diff options
Diffstat (limited to 'erts/emulator')
157 files changed, 14173 insertions, 12934 deletions
diff --git a/erts/emulator/Makefile.in b/erts/emulator/Makefile.in index fdaf253fe5..9c79bf3da0 100644 --- a/erts/emulator/Makefile.in +++ b/erts/emulator/Makefile.in @@ -634,6 +634,7 @@ GENERATE += $(TTF_DIR)/driver_tab.c PRELOAD_BEAM = $(ERL_TOP)/erts/preloaded/ebin/otp_ring0.beam \ $(ERL_TOP)/erts/preloaded/ebin/erts_code_purger.beam \ $(ERL_TOP)/erts/preloaded/ebin/init.beam \ + $(ERL_TOP)/erts/preloaded/ebin/prim_buffer.beam \ $(ERL_TOP)/erts/preloaded/ebin/prim_eval.beam \ $(ERL_TOP)/erts/preloaded/ebin/prim_inet.beam \ $(ERL_TOP)/erts/preloaded/ebin/prim_file.beam \ @@ -785,6 +786,9 @@ $(OBJDIR)/%.o: drivers/$(ERLANG_OSTYPE)/%.c $(OBJDIR)/%.o: nifs/common/%.c $(V_CC) $(CFLAGS) -DLIBSCTP=$(LIBSCTP) $(INCLUDES) -Inifs/common -Inifs/$(ERLANG_OSTYPE) -c $< -o $@ +$(OBJDIR)/%.o: nifs/$(ERLANG_OSTYPE)/%.c + $(V_CC) $(CFLAGS) $(INCLUDES) -Inifs/common -Inifs/$(ERLANG_OSTYPE) -I../etc/$(ERLANG_OSTYPE) -c $< -o $@ + # ---------------------------------------------------------------------- # Specials # @@ -873,17 +877,17 @@ RUN_OBJS += \ LTTNG_OBJS = $(OBJDIR)/erlang_lttng.o NIF_OBJS = \ $(OBJDIR)/erl_tracer_nif.o \ + $(OBJDIR)/prim_buffer_nif.o \ + $(OBJDIR)/prim_file_nif.o \ $(OBJDIR)/zlib_nif.o ifeq ($(TARGET),win32) DRV_OBJS = \ $(OBJDIR)/registry_drv.o \ - $(OBJDIR)/efile_drv.o \ $(OBJDIR)/inet_drv.o \ $(OBJDIR)/ram_file_drv.o \ $(OBJDIR)/ttsl_drv.o OS_OBJS = \ - $(OBJDIR)/win_efile.o \ $(OBJDIR)/win_con.o \ $(OBJDIR)/dll_sys.o \ $(OBJDIR)/driver_tab.o \ @@ -892,22 +896,23 @@ OS_OBJS = \ $(OBJDIR)/sys_time.o \ $(OBJDIR)/sys_interrupt.o \ $(OBJDIR)/sys_env.o \ - $(OBJDIR)/dosmap.o + $(OBJDIR)/dosmap.o \ + $(OBJDIR)/win_prim_file.o else OS_OBJS = \ $(OBJDIR)/sys.o \ $(OBJDIR)/sys_drivers.o \ + $(OBJDIR)/sys_env.o \ $(OBJDIR)/sys_uds.o \ $(OBJDIR)/driver_tab.o \ - $(OBJDIR)/unix_efile.o \ + $(OBJDIR)/elib_memmove.o \ $(OBJDIR)/gzio.o \ - $(OBJDIR)/elib_memmove.o + $(OBJDIR)/unix_prim_file.o OS_OBJS += $(OBJDIR)/sys_float.o \ $(OBJDIR)/sys_time.o DRV_OBJS = \ - $(OBJDIR)/efile_drv.o \ $(OBJDIR)/inet_drv.o \ $(OBJDIR)/ram_file_drv.o \ $(OBJDIR)/ttsl_drv.o @@ -940,7 +945,8 @@ endif OS_OBJS += $(OBJDIR)/erl_poll.o \ $(OBJDIR)/erl_check_io.o \ $(OBJDIR)/erl_mseg.o \ - $(OBJDIR)/erl_mmap.o \ + $(OBJDIR)/erl_mmap.o \ + $(OBJDIR)/erl_osenv.o \ $(OBJDIR)/erl_$(ERLANG_OSTYPE)_sys_ddll.o \ $(OBJDIR)/erl_mtrace_sys_wrap.o \ $(OBJDIR)/erl_sys_common_misc.o \ @@ -1135,6 +1141,7 @@ BEAM_SRC=$(wildcard beam/*.c) DRV_COMMON_SRC=$(wildcard drivers/common/*.c) DRV_OSTYPE_SRC=$(wildcard drivers/$(ERLANG_OSTYPE)/*.c) NIF_COMMON_SRC=$(wildcard nifs/common/*.c) +NIF_OSTYPE_SRC=$(wildcard nifs/$(ERLANG_OSTYPE)/*.c) ALL_SYS_SRC=$(wildcard sys/$(ERLANG_OSTYPE)/*.c) $(wildcard sys/common/*.c) # We use $(shell ls) here instead of wildcard as $(wildcard ) resolved at # loadtime of the makefile and at that time these files are not generated yet. @@ -1147,7 +1154,10 @@ ifeq ($(TARGET),win32) #DEP_CC=$(EMU_CC) DEP_CC=$(CC) -DEP_FLAGS=-MM $(subst -O2,,$(CFLAGS)) $(INCLUDES) -I../etc/win32 -Idrivers/common -Idrivers/$(ERLANG_OSTYPE) +DEP_FLAGS=-MM $(subst -O2,,$(CFLAGS)) $(INCLUDES) -I../etc/win32 \ + -Idrivers/common -Idrivers/$(ERLANG_OSTYPE) \ + -Inifs/common -Inifs/$(ERLANG_OSTYPE) + # ifeq (@MIXED_CYGWIN_VC@,yes) # VC++ used for compiling. If __GNUC__ is defined we will include # other headers then when compiling which will result in faulty @@ -1167,7 +1177,9 @@ MG_FLAG=-MG endif DEP_CC=$(CC) -DEP_FLAGS=-MM $(MG_FLAG) $(CFLAGS) $(INCLUDES) -Inifs/common -Idrivers/common -Idrivers/$(ERLANG_OSTYPE) +DEP_FLAGS=-MM $(MG_FLAG) $(CFLAGS) $(INCLUDES) \ + -Idrivers/common -Idrivers/$(ERLANG_OSTYPE) \ + -Inifs/common -Inifs/$(ERLANG_OSTYPE) SYS_SRC=$(ALL_SYS_SRC) endif @@ -1198,6 +1210,8 @@ $(TTF_DIR)/depend.mk: $(TTF_DIR)/GENERATED $(PRELOAD_SRC) | $(SED_DEPEND) >> $(TTF_DIR)/depend.mk $(V_at)$(DEP_CC) $(DEP_FLAGS) $(NIF_COMMON_SRC) \ | $(SED_DEPEND) >> $(TTF_DIR)/depend.mk + $(V_at)$(DEP_CC) $(DEP_FLAGS) -I../etc/$(ERLANG_OSTYPE) $(NIF_OSTYPE_SRC) \ + | $(SED_DEPEND) >> $(TTF_DIR)/depend.mk $(V_at)$(DEP_CC) $(DEP_FLAGS) $(SYS_SRC) \ | $(SED_DEPEND) >> $(TTF_DIR)/depend.mk $(V_at)$(DEP_CC) $(DEP_FLAGS) $(TARGET_SRC) \ diff --git a/erts/emulator/beam/atom.c b/erts/emulator/beam/atom.c index bbe1cb3e11..e5b7616a0d 100644 --- a/erts/emulator/beam/atom.c +++ b/erts/emulator/beam/atom.c @@ -452,7 +452,7 @@ init_atom_table(void) /* Ordinary atoms */ for (i = 0; erl_atom_names[i] != 0; i++) { int ix; - a.len = strlen(erl_atom_names[i]); + a.len = sys_strlen(erl_atom_names[i]); a.latin1_chars = a.len; a.name = (byte*)erl_atom_names[i]; a.slot.index = i; diff --git a/erts/emulator/beam/atom.h b/erts/emulator/beam/atom.h index be998a46bd..385120a8d9 100644 --- a/erts/emulator/beam/atom.h +++ b/erts/emulator/beam/atom.h @@ -36,7 +36,7 @@ /* Internal atom cache needs MAX_ATOM_TABLE_SIZE to be less than an unsigned 32 bit integer. See external.c(erts_encode_ext_dist_header_setup) for more details. */ -#define MAX_ATOM_TABLE_SIZE ((MAX_ATOM_INDEX + 1 < (UWORD_CONSTANT(1) << 32)) ? MAX_ATOM_INDEX + 1 : (UWORD_CONSTANT(1) << 32)) +#define MAX_ATOM_TABLE_SIZE ((MAX_ATOM_INDEX + 1 < (UWORD_CONSTANT(1) << 32)) ? MAX_ATOM_INDEX + 1 : ((UWORD_CONSTANT(1) << 31) - 1)) /* Here we use maximum signed interger value to avoid integer overflow */ #else #define MAX_ATOM_TABLE_SIZE (MAX_ATOM_INDEX + 1) #endif diff --git a/erts/emulator/beam/atom.names b/erts/emulator/beam/atom.names index fc55b687d4..38b5f0c5e3 100644 --- a/erts/emulator/beam/atom.names +++ b/erts/emulator/beam/atom.names @@ -48,7 +48,7 @@ atom Empty='' # # Used in the Beam emulator loop. (Smaller literals usually means tighter code.) # -atom fun infinity timeout normal call return +atom infinity timeout normal call return atom throw error exit atom undefined @@ -69,11 +69,9 @@ atom DOWN='DOWN' atom UP='UP' atom EXIT='EXIT' atom abort -atom aborted atom abs_path atom absoluteURI atom ac -atom accessor atom active atom active_tasks atom active_tasks_all @@ -88,7 +86,6 @@ atom allocated_areas atom allocator atom allocator_sizes atom alloc_util_allocators -atom allow_gc atom allow_passive_connect atom already_loaded atom amd64 @@ -108,6 +105,7 @@ atom asynchronous atom atom atom atom_used atom attributes +atom auto_connect atom await_microstate_accounting_modifications atom await_port_send_result atom await_proc_exit @@ -121,9 +119,7 @@ atom bag atom band atom big atom bif_return_trap -atom bif_timer_server atom binary -atom binary_bin_to_list_trap atom binary_copy_trap atom binary_find_trap atom binary_longest_prefix_trap @@ -178,7 +174,6 @@ atom convert_time_unit atom connect atom connected atom connection_closed -atom cons atom const atom context_switches atom control @@ -198,9 +193,6 @@ atom debug_flags atom decimals atom default atom delay_trap -atom dexit -atom depth -atom dgroup_leader atom dictionary atom dirty_bif_exception atom dirty_bif_result @@ -221,7 +213,6 @@ atom dist_ctrl_put_data atom dist_data atom Div='/' atom div -atom dlink atom dmonitor_node atom dmonitor_p atom DollarDollar='$$' @@ -230,9 +221,7 @@ atom dollar_endonly atom dotall atom driver atom driver_options -atom dsend atom dsend_continue_trap -atom dunlink atom duplicate_bag atom duplicated atom dupnames @@ -250,15 +239,14 @@ atom Eqeq='==' atom erl_tracer atom erlang atom erl_signal_server -atom ERROR='ERROR' atom error_handler atom error_logger atom erts_code_purger atom erts_debug +atom erts_dflags atom erts_internal atom ets atom ETS_TRANSFER='ETS-TRANSFER' -atom event atom exact_reductions atom exception_from atom exception_trace @@ -285,27 +273,20 @@ atom force atom format_cpu_topology atom free atom fullsweep_after -atom fullsweep_if_old_binaries -atom fun -atom function atom functions atom function_clause atom garbage_collecting atom garbage_collection atom garbage_collection_info -atom gc_end atom gc_major_end atom gc_major_start atom gc_max_heap_size atom gc_minor_end atom gc_minor_start -atom gc_start atom Ge='>=' atom generational -atom get_data atom get_seq_token atom get_tcw -atom getenv atom gather_gc_info_result atom gather_io_bytes atom gather_microstate_accounting_result @@ -347,11 +328,11 @@ atom initial_call atom input atom internal atom internal_error -atom internal_status atom instruction_counts atom invalid atom is_constant atom is_seq_trace +atom iterator atom io atom keypos atom kill @@ -392,14 +373,12 @@ atom match_spec_result atom max atom maximum atom max_heap_size -atom max_tables max_processes atom mbuf_size atom md5 atom memory atom memory_internal atom memory_types atom message -atom message_binary atom message_queue_data atom message_queue_len atom messages @@ -446,21 +425,18 @@ atom new_processes atom new_ports atom new_uniq atom newline -atom next atom no atom nomatch atom none atom no_auto_capture atom noconnect atom noconnection -atom nocookie atom node atom node_type atom nodedown atom nodedown_reason atom nodeup atom noeol -atom nofile atom noproc atom normal atom nosuspend @@ -482,7 +458,6 @@ atom notempty_atstart atom notify atom notsup atom nouse_stdio -atom objects atom off_heap atom offset atom ok @@ -552,7 +527,6 @@ atom re_run_trap atom read_concurrency atom ready_input atom ready_output -atom ready_async atom reason atom receive atom recent_size @@ -580,7 +554,8 @@ atom running_procs atom runtime atom safe atom save_calls -atom scheduler +atom sbct +atom scheduler atom scheduler_id atom scheduler_wall_time atom scheduler_wall_time_all @@ -598,7 +573,6 @@ atom sequential_trace_token atom serial atom set atom set_cpu_topology -atom set_data atom set_on_first_link atom set_on_first_spawn atom set_on_link @@ -606,8 +580,6 @@ atom set_on_spawn atom set_seq_token atom set_tcw atom set_tcw_fake -atom separate -atom shared atom sighup atom sigterm atom sigusr1 @@ -623,7 +595,6 @@ atom sigtstp atom sigquit atom silent atom size -atom sl_alloc atom spawn_executable atom spawn_driver atom spawned @@ -631,7 +602,6 @@ atom ssl_tls atom stack_size atom start atom status -atom static atom stderr_to_stdout atom stop atom stream @@ -641,13 +611,11 @@ atom sunrm atom suspend atom suspended atom suspending -atom sys_misc atom system -atom system_error +atom system_flag_scheduler_wall_time atom system_limit atom system_version atom system_architecture -atom SYSTEM='SYSTEM' atom table atom term_to_binary_trap atom this @@ -665,7 +633,7 @@ atom total_heap_size atom total_run_queue_lengths atom total_run_queue_lengths_all atom tpkt -atom trace trace_ts traced +atom trace traced atom trace_control_word atom trace_status atom tracer @@ -674,7 +642,6 @@ atom trim atom trim_all atom try_clause atom true -atom tuple atom type atom ucompile atom ucp @@ -691,14 +658,11 @@ atom unblock_normal atom uniq atom unless_suspending atom unloaded -atom unloading atom unloaded_only atom unload_cancelled atom value -atom values atom version atom visible -atom wait atom waiting atom wall_clock atom warning diff --git a/erts/emulator/beam/beam_bp.c b/erts/emulator/beam/beam_bp.c index fe1e15701b..0832b3f374 100644 --- a/erts/emulator/beam/beam_bp.c +++ b/erts/emulator/beam/beam_bp.c @@ -998,7 +998,9 @@ do_call_trace(Process* c_p, ErtsCodeInfo* info, Eterm* reg, fixup_cp_before_trace(c_p, &return_to_trace); + ERTS_UNREQ_PROC_MAIN_LOCK(c_p); flags = erts_call_trace(c_p, info, ms, reg, local, &tracer); + ERTS_REQ_PROC_MAIN_LOCK(c_p); /* restore cp after potential fixup */ c_p->cp = cp_save; diff --git a/erts/emulator/beam/beam_debug.c b/erts/emulator/beam/beam_debug.c index 509aa2a84f..5eb68b817e 100644 --- a/erts/emulator/beam/beam_debug.c +++ b/erts/emulator/beam/beam_debug.c @@ -40,6 +40,7 @@ #include "erl_binary.h" #include "erl_thr_progress.h" #include "erl_nfunc_sched.h" +#include "beam_catches.h" #ifdef ARCH_64 # define HEXF "%016bpX" @@ -55,6 +56,7 @@ static int print_op(fmtfn_t to, void *to_arg, int op, int size, BeamInstr* addr) static void print_bif_name(fmtfn_t to, void* to_arg, BifFunction bif); static BeamInstr* f_to_addr(BeamInstr* base, int op, BeamInstr* ap); static BeamInstr* f_to_addr_packed(BeamInstr* base, int op, Sint32* ap); +static void print_byte_string(fmtfn_t to, void *to_arg, byte* str, Uint bytes); BIF_RETTYPE erts_debug_same_2(BIF_ALIST_2) @@ -226,11 +228,11 @@ erts_debug_instructions_0(BIF_ALIST_0) Eterm res = NIL; for (i = 0; i < num_instructions; i++) { - needed += 2*strlen(opc[i].name); + needed += 2*sys_strlen(opc[i].name); } hp = HAlloc(BIF_P, needed); for (i = num_instructions-1; i >= 0; i--) { - Eterm s = erts_bld_string_n(&hp, 0, opc[i].name, strlen(opc[i].name)); + Eterm s = erts_bld_string_n(&hp, 0, opc[i].name, sys_strlen(opc[i].name)); res = erts_bld_cons(&hp, 0, s, res); } return res; @@ -396,6 +398,7 @@ print_op(fmtfn_t to, void *to_arg, int op, int size, BeamInstr* addr) BeamInstr args[8]; /* Arguments for this instruction. */ BeamInstr* ap; /* Pointer to arguments. */ BeamInstr* unpacked; /* Unpacked arguments */ + BeamInstr* first_arg; /* First argument */ start_prog = opc[op].pack; @@ -428,7 +431,7 @@ print_op(fmtfn_t to, void *to_arg, int op, int size, BeamInstr* addr) * the packing program backwards and in reverse. */ - prog = start_prog + strlen(start_prog); + prog = start_prog + sys_strlen(start_prog); while (start_prog < prog) { prog--; switch (*prog) { @@ -480,6 +483,8 @@ print_op(fmtfn_t to, void *to_arg, int op, int size, BeamInstr* addr) ap = args; } + first_arg = ap; + /* * Print the name and all operands of the instructions. */ @@ -570,24 +575,60 @@ print_op(fmtfn_t to, void *to_arg, int op, int size, BeamInstr* addr) } break; } + case op_i_make_fun_Wt: + if (*sign == 'W') { + ErlFunEntry* fe = (ErlFunEntry *) *ap; + ErtsCodeMFA* cmfa = find_function_from_pc(fe->address); + erts_print(to, to_arg, "%T:%T/%bpu", cmfa->module, + cmfa->function, cmfa->arity); + } else { + erts_print(to, to_arg, "%d", *ap); + } + break; + case op_i_bs_match_string_xfWW: + if (ap - first_arg < 3) { + erts_print(to, to_arg, "%d", *ap); + } else { + Uint bits = ap[-1]; + Uint bytes = (bits+7)/8; + byte* str = (byte *) *ap; + print_byte_string(to, to_arg, str, bytes); + } + break; + case op_bs_put_string_WW: + if (ap - first_arg == 0) { + erts_print(to, to_arg, "%d", *ap); + } else { + Uint bytes = ap[-1]; + byte* str = (byte *) ap[0]; + print_byte_string(to, to_arg, str, bytes); + } + break; default: erts_print(to, to_arg, "%d", *ap); } ap++; break; case 'f': /* Destination label */ - { - BeamInstr* target = f_to_addr(addr, op, ap); - ErtsCodeMFA* cmfa = find_function_from_pc(target); - if (!cmfa || erts_codemfa_to_code(cmfa) != target) { - erts_print(to, to_arg, "f(" HEXF ")", target); - } else { - erts_print(to, to_arg, "%T:%T/%bpu", cmfa->module, - cmfa->function, cmfa->arity); - } - ap++; - } - break; + switch (op) { + case op_catch_yf: + erts_print(to, to_arg, "f(" HEXF ")", catch_pc((BeamInstr)*ap)); + break; + default: + { + BeamInstr* target = f_to_addr(addr, op, ap); + ErtsCodeMFA* cmfa = find_function_from_pc(target); + if (!cmfa || erts_codemfa_to_code(cmfa) != target) { + erts_print(to, to_arg, "f(" HEXF ")", target); + } else { + erts_print(to, to_arg, "%T:%T/%bpu", cmfa->module, + cmfa->function, cmfa->arity); + } + ap++; + } + break; + } + break; case 'p': /* Pointer (to label) */ { BeamInstr* target = f_to_addr(addr, op, ap); @@ -848,6 +889,14 @@ static BeamInstr* f_to_addr_packed(BeamInstr* base, int op, Sint32* ap) return base - 1 + opc[op].adjust + *ap; } +static void print_byte_string(fmtfn_t to, void *to_arg, byte* str, Uint bytes) +{ + Uint i; + + for (i = 0; i < bytes; i++) { + erts_print(to, to_arg, "%02X", str[i]); + } +} /* * Dirty BIF testing. diff --git a/erts/emulator/beam/beam_emu.c b/erts/emulator/beam/beam_emu.c index 60d0008d8f..fbd0e38735 100644 --- a/erts/emulator/beam/beam_emu.c +++ b/erts/emulator/beam/beam_emu.c @@ -400,12 +400,13 @@ static BeamInstr* apply_fun(Process* p, Eterm fun, Eterm args, Eterm* reg) NOINLINE; static Eterm new_fun(Process* p, Eterm* reg, ErlFunEntry* fe, int num_free) NOINLINE; -static Eterm new_map(Process* p, Eterm* reg, Uint live, Uint n, BeamInstr* ptr) NOINLINE; -static Eterm new_small_map_lit(Process* p, Eterm* reg, Eterm keys_literal, +static Eterm erts_gc_new_map(Process* p, Eterm* reg, Uint live, + Uint n, BeamInstr* ptr) NOINLINE; +static Eterm erts_gc_new_small_map_lit(Process* p, Eterm* reg, Eterm keys_literal, Uint live, BeamInstr* ptr) NOINLINE; -static Eterm update_map_assoc(Process* p, Eterm* reg, Uint live, +static Eterm erts_gc_update_map_assoc(Process* p, Eterm* reg, Uint live, Uint n, BeamInstr* new_p) NOINLINE; -static Eterm update_map_exact(Process* p, Eterm* reg, Uint live, +static Eterm erts_gc_update_map_exact(Process* p, Eterm* reg, Uint live, Uint n, Eterm* new_p) NOINLINE; static Eterm get_map_element(Eterm map, Eterm key); static Eterm get_map_element_hash(Eterm map, Eterm key, Uint32 hx); @@ -748,7 +749,7 @@ void process_main(Eterm * x_reg_array, FloatDef* f_reg_array) dtrace_proc_str(c_p, process_buf); if (ERTS_PROC_IS_EXITING(c_p)) { - strcpy(fun_buf, "<exiting>"); + sys_strcpy(fun_buf, "<exiting>"); } else { ErtsCodeMFA *cmfa = find_function_from_pc(c_p->i); if (cmfa) { @@ -1228,7 +1229,7 @@ void erts_dirty_process_main(ErtsSchedulerData *esdp) dtrace_proc_str(c_p, process_buf); if (ERTS_PROC_IS_EXITING(c_p)) { - strcpy(fun_buf, "<exiting>"); + sys_strcpy(fun_buf, "<exiting>"); } else { ErtsCodeMFA *cmfa = find_function_from_pc(c_p->i); if (cmfa) { @@ -1453,6 +1454,7 @@ handle_error(Process* c_p, BeamInstr* pc, Eterm* reg, ErtsCodeMFA *bif_mfa) reg[3] = c_p->ftrace; if ((new_pc = next_catch(c_p, reg))) { c_p->cp = 0; /* To avoid keeping stale references. */ + c_p->msg.saved_last = 0; /* No longer safe to use this position */ return new_pc; } if (c_p->catches > 0) erts_exit(ERTS_ERROR_EXIT, "Catch not found"); @@ -2755,7 +2757,7 @@ do { \ static Eterm -new_map(Process* p, Eterm* reg, Uint live, Uint n, BeamInstr* ptr) +erts_gc_new_map(Process* p, Eterm* reg, Uint live, Uint n, BeamInstr* ptr) { Uint i; Uint need = n + 1 /* hdr */ + 1 /*size*/ + 1 /* ptr */ + 1 /* arity */; @@ -2812,7 +2814,8 @@ new_map(Process* p, Eterm* reg, Uint live, Uint n, BeamInstr* ptr) } static Eterm -new_small_map_lit(Process* p, Eterm* reg, Eterm keys_literal, Uint live, BeamInstr* ptr) +erts_gc_new_small_map_lit(Process* p, Eterm* reg, Eterm keys_literal, + Uint live, BeamInstr* ptr) { Eterm* keys = tuple_val(keys_literal); Uint n = arityval(*keys); @@ -2846,7 +2849,8 @@ new_small_map_lit(Process* p, Eterm* reg, Eterm keys_literal, Uint live, BeamIns } static Eterm -update_map_assoc(Process* p, Eterm* reg, Uint live, Uint n, BeamInstr* new_p) +erts_gc_update_map_assoc(Process* p, Eterm* reg, Uint live, + Uint n, BeamInstr* new_p) { Uint num_old; Uint num_updates; @@ -2892,7 +2896,7 @@ update_map_assoc(Process* p, Eterm* reg, Uint live, Uint n, BeamInstr* new_p) */ if (num_old == 0) { - return new_map(p, reg, live, n, new_p); + return erts_gc_new_map(p, reg, live, n, new_p); } /* @@ -3048,7 +3052,7 @@ update_map_assoc(Process* p, Eterm* reg, Uint live, Uint n, BeamInstr* new_p) */ static Eterm -update_map_exact(Process* p, Eterm* reg, Uint live, Uint n, Eterm* new_p) +erts_gc_update_map_exact(Process* p, Eterm* reg, Uint live, Uint n, Eterm* new_p) { Uint i; Uint num_old; diff --git a/erts/emulator/beam/beam_load.c b/erts/emulator/beam/beam_load.c index 7331c331a6..50498cb6cf 100644 --- a/erts/emulator/beam/beam_load.c +++ b/erts/emulator/beam/beam_load.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 1996-2017. All Rights Reserved. + * Copyright Ericsson AB 1996-2018. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -40,6 +40,7 @@ #include "erl_zlib.h" #include "erl_map.h" #include "erl_process_dict.h" +#include "erl_unicode.h" #ifdef HIPE #include "hipe_bif0.h" @@ -458,7 +459,7 @@ typedef struct LoaderState { #ifdef DEBUG # define GARBAGE 0xCC -# define DEBUG_INIT_GENOP(Dst) memset(Dst, GARBAGE, sizeof(GenOp)) +# define DEBUG_INIT_GENOP(Dst) sys_memset(Dst, GARBAGE, sizeof(GenOp)) #else # define DEBUG_INIT_GENOP(Dst) #endif @@ -1421,7 +1422,7 @@ load_atom_table(LoaderState* stp, ErtsAtomEncoding enc) Atom* ap; ap = atom_tab(atom_val(stp->atom[1])); - memcpy(sbuf, ap->name, ap->len); + sys_memcpy(sbuf, ap->name, ap->len); sbuf[ap->len] = '\0'; LoadError1(stp, "module name in object code is %s", sbuf); } @@ -1806,7 +1807,7 @@ read_line_table(LoaderState* stp) GetInt(stp, 2, n); GetString(stp, fname, n); - stp->fname[i] = erts_atom_put(fname, n, ERTS_ATOM_ENC_LATIN1, 1); + stp->fname[i] = erts_atom_put(fname, n, ERTS_ATOM_ENC_UTF8, 1); } } @@ -2088,7 +2089,7 @@ load_code(LoaderState* stp) erts_alloc(ERTS_ALC_T_LOADER_TMP, (arity+last_op->a[arg].val) *sizeof(GenOpArg)); - memcpy(last_op->a, last_op->def_args, + sys_memcpy(last_op->a, last_op->def_args, arity*sizeof(GenOpArg)); arity += last_op->a[arg].val; break; @@ -2384,8 +2385,18 @@ load_code(LoaderState* stp) code[ci++] = NIL; break; case TAG_q: - new_literal_patch(stp, ci); - code[ci++] = tmp_op->a[arg].val; + { + BeamInstr val = tmp_op->a[arg].val; + Eterm term = stp->literals[val].term; + new_literal_patch(stp, ci); + code[ci++] = val; + switch (loader_tag(term)) { + case LOADER_X_REG: + case LOADER_Y_REG: + LoadError1(stp, "the term '%T' would be confused " + "with a register", term); + } + } break; default: LoadError1(stp, "bad tag %d for general source", @@ -4904,7 +4915,7 @@ freeze_code(LoaderState* stp) line_items[i] = codev + stp->ci - 1; line_tab->fname_ptr = (Eterm*) &line_items[i + 1]; - memcpy(line_tab->fname_ptr, stp->fname, stp->num_fnames*sizeof(Eterm)); + sys_memcpy(line_tab->fname_ptr, stp->fname, stp->num_fnames*sizeof(Eterm)); line_tab->loc_size = stp->loc_size; if (stp->loc_size == 2) { @@ -5487,8 +5498,8 @@ transform_engine(LoaderState* st) case TOP_store_rest_args: { GENOP_ARITY(instr, instr->arity+num_rest_args); - memcpy(instr->a, instr->def_args, ap*sizeof(GenOpArg)); - memcpy(instr->a+ap, rest_args, num_rest_args*sizeof(GenOpArg)); + sys_memcpy(instr->a, instr->def_args, ap*sizeof(GenOpArg)); + sys_memcpy(instr->a+ap, rest_args, num_rest_args*sizeof(GenOpArg)); ap += num_rest_args; } break; @@ -6018,13 +6029,36 @@ erts_release_literal_area(ErtsLiteralArea* literal_area) return; oh = literal_area->off_heap; - + while (oh) { - Binary* bptr; - ASSERT(thing_subtag(oh->thing_word) == REFC_BINARY_SUBTAG); - bptr = ((ProcBin*)oh)->val; - erts_bin_release(bptr); - oh = oh->next; + switch (thing_subtag(oh->thing_word)) { + case REFC_BINARY_SUBTAG: + { + Binary* bptr = ((ProcBin*)oh)->val; + erts_bin_release(bptr); + break; + } + case FUN_SUBTAG: + { + ErlFunEntry* fe = ((ErlFunThing*)oh)->fe; + if (erts_refc_dectest(&fe->refc, 0) == 0) { + erts_erase_fun_entry(fe); + } + break; + } + case REF_SUBTAG: + { + ErtsMagicBinary *bptr; + ASSERT(is_magic_ref_thing(oh)); + bptr = ((ErtsMRefThing *) oh)->mb; + erts_bin_release((Binary *) bptr); + break; + } + default: + ASSERT(is_external_header(oh->thing_word)); + erts_deref_node_entry(((ExternalThing*)oh)->node); + } + oh = oh->next; } erts_free(ERTS_ALC_T_LITERAL, literal_area); } @@ -6221,8 +6255,7 @@ erts_build_mfa_item(FunctionInfo* fi, Eterm* hp, Eterm args, Eterm* mfa_p) file_term = buf_to_intlist(&hp, ".erl", 4, NIL); file_term = buf_to_intlist(&hp, (char*)ap->name, ap->len, file_term); } else { - Atom* ap = atom_tab(atom_val((fi->fname_ptr)[file-1])); - file_term = buf_to_intlist(&hp, (char*)ap->name, ap->len, NIL); + file_term = erts_atom_to_string(&hp, (fi->fname_ptr)[file-1]); } tuple = TUPLE2(hp, am_line, make_small(line)); @@ -6414,7 +6447,7 @@ stub_copy_info(LoaderState* stp, Sint decoded_size; Uint size = stp->chunks[chunk].size; if (size != 0) { - memcpy(info, stp->chunks[chunk].start, size); + sys_memcpy(info, stp->chunks[chunk].start, size); *ptr_word = info; decoded_size = erts_decode_ext_size(info, size); if (decoded_size < 0) { @@ -6951,6 +6984,8 @@ int erts_commit_hipe_patch_load(Eterm hipe_magic_bin) hipe_stp->new_hipe_refs = NULL; hipe_stp->new_hipe_sdesc = NULL; + hipe_redirect_to_module(modp); + return 1; } @@ -6982,8 +7017,8 @@ void dbg_set_traced_mfa(const char* m, const char* f, Uint a) { unsigned i = dbg_trace_ix++; ASSERT(i < MFA_MAX); - dbg_trace_m[i] = am_atom_put(m, strlen(m)); - dbg_trace_f[i] = am_atom_put(f, strlen(f)); + dbg_trace_m[i] = am_atom_put(m, sys_strlen(m)); + dbg_trace_f[i] = am_atom_put(f, sys_strlen(f)); dbg_trace_a[i] = a; } diff --git a/erts/emulator/beam/beam_ranges.c b/erts/emulator/beam/beam_ranges.c index 01bda7f3c1..f0c9496341 100644 --- a/erts/emulator/beam/beam_ranges.c +++ b/erts/emulator/beam/beam_ranges.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2012-2016. All Rights Reserved. + * Copyright Ericsson AB 2012-2018. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -26,6 +26,7 @@ #include "erl_vm.h" #include "global.h" #include "beam_load.h" +#include "erl_unicode.h" typedef struct { BeamInstr* start; /* Pointer to start of module. */ @@ -341,8 +342,7 @@ lookup_loc(FunctionInfo* fi, const BeamInstr* pc, Atom* mod_atom = atom_tab(atom_val(fi->mfa->module)); fi->needed += 2*(mod_atom->len+4); } else { - Atom* ap = atom_tab(atom_val((fi->fname_ptr)[file-1])); - fi->needed += 2*ap->len; + fi->needed += 2*erts_atom_to_string_length((fi->fname_ptr)[file-1]); } return; } else { diff --git a/erts/emulator/beam/bif.c b/erts/emulator/beam/bif.c index 4b11884f38..652b95105f 100644 --- a/erts/emulator/beam/bif.c +++ b/erts/emulator/beam/bif.c @@ -59,6 +59,7 @@ Export *erts_convert_time_unit_trap = NULL; static Export *await_msacc_mod_trap = NULL; static erts_atomic32_t msacc; +static Export *system_flag_scheduler_wall_time_trap; static Export *await_sched_wall_time_mod_trap; static erts_atomic32_t sched_wall_time; @@ -227,17 +228,38 @@ BIF_RETTYPE link_1(BIF_ALIST_1) goto res_no_proc; } - code = erts_dsig_prepare(&dsd, dep, BIF_P, ERTS_DSP_RLOCK, 0); + code = erts_dsig_prepare(&dsd, dep, BIF_P, + (ERTS_PROC_LOCK_MAIN | ERTS_PROC_LOCK_LINK), + ERTS_DSP_RLOCK, 0, 1); switch (code) { case ERTS_DSIG_PREP_NOT_ALIVE: - /* Let the dlink trap handle it */ - case ERTS_DSIG_PREP_NOT_CONNECTED: - erts_proc_unlock(BIF_P, ERTS_PROC_LOCK_LINK); - BIF_TRAP1(dlink_trap, BIF_P, BIF_ARG_1); - + case ERTS_DSIG_PREP_NOT_CONNECTED: { + ErtsProcLocks locks = ERTS_PROC_LOCK_MAIN | ERTS_PROC_LOCK_LINK; + erts_aint32_t state; + erts_proc_lock(BIF_P, (ERTS_PROC_LOCKS_ALL & ~locks)); + locks = ERTS_PROC_LOCKS_ALL; + erts_send_exit_signal(BIF_P, BIF_ARG_1, BIF_P, &locks, + am_noconnection, NIL, NULL, 0); + erts_proc_unlock(BIF_P, locks & ERTS_PROC_LOCKS_ALL_MINOR); + + /* + * Copy-paste from old dist_exit_3, not sure if we really + * need erts_handle_pending_exit when exit_2 does not. + */ + state = erts_atomic32_read_acqb(&BIF_P->state); + if (state & (ERTS_PSFLG_EXITING|ERTS_PSFLG_PENDING_EXIT)) { + if (state & ERTS_PSFLG_PENDING_EXIT) + erts_handle_pending_exit(BIF_P, ERTS_PROC_LOCK_MAIN); + ERTS_BIF_EXITED(BIF_P); + } + BIF_RET(am_true); + } + case ERTS_DSIG_PREP_PENDING: case ERTS_DSIG_PREP_CONNECTED: - /* We are connected. Setup link and send link signal */ - + /* + * We have (pending) connection. + * Setup link and enqueue link signal. + */ erts_de_links_lock(dep); erts_add_link(&ERTS_P_LINKS(BIF_P), LINK_PID, BIF_ARG_1); @@ -256,8 +278,7 @@ BIF_RETTYPE link_1(BIF_ALIST_1) ERTS_BIF_YIELD_RETURN(BIF_P, am_true); BIF_RET(am_true); default: - ASSERT(! "Invalid dsig prepare result"); - BIF_ERROR(BIF_P, EXC_INTERNAL_ERROR); + ERTS_ASSERT(! "Invalid dsig prepare result"); } } } @@ -292,7 +313,8 @@ remote_demonitor(Process *c_p, DistEntry *dep, Eterm ref, Eterm to) ERTS_LC_ASSERT((ERTS_PROC_LOCK_MAIN|ERTS_PROC_LOCK_LINK) == erts_proc_lc_my_proc_locks(c_p)); - code = erts_dsig_prepare(&dsd, dep, c_p, ERTS_DSP_RLOCK, 0); + code = erts_dsig_prepare(&dsd, dep, c_p, ERTS_PROC_LOCK_MAIN, + ERTS_DSP_RLOCK, 0, 0); switch (code) { case ERTS_DSIG_PREP_NOT_ALIVE: case ERTS_DSIG_PREP_NOT_CONNECTED: @@ -313,6 +335,7 @@ remote_demonitor(Process *c_p, DistEntry *dep, Eterm ref, Eterm to) res = am_true; break; + case ERTS_DSIG_PREP_PENDING: case ERTS_DSIG_PREP_CONNECTED: erts_de_links_lock(dep); @@ -347,8 +370,7 @@ remote_demonitor(Process *c_p, DistEntry *dep, Eterm ref, Eterm to) } break; default: - ASSERT(! "Invalid dsig prepare result"); - return am_internal_error; + ERTS_ASSERT(! "Invalid dsig prepare result"); } @@ -767,23 +789,20 @@ remote_monitor(Process *p, Eterm bifarg1, Eterm bifarg2, BIF_RETTYPE ret; int code; + ASSERT(dep); erts_proc_lock(p, ERTS_PROC_LOCK_LINK); - code = erts_dsig_prepare(&dsd, dep, p, ERTS_DSP_RLOCK, 0); + code = erts_dsig_prepare(&dsd, dep, + p, (ERTS_PROC_LOCK_MAIN | ERTS_PROC_LOCK_LINK), + ERTS_DSP_RLOCK, 0, 1); switch (code) { case ERTS_DSIG_PREP_NOT_ALIVE: - /* Let the dmonitor_p trap handle it */ case ERTS_DSIG_PREP_NOT_CONNECTED: erts_proc_unlock(p, ERTS_PROC_LOCK_LINK); ERTS_BIF_PREP_TRAP2(ret, dmonitor_p_trap, p, bifarg1, bifarg2); break; + case ERTS_DSIG_PREP_PENDING: case ERTS_DSIG_PREP_CONNECTED: - if (!(dep->flags & DFLAG_DIST_MONITOR) - || (byname && !(dep->flags & DFLAG_DIST_MONITOR_NAME))) { - erts_de_runlock(dep); - erts_proc_unlock(p, ERTS_PROC_LOCK_LINK); - ERTS_BIF_PREP_ERROR(ret, p, BADARG); - } - else { + { Eterm p_trgt, p_name, d_name, mon_ref; mon_ref = erts_make_ref(p); @@ -818,9 +837,7 @@ remote_monitor(Process *p, Eterm bifarg1, Eterm bifarg2, } break; default: - ASSERT(! "Invalid dsig prepare result"); - ERTS_BIF_PREP_ERROR(ret, p, EXC_INTERNAL_ERROR); - break; + ERTS_ASSERT(! "Invalid dsig prepare result"); } BIF_RET(ret); @@ -888,17 +905,17 @@ local_port: if (!erts_is_alive && remote_node != am_Noname) { goto badarg; /* Remote monitor from (this) undistributed node */ } - dep = erts_sysname_to_connected_dist_entry(remote_node); + dep = erts_find_or_insert_dist_entry(remote_node); if (dep == erts_this_dist_entry) { ret = local_name_monitor(BIF_P, BIF_ARG_1, name); } else { ret = remote_monitor(BIF_P, BIF_ARG_1, BIF_ARG_2, dep, name, 1); } + erts_deref_dist_entry(dep); } else { badarg: ERTS_BIF_PREP_ERROR(ret, BIF_P, BADARG); } - return ret; } @@ -1158,21 +1175,14 @@ BIF_RETTYPE unlink_1(BIF_ALIST_1) BIF_RET(am_true); } - code = erts_dsig_prepare(&dsd, dep, BIF_P, ERTS_DSP_NO_LOCK, 0); + code = erts_dsig_prepare(&dsd, dep, BIF_P, ERTS_PROC_LOCK_MAIN, + ERTS_DSP_NO_LOCK, 0, 0); switch (code) { case ERTS_DSIG_PREP_NOT_ALIVE: case ERTS_DSIG_PREP_NOT_CONNECTED: -#if 1 BIF_RET(am_true); -#else - /* - * This is how we used to do it, but the link is obviously not - * active, so I see no point in setting up a connection. - * /Rickard - */ - BIF_TRAP1(dunlink_trap, BIF_P, BIF_ARG_1); -#endif + case ERTS_DSIG_PREP_PENDING: case ERTS_DSIG_PREP_CONNECTED: erts_remove_dist_link(&dld, BIF_P->common.id, BIF_ARG_1, dep); code = erts_dsig_send_unlink(&dsd, BIF_P->common.id, BIF_ARG_1); @@ -1545,22 +1555,24 @@ BIF_RETTYPE exit_2(BIF_ALIST_2) DistEntry *dep; dep = external_pid_dist_entry(BIF_ARG_1); + ERTS_ASSERT(dep); if(dep == erts_this_dist_entry) BIF_RET(am_true); - code = erts_dsig_prepare(&dsd, dep, BIF_P, ERTS_DSP_NO_LOCK, 0); + code = erts_dsig_prepare(&dsd, dep, BIF_P, ERTS_PROC_LOCK_MAIN, + ERTS_DSP_NO_LOCK, 0, 1); switch (code) { case ERTS_DSIG_PREP_NOT_ALIVE: case ERTS_DSIG_PREP_NOT_CONNECTED: - BIF_TRAP2(dexit_trap, BIF_P, BIF_ARG_1, BIF_ARG_2); + BIF_RET(am_true); + case ERTS_DSIG_PREP_PENDING: case ERTS_DSIG_PREP_CONNECTED: code = erts_dsig_send_exit2(&dsd, BIF_P->common.id, BIF_ARG_1, BIF_ARG_2); if (code == ERTS_DSIG_SEND_YIELD) ERTS_BIF_YIELD_RETURN(BIF_P, am_true); BIF_RET(am_true); default: - ASSERT(! "Invalid dsig prepare result"); - BIF_ERROR(BIF_P, EXC_INTERNAL_ERROR); + ERTS_ASSERT(! "Invalid dsig prepare result"); } } else if (is_not_internal_pid(BIF_ARG_1)) { @@ -1732,7 +1744,6 @@ BIF_RETTYPE process_flag_2(BIF_ALIST_2) else if (BIF_ARG_1 == am_scheduler) { ErtsRunQueue *old, *new, *curr; Sint sched; - erts_aint32_t state; if (!is_small(BIF_ARG_2)) goto error; @@ -1741,23 +1752,23 @@ BIF_RETTYPE process_flag_2(BIF_ALIST_2) goto error; if (sched == 0) { + old = erts_bind_runq_proc(BIF_P, 0); new = NULL; - state = erts_atomic32_read_band_mb(&BIF_P->state, - ~ERTS_PSFLG_BOUND); } else { + int bound = !0; new = erts_schedid2runq(sched); - erts_atomic_set_nob(&BIF_P->run_queue, (erts_aint_t) new); - state = erts_atomic32_read_bor_mb(&BIF_P->state, - ERTS_PSFLG_BOUND); + old = erts_set_runq_proc(BIF_P, new, &bound); + if (!bound) + old = NULL; } + old_value = old ? make_small(old->ix+1) : make_small(0); + curr = erts_proc_sched_data(BIF_P)->run_queue; - old = (ERTS_PSFLG_BOUND & state) ? curr : NULL; ASSERT(!old || old == curr); - old_value = old ? make_small(old->ix+1) : make_small(0); if (new && new != curr) ERTS_BIF_YIELD_RETURN_X(BIF_P, old_value, am_scheduler); else @@ -1964,7 +1975,7 @@ ebif_bang_2(BIF_ALIST_2) * Send a message to Process, Port or Registered Process. * Returns non-negative reduction bump or negative result code. */ -#define SEND_TRAP (-1) +#define SEND_NOCONNECT (-1) #define SEND_YIELD (-2) #define SEND_YIELD_RETURN (-3) #define SEND_BADARG (-4) @@ -1980,20 +1991,22 @@ static Sint remote_send(Process *p, DistEntry *dep, { Sint res; int code; - ASSERT(is_atom(to) || is_external_pid(to)); ctx->dep = dep; - code = erts_dsig_prepare(&ctx->dsd, dep, p, ERTS_DSP_NO_LOCK, !ctx->suspend); + code = erts_dsig_prepare(&ctx->dsd, dep, p, ERTS_PROC_LOCK_MAIN, + ERTS_DSP_NO_LOCK, + !ctx->suspend, ctx->connect); switch (code) { case ERTS_DSIG_PREP_NOT_ALIVE: case ERTS_DSIG_PREP_NOT_CONNECTED: - res = SEND_TRAP; + res = SEND_NOCONNECT; break; case ERTS_DSIG_PREP_WOULD_SUSPEND: ASSERT(!ctx->suspend); res = SEND_YIELD; break; + case ERTS_DSIG_PREP_PENDING: case ERTS_DSIG_PREP_CONNECTED: { if (is_atom(to)) @@ -2170,6 +2183,7 @@ do_send(Process *p, Eterm to, Eterm msg, Eterm *refp, ErtsSendContext *ctx) } return ret_val; } else if (is_tuple(to)) { /* Remote send */ + int deref_dep = 0; int ret; tp = tuple_val(to); if (*tp != make_arityval(2)) @@ -2177,11 +2191,10 @@ do_send(Process *p, Eterm to, Eterm msg, Eterm *refp, ErtsSendContext *ctx) if (is_not_atom(tp[1]) || is_not_atom(tp[2])) return SEND_BADARG; - /* sysname_to_connected_dist_entry will return NULL if there - is no dist_entry or the dist_entry has no port, + /* erts_find_dist_entry will return NULL if there is no dist_entry but remote_send() will handle that. */ - dep = erts_sysname_to_connected_dist_entry(tp[2]); + dep = erts_find_dist_entry(tp[2]); if (dep == erts_this_dist_entry) { Eterm id; @@ -2205,13 +2218,20 @@ do_send(Process *p, Eterm to, Eterm msg, Eterm *refp, ErtsSendContext *ctx) } return 0; } + if (dep == NULL) { + dep = erts_find_or_insert_dist_entry(tp[2]); + ASSERT(dep != erts_this_dist_entry); + deref_dep = 1; + } + ctx->dsd.node = tp[2]; ret = remote_send(p, dep, tp[1], to, msg, ctx); if (ret == SEND_YIELD_CONTINUE) { - if (dep) - erts_ref_dist_entry(dep); - ctx->dep_to_deref = dep; + erts_ref_dist_entry(ctx->dep); + ctx->deref_dep = 1; } + if (deref_dep) + erts_deref_dist_entry(dep); return ret; } else { if (IS_TRACED_FL(p, F_TRACE_SEND)) @@ -2223,20 +2243,15 @@ do_send(Process *p, Eterm to, Eterm msg, Eterm *refp, ErtsSendContext *ctx) send_message: { ErtsProcLocks rp_locks = 0; - Sint res; if (p == rp) rp_locks |= ERTS_PROC_LOCK_MAIN; /* send to local process */ - res = erts_send_message(p, rp, &rp_locks, msg, 0); - if (erts_use_sender_punish) - res *= 4; - else - res = 0; + erts_send_message(p, rp, &rp_locks, msg, 0); erts_proc_unlock(rp, p == rp ? (rp_locks & ~ERTS_PROC_LOCK_MAIN) : rp_locks); - return res; + return 0; } } @@ -2251,7 +2266,6 @@ BIF_RETTYPE send_3(BIF_ALIST_3) Eterm msg = BIF_ARG_2; Eterm opts = BIF_ARG_3; - int connect = !0; Eterm l = opts; Sint result; @@ -2262,14 +2276,15 @@ BIF_RETTYPE send_3(BIF_ALIST_3) UseTmpHeap(sizeof(ErtsSendContext)/sizeof(Eterm), BIF_P); ctx->suspend = !0; - ctx->dep_to_deref = NULL; + ctx->connect = !0; + ctx->deref_dep = 0; ctx->return_term = am_ok; ctx->dss.reds = (Sint) (ERTS_BIF_REDS_LEFT(p) * TERM_TO_BINARY_LOOP_FACTOR); ctx->dss.phase = ERTS_DSIG_SEND_PHASE_INIT; while (is_list(l)) { if (CAR(list_val(l)) == am_noconnect) { - connect = 0; + ctx->connect = 0; } else if (CAR(list_val(l)) == am_nosuspend) { ctx->suspend = 0; } else { @@ -2291,8 +2306,8 @@ BIF_RETTYPE send_3(BIF_ALIST_3) result = do_send(p, to, msg, &ref, ctx); ERTS_MSACC_POP_STATE_M_X(); - if (result > 0) { - ERTS_VBUMP_REDS(p, result); + if (result >= 0) { + ERTS_VBUMP_REDS(p, 4); if (ERTS_IS_PROC_OUT_OF_REDS(p)) goto yield_return; ERTS_BIF_PREP_RET(retval, am_ok); @@ -2300,15 +2315,9 @@ BIF_RETTYPE send_3(BIF_ALIST_3) } switch (result) { - case 0: - /* May need to yield even though we do not bump reds here... */ - if (ERTS_IS_PROC_OUT_OF_REDS(p)) - goto yield_return; - ERTS_BIF_PREP_RET(retval, am_ok); - break; - case SEND_TRAP: - if (connect) { - ERTS_BIF_PREP_TRAP3(retval, dsend3_trap, p, to, msg, opts); + case SEND_NOCONNECT: + if (ctx->connect) { + ERTS_BIF_PREP_RET(retval, am_ok); } else { ERTS_BIF_PREP_RET(retval, am_noconnect); } @@ -2412,7 +2421,8 @@ Eterm erl_send(Process *p, Eterm to, Eterm msg) ref = NIL; #endif ctx->suspend = !0; - ctx->dep_to_deref = NULL; + ctx->connect = !0; + ctx->deref_dep = 0; ctx->return_term = msg; ctx->dss.reds = (Sint) (ERTS_BIF_REDS_LEFT(p) * TERM_TO_BINARY_LOOP_FACTOR); ctx->dss.phase = ERTS_DSIG_SEND_PHASE_INIT; @@ -2421,8 +2431,8 @@ Eterm erl_send(Process *p, Eterm to, Eterm msg) ERTS_MSACC_POP_STATE_M_X(); - if (result > 0) { - ERTS_VBUMP_REDS(p, result); + if (result >= 0) { + ERTS_VBUMP_REDS(p, 4); if (ERTS_IS_PROC_OUT_OF_REDS(p)) goto yield_return; ERTS_BIF_PREP_RET(retval, msg); @@ -2430,15 +2440,9 @@ Eterm erl_send(Process *p, Eterm to, Eterm msg) } switch (result) { - case 0: - /* May need to yield even though we do not bump reds here... */ - if (ERTS_IS_PROC_OUT_OF_REDS(p)) - goto yield_return; + case SEND_NOCONNECT: ERTS_BIF_PREP_RET(retval, msg); break; - case SEND_TRAP: - ERTS_BIF_PREP_TRAP2(retval, dsend2_trap, p, to, msg); - break; case SEND_YIELD: ERTS_BIF_PREP_YIELD2(retval, bif_export[BIF_send_2], p, to, msg); break; @@ -3097,7 +3101,7 @@ BIF_RETTYPE list_to_integer_2(BIF_ALIST_2) static int do_float_to_charbuf(Process *p, Eterm efloat, Eterm list, char *fbuf, int sizeof_fbuf) { - const static int arity_two = make_arityval(2); + Eterm arity_two = make_arityval(2); int decimals = SYS_DEFAULT_FLOAT_DECIMALS; int compact = 0; enum fmt_type_ { @@ -3461,7 +3465,7 @@ BIF_RETTYPE binary_to_float_1(BIF_ALIST_1) if (bit_offs) erts_copy_bits(bytes, bit_offs, 1, buf, 0, 1, size*8); else - memcpy(buf, bytes, size); + sys_memcpy(buf, bytes, size); buf[size] = '\0'; @@ -4195,10 +4199,10 @@ BIF_RETTYPE list_to_port_1(BIF_ALIST_1) buf[i] = '\0'; /* null terminal */ cp = &buf[0]; - if (strncmp("#Port<", cp, 6) != 0) + if (sys_strncmp("#Port<", cp, 6) != 0) goto bad; - cp += 6; /* strlen("#Port<") */ + cp += 6; /* sys_strlen("#Port<") */ if (sscanf(cp, "%u.%u>", (unsigned int*)&n, (unsigned int*)&p) < 2) goto bad; @@ -4384,23 +4388,24 @@ BIF_RETTYPE group_leader_2(BIF_ALIST_2) int code; ErtsDSigData dsd; dep = external_pid_dist_entry(BIF_ARG_2); + ERTS_ASSERT(dep); if(dep == erts_this_dist_entry) BIF_ERROR(BIF_P, BADARG); - code = erts_dsig_prepare(&dsd, dep, BIF_P, ERTS_DSP_NO_LOCK, 0); + code = erts_dsig_prepare(&dsd, dep, BIF_P, ERTS_PROC_LOCK_MAIN, + ERTS_DSP_NO_LOCK, 0, 1); switch (code) { case ERTS_DSIG_PREP_NOT_ALIVE: - BIF_RET(am_true); case ERTS_DSIG_PREP_NOT_CONNECTED: - BIF_TRAP2(dgroup_leader_trap, BIF_P, BIF_ARG_1, BIF_ARG_2); + BIF_RET(am_true); + case ERTS_DSIG_PREP_PENDING: case ERTS_DSIG_PREP_CONNECTED: code = erts_dsig_send_group_leader(&dsd, BIF_ARG_1, BIF_ARG_2); if (code == ERTS_DSIG_SEND_YIELD) ERTS_BIF_YIELD_RETURN(BIF_P, am_true); BIF_RET(am_true); default: - ASSERT(! "Invalid dsig prepare result"); - BIF_ERROR(BIF_P, EXC_INTERNAL_ERROR); + ERTS_ASSERT(! "Invalid dsig prepare result"); } } else if (is_internal_pid(BIF_ARG_2)) { @@ -4472,7 +4477,7 @@ BIF_RETTYPE group_leader_2(BIF_ALIST_2) BIF_ERROR(BIF_P, BADARG); } } - + BIF_RETTYPE system_flag_2(BIF_ALIST_2) { Sint n; @@ -4690,17 +4695,9 @@ BIF_RETTYPE system_flag_2(BIF_ALIST_2) BIF_RET(am_true); } else if (BIF_ARG_1 == am_scheduler_wall_time) { - if (BIF_ARG_2 == am_true || BIF_ARG_2 == am_false) { - erts_aint32_t new = BIF_ARG_2 == am_true ? 1 : 0; - erts_aint32_t old = erts_atomic32_xchg_nob(&sched_wall_time, - new); - Eterm ref = erts_sched_wall_time_request(BIF_P, 1, new, 0, 0); - ASSERT(is_value(ref)); - BIF_TRAP2(await_sched_wall_time_mod_trap, - BIF_P, - ref, - old ? am_true : am_false); - } + if (BIF_ARG_2 == am_true || BIF_ARG_2 == am_false) + BIF_TRAP1(system_flag_scheduler_wall_time_trap, + BIF_P, BIF_ARG_2); } else if (BIF_ARG_1 == am_dirty_cpu_schedulers_online) { Sint old_no; if (!is_small(BIF_ARG_2)) @@ -4807,11 +4804,24 @@ BIF_RETTYPE system_flag_2(BIF_ALIST_2) "scheduled for removal in Erlang/OTP 18. For more\n" "information see the erlang:system_flag/2 documentation.\n"); return erts_bind_schedulers(BIF_P, BIF_ARG_2); + } else if (ERTS_IS_ATOM_STR("erts_alloc", BIF_ARG_1)) { + return erts_alloc_set_dyn_param(BIF_P, BIF_ARG_2); } error: BIF_ERROR(BIF_P, BADARG); } +BIF_RETTYPE erts_internal_scheduler_wall_time_1(BIF_ALIST_1) +{ + erts_aint32_t new = BIF_ARG_1 == am_true ? 1 : 0; + erts_aint32_t old = erts_atomic32_xchg_nob(&sched_wall_time, + new); + Eterm ref = erts_sched_wall_time_request(BIF_P, 1, new, 0, 0); + ASSERT(is_value(ref)); + BIF_TRAP2(await_sched_wall_time_mod_trap, + BIF_P, ref, old ? am_true : am_false); +} + /**********************************************************************/ BIF_RETTYPE phash_2(BIF_ALIST_2) @@ -5070,8 +5080,10 @@ void erts_init_bif(void) await_proc_exit_trap = erts_export_put(am_erlang,am_await_proc_exit,3); await_port_send_result_trap = erts_export_put(am_erts_internal, am_await_port_send_result, 3); + system_flag_scheduler_wall_time_trap + = erts_export_put(am_erts_internal, am_system_flag_scheduler_wall_time, 1); await_sched_wall_time_mod_trap - = erts_export_put(am_erlang, am_await_sched_wall_time_modifications, 2); + = erts_export_put(am_erts_internal, am_await_sched_wall_time_modifications, 2); await_msacc_mod_trap = erts_export_put(am_erts_internal, am_await_microstate_accounting_modifications, 3); diff --git a/erts/emulator/beam/bif.tab b/erts/emulator/beam/bif.tab index f7b4451890..be653ee2a0 100644 --- a/erts/emulator/beam/bif.tab +++ b/erts/emulator/beam/bif.tab @@ -153,14 +153,12 @@ bif erlang:whereis/1 bif erlang:spawn_opt/1 bif erlang:setnode/2 bif erlang:setnode/3 -bif erlang:dist_exit/3 bif erlang:dist_get_stat/1 bif erlang:dist_ctrl_input_handler/2 bif erlang:dist_ctrl_put_data/2 bif erlang:dist_ctrl_get_data/1 bif erlang:dist_ctrl_get_data_notification/1 - # Static native functions in erts_internal bif erts_internal:port_info/1 bif erts_internal:port_info/2 @@ -187,6 +185,8 @@ bif erts_internal:system_check/1 bif erts_internal:release_literal_area_switch/0 +bif erts_internal:scheduler_wall_time/1 + # inet_db support bif erlang:port_set_data/2 bif erlang:port_get_data/1 @@ -376,9 +376,10 @@ bif ets:match_spec_run_r/3 # Bifs in os module. # -bif os:putenv/2 -bif os:getenv/0 -bif os:getenv/1 +bif os:get_env_var/1 +bif os:set_env_var/2 +bif os:unset_env_var/1 +bif os:list_env_vars/0 bif os:getpid/0 bif os:timestamp/0 bif os:system_time/0 @@ -550,9 +551,6 @@ bif binary:last/1 bif binary:at/2 bif binary:part/2 binary_binary_part_2 bif binary:part/3 binary_binary_part_3 -bif binary:bin_to_list/1 -bif binary:bin_to_list/2 -bif binary:bin_to_list/3 bif binary:list_to_bin/1 bif binary:copy/1 bif binary:copy/2 @@ -619,7 +617,6 @@ bif erlang:float_to_binary/2 bif erlang:binary_to_float/1 bif io:printable_range/0 -bif os:unsetenv/1 # # New in 17.0 @@ -629,7 +626,6 @@ bif re:inspect/2 ubif erlang:is_map/1 gcbif erlang:map_size/1 -bif maps:to_list/1 bif maps:find/2 bif maps:get/2 bif maps:from_list/1 @@ -684,10 +680,18 @@ bif math:floor/1 bif math:ceil/1 bif math:fmod/2 bif os:set_signal/2 -bif erts_internal:maps_to_list/2 # # New in 20.1 # - bif erlang:iolist_to_iovec/1 + +# +# New in 21.0 +# + +bif erts_internal:get_dflags/0 +bif erts_internal:new_connection/1 +bif erts_internal:abort_connection/2 +bif erts_internal:map_next/3 +bif ets:whereis/1 diff --git a/erts/emulator/beam/big.c b/erts/emulator/beam/big.c index 5eaf262cd8..c5cb268f09 100644 --- a/erts/emulator/beam/big.c +++ b/erts/emulator/beam/big.c @@ -2549,12 +2549,17 @@ int term_equals_2pow32(Eterm x) } } +static ERTS_INLINE int c2int_is_valid_char(byte ch, int base) { + if (base <= 10) + return (ch >= '0' && ch < ('0' + base)); + else + return (ch >= '0' && ch <= '9') + || (ch >= 'A' && ch < ('A' + base - 10)) + || (ch >= 'a' && ch < ('a' + base - 10)); +} + static ERTS_INLINE int c2int_is_invalid_char(byte ch, int base) { - return (ch < '0' - || (ch > ('0' + base - 1) - && !(base > 10 - && ((ch >= 'a' && ch < ('a' + base - 10)) - || (ch >= 'A' && ch < ('A' + base - 10)))))); + return !c2int_is_valid_char(ch, base); } static ERTS_INLINE byte c2int_digit_from_base(byte ch) { diff --git a/erts/emulator/beam/binary.c b/erts/emulator/beam/binary.c index ca3e48e205..95d324d2c1 100644 --- a/erts/emulator/beam/binary.c +++ b/erts/emulator/beam/binary.c @@ -60,14 +60,36 @@ erts_init_binary(void) } +static ERTS_INLINE +Eterm build_proc_bin(ErlOffHeap* ohp, Eterm* hp, Binary* bptr) +{ + ProcBin* pb = (ProcBin *) hp; + pb->thing_word = HEADER_PROC_BIN; + pb->size = bptr->orig_size; + pb->next = ohp->first; + ohp->first = (struct erl_off_heap_header*)pb; + pb->val = bptr; + pb->bytes = (byte*) bptr->orig_bytes; + pb->flags = 0; + OH_OVERHEAD(ohp, pb->size / sizeof(Eterm)); + + return make_binary(pb); +} + +/** @brief Initiate a ProcBin for a full Binary. + * @param hp must point to PROC_BIN_SIZE available heap words. + */ +Eterm erts_build_proc_bin(ErlOffHeap* ohp, Eterm* hp, Binary* bptr) +{ + return build_proc_bin(ohp, hp, bptr); +} + /* * Create a brand new binary from scratch. */ - Eterm new_binary(Process *p, byte *buf, Uint len) { - ProcBin* pb; Binary* bptr; if (len <= ERL_ONHEAP_BIN_LIMIT) { @@ -88,23 +110,7 @@ new_binary(Process *p, byte *buf, Uint len) sys_memcpy(bptr->orig_bytes, buf, len); } - /* - * Now allocate the ProcBin on the heap. - */ - pb = (ProcBin *) HAlloc(p, PROC_BIN_SIZE); - pb->thing_word = HEADER_PROC_BIN; - pb->size = len; - pb->next = MSO(p).first; - MSO(p).first = (struct erl_off_heap_header*)pb; - pb->val = bptr; - pb->bytes = (byte*) bptr->orig_bytes; - pb->flags = 0; - - /* - * Miscellaneous updates. Return the tagged binary. - */ - OH_OVERHEAD(&(MSO(p)), pb->size / sizeof(Eterm)); - return make_binary(pb); + return build_proc_bin(&MSO(p), HAlloc(p, PROC_BIN_SIZE), bptr); } /* @@ -113,7 +119,6 @@ new_binary(Process *p, byte *buf, Uint len) Eterm erts_new_mso_binary(Process *p, byte *buf, Uint len) { - ProcBin* pb; Binary* bptr; /* @@ -124,23 +129,7 @@ Eterm erts_new_mso_binary(Process *p, byte *buf, Uint len) sys_memcpy(bptr->orig_bytes, buf, len); } - /* - * Now allocate the ProcBin on the heap. - */ - pb = (ProcBin *) HAlloc(p, PROC_BIN_SIZE); - pb->thing_word = HEADER_PROC_BIN; - pb->size = len; - pb->next = MSO(p).first; - MSO(p).first = (struct erl_off_heap_header*)pb; - pb->val = bptr; - pb->bytes = (byte*) bptr->orig_bytes; - pb->flags = 0; - - /* - * Miscellaneous updates. Return the tagged binary. - */ - OH_OVERHEAD(&(MSO(p)), pb->size / sizeof(Eterm)); - return make_binary(pb); + return build_proc_bin(&MSO(p), HAlloc(p, PROC_BIN_SIZE), bptr); } /* @@ -964,7 +953,10 @@ HIPE_WRAPPER_BIF_DISABLE_GC(iolist_to_binary, 1) BIF_RETTYPE iolist_to_binary_1(BIF_ALIST_1) { if (is_binary(BIF_ARG_1)) { - BIF_RET(BIF_ARG_1); + if (binary_bitsize(BIF_ARG_1) == 0) { + BIF_RET(BIF_ARG_1); + } + BIF_ERROR(BIF_P, BADARG); } return erts_list_to_binary_bif(BIF_P, BIF_ARG_1, bif_export[BIF_iolist_to_binary_1]); } diff --git a/erts/emulator/beam/break.c b/erts/emulator/beam/break.c index a46063142e..3967f7f7fc 100644 --- a/erts/emulator/beam/break.c +++ b/erts/emulator/beam/break.c @@ -333,6 +333,12 @@ print_process_info(fmtfn_t to, void *to_arg, Process *p) erts_print(to, to_arg, "Heap unused: %bpu\n", (p->hend - p->htop)); erts_print(to, to_arg, "OldHeap unused: %bpu\n", (OLD_HEAP(p) == NULL) ? 0 : (OLD_HEND(p) - OLD_HTOP(p)) ); + erts_print(to, to_arg, "BinVHeap: %b64u\n", p->off_heap.overhead); + erts_print(to, to_arg, "OldBinVHeap: %b64u\n", BIN_OLD_VHEAP(p)); + erts_print(to, to_arg, "BinVHeap unused: %b64u\n", + BIN_VHEAP_SZ(p) - p->off_heap.overhead); + erts_print(to, to_arg, "OldBinVHeap unused: %b64u\n", + BIN_OLD_VHEAP_SZ(p) - BIN_OLD_VHEAP(p)); erts_print(to, to_arg, "Memory: %beu\n", erts_process_memory(p, !0)); if (garbing) { @@ -355,7 +361,7 @@ print_process_info(fmtfn_t to, void *to_arg, Process *p) static void print_garb_info(fmtfn_t to, void *to_arg, Process* p) { - /* ERTS_SMP: A scheduler is probably concurrently doing gc... */ + /* A scheduler is probably concurrently doing gc... */ if (!ERTS_IS_CRASH_DUMPING) return; erts_print(to, to_arg, "New heap start: %bpX\n", p->heap); @@ -485,9 +491,7 @@ loaded(fmtfn_t to, void *to_arg) static void dump_attributes(fmtfn_t to, void *to_arg, byte* ptr, int size) { - while (size-- > 0) { - erts_print(to, to_arg, "%02X", *ptr++); - } + erts_print_base64(to, to_arg, ptr, size); erts_print(to, to_arg, "\n"); } @@ -502,7 +506,7 @@ do_break(void) /* check if we're in console mode and, if so, halt immediately if break is called */ mode = erts_read_env("ERL_CONSOLE_MODE"); - if (mode && strcmp(mode, "window") != 0) + if (mode && sys_strcmp(mode, "window") != 0) erts_exit(0, ""); erts_free_read_env(mode); #endif /* __WIN32__ */ @@ -774,16 +778,16 @@ erl_crash_dump_v(char *file, int line, char* fmt, va_list args) * - write dump until alarm or file is written completely */ - if (erts_sys_getenv__("ERL_CRASH_DUMP_SECONDS", env, &envsz) != 0) { - env_erl_crash_dump_seconds_set = 0; - secs = -1; + if (erts_sys_explicit_8bit_getenv("ERL_CRASH_DUMP_SECONDS", env, &envsz) == 1) { + env_erl_crash_dump_seconds_set = 1; + secs = atoi(env); } else { - env_erl_crash_dump_seconds_set = 1; - secs = atoi(env); + env_erl_crash_dump_seconds_set = 0; + secs = -1; } if (secs == 0) { - return; + return; } /* erts_sys_prepare_crash_dump returns 1 if heart port is found, otherwise 0 @@ -799,7 +803,7 @@ erl_crash_dump_v(char *file, int line, char* fmt, va_list args) crash_dump_limit = ERTS_SINT64_MAX; envsz = sizeof(env); - if (erts_sys_getenv__("ERL_CRASH_DUMP_BYTES", env, &envsz) == 0) { + if (erts_sys_explicit_8bit_getenv("ERL_CRASH_DUMP_BYTES", env, &envsz) == 1) { Sint64 limit; char* endptr; errno = 0; @@ -812,7 +816,7 @@ erl_crash_dump_v(char *file, int line, char* fmt, va_list args) } } - if (erts_sys_getenv__("ERL_CRASH_DUMP",&dumpnamebuf[0],&dumpnamebufsize) != 0) + if (erts_sys_explicit_8bit_getenv("ERL_CRASH_DUMP",&dumpnamebuf[0],&dumpnamebufsize) != 1) dumpname = "erl_crash.dump"; else dumpname = &dumpnamebuf[0]; @@ -845,7 +849,7 @@ erl_crash_dump_v(char *file, int line, char* fmt, va_list args) } time(&now); - erts_cbprintf(to, to_arg, "=erl_crash_dump:0.4\n%s", ctime(&now)); + erts_cbprintf(to, to_arg, "=erl_crash_dump:0.5\n%s", ctime(&now)); if (file != NULL) erts_cbprintf(to, to_arg, "The error occurred in file %s, line %d\n", file, line); @@ -890,6 +894,21 @@ erl_crash_dump_v(char *file, int line, char* fmt, va_list args) erts_print_scheduler_info(to, to_arg, ERTS_SCHEDULER_IX(i)), erts_cbprintf(to, to_arg, "** crashed **\n")); } + for (i = 0; i < erts_no_dirty_cpu_schedulers; i++) { + ERTS_SYS_TRY_CATCH( + erts_print_scheduler_info(to, to_arg, ERTS_DIRTY_CPU_SCHEDULER_IX(i)), + erts_cbprintf(to, to_arg, "** crashed **\n")); + } + erts_cbprintf(to, to_arg, "=dirty_cpu_run_queue\n"); + erts_print_run_queue_info(to, to_arg, ERTS_DIRTY_CPU_RUNQ); + + for (i = 0; i < erts_no_dirty_io_schedulers; i++) { + ERTS_SYS_TRY_CATCH( + erts_print_scheduler_info(to, to_arg, ERTS_DIRTY_IO_SCHEDULER_IX(i)), + erts_cbprintf(to, to_arg, "** crashed **\n")); + } + erts_cbprintf(to, to_arg, "=dirty_io_run_queue\n"); + erts_print_run_queue_info(to, to_arg, ERTS_DIRTY_IO_RUNQ); #endif @@ -956,3 +975,28 @@ erl_crash_dump_v(char *file, int line, char* fmt, va_list args) erts_fprintf(stderr,"done\n"); } +void +erts_print_base64(fmtfn_t to, void *to_arg, byte* src, Uint size) +{ + static const byte base64_chars[] = + "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"; + + while (size >= 3) { + erts_putc(to, to_arg, base64_chars[src[0] >> 2]); + erts_putc(to, to_arg, base64_chars[((src[0] & 0x03) << 4) | (src[1] >> 4)]); + erts_putc(to, to_arg, base64_chars[((src[1] & 0x0f) << 2) | (src[2] >> 6)]); + erts_putc(to, to_arg, base64_chars[src[2] & 0x3f]); + size -= 3; + src += 3; + } + if (size == 1) { + erts_putc(to, to_arg, base64_chars[src[0] >> 2]); + erts_putc(to, to_arg, base64_chars[(src[0] & 0x03) << 4]); + erts_print(to, to_arg, "=="); + } else if (size == 2) { + erts_putc(to, to_arg, base64_chars[src[0] >> 2]); + erts_putc(to, to_arg, base64_chars[((src[0] & 0x03) << 4) | (src[1] >> 4)]); + erts_putc(to, to_arg, base64_chars[(src[1] & 0x0f) << 2]); + erts_putc(to, to_arg, '='); + } +} diff --git a/erts/emulator/beam/bs_instrs.tab b/erts/emulator/beam/bs_instrs.tab index 9f03b19731..94e0000c8b 100644 --- a/erts/emulator/beam/bs_instrs.tab +++ b/erts/emulator/beam/bs_instrs.tab @@ -919,9 +919,23 @@ i_bs_get_integer(Fail, Live, FlagsAndUnit, Ms, Sz, Dst) { } i_bs_get_utf8(Ctx, Fail, Dst) { + Eterm result; ErlBinMatchBuffer* mb = ms_matchbuffer($Ctx); - Eterm result = erts_bs_get_utf8(mb); + if (mb->size - mb->offset < 8) { + $FAIL($Fail); + } + if (BIT_OFFSET(mb->offset) != 0) { + result = erts_bs_get_utf8(mb); + } else { + byte b = mb->base[BYTE_OFFSET(mb->offset)]; + if (b < 128) { + result = make_small(b); + mb->offset += 8; + } else { + result = erts_bs_get_utf8(mb); + } + } if (is_non_value(result)) { $FAIL($Fail); } @@ -976,6 +990,9 @@ ctx_to_bin.execute() { Uint hole_size; Uint orig = mb->orig; ErlSubBin* sb = (ErlSubBin *) boxed_val(context); + /* Since we're going to overwrite the match state with the result, an + * ErlBinMatchState must be at least as large as an ErlSubBin. */ + ERTS_CT_ASSERT(sizeof(ErlSubBin) <= sizeof(ErlBinMatchState)); hole_size = 1 + header_arity(sb->thing_word) - ERL_SUB_BIN_SIZE; sb->thing_word = HEADER_SUB_BIN; sb->size = BYTE_OFFSET(size); diff --git a/erts/emulator/beam/copy.c b/erts/emulator/beam/copy.c index 10bf197405..7769a914db 100644 --- a/erts/emulator/beam/copy.c +++ b/erts/emulator/beam/copy.c @@ -611,7 +611,7 @@ Eterm copy_struct_x(Eterm obj, Uint sz, Eterm** hpp, ErlOffHeap* off_heap, Uint Eterm* htop; Eterm* hbot; Eterm* hp; - Eterm* objp; + Eterm* ERTS_RESTRICT objp; Eterm* tp; Eterm res; Eterm elem; @@ -1821,7 +1821,8 @@ all_clean: * * NOTE: Assumes that term is a tuple (ptr is an untagged tuple ptr). */ -Eterm copy_shallow(Eterm* ptr, Uint sz, Eterm** hpp, ErlOffHeap* off_heap) +Eterm copy_shallow(Eterm* ERTS_RESTRICT ptr, Uint sz, Eterm** hpp, + ErlOffHeap* off_heap) { Eterm* tp = ptr; Eterm* hp = *hpp; @@ -1985,7 +1986,7 @@ move_one_frag(Eterm** hpp, ErlHeapFragment* frag, ErlOffHeap* off_heap, int lite if (is_header(val)) { struct erl_off_heap_header* hdr = (struct erl_off_heap_header*)hp; ASSERT(ptr + header_arity(val) < end); - move_boxed(&ptr, val, &hp, &dummy_ref); + ptr = move_boxed(ptr, val, &hp, &dummy_ref); switch (val & _HEADER_SUBTAG_MASK) { case REF_SUBTAG: if (is_ordinary_ref_thing(hdr)) @@ -2002,7 +2003,7 @@ move_one_frag(Eterm** hpp, ErlHeapFragment* frag, ErlOffHeap* off_heap, int lite } else { /* must be a cons cell */ ASSERT(ptr+1 < end); - move_cons(&ptr, val, &hp, &dummy_ref); + move_cons(ptr, val, &hp, &dummy_ref); ptr += 2; } } diff --git a/erts/emulator/beam/dist.c b/erts/emulator/beam/dist.c index bc168fc58d..cd799e04b8 100644 --- a/erts/emulator/beam/dist.c +++ b/erts/emulator/beam/dist.c @@ -103,21 +103,12 @@ dist_msg_dbg(ErtsDistExternal *edep, char *what, byte *buf, int sz) -#define PASS_THROUGH 'p' /* This code should go */ - int erts_is_alive; /* System must be blocked on change */ int erts_dist_buf_busy_limit; /* distribution trap functions */ -Export* dsend2_trap = NULL; -Export* dsend3_trap = NULL; -/*Export* dsend_nosuspend_trap = NULL;*/ -Export* dlink_trap = NULL; -Export* dunlink_trap = NULL; Export* dmonitor_node_trap = NULL; -Export* dgroup_leader_trap = NULL; -Export* dexit_trap = NULL; Export* dmonitor_p_trap = NULL; /* local variables */ @@ -129,6 +120,7 @@ static void clear_dist_entry(DistEntry*); static int dsig_send_ctl(ErtsDSigData* dsdp, Eterm ctl, int force_busy); static void send_nodes_mon_msgs(Process *, Eterm, Eterm, Eterm, Eterm); static void init_nodes_monitors(void); +static Sint abort_connection(DistEntry* dep, Uint32 conn_id); static erts_atomic_t no_caches; static erts_atomic_t no_nodes; @@ -383,22 +375,24 @@ static void doit_node_link_net_exits(ErtsLink *lnk, void *vnecp) if (!rp) goto done; erts_proc_lock(rp, rp_locks); - rlnk = erts_remove_link(&ERTS_P_LINKS(rp), name); - if (rlnk != NULL) { - ASSERT(is_atom(rlnk->pid) && (rlnk->type == LINK_NODE)); - erts_destroy_link(rlnk); - } - n = ERTS_LINK_REFC(lnk); - for (i = 0; i < n; ++i) { - Eterm tup; - Eterm *hp; - ErtsMessage *msgp; - - msgp = erts_alloc_message_heap(rp, &rp_locks, - 3, &hp, &ohp); - tup = TUPLE2(hp, am_nodedown, name); - erts_queue_message(rp, rp_locks, msgp, tup, am_system); - } + if (!ERTS_PROC_IS_EXITING(rp)) { + rlnk = erts_remove_link(&ERTS_P_LINKS(rp), name); + if (rlnk != NULL) { + ASSERT(is_atom(rlnk->pid) && (rlnk->type == LINK_NODE)); + erts_destroy_link(rlnk); + } + n = ERTS_LINK_REFC(lnk); + for (i = 0; i < n; ++i) { + Eterm tup; + Eterm *hp; + ErtsMessage *msgp; + + msgp = erts_alloc_message_heap(rp, &rp_locks, + 3, &hp, &ohp); + tup = TUPLE2(hp, am_nodedown, name); + erts_queue_message(rp, rp_locks, msgp, tup, am_system); + } + } erts_proc_unlock(rp, rp_locks); } done: @@ -484,41 +478,55 @@ int erts_do_net_exits(DistEntry *dep, Eterm reason) if (dep == erts_this_dist_entry) { /* Net kernel has died (clean up!!) */ DistEntry *tdep; - int no_dist_ctrl = 0; + int no_dist_ctrl; + int no_pending; Eterm nd_reason = (reason == am_no_network ? am_no_network : am_net_kernel_terminated); + int i = 0; + Eterm *dist_ctrl; + DistEntry** pending; + + ERTS_UNDEF(dist_ctrl, NULL); + ERTS_UNDEF(pending, NULL); + erts_rwmtx_rlock(&erts_dist_table_rwmtx); - for (tdep = erts_hidden_dist_entries; tdep; tdep = tdep->next) - no_dist_ctrl++; - for (tdep = erts_visible_dist_entries; tdep; tdep = tdep->next) - no_dist_ctrl++; + no_dist_ctrl = (erts_no_of_hidden_dist_entries + + erts_no_of_visible_dist_entries); + no_pending = erts_no_of_pending_dist_entries; /* KILL all port controllers */ - if (no_dist_ctrl == 0) - erts_rwmtx_runlock(&erts_dist_table_rwmtx); - else { - Eterm def_buf[128]; - int i = 0; - Eterm *dist_ctrl; - - if (no_dist_ctrl <= sizeof(def_buf)/sizeof(def_buf[0])) - dist_ctrl = &def_buf[0]; - else - dist_ctrl = erts_alloc(ERTS_ALC_T_TMP, - sizeof(Eterm)*no_dist_ctrl); + if (no_dist_ctrl) { + dist_ctrl = erts_alloc(ERTS_ALC_T_TMP, + sizeof(Eterm)*no_dist_ctrl); for (tdep = erts_hidden_dist_entries; tdep; tdep = tdep->next) { ASSERT(is_internal_port(tdep->cid) || is_internal_pid(tdep->cid)); + ASSERT(i < no_dist_ctrl); dist_ctrl[i++] = tdep->cid; } for (tdep = erts_visible_dist_entries; tdep; tdep = tdep->next) { ASSERT(is_internal_port(tdep->cid) || is_internal_pid(tdep->cid)); + ASSERT(i < no_dist_ctrl); dist_ctrl[i++] = tdep->cid; } - erts_rwmtx_runlock(&erts_dist_table_rwmtx); + ASSERT(i == no_dist_ctrl); + } + if (no_pending) { + pending = erts_alloc(ERTS_ALC_T_TMP, sizeof(DistEntry*)*no_pending); + i = 0; + for (tdep = erts_pending_dist_entries; tdep; tdep = tdep->next) { + ASSERT(is_nil(tdep->cid)); + ASSERT(i < no_pending); + pending[i++] = tdep; + erts_ref_dist_entry(tdep); + } + ASSERT(i == no_pending); + } + erts_rwmtx_runlock(&erts_dist_table_rwmtx); - for (i = 0; i < no_dist_ctrl; i++) { + if (no_dist_ctrl) { + for (i = 0; i < no_dist_ctrl; i++) { if (is_internal_pid(dist_ctrl[i])) schedule_kill_dist_ctrl_proc(dist_ctrl[i]); else { @@ -532,11 +540,18 @@ int erts_do_net_exits(DistEntry *dep, Eterm reason) prt, dist_ctrl[i], nd_reason, NULL); } } - } + } + erts_free(ERTS_ALC_T_TMP, dist_ctrl); + } + + if (no_pending) { + for (i = 0; i < no_pending; i++) { + abort_connection(pending[i], pending[i]->connection_id); + erts_deref_dist_entry(pending[i]); + } + erts_free(ERTS_ALC_T_TMP, pending); + } - if (dist_ctrl != &def_buf[0]) - erts_free(ERTS_ALC_T_TMP, dist_ctrl); - } /* * When last dist ctrl exits, node will be taken @@ -573,13 +588,13 @@ int erts_do_net_exits(DistEntry *dep, Eterm reason) erts_port_task_abort(&dep->dist_cmd); } - if (dep->status & ERTS_DE_SFLG_EXITING) { + if (dep->state == ERTS_DE_STATE_EXITING) { #ifdef DEBUG ASSERT(erts_atomic32_read_nob(&dep->qflgs) & ERTS_DE_QFLG_EXIT); #endif } else { - dep->status |= ERTS_DE_SFLG_EXITING; + dep->state = ERTS_DE_STATE_EXITING; erts_mtx_lock(&dep->qlock); ASSERT(!(erts_atomic32_read_nob(&dep->qflgs) & ERTS_DE_QFLG_EXIT)); erts_atomic32_read_bor_relb(&dep->qflgs, ERTS_DE_QFLG_EXIT); @@ -613,7 +628,6 @@ int erts_do_net_exits(DistEntry *dep, Eterm reason) reason == am_normal ? am_connection_closed : reason); clear_dist_entry(dep); - } dec_no_nodes(); @@ -627,6 +641,14 @@ trap_function(Eterm func, int arity) return erts_export_put(am_erlang, func, arity); } +/* + * Sync with dist_util.erl: + * + * -record(erts_dflags, + * {default, mandatory, addable, rejectable, strict_order}). + */ +static Eterm erts_dflags_record; + void init_dist(void) { init_nodes_monitors(); @@ -638,18 +660,21 @@ void init_dist(void) erts_atomic_init_nob(&no_caches, 0); /* Lookup/Install all references to trap functions */ - dsend2_trap = trap_function(am_dsend,2); - dsend3_trap = trap_function(am_dsend,3); - /* dsend_nosuspend_trap = trap_function(am_dsend_nosuspend,2);*/ - dlink_trap = trap_function(am_dlink,1); - dunlink_trap = trap_function(am_dunlink,1); dmonitor_node_trap = trap_function(am_dmonitor_node,3); - dgroup_leader_trap = trap_function(am_dgroup_leader,2); - dexit_trap = trap_function(am_dexit, 2); dmonitor_p_trap = trap_function(am_dmonitor_p, 2); dist_ctrl_put_data_trap = erts_export_put(am_erts_internal, am_dist_ctrl_put_data, 2); + { + Eterm* hp = erts_alloc(ERTS_ALC_T_LITERAL, (1+6)*sizeof(Eterm)); + erts_dflags_record = TUPLE6(hp, am_erts_dflags, + make_small(DFLAG_DIST_DEFAULT), + make_small(DFLAG_DIST_MANDATORY), + make_small(DFLAG_DIST_ADDABLE), + make_small(DFLAG_DIST_REJECTABLE), + make_small(DFLAG_DIST_STRICT_ORDER)); + erts_set_literal_tag(&erts_dflags_record, hp, (1+6)); + } } #define ErtsDistOutputBuf2Binary(OB) \ @@ -664,6 +689,7 @@ alloc_dist_obuf(Uint size) obuf = (ErtsDistOutputBuf *) &bin->orig_bytes[0]; #ifdef DEBUG obuf->dbg_pattern = ERTS_DIST_OUTPUT_BUF_DBG_PATTERN; + obuf->alloc_endp = obuf->data + size; ASSERT(bin == ErtsDistOutputBuf2Binary(obuf)); #endif return obuf; @@ -684,31 +710,11 @@ size_obuf(ErtsDistOutputBuf *obuf) return bin->orig_size; } -static void clear_dist_entry(DistEntry *dep) +static ErtsDistOutputBuf* clear_de_out_queues(DistEntry* dep) { - Sint obufsize = 0; - ErtsAtomCache *cache; - ErtsProcList *suspendees; ErtsDistOutputBuf *obuf; - erts_de_rwlock(dep); - erts_atomic_set_nob(&dep->input_handler, - (erts_aint_t) NIL); - cache = dep->cache; - dep->cache = NULL; - -#ifdef DEBUG - erts_de_links_lock(dep); - ASSERT(!dep->nlinks); - ASSERT(!dep->node_links); - ASSERT(!dep->monitors); - erts_de_links_unlock(dep); -#endif - - erts_mtx_lock(&dep->qlock); - - erts_atomic64_set_nob(&dep->in, 0); - erts_atomic64_set_nob(&dep->out, 0); + ERTS_LC_ASSERT(erts_lc_mtx_is_locked(&dep->qlock)); if (!dep->out_queue.last) obuf = dep->finalized_out_queue.first; @@ -728,17 +734,13 @@ static void clear_dist_entry(DistEntry *dep) dep->tmp_out_queue.last = NULL; dep->finalized_out_queue.first = NULL; dep->finalized_out_queue.last = NULL; - dep->status = 0; - suspendees = get_suspended_on_de(dep, ERTS_DE_QFLGS_ALL); - - erts_mtx_unlock(&dep->qlock); - erts_atomic_set_nob(&dep->dist_cmd_scheduled, 0); - dep->send = NULL; - erts_de_rwunlock(dep); - erts_resume_processes(suspendees); + return obuf; +} - delete_cache(cache); +static void free_de_out_queues(DistEntry* dep, ErtsDistOutputBuf *obuf) +{ + Sint obufsize = 0; while (obuf) { ErtsDistOutputBuf *fobuf; @@ -750,13 +752,56 @@ static void clear_dist_entry(DistEntry *dep) if (obufsize) { erts_mtx_lock(&dep->qlock); - ASSERT(erts_atomic_read_nob(&dep->qsize) >= obufsize); + ASSERT(erts_atomic_read_nob(&dep->qsize) >= obufsize); erts_atomic_add_nob(&dep->qsize, (erts_aint_t) -obufsize); erts_mtx_unlock(&dep->qlock); } } +static void clear_dist_entry(DistEntry *dep) +{ + ErtsAtomCache *cache; + ErtsProcList *suspendees; + ErtsDistOutputBuf *obuf; + + erts_de_rwlock(dep); + erts_atomic_set_nob(&dep->input_handler, + (erts_aint_t) NIL); + cache = dep->cache; + dep->cache = NULL; + +#ifdef DEBUG + erts_de_links_lock(dep); + ASSERT(!dep->nlinks); + ASSERT(!dep->node_links); + ASSERT(!dep->monitors); + erts_de_links_unlock(dep); +#endif + + erts_mtx_lock(&dep->qlock); + + erts_atomic64_set_nob(&dep->in, 0); + erts_atomic64_set_nob(&dep->out, 0); + + obuf = clear_de_out_queues(dep); + dep->state = ERTS_DE_STATE_IDLE; + suspendees = get_suspended_on_de(dep, ERTS_DE_QFLGS_ALL); + + erts_mtx_unlock(&dep->qlock); + erts_atomic_set_nob(&dep->dist_cmd_scheduled, 0); + dep->send = NULL; + erts_de_rwunlock(dep); + + erts_resume_processes(suspendees); + + delete_cache(cache); + + free_de_out_queues(dep, obuf); + if (dep->transcode_ctx) + transcode_free_ctx(dep); +} + int erts_dsend_context_dtor(Binary* ctx_bin) { ErtsSendContext* ctx = ERTS_MAGIC_BIN_DATA(ctx_bin); @@ -772,8 +817,8 @@ int erts_dsend_context_dtor(Binary* ctx_bin) if (ctx->dss.phase >= ERTS_DSIG_SEND_PHASE_ALLOC && ctx->dss.obuf) { free_dist_obuf(ctx->dss.obuf); } - if (ctx->dep_to_deref) - erts_deref_dist_entry(ctx->dep_to_deref); + if (ctx->deref_dep) + erts_deref_dist_entry(ctx->dep); return 1; } @@ -852,6 +897,13 @@ erts_dsig_send_m_exit(ErtsDSigData *dsdp, Eterm watcher, Eterm watched, DeclareTmpHeapNoproc(ctl_heap,6); int res; + if (~dsdp->dep->flags & (DFLAG_DIST_MONITOR | DFLAG_DIST_MONITOR_NAME)) { + /* + * Receiver does not support DOP_MONITOR_P_EXIT (see dsig_send_monitor) + */ + return ERTS_DSIG_SEND_OK; + } + UseTmpHeapNoproc(6); ctl = TUPLE5(&ctl_heap[0], make_small(DOP_MONITOR_P_EXIT), @@ -879,6 +931,16 @@ erts_dsig_send_monitor(ErtsDSigData *dsdp, Eterm watcher, Eterm watched, DeclareTmpHeapNoproc(ctl_heap,5); int res; + if (~dsdp->dep->flags & (DFLAG_DIST_MONITOR | DFLAG_DIST_MONITOR_NAME)) { + /* + * Receiver does not support DOP_MONITOR_P. + * Just avoid sending it and by doing that reduce this monitor + * to only supervise the connection. This will work for simple c-nodes + * with a 1-to-1 relation between "Erlang process" and OS-process. + */ + return ERTS_DSIG_SEND_OK; + } + UseTmpHeapNoproc(5); ctl = TUPLE4(&ctl_heap[0], make_small(DOP_MONITOR_P), @@ -901,6 +963,13 @@ erts_dsig_send_demonitor(ErtsDSigData *dsdp, Eterm watcher, DeclareTmpHeapNoproc(ctl_heap,5); int res; + if (~dsdp->dep->flags & (DFLAG_DIST_MONITOR | DFLAG_DIST_MONITOR_NAME)) { + /* + * Receiver does not support DOP_DEMONITOR_P (see dsig_send_monitor) + */ + return ERTS_DSIG_SEND_OK; + } + UseTmpHeapNoproc(5); ctl = TUPLE4(&ctl_heap[0], make_small(DOP_DEMONITOR_P), @@ -1161,7 +1230,7 @@ erts_dsig_send_group_leader(ErtsDSigData *dsdp, Eterm leader, Eterm remote) # define PURIFY_MSG(msg) \ do { \ char buf__[1]; size_t bufsz__ = sizeof(buf__); \ - if (erts_sys_getenv_raw("VALGRIND_LOG_XML", buf__, &bufsz__) >= 0) { \ + if (erts_sys_explicit_8bit_getenv("VALGRIND_LOG_XML", buf__, &bufsz__) >= 0) { \ VALGRIND_PRINTF_XML("<erlang_error_log>" \ "%s, line %d: %s</erlang_error_log>\n", \ __FILE__, __LINE__, msg); \ @@ -1324,7 +1393,7 @@ int erts_net_message(Port *prt, /* This is tricky (we MUST force a distributed send) */ ErtsDSigData dsd; int code; - code = erts_dsig_prepare(&dsd, dep, NULL, ERTS_DSP_NO_LOCK, 0); + code = erts_dsig_prepare(&dsd, dep, NULL, 0, ERTS_DSP_NO_LOCK, 0, 0); if (code == ERTS_DSIG_PREP_CONNECTED) { code = erts_dsig_send_exit(&dsd, to, from, am_noproc); ASSERT(code == ERTS_DSIG_SEND_OK); @@ -1416,7 +1485,7 @@ int erts_net_message(Port *prt, if (!rp) { ErtsDSigData dsd; int code; - code = erts_dsig_prepare(&dsd, dep, NULL, ERTS_DSP_NO_LOCK, 0); + code = erts_dsig_prepare(&dsd, dep, NULL, 0, ERTS_DSP_NO_LOCK, 0, 0); if (code == ERTS_DSIG_PREP_CONNECTED) { code = erts_dsig_send_m_exit(&dsd, watcher, watched, ref, am_noproc); @@ -1879,33 +1948,32 @@ erts_dsig_send(ErtsDSigData *dsdp, struct erts_dsig_send_context* ctx) if (ctx->flags & DFLAG_DIST_HDR_ATOM_CACHE) { ctx->acmp = erts_get_atom_cache_map(ctx->c_p); - ctx->pass_through_size = 0; + ctx->max_finalize_prepend = 0; } else { ctx->acmp = NULL; - ctx->pass_through_size = 1; + ctx->max_finalize_prepend = 3; } #ifdef ERTS_DIST_MSG_DBG - erts_fprintf(stderr, ">>%s CTL: %T\n", ctx->pass_through_size ? "P" : " ", ctx->ctl); - if (is_value(ctx->msg)) - erts_fprintf(stderr, " MSG: %T\n", ctx->msg); + erts_fprintf(stderr, ">> CTL: %T\n", ctx->ctl); + if (is_value(ctx->msg)) + erts_fprintf(stderr, " MSG: %T\n", ctx->msg); #endif - ctx->data_size = ctx->pass_through_size; + ctx->data_size = ctx->max_finalize_prepend; erts_reset_atom_cache_map(ctx->acmp); erts_encode_dist_ext_size(ctx->ctl, ctx->flags, ctx->acmp, &ctx->data_size); - if (is_value(ctx->msg)) { - ctx->u.sc.wstack.wstart = NULL; - ctx->u.sc.flags = ctx->flags; - ctx->u.sc.level = 0; - ctx->phase = ERTS_DSIG_SEND_PHASE_MSG_SIZE; - } else { - ctx->phase = ERTS_DSIG_SEND_PHASE_ALLOC; - } - break; + if (is_non_value(ctx->msg)) { + ctx->phase = ERTS_DSIG_SEND_PHASE_ALLOC; + break; + } + ctx->u.sc.wstack.wstart = NULL; + ctx->u.sc.flags = ctx->flags; + ctx->u.sc.level = 0; + ctx->phase = ERTS_DSIG_SEND_PHASE_MSG_SIZE; case ERTS_DSIG_SEND_PHASE_MSG_SIZE: if (erts_encode_dist_ext_size_int(ctx->msg, ctx, &ctx->data_size)) { retval = ERTS_DSIG_SEND_CONTINUE; @@ -1920,23 +1988,25 @@ erts_dsig_send(ErtsDSigData *dsdp, struct erts_dsig_send_context* ctx) ctx->data_size += ctx->dhdr_ext_size; ctx->obuf = alloc_dist_obuf(ctx->data_size); - ctx->obuf->ext_endp = &ctx->obuf->data[0] + ctx->pass_through_size + ctx->dhdr_ext_size; + ctx->obuf->ext_endp = &ctx->obuf->data[0] + ctx->max_finalize_prepend + ctx->dhdr_ext_size; /* Encode internal version of dist header */ ctx->obuf->extp = erts_encode_ext_dist_header_setup(ctx->obuf->ext_endp, ctx->acmp); /* Encode control message */ erts_encode_dist_ext(ctx->ctl, &ctx->obuf->ext_endp, ctx->flags, ctx->acmp, NULL, NULL); - if (is_value(ctx->msg)) { - ctx->u.ec.flags = ctx->flags; - ctx->u.ec.level = 0; - ctx->u.ec.wstack.wstart = NULL; - ctx->phase = ERTS_DSIG_SEND_PHASE_MSG_ENCODE; - } else { - ctx->phase = ERTS_DSIG_SEND_PHASE_FIN; - } - break; - - case ERTS_DSIG_SEND_PHASE_MSG_ENCODE: + if (is_non_value(ctx->msg)) { + ctx->obuf->msg_start = NULL; + ctx->phase = ERTS_DSIG_SEND_PHASE_FIN; + break; + } + ctx->u.ec.flags = ctx->flags; + ctx->u.ec.hopefull_flags = 0; + ctx->u.ec.level = 0; + ctx->u.ec.wstack.wstart = NULL; + ctx->obuf->msg_start = ctx->obuf->ext_endp; + + ctx->phase = ERTS_DSIG_SEND_PHASE_MSG_ENCODE; + case ERTS_DSIG_SEND_PHASE_MSG_ENCODE: if (erts_encode_dist_ext(ctx->msg, &ctx->obuf->ext_endp, ctx->flags, ctx->acmp, &ctx->u.ec, &ctx->reds)) { retval = ERTS_DSIG_SEND_CONTINUE; goto done; @@ -1949,11 +2019,12 @@ erts_dsig_send(ErtsDSigData *dsdp, struct erts_dsig_send_context* ctx) int resume = 0; ASSERT(ctx->obuf->extp < ctx->obuf->ext_endp); - ASSERT(&ctx->obuf->data[0] <= ctx->obuf->extp - ctx->pass_through_size); + ASSERT(&ctx->obuf->data[0] <= ctx->obuf->extp - ctx->max_finalize_prepend); ASSERT(ctx->obuf->ext_endp <= &ctx->obuf->data[0] + ctx->data_size); ctx->data_size = ctx->obuf->ext_endp - ctx->obuf->extp; + ctx->obuf->hopefull_flags = ctx->u.ec.hopefull_flags; /* * Signal encoded; now verify that the connection still exists, * and if so enqueue the signal and schedule it for send. @@ -1961,9 +2032,9 @@ erts_dsig_send(ErtsDSigData *dsdp, struct erts_dsig_send_context* ctx) ctx->obuf->next = NULL; erts_de_rlock(dep); cid = dep->cid; - if (cid != dsdp->cid - || dep->connection_id != dsdp->connection_id - || dep->status & ERTS_DE_SFLG_EXITING) { + if (dep->state == ERTS_DE_STATE_EXITING + || dep->state == ERTS_DE_STATE_IDLE + || dep->connection_id != dsdp->connection_id) { /* Not the same connection as when we started; drop message... */ erts_de_runlock(dep); free_dist_obuf(ctx->obuf); @@ -2037,8 +2108,13 @@ erts_dsig_send(ErtsDSigData *dsdp, struct erts_dsig_send_context* ctx) } erts_mtx_unlock(&dep->qlock); - if (is_internal_port(dep->cid)) - erts_schedule_dist_command(NULL, dep); + if (dep->state != ERTS_DE_STATE_PENDING) { + if (is_internal_port(dep->cid)) + erts_schedule_dist_command(NULL, dep); + } + else { + notify_proc = NIL; + } erts_de_runlock(dep); if (is_internal_pid(notify_proc)) notify_dist_data(ctx->c_p, notify_proc); @@ -2203,7 +2279,6 @@ dist_port_commandv(Port *prt, ErtsDistOutputBuf *obuf) #endif #define ERTS_PORT_REDS_DIST_CMD_START 5 -#define ERTS_PORT_REDS_DIST_CMD_FINALIZE 3 #define ERTS_PORT_REDS_DIST_CMD_EXIT 200 #define ERTS_PORT_REDS_DIST_CMD_RESUMED 5 #define ERTS_PORT_REDS_DIST_CMD_DATA(SZ) \ @@ -2212,10 +2287,10 @@ dist_port_commandv(Port *prt, ErtsDistOutputBuf *obuf) : ((((Sint) (SZ)) >> 10) & ((Sint) ERTS_PORT_REDS_MASK__))) int -erts_dist_command(Port *prt, int reds_limit) +erts_dist_command(Port *prt, int initial_reds) { - Sint reds = ERTS_PORT_REDS_DIST_CMD_START; - Uint32 status; + Sint reds = initial_reds - ERTS_PORT_REDS_DIST_CMD_START; + enum dist_entry_state state; Uint32 flags; Sint qsize, obufsize = 0; ErtsDistOutputQueue oq, foq; @@ -2230,15 +2305,18 @@ erts_dist_command(Port *prt, int reds_limit) erts_de_rlock(dep); flags = dep->flags; - status = dep->status; + state = dep->state; send = dep->send; erts_de_runlock(dep); - if (status & ERTS_DE_SFLG_EXITING) { + if (state == ERTS_DE_STATE_EXITING) { erts_deliver_port_exit(prt, prt->common.id, am_killed, 0, 1); - return reds + ERTS_PORT_REDS_DIST_CMD_EXIT; + reds -= ERTS_PORT_REDS_DIST_CMD_EXIT; + return initial_reds - reds; } + ASSERT(state != ERTS_DE_STATE_PENDING); + ASSERT(send); /* @@ -2263,7 +2341,7 @@ erts_dist_command(Port *prt, int reds_limit) sched_flags = erts_atomic32_read_nob(&prt->sched.flags); - if (reds > reds_limit) + if (reds < 0) goto preempted; if (!(sched_flags & ERTS_PTS_FLG_BUSY_PORT) && foq.first) { @@ -2278,13 +2356,13 @@ erts_dist_command(Port *prt, int reds_limit) erts_fprintf(stderr, ">> "); bw(foq.first->extp, size); #endif - reds += ERTS_PORT_REDS_DIST_CMD_DATA(size); - fob = foq.first; - obufsize += size_obuf(fob); - foq.first = foq.first->next; - free_dist_obuf(fob); + reds -= ERTS_PORT_REDS_DIST_CMD_DATA(size); + fob = foq.first; + obufsize += size_obuf(fob); + foq.first = foq.first->next; + free_dist_obuf(fob); sched_flags = erts_atomic32_read_nob(&prt->sched.flags); - preempt = reds > reds_limit || (sched_flags & ERTS_PTS_FLG_EXIT); + preempt = reds < 0 || (sched_flags & ERTS_PTS_FLG_EXIT); if (sched_flags & ERTS_PTS_FLG_BUSY_PORT) break; } while (foq.first && !preempt); @@ -2297,81 +2375,71 @@ erts_dist_command(Port *prt, int reds_limit) if (sched_flags & ERTS_PTS_FLG_BUSY_PORT) { if (oq.first) { ErtsDistOutputBuf *ob; - int preempt; + ErtsDistOutputBuf *last_finalized = NULL; finalize_only: - preempt = 0; ob = oq.first; ASSERT(ob); do { - ob->extp = erts_encode_ext_dist_header_finalize(ob->extp, - dep->cache, - flags); - if (!(flags & DFLAG_DIST_HDR_ATOM_CACHE)) - *--ob->extp = PASS_THROUGH; /* Old node; 'pass through' - needed */ - ASSERT(&ob->data[0] <= ob->extp && ob->extp < ob->ext_endp); - reds += ERTS_PORT_REDS_DIST_CMD_FINALIZE; - preempt = reds > reds_limit; - if (preempt) - break; - ob = ob->next; + reds = erts_encode_ext_dist_header_finalize(ob, dep, flags, reds); + if (reds < 0) + break; + last_finalized = ob; + ob = ob->next; } while (ob); - /* - * At least one buffer was finalized; if we got preempted, - * ob points to the last buffer that we finalized. - */ - if (foq.last) - foq.last->next = oq.first; - else - foq.first = oq.first; - if (!preempt) { - /* All buffers finalized */ - foq.last = oq.last; - oq.first = oq.last = NULL; - } - else { - /* Not all buffers finalized; split oq. */ - foq.last = ob; - oq.first = ob->next; - if (oq.first) - ob->next = NULL; - else - oq.last = NULL; - } - if (preempt) - goto preempted; + if (last_finalized) { + /* + * At least one buffer was finalized; if we got preempted, + * ob points to the next buffer to continue finalize. + */ + if (foq.last) + foq.last->next = oq.first; + else + foq.first = oq.first; + foq.last = last_finalized; + if (!ob) { + /* All buffers finalized */ + ASSERT(foq.last == oq.last); + ASSERT(foq.last->next == NULL); + oq.first = oq.last = NULL; + } + else { + /* Not all buffers finalized; split oq. */ + ASSERT(foq.last->next == ob); + foq.last->next = NULL; + oq.first = ob; + } + } + if (reds <= 0) + goto preempted; } } else { int de_busy; int preempt = 0; while (oq.first && !preempt) { - ErtsDistOutputBuf *fob; - Uint size; - oq.first->extp - = erts_encode_ext_dist_header_finalize(oq.first->extp, - dep->cache, - flags); - reds += ERTS_PORT_REDS_DIST_CMD_FINALIZE; - if (!(flags & DFLAG_DIST_HDR_ATOM_CACHE)) - *--oq.first->extp = PASS_THROUGH; /* Old node; 'pass through' - needed */ - ASSERT(&oq.first->data[0] <= oq.first->extp - && oq.first->extp < oq.first->ext_endp); - size = (*send)(prt, oq.first); + ErtsDistOutputBuf *fob; + Uint size; + reds = erts_encode_ext_dist_header_finalize(oq.first, dep, flags, reds); + if (reds < 0) { + preempt = 1; + break; + } + ASSERT(&oq.first->data[0] <= oq.first->extp + && oq.first->extp <= oq.first->ext_endp); + size = (*send)(prt, oq.first); erts_atomic64_inc_nob(&dep->out); - esdp->io.out += (Uint64) size; + esdp->io.out += (Uint64) size; #ifdef ERTS_RAW_DIST_MSG_DBG erts_fprintf(stderr, ">> "); bw(oq.first->extp, size); #endif - reds += ERTS_PORT_REDS_DIST_CMD_DATA(size); - fob = oq.first; - obufsize += size_obuf(fob); - oq.first = oq.first->next; - free_dist_obuf(fob); + reds -= ERTS_PORT_REDS_DIST_CMD_DATA(size); + fob = oq.first; + obufsize += size_obuf(fob); + oq.first = oq.first->next; + free_dist_obuf(fob); sched_flags = erts_atomic32_read_nob(&prt->sched.flags); - preempt = reds > reds_limit || (sched_flags & ERTS_PTS_FLG_EXIT); + preempt = reds <= 0 || (sched_flags & ERTS_PTS_FLG_EXIT); if ((sched_flags & ERTS_PTS_FLG_BUSY_PORT) && oq.first && !preempt) goto finalize_only; } @@ -2411,7 +2479,7 @@ erts_dist_command(Port *prt, int reds_limit) erts_mtx_unlock(&dep->qlock); resumed = erts_resume_processes(suspendees); - reds += resumed*ERTS_PORT_REDS_DIST_CMD_RESUMED; + reds -= resumed*ERTS_PORT_REDS_DIST_CMD_RESUMED; } else erts_mtx_unlock(&dep->qlock); @@ -2434,8 +2502,7 @@ erts_dist_command(Port *prt, int reds_limit) erts_mtx_unlock(&dep->qlock); } - ASSERT(foq.first || !foq.last); - ASSERT(!foq.first || foq.last); + ASSERT(!!foq.first == !!foq.last); ASSERT(!dep->finalized_out_queue.first); ASSERT(!dep->finalized_out_queue.last); @@ -2445,10 +2512,10 @@ erts_dist_command(Port *prt, int reds_limit) } /* Avoid wrapping reduction counter... */ - if (reds > INT_MAX/2) - reds = INT_MAX/2; + if (reds < INT_MIN/2) + reds = INT_MIN/2; - return reds; + return initial_reds - reds; preempted: /* @@ -2456,8 +2523,7 @@ erts_dist_command(Port *prt, int reds_limit) * since last call to driver. */ - ASSERT(oq.first || !oq.last); - ASSERT(!oq.first || oq.last); + ASSERT(!!oq.first == !!oq.last); if (sched_flags & ERTS_PTS_FLG_EXIT) { /* @@ -2481,12 +2547,6 @@ erts_dist_command(Port *prt, int reds_limit) foq.first = NULL; foq.last = NULL; - -#ifdef DEBUG - erts_mtx_lock(&dep->qlock); - ASSERT(erts_atomic_read_nob(&dep->qsize) == obufsize); - erts_mtx_unlock(&dep->qlock); -#endif } else { if (oq.first) { @@ -2776,7 +2836,8 @@ BIF_RETTYPE dist_ctrl_get_data_1(BIF_ALIST_1) { DistEntry *dep = ERTS_PROC_GET_DIST_ENTRY(BIF_P); - int reds = 1; + const Sint initial_reds = ERTS_BIF_REDS_LEFT(BIF_P); + Sint reds = initial_reds; ErtsDistOutputBuf *obuf; Eterm *hp; ProcBin *pb; @@ -2790,7 +2851,7 @@ dist_ctrl_get_data_1(BIF_ALIST_1) erts_de_rlock(dep); - if (dep->status & ERTS_DE_SFLG_EXITING) + if (dep->state == ERTS_DE_STATE_EXITING) goto return_none; ASSERT(dep->cid == BIF_P->common.id); @@ -2807,6 +2868,7 @@ dist_ctrl_get_data_1(BIF_ALIST_1) { if (!dep->tmp_out_queue.first) { ASSERT(!dep->tmp_out_queue.last); + ASSERT(!dep->transcode_ctx); qsize = erts_atomic_read_acqb(&dep->qsize); if (qsize > 0) { erts_mtx_lock(&dep->qlock); @@ -2824,21 +2886,18 @@ dist_ctrl_get_data_1(BIF_ALIST_1) erts_de_runlock(dep); BIF_RET(am_none); } - else { - obuf = dep->tmp_out_queue.first; - dep->tmp_out_queue.first = obuf->next; - if (!obuf->next) - dep->tmp_out_queue.last = NULL; + + obuf = dep->tmp_out_queue.first; + reds = erts_encode_ext_dist_header_finalize(obuf, dep, dep->flags, reds); + if (reds < 0) { + erts_de_runlock(dep); + ERTS_BIF_YIELD1(bif_export[BIF_dist_ctrl_get_data_1], + BIF_P, BIF_ARG_1); } - obuf->extp = erts_encode_ext_dist_header_finalize(obuf->extp, - dep->cache, - dep->flags); - reds += ERTS_PORT_REDS_DIST_CMD_FINALIZE; - if (!(dep->flags & DFLAG_DIST_HDR_ATOM_CACHE)) - *--obuf->extp = PASS_THROUGH; /* 'pass through' needed */ - ASSERT(&obuf->data[0] <= obuf->extp - && obuf->extp < obuf->ext_endp); + dep->tmp_out_queue.first = obuf->next; + if (!obuf->next) + dep->tmp_out_queue.last = NULL; } erts_atomic64_inc_nob(&dep->out); @@ -2866,11 +2925,11 @@ dist_ctrl_get_data_1(BIF_ALIST_1) erts_mtx_unlock(&dep->qlock); if (resume_procs) { int resumed = erts_resume_processes(resume_procs); - reds += resumed*ERTS_PORT_REDS_DIST_CMD_RESUMED; + reds -= resumed*ERTS_PORT_REDS_DIST_CMD_RESUMED; } } - BIF_RET2(make_binary(pb), reds); + BIF_RET2(make_binary(pb), (initial_reds - reds)); } void @@ -2892,24 +2951,31 @@ erts_dist_port_not_busy(Port *prt) erts_schedule_dist_command(prt, NULL); } +static void kill_connection(DistEntry *dep) +{ + ERTS_LC_ASSERT(erts_lc_is_de_rwlocked(dep)); + ASSERT(dep->state == ERTS_DE_STATE_CONNECTED); + + dep->state = ERTS_DE_STATE_EXITING; + erts_mtx_lock(&dep->qlock); + ASSERT(!(erts_atomic32_read_nob(&dep->qflgs) & ERTS_DE_QFLG_EXIT)); + erts_atomic32_read_bor_nob(&dep->qflgs, ERTS_DE_QFLG_EXIT); + erts_mtx_unlock(&dep->qlock); + + if (is_internal_port(dep->cid)) + erts_schedule_dist_command(NULL, dep); + else if (is_internal_pid(dep->cid)) + schedule_kill_dist_ctrl_proc(dep->cid); +} + void erts_kill_dist_connection(DistEntry *dep, Uint32 connection_id) { erts_de_rwlock(dep); if (connection_id == dep->connection_id - && !(dep->status & ERTS_DE_SFLG_EXITING)) { - - dep->status |= ERTS_DE_SFLG_EXITING; - - erts_mtx_lock(&dep->qlock); - ASSERT(!(erts_atomic32_read_nob(&dep->qflgs) & ERTS_DE_QFLG_EXIT)); - erts_atomic32_read_bor_nob(&dep->qflgs, ERTS_DE_QFLG_EXIT); - erts_mtx_unlock(&dep->qlock); + && dep->state == ERTS_DE_STATE_CONNECTED) { - if (is_internal_port(dep->cid)) - erts_schedule_dist_command(NULL, dep); - else if (is_internal_pid(dep->cid)) - schedule_kill_dist_ctrl_proc(dep->cid); + kill_connection(dep); } erts_de_rwunlock(dep); } @@ -3069,6 +3135,10 @@ int distribution_info(fmtfn_t to, void *arg) /* Called by break handler */ info_dist_entry(to, arg, dep, 0, 1); } + for (dep = erts_pending_dist_entries; dep; dep = dep->next) { + info_dist_entry(to, arg, dep, 0, 0); + } + for (dep = erts_not_connected_dist_entries; dep; dep = dep->next) { if (dep != erts_this_dist_entry) { info_dist_entry(to, arg, dep, 0, 0); @@ -3091,7 +3161,6 @@ int distribution_info(fmtfn_t to, void *arg) /* Called by break handler */ monitor_node -- turn on/off node monitoring node controller only: - dist_exit/3 -- send exit signals from remote to local process dist_link/2 -- link a remote process to a local dist_unlink/2 -- unlink a remote from a local ****************************************************************************/ @@ -3101,15 +3170,6 @@ int distribution_info(fmtfn_t to, void *arg) /* Called by break handler */ /********************************************************************** ** Set the node name of current node fail if node already is set. ** setnode(name@host, Creation) - ** loads functions pointer to trap_functions from module erlang. - ** erlang:dsend/2 - ** erlang:dlink/1 - ** erlang:dunlink/1 - ** erlang:dmonitor_node/3 - ** erlang:dgroup_leader/2 - ** erlang:dexit/2 - ** -- are these needed ? - ** dexit/1 ***********************************************************************/ BIF_RETTYPE setnode_2(BIF_ALIST_2) @@ -3133,15 +3193,8 @@ BIF_RETTYPE setnode_2(BIF_ALIST_2) goto error; /* Check that all trap functions are defined !! */ - if (dsend2_trap->addressv[0] == NULL || - dsend3_trap->addressv[0] == NULL || - /* dsend_nosuspend_trap->address == NULL ||*/ - dlink_trap->addressv[0] == NULL || - dunlink_trap->addressv[0] == NULL || - dmonitor_node_trap->addressv[0] == NULL || - dgroup_leader_trap->addressv[0] == NULL || - dmonitor_p_trap->addressv[0] == NULL || - dexit_trap->addressv[0] == NULL) { + if (dmonitor_node_trap->addressv[0] == NULL || + dmonitor_p_trap->addressv[0] == NULL) { goto error; } @@ -3218,6 +3271,8 @@ BIF_RETTYPE setnode_3(BIF_ALIST_3) ErtsProcLocks proc_unlock = 0; Process *proc; Port *pp = NULL; + Eterm notify_proc; + erts_aint32_t qflgs; /* * Check and pick out arguments @@ -3245,21 +3300,25 @@ BIF_RETTYPE setnode_3(BIF_ALIST_3) if (!is_atom(ic) || !is_atom(oc)) goto badarg; - /* DFLAG_EXTENDED_REFERENCES is compulsory from R9 and forward */ - if (!(DFLAG_EXTENDED_REFERENCES & flags)) { + if (~flags & DFLAG_DIST_MANDATORY) { erts_dsprintf_buf_t *dsbufp = erts_create_logger_dsbuf(); erts_dsprintf(dsbufp, "%T", BIF_P->common.id); if (BIF_P->common.u.alive.reg) erts_dsprintf(dsbufp, " (%T)", BIF_P->common.u.alive.reg->name); erts_dsprintf(dsbufp, " attempted to enable connection to node %T " - "which is not able to handle extended references.\n", + "which does not support all mandatory capabilities.\n", BIF_ARG_1); erts_send_error_to_logger(BIF_P->group_leader, dsbufp); goto badarg; } /* + * ToDo: Should we not pass connection_id as well + * to make sure it's the right connection we commit. + */ + + /* * Arguments seem to be in order. */ @@ -3302,6 +3361,23 @@ BIF_RETTYPE setnode_3(BIF_ALIST_3) goto badarg; } + if (dep->state == ERTS_DE_STATE_EXITING) { + /* Suspend on dist entry waiting for the exit to finish */ + ErtsProcList *plp = erts_proclist_create(BIF_P); + plp->next = NULL; + erts_suspend(BIF_P, ERTS_PROC_LOCK_MAIN, NULL); + erts_mtx_lock(&dep->qlock); + erts_proclist_store_last(&dep->suspended, plp); + erts_mtx_unlock(&dep->qlock); + goto yield; + } + if (dep->state != ERTS_DE_STATE_PENDING) { + if (dep->state == ERTS_DE_STATE_IDLE) + erts_set_dist_entry_pending(dep); + else + goto badarg; + } + if (is_not_nil(dep->cid)) goto badarg; @@ -3334,7 +3410,7 @@ BIF_RETTYPE setnode_3(BIF_ALIST_3) goto done; /* Already set */ } - if (dep->status & ERTS_DE_SFLG_EXITING) { + if (dep->state == ERTS_DE_STATE_EXITING) { /* Suspend on dist entry waiting for the exit to finish */ ErtsProcList *plp = erts_proclist_create(BIF_P); plp->next = NULL; @@ -3344,8 +3420,12 @@ BIF_RETTYPE setnode_3(BIF_ALIST_3) erts_mtx_unlock(&dep->qlock); goto yield; } - - ASSERT(!(dep->status & ERTS_DE_SFLG_EXITING)); + if (dep->state != ERTS_DE_STATE_PENDING) { + if (dep->state == ERTS_DE_STATE_IDLE) + erts_set_dist_entry_pending(dep); + else + goto badarg; + } if (pp->dist_entry || is_not_nil(dep->cid)) goto badarg; @@ -3376,7 +3456,8 @@ BIF_RETTYPE setnode_3(BIF_ALIST_3) dep->creation = 0; #ifdef DEBUG - ASSERT(erts_atomic_read_nob(&dep->qsize) == 0); + ASSERT(erts_atomic_read_nob(&dep->qsize) == 0 + || (dep->state == ERTS_DE_STATE_PENDING)); #endif if (flags & DFLAG_DIST_HDR_ATOM_CACHE) @@ -3384,7 +3465,26 @@ BIF_RETTYPE setnode_3(BIF_ALIST_3) erts_set_dist_entry_connected(dep, BIF_ARG_2, flags); + notify_proc = NIL; + if (erts_atomic_read_nob(&dep->qsize)) { + if (is_internal_port(dep->cid)) { + erts_schedule_dist_command(NULL, dep); + } + else { + qflgs = erts_atomic32_read_nob(&dep->qflgs); + if (qflgs & ERTS_DE_QFLG_REQ_INFO) { + qflgs = erts_atomic32_read_band_mb(&dep->qflgs, + ~ERTS_DE_QFLG_REQ_INFO); + if (qflgs & ERTS_DE_QFLG_REQ_INFO) { + notify_proc = dep->cid; + ASSERT(is_internal_pid(notify_proc)); + } + } + } + } erts_de_rwunlock(dep); + if (is_internal_pid(notify_proc)) + notify_dist_data(BIF_P, notify_proc); ERTS_BIF_PREP_RET(ret, erts_make_dhandle(BIF_P, dep)); @@ -3426,79 +3526,189 @@ BIF_RETTYPE setnode_3(BIF_ALIST_3) goto done; } +BIF_RETTYPE erts_internal_get_dflags_0(BIF_ALIST_0) +{ + return erts_dflags_record; +} -/**********************************************************************/ -/* dist_exit(Local, Term, Remote) -> Bool */ +BIF_RETTYPE erts_internal_new_connection_1(BIF_ALIST_1) +{ + DistEntry* dep; + Uint32 conn_id; + Eterm* hp; + Eterm dhandle; + + if (is_not_atom(BIF_ARG_1)) { + BIF_ERROR(BIF_P, BADARG); + } + dep = erts_find_or_insert_dist_entry(BIF_ARG_1); + + if (dep == erts_this_dist_entry) { + erts_deref_dist_entry(dep); + BIF_ERROR(BIF_P, BADARG); + } + + erts_de_rwlock(dep); + + switch (dep->state) { + case ERTS_DE_STATE_PENDING: + case ERTS_DE_STATE_CONNECTED: + conn_id = dep->connection_id; + break; + case ERTS_DE_STATE_IDLE: + erts_set_dist_entry_pending(dep); + conn_id = dep->connection_id; + break; + case ERTS_DE_STATE_EXITING: + conn_id = (dep->connection_id + 1) & ERTS_DIST_CON_ID_MASK; + break; + default: + erts_exit(ERTS_ABORT_EXIT, "Invalid dep->state (%d)\n", dep->state); + } + erts_de_rwunlock(dep); + hp = HAlloc(BIF_P, 3 + ERTS_MAGIC_REF_THING_SIZE); + dhandle = erts_build_dhandle(&hp, &BIF_P->off_heap, dep); + erts_deref_dist_entry(dep); + BIF_RET(TUPLE2(hp, make_small(conn_id), dhandle)); +} -BIF_RETTYPE dist_exit_3(BIF_ALIST_3) +static Sint abort_connection(DistEntry* dep, Uint32 conn_id) { - Eterm local; - Eterm remote; - DistEntry *rdep; + erts_de_rwlock(dep); - local = BIF_ARG_1; - remote = BIF_ARG_3; + if (dep->connection_id != conn_id) + ; + else if (dep->state == ERTS_DE_STATE_CONNECTED) { + kill_connection(dep); + } + else if (dep->state == ERTS_DE_STATE_PENDING) { + NetExitsContext nec = {dep}; + ErtsLink *nlinks; + ErtsLink *node_links; + ErtsMonitor *monitors; + ErtsAtomCache *cache; + ErtsDistOutputBuf *obuf; + ErtsProcList *resume_procs; + Sint reds = 0; + + ASSERT(is_nil(dep->cid)); - /* Check that remote is a remote process */ - if (is_not_external_pid(remote)) - goto error; + erts_de_links_lock(dep); + monitors = dep->monitors; + nlinks = dep->nlinks; + node_links = dep->node_links; + dep->monitors = NULL; + dep->nlinks = NULL; + dep->node_links = NULL; + erts_de_links_unlock(dep); - rdep = external_dist_entry(remote); - - if(rdep == erts_this_dist_entry) - goto error; + cache = dep->cache; + dep->cache = NULL; + erts_mtx_lock(&dep->qlock); + obuf = dep->out_queue.first; + dep->out_queue.first = NULL; + dep->out_queue.last = NULL; + ASSERT(!dep->tmp_out_queue.first); + ASSERT(!dep->finalized_out_queue.first); + resume_procs = get_suspended_on_de(dep, ERTS_DE_QFLGS_ALL); + erts_mtx_unlock(&dep->qlock); + erts_atomic_set_nob(&dep->dist_cmd_scheduled, 0); + dep->send = NULL; - /* Check that local is local */ - if (is_internal_pid(local)) { - Process *lp; - ErtsProcLocks lp_locks; - if (BIF_P->common.id == local) { - lp_locks = ERTS_PROC_LOCKS_ALL; - lp = BIF_P; - erts_proc_lock(BIF_P, ERTS_PROC_LOCKS_ALL_MINOR); - } - else { - lp_locks = ERTS_PROC_LOCKS_XSIG_SEND; - lp = erts_pid2proc(BIF_P, ERTS_PROC_LOCK_MAIN, - local, lp_locks); - if (!lp) { - BIF_RET(am_true); /* ignore */ - } - } - - (void) erts_send_exit_signal(BIF_P, - remote, - lp, - &lp_locks, - BIF_ARG_2, - NIL, - NULL, - 0); - if (lp == BIF_P) - lp_locks &= ~ERTS_PROC_LOCK_MAIN; - erts_proc_unlock(lp, lp_locks); - if (lp == BIF_P) { - erts_aint32_t state = erts_atomic32_read_acqb(&BIF_P->state); - /* - * We may have exited current process and may have to take action. - */ - if (state & (ERTS_PSFLG_EXITING|ERTS_PSFLG_PENDING_EXIT)) { - if (state & ERTS_PSFLG_PENDING_EXIT) - erts_handle_pending_exit(BIF_P, ERTS_PROC_LOCK_MAIN); - ERTS_BIF_EXITED(BIF_P); - } - } + erts_set_dist_entry_not_connected(dep); + + erts_de_rwunlock(dep); + + erts_sweep_monitors(monitors, &doit_monitor_net_exits, &nec); + erts_sweep_links(nlinks, &doit_link_net_exits, &nec); + erts_sweep_links(node_links, &doit_node_link_net_exits, &nec); + + if (resume_procs) { + int resumed = erts_resume_processes(resume_procs); + reds += resumed*ERTS_PORT_REDS_DIST_CMD_RESUMED; + } + + delete_cache(cache); + free_de_out_queues(dep, obuf); + + /* + * We wait to make DistEntry idle and accept new connection attempts + * until all is cleared and deallocated. This to get some back pressure + * against repeated failing connection attempts saturating all CPUs + * with cleanup jobs. + */ + erts_de_rwlock(dep); + ASSERT(dep->state == ERTS_DE_STATE_EXITING); + dep->state = ERTS_DE_STATE_IDLE; + erts_de_rwunlock(dep); + return reds; } - else if (is_external_pid(local) - && external_dist_entry(local) == erts_this_dist_entry) { - BIF_RET(am_true); /* ignore */ + erts_de_rwunlock(dep); + return 0; +} + +BIF_RETTYPE erts_internal_abort_connection_2(BIF_ALIST_2) +{ + DistEntry* dep; + Eterm* tp; + + if (is_not_atom(BIF_ARG_1) || is_not_tuple_arity(BIF_ARG_2, 2)) { + BIF_ERROR(BIF_P, BADARG); + } + tp = tuple_val(BIF_ARG_2); + dep = erts_dhandle_to_dist_entry(tp[2]); + if (is_not_small(tp[1]) || dep != erts_find_dist_entry(BIF_ARG_1) + || dep == erts_this_dist_entry) { + BIF_ERROR(BIF_P, BADARG); + } + + if (dep) { + Sint reds = abort_connection(dep, unsigned_val(tp[1])); + BUMP_REDS(BIF_P, reds); } - else - goto error; BIF_RET(am_true); +} - error: - BIF_ERROR(BIF_P, BADARG); +int erts_auto_connect(DistEntry* dep, Process *proc, ErtsProcLocks proc_locks) +{ + erts_de_rwlock(dep); + if (dep->state != ERTS_DE_STATE_IDLE) { + erts_de_rwunlock(dep); + } + else { + Process* net_kernel; + ErtsProcLocks nk_locks = ERTS_PROC_LOCK_MSGQ; + Eterm *hp; + ErlOffHeap *ohp; + ErtsMessage *mp; + Eterm msg, dhandle; + Uint32 conn_id; + + erts_set_dist_entry_pending(dep); + conn_id = dep->connection_id; + erts_de_rwunlock(dep); + + net_kernel = erts_whereis_process(proc, proc_locks, + am_net_kernel, nk_locks, 0); + if (!net_kernel) { + abort_connection(dep, conn_id); + return 0; + } + + /* + * Send {auto_connect, Node, ConnId, DHandle} to net_kernel + */ + mp = erts_alloc_message_heap(net_kernel, &nk_locks, + 5 + ERTS_MAGIC_REF_THING_SIZE, + &hp, &ohp); + dhandle = erts_build_dhandle(&hp, ohp, dep); + msg = TUPLE4(hp, am_auto_connect, dep->sysname, make_small(conn_id), + dhandle); + erts_queue_message(net_kernel, nk_locks, mp, msg, proc->common.id); + erts_proc_unlock(net_kernel, nk_locks); + } + + return 1; } /**********************************************************************/ @@ -3574,9 +3784,11 @@ BIF_RETTYPE nodes_1(BIF_ALIST_1) ASSERT(erts_no_of_not_connected_dist_entries > 0); ASSERT(erts_no_of_hidden_dist_entries >= 0); + ASSERT(erts_no_of_pending_dist_entries >= 0); ASSERT(erts_no_of_visible_dist_entries >= 0); if(not_connected) - length += (erts_no_of_not_connected_dist_entries - 1); + length += ((erts_no_of_not_connected_dist_entries - 1) + + erts_no_of_pending_dist_entries); if(hidden) length += erts_no_of_hidden_dist_entries; if(visible) @@ -3596,13 +3808,18 @@ BIF_RETTYPE nodes_1(BIF_ALIST_1) #ifdef DEBUG endp = hp + length*2; #endif - if(not_connected) + if(not_connected) { for(dep = erts_not_connected_dist_entries; dep; dep = dep->next) { if (dep != erts_this_dist_entry) { result = CONS(hp, dep->sysname, result); hp += 2; } + } + for(dep = erts_pending_dist_entries; dep; dep = dep->next) { + result = CONS(hp, dep->sysname, result); + hp += 2; } + } if(hidden) for(dep = erts_hidden_dist_entries; dep; dep = dep->next) { result = CONS(hp, dep->sysname, result); @@ -3647,11 +3864,17 @@ monitor_node(Process* p, Eterm Node, Eterm Bool, Eterm Options) DistEntry *dep; ErtsLink *lnk; Eterm l; + int async_connect = 1; for (l = Options; l != NIL && is_list(l); l = CDR(list_val(l))) { Eterm t = CAR(list_val(l)); - /* allow_passive_connect the only available option right now */ - if (t != am_allow_passive_connect) { + if (t == am_allow_passive_connect) { + /* + * Handle this horrible feature by falling back on old synchronous + * auto-connect (if needed) + */ + async_connect = 0; + } else { BIF_ERROR(p, BADARG); } } @@ -3665,50 +3888,91 @@ monitor_node(Process* p, Eterm Node, Eterm Bool, Eterm Options) && (Node != erts_this_node->sysname))) { BIF_ERROR(p, BADARG); } - dep = erts_sysname_to_connected_dist_entry(Node); - if (!dep) { - do_trap: - BIF_TRAP3(dmonitor_node_trap, p, Node, Bool, Options); - } - if (dep == erts_this_dist_entry) - goto done; - - erts_proc_lock(p, ERTS_PROC_LOCK_LINK); - erts_de_rlock(dep); - if (ERTS_DE_IS_NOT_CONNECTED(dep)) { - erts_proc_unlock(p, ERTS_PROC_LOCK_LINK); - erts_de_runlock(dep); - goto do_trap; - } - erts_de_links_lock(dep); - erts_de_runlock(dep); if (Bool == am_true) { - ASSERT(dep->cid != NIL); - lnk = erts_add_or_lookup_link(&(dep->node_links), LINK_NODE, - p->common.id); - ++ERTS_LINK_REFC(lnk); - lnk = erts_add_or_lookup_link(&ERTS_P_LINKS(p), LINK_NODE, Node); - ++ERTS_LINK_REFC(lnk); - } - else { - lnk = erts_lookup_link(dep->node_links, p->common.id); - if (lnk != NULL) { - if ((--ERTS_LINK_REFC(lnk)) == 0) { - erts_destroy_link(erts_remove_link(&(dep->node_links), - p->common.id)); - } - } - lnk = erts_lookup_link(ERTS_P_LINKS(p), Node); - if (lnk != NULL) { - if ((--ERTS_LINK_REFC(lnk)) == 0) { - erts_destroy_link(erts_remove_link(&ERTS_P_LINKS(p), - Node)); + ErtsDSigData dsd; + dsd.node = Node; + dep = erts_find_or_insert_dist_entry(Node); + if (dep == erts_this_dist_entry) + goto done; + + erts_proc_lock(p, ERTS_PROC_LOCK_LINK); + + switch (erts_dsig_prepare(&dsd, dep, p, + (ERTS_PROC_LOCK_MAIN | ERTS_PROC_LOCK_LINK), + ERTS_DSP_RLOCK, 0, async_connect)) { + case ERTS_DSIG_PREP_NOT_ALIVE: + case ERTS_DSIG_PREP_NOT_CONNECTED: + /* Trap to either send 'nodedown' or do passive connection attempt */ + trap: + erts_proc_unlock(p, ERTS_PROC_LOCK_LINK); + erts_deref_dist_entry(dep); + BIF_TRAP3(dmonitor_node_trap, p, Node, Bool, Options); + case ERTS_DSIG_PREP_PENDING: + if (!async_connect) { + /* + * Pending connection may fail, so we must trap + * to ensure passive connection attempt + */ + erts_de_runlock(dep); + goto trap; } - } + /*fall through*/ + case ERTS_DSIG_PREP_CONNECTED: + erts_de_links_lock(dep); + erts_de_runlock(dep); + lnk = erts_add_or_lookup_link(&(dep->node_links), LINK_NODE, + p->common.id); + ++ERTS_LINK_REFC(lnk); + lnk = erts_add_or_lookup_link(&ERTS_P_LINKS(p), LINK_NODE, Node); + ++ERTS_LINK_REFC(lnk); + erts_de_links_unlock(dep); + break; + default: + ERTS_ASSERT(! "Invalid dsig prepare result"); + } + erts_deref_dist_entry(dep); + } + else { /* Bool == false */ + dep = erts_sysname_to_connected_dist_entry(Node); + if (!dep) { + /* + * Before OTP-21 this case triggered auto-connect + * and a 'nodedown' message if that failed. + * Now it's a simple no-op which feels more reasonable. + */ + BIF_RET(am_true); + } + if (dep == erts_this_dist_entry) + goto done; + + erts_proc_lock(p, ERTS_PROC_LOCK_LINK); + erts_de_rlock(dep); + if (dep->state == ERTS_DE_STATE_IDLE) { + ASSERT(!dep->node_links); + erts_proc_unlock(p, ERTS_PROC_LOCK_LINK); + erts_de_runlock(dep); + goto done; + } + erts_de_links_lock(dep); + erts_de_runlock(dep); + lnk = erts_lookup_link(dep->node_links, p->common.id); + if (lnk != NULL) { + if ((--ERTS_LINK_REFC(lnk)) == 0) { + erts_destroy_link(erts_remove_link(&(dep->node_links), + p->common.id)); + } + } + lnk = erts_lookup_link(ERTS_P_LINKS(p), Node); + if (lnk != NULL) { + if ((--ERTS_LINK_REFC(lnk)) == 0) { + erts_destroy_link(erts_remove_link(&ERTS_P_LINKS(p), + Node)); + } + } + erts_de_links_unlock(dep); } - erts_de_links_unlock(dep); erts_proc_unlock(p, ERTS_PROC_LOCK_LINK); done: diff --git a/erts/emulator/beam/dist.h b/erts/emulator/beam/dist.h index d4765c50b8..b1b7ce9c78 100644 --- a/erts/emulator/beam/dist.h +++ b/erts/emulator/beam/dist.h @@ -40,11 +40,51 @@ #define DFLAG_UNICODE_IO 0x1000 #define DFLAG_DIST_HDR_ATOM_CACHE 0x2000 #define DFLAG_SMALL_ATOM_TAGS 0x4000 -#define DFLAG_INTERNAL_TAGS 0x8000 +#define DFLAG_INTERNAL_TAGS 0x8000 /* used by ETS 'compressed' option */ #define DFLAG_UTF8_ATOMS 0x10000 #define DFLAG_MAP_TAG 0x20000 #define DFLAG_BIG_CREATION 0x40000 #define DFLAG_SEND_SENDER 0x80000 +#define DFLAG_NO_MAGIC 0x100000 /* internal for pending connection */ + +/* Mandatory flags for distribution */ +#define DFLAG_DIST_MANDATORY (DFLAG_EXTENDED_REFERENCES \ + | DFLAG_EXTENDED_PIDS_PORTS \ + | DFLAG_UTF8_ATOMS \ + | DFLAG_NEW_FUN_TAGS) + +/* + * Additional optimistic flags when encoding toward pending connection. + * If remote node (erl_interface) does not supporting these then we may need + * to transcode messages enqueued before connection setup was finished. + */ +#define DFLAG_DIST_HOPEFULLY (DFLAG_EXPORT_PTR_TAG \ + | DFLAG_BIT_BINARIES \ + | DFLAG_DIST_MONITOR \ + | DFLAG_DIST_MONITOR_NAME) + +/* Our preferred set of flags. Used for connection setup handshake */ +#define DFLAG_DIST_DEFAULT (DFLAG_DIST_MANDATORY | DFLAG_DIST_HOPEFULLY \ + | DFLAG_FUN_TAGS \ + | DFLAG_NEW_FLOATS \ + | DFLAG_UNICODE_IO \ + | DFLAG_DIST_HDR_ATOM_CACHE \ + | DFLAG_SMALL_ATOM_TAGS \ + | DFLAG_UTF8_ATOMS \ + | DFLAG_MAP_TAG \ + | DFLAG_BIG_CREATION \ + | DFLAG_SEND_SENDER) + +/* Flags addable by local distr implementations */ +#define DFLAG_DIST_ADDABLE DFLAG_DIST_DEFAULT + +/* Flags rejectable by local distr implementation */ +#define DFLAG_DIST_REJECTABLE (DFLAG_DIST_HDR_ATOM_CACHE \ + | DFLAG_HIDDEN_ATOM_CACHE \ + | DFLAG_ATOM_CACHE) + +/* Flags for all features needing strict order delivery */ +#define DFLAG_DIST_STRICT_ORDER DFLAG_DIST_HDR_ATOM_CACHE /* All flags that should be enabled when term_to_binary/1 is used. */ #define TERM_TO_BINARY_DFLAGS (DFLAG_EXTENDED_REFERENCES \ @@ -79,52 +119,34 @@ #define DOP_SEND_SENDER_TT 23 /* distribution trap functions */ -extern Export* dsend2_trap; -extern Export* dsend3_trap; -extern Export* dlink_trap; -extern Export* dunlink_trap; extern Export* dmonitor_node_trap; -extern Export* dgroup_leader_trap; -extern Export* dexit_trap; extern Export* dmonitor_p_trap; typedef enum { ERTS_DSP_NO_LOCK, - ERTS_DSP_RLOCK, - ERTS_DSP_RWLOCK + ERTS_DSP_RLOCK } ErtsDSigPrepLock; typedef struct { Process *proc; DistEntry *dep; + Eterm node; /* used if dep == NULL */ Eterm cid; Eterm connection_id; int no_suspend; } ErtsDSigData; -#define ERTS_DE_IS_NOT_CONNECTED(DEP) \ - (ERTS_LC_ASSERT(erts_lc_rwmtx_is_rlocked(&(DEP)->rwmtx) \ - || erts_lc_rwmtx_is_rwlocked(&(DEP)->rwmtx)), \ - (is_nil((DEP)->cid) || ((DEP)->status & ERTS_DE_SFLG_EXITING))) - -#define ERTS_DE_IS_CONNECTED(DEP) \ - (!ERTS_DE_IS_NOT_CONNECTED((DEP))) - #define ERTS_DE_BUSY_LIMIT (1024*1024) extern int erts_dist_buf_busy_limit; extern int erts_is_alive; /* * erts_dsig_prepare() prepares a send of a distributed signal. - * One of the values defined below are returned. If the returned - * value is another than ERTS_DSIG_PREP_CONNECTED, the - * distributed signal cannot be sent before appropriate actions - * have been taken. Appropriate actions would typically be setting - * up the connection. + * One of the values defined below are returned. */ -/* Connected; signal can be sent. */ +/* Connected; signals can be enqueued and sent. */ #define ERTS_DSIG_PREP_CONNECTED 0 /* Not connected; connection needs to be set up. */ #define ERTS_DSIG_PREP_NOT_CONNECTED 1 @@ -132,43 +154,80 @@ extern int erts_is_alive; #define ERTS_DSIG_PREP_WOULD_SUSPEND 2 /* System not alive (distributed) */ #define ERTS_DSIG_PREP_NOT_ALIVE 3 +/* Pending connection; signals can be enqueued */ +#define ERTS_DSIG_PREP_PENDING 4 ERTS_GLB_INLINE int erts_dsig_prepare(ErtsDSigData *, - DistEntry *, + DistEntry*, Process *, + ErtsProcLocks, ErtsDSigPrepLock, + int, int); ERTS_GLB_INLINE void erts_schedule_dist_command(Port *, DistEntry *); +int erts_auto_connect(DistEntry* dep, Process *proc, ErtsProcLocks proc_locks); + #if ERTS_GLB_INLINE_INCL_FUNC_DEF ERTS_GLB_INLINE int erts_dsig_prepare(ErtsDSigData *dsdp, DistEntry *dep, Process *proc, + ErtsProcLocks proc_locks, ErtsDSigPrepLock dspl, - int no_suspend) + int no_suspend, + int connect) { - int failure; + int res; + if (!erts_is_alive) return ERTS_DSIG_PREP_NOT_ALIVE; - if (!dep) - return ERTS_DSIG_PREP_NOT_CONNECTED; - if (dspl == ERTS_DSP_RWLOCK) - erts_de_rwlock(dep); - else - erts_de_rlock(dep); - if (ERTS_DE_IS_NOT_CONNECTED(dep)) { - failure = ERTS_DSIG_PREP_NOT_CONNECTED; + if (!dep) { + ASSERT(!connect); + return ERTS_DSIG_PREP_NOT_CONNECTED; + } + +#ifdef ERTS_ENABLE_LOCK_CHECK + if (connect) { + erts_proc_lc_might_unlock(proc, proc_locks); + } +#endif + +retry: + erts_de_rlock(dep); + + if (dep->state == ERTS_DE_STATE_CONNECTED) { + res = ERTS_DSIG_PREP_CONNECTED; + } + else if (dep->state == ERTS_DE_STATE_PENDING) { + res = ERTS_DSIG_PREP_PENDING; + } + else if (dep->state == ERTS_DE_STATE_EXITING) { + res = ERTS_DSIG_PREP_NOT_CONNECTED; + goto fail; + } + else if (connect) { + ASSERT(dep->state == ERTS_DE_STATE_IDLE); + erts_de_runlock(dep); + if (!erts_auto_connect(dep, proc, proc_locks)) { + return ERTS_DSIG_PREP_NOT_ALIVE; + } + goto retry; + } + else { + ASSERT(dep->state == ERTS_DE_STATE_IDLE); + res = ERTS_DSIG_PREP_NOT_CONNECTED; goto fail; } + if (no_suspend) { - if (erts_atomic32_read_acqb(&dep->qflgs) & ERTS_DE_QFLG_BUSY) { - failure = ERTS_DSIG_PREP_WOULD_SUSPEND; + if (erts_atomic32_read_acqb(&dep->qflgs) & ERTS_DE_QFLG_BUSY) { + res = ERTS_DSIG_PREP_WOULD_SUSPEND; goto fail; - } + } } dsdp->proc = proc; dsdp->dep = dep; @@ -177,15 +236,11 @@ erts_dsig_prepare(ErtsDSigData *dsdp, dsdp->no_suspend = no_suspend; if (dspl == ERTS_DSP_NO_LOCK) erts_de_runlock(dep); - return ERTS_DSIG_PREP_CONNECTED; + return res; fail: - if (dspl == ERTS_DSP_RWLOCK) - erts_de_rwunlock(dep); - else - erts_de_runlock(dep); - return failure; - + erts_de_runlock(dep); + return res; } ERTS_GLB_INLINE @@ -291,6 +346,7 @@ typedef struct TTBSizeContext_ { typedef struct TTBEncodeContext_ { Uint flags; + Uint hopefull_flags; int level; byte* ep; Eterm obj; @@ -332,7 +388,7 @@ struct erts_dsig_send_context { Eterm ctl; Eterm msg; int force_busy; - Uint32 pass_through_size; + Uint32 max_finalize_prepend; Uint data_size, dhdr_ext_size; ErtsAtomCacheMap *acmp; ErtsDistOutputBuf *obuf; @@ -346,11 +402,12 @@ struct erts_dsig_send_context { typedef struct { int suspend; + int connect; Eterm ctl_heap[6]; ErtsDSigData dsd; - DistEntry* dep_to_deref; DistEntry *dep; + int deref_dep; struct erts_dsig_send_context dss; Eterm return_term; diff --git a/erts/emulator/beam/erl_afit_alloc.c b/erts/emulator/beam/erl_afit_alloc.c index 4ebe37ee1d..23efe3bba4 100644 --- a/erts/emulator/beam/erl_afit_alloc.c +++ b/erts/emulator/beam/erl_afit_alloc.c @@ -181,7 +181,7 @@ static struct { static void ERTS_INLINE atom_init(Eterm *atom, char *name) { - *atom = am_atom_put(name, strlen(name)); + *atom = am_atom_put(name, sys_strlen(name)); } #define AM_INIT(AM) atom_init(&am.AM, #AM) diff --git a/erts/emulator/beam/erl_alloc.c b/erts/emulator/beam/erl_alloc.c index 88285d8be6..fa49096d2c 100644 --- a/erts/emulator/beam/erl_alloc.c +++ b/erts/emulator/beam/erl_alloc.c @@ -160,7 +160,7 @@ enum allctr_type { GOODFIT, BESTFIT, AFIT, - AOFIRSTFIT + FIRSTFIT }; struct au_init { @@ -500,8 +500,9 @@ set_default_test_alloc_opts(struct au_init *ip) SET_DEFAULT_ALLOC_OPTS(ip); ip->enable = 0; /* Disabled by default */ ip->thr_spec = -1 * erts_no_schedulers; - ip->atype = AOFIRSTFIT; - ip->init.aoff.flavor = AOFF_BF; + ip->atype = FIRSTFIT; + ip->init.aoff.crr_order = FF_AOFF; + ip->init.aoff.blk_order = FF_BF; ip->init.util.name_prefix = "test_"; ip->init.util.alloc_no = ERTS_ALC_A_TEST; ip->init.util.mmbcs = 0; /* Main carrier size */ @@ -599,10 +600,10 @@ static ERTS_INLINE int strategy_support_carrier_migration(struct au_init *auip) { /* - * Currently only aoff, aoffcbf and aoffcaobf support carrier + * Currently only aoff* and ageff* support carrier * migration, i.e, type AOFIRSTFIT. */ - return auip->atype == AOFIRSTFIT; + return auip->atype == FIRSTFIT; } static ERTS_INLINE void @@ -617,8 +618,9 @@ adjust_carrier_migration_support(struct au_init *auip) */ if (!strategy_support_carrier_migration(auip)) { /* Default to aoffcbf */ - auip->atype = AOFIRSTFIT; - auip->init.aoff.flavor = AOFF_BF; + auip->atype = FIRSTFIT; + auip->init.aoff.crr_order = FF_AOFF; + auip->init.aoff.blk_order = FF_BF; } } } @@ -1132,7 +1134,7 @@ start_au_allocator(ErtsAlcType_t alctr_n, &init->init.af, &init->init.util); break; - case AOFIRSTFIT: + case FIRSTFIT: as = erts_aoffalc_start((AOFFAllctr_t *) as0, &init->init.aoff, &init->init.util); @@ -1217,31 +1219,41 @@ get_bool_value(char *param_end, char** argv, int* ip) { char *param = argv[*ip]+1; char *value = get_value(param_end, argv, ip); - if (strcmp(value, "true") == 0) + if (sys_strcmp(value, "true") == 0) return 1; - else if (strcmp(value, "false") == 0) + else if (sys_strcmp(value, "false") == 0) return 0; else bad_value(param, param_end, value); return -1; } +static Uint kb_to_bytes(Sint kb, Uint *bytes) +{ + const Uint max = ((~((Uint) 0))/1024) + 1; + + if (kb < 0 || (Uint)kb > max) + return 0; + if ((Uint)kb == max) + *bytes = ~((Uint) 0); + else + *bytes = ((Uint) kb)*1024; + return 1; +} + static Uint get_kb_value(char *param_end, char** argv, int* ip) { Sint tmp; - Uint max = ((~((Uint) 0))/1024) + 1; + Uint bytes = 0; char *rest; char *param = argv[*ip]+1; char *value = get_value(param_end, argv, ip); errno = 0; tmp = (Sint) ErtsStrToSint(value, &rest, 10); - if (errno != 0 || rest == value || tmp < 0 || max < ((Uint) tmp)) + if (errno != 0 || rest == value || !kb_to_bytes(tmp, &bytes)) bad_value(param, param_end, value); - if (max == (Uint) tmp) - return ~((Uint) 0); - else - return ((Uint) tmp)*1024; + return bytes; } static UWord @@ -1328,49 +1340,79 @@ handle_au_arg(struct au_init *auip, switch (sub_param[0]) { case 'a': - if (has_prefix("acul", sub_param)) { - if (!auip->carrier_migration_allowed) { - if (!u_switch) - goto bad_switch; - else { - /* ignore */ - (void) get_acul_value(auip, sub_param + 4, argv, ip); - break; - } - } - auip->init.util.acul = get_acul_value(auip, sub_param + 4, argv, ip); - } + if (sub_param[1] == 'c') { /* Migration parameters "ac*" */ + UWord value; + UWord* wp; + if (!auip->carrier_migration_allowed && !u_switch) + goto bad_switch; + + if (has_prefix("acul", sub_param)) { + value = get_acul_value(auip, sub_param + 4, argv, ip); + wp = &auip->init.util.acul; + } + else if (has_prefix("acnl", sub_param)) { + value = get_amount_value(sub_param + 4, argv, ip); + wp = &auip->init.util.acnl; + } + else if (has_prefix("acfml", sub_param)) { + value = get_amount_value(sub_param + 5, argv, ip); + wp = &auip->init.util.acfml; + } + else + goto bad_switch; + + if (auip->carrier_migration_allowed) + *wp = value; + } else if(has_prefix("asbcst", sub_param)) { auip->init.util.asbcst = get_kb_value(sub_param + 6, argv, ip); } else if(has_prefix("as", sub_param)) { char *alg = get_value(sub_param + 2, argv, ip); - if (strcmp("bf", alg) == 0) { + if (sys_strcmp("bf", alg) == 0) { auip->atype = BESTFIT; auip->init.bf.ao = 0; } - else if (strcmp("aobf", alg) == 0) { + else if (sys_strcmp("aobf", alg) == 0) { auip->atype = BESTFIT; auip->init.bf.ao = 1; } - else if (strcmp("gf", alg) == 0) { + else if (sys_strcmp("gf", alg) == 0) { auip->atype = GOODFIT; } - else if (strcmp("af", alg) == 0) { + else if (sys_strcmp("af", alg) == 0) { auip->atype = AFIT; } - else if (strcmp("aoff", alg) == 0) { - auip->atype = AOFIRSTFIT; - auip->init.aoff.flavor = AOFF_AOFF; + else if (sys_strcmp("aoff", alg) == 0) { + auip->atype = FIRSTFIT; + auip->init.aoff.crr_order = FF_AOFF; + auip->init.aoff.blk_order = FF_AOFF; } - else if (strcmp("aoffcbf", alg) == 0) { - auip->atype = AOFIRSTFIT; - auip->init.aoff.flavor = AOFF_BF; + else if (sys_strcmp("aoffcbf", alg) == 0) { + auip->atype = FIRSTFIT; + auip->init.aoff.crr_order = FF_AOFF; + auip->init.aoff.blk_order = FF_BF; } - else if (strcmp("aoffcaobf", alg) == 0) { - auip->atype = AOFIRSTFIT; - auip->init.aoff.flavor = AOFF_AOBF; + else if (sys_strcmp("aoffcaobf", alg) == 0) { + auip->atype = FIRSTFIT; + auip->init.aoff.crr_order = FF_AOFF; + auip->init.aoff.blk_order = FF_AOBF; } + else if (sys_strcmp("ageffcaoff", alg) == 0) { + auip->atype = FIRSTFIT; + auip->init.aoff.crr_order = FF_AGEFF; + auip->init.aoff.blk_order = FF_AOFF; + } + else if (sys_strcmp("ageffcbf", alg) == 0) { + auip->atype = FIRSTFIT; + auip->init.aoff.crr_order = FF_AGEFF; + auip->init.aoff.blk_order = FF_BF; + } + else if (sys_strcmp("ageffcaobf", alg) == 0) { + auip->atype = FIRSTFIT; + auip->init.aoff.crr_order = FF_AGEFF; + auip->init.aoff.blk_order = FF_AOBF; + } else { bad_value(param, sub_param + 1, alg); } @@ -1646,7 +1688,7 @@ handle_args(int *argc, char **argv, erts_alc_hndl_args_init_t *init) } else if (has_prefix("e", param+2)) { arg = get_value(param+3, argv, &i); - if (strcmp("true", arg) != 0) + if (sys_strcmp("true", arg) != 0) bad_value(param, param+3, arg); } else @@ -1658,20 +1700,20 @@ handle_args(int *argc, char **argv, erts_alc_hndl_args_init_t *init) case 'a': { int a; arg = get_value(argv[i]+4, argv, &i); - if (strcmp("min", arg) == 0) { + if (sys_strcmp("min", arg) == 0) { for (a = 0; a < aui_sz; a++) aui[a]->enable = 0; } - else if (strcmp("max", arg) == 0) { + else if (sys_strcmp("max", arg) == 0) { for (a = 0; a < aui_sz; a++) aui[a]->enable = 1; } - else if (strcmp("config", arg) == 0) { + else if (sys_strcmp("config", arg) == 0) { init->erts_alloc_config = 1; } - else if (strcmp("r9c", arg) == 0 - || strcmp("r10b", arg) == 0 - || strcmp("r11b", arg) == 0) { + else if (sys_strcmp("r9c", arg) == 0 + || sys_strcmp("r10b", arg) == 0 + || sys_strcmp("r11b", arg) == 0) { set_default_sl_alloc_opts(&init->sl_alloc); set_default_std_alloc_opts(&init->std_alloc); set_default_ll_alloc_opts(&init->ll_alloc); @@ -1683,7 +1725,7 @@ handle_args(int *argc, char **argv, erts_alc_hndl_args_init_t *init) set_default_driver_alloc_opts(&init->fix_alloc); init->driver_alloc.enable = 0; - if (strcmp("r9c", arg) == 0) { + if (sys_strcmp("r9c", arg) == 0) { init->sl_alloc.enable = 0; init->std_alloc.enable = 0; init->binary_alloc.enable = 0; @@ -1710,18 +1752,18 @@ handle_args(int *argc, char **argv, erts_alc_hndl_args_init_t *init) switch (argv[i][3]) { case 's': arg = get_value(argv[i]+4, argv, &i); - if (strcmp("true", arg) == 0) + if (sys_strcmp("true", arg) == 0) init->instr.stat = 1; - else if (strcmp("false", arg) == 0) + else if (sys_strcmp("false", arg) == 0) init->instr.stat = 0; else bad_value(param, param+3, arg); break; case 'm': arg = get_value(argv[i]+4, argv, &i); - if (strcmp("true", arg) == 0) + if (sys_strcmp("true", arg) == 0) init->instr.map = 1; - else if (strcmp("false", arg) == 0) + else if (sys_strcmp("false", arg) == 0) init->instr.map = 0; else bad_value(param, param+3, arg); @@ -1736,9 +1778,9 @@ handle_args(int *argc, char **argv, erts_alc_hndl_args_init_t *init) case 'l': if (has_prefix("pm", param+2)) { arg = get_value(argv[i]+5, argv, &i); - if (strcmp("all", arg) == 0) + if (sys_strcmp("all", arg) == 0) lock_all_physical_memory = 1; - else if (strcmp("no", arg) == 0) + else if (sys_strcmp("no", arg) == 0) lock_all_physical_memory = 0; else bad_value(param, param+4, arg); @@ -1788,8 +1830,8 @@ handle_args(int *argc, char **argv, erts_alc_hndl_args_init_t *init) /* || init->instr.stat || init->instr.map */) { while (i < *argc) { - if(strcmp(argv[i], "-sname") == 0 - || strcmp(argv[i], "-name") == 0) { + if(sys_strcmp(argv[i], "-sname") == 0 + || sys_strcmp(argv[i], "-name") == 0) { if (i + 1 <*argc) { init->instr.nodename = argv[i+1]; break; @@ -2650,7 +2692,7 @@ erts_allocated_areas(fmtfn_t *print_to_p, void *print_to_arg, void *proc) Eterm atom; if (hpp) atom = am_atom_put(values[i].name, - (int) strlen(values[i].name)); + (int) sys_strlen(values[i].name)); else atom = am_true; @@ -2841,7 +2883,7 @@ erts_allocator_options(void *proc) for (a = ERTS_ALC_A_MIN; a <= ERTS_ALC_A_MAX; a++) { Eterm tmp = NIL; atoms[length] = am_atom_put((char *) ERTS_ALC_A2AD(a), - strlen(ERTS_ALC_A2AD(a))); + sys_strlen(ERTS_ALC_A2AD(a))); if (erts_allctrs_info[a].enabled) { if (erts_allctrs_info[a].alloc_util) { Allctr_t *allctr; @@ -2935,7 +2977,7 @@ erts_allocator_options(void *proc) for (a = ERTS_ALC_A_MIN; a <= ERTS_ALC_A_MAX; a++) { if (erts_allctrs_info[a].enabled) { terms[length++] = am_atom_put((char *) ERTS_ALC_A2AD(a), - strlen(ERTS_ALC_A2AD(a))); + sys_strlen(ERTS_ALC_A2AD(a))); } } @@ -3392,6 +3434,65 @@ erts_request_alloc_info(struct process *c_p, return 1; } +Eterm erts_alloc_set_dyn_param(Process* c_p, Eterm tuple) +{ + ErtsAllocatorThrSpec_t *tspec; + ErtsAlcType_t ai; + Allctr_t* allctr; + Eterm* tp; + Eterm res; + + if (!is_tuple_arity(tuple, 3)) + goto badarg; + + tp = tuple_val(tuple); + + /* + * Ex: {ets_alloc, sbct, 256000} + */ + if (!is_atom(tp[1]) || !is_atom(tp[2]) || !is_integer(tp[3])) + goto badarg; + + for (ai = ERTS_ALC_A_MIN; ai <= ERTS_ALC_A_MAX; ai++) + if (erts_is_atom_str(erts_alc_a2ad[ai], tp[1], 0)) + break; + + if (ai > ERTS_ALC_A_MAX) + goto badarg; + + if (!erts_allctrs_info[ai].enabled || + !erts_allctrs_info[ai].alloc_util) { + return am_notsup; + } + + if (tp[2] == am_sbct) { + Uint sbct; + int i, ok; + + if (!term_to_Uint(tp[3], &sbct)) + goto badarg; + + tspec = &erts_allctr_thr_spec[ai]; + if (tspec->enabled) { + ok = 0; + for (i = 0; i < tspec->size; i++) { + allctr = tspec->allctr[i]; + ok |= allctr->try_set_dyn_param(allctr, am_sbct, sbct); + } + } + else { + allctr = erts_allctrs_info[ai].extra; + ok = allctr->try_set_dyn_param(allctr, am_sbct, sbct); + } + return ok ? am_ok : am_notsup; + } + return am_notsup; + +badarg: + ERTS_BIF_PREP_ERROR(res, c_p, EXC_BADARG); + return res; +} + /* * The allocator wrapper prelocking stuff below is about the locking order. * It only affects wrappers (erl_mtrace.c and erl_instrument.c) that keep locks @@ -3528,7 +3629,7 @@ UWord erts_alc_test(UWord op, UWord a1, UWord a2, UWord a3) &init.init.af, &init.init.util); break; - case AOFIRSTFIT: + case FIRSTFIT: allctr = erts_aoffalc_start((AOFFAllctr_t *) erts_alloc(ERTS_ALC_T_UNDEF, sizeof(AOFFAllctr_t)), @@ -3622,7 +3723,9 @@ UWord erts_alc_test(UWord op, UWord a1, UWord a2, UWord a3) case 0xf15: erts_free(ERTS_ALC_T_TEST, (void*)a1); return 0; - case 0xf16: { + case 0xf16: return (UWord) erts_realloc(ERTS_ALC_T_TEST, (void*)a1, (Uint)a2); + + case 0xf17: { Uint extra_hdr_sz = UNIT_CEILING((Uint)a1); ErtsAllocatorThrSpec_t* ts = &erts_allctr_thr_spec[ERTS_ALC_A_TEST]; Uint offset = ts->allctr[0]->mbc_header_size; @@ -3649,7 +3752,7 @@ UWord erts_alc_test(UWord op, UWord a1, UWord a2, UWord a3) *(void**)a3 = orig_destroying_mbc; return offset; } - case 0xf17: { + case 0xf18: { ErtsAllocatorThrSpec_t* ts = &erts_allctr_thr_spec[ERTS_ALC_A_TEST]; return ts->allctr[0]->largest_mbc_size; } @@ -3858,7 +3961,7 @@ set_memory_fence(void *ptr, Uint sz, ErtsAlcType_t n) *(ui_ptr++) = sz; *(ui_ptr++) = pattern; - memcpy((void *) (((char *) ui_ptr)+sz), (void *) &pattern, sizeof(UWord)); + sys_memcpy((void *) (((char *) ui_ptr)+sz), (void *) &pattern, sizeof(UWord)); #ifdef HARD_DEBUG *mblkpp = hdbg_alloc((void *) ui_ptr, sz, n); @@ -3898,7 +4001,7 @@ check_memory_fence(void *ptr, Uint *size, ErtsAlcType_t n, int func) (UWord) ptr); } - memcpy((void *) &post_pattern, (void *) (((char *)ptr)+sz), sizeof(UWord)); + sys_memcpy((void *) &post_pattern, (void *) (((char *)ptr)+sz), sizeof(UWord)); if (post_pattern != MK_PATTERN(n) || pre_pattern != post_pattern) { diff --git a/erts/emulator/beam/erl_alloc.h b/erts/emulator/beam/erl_alloc.h index 117f96a4ad..578a3717d9 100644 --- a/erts/emulator/beam/erl_alloc.h +++ b/erts/emulator/beam/erl_alloc.h @@ -126,8 +126,10 @@ typedef struct { void *extra; } ErtsAllocatorFunctions_t; -extern ErtsAllocatorFunctions_t erts_allctrs[ERTS_ALC_A_MAX+1]; -extern ErtsAllocatorInfo_t erts_allctrs_info[ERTS_ALC_A_MAX+1]; +extern ErtsAllocatorFunctions_t + ERTS_WRITE_UNLIKELY(erts_allctrs[ERTS_ALC_A_MAX+1]); +extern ErtsAllocatorInfo_t + ERTS_WRITE_UNLIKELY(erts_allctrs_info[ERTS_ALC_A_MAX+1]); typedef struct { int enabled; @@ -144,7 +146,7 @@ typedef struct ErtsAllocatorWrapper_t_ { void (*unlock)(void); struct ErtsAllocatorWrapper_t_* next; }ErtsAllocatorWrapper_t; -ErtsAllocatorWrapper_t *erts_allctr_wrappers; +extern ErtsAllocatorWrapper_t *erts_allctr_wrappers; extern int erts_allctr_wrapper_prelocked; extern erts_tsd_key_t erts_allctr_prelock_tsd_key; void erts_allctr_wrapper_prelock_init(ErtsAllocatorWrapper_t* wrapper); @@ -169,6 +171,8 @@ __decl_noreturn void erts_realloc_n_enomem(ErtsAlcType_t,void*,Uint) __decl_noreturn void erts_alc_fatal_error(int,int,ErtsAlcType_t,...) __noreturn; +Eterm erts_alloc_set_dyn_param(struct process*, Eterm); + #undef ERTS_HAVE_IS_IN_LITERAL_RANGE #if defined(ARCH_32) || defined(ERTS_HAVE_OS_PHYSICAL_MEMORY_RESERVATION) # define ERTS_HAVE_IS_IN_LITERAL_RANGE @@ -430,7 +434,7 @@ NAME##_free(TYPE *p) \ #ifdef DEBUG #define ERTS_PRE_ALLOC_SIZE(SZ) ((SZ) < 1000 ? (SZ)/10 + 10 : 100) -#define ERTS_PRE_ALLOC_CLOBBER(P, T) memset((void *) (P), 0xfd, sizeof(T)) +#define ERTS_PRE_ALLOC_CLOBBER(P, T) sys_memset((void *) (P), 0xfd, sizeof(T)) #else #define ERTS_PRE_ALLOC_SIZE(SZ) ((SZ) > 1 ? (SZ) : 1) #define ERTS_PRE_ALLOC_CLOBBER(P, T) diff --git a/erts/emulator/beam/erl_alloc.types b/erts/emulator/beam/erl_alloc.types index 2960272eab..8107f133aa 100644 --- a/erts/emulator/beam/erl_alloc.types +++ b/erts/emulator/beam/erl_alloc.types @@ -261,6 +261,7 @@ type MINDIRECTION FIXED_SIZE SYSTEM magic_indirection type BINARY_FIND SHORT_LIVED PROCESSES binary_find type OPEN_PORT_ENV TEMPORARY SYSTEM open_port_env type CRASH_DUMP STANDARD SYSTEM crash_dump +type DIST_TRANSCODE SHORT_LIVED SYSTEM dist_transcode_context type THR_Q_EL STANDARD SYSTEM thr_q_element type THR_Q_EL_SL FIXED_SIZE SYSTEM sl_thr_q_element @@ -282,6 +283,8 @@ type THR_PRGR_DATA LONG_LIVED SYSTEM thr_prgr_data type T_THR_PRGR_DATA SHORT_LIVED SYSTEM temp_thr_prgr_data type RELEASE_LAREA SHORT_LIVED SYSTEM release_literal_area +type ENVIRONMENT SYSTEM SYSTEM environment + # # Types used for special emulators # @@ -369,8 +372,6 @@ type SYS_READ_BUF TEMPORARY SYSTEM sys_read_buf type FD_TAB LONG_LIVED SYSTEM fd_tab type FD_ENTRY_BUF STANDARD SYSTEM fd_entry_buf type CS_PROG_PATH LONG_LIVED SYSTEM cs_prog_path -type ENVIRONMENT TEMPORARY SYSTEM environment -type PUTENV_STR SYSTEM SYSTEM putenv_string type PRT_REP_EXIT STANDARD SYSTEM port_report_exit type SYS_BLOCKING STANDARD SYSTEM sys_blocking @@ -382,9 +383,7 @@ type SYS_WRITE_BUF TEMPORARY SYSTEM sys_write_buf type DRV_DATA_BUF SYSTEM SYSTEM drv_data_buf type PRELOADED LONG_LIVED SYSTEM preloaded -type PUTENV_STR SYSTEM SYSTEM putenv_string type WAITER_OBJ LONG_LIVED SYSTEM waiter_object -type ENVIRONMENT SYSTEM SYSTEM environment type CON_VPRINTF_BUF TEMPORARY SYSTEM con_vprintf_buf +endif diff --git a/erts/emulator/beam/erl_alloc_util.c b/erts/emulator/beam/erl_alloc_util.c index 4d4bddb93f..d178c2c2c2 100644 --- a/erts/emulator/beam/erl_alloc_util.c +++ b/erts/emulator/beam/erl_alloc_util.c @@ -362,8 +362,10 @@ do { \ #define ERTS_CRR_ALCTR_FLG_IN_POOL (((erts_aint_t) 1) << 0) #define ERTS_CRR_ALCTR_FLG_BUSY (((erts_aint_t) 1) << 1) +#define ERTS_CRR_ALCTR_FLG_HOMECOMING (((erts_aint_t) 1) << 2) #define ERTS_CRR_ALCTR_FLG_MASK (ERTS_CRR_ALCTR_FLG_IN_POOL | \ - ERTS_CRR_ALCTR_FLG_BUSY) + ERTS_CRR_ALCTR_FLG_BUSY | \ + ERTS_CRR_ALCTR_FLG_HOMECOMING) #define SBC_HEADER_SIZE \ (UNIT_CEILING(offsetof(Carrier_t, cpool) \ @@ -563,7 +565,7 @@ do { \ DEBUG_CHECK_CARRIER_NO_SZ((AP)); \ } while (0) -#define STAT_MBC_CPOOL_INSERT(AP, CRR) \ +#define STAT_MBC_ABANDON(AP, CRR) \ do { \ UWord csz__ = CARRIER_SZ((CRR)); \ if (IS_MSEG_CARRIER((CRR))) \ @@ -1153,89 +1155,23 @@ unlink_carrier(CarrierList_t *cl, Carrier_t *crr) ASSERT(crr->next); crr->next->prev = crr->prev; } -} - - #ifdef DEBUG -static int is_in_list(ErtsDoubleLink_t* sentinel, ErtsDoubleLink_t* node) -{ - ErtsDoubleLink_t* p; - - ASSERT(node != sentinel); - for (p = sentinel->next; p != sentinel; p = p->next) { - if (p == node) - return 1; - } - return 0; -} -#endif /* DEBUG */ - -static ERTS_INLINE void -link_edl_after(ErtsDoubleLink_t* after_me, ErtsDoubleLink_t* node) -{ - ErtsDoubleLink_t* before_me = after_me->next; - ASSERT(node != after_me && node != before_me); - node->next = before_me; - node->prev = after_me; - before_me->prev = node; - after_me->next = node; -} - -static ERTS_INLINE void -link_edl_before(ErtsDoubleLink_t* before_me, ErtsDoubleLink_t* node) -{ - ErtsDoubleLink_t* after_me = before_me->prev; - ASSERT(node != before_me && node != after_me); - node->next = before_me; - node->prev = after_me; - before_me->prev = node; - after_me->next = node; -} - -static ERTS_INLINE void -unlink_edl(ErtsDoubleLink_t* node) -{ - node->next->prev = node->prev; - node->prev->next = node->next; -} - -static ERTS_INLINE void -relink_edl_before(ErtsDoubleLink_t* before_me, ErtsDoubleLink_t* node) -{ - if (node != before_me && node != before_me->prev) { - unlink_edl(node); - link_edl_before(before_me, node); - } + crr->next = crr; + crr->prev = crr; +#endif } static ERTS_INLINE int is_abandoned(Carrier_t *crr) { - return crr->cpool.abandoned.next != NULL; -} - -static ERTS_INLINE void -link_abandoned_carrier(ErtsDoubleLink_t* list, Carrier_t *crr) -{ - ASSERT(!is_abandoned(crr)); - - link_edl_after(list, &crr->cpool.abandoned); - - ASSERT(crr->cpool.abandoned.next != &crr->cpool.abandoned); - ASSERT(crr->cpool.abandoned.prev != &crr->cpool.abandoned); + return crr->cpool.state != ERTS_MBC_IS_HOME; } static ERTS_INLINE void unlink_abandoned_carrier(Carrier_t *crr) { - ASSERT(is_in_list(&crr->cpool.orig_allctr->cpool.pooled_list, - &crr->cpool.abandoned) || - is_in_list(&crr->cpool.orig_allctr->cpool.traitor_list, - &crr->cpool.abandoned)); - - unlink_edl(&crr->cpool.abandoned); - - crr->cpool.abandoned.next = NULL; - crr->cpool.abandoned.prev = NULL; + if (crr->cpool.state == ERTS_MBC_WAS_POOLED) { + aoff_remove_pooled_mbc(crr->cpool.orig_allctr, crr); + } } static ERTS_INLINE void @@ -1243,24 +1179,19 @@ clear_busy_pool_carrier(Allctr_t *allctr, Carrier_t *crr) { if (crr) { erts_aint_t max_size; - erts_aint_t new_val; + erts_aint_t iallctr; max_size = (erts_aint_t) allctr->largest_fblk_in_mbc(allctr, crr); erts_atomic_set_nob(&crr->cpool.max_size, max_size); - new_val = (((erts_aint_t) allctr)|ERTS_CRR_ALCTR_FLG_IN_POOL); + iallctr = erts_atomic_read_nob(&crr->allctr); + ERTS_ALC_CPOOL_ASSERT((iallctr & ~ERTS_CRR_ALCTR_FLG_HOMECOMING) + == ((erts_aint_t)allctr | + ERTS_CRR_ALCTR_FLG_IN_POOL | + ERTS_CRR_ALCTR_FLG_BUSY)); -#ifdef ERTS_ALC_CPOOL_DEBUG - { - erts_aint_t old_val = new_val|ERTS_CRR_ALCTR_FLG_BUSY; - - ERTS_ALC_CPOOL_ASSERT(old_val - == erts_atomic_xchg_relb(&crr->allctr, - new_val)); - } -#else - erts_atomic_set_relb(&crr->allctr, new_val); -#endif + iallctr &= ~ERTS_CRR_ALCTR_FLG_BUSY; + erts_atomic_set_relb(&crr->allctr, iallctr); } } @@ -1658,6 +1589,11 @@ dealloc_mbc(Allctr_t *allctr, Carrier_t *crr) } +static void set_new_allctr_abandon_limit(Allctr_t*); +static void abandon_carrier(Allctr_t*, Carrier_t*); +static void poolify_my_carrier(Allctr_t*, Carrier_t*); +static void enqueue_homecoming(Allctr_t*, Carrier_t*); + static ERTS_INLINE Allctr_t* get_pref_allctr(void *extra) { @@ -1724,9 +1660,23 @@ get_used_allctr(Allctr_t *pref_allctr, int pref_lock, void *p, UWord *sizep, erts_aint_t act; ERTS_ALC_CPOOL_ASSERT(!(iallctr & ERTS_CRR_ALCTR_FLG_BUSY)); - act = erts_atomic_cmpxchg_ddrb(&crr->allctr, - iallctr|ERTS_CRR_ALCTR_FLG_BUSY, - iallctr); + if (iallctr & ERTS_CRR_ALCTR_FLG_HOMECOMING) { + /* + * This carrier has just been given back to us by writing + * to crr->allctr with a write barrier (see abandon_carrier). + * + * We need a mathing read barrier to guarantee a correct view + * of the carrier for deallocation work. + */ + act = erts_atomic_cmpxchg_rb(&crr->allctr, + iallctr|ERTS_CRR_ALCTR_FLG_BUSY, + iallctr); + } + else { + act = erts_atomic_cmpxchg_ddrb(&crr->allctr, + iallctr|ERTS_CRR_ALCTR_FLG_BUSY, + iallctr); + } if (act == iallctr) { *busy_pcrr_pp = crr; break; @@ -1742,13 +1692,6 @@ get_used_allctr(Allctr_t *pref_allctr, int pref_lock, void *p, UWord *sizep, erts_mtx_unlock(&pref_allctr->mutex); } } - - ERTS_ALC_CPOOL_ASSERT( - (((iallctr & ~ERTS_CRR_ALCTR_FLG_MASK) == (erts_aint_t) pref_allctr) - ? (((iallctr & ERTS_CRR_ALCTR_FLG_MASK) == ERTS_CRR_ALCTR_FLG_IN_POOL) - || ((iallctr & ERTS_CRR_ALCTR_FLG_MASK) == 0)) - : 1)); - return used_allctr; } } @@ -2000,9 +1943,9 @@ handle_delayed_fix_dealloc(Allctr_t *allctr, void *ptr) /* Carrier migrated; need to redirect block to new owner... */ int cinit = used_allctr->dd.ix - allctr->dd.ix; - ERTS_ALC_CPOOL_ASSERT(!busy_pcrr_p); + ERTS_ALC_CPOOL_ASSERT(!busy_pcrr_p); - DEC_CC(allctr->calls.this_free); + DEC_CC(allctr->calls.this_free); ((ErtsAllctrFixDDBlock_t *) ptr)->fix_type = type; if (ddq_enqueue(&used_allctr->dd.q, ptr, cinit)) @@ -2011,8 +1954,9 @@ handle_delayed_fix_dealloc(Allctr_t *allctr, void *ptr) } } -static void -schedule_dealloc_carrier(Allctr_t *allctr, Carrier_t *crr); +static void schedule_dealloc_carrier(Allctr_t*, Carrier_t*); +static void dealloc_my_carrier(Allctr_t*, Carrier_t*); + static ERTS_INLINE int handle_delayed_dealloc(Allctr_t *allctr, @@ -2074,39 +2018,61 @@ handle_delayed_dealloc(Allctr_t *allctr, res = 1; blk = UMEM2BLK(ptr); - if (IS_FREE_LAST_MBC_BLK(blk)) { + if (blk->bhdr == HOMECOMING_MBC_BLK_HDR) { /* * A multiblock carrier that previously has been migrated away - * from us and now is back to be deallocated. For more info - * see schedule_dealloc_carrier(). - * - * Note that we cannot use FBLK_TO_MBC(blk) since it - * data has been overwritten by the queue. + * from us, was sent back to us either because + * - it became empty and we need to deallocated it, or + * - it was inserted into the pool and we need to update our pooled_tree */ - Carrier_t *crr = FIRST_BLK_TO_MBC(allctr, blk); - - /* Restore word overwritten by the dd-queue as it will be read - * if this carrier is pulled from dc_list by cpool_fetch() - */ - ERTS_ALC_CPOOL_ASSERT(FBLK_TO_MBC(blk) != crr); - ERTS_CT_ASSERT(sizeof(ErtsAllctrDDBlock_t) == sizeof(void*)); -#ifdef MBC_ABLK_OFFSET_BITS - blk->u.carrier = crr; -#else - blk->carrier = crr; -#endif + Carrier_t *crr = ErtsContainerStruct(blk, Carrier_t, + cpool.homecoming_dd.blk); + Block_t* first_blk = MBC_TO_FIRST_BLK(allctr, crr); + erts_aint_t iallctr; ERTS_ALC_CPOOL_ASSERT(ERTS_ALC_IS_CPOOL_ENABLED(allctr)); ERTS_ALC_CPOOL_ASSERT(allctr == crr->cpool.orig_allctr); - ERTS_ALC_CPOOL_ASSERT(((erts_aint_t) allctr) - != (erts_atomic_read_nob(&crr->allctr) - & ~ERTS_CRR_ALCTR_FLG_MASK)); - - erts_atomic_set_nob(&crr->allctr, ((erts_aint_t) allctr)); - schedule_dealloc_carrier(allctr, crr); + iallctr = erts_atomic_read_nob(&crr->allctr); + ASSERT(iallctr & ERTS_CRR_ALCTR_FLG_HOMECOMING); + while (1) { + if ((iallctr & (~ERTS_CRR_ALCTR_FLG_MASK | + ERTS_CRR_ALCTR_FLG_IN_POOL)) + == (erts_aint_t)allctr) { + /* + * Carrier is home (mine and not in pool) + */ + ASSERT(!(iallctr & ERTS_CRR_ALCTR_FLG_BUSY)); + erts_atomic_set_nob(&crr->allctr, (erts_aint_t)allctr); + if (IS_FREE_LAST_MBC_BLK(first_blk)) + dealloc_my_carrier(allctr, crr); + else + ASSERT(crr->cpool.state == ERTS_MBC_IS_HOME); + } + else { + erts_aint_t exp = iallctr; + erts_aint_t want = iallctr & ~ERTS_CRR_ALCTR_FLG_HOMECOMING; + + iallctr = erts_atomic_cmpxchg_nob(&crr->allctr, + want, + exp); + if (iallctr != exp) + continue; /* retry */ + + ASSERT(crr->cpool.state != ERTS_MBC_IS_HOME); + unlink_abandoned_carrier(crr); + if (iallctr & ERTS_CRR_ALCTR_FLG_IN_POOL) + poolify_my_carrier(allctr, crr); + else + crr->cpool.state = ERTS_MBC_WAS_TRAITOR; + } + break; + } } else { + ASSERT(IS_SBC_BLK(blk) || (ABLK_TO_MBC(blk) != + ErtsContainerStruct(blk, Carrier_t, + cpool.homecoming_dd.blk))); INC_CC(allctr->calls.this_free); @@ -2148,20 +2114,26 @@ enqueue_dealloc_other_instance(ErtsAlcType_t type, erts_alloc_notify_delayed_dealloc(allctr->ix); } - -static void -set_new_allctr_abandon_limit(Allctr_t *allctr); -static void -abandon_carrier(Allctr_t *allctr, Carrier_t *crr); - +static ERTS_INLINE void +update_pooled_tree(Allctr_t *allctr, Carrier_t *crr, Uint blk_sz) +{ + if (allctr == crr->cpool.orig_allctr && crr->cpool.state == ERTS_MBC_WAS_POOLED) { + /* + * Update pooled_tree with a potentially new (larger) max_sz + */ + AOFF_RBTree_t* crr_node = &crr->cpool.pooled; + if (blk_sz > crr_node->hdr.bhdr) { + crr_node->hdr.bhdr = blk_sz; + erts_aoff_larger_max_size(crr_node); + } + } +} static ERTS_INLINE void check_abandon_carrier(Allctr_t *allctr, Block_t *fblk, Carrier_t **busy_pcrr_pp) { Carrier_t *crr; - - if (busy_pcrr_pp && *busy_pcrr_pp) - return; + UWord ncrr_in_pool, largest_fblk; if (!ERTS_ALC_IS_CPOOL_ENABLED(allctr)) return; @@ -2170,8 +2142,7 @@ check_abandon_carrier(Allctr_t *allctr, Block_t *fblk, Carrier_t **busy_pcrr_pp) if (--allctr->cpool.check_limit_count <= 0) set_new_allctr_abandon_limit(allctr); - if (!erts_thr_progress_is_managed_thread()) - return; + ASSERT(erts_thr_progress_is_managed_thread()); if (allctr->cpool.disable_abandon) return; @@ -2179,6 +2150,9 @@ check_abandon_carrier(Allctr_t *allctr, Block_t *fblk, Carrier_t **busy_pcrr_pp) if (allctr->mbcs.blocks.curr.size > allctr->cpool.abandon_limit) return; + ncrr_in_pool = erts_atomic_read_nob(&allctr->cpool.stat.no_carriers); + if (ncrr_in_pool >= allctr->cpool.in_pool_limit) + return; crr = FBLK_TO_MBC(fblk); @@ -2189,9 +2163,14 @@ check_abandon_carrier(Allctr_t *allctr, Block_t *fblk, Carrier_t **busy_pcrr_pp) return; if (crr->cpool.thr_prgr != ERTS_THR_PRGR_INVALID - && !erts_thr_progress_has_reached(crr->cpool.thr_prgr)) - return; + && !erts_thr_progress_has_reached(crr->cpool.thr_prgr)) + return; + + largest_fblk = allctr->largest_fblk_in_mbc(allctr, crr); + if (largest_fblk < allctr->cpool.fblk_min_limit) + return; + erts_atomic_set_nob(&crr->cpool.max_size, largest_fblk); abandon_carrier(allctr, crr); } @@ -2237,6 +2216,7 @@ dealloc_block(Allctr_t *allctr, void *ptr, ErtsAlcFixList_t *fix, int dec_cc_on_ else { Carrier_t *busy_pcrr_p; Allctr_t *used_allctr; + used_allctr = get_used_allctr(allctr, ERTS_ALC_TS_PREF_LOCK_NO, ptr, NULL, &busy_pcrr_p); if (used_allctr == allctr) { @@ -2253,10 +2233,10 @@ dealloc_block(Allctr_t *allctr, void *ptr, ErtsAlcFixList_t *fix, int dec_cc_on_ /* Carrier migrated; need to redirect block to new owner... */ int cinit = used_allctr->dd.ix - allctr->dd.ix; - ERTS_ALC_CPOOL_ASSERT(!busy_pcrr_p); + ERTS_ALC_CPOOL_ASSERT(!busy_pcrr_p); - if (dec_cc_on_redirect) - DEC_CC(allctr->calls.this_free); + if (dec_cc_on_redirect) + DEC_CC(allctr->calls.this_free); if (ddq_enqueue(&used_allctr->dd.q, ptr, cinit)) erts_alloc_notify_delayed_dealloc(used_allctr->ix); } @@ -2500,15 +2480,16 @@ mbc_free(Allctr_t *allctr, void *p, Carrier_t **busy_pcrr_pp) ASSERT(blk_sz % sizeof(Unit_t) == 0); ASSERT(IS_MBC_BLK(blk)); - if (is_first_blk - && is_last_blk - && allctr->main_carrier != FIRST_BLK_TO_MBC(allctr, blk)) { - destroy_carrier(allctr, blk, busy_pcrr_pp); + if (is_first_blk && is_last_blk && crr != allctr->main_carrier) { + destroy_carrier(allctr, blk, busy_pcrr_pp); } else { (*allctr->link_free_block)(allctr, blk); HARD_CHECK_BLK_CARRIER(allctr, blk); - check_abandon_carrier(allctr, blk, busy_pcrr_pp); + if (busy_pcrr_pp && *busy_pcrr_pp) + update_pooled_tree(allctr, crr, blk_sz); + else + check_abandon_carrier(allctr, blk, busy_pcrr_pp); } } @@ -2542,8 +2523,19 @@ mbc_realloc(Allctr_t *allctr, void *p, Uint size, Uint32 alcu_flgs, return NULL; #else /* !MBC_REALLOC_ALWAYS_MOVES */ - if (busy_pcrr_pp && *busy_pcrr_pp) - goto realloc_move; /* Don't want to use carrier in pool */ + if (busy_pcrr_pp && *busy_pcrr_pp) { + /* + * Don't want to use carrier in pool + */ + new_p = mbc_alloc(allctr, size); + if (!new_p) + return NULL; + new_blk = UMEM2BLK(new_p); + ASSERT(!(IS_MBC_BLK(new_blk) && ABLK_TO_MBC(new_blk) == *busy_pcrr_pp)); + sys_memcpy(new_p, p, MIN(size, old_blk_sz - ABLK_HDR_SZ)); + mbc_free(allctr, p, busy_pcrr_pp); + return new_p; + } get_blk_sz = blk_sz = UMEMSZ2BLKSZ(allctr, size); @@ -2776,7 +2768,7 @@ mbc_realloc(Allctr_t *allctr, void *p, Uint size, Uint32 alcu_flgs, if (cand_blk_sz < get_blk_sz) { /* We wont fit in cand_blk get a new one */ - realloc_move: + #endif /* !MBC_REALLOC_ALWAYS_MOVES */ new_p = mbc_alloc(allctr, size); @@ -2880,8 +2872,7 @@ mbc_realloc(Allctr_t *allctr, void *p, Uint size, Uint32 alcu_flgs, #define ERTS_ALC_MAX_DEALLOC_CARRIER 10 -#define ERTS_ALC_CPOOL_MAX_FETCH_INSPECT 20 -#define ERTS_ALC_CPOOL_MAX_TRAITOR_INSPECT 10 +#define ERTS_ALC_CPOOL_MAX_FETCH_INSPECT 100 #define ERTS_ALC_CPOOL_CHECK_LIMIT_COUNT 100 #define ERTS_ALC_CPOOL_MAX_FAILED_STAT_READS 3 @@ -3045,19 +3036,18 @@ cpool_insert(Allctr_t *allctr, Carrier_t *crr) ErtsAlcCPoolData_t *cpd1p, *cpd2p; erts_aint_t val; ErtsAlcCPoolData_t *sentinel = &carrier_pool[allctr->alloc_no].sentinel; + Allctr_t *orig_allctr = crr->cpool.orig_allctr; ERTS_ALC_CPOOL_ASSERT(allctr->alloc_no == ERTS_ALC_A_INVALID /* testcase */ || erts_thr_progress_is_managed_thread()); - ERTS_ALC_CPOOL_ASSERT(erts_atomic_read_nob(&crr->allctr) - == (erts_aint_t) allctr); - erts_atomic_add_nob(&allctr->cpool.stat.blocks_size, + erts_atomic_add_nob(&orig_allctr->cpool.stat.blocks_size, (erts_aint_t) crr->cpool.blocks_size); - erts_atomic_add_nob(&allctr->cpool.stat.no_blocks, + erts_atomic_add_nob(&orig_allctr->cpool.stat.no_blocks, (erts_aint_t) crr->cpool.blocks); - erts_atomic_add_nob(&allctr->cpool.stat.carriers_size, + erts_atomic_add_nob(&orig_allctr->cpool.stat.carriers_size, (erts_aint_t) CARRIER_SZ(crr)); - erts_atomic_inc_nob(&allctr->cpool.stat.no_carriers); + erts_atomic_inc_nob(&orig_allctr->cpool.stat.no_carriers); /* * We search in 'next' direction and begin by passing @@ -3118,8 +3108,6 @@ cpool_insert(Allctr_t *allctr, Carrier_t *crr) (erts_aint_t) &crr->cpool, (erts_aint_t) cpd1p); - erts_atomic_set_wb(&crr->allctr, - ((erts_aint_t) allctr)|ERTS_CRR_ALCTR_FLG_IN_POOL); LTTNG3(carrier_pool_put, ERTS_ALC_A2AD(allctr->alloc_no), allctr->ix, CARRIER_SZ(crr)); } @@ -3221,130 +3209,126 @@ cpool_delete(Allctr_t *allctr, Allctr_t *prev_allctr, Carrier_t *crr) static Carrier_t * cpool_fetch(Allctr_t *allctr, UWord size) { - int i, i_stop, has_passed_sentinel; + enum { IGNORANT, HAS_SEEN_SENTINEL, THE_LAST_ONE } loop_state; + int i; Carrier_t *crr; + Carrier_t *reinsert_crr = NULL; ErtsAlcCPoolData_t *cpdp; - ErtsAlcCPoolData_t *cpool_entrance; + ErtsAlcCPoolData_t *cpool_entrance = NULL; ErtsAlcCPoolData_t *sentinel; - ErtsDoubleLink_t* dl; - ErtsDoubleLink_t* first_old_traitor; ERTS_ALC_CPOOL_ASSERT(allctr->alloc_no == ERTS_ALC_A_INVALID /* testcase */ || erts_thr_progress_is_managed_thread()); i = ERTS_ALC_CPOOL_MAX_FETCH_INSPECT; - first_old_traitor = allctr->cpool.traitor_list.next; - cpool_entrance = NULL; LTTNG3(carrier_pool_get, ERTS_ALC_A2AD(allctr->alloc_no), allctr->ix, (unsigned long)size); /* - * Search my own pooled_list, + * Search my own pooled_tree, * i.e my abandoned carriers that were in the pool last time I checked. */ + do { + erts_aint_t exp, act; + + crr = aoff_lookup_pooled_mbc(allctr, size); + if (!crr) + break; + + ASSERT(crr->cpool.state == ERTS_MBC_WAS_POOLED); + ASSERT(crr->cpool.orig_allctr == allctr); + + aoff_remove_pooled_mbc(allctr, crr); + + exp = erts_atomic_read_nob(&crr->allctr); + if (exp & ERTS_CRR_ALCTR_FLG_IN_POOL) { + ASSERT((exp & ~ERTS_CRR_ALCTR_FLG_MASK) == (erts_aint_t)allctr); + if (erts_atomic_read_nob(&crr->cpool.max_size) < size) { + /* + * This carrier has been fetched and inserted back again + * by a foreign allocator. That's why it has a stale search size. + */ + ASSERT(exp & ERTS_CRR_ALCTR_FLG_HOMECOMING); + crr->cpool.pooled.hdr.bhdr = erts_atomic_read_nob(&crr->cpool.max_size); + aoff_add_pooled_mbc(allctr, crr); + INC_CC(allctr->cpool.stat.skip_size); + continue; + } + else if (exp & ERTS_CRR_ALCTR_FLG_BUSY) { + /* + * This must be our own carrier as part of a realloc call. + * Skip it to make things simpler. + * Must wait to re-insert to not be found again by lookup. + */ + ASSERT(!reinsert_crr); + reinsert_crr = crr; + INC_CC(allctr->cpool.stat.skip_busy); + continue; + } - dl = allctr->cpool.pooled_list.next; - while(dl != &allctr->cpool.pooled_list) { - erts_aint_t exp, act; - crr = (Carrier_t *) (((char *) dl) - offsetof(Carrier_t, cpool.abandoned)); - - ASSERT(!is_in_list(&allctr->cpool.traitor_list, dl)); - ASSERT(crr->cpool.orig_allctr == allctr); - dl = dl->next; - exp = erts_atomic_read_rb(&crr->allctr); - if ((exp & ERTS_CRR_ALCTR_FLG_MASK) == ERTS_CRR_ALCTR_FLG_IN_POOL - && erts_atomic_read_nob(&crr->cpool.max_size) >= size) { - /* Try to fetch it... */ - act = erts_atomic_cmpxchg_mb(&crr->allctr, - (erts_aint_t) allctr, - exp); - if (act == exp) { - cpool_delete(allctr, ((Allctr_t *) (act & ~ERTS_CRR_ALCTR_FLG_MASK)), crr); - unlink_abandoned_carrier(crr); - - /* Move sentinel to continue next search from here */ - relink_edl_before(dl, &allctr->cpool.pooled_list); - return crr; - } - exp = act; - } - if (exp & ERTS_CRR_ALCTR_FLG_IN_POOL) { - if (!cpool_entrance) - cpool_entrance = &crr->cpool; - } - else { /* Not in pool, move to traitor_list */ - unlink_abandoned_carrier(crr); - link_abandoned_carrier(&allctr->cpool.traitor_list, crr); - } - if (--i <= 0) { - /* Move sentinel to continue next search from here */ - relink_edl_before(dl, &allctr->cpool.pooled_list); - return NULL; - } - } + /* Try to fetch it... */ + act = erts_atomic_cmpxchg_mb(&crr->allctr, + exp & ~ERTS_CRR_ALCTR_FLG_IN_POOL, + exp); + if (act == exp) { + cpool_delete(allctr, allctr, crr); + crr->cpool.state = ERTS_MBC_IS_HOME; + + if (reinsert_crr) + aoff_add_pooled_mbc(allctr, reinsert_crr); + return crr; + } + exp = act; + INC_CC(allctr->cpool.stat.skip_race); + } + else + INC_CC(allctr->cpool.stat.skip_not_pooled); - /* Now search traitor_list. - * i.e carriers employed by other allocators last time I checked. - * They might have been abandoned since then. - */ + /* Not in pool anymore */ + ASSERT(!(exp & ERTS_CRR_ALCTR_FLG_BUSY)); + crr->cpool.state = ERTS_MBC_WAS_TRAITOR; - i_stop = (i < ERTS_ALC_CPOOL_MAX_TRAITOR_INSPECT ? - 0 : i - ERTS_ALC_CPOOL_MAX_TRAITOR_INSPECT); - dl = first_old_traitor; - while(dl != &allctr->cpool.traitor_list) { - erts_aint_t exp, act; - crr = (Carrier_t *) (((char *) dl) - offsetof(Carrier_t, cpool.abandoned)); - ASSERT(dl != &allctr->cpool.pooled_list); - ASSERT(crr->cpool.orig_allctr == allctr); - dl = dl->next; - exp = erts_atomic_read_rb(&crr->allctr); - if (exp & ERTS_CRR_ALCTR_FLG_IN_POOL) { - if (!(exp & ERTS_CRR_ALCTR_FLG_BUSY) - && erts_atomic_read_nob(&crr->cpool.max_size) >= size) { - /* Try to fetch it... */ - act = erts_atomic_cmpxchg_mb(&crr->allctr, - (erts_aint_t) allctr, - exp); - if (act == exp) { - cpool_delete(allctr, ((Allctr_t *) (act & ~ERTS_CRR_ALCTR_FLG_MASK)), crr); - unlink_abandoned_carrier(crr); + }while (--i > 0); - /* Move sentinel to continue next search from here */ - relink_edl_before(dl, &allctr->cpool.traitor_list); - return crr; - } - exp = act; - } - if (exp & ERTS_CRR_ALCTR_FLG_IN_POOL) { - if (!cpool_entrance) - cpool_entrance = &crr->cpool; + if (reinsert_crr) + aoff_add_pooled_mbc(allctr, reinsert_crr); - /* Move to pooled_list */ - unlink_abandoned_carrier(crr); - link_abandoned_carrier(&allctr->cpool.pooled_list, crr); - } - } - if (--i <= i_stop) { - /* Move sentinel to continue next search from here */ - relink_edl_before(dl, &allctr->cpool.traitor_list); - if (i > 0) - break; - else - return NULL; - } + /* + * Try find a nice cpool_entrance + */ + while (allctr->cpool.pooled_tree) { + erts_aint_t iallctr; + + crr = ErtsContainerStruct(allctr->cpool.pooled_tree, Carrier_t, cpool.pooled); + iallctr = erts_atomic_read_nob(&crr->allctr); + if (iallctr & ERTS_CRR_ALCTR_FLG_IN_POOL) { + cpool_entrance = &crr->cpool; + break; + } + /* Not in pool anymore */ + ASSERT(!(iallctr & ERTS_CRR_ALCTR_FLG_BUSY)); + aoff_remove_pooled_mbc(allctr, crr); + crr->cpool.state = ERTS_MBC_WAS_TRAITOR; + + if (--i <= 0) { + INC_CC(allctr->cpool.stat.fail_pooled); + return NULL; + } } + /* * Finally search the shared pool and try employ foreign carriers */ - sentinel = &carrier_pool[allctr->alloc_no].sentinel; if (cpool_entrance) { - /* We saw a pooled carried above, use it as entrance into the pool + /* + * We saw a pooled carried above, use it as entrance into the pool */ cpdp = cpool_entrance; } else { - /* No pooled carried seen above. Start search at cpool sentinel, + /* + * No pooled carried seen above. Start search at cpool sentinel, * but begin by passing one element before trying to fetch. * This in order to avoid contention with threads inserting elements. */ @@ -3354,8 +3338,8 @@ cpool_fetch(Allctr_t *allctr, UWord size) goto check_dc_list; } - has_passed_sentinel = 0; - while (1) { + loop_state = IGNORANT; + do { erts_aint_t exp; cpdp = cpool_aint2cpd(cpool_read(&cpdp->prev)); if (cpdp == cpool_entrance) { @@ -3364,38 +3348,52 @@ cpool_fetch(Allctr_t *allctr, UWord size) if (cpdp == sentinel) break; } - i = 0; /* Last one to inspect */ + loop_state = THE_LAST_ONE; } else if (cpdp == sentinel) { - if (has_passed_sentinel) { + if (loop_state == HAS_SEEN_SENTINEL) { /* We been here before. cpool_entrance must have been removed */ + INC_CC(allctr->cpool.stat.entrance_removed); break; } cpdp = cpool_aint2cpd(cpool_read(&cpdp->prev)); if (cpdp == sentinel) break; - has_passed_sentinel = 1; + loop_state = HAS_SEEN_SENTINEL; } - crr = (Carrier_t *)(((char *)cpdp) - offsetof(Carrier_t, cpool)); + crr = ErtsContainerStruct(cpdp, Carrier_t, cpool); exp = erts_atomic_read_rb(&crr->allctr); - if (((exp & (ERTS_CRR_ALCTR_FLG_MASK)) == ERTS_CRR_ALCTR_FLG_IN_POOL) - && (erts_atomic_read_nob(&cpdp->max_size) >= size)) { + + if (erts_atomic_read_nob(&cpdp->max_size) < size) { + INC_CC(allctr->cpool.stat.skip_size); + } + else if ((exp & (ERTS_CRR_ALCTR_FLG_IN_POOL | ERTS_CRR_ALCTR_FLG_BUSY)) + == ERTS_CRR_ALCTR_FLG_IN_POOL) { erts_aint_t act; - /* Try to fetch it... */ - act = erts_atomic_cmpxchg_mb(&crr->allctr, - (erts_aint_t) allctr, - exp); + erts_aint_t want = (((erts_aint_t) allctr) + | (exp & ERTS_CRR_ALCTR_FLG_HOMECOMING)); + /* Try to fetch it... */ + act = erts_atomic_cmpxchg_mb(&crr->allctr, want, exp); if (act == exp) { cpool_delete(allctr, ((Allctr_t *) (act & ~ERTS_CRR_ALCTR_FLG_MASK)), crr); if (crr->cpool.orig_allctr == allctr) { unlink_abandoned_carrier(crr); - } + crr->cpool.state = ERTS_MBC_IS_HOME; + } return crr; } } - if (--i <= 0) + + if (exp & ERTS_CRR_ALCTR_FLG_BUSY) + INC_CC(allctr->cpool.stat.skip_busy); + else + INC_CC(allctr->cpool.stat.skip_race); + + if (--i <= 0) { + INC_CC(allctr->cpool.stat.fail_shared); return NULL; - } + } + }while (loop_state != THE_LAST_ONE); check_dc_list: /* Last; check our own pending dealloc carrier list... */ @@ -3404,23 +3402,23 @@ check_dc_list: if (erts_atomic_read_nob(&crr->cpool.max_size) >= size) { Block_t* blk; unlink_carrier(&allctr->cpool.dc_list, crr); -#ifdef ERTS_ALC_CPOOL_DEBUG - ERTS_ALC_CPOOL_ASSERT(erts_atomic_xchg_nob(&crr->allctr, - ((erts_aint_t) allctr)) - == (((erts_aint_t) allctr) & ~ERTS_CRR_ALCTR_FLG_MASK)); -#else - erts_atomic_set_nob(&crr->allctr, ((erts_aint_t) allctr)); -#endif + ERTS_ALC_CPOOL_ASSERT(erts_atomic_read_nob(&crr->allctr) + == ((erts_aint_t) allctr)); blk = MBC_TO_FIRST_BLK(allctr, crr); ASSERT(FBLK_TO_MBC(blk) == crr); allctr->link_free_block(allctr, blk); return crr; } crr = crr->prev; - if (--i <= 0) + if (--i <= 0) { + INC_CC(allctr->cpool.stat.fail_pend_dealloc); return NULL; + } } + if (i != ERTS_ALC_CPOOL_MAX_FETCH_INSPECT) + INC_CC(allctr->cpool.stat.fail); + return NULL; } @@ -3475,9 +3473,6 @@ static void schedule_dealloc_carrier(Allctr_t *allctr, Carrier_t *crr) { Allctr_t *orig_allctr; - Block_t *blk; - int check_pending_dealloc; - erts_aint_t max_size; ASSERT(IS_MB_CARRIER(crr)); @@ -3488,9 +3483,17 @@ schedule_dealloc_carrier(Allctr_t *allctr, Carrier_t *crr) orig_allctr = crr->cpool.orig_allctr; - if (allctr != orig_allctr) { - int cinit = orig_allctr->dd.ix - allctr->dd.ix; - + if (allctr == orig_allctr) { + if (!(erts_atomic_read_nob(&crr->allctr) & ERTS_CRR_ALCTR_FLG_HOMECOMING)) { + dealloc_my_carrier(allctr, crr); + } + /*else + * Carrier was abandoned earlier by other thread and + * is still waiting for us in dd-queue. + * handle_delayed_dealloc() will handle it when crr is dequeued. + */ + } + else { /* * We send the carrier to its origin for deallocation. * This in order: @@ -3499,29 +3502,39 @@ schedule_dealloc_carrier(Allctr_t *allctr, Carrier_t *crr) * - to ensure that we always only reuse empty carriers * originating from our own thread specific mseg_alloc * instance which is beneficial on NUMA systems. - * - * The receiver will recognize that this is a carrier to - * deallocate (and not a block which is the common case) - * since the block is an mbc block that is free and last - * in the carrier. */ - blk = MBC_TO_FIRST_BLK(allctr, crr); - ERTS_ALC_CPOOL_ASSERT(IS_FREE_LAST_MBC_BLK(blk)); - - ERTS_ALC_CPOOL_ASSERT(IS_MBC_FIRST_ABLK(allctr, blk)); - ERTS_ALC_CPOOL_ASSERT(crr == FBLK_TO_MBC(blk)); - ERTS_ALC_CPOOL_ASSERT(crr == FIRST_BLK_TO_MBC(allctr, blk)); - ERTS_ALC_CPOOL_ASSERT(((erts_aint_t) allctr) - == (erts_atomic_read_nob(&crr->allctr) - & ~ERTS_CRR_ALCTR_FLG_MASK)); + erts_aint_t iallctr; +#ifdef ERTS_ALC_CPOOL_DEBUG + Block_t* first_blk = MBC_TO_FIRST_BLK(allctr, crr); + ERTS_ALC_CPOOL_ASSERT(IS_FREE_LAST_MBC_BLK(first_blk)); + + ERTS_ALC_CPOOL_ASSERT(IS_MBC_FIRST_ABLK(allctr, first_blk)); + ERTS_ALC_CPOOL_ASSERT(crr == FBLK_TO_MBC(first_blk)); + ERTS_ALC_CPOOL_ASSERT(crr == FIRST_BLK_TO_MBC(allctr, first_blk)); + ERTS_ALC_CPOOL_ASSERT((erts_atomic_read_nob(&crr->allctr) + & ~ERTS_CRR_ALCTR_FLG_HOMECOMING) + == (erts_aint_t) allctr); +#endif - if (ddq_enqueue(&orig_allctr->dd.q, BLK2UMEM(blk), cinit)) - erts_alloc_notify_delayed_dealloc(orig_allctr->ix); - return; + iallctr = (erts_aint_t)orig_allctr | ERTS_CRR_ALCTR_FLG_HOMECOMING; + if (!(erts_atomic_xchg_nob(&crr->allctr, iallctr) + & ERTS_CRR_ALCTR_FLG_HOMECOMING)) { + enqueue_homecoming(allctr, crr); + } } +} + +static void dealloc_my_carrier(Allctr_t *allctr, Carrier_t *crr) +{ + Block_t *blk; + int check_pending_dealloc; + erts_aint_t max_size; - if (is_abandoned(crr)) - unlink_abandoned_carrier(crr); + ERTS_ALC_CPOOL_ASSERT(allctr == crr->cpool.orig_allctr); + if (is_abandoned(crr)) { + unlink_abandoned_carrier(crr); + crr->cpool.state = ERTS_MBC_IS_HOME; + } if (crr->cpool.thr_prgr == ERTS_THR_PRGR_INVALID || erts_thr_progress_has_reached(crr->cpool.thr_prgr)) { @@ -3553,6 +3566,7 @@ schedule_dealloc_carrier(Allctr_t *allctr, Carrier_t *crr) static ERTS_INLINE void cpool_init_carrier_data(Allctr_t *allctr, Carrier_t *crr) { + crr->cpool.homecoming_dd.blk.bhdr = HOMECOMING_MBC_BLK_HDR; erts_atomic_init_nob(&crr->cpool.next, ERTS_AINT_NULL); erts_atomic_init_nob(&crr->cpool.prev, ERTS_AINT_NULL); crr->cpool.orig_allctr = allctr; @@ -3571,8 +3585,7 @@ cpool_init_carrier_data(Allctr_t *allctr, Carrier_t *crr) limit = (csz/100)*allctr->cpool.util_limit; crr->cpool.abandon_limit = limit; } - crr->cpool.abandoned.next = NULL; - crr->cpool.abandoned.prev = NULL; + crr->cpool.state = ERTS_MBC_IS_HOME; } static void @@ -3598,23 +3611,62 @@ set_new_allctr_abandon_limit(Allctr_t *allctr) static void abandon_carrier(Allctr_t *allctr, Carrier_t *crr) { - erts_aint_t max_size; + erts_aint_t iallctr; - STAT_MBC_CPOOL_INSERT(allctr, crr); + STAT_MBC_ABANDON(allctr, crr); unlink_carrier(&allctr->mbc_list, crr); - if (crr->cpool.orig_allctr == allctr) { - link_abandoned_carrier(&allctr->cpool.pooled_list, crr); + allctr->remove_mbc(allctr, crr); + set_new_allctr_abandon_limit(allctr); + + cpool_insert(allctr, crr); + + + iallctr = erts_atomic_read_nob(&crr->allctr); + if (allctr == crr->cpool.orig_allctr) { + /* preserve HOMECOMING flag */ + ASSERT((iallctr & ~ERTS_CRR_ALCTR_FLG_HOMECOMING) == (erts_aint_t)allctr); + erts_atomic_set_wb(&crr->allctr, iallctr | ERTS_CRR_ALCTR_FLG_IN_POOL); + poolify_my_carrier(allctr, crr); } + else { + ASSERT((iallctr & ~ERTS_CRR_ALCTR_FLG_HOMECOMING) == (erts_aint_t)allctr); + iallctr = ((erts_aint_t)crr->cpool.orig_allctr | + ERTS_CRR_ALCTR_FLG_HOMECOMING | + ERTS_CRR_ALCTR_FLG_IN_POOL); + if (!(erts_atomic_xchg_wb(&crr->allctr, iallctr) + & ERTS_CRR_ALCTR_FLG_HOMECOMING)) { + + enqueue_homecoming(allctr, crr); + } + } +} - allctr->remove_mbc(allctr, crr); +static void +enqueue_homecoming(Allctr_t* allctr, Carrier_t* crr) +{ + Allctr_t* orig_allctr = crr->cpool.orig_allctr; + const int cinit = orig_allctr->dd.ix - allctr->dd.ix; + Block_t* dd_blk = &crr->cpool.homecoming_dd.blk; - max_size = (erts_aint_t) allctr->largest_fblk_in_mbc(allctr, crr); - erts_atomic_set_nob(&crr->cpool.max_size, max_size); + /* + * The receiver will recognize this as a carrier + * (and not a block which is the common case) + * since the block header is HOMECOMING_MBC_BLK_HDR. + */ + ASSERT(dd_blk->bhdr == HOMECOMING_MBC_BLK_HDR); + if (ddq_enqueue(&orig_allctr->dd.q, BLK2UMEM(dd_blk), cinit)) + erts_alloc_notify_delayed_dealloc(orig_allctr->ix); +} - cpool_insert(allctr, crr); +static void +poolify_my_carrier(Allctr_t *allctr, Carrier_t *crr) +{ + ERTS_ALC_CPOOL_ASSERT(allctr == crr->cpool.orig_allctr); - set_new_allctr_abandon_limit(allctr); + crr->cpool.pooled.hdr.bhdr = erts_atomic_read_nob(&crr->cpool.max_size); + aoff_add_pooled_mbc(allctr, crr); + crr->cpool.state = ERTS_MBC_WAS_POOLED; } static void @@ -3771,6 +3823,7 @@ create_carrier(Allctr_t *allctr, Uint umem_sz, UWord flags) crr = cpool_fetch(allctr, blk_sz); if (crr) { STAT_MBC_CPOOL_FETCH(allctr, crr); + INC_CC(allctr->cpool.stat.fetch); link_carrier(&allctr->mbc_list, crr); (*allctr->add_mbc)(allctr, crr); blk = (*allctr->get_free_block)(allctr, blk_sz, NULL, 0); @@ -4128,13 +4181,18 @@ destroy_carrier(Allctr_t *allctr, Block_t *blk, Carrier_t **busy_pcrr_pp) #endif if (busy_pcrr_pp && *busy_pcrr_pp) { + erts_aint_t iallctr = erts_atomic_read_nob(&crr->allctr); ERTS_ALC_CPOOL_ASSERT(*busy_pcrr_pp == crr); - *busy_pcrr_pp = NULL; - ERTS_ALC_CPOOL_ASSERT(erts_atomic_read_nob(&crr->allctr) - == (((erts_aint_t) allctr) - | ERTS_CRR_ALCTR_FLG_IN_POOL - | ERTS_CRR_ALCTR_FLG_BUSY)); - erts_atomic_set_nob(&crr->allctr, ((erts_aint_t) allctr)); + ERTS_ALC_CPOOL_ASSERT((iallctr & ~ERTS_CRR_ALCTR_FLG_HOMECOMING) + == (((erts_aint_t) allctr) + | ERTS_CRR_ALCTR_FLG_IN_POOL + | ERTS_CRR_ALCTR_FLG_BUSY)); + ERTS_ALC_CPOOL_ASSERT(allctr == crr->cpool.orig_allctr); + + *busy_pcrr_pp = NULL; + erts_atomic_set_nob(&crr->allctr, + (iallctr & ~(ERTS_CRR_ALCTR_FLG_IN_POOL | + ERTS_CRR_ALCTR_FLG_BUSY))); cpool_delete(allctr, allctr, crr); } else @@ -4184,7 +4242,6 @@ static struct { Eterm e; Eterm t; Eterm ramv; - Eterm sbct; #if HAVE_ERTS_MSEG Eterm asbcst; Eterm rsbcst; @@ -4201,6 +4258,8 @@ static struct { Eterm smbcs; Eterm mbcgs; Eterm acul; + Eterm acnl; + Eterm acfml; #if HAVE_ERTS_MSEG Eterm mmc; @@ -4212,6 +4271,17 @@ static struct { Eterm mbcs; Eterm mbcs_pool; + Eterm fetch; + Eterm fail_pooled; + Eterm fail_shared; + Eterm fail_pend_dealloc; + Eterm fail; + Eterm skip_size; + Eterm skip_busy; + Eterm skip_not_pooled; + Eterm skip_homecoming; + Eterm skip_race; + Eterm entrance_removed; Eterm sbcs; Eterm sys_alloc_carriers_size; @@ -4245,7 +4315,7 @@ static Eterm fix_type_atoms[ERTS_ALC_NO_FIXED_SIZES]; static ERTS_INLINE void atom_init(Eterm *atom, char *name) { - *atom = am_atom_put(name, strlen(name)); + *atom = am_atom_put(name, sys_strlen(name)); } #define AM_INIT(AM) atom_init(&am.AM, #AM) @@ -4272,7 +4342,6 @@ init_atoms(Allctr_t *allctr) AM_INIT(e); AM_INIT(t); AM_INIT(ramv); - AM_INIT(sbct); #if HAVE_ERTS_MSEG AM_INIT(asbcst); AM_INIT(rsbcst); @@ -4289,6 +4358,8 @@ init_atoms(Allctr_t *allctr) AM_INIT(smbcs); AM_INIT(mbcgs); AM_INIT(acul); + AM_INIT(acnl); + AM_INIT(acfml); #if HAVE_ERTS_MSEG AM_INIT(mmc); @@ -4300,6 +4371,17 @@ init_atoms(Allctr_t *allctr) AM_INIT(mbcs); AM_INIT(mbcs_pool); + AM_INIT(fetch); + AM_INIT(fail_pooled); + AM_INIT(fail_shared); + AM_INIT(fail_pend_dealloc); + AM_INIT(fail); + AM_INIT(skip_size); + AM_INIT(skip_busy); + AM_INIT(skip_not_pooled); + AM_INIT(skip_homecoming); + AM_INIT(skip_race); + AM_INIT(entrance_removed); AM_INIT(sbcs); AM_INIT(sys_alloc_carriers_size); @@ -4334,7 +4416,7 @@ init_atoms(Allctr_t *allctr) for (ix = 0; ix < ERTS_ALC_NO_FIXED_SIZES; ix++) { ErtsAlcType_t n = ERTS_ALC_N_MIN_A_FIXED_SIZE + ix; char *name = (char *) ERTS_ALC_N2TD(n); - size_t len = strlen(name); + size_t len = sys_strlen(name); fix_type_atoms[ix] = am_atom_put(name, len); } } @@ -4583,9 +4665,56 @@ info_cpool(Allctr_t *allctr, if (hpp || szp) { res = NIL; + + if (!sz_only) { + add_3tup(hpp, szp, &res, am.fail_pooled, + bld_unstable_uint(hpp, szp, ERTS_ALC_CC_GIGA_VAL(allctr->cpool.stat.fail_pooled)), + bld_unstable_uint(hpp, szp, ERTS_ALC_CC_VAL(allctr->cpool.stat.fail_pooled))); + + add_3tup(hpp, szp, &res, am.fail_shared, + bld_unstable_uint(hpp, szp, ERTS_ALC_CC_GIGA_VAL(allctr->cpool.stat.fail_shared)), + bld_unstable_uint(hpp, szp, ERTS_ALC_CC_VAL(allctr->cpool.stat.fail_shared))); + + add_3tup(hpp, szp, &res, am.fail_pend_dealloc, + bld_unstable_uint(hpp, szp, ERTS_ALC_CC_GIGA_VAL(allctr->cpool.stat.fail_pend_dealloc)), + bld_unstable_uint(hpp, szp, ERTS_ALC_CC_VAL(allctr->cpool.stat.fail_pend_dealloc))); + + add_3tup(hpp, szp, &res, am.fail, + bld_unstable_uint(hpp, szp, ERTS_ALC_CC_GIGA_VAL(allctr->cpool.stat.fail)), + bld_unstable_uint(hpp, szp, ERTS_ALC_CC_VAL(allctr->cpool.stat.fail))); + + add_3tup(hpp, szp, &res, am.fetch, + bld_unstable_uint(hpp, szp, ERTS_ALC_CC_GIGA_VAL(allctr->cpool.stat.fetch)), + bld_unstable_uint(hpp, szp, ERTS_ALC_CC_VAL(allctr->cpool.stat.fetch))); + + add_3tup(hpp, szp, &res, am.skip_size, + bld_unstable_uint(hpp, szp, ERTS_ALC_CC_GIGA_VAL(allctr->cpool.stat.skip_size)), + bld_unstable_uint(hpp, szp, ERTS_ALC_CC_VAL(allctr->cpool.stat.skip_size))); + + add_3tup(hpp, szp, &res, am.skip_busy, + bld_unstable_uint(hpp, szp, ERTS_ALC_CC_GIGA_VAL(allctr->cpool.stat.skip_busy)), + bld_unstable_uint(hpp, szp, ERTS_ALC_CC_VAL(allctr->cpool.stat.skip_busy))); + + add_3tup(hpp, szp, &res, am.skip_not_pooled, + bld_unstable_uint(hpp, szp, ERTS_ALC_CC_GIGA_VAL(allctr->cpool.stat.skip_not_pooled)), + bld_unstable_uint(hpp, szp, ERTS_ALC_CC_VAL(allctr->cpool.stat.skip_not_pooled))); + + add_3tup(hpp, szp, &res, am.skip_homecoming, + bld_unstable_uint(hpp, szp, ERTS_ALC_CC_GIGA_VAL(allctr->cpool.stat.skip_homecoming)), + bld_unstable_uint(hpp, szp, ERTS_ALC_CC_VAL(allctr->cpool.stat.skip_homecoming))); + + add_3tup(hpp, szp, &res, am.skip_race, + bld_unstable_uint(hpp, szp, ERTS_ALC_CC_GIGA_VAL(allctr->cpool.stat.skip_race)), + bld_unstable_uint(hpp, szp, ERTS_ALC_CC_VAL(allctr->cpool.stat.skip_race))); + + add_3tup(hpp, szp, &res, am.entrance_removed, + bld_unstable_uint(hpp, szp, ERTS_ALC_CC_GIGA_VAL(allctr->cpool.stat.entrance_removed)), + bld_unstable_uint(hpp, szp, ERTS_ALC_CC_VAL(allctr->cpool.stat.entrance_removed))); + add_2tup(hpp, szp, &res, am.carriers_size, bld_unstable_uint(hpp, szp, csz)); + } if (!sz_only) add_2tup(hpp, szp, &res, am.carriers, @@ -4725,20 +4854,20 @@ make_name_atoms(Allctr_t *allctr) char realloc[] = "realloc"; char free[] = "free"; char buf[MAX_ATOM_CHARACTERS]; - size_t prefix_len = strlen(allctr->name_prefix); + size_t prefix_len = sys_strlen(allctr->name_prefix); if (prefix_len > MAX_ATOM_CHARACTERS + sizeof(realloc) - 1) erts_exit(ERTS_ERROR_EXIT,"Too long allocator name: %salloc\n",allctr->name_prefix); - memcpy((void *) buf, (void *) allctr->name_prefix, prefix_len); + sys_memcpy((void *) buf, (void *) allctr->name_prefix, prefix_len); - memcpy((void *) &buf[prefix_len], (void *) alloc, sizeof(alloc) - 1); + sys_memcpy((void *) &buf[prefix_len], (void *) alloc, sizeof(alloc) - 1); allctr->name.alloc = am_atom_put(buf, prefix_len + sizeof(alloc) - 1); - memcpy((void *) &buf[prefix_len], (void *) realloc, sizeof(realloc) - 1); + sys_memcpy((void *) &buf[prefix_len], (void *) realloc, sizeof(realloc) - 1); allctr->name.realloc = am_atom_put(buf, prefix_len + sizeof(realloc) - 1); - memcpy((void *) &buf[prefix_len], (void *) free, sizeof(free) - 1); + sys_memcpy((void *) &buf[prefix_len], (void *) free, sizeof(free) - 1); allctr->name.free = am_atom_put(buf, prefix_len + sizeof(free) - 1); } @@ -4844,7 +4973,7 @@ info_options(Allctr_t *allctr, Uint *szp) { Eterm res = THE_NON_VALUE; - int acul; + UWord acul, acnl, acfml; if (!allctr) { if (print_to_p) @@ -4857,6 +4986,8 @@ info_options(Allctr_t *allctr, } acul = allctr->cpool.util_limit; + acnl = allctr->cpool.in_pool_limit; + acfml = allctr->cpool.fblk_min_limit; if (print_to_p) { char topt[21]; /* Enough for any 64-bit integer */ @@ -4884,7 +5015,7 @@ info_options(Allctr_t *allctr, "option lmbcs: %beu\n" "option smbcs: %beu\n" "option mbcgs: %beu\n" - "option acul: %d\n", + "option acul: %bpu\n", topt, allctr->ramv ? "true" : "false", allctr->sbc_threshold, @@ -4909,9 +5040,15 @@ info_options(Allctr_t *allctr, hpp, szp); if (hpp || szp) { + add_2tup(hpp, szp, &res, + am.acfml, + bld_uint(hpp, szp, acfml)); + add_2tup(hpp, szp, &res, + am.acnl, + bld_uint(hpp, szp, acnl)); add_2tup(hpp, szp, &res, am.acul, - bld_uint(hpp, szp, (UWord) acul)); + bld_uint(hpp, szp, acul)); add_2tup(hpp, szp, &res, am.mbcgs, bld_uint(hpp, szp, allctr->mbc_growth_stages)); @@ -4947,7 +5084,7 @@ info_options(Allctr_t *allctr, bld_uint(hpp, szp, allctr->mseg_opt.abs_shrink_th)); #endif add_2tup(hpp, szp, &res, - am.sbct, + am_sbct, bld_uint(hpp, szp, allctr->sbc_threshold)); add_2tup(hpp, szp, &res, am.ramv, allctr->ramv ? am_true : am_false); add_2tup(hpp, szp, &res, am.t, (allctr->t ? am_true : am_false)); @@ -5481,12 +5618,13 @@ erts_alcu_free_thr_pref(ErtsAlcType_t type, void *extra, void *p) pref_allctr = get_pref_allctr(extra); used_allctr = get_used_allctr(pref_allctr, ERTS_ALC_TS_PREF_LOCK_IF_USED, p, NULL, &busy_pcrr_p); - if (pref_allctr != used_allctr) + if (pref_allctr != used_allctr) { enqueue_dealloc_other_instance(type, - used_allctr, - p, - (used_allctr->dd.ix - - pref_allctr->dd.ix)); + used_allctr, + p, + (used_allctr->dd.ix + - pref_allctr->dd.ix)); + } else { ERTS_ALCU_DBG_CHK_THR_ACCESS(used_allctr); do_erts_alcu_free(type, used_allctr, p, &busy_pcrr_p); @@ -5854,6 +5992,37 @@ erts_alcu_realloc_mv_thr_pref(ErtsAlcType_t type, void *extra, +static Uint adjust_sbct(Allctr_t* allctr, Uint sbct) +{ +#ifndef ARCH_64 + if (sbct > 0) { + Uint max_mbc_block_sz = UNIT_CEILING(sbct - 1 + ABLK_HDR_SZ); + if (max_mbc_block_sz + UNIT_FLOOR(allctr->min_block_size - 1) > MBC_ABLK_SZ_MASK + || max_mbc_block_sz < sbct) { /* wrap around */ + /* + * By limiting sbc_threshold to (hard limit - min_block_size) + * we avoid having to split off free "residue blocks" + * smaller than min_block_size. + */ + max_mbc_block_sz = MBC_ABLK_SZ_MASK - UNIT_FLOOR(allctr->min_block_size - 1); + sbct = max_mbc_block_sz - ABLK_HDR_SZ + 1; + } + } +#endif + return sbct; +} + +int erts_alcu_try_set_dyn_param(Allctr_t* allctr, Eterm param, Uint value) +{ + const Uint MIN_DYN_SBCT = 4000; /* a lame catastrophe prevention */ + + if (param == am_sbct && value >= MIN_DYN_SBCT) { + allctr->sbc_threshold = adjust_sbct(allctr, value); + return 1; + } + return 0; +} + /* ------------------------------------------------------------------------- */ int @@ -5941,10 +6110,7 @@ erts_alcu_start(Allctr_t *allctr, AllctrInit_t *init) allctr->min_block_size = sz; } - allctr->cpool.pooled_list.next = &allctr->cpool.pooled_list; - allctr->cpool.pooled_list.prev = &allctr->cpool.pooled_list; - allctr->cpool.traitor_list.next = &allctr->cpool.traitor_list; - allctr->cpool.traitor_list.prev = &allctr->cpool.traitor_list; + allctr->cpool.pooled_tree = NULL; allctr->cpool.dc_list.first = NULL; allctr->cpool.dc_list.last = NULL; allctr->cpool.abandon_limit = 0; @@ -5954,24 +6120,18 @@ erts_alcu_start(Allctr_t *allctr, AllctrInit_t *init) erts_atomic_init_nob(&allctr->cpool.stat.carriers_size, 0); erts_atomic_init_nob(&allctr->cpool.stat.no_carriers, 0); allctr->cpool.check_limit_count = ERTS_ALC_CPOOL_CHECK_LIMIT_COUNT; - allctr->cpool.util_limit = init->ts ? 0 : init->acul; - - allctr->sbc_threshold = init->sbct; -#ifndef ARCH_64 - if (allctr->sbc_threshold > 0) { - Uint max_mbc_block_sz = UNIT_CEILING(allctr->sbc_threshold - 1 + ABLK_HDR_SZ); - if (max_mbc_block_sz + UNIT_FLOOR(allctr->min_block_size - 1) > MBC_ABLK_SZ_MASK - || max_mbc_block_sz < allctr->sbc_threshold) { /* wrap around */ - /* - * By limiting sbc_threshold to (hard limit - min_block_size) - * we avoid having to split off free "residue blocks" - * smaller than min_block_size. - */ - max_mbc_block_sz = MBC_ABLK_SZ_MASK - UNIT_FLOOR(allctr->min_block_size - 1); - allctr->sbc_threshold = max_mbc_block_sz - ABLK_HDR_SZ + 1; - } + if (!init->ts && init->acul && init->acnl) { + allctr->cpool.util_limit = init->acul; + allctr->cpool.in_pool_limit = init->acnl; + allctr->cpool.fblk_min_limit = init->acfml; } -#endif + else { + allctr->cpool.util_limit = 0; + allctr->cpool.in_pool_limit = 0; + allctr->cpool.fblk_min_limit = 0; + } + + allctr->sbc_threshold = adjust_sbct(allctr, init->sbct); #if HAVE_ERTS_MSEG if (allctr->mseg_opt.abs_shrink_th > ~((UWord) 0) / 100) @@ -6022,6 +6182,9 @@ erts_alcu_start(Allctr_t *allctr, AllctrInit_t *init) allctr->sys_realloc = &erts_alcu_sys_realloc; allctr->sys_dealloc = &erts_alcu_sys_dealloc; } + + allctr->try_set_dyn_param = &erts_alcu_try_set_dyn_param; + #if HAVE_ERTS_MSEG if (init->mseg_alloc) { ASSERT(init->mseg_realloc && init->mseg_dealloc); @@ -6036,6 +6199,7 @@ erts_alcu_start(Allctr_t *allctr, AllctrInit_t *init) allctr->mseg_realloc = &erts_alcu_mseg_realloc; allctr->mseg_dealloc = &erts_alcu_mseg_dealloc; } + /* If a custom carrier alloc function is specified, make sure it's used */ if (init->mseg_alloc && !init->sys_alloc) { allctr->crr_set_flgs = CFLG_FORCE_MSEG; diff --git a/erts/emulator/beam/erl_alloc_util.h b/erts/emulator/beam/erl_alloc_util.h index faeb5ef368..9a6de2bb75 100644 --- a/erts/emulator/beam/erl_alloc_util.h +++ b/erts/emulator/beam/erl_alloc_util.h @@ -61,7 +61,9 @@ typedef struct { UWord lmbcs; UWord smbcs; UWord mbcgs; - int acul; + UWord acul; + UWord acnl; + UWord acfml; void *fix; size_t *fix_type_size; @@ -116,6 +118,8 @@ typedef struct { 1024*1024, /* (bytes) smbcs: smallest mbc size */\ 10, /* (amount) mbcgs: mbc growth stages */\ 0, /* (%) acul: abandon carrier utilization limit */\ + 1000, /* (amount) acnl: abandoned carriers number limit */\ + 0, /* (bytes) acfml: abandoned carrier fblk min limit */\ /* --- Data not options -------------------------------------------- */\ NULL, /* (ptr) fix */\ NULL /* (ptr) fix_type_size */\ @@ -149,6 +153,8 @@ typedef struct { 128*1024, /* (bytes) smbcs: smallest mbc size */\ 10, /* (amount) mbcgs: mbc growth stages */\ 0, /* (%) acul: abandon carrier utilization limit */\ + 1000, /* (amount) acnl: abandoned carriers number limit */\ + 0, /* (bytes) acfml: abandoned carrier fblk min limit */\ /* --- Data not options -------------------------------------------- */\ NULL, /* (ptr) fix */\ NULL /* (ptr) fix_type_size */\ @@ -216,6 +222,8 @@ void erts_alcu_literal_32_sys_dealloc(Allctr_t*, void *ptr, Uint size, int supe void erts_lcnt_update_allocator_locks(int enable); #endif +int erts_alcu_try_set_dyn_param(Allctr_t*, Eterm param, Uint value); + #endif /* !ERL_ALLOC_UTIL__ */ #if defined(GET_ERL_ALLOC_UTIL_IMPL) && !defined(ERL_ALLOC_UTIL_IMPL__) @@ -296,41 +304,7 @@ void erts_lcnt_update_allocator_locks(int enable); typedef union {char c[ERTS_ALLOC_ALIGN_BYTES]; long l; double d;} Unit_t; - -typedef struct ErtsDoubleLink_t_ { - struct ErtsDoubleLink_t_ *next; - struct ErtsDoubleLink_t_ *prev; -}ErtsDoubleLink_t; - -typedef struct { - erts_atomic_t next; - erts_atomic_t prev; - Allctr_t *orig_allctr; /* read-only while carrier is alive */ - ErtsThrPrgrVal thr_prgr; - erts_atomic_t max_size; - UWord abandon_limit; - UWord blocks; - UWord blocks_size; - ErtsDoubleLink_t abandoned; /* node in pooled_list or traitor_list */ -} ErtsAlcCPoolData_t; - - typedef struct Carrier_t_ Carrier_t; -struct Carrier_t_ { - UWord chdr; - Carrier_t *next; - Carrier_t *prev; - erts_atomic_t allctr; - ErtsAlcCPoolData_t cpool; /* Overwritten by block if sbc */ -}; - -#define ERTS_ALC_CARRIER_TO_ALLCTR(C) \ - ((Allctr_t *) (erts_atomic_read_nob(&(C)->allctr) & ~FLG_MASK)) - -typedef struct { - Carrier_t *first; - Carrier_t *last; -} CarrierList_t; typedef struct { UWord bhdr; @@ -344,6 +318,22 @@ typedef struct { #endif } Block_t; +typedef union ErtsAllctrDDBlock_t_ ErtsAllctrDDBlock_t; + +union ErtsAllctrDDBlock_t_ { + erts_atomic_t atmc_next; + ErtsAllctrDDBlock_t *ptr_next; +}; + +typedef struct { + Block_t blk; +#if !MBC_ABLK_OFFSET_BITS + ErtsAllctrDDBlock_t umem_; +#endif +} ErtsFakeDDBlock_t; + + + #define THIS_FREE_BLK_HDR_FLG (((UWord) 1) << 0) #define PREV_FREE_BLK_HDR_FLG (((UWord) 1) << 1) #define LAST_BLK_HDR_FLG (((UWord) 1) << 2) @@ -352,14 +342,13 @@ typedef struct { (THIS_FREE_BLK_HDR_FLG | PREV_FREE_BLK_HDR_FLG | LAST_BLK_HDR_FLG) /* - * FREE_LAST_MBC_BLK_HDR_FLGS is a special flag combo used for - * distinguishing empty mbc's from allocated blocks in - * handle_delayed_dealloc(). + * HOMECOMING_MBC_BLK_HDR is a special block header combo used for + * distinguishing MBC's from allocated blocks in handle_delayed_dealloc(). */ -#define FREE_LAST_MBC_BLK_HDR_FLGS (THIS_FREE_BLK_HDR_FLG | LAST_BLK_HDR_FLG) +#define HOMECOMING_MBC_BLK_HDR (THIS_FREE_BLK_HDR_FLG | LAST_BLK_HDR_FLG) #define IS_FREE_LAST_MBC_BLK(B) \ - (((B)->bhdr & FLG_MASK) == FREE_LAST_MBC_BLK_HDR_FLGS) + (((B)->bhdr & FLG_MASK) == (THIS_FREE_BLK_HDR_FLG | LAST_BLK_HDR_FLG)) #define IS_SBC_BLK(B) (((B)->bhdr & FLG_MASK) == SBC_BLK_HDR_FLG) #define IS_MBC_BLK(B) (!IS_SBC_BLK((B))) @@ -383,6 +372,57 @@ typedef struct { typedef UWord FreeBlkFtr_t; /* Footer of a free block */ +/* This AOFF stuff really belong in erl_ao_firstfit_alloc.h */ +typedef struct AOFF_RBTree_t_ AOFF_RBTree_t; +struct AOFF_RBTree_t_ { + Block_t hdr; + AOFF_RBTree_t *parent; + AOFF_RBTree_t *left; + AOFF_RBTree_t *right; + Uint32 flags; + Uint32 max_sz; /* of all blocks in this sub-tree */ +}; + +void aoff_add_pooled_mbc(Allctr_t*, Carrier_t*); +void aoff_remove_pooled_mbc(Allctr_t*, Carrier_t*); +Carrier_t* aoff_lookup_pooled_mbc(Allctr_t*, Uint size); +void erts_aoff_larger_max_size(AOFF_RBTree_t *node); + +typedef struct { + ErtsFakeDDBlock_t homecoming_dd; + erts_atomic_t next; + erts_atomic_t prev; + Allctr_t *orig_allctr; /* read-only while carrier is alive */ + ErtsThrPrgrVal thr_prgr; + erts_atomic_t max_size; + UWord abandon_limit; + UWord blocks; + UWord blocks_size; + enum { + ERTS_MBC_IS_HOME, + ERTS_MBC_WAS_POOLED, + ERTS_MBC_WAS_TRAITOR + } state; + AOFF_RBTree_t pooled; /* node in pooled_tree */ +} ErtsAlcCPoolData_t; + +struct Carrier_t_ { + UWord chdr; + Carrier_t *next; + Carrier_t *prev; + erts_atomic_t allctr; + ErtsAlcCPoolData_t cpool; /* Overwritten by block if sbc */ +}; + +#define ERTS_ALC_CARRIER_TO_ALLCTR(C) \ + ((Allctr_t *) (erts_atomic_read_nob(&(C)->allctr) & ~FLG_MASK)) + +typedef struct { + Carrier_t *first; + Carrier_t *last; +} CarrierList_t; + + typedef Uint64 CallCounter_t; typedef struct { @@ -419,13 +459,6 @@ typedef struct { #endif -typedef union ErtsAllctrDDBlock_t_ ErtsAllctrDDBlock_t; - -union ErtsAllctrDDBlock_t_ { - erts_atomic_t atmc_next; - ErtsAllctrDDBlock_t *ptr_next; -}; - typedef struct { ErtsAllctrDDBlock_t marker; erts_atomic_t last; @@ -537,25 +570,37 @@ struct Allctr_t_ { UWord crr_set_flgs; UWord crr_clr_flgs; - /* Carriers */ + /* Carriers *employed* by this allocator */ CarrierList_t mbc_list; CarrierList_t sbc_list; struct { - /* pooled_list, traitor list and dc_list contain only - carriers _created_ by this allocator */ - ErtsDoubleLink_t pooled_list; - ErtsDoubleLink_t traitor_list; + /* pooled_tree and dc_list contain only + carriers *created* by this allocator */ + AOFF_RBTree_t* pooled_tree; CarrierList_t dc_list; UWord abandon_limit; int disable_abandon; int check_limit_count; - int util_limit; + UWord util_limit; /* acul */ + UWord in_pool_limit; /* acnl */ + UWord fblk_min_limit; /* acmfl */ struct { erts_atomic_t blocks_size; erts_atomic_t no_blocks; erts_atomic_t carriers_size; erts_atomic_t no_carriers; + CallCounter_t fail_pooled; + CallCounter_t fail_shared; + CallCounter_t fail_pend_dealloc; + CallCounter_t fail; + CallCounter_t fetch; + CallCounter_t skip_size; + CallCounter_t skip_busy; + CallCounter_t skip_not_pooled; + CallCounter_t skip_homecoming; + CallCounter_t skip_race; + CallCounter_t entrance_removed; } stat; } cpool; @@ -589,6 +634,8 @@ struct Allctr_t_ { void* (*sys_realloc)(Allctr_t *allctr, void *ptr, Uint *size_p, Uint old_size, int superalign); void (*sys_dealloc)(Allctr_t *allctr, void *ptr, Uint size, int superalign); + int (*try_set_dyn_param)(Allctr_t*, Eterm param, Uint value); + void (*init_atoms) (void); #ifdef ERTS_ALLOC_UTIL_HARD_DEBUG diff --git a/erts/emulator/beam/erl_ao_firstfit_alloc.c b/erts/emulator/beam/erl_ao_firstfit_alloc.c index 05ba1f9891..0c5545401a 100644 --- a/erts/emulator/beam/erl_ao_firstfit_alloc.c +++ b/erts/emulator/beam/erl_ao_firstfit_alloc.c @@ -20,7 +20,7 @@ /* - * Description: An "address order first fit" allocator + * Description: A family of "first fit" allocator strategies * based on a Red-Black (binary search) Tree. The search, * insert, and delete operations are all O(log n) operations * on a Red-Black Tree. @@ -40,6 +40,10 @@ * sorting order. Blocks within the same carrier are sorted * wrt size instead of address. The 'max_sz' field is maintained * in order to dismiss entire carriers with too small blocks. + * Age Order: + * Carriers are ordered by creation time instead of address. + * Oldest carrier with a large enough free block is chosen. + * No age order supported for blocks. * * Authors: Rickard Green/Sverker Eriksson */ @@ -53,10 +57,12 @@ #include "erl_ao_firstfit_alloc.h" #ifdef DEBUG +# define IS_DEBUG 1 #if 0 #define HARD_DEBUG #endif #else +# define IS_DEBUG 0 #undef HARD_DEBUG #endif @@ -92,18 +98,6 @@ #define RBT_ASSERT(x) #endif - -/* Types... */ -typedef struct AOFF_RBTree_t_ AOFF_RBTree_t; - -struct AOFF_RBTree_t_ { - Block_t hdr; - AOFF_RBTree_t *parent; - AOFF_RBTree_t *left; - AOFF_RBTree_t *right; - Uint32 flags; - Uint32 max_sz; /* of all blocks in this sub-tree */ -}; #define AOFF_BLK_SZ(B) MBC_FBLK_SZ(&(B)->hdr) /* BF block nodes keeps list of all with equal size @@ -121,6 +115,7 @@ typedef struct AOFF_Carrier_t_ AOFF_Carrier_t; struct AOFF_Carrier_t_ { Carrier_t crr; AOFF_RBTree_t rbt_node; /* My node in the carrier tree */ + Sint64 birth_time; AOFF_RBTree_t* root; /* Root of my block tree */ }; #define RBT_NODE_TO_MBC(PTR) ErtsContainerStruct((PTR), AOFF_Carrier_t, rbt_node) @@ -136,12 +131,12 @@ struct AOFF_Carrier_t_ { */ #ifdef HARD_DEBUG -# define HARD_CHECK_IS_MEMBER(ROOT,NODE) rbt_assert_is_member(ROOT,NODE) -# define HARD_CHECK_TREE(CRR,FLV,ROOT,SZ) check_tree(CRR, FLV, ROOT, SZ) -static AOFF_RBTree_t * check_tree(Carrier_t* within_crr, enum AOFF_Flavor flavor, AOFF_RBTree_t* root, Uint); +# define HARD_CHECK_IS_MEMBER(ROOT,NODE) ASSERT(rbt_is_member(ROOT,NODE)) +# define HARD_CHECK_TREE(CRR,ORDER,ROOT,SZ) check_tree(CRR, ORDER, ROOT, SZ) +static AOFF_RBTree_t * check_tree(Carrier_t*, enum AOFFSortOrder, AOFF_RBTree_t*, Uint); #else # define HARD_CHECK_IS_MEMBER(ROOT,NODE) -# define HARD_CHECK_TREE(CRR,FLV,ROOT,SZ) +# define HARD_CHECK_TREE(CRR,ORDER,ROOT,SZ) #endif @@ -179,25 +174,63 @@ static ERTS_INLINE void lower_max_size(AOFF_RBTree_t *node, else ASSERT(new_max == old_max); } -static ERTS_INLINE SWord cmp_blocks(enum AOFF_Flavor flavor, +/* + * Set possibly new larger 'max_sz' of node and propagate change toward root + */ +void erts_aoff_larger_max_size(AOFF_RBTree_t *node) +{ + AOFF_RBTree_t* x = node; + const Uint new_sz = node->hdr.bhdr; + + ASSERT(!x->left || x->left->max_sz <= x->max_sz); + ASSERT(!x->right || x->right->max_sz <= x->max_sz); + + while (new_sz > x->max_sz) { + x->max_sz = new_sz; + x = x->parent; + if (!x) + break; + } +} + +/* Compare nodes for both carrier and block trees */ +static ERTS_INLINE SWord cmp_blocks(enum AOFFSortOrder order, AOFF_RBTree_t* lhs, AOFF_RBTree_t* rhs) { ASSERT(lhs != rhs); - ASSERT(flavor == AOFF_AOFF || FBLK_TO_MBC(&lhs->hdr) == FBLK_TO_MBC(&rhs->hdr)); - if (flavor != AOFF_AOFF) { - SWord diff = (SWord)AOFF_BLK_SZ(lhs) - (SWord)AOFF_BLK_SZ(rhs); - if (diff || flavor == AOFF_BF) return diff; + if (order == FF_AGEFF) { + AOFF_Carrier_t* lc = RBT_NODE_TO_MBC(lhs); + AOFF_Carrier_t* rc = RBT_NODE_TO_MBC(rhs); + Sint64 diff = lc->birth_time - rc->birth_time; + #ifdef ARCH_64 + if (diff) + return diff; + #else + if (diff < 0) + return -1; + else if (diff > 0) + return 1; + #endif + } + else { + ASSERT(order == FF_AOFF || FBLK_TO_MBC(&lhs->hdr) == FBLK_TO_MBC(&rhs->hdr)); + if (order != FF_AOFF) { + SWord diff = (SWord)AOFF_BLK_SZ(lhs) - (SWord)AOFF_BLK_SZ(rhs); + if (diff || order == FF_BF) return diff; + } } return (char*)lhs - (char*)rhs; } -static ERTS_INLINE SWord cmp_cand_blk(enum AOFF_Flavor flavor, +/* Compare candidate block. Only for block tree */ +static ERTS_INLINE SWord cmp_cand_blk(enum AOFFSortOrder order, Block_t* cand_blk, AOFF_RBTree_t* rhs) { - if (flavor != AOFF_AOFF) { + ASSERT(order != FF_AGEFF); + if (order != FF_AOFF) { if (BLK_TO_MBC(cand_blk) == FBLK_TO_MBC(&rhs->hdr)) { SWord diff = (SWord)MBC_BLK_SZ(cand_blk) - (SWord)MBC_FBLK_SZ(&rhs->hdr); - if (diff || flavor == AOFF_BF) return diff; + if (diff || order == FF_BF) return diff; } } return (char*)cand_blk - (char*)rhs; @@ -218,11 +251,8 @@ static UWord aoff_largest_fblk_in_mbc(Allctr_t*, Carrier_t*); /* Generic tree functions used by both carrier and block trees. */ static void rbt_delete(AOFF_RBTree_t** root, AOFF_RBTree_t* del); -static void rbt_insert(enum AOFF_Flavor flavor, AOFF_RBTree_t** root, AOFF_RBTree_t* blk); +static void rbt_insert(enum AOFFSortOrder, AOFF_RBTree_t** root, AOFF_RBTree_t* blk); static AOFF_RBTree_t* rbt_search(AOFF_RBTree_t* root, Uint size); -#ifdef HARD_DEBUG -static int rbt_assert_is_member(AOFF_RBTree_t* root, AOFF_RBTree_t* node); -#endif static Eterm info_options(Allctr_t *, char *, fmtfn_t *, void *, Uint **, Uint *); static void init_atoms(void); @@ -230,10 +260,17 @@ static void init_atoms(void); static int atoms_initialized = 0; +#ifndef ERTS_HAVE_OS_MONOTONIC_TIME_SUPPORT +static erts_atomic64_t birth_time_counter; +#endif + void erts_aoffalc_init(void) { atoms_initialized = 0; +#ifndef ERTS_HAVE_OS_MONOTONIC_TIME_SUPPORT + erts_atomic64_init_nob(&birth_time_counter, 0); +#endif } Allctr_t * @@ -254,11 +291,12 @@ erts_aoffalc_start(AOFFAllctr_t *alc, sys_memcpy((void *) alc, (void *) &zero.allctr, sizeof(AOFFAllctr_t)); - alc->flavor = aoffinit->flavor; + alc->blk_order = aoffinit->blk_order; + alc->crr_order = aoffinit->crr_order; allctr->mbc_header_size = sizeof(AOFF_Carrier_t); allctr->min_mbc_size = MIN_MBC_SZ; allctr->min_mbc_first_free_size = MIN_MBC_FIRST_FREE_SZ; - allctr->min_block_size = (aoffinit->flavor == AOFF_BF ? + allctr->min_block_size = (aoffinit->blk_order == FF_BF ? sizeof(AOFF_RBTreeList_t):sizeof(AOFF_RBTree_t)); allctr->vsn_str = ERTS_ALC_AOFF_ALLOC_VSN_STR; @@ -487,9 +525,9 @@ aoff_unlink_free_block(Allctr_t *allctr, Block_t *blk) AOFF_Carrier_t *crr = (AOFF_Carrier_t*) FBLK_TO_MBC(&del->hdr); ASSERT(crr->rbt_node.hdr.bhdr == crr->root->max_sz); - HARD_CHECK_TREE(&crr->crr, alc->flavor, crr->root, 0); + HARD_CHECK_TREE(&crr->crr, alc->blk_order, crr->root, 0); - if (alc->flavor == AOFF_BF) { + if (alc->blk_order == FF_BF) { ASSERT(del->flags & IS_BF_FLG); if (IS_LIST_ELEM(del)) { /* Remove from list */ @@ -510,14 +548,14 @@ aoff_unlink_free_block(Allctr_t *allctr, Block_t *blk) replace(&crr->root, (AOFF_RBTree_t*)del, LIST_NEXT(del)); - HARD_CHECK_TREE(&crr->crr, alc->flavor, crr->root, 0); + HARD_CHECK_TREE(&crr->crr, alc->blk_order, crr->root, 0); return; } } rbt_delete(&crr->root, (AOFF_RBTree_t*)del); - HARD_CHECK_TREE(&crr->crr, alc->flavor, crr->root, 0); + HARD_CHECK_TREE(&crr->crr, alc->blk_order, crr->root, 0); /* Update the carrier tree with a potentially new (lower) max_sz */ @@ -715,32 +753,33 @@ aoff_link_free_block(Allctr_t *allctr, Block_t *block) ASSERT(allctr == ERTS_ALC_CARRIER_TO_ALLCTR(&blk_crr->crr)); ASSERT(blk_crr->rbt_node.hdr.bhdr == (blk_crr->root ? blk_crr->root->max_sz : 0)); - HARD_CHECK_TREE(&blk_crr->crr, alc->flavor, blk_crr->root, 0); + HARD_CHECK_TREE(&blk_crr->crr, alc->blk_order, blk_crr->root, 0); - rbt_insert(alc->flavor, &blk_crr->root, blk); + rbt_insert(alc->blk_order, &blk_crr->root, blk); - /* Update the carrier tree with a potentially new (larger) max_sz - */ + /* + * Update carrier tree with a potentially new (larger) max_sz + */ crr_node = &blk_crr->rbt_node; if (blk_sz > crr_node->hdr.bhdr) { - ASSERT(blk_sz == blk_crr->root->max_sz); - crr_node->hdr.bhdr = blk_sz; - while (blk_sz > crr_node->max_sz) { - crr_node->max_sz = blk_sz; - crr_node = crr_node->parent; - if (!crr_node) break; - } + ASSERT(blk_sz == blk_crr->root->max_sz); + crr_node->hdr.bhdr = blk_sz; + while (blk_sz > crr_node->max_sz) { + crr_node->max_sz = blk_sz; + crr_node = crr_node->parent; + if (!crr_node) break; + } } - HARD_CHECK_TREE(&blk_crr->crr, alc->flavor, blk_crr->root, 0); + HARD_CHECK_TREE(NULL, alc->crr_order, alc->mbc_root, 0); } static void -rbt_insert(enum AOFF_Flavor flavor, AOFF_RBTree_t** root, AOFF_RBTree_t* blk) +rbt_insert(enum AOFFSortOrder order, AOFF_RBTree_t** root, AOFF_RBTree_t* blk) { Uint blk_sz = AOFF_BLK_SZ(blk); #ifdef DEBUG - blk->flags = (flavor == AOFF_BF) ? IS_BF_FLG : 0; + blk->flags = (order == FF_BF) ? IS_BF_FLG : 0; #else blk->flags = 0; #endif @@ -760,7 +799,7 @@ rbt_insert(enum AOFF_Flavor flavor, AOFF_RBTree_t** root, AOFF_RBTree_t* blk) if (x->max_sz < blk_sz) { x->max_sz = blk_sz; } - diff = cmp_blocks(flavor, blk, x); + diff = cmp_blocks(order, blk, x); if (diff < 0) { if (!x->left) { blk->parent = x; @@ -778,7 +817,7 @@ rbt_insert(enum AOFF_Flavor flavor, AOFF_RBTree_t** root, AOFF_RBTree_t* blk) x = x->right; } else { - ASSERT(flavor == AOFF_BF); + ASSERT(order == FF_BF); ASSERT(blk->flags & IS_BF_FLG); ASSERT(x->flags & IS_BF_FLG); SET_LIST_ELEM(blk); @@ -798,7 +837,7 @@ rbt_insert(enum AOFF_Flavor flavor, AOFF_RBTree_t** root, AOFF_RBTree_t* blk) if (IS_RED(blk->parent)) tree_insert_fixup(root, blk); } - if (flavor == AOFF_BF) { + if (order == FF_BF) { SET_TREE_NODE(blk); LIST_NEXT(blk) = NULL; } @@ -826,6 +865,16 @@ rbt_search(AOFF_RBTree_t* root, Uint size) } } +Carrier_t* aoff_lookup_pooled_mbc(Allctr_t* allctr, Uint size) +{ + AOFF_RBTree_t* node; + + if (!allctr->cpool.pooled_tree) + return NULL; + node = rbt_search(allctr->cpool.pooled_tree, size); + return node ? ErtsContainerStruct(node, Carrier_t, cpool.pooled) : NULL; +} + static Block_t * aoff_get_free_block(Allctr_t *allctr, Uint size, Block_t *cand_blk, Uint cand_size) @@ -850,7 +899,7 @@ aoff_get_free_block(Allctr_t *allctr, Uint size, /* Get block within carrier tree */ #ifdef HARD_DEBUG - dbg_blk = HARD_CHECK_TREE(&crr->crr, alc->flavor, crr->root, size); + dbg_blk = HARD_CHECK_TREE(&crr->crr, alc->blk_order, crr->root, size); #endif blk = rbt_search(crr->root, size); @@ -863,7 +912,7 @@ aoff_get_free_block(Allctr_t *allctr, Uint size, if (!blk) return NULL; - if (cand_blk && cmp_cand_blk(alc->flavor, cand_blk, blk) < 0) { + if (cand_blk && cmp_cand_blk(alc->blk_order, cand_blk, blk) < 0) { return NULL; /* cand_blk was better */ } @@ -872,23 +921,32 @@ aoff_get_free_block(Allctr_t *allctr, Uint size, return (Block_t *) blk; } +static ERTS_INLINE Sint64 get_birth_time(void) +{ +#ifdef ERTS_HAVE_OS_MONOTONIC_TIME_SUPPORT + return (Sint64) erts_os_monotonic_time(); +#else + return (Sint64) erts_atomic64_inc_read_nob(&birth_time_counter); +#endif +} + static void aoff_creating_mbc(Allctr_t *allctr, Carrier_t *carrier) { AOFFAllctr_t *alc = (AOFFAllctr_t *) allctr; AOFF_Carrier_t *crr = (AOFF_Carrier_t*) carrier; AOFF_RBTree_t **root = &alc->mbc_root; - HARD_CHECK_TREE(NULL, 0, *root, 0); + HARD_CHECK_TREE(NULL, alc->crr_order, *root, 0); - /* Link carrier in address order tree - */ crr->rbt_node.hdr.bhdr = 0; - rbt_insert(AOFF_AOFF, root, &crr->rbt_node); + if (alc->crr_order == FF_AGEFF || IS_DEBUG) + crr->birth_time = get_birth_time(); + rbt_insert(alc->crr_order, root, &crr->rbt_node); /* aoff_link_free_block will add free block later */ crr->root = NULL; - HARD_CHECK_TREE(NULL, 0, *root, 0); + HARD_CHECK_TREE(NULL, alc->crr_order, *root, 0); } #define IS_CRR_IN_TREE(CRR,ROOT) \ @@ -911,27 +969,38 @@ static void aoff_add_mbc(Allctr_t *allctr, Carrier_t *carrier) AOFF_RBTree_t **root = &alc->mbc_root; ASSERT(!IS_CRR_IN_TREE(crr, *root)); - HARD_CHECK_TREE(NULL, 0, *root, 0); + HARD_CHECK_TREE(NULL, alc->crr_order, *root, 0); + + rbt_insert(alc->crr_order, root, &crr->rbt_node); + + HARD_CHECK_TREE(NULL, alc->crr_order, *root, 0); +} + +void aoff_add_pooled_mbc(Allctr_t *allctr, Carrier_t *crr) +{ + AOFF_RBTree_t **root = &allctr->cpool.pooled_tree; + + ASSERT(allctr == crr->cpool.orig_allctr); + HARD_CHECK_TREE(NULL, 0, *root, 0); /* Link carrier in address order tree */ - rbt_insert(AOFF_AOFF, root, &crr->rbt_node); + rbt_insert(FF_AOFF, root, &crr->cpool.pooled); HARD_CHECK_TREE(NULL, 0, *root, 0); } static void aoff_remove_mbc(Allctr_t *allctr, Carrier_t *carrier) { - AOFFAllctr_t *alc = (AOFFAllctr_t *) allctr; - AOFF_Carrier_t *crr = (AOFF_Carrier_t*) carrier; - AOFF_RBTree_t **root = &alc->mbc_root; + AOFF_RBTree_t **root = &((AOFFAllctr_t*)allctr)->mbc_root; + AOFF_Carrier_t *crr = (AOFF_Carrier_t*)carrier; ASSERT(allctr == ERTS_ALC_CARRIER_TO_ALLCTR(carrier)); if (!IS_CRR_IN_TREE(crr,*root)) - return; + return; - HARD_CHECK_TREE(NULL, 0, *root, 0); + HARD_CHECK_TREE(NULL, alc->crr_order, *root, 0); rbt_delete(root, &crr->rbt_node); crr->rbt_node.parent = NULL; @@ -939,9 +1008,27 @@ static void aoff_remove_mbc(Allctr_t *allctr, Carrier_t *carrier) crr->rbt_node.right = NULL; crr->rbt_node.max_sz = crr->rbt_node.hdr.bhdr; - HARD_CHECK_TREE(NULL, 0, *root, 0); + HARD_CHECK_TREE(NULL, alc->crr_order, *root, 0); +} + +void aoff_remove_pooled_mbc(Allctr_t *allctr, Carrier_t *crr) +{ + ASSERT(allctr == crr->cpool.orig_allctr); + + HARD_CHECK_TREE(NULL, 0, allctr->cpool.pooled_tree, 0); + + rbt_delete(&allctr->cpool.pooled_tree, &crr->cpool.pooled); +#ifdef DEBUG + crr->cpool.pooled.parent = NULL; + crr->cpool.pooled.left = NULL; + crr->cpool.pooled.right = NULL; + crr->cpool.pooled.max_sz = 0; +#endif + HARD_CHECK_TREE(NULL, 0, allctr->cpool.pooled_tree, 0); + } + static UWord aoff_largest_fblk_in_mbc(Allctr_t* allctr, Carrier_t* carrier) { AOFF_Carrier_t *crr = (AOFF_Carrier_t*) carrier; @@ -955,47 +1042,35 @@ static UWord aoff_largest_fblk_in_mbc(Allctr_t* allctr, Carrier_t* carrier) * info_options() */ +static const char* flavor_str[2][3] = { + {"ageffcaoff", "ageffcaobf", "ageffcbf"}, + { "aoff", "aoffcaobf", "aoffcbf"} +}; +static Eterm flavor_atoms[2][3]; + static struct { Eterm as; - Eterm aoff; - Eterm aoffcaobf; - Eterm aoffcbf; -#ifdef DEBUG - Eterm end_of_atoms; -#endif } am; -static void ERTS_INLINE atom_init(Eterm *atom, char *name) +static void ERTS_INLINE atom_init(Eterm *atom, const char *name) { - *atom = am_atom_put(name, strlen(name)); + *atom = am_atom_put(name, sys_strlen(name)); } #define AM_INIT(AM) atom_init(&am.AM, #AM) static void init_atoms(void) { -#ifdef DEBUG - Eterm *atom; -#endif + int i, j; if (atoms_initialized) return; -#ifdef DEBUG - for (atom = (Eterm *) &am; atom <= &am.end_of_atoms; atom++) { - *atom = THE_NON_VALUE; - } -#endif AM_INIT(as); - AM_INIT(aoff); - AM_INIT(aoffcaobf); - AM_INIT(aoffcbf); -#ifdef DEBUG - for (atom = (Eterm *) &am; atom < &am.end_of_atoms; atom++) { - ASSERT(*atom != THE_NON_VALUE); - } -#endif + for (i = 0; i < 2; i++) + for (j = 0; j < 3; j++) + atom_init(&flavor_atoms[i][j], flavor_str[i][j]); atoms_initialized = 1; } @@ -1021,15 +1096,16 @@ info_options(Allctr_t *allctr, { AOFFAllctr_t* alc = (AOFFAllctr_t*) allctr; Eterm res = THE_NON_VALUE; - const char* flavor_str[3] = {"aoff", "aoffcaobf", "aoffcbf"}; - Eterm flavor_atom[3] = {am.aoff, am.aoffcaobf, am.aoffcbf}; + + ASSERT(alc->crr_order >= 0 && alc->crr_order <= 1); + ASSERT(alc->blk_order >= 1 && alc->blk_order <= 3); if (print_to_p) { erts_print(*print_to_p, print_to_arg, "%sas: %s\n", prefix, - flavor_str[alc->flavor]); + flavor_str[alc->crr_order][alc->blk_order-1]); } if (hpp || szp) { @@ -1039,7 +1115,8 @@ info_options(Allctr_t *allctr, __FILE__, __LINE__);; res = NIL; - add_2tup(hpp, szp, &res, am.as, flavor_atom[alc->flavor]); + add_2tup(hpp, szp, &res, am.as, + flavor_atoms[alc->crr_order][alc->blk_order-1]); } return res; @@ -1057,7 +1134,7 @@ UWord erts_aoffalc_test(UWord op, UWord a1, UWord a2) { switch (op) { - case 0x500: return (UWord) ((AOFFAllctr_t *) a1)->flavor == AOFF_AOBF; + case 0x500: return (UWord) ((AOFFAllctr_t *) a1)->blk_order == FF_AOBF; case 0x501: { AOFF_RBTree_t *node = ((AOFFAllctr_t *) a1)->mbc_root; Uint size = (Uint) a2; @@ -1072,7 +1149,7 @@ erts_aoffalc_test(UWord op, UWord a1, UWord a2) case 0x507: return (UWord) IS_TREE_NODE((AOFF_RBTree_t *) a1); case 0x508: return (UWord) 0; /* IS_BF_ALGO */ case 0x509: return (UWord) ((AOFF_RBTree_t *) a1)->max_sz; - case 0x50a: return (UWord) ((AOFFAllctr_t *) a1)->flavor == AOFF_BF; + case 0x50a: return (UWord) ((AOFFAllctr_t *) a1)->blk_order == FF_BF; case 0x50b: return (UWord) LIST_PREV(a1); default: ASSERT(0); return ~((UWord) 0); } @@ -1085,12 +1162,13 @@ erts_aoffalc_test(UWord op, UWord a1, UWord a2) #ifdef HARD_DEBUG - -static int rbt_assert_is_member(AOFF_RBTree_t* root, AOFF_RBTree_t* node) +static int rbt_is_member(AOFF_RBTree_t* root, AOFF_RBTree_t* node) { while (node != root) { - ASSERT(node->parent); - ASSERT(node->parent->left == node || node->parent->right == node); + if (!node->parent || (node->parent->left != node && + node->parent->right != node)) { + return 0; + } node = node->parent; } return 1; @@ -1132,7 +1210,7 @@ static void print_tree(AOFF_RBTree_t*); */ static AOFF_RBTree_t * -check_tree(Carrier_t* within_crr, enum AOFF_Flavor flavor, AOFF_RBTree_t* root, Uint size) +check_tree(Carrier_t* within_crr, enum AOFFSortOrder order, AOFF_RBTree_t* root, Uint size) { AOFF_RBTree_t *res = NULL; Sint blacks; @@ -1144,7 +1222,8 @@ check_tree(Carrier_t* within_crr, enum AOFF_Flavor flavor, AOFF_RBTree_t* root, #ifdef PRINT_TREE print_tree(root); #endif - ASSERT(within_crr || flavor == AOFF_AOFF); + ASSERT((within_crr && order >= FF_AOFF) || + (!within_crr && order <= FF_AOFF)); if (!root) return res; @@ -1202,7 +1281,7 @@ check_tree(Carrier_t* within_crr, enum AOFF_Flavor flavor, AOFF_RBTree_t* root, ASSERT(((char*)x + AOFF_BLK_SZ(x)) <= ((char*)crr + CARRIER_SZ(crr))); } - if (flavor == AOFF_BF) { + if (order == FF_BF) { AOFF_RBTree_t* y = x; AOFF_RBTree_t* nxt = LIST_NEXT(y); ASSERT(IS_TREE_NODE(x)); @@ -1225,13 +1304,13 @@ check_tree(Carrier_t* within_crr, enum AOFF_Flavor flavor, AOFF_RBTree_t* root, if (x->left) { ASSERT(x->left->parent == x); - ASSERT(cmp_blocks(flavor, x->left, x) < 0); + ASSERT(cmp_blocks(order, x->left, x) < 0); ASSERT(x->left->max_sz <= x->max_sz); } if (x->right) { ASSERT(x->right->parent == x); - ASSERT(cmp_blocks(flavor, x->right, x) > 0); + ASSERT(cmp_blocks(order, x->right, x) > 0); ASSERT(x->right->max_sz <= x->max_sz); } ASSERT(x->max_sz >= AOFF_BLK_SZ(x)); @@ -1240,7 +1319,7 @@ check_tree(Carrier_t* within_crr, enum AOFF_Flavor flavor, AOFF_RBTree_t* root, || x->max_sz == (x->right ? x->right->max_sz : 0)); if (size && AOFF_BLK_SZ(x) >= size) { - if (!res || cmp_blocks(flavor, x, res) < 0) { + if (!res || cmp_blocks(order, x, res) < 0) { res = x; } } diff --git a/erts/emulator/beam/erl_ao_firstfit_alloc.h b/erts/emulator/beam/erl_ao_firstfit_alloc.h index 7349c6ab19..9cf4fc81a8 100644 --- a/erts/emulator/beam/erl_ao_firstfit_alloc.h +++ b/erts/emulator/beam/erl_ao_firstfit_alloc.h @@ -28,14 +28,16 @@ typedef struct AOFFAllctr_t_ AOFFAllctr_t; -enum AOFF_Flavor { - AOFF_AOFF = 0, - AOFF_AOBF = 1, - AOFF_BF = 2 +enum AOFFSortOrder { + FF_AGEFF = 0, + FF_AOFF = 1, + FF_AOBF = 2, + FF_BF = 3 }; typedef struct { - enum AOFF_Flavor flavor; + enum AOFFSortOrder blk_order; + enum AOFFSortOrder crr_order; } AOFFAllctrInit_t; #define ERTS_DEFAULT_AOFF_ALLCTR_INIT {0/*dummy*/} @@ -53,12 +55,12 @@ Allctr_t *erts_aoffalc_start(AOFFAllctr_t *, AOFFAllctrInit_t*, AllctrInit_t *); #define GET_ERL_ALLOC_UTIL_IMPL #include "erl_alloc_util.h" - struct AOFFAllctr_t_ { Allctr_t allctr; /* Has to be first! */ struct AOFF_RBTree_t_* mbc_root; - enum AOFF_Flavor flavor; + enum AOFFSortOrder blk_order; + enum AOFFSortOrder crr_order; }; UWord erts_aoffalc_test(UWord, UWord, UWord); diff --git a/erts/emulator/beam/erl_bestfit_alloc.c b/erts/emulator/beam/erl_bestfit_alloc.c index 6173c408e1..85fc4c3a85 100644 --- a/erts/emulator/beam/erl_bestfit_alloc.c +++ b/erts/emulator/beam/erl_bestfit_alloc.c @@ -875,7 +875,7 @@ static struct { static void ERTS_INLINE atom_init(Eterm *atom, char *name) { - *atom = am_atom_put(name, strlen(name)); + *atom = am_atom_put(name, sys_strlen(name)); } #define AM_INIT(AM) atom_init(&am.AM, #AM) diff --git a/erts/emulator/beam/erl_bif_binary.c b/erts/emulator/beam/erl_bif_binary.c index 4cafa499a9..469f6a1ea8 100644 --- a/erts/emulator/beam/erl_bif_binary.c +++ b/erts/emulator/beam/erl_bif_binary.c @@ -61,8 +61,6 @@ static Export binary_longest_prefix_trap_export; static BIF_RETTYPE binary_longest_prefix_trap(BIF_ALIST_3); static Export binary_longest_suffix_trap_export; static BIF_RETTYPE binary_longest_suffix_trap(BIF_ALIST_3); -static Export binary_bin_to_list_trap_export; -static BIF_RETTYPE binary_bin_to_list_trap(BIF_ALIST_3); static Export binary_copy_trap_export; static BIF_RETTYPE binary_copy_trap(BIF_ALIST_2); static Uint max_loop_limit; @@ -86,10 +84,6 @@ void erts_init_bif_binary(void) am_erlang, am_binary_longest_suffix_trap, 3, &binary_longest_suffix_trap); - erts_init_trap_export(&binary_bin_to_list_trap_export, - am_erlang, am_binary_bin_to_list_trap, 3, - &binary_bin_to_list_trap); - erts_init_trap_export(&binary_copy_trap_export, am_erlang, am_binary_copy_trap, 2, &binary_copy_trap); @@ -371,7 +365,7 @@ static ACTrie *create_acdata(MyAllocator *my, Uint len, acn->d = 0; acn->final = 0; acn->h = NULL; - memset(acn->g, 0, sizeof(ACNode *) * ALPHABET_SIZE); + sys_memset(acn->g, 0, sizeof(ACNode *) * ALPHABET_SIZE); #ifdef HARDDEBUG act->idc = 0; acn->id = 0; @@ -394,7 +388,7 @@ static BMData *create_bmdata(MyAllocator *my, byte *x, Uint len, init_my_allocator(my, datasize, data); bmd = my_alloc(my, sizeof(BMData)); bmd->x = my_alloc(my,len); - memcpy(bmd->x,x,len); + sys_memcpy(bmd->x,x,len); bmd->len = len; bmd->goodshift = my_alloc(my,sizeof(Uint) * len); *the_bin = mb; @@ -439,7 +433,7 @@ static void ac_add_one_pattern(MyAllocator *my, ACTrie *act, byte *x, Uint len) nn->d = i+1; nn->h = act->root; nn->final = 0; - memset(nn->g, 0, sizeof(ACNode *) * ALPHABET_SIZE); + sys_memset(nn->g, 0, sizeof(ACNode *) * ALPHABET_SIZE); acn->g[x[i]] = nn; ++i; acn = nn; @@ -2270,7 +2264,7 @@ static BIF_RETTYPE do_longest_common(Process *p, Eterm list, int direction) if (cd[i].type == CL_TYPE_HEAP_NOALLOC) { unsigned char *tmp = cd[i].buff; cd[i].buff = erts_alloc(ERTS_ALC_T_BINARY_BUFFER, cd[i].bufflen); - memcpy(cd[i].buff,tmp,cd[i].bufflen); + sys_memcpy(cd[i].buff,tmp,cd[i].bufflen); cd[i].type = CL_TYPE_HEAP; } } @@ -2440,191 +2434,6 @@ BIF_RETTYPE binary_at_2(BIF_ALIST_2) BIF_ERROR(BIF_P,BADARG); } -#define BIN_TO_LIST_OK 0 -#define BIN_TO_LIST_TRAP 1 -/* No badarg, checked before call */ - -#define BIN_TO_LIST_LOOP_FACTOR 10 - -static int do_bin_to_list(Process *p, byte *bytes, Uint bit_offs, - Uint start, Sint *lenp, Eterm *termp) -{ - Uint reds = get_reds(p, BIN_TO_LIST_LOOP_FACTOR); /* reds can never be 0 */ - Uint len = *lenp; - Uint loops; - Eterm *hp; - Eterm term = *termp; - Uint n; - - ASSERT(reds > 0); - - loops = MIN(reds,len); - - BUMP_REDS(p, loops / BIN_TO_LIST_LOOP_FACTOR); - - hp = HAlloc(p,2*loops); - while (loops--) { - --len; - if (bit_offs) { - n = ((((Uint) bytes[start+len]) << bit_offs) | - (((Uint) bytes[start+len+1]) >> (8-bit_offs))) & 0xFF; - } else { - n = bytes[start+len]; - } - - term = CONS(hp,make_small(n),term); - hp +=2; - } - *termp = term; - *lenp = len; - if (len) { - BUMP_ALL_REDS(p); - return BIN_TO_LIST_TRAP; - } - return BIN_TO_LIST_OK; -} - - -static BIF_RETTYPE do_trap_bin_to_list(Process *p, Eterm binary, - Uint start, Sint len, Eterm sofar) -{ - Eterm *hp; - Eterm blob; - - hp = HAlloc(p,3); - hp[0] = make_pos_bignum_header(2); - hp[1] = start; - hp[2] = (Uint) len; - blob = make_big(hp); - BIF_TRAP3(&binary_bin_to_list_trap_export, p, binary, blob, sofar); -} - -static BIF_RETTYPE binary_bin_to_list_trap(BIF_ALIST_3) -{ - Eterm *ptr; - Uint start; - Sint len; - byte *bytes; - Uint bit_offs; - Uint bit_size; - Eterm res = BIF_ARG_3; - - ptr = big_val(BIF_ARG_2); - start = ptr[1]; - len = (Sint) ptr[2]; - - ERTS_GET_BINARY_BYTES(BIF_ARG_1,bytes,bit_offs,bit_size); - if (do_bin_to_list(BIF_P, bytes, bit_offs, start, &len, &res) == - BIN_TO_LIST_OK) { - BIF_RET(res); - } - return do_trap_bin_to_list(BIF_P,BIF_ARG_1,start,len,res); -} - -static BIF_RETTYPE binary_bin_to_list_common(Process *p, - Eterm bin, - Eterm epos, - Eterm elen) -{ - Uint pos; - Sint len; - size_t sz; - byte *bytes; - Uint bit_offs; - Uint bit_size; - Eterm res = NIL; - - if (is_not_binary(bin)) { - goto badarg; - } - if (!term_to_Uint(epos, &pos)) { - goto badarg; - } - if (!term_to_Sint(elen, &len)) { - goto badarg; - } - if (len < 0) { - Uint lentmp = -(Uint)len; - /* overflow */ - if ((Sint)lentmp < 0) { - goto badarg; - } - len = lentmp; - if (len > pos) { - goto badarg; - } - pos -= len; - } - /* overflow */ - if ((pos + len) < pos || (len > 0 && (pos + len) == pos)) { - goto badarg; - } - sz = binary_size(bin); - - if (pos+len > sz) { - goto badarg; - } - ERTS_GET_BINARY_BYTES(bin,bytes,bit_offs,bit_size); - if (bit_size != 0) { - goto badarg; - } - if(do_bin_to_list(p, bytes, bit_offs, pos, &len, &res) == - BIN_TO_LIST_OK) { - BIF_RET(res); - } - return do_trap_bin_to_list(p,bin,pos,len,res); - - badarg: - BIF_ERROR(p,BADARG); -} - -BIF_RETTYPE binary_bin_to_list_3(BIF_ALIST_3) -{ - return binary_bin_to_list_common(BIF_P,BIF_ARG_1,BIF_ARG_2,BIF_ARG_3); -} - -BIF_RETTYPE binary_bin_to_list_2(BIF_ALIST_2) -{ - Eterm *tp; - - if (is_not_tuple(BIF_ARG_2)) { - goto badarg; - } - tp = tuple_val(BIF_ARG_2); - if (arityval(*tp) != 2) { - goto badarg; - } - return binary_bin_to_list_common(BIF_P,BIF_ARG_1,tp[1],tp[2]); - badarg: - BIF_ERROR(BIF_P,BADARG); -} - -BIF_RETTYPE binary_bin_to_list_1(BIF_ALIST_1) -{ - Uint pos = 0; - Sint len; - byte *bytes; - Uint bit_offs; - Uint bit_size; - Eterm res = NIL; - - if (is_not_binary(BIF_ARG_1)) { - goto badarg; - } - len = binary_size(BIF_ARG_1); - ERTS_GET_BINARY_BYTES(BIF_ARG_1,bytes,bit_offs,bit_size); - if (bit_size != 0) { - goto badarg; - } - if(do_bin_to_list(BIF_P, bytes, bit_offs, pos, &len, &res) == - BIN_TO_LIST_OK) { - BIF_RET(res); - } - return do_trap_bin_to_list(BIF_P,BIF_ARG_1,pos,len,res); - badarg: - BIF_ERROR(BIF_P,BADARG); -} - HIPE_WRAPPER_BIF_DISABLE_GC(binary_list_to_bin, 1) BIF_RETTYPE binary_list_to_bin_1(BIF_ALIST_1) @@ -2747,7 +2556,7 @@ static BIF_RETTYPE do_binary_copy(Process *p, Eterm bin, Eterm en) 0); if (!(cbs->temp_alloc)) { /* alignment not needed, need to copy */ byte *tmp = erts_alloc(ERTS_ALC_T_BINARY_BUFFER,size); - memcpy(tmp,cbs->source,size); + sys_memcpy(tmp,cbs->source,size); cbs->source = tmp; cbs->source_type = BC_TYPE_HEAP; } else { @@ -2760,7 +2569,7 @@ static BIF_RETTYPE do_binary_copy(Process *p, Eterm bin, Eterm en) pos = 0; i = 0; while (pos < reds) { - memcpy(t+pos,cbs->source, size); + sys_memcpy(t+pos,cbs->source, size); pos += size; ++i; } @@ -2785,7 +2594,7 @@ static BIF_RETTYPE do_binary_copy(Process *p, Eterm bin, Eterm en) } pos = 0; while (pos < target_size) { - memcpy(t+pos,source, size); + sys_memcpy(t+pos,source, size); pos += size; } erts_free_aligned_binary_bytes(temp_alloc); @@ -2815,7 +2624,7 @@ BIF_RETTYPE binary_copy_trap(BIF_ALIST_2) if ((n-1) * size >= reds) { Uint i = 0; while ((pos - opos) < reds) { - memcpy(t+pos,cbs->source, size); + sys_memcpy(t+pos,cbs->source, size); pos += size; ++i; } @@ -2825,28 +2634,21 @@ BIF_RETTYPE binary_copy_trap(BIF_ALIST_2) BIF_TRAP2(&binary_copy_trap_export, BIF_P, BIF_ARG_1, BIF_ARG_2); } else { Binary *save; - ProcBin* pb; + Eterm resbin; Uint target_size = cbs->result->orig_size; while (pos < target_size) { - memcpy(t+pos,cbs->source, size); + sys_memcpy(t+pos,cbs->source, size); pos += size; } - save = cbs->result; + save = cbs->result; cbs->result = NULL; cleanup_copy_bin_state(mb); /* now cbs is dead */ - pb = (ProcBin *) HAlloc(BIF_P, PROC_BIN_SIZE); - pb->thing_word = HEADER_PROC_BIN; - pb->size = target_size; - pb->next = MSO(BIF_P).first; - MSO(BIF_P).first = (struct erl_off_heap_header*) pb; - pb->val = save; - pb->bytes = t; - pb->flags = 0; - - OH_OVERHEAD(&(MSO(BIF_P)), target_size / sizeof(Eterm)); - BUMP_REDS(BIF_P,(pos - opos) / BINARY_COPY_LOOP_FACTOR); - BIF_RET(make_binary(pb)); + resbin = erts_build_proc_bin(&MSO(BIF_P), + HAlloc(BIF_P, PROC_BIN_SIZE), + save); + BUMP_REDS(BIF_P,(pos - opos) / BINARY_COPY_LOOP_FACTOR); + BIF_RET(resbin); } } @@ -3227,7 +3029,7 @@ static void dump_bm_data(BMData *bm) static void dump_ac_node(ACNode *node, int indent, int ch) { int i; char *spaces = erts_alloc(ERTS_ALC_T_TMP, 10 * indent + 1); - memset(spaces,' ',10*indent); + sys_memset(spaces,' ',10*indent); spaces[10*indent] = '\0'; erts_printf("%s-> %c\n",spaces,ch); erts_printf("%sId: %u\n",spaces,(unsigned) node->id); diff --git a/erts/emulator/beam/erl_bif_chksum.c b/erts/emulator/beam/erl_bif_chksum.c index 9417803e14..9095bcd380 100644 --- a/erts/emulator/beam/erl_bif_chksum.c +++ b/erts/emulator/beam/erl_bif_chksum.c @@ -516,7 +516,7 @@ md5_2(BIF_ALIST_2) /* No need to check context, this function cannot be called with unaligned or badly sized context as it's always trapped to. */ bytes = binary_bytes(BIF_ARG_1); - memcpy(&context,bytes,sizeof(MD5_CTX)); + sys_memcpy(&context,bytes,sizeof(MD5_CTX)); rest = do_chksum(&md5_wrap,BIF_P,BIF_ARG_2,100,(void *) &context,&res, &err); if (err != 0) { @@ -564,7 +564,7 @@ md5_update_2(BIF_ALIST_2) erts_free_aligned_binary_bytes(temp_alloc); BIF_ERROR(BIF_P, BADARG); } - memcpy(&context,bytes,sizeof(MD5_CTX)); + sys_memcpy(&context,bytes,sizeof(MD5_CTX)); erts_free_aligned_binary_bytes(temp_alloc); rest = do_chksum(&md5_wrap,BIF_P,BIF_ARG_2,100,(void *) &context,&res, &err); @@ -599,7 +599,7 @@ md5_final_1(BIF_ALIST_1) goto error; } bin = erts_new_heap_binary(BIF_P, (byte *)NULL, 16, &result); - memcpy(&ctx_copy, context, sizeof(MD5_CTX)); + sys_memcpy(&ctx_copy, context, sizeof(MD5_CTX)); erts_free_aligned_binary_bytes(temp_alloc); MD5Final(result, &ctx_copy); BIF_RET(bin); diff --git a/erts/emulator/beam/erl_bif_ddll.c b/erts/emulator/beam/erl_bif_ddll.c index f673ef3194..579e9b12f4 100644 --- a/erts/emulator/beam/erl_bif_ddll.c +++ b/erts/emulator/beam/erl_bif_ddll.c @@ -1412,7 +1412,7 @@ static Eterm copy_ref(Eterm ref, Eterm *hp) ErtsORefThing *ptr; ASSERT(is_internal_ordinary_ref(ref)); ptr = ordinary_ref_thing_ptr(ref); - memcpy(hp, ptr, sizeof(ErtsORefThing)); + sys_memcpy(hp, ptr, sizeof(ErtsORefThing)); return (make_internal_ref(hp)); } @@ -1454,9 +1454,9 @@ static void set_driver_reloading(DE_Handle *dh, Process *proc, char *path, char dh->procs = p; dh->status = ERL_DE_RELOAD; dh->reload_full_path = erts_alloc(ERTS_ALC_T_DDLL_HANDLE, sys_strlen(path) + 1); - strcpy(dh->reload_full_path,path); + sys_strcpy(dh->reload_full_path,path); dh->reload_driver_name = erts_alloc(ERTS_ALC_T_DDLL_HANDLE, sys_strlen(name) + 1); - strcpy(dh->reload_driver_name,name); + sys_strcpy(dh->reload_driver_name,name); dh->reload_flags = flags; } @@ -1501,7 +1501,7 @@ static int do_load_driver_entry(DE_Handle *dh, char *path, char *name) goto error; } - if (strcmp(name, dp->driver_name) != 0) { + if (sys_strcmp(name, dp->driver_name) != 0) { res = ERL_DE_LOAD_ERROR_BAD_NAME; goto error; } @@ -1799,7 +1799,7 @@ static char *pick_list_or_atom(Eterm name_term) goto error; } name = erts_alloc(ERTS_ALC_T_DDLL_TMP_BUF, ap->len + 1); - memcpy(name,ap->name,ap->len); + sys_memcpy(name,ap->name,ap->len); name[ap->len] = '\0'; } else { if (erts_iolist_size(name_term, &name_len)) { @@ -1870,7 +1870,7 @@ static erts_driver_t *lookup_driver(char *name) { erts_driver_t *drv; assert_drv_list_locked(); - for (drv = driver_list; drv != NULL && strcmp(drv->name, name); drv = drv->next) + for (drv = driver_list; drv != NULL && sys_strcmp(drv->name, name); drv = drv->next) ; return drv; } diff --git a/erts/emulator/beam/erl_bif_info.c b/erts/emulator/beam/erl_bif_info.c index 27bbf70c0b..6475f04c56 100644 --- a/erts/emulator/beam/erl_bif_info.c +++ b/erts/emulator/beam/erl_bif_info.c @@ -1754,7 +1754,7 @@ static int check_if_xml(void) { char buf[1]; size_t bufsz = sizeof(buf); - return erts_sys_getenv_raw("VALGRIND_LOG_XML", buf, &bufsz) >= 0; + return erts_sys_explicit_8bit_getenv("VALGRIND_LOG_XML", buf, &bufsz) >= 0; } #else #define check_if_xml() 0 @@ -2378,12 +2378,12 @@ BIF_RETTYPE system_info_1(BIF_ALIST_1) BIF_RET(os_version_tuple); } else if (BIF_ARG_1 == am_version) { - int n = strlen(ERLANG_VERSION); + int n = sys_strlen(ERLANG_VERSION); hp = HAlloc(BIF_P, ((sizeof ERLANG_VERSION)-1) * 2); BIF_RET(buf_to_intlist(&hp, ERLANG_VERSION, n, NIL)); } else if (BIF_ARG_1 == am_machine) { - int n = strlen(EMULATOR); + int n = sys_strlen(EMULATOR); hp = HAlloc(BIF_P, n*2); BIF_RET(buf_to_intlist(&hp, EMULATOR, n, NIL)); } @@ -2409,7 +2409,7 @@ BIF_RETTYPE system_info_1(BIF_ALIST_1) res = erts_bld_cons(hpp, hszp, erts_bld_tuple(hpp, hszp, 2, erts_atom_put((byte *)opc[i].name, - strlen(opc[i].name), + sys_strlen(opc[i].name), ERTS_ATOM_ENC_LATIN1, 1), erts_bld_uint(hpp, hszp, @@ -2805,21 +2805,21 @@ BIF_RETTYPE system_info_1(BIF_ALIST_1) Uint sz; Eterm res = NIL, tup, text; Eterm *hp = HAlloc(BIF_P, 3*(2 + 3) + /* three 2-tuples and three cons */ - 2*(strlen(erts_build_flags_CONFIG_H) + - strlen(erts_build_flags_CFLAGS) + - strlen(erts_build_flags_LDFLAGS))); + 2*(sys_strlen(erts_build_flags_CONFIG_H) + + sys_strlen(erts_build_flags_CFLAGS) + + sys_strlen(erts_build_flags_LDFLAGS))); - sz = strlen(erts_build_flags_CONFIG_H); + sz = sys_strlen(erts_build_flags_CONFIG_H); text = buf_to_intlist(&hp, erts_build_flags_CONFIG_H, sz, NIL); tup = TUPLE2(hp, am_config_h, text); hp += 3; res = CONS(hp, tup, res); hp += 2; - sz = strlen(erts_build_flags_CFLAGS); + sz = sys_strlen(erts_build_flags_CFLAGS); text = buf_to_intlist(&hp, erts_build_flags_CFLAGS, sz, NIL); tup = TUPLE2(hp, am_cflags, text); hp += 3; res = CONS(hp, tup, res); hp += 2; - sz = strlen(erts_build_flags_LDFLAGS); + sz = sys_strlen(erts_build_flags_LDFLAGS); text = buf_to_intlist(&hp, erts_build_flags_LDFLAGS, sz, NIL); tup = TUPLE2(hp, am_ldflags, text); hp += 3; res = CONS(hp, tup, res); hp += 2; @@ -3830,10 +3830,10 @@ BIF_RETTYPE erts_debug_get_internal_state_1(BIF_ALIST_1) BIF_RET(res); } } - else if (ERTS_IS_ATOM_STR("term_to_binary_no_funs", tp[1])) { - Uint dflags = (DFLAG_EXTENDED_REFERENCES | - DFLAG_EXTENDED_PIDS_PORTS | - DFLAG_BIT_BINARIES); + else if (ERTS_IS_ATOM_STR("term_to_binary_tuple_fallbacks", tp[1])) { + Uint dflags = (TERM_TO_BINARY_DFLAGS + & ~DFLAG_EXPORT_PTR_TAG + & ~DFLAG_BIT_BINARIES); BIF_RET(erts_term_to_binary(BIF_P, tp[2], 0, dflags)); } else if (ERTS_IS_ATOM_STR("dist_ctrl", tp[1])) { @@ -3883,7 +3883,7 @@ BIF_RETTYPE erts_debug_get_internal_state_1(BIF_ALIST_1) while (ix >= atom_table_size()) { char tmp[20]; erts_snprintf(tmp, sizeof(tmp), "am%x", atom_table_size()); - erts_atom_put((byte *) tmp, strlen(tmp), ERTS_ATOM_ENC_LATIN1, 1); + erts_atom_put((byte *) tmp, sys_strlen(tmp), ERTS_ATOM_ENC_LATIN1, 1); } return make_atom(ix); } @@ -4334,6 +4334,19 @@ BIF_RETTYPE erts_debug_set_internal_state_2(BIF_ALIST_2) BIF_RET(res); } } + else if (ERTS_IS_ATOM_STR("binary", BIF_ARG_1)) { + Sint64 size; + if (term_to_Sint64(BIF_ARG_2, &size)) { + Binary* refbin = erts_bin_drv_alloc_fnf(size); + if (!refbin) + BIF_RET(am_false); + sys_memset(refbin->orig_bytes, 0, size); + BIF_RET(erts_build_proc_bin(&MSO(BIF_P), + HAlloc(BIF_P, PROC_BIN_SIZE), + refbin)); + } + } + } BIF_ERROR(BIF_P, BADARG); @@ -4436,7 +4449,7 @@ static Eterm lcnt_build_lock_stats_term(Eterm **hpp, Uint *szp, erts_lcnt_lock_s file = stats->file ? stats->file : "undefined"; - af = erts_atom_put((byte *)file, strlen(file), ERTS_ATOM_ENC_LATIN1, 1); + af = erts_atom_put((byte *)file, sys_strlen(file), ERTS_ATOM_ENC_LATIN1, 1); uil = erts_bld_uint( hpp, szp, stats->line); tloc = erts_bld_tuple(hpp, szp, 2, af, uil); @@ -4465,7 +4478,7 @@ static Eterm lcnt_build_lock_stats_term(Eterm **hpp, Uint *szp, erts_lcnt_lock_s static Eterm lcnt_pretty_print_lock_id(erts_lcnt_lock_info_t *info) { Eterm id = info->id; - if((info->flags & ERTS_LOCK_FLAGS_MASK_TYPE) == ERTS_LOCK_TYPE_PROCLOCK) { + if((info->flags & ERTS_LOCK_FLAGS_MASK_TYPE) == ERTS_LOCK_FLAGS_TYPE_PROCLOCK) { /* Use registered names as id's for process locks if available. Thread * progress is delayed since we may be running on a dirty scheduler. */ ErtsThrPrgrDelayHandle delay_handle; @@ -4482,7 +4495,7 @@ static Eterm lcnt_pretty_print_lock_id(erts_lcnt_lock_info_t *info) { } else if(info->flags & ERTS_LOCK_FLAGS_CATEGORY_ALLOCATOR) { if(is_small(id) && !sys_strcmp(info->name, "alcu_allocator")) { const char *name = (const char*)ERTS_ALC_A2AD(signed_val(id)); - id = erts_atom_put((byte*)name, strlen(name), ERTS_ATOM_ENC_LATIN1, 1); + id = erts_atom_put((byte*)name, sys_strlen(name), ERTS_ATOM_ENC_LATIN1, 1); } } @@ -4502,8 +4515,8 @@ static Eterm lcnt_build_lock_term(Eterm **hpp, Uint *szp, lcnt_sample_t *sample, lock_desc = erts_lock_flags_get_type_name(info->flags); - type = erts_atom_put((byte*)lock_desc, strlen(lock_desc), ERTS_ATOM_ENC_LATIN1, 1); - name = erts_atom_put((byte*)info->name, strlen(info->name), ERTS_ATOM_ENC_LATIN1, 1); + type = erts_atom_put((byte*)lock_desc, sys_strlen(lock_desc), ERTS_ATOM_ENC_LATIN1, 1); + name = erts_atom_put((byte*)info->name, sys_strlen(info->name), ERTS_ATOM_ENC_LATIN1, 1); /* Only attempt to resolve ids when actually emitting the term. This ought * to be safe since all immediates are the same size. */ @@ -4539,11 +4552,11 @@ static Eterm lcnt_build_result_term(Eterm **hpp, Uint *szp, erts_lcnt_time_t *du dtns = bld_unstable_uint64(hpp, szp, duration->ns); tdt = erts_bld_tuple(hpp, szp, 2, dts, dtns); - adur = erts_atom_put((byte *)str_duration, strlen(str_duration), ERTS_ATOM_ENC_LATIN1, 1); + adur = erts_atom_put((byte *)str_duration, sys_strlen(str_duration), ERTS_ATOM_ENC_LATIN1, 1); tdur = erts_bld_tuple(hpp, szp, 2, adur, tdt); /* lock tuple */ - aloc = erts_atom_put((byte *)str_locks, strlen(str_locks), ERTS_ATOM_ENC_LATIN1, 1); + aloc = erts_atom_put((byte *)str_locks, sys_strlen(str_locks), ERTS_ATOM_ENC_LATIN1, 1); for(i = 0; i < current_locks->size; i++) { lloc = lcnt_build_lock_term(hpp, szp, ¤t_locks->elements[i], lloc); @@ -4597,7 +4610,7 @@ static Eterm lcnt_build_category_list(Eterm **hpp, Uint *szp, erts_lock_flags_t for(i = 0; lcnt_category_map[i].name != NULL; i++) { if(mask & lcnt_category_map[i].flag) { Eterm category = erts_atom_put((byte*)lcnt_category_map[i].name, - strlen(lcnt_category_map[i].name), + sys_strlen(lcnt_category_map[i].name), ERTS_ATOM_ENC_UTF8, 0); res = erts_bld_cons(hpp, szp, category, res); @@ -4727,14 +4740,14 @@ BIF_RETTYPE erts_debug_lcnt_control_2(BIF_ALIST_2) static void os_info_init(void) { - Eterm type = erts_atom_put((byte *) os_type, strlen(os_type), ERTS_ATOM_ENC_LATIN1, 1); + Eterm type = erts_atom_put((byte *) os_type, sys_strlen(os_type), ERTS_ATOM_ENC_LATIN1, 1); Eterm flav; int major, minor, build; char* buf = erts_alloc(ERTS_ALC_T_TMP, 1024); /* More than enough */ Eterm* hp; os_flavor(buf, 1024); - flav = erts_atom_put((byte *) buf, strlen(buf), ERTS_ATOM_ENC_LATIN1, 1); + flav = erts_atom_put((byte *) buf, sys_strlen(buf), ERTS_ATOM_ENC_LATIN1, 1); erts_free(ERTS_ALC_T_TMP, (void *) buf); hp = erts_alloc(ERTS_ALC_T_LITERAL, (3+4)*sizeof(Eterm)); os_type_tuple = TUPLE2(hp, type, flav); @@ -4758,7 +4771,7 @@ erts_bif_info_init(void) alloc_info_trap = erts_export_put(am_erlang, am_alloc_info, 1); alloc_sizes_trap = erts_export_put(am_erlang, am_alloc_sizes, 1); gather_sched_wall_time_res_trap - = erts_export_put(am_erlang, am_gather_sched_wall_time_result, 1); + = erts_export_put(am_erts_internal, am_gather_sched_wall_time_result, 1); gather_gc_info_res_trap = erts_export_put(am_erlang, am_gather_gc_info_result, 1); gather_io_bytes_trap diff --git a/erts/emulator/beam/erl_bif_os.c b/erts/emulator/beam/erl_bif_os.c index 910325a2f4..ce2b27409b 100644 --- a/erts/emulator/beam/erl_bif_os.c +++ b/erts/emulator/beam/erl_bif_os.c @@ -36,8 +36,7 @@ #include "big.h" #include "dist.h" #include "erl_version.h" - -static int check_env_name(char *name); +#include "erl_osenv.h" /* * Return the pid for the Erlang process in the host OS. @@ -67,148 +66,78 @@ BIF_RETTYPE os_getpid_0(BIF_ALIST_0) BIF_RET(buf_to_intlist(&hp, pid_string, n, NIL)); } -BIF_RETTYPE os_getenv_0(BIF_ALIST_0) +static void os_getenv_foreach(Process *process, Eterm *result, Eterm key, Eterm value) { - GETENV_STATE state; - char *cp; - Eterm* hp; - Eterm ret; - Eterm str; + Eterm kvp_term, *hp; - init_getenv_state(&state); + hp = HAlloc(process, 5); + kvp_term = TUPLE2(hp, key, value); + hp += 3; - ret = NIL; - while ((cp = getenv_string(&state)) != NULL) { - str = erts_convert_native_to_filename(BIF_P,(byte *)cp); - hp = HAlloc(BIF_P, 2); - ret = CONS(hp, str, ret); - } + (*result) = CONS(hp, kvp_term, (*result)); +} - fini_getenv_state(&state); +BIF_RETTYPE os_list_env_vars_0(BIF_ALIST_0) +{ + const erts_osenv_t *global_env; + Eterm result = NIL; + + global_env = erts_sys_rlock_global_osenv(); + erts_osenv_foreach_term(global_env, BIF_P, &result, (void*)&os_getenv_foreach); + erts_sys_runlock_global_osenv(); - return ret; + return result; } -#define STATIC_BUF_SIZE 1024 -BIF_RETTYPE os_getenv_1(BIF_ALIST_1) +BIF_RETTYPE os_get_env_var_1(BIF_ALIST_1) { - Process* p = BIF_P; - Eterm str; - Sint len; - int res; - char *key_str, *val; - char buf[STATIC_BUF_SIZE]; - size_t val_size = sizeof(buf); - - key_str = erts_convert_filename_to_native(BIF_ARG_1,buf,STATIC_BUF_SIZE, - ERTS_ALC_T_TMP,1,0,&len); - - if (!check_env_name(key_str)) { - if (key_str && key_str != &buf[0]) - erts_free(ERTS_ALC_T_TMP, key_str); - BIF_ERROR(p, BADARG); - } + const erts_osenv_t *global_env; + Eterm out_term; + int error; - if (key_str != &buf[0]) - val = &buf[0]; - else { - /* len includes zero byte */ - val_size -= len; - val = &buf[len]; - } - res = erts_sys_getenv(key_str, val, &val_size); - - if (res < 0) { - no_var: - str = am_false; - } else { - if (res > 0) { - val = erts_alloc(ERTS_ALC_T_TMP, val_size); - while (1) { - res = erts_sys_getenv(key_str, val, &val_size); - if (res == 0) - break; - else if (res < 0) - goto no_var; - else - val = erts_realloc(ERTS_ALC_T_TMP, val, val_size); - } - } - str = erts_convert_native_to_filename(p,(byte *)val); - } - if (key_str != &buf[0]) - erts_free(ERTS_ALC_T_TMP, key_str); - if (val < &buf[0] || &buf[sizeof(buf)-1] < val) - erts_free(ERTS_ALC_T_TMP, val); - BIF_RET(str); + global_env = erts_sys_rlock_global_osenv(); + error = erts_osenv_get_term(global_env, BIF_P, BIF_ARG_1, &out_term); + erts_sys_runlock_global_osenv(); + + if (error == 0) { + return am_false; + } else if (error < 0) { + BIF_ERROR(BIF_P, BADARG); + } + + return out_term; } -BIF_RETTYPE os_putenv_2(BIF_ALIST_2) +BIF_RETTYPE os_set_env_var_2(BIF_ALIST_2) { - char def_buf_key[STATIC_BUF_SIZE]; - char def_buf_value[STATIC_BUF_SIZE]; - char *key_buf = NULL, *value_buf = NULL; - - key_buf = erts_convert_filename_to_native(BIF_ARG_1,def_buf_key, - STATIC_BUF_SIZE, - ERTS_ALC_T_TMP,0,0,NULL); - if (!check_env_name(key_buf)) - goto badarg; - - value_buf = erts_convert_filename_to_native(BIF_ARG_2,def_buf_value, - STATIC_BUF_SIZE, - ERTS_ALC_T_TMP,1,0, - NULL); - if (!value_buf) - goto badarg; - - if (erts_sys_putenv(key_buf, value_buf)) { - if (key_buf != def_buf_key) { - erts_free(ERTS_ALC_T_TMP, key_buf); - } - if (value_buf != def_buf_value) { - erts_free(ERTS_ALC_T_TMP, value_buf); - } - BIF_ERROR(BIF_P, BADARG); - } - if (key_buf != def_buf_key) { - erts_free(ERTS_ALC_T_TMP, key_buf); - } - if (value_buf != def_buf_value) { - erts_free(ERTS_ALC_T_TMP, value_buf); + erts_osenv_t *global_env; + int error; + + global_env = erts_sys_rwlock_global_osenv(); + error = erts_osenv_put_term(global_env, BIF_ARG_1, BIF_ARG_2); + erts_sys_rwunlock_global_osenv(); + + if (error < 0) { + BIF_ERROR(BIF_P, BADARG); } - BIF_RET(am_true); -badarg: - if (key_buf && key_buf != def_buf_key) - erts_free(ERTS_ALC_T_TMP, key_buf); - if (value_buf && value_buf != def_buf_value) - erts_free(ERTS_ALC_T_TMP, value_buf); - BIF_ERROR(BIF_P, BADARG); + BIF_RET(am_true); } -BIF_RETTYPE os_unsetenv_1(BIF_ALIST_1) +BIF_RETTYPE os_unset_env_var_1(BIF_ALIST_1) { - char *key_buf; - char buf[STATIC_BUF_SIZE]; + erts_osenv_t *global_env; + int error; - key_buf = erts_convert_filename_to_native(BIF_ARG_1,buf,STATIC_BUF_SIZE, - ERTS_ALC_T_TMP,0,0,NULL); - if (!check_env_name(key_buf)) - goto badarg; + global_env = erts_sys_rwlock_global_osenv(); + error = erts_osenv_unset_term(global_env, BIF_ARG_1); + erts_sys_rwunlock_global_osenv(); - if (erts_sys_unsetenv(key_buf)) - goto badarg; - - if (key_buf != buf) { - erts_free(ERTS_ALC_T_TMP, key_buf); + if (error < 0) { + BIF_ERROR(BIF_P, BADARG); } - BIF_RET(am_true); -badarg: - if (key_buf && key_buf != buf) - erts_free(ERTS_ALC_T_TMP, key_buf); - BIF_ERROR(BIF_P, BADARG); + BIF_RET(am_true); } BIF_RETTYPE os_set_signal_2(BIF_ALIST_2) { @@ -224,27 +153,3 @@ BIF_RETTYPE os_set_signal_2(BIF_ALIST_2) { error: BIF_ERROR(BIF_P, BADARG); } - -static int -check_env_name(char *raw_name) -{ - byte *c = (byte *) raw_name; - int encoding; - - if (!c) - return 0; - - encoding = erts_get_native_filename_encoding(); - - if (erts_raw_env_char_is_7bit_ascii_char('\0', c, encoding)) - return 0; /* Do not allow empty name... */ - - /* Verify no '=' characters in variable name... */ - do { - if (erts_raw_env_char_is_7bit_ascii_char('=', c, encoding)) - return 0; - c = erts_raw_env_next_char(c, encoding); - } while (!erts_raw_env_char_is_7bit_ascii_char('\0', c, encoding)); - - return 1; /* Seems ok... */ -} diff --git a/erts/emulator/beam/erl_bif_port.c b/erts/emulator/beam/erl_bif_port.c index c4a4dd5863..1feb892b93 100644 --- a/erts/emulator/beam/erl_bif_port.c +++ b/erts/emulator/beam/erl_bif_port.c @@ -45,7 +45,7 @@ #include "dtrace-wrapper.h" static Port *open_port(Process* p, Eterm name, Eterm settings, int *err_typep, int *err_nump); -static char* convert_environment(Eterm env); +static int merge_global_environment(erts_osenv_t *env, Eterm key_value_pairs); static char **convert_args(Eterm); static void free_args(char **); @@ -71,7 +71,7 @@ BIF_RETTYPE erts_internal_open_port_2(BIF_ALIST_2) BIF_ERROR(BIF_P, EXC_INTERNAL_ERROR); } else if (err_type == -2) { str = erl_errno_id(err_num); - res = erts_atom_put((byte *) str, strlen(str), ERTS_ATOM_ENC_LATIN1, 1); + res = erts_atom_put((byte *) str, sys_strlen(str), ERTS_ATOM_ENC_LATIN1, 1); } else { res = am_einval; } @@ -639,6 +639,27 @@ BIF_RETTYPE port_get_data_1(BIF_ALIST_1) BIF_RET(res); } +Eterm erts_port_data_read(Port* prt) +{ + Eterm res; + erts_aint_t data; + + data = erts_atomic_read_ddrb(&prt->data); + if (data == (erts_aint_t)NULL) + return am_undefined; /* Port terminated by racing thread */ + + if ((data & 0x3) != 0) { + res = (Eterm) (UWord) data; + ASSERT(is_immed(res)); + } + else { + ErtsPortDataHeap *pdhp = (ErtsPortDataHeap *) data; + res = pdhp->data; + } + return res; +} + + /* * Open a port. Most of the work is not done here but rather in * the file io.c. @@ -651,6 +672,7 @@ BIF_RETTYPE port_get_data_1(BIF_ALIST_1) static Port * open_port(Process* p, Eterm name, Eterm settings, int *err_typep, int *err_nump) { + int merged_environment = 0; Sint i; Eterm option; Uint arity; @@ -672,12 +694,13 @@ open_port(Process* p, Eterm name, Eterm settings, int *err_typep, int *err_nump) opts.read_write = 0; opts.hide_window = 0; opts.wd = NULL; - opts.envir = NULL; opts.exit_status = 0; opts.overlapped_io = 0; opts.spawn_type = ERTS_SPAWN_ANY; opts.argv = NULL; opts.parallelism = erts_port_parallelism; + erts_osenv_init(&opts.envir); + linebuf = 0; *err_nump = 0; @@ -718,11 +741,16 @@ open_port(Process* p, Eterm name, Eterm settings, int *err_typep, int *err_nump) goto badarg; } } else if (option == am_env) { - if (opts.envir) /* ignore previous env option... */ - erts_free(ERTS_ALC_T_OPEN_PORT_ENV, opts.envir); - opts.envir = convert_environment(*tp); - if (!opts.envir) - goto badarg; + if (merged_environment) { + /* Ignore previous env option */ + erts_osenv_clear(&opts.envir); + } + + merged_environment = 1; + + if (merge_global_environment(&opts.envir, *tp)) { + goto badarg; + } } else if (option == am_args) { char **av; char **oav = opts.argv; @@ -807,6 +835,12 @@ open_port(Process* p, Eterm name, Eterm settings, int *err_typep, int *err_nump) if((linebuf && opts.packet_bytes) || (opts.redir_stderr && !opts.use_stdio)) { goto badarg; +} + + /* If we lacked an env option, fill in the global environment without + * changes. */ + if (!merged_environment) { + merge_global_environment(&opts.envir, NIL); } /* @@ -956,8 +990,7 @@ open_port(Process* p, Eterm name, Eterm settings, int *err_typep, int *err_nump) erts_atomic32_read_bor_relb(&port->state, sflgs); do_return: - if (opts.envir) - erts_free(ERTS_ALC_T_OPEN_PORT_ENV, opts.envir); + erts_osenv_clear(&opts.envir); if (name_buf) erts_free(ERTS_ALC_T_TMP, (void *) name_buf); if (opts.argv) { @@ -977,6 +1010,45 @@ open_port(Process* p, Eterm name, Eterm settings, int *err_typep, int *err_nump) goto do_return; } +/* Merges the the global environment and the given {Key, Value} list into env, + * unsetting all keys whose value is either 'false' or NIL. The behavior on + * NIL is undocumented and perhaps surprising, but the previous implementation + * worked in this manner. */ +static int merge_global_environment(erts_osenv_t *env, Eterm key_value_pairs) { + const erts_osenv_t *global_env = erts_sys_rlock_global_osenv(); + erts_osenv_merge(env, global_env, 0); + erts_sys_runlock_global_osenv(); + + while (is_list(key_value_pairs)) { + Eterm *cell, *tuple; + + cell = list_val(key_value_pairs); + + if(!is_tuple_arity(CAR(cell), 2)) { + return -1; + } + + tuple = tuple_val(CAR(cell)); + key_value_pairs = CDR(cell); + + if(is_nil(tuple[2]) || tuple[2] == am_false) { + if(erts_osenv_unset_term(env, tuple[1]) < 0) { + return -1; + } + } else { + if(erts_osenv_put_term(env, tuple[1], tuple[2]) < 0) { + return -1; + } + } + } + + if(!is_nil(key_value_pairs)) { + return -1; + } + + return 0; +} + /* Arguments can be given i unicode and as raw binaries, convert filename is used to convert */ static char **convert_args(Eterm l) { @@ -1024,130 +1096,6 @@ static void free_args(char **av) erts_free(ERTS_ALC_T_TMP, av); } -#ifdef DEBUG -#define ERTS_CONV_ENV_BUF_EXTRA 2 -#else -#define ERTS_CONV_ENV_BUF_EXTRA 1024 -#endif - -static char* convert_environment(Eterm env) -{ - /* - * Returns environment buffer in memory allocated - * as ERTS_ALC_T_OPEN_PORT_ENV. Caller *needs* - * to deallocate... - */ - - Sint size, alloc_size; - byte* bytes; - int encoding = erts_get_native_filename_encoding(); - - alloc_size = ERTS_CONV_ENV_BUF_EXTRA; - bytes = erts_alloc(ERTS_ALC_T_OPEN_PORT_ENV, - alloc_size); - size = 0; - - /* ERTS_CONV_ENV_BUF_EXTRA >= for end delimiter... */ - ERTS_CT_ASSERT(ERTS_CONV_ENV_BUF_EXTRA >= 2); - - while (is_list(env)) { - Sint var_sz, val_sz, need; - byte *str, *limit; - Eterm tmp, *tp, *consp; - - consp = list_val(env); - tmp = CAR(consp); - if (is_not_tuple_arity(tmp, 2)) - goto error; - - tp = tuple_val(tmp); - - /* Check encoding of env variable... */ - if (is_not_list(tp[1])) - goto error; - var_sz = erts_native_filename_need(tp[1], encoding); - if (var_sz <= 0) - goto error; - /* Check encoding of value... */ - if (tp[2] == am_false || is_nil(tp[2])) - val_sz = 0; - else if (is_not_list(tp[2])) - goto error; - else { - val_sz = erts_native_filename_need(tp[2], encoding); - if (val_sz < 0) - goto error; - } - - /* Ensure enough memory... */ - need = size; - need += var_sz + val_sz; - /* '=' and '\0' */ - need += 2 * erts_raw_env_7bit_ascii_char_need(encoding); - if (need > alloc_size) { - alloc_size = (need - alloc_size) + alloc_size; - alloc_size += ERTS_CONV_ENV_BUF_EXTRA; - bytes = erts_realloc(ERTS_ALC_T_OPEN_PORT_ENV, - bytes, alloc_size); - } - - /* Write environment variable name... */ - str = bytes + size; - erts_native_filename_put(tp[1], encoding, str); - /* empty variable name is not allowed... */ - if (erts_raw_env_char_is_7bit_ascii_char('\0', str, encoding)) - goto error; - - /* - * Drop null characters at the end and verify that we do - * not have any '=' characters in the name... - */ - limit = str + var_sz; - while (str < limit) { - if (erts_raw_env_char_is_7bit_ascii_char('\0', str, encoding)) - break; - if (erts_raw_env_char_is_7bit_ascii_char('=', str, encoding)) - goto error; - str = erts_raw_env_next_char(str, encoding); - } - - /* Write the equals sign... */ - str = erts_raw_env_7bit_ascii_char_put('=', str, encoding); - - /* Write the value... */ - if (val_sz > 0) { - limit = str + val_sz; - erts_native_filename_put(tp[2], encoding, str); - while (str < limit) { - if (erts_raw_env_char_is_7bit_ascii_char('\0', str, encoding)) - break; - str = erts_raw_env_next_char(str, encoding); - } - } - - /* Delimit... */ - str = erts_raw_env_7bit_ascii_char_put('\0', str, encoding); - - size = str - bytes; - ASSERT(size <= alloc_size); - - env = CDR(consp); - } - - /* End delimit... */ - (void) erts_raw_env_7bit_ascii_char_put('\0', &bytes[size], encoding); - - if (is_nil(env)) - return (char *) bytes; - -error: - - if (bytes) - erts_free(ERTS_ALC_T_OPEN_PORT_ENV, bytes); - - return (char *) NULL; /* error... */ -} - /* ------------ decode_packet() and friends: */ struct packet_callback_args @@ -1197,7 +1145,7 @@ http_bld_string(struct packet_callback_args* pca, Uint **hpp, Uint *szp, ErlHeapBin* bin = (ErlHeapBin*) *hpp; bin->thing_word = header_heap_bin(len); bin->size = len; - memcpy(bin->data, str, len); + sys_memcpy(bin->data, str, len); *hpp += size; } } @@ -1375,9 +1323,9 @@ int ssl_tls_erl(void* arg, unsigned type, unsigned major, unsigned minor, Eterm bin = new_binary(pca->p, NULL, plen+len); byte* bin_ptr = binary_bytes(bin); - memcpy(bin_ptr+plen, buf, len); + sys_memcpy(bin_ptr+plen, buf, len); if (plen) { - memcpy(bin_ptr, prefix, plen); + sys_memcpy(bin_ptr, prefix, plen); } /* {ssl_tls,NIL,ContentType,{Major,Minor},Bin} */ diff --git a/erts/emulator/beam/erl_bif_re.c b/erts/emulator/beam/erl_bif_re.c index bc819505e7..4d769c2d46 100644 --- a/erts/emulator/beam/erl_bif_re.c +++ b/erts/emulator/beam/erl_bif_re.c @@ -514,7 +514,7 @@ re_version_0(BIF_ALIST_0) Eterm ret; size_t version_size = 0; byte *version = (byte *) erts_pcre_version(); - version_size = strlen((const char *) version); + version_size = sys_strlen((const char *) version); ret = new_binary(BIF_P, version, version_size); BIF_RET(ret); } @@ -998,7 +998,7 @@ build_capture(Eterm capture_spec[CAPSPEC_SIZE], const pcre *code) has_dupnames = ((options & PCRE_DUPNAMES) != 0); for(i=0;i<top;++i) { - if (last == NULL || !has_dupnames || strcmp((char *) last+2,(char *) nametable+2)) { + if (last == NULL || !has_dupnames || sys_strcmp((char *) last+2,(char *) nametable+2)) { ASSERT(ri->num_spec >= 0); ++(ri->num_spec); if(ri->num_spec > sallocated) { @@ -1048,7 +1048,7 @@ build_capture(Eterm capture_spec[CAPSPEC_SIZE], const pcre *code) (tmpbsiz = ap->len + 1)); } } - memcpy(tmpb,ap->name,ap->len); + sys_memcpy(tmpb,ap->name,ap->len); tmpb[ap->len] = '\0'; } else { ErlDrvSizeT slen; @@ -1210,7 +1210,7 @@ re_run(Process *p, Eterm arg1, Eterm arg2, Eterm arg3) erts_pcre_fullinfo(result, NULL, PCRE_INFO_CAPTURECOUNT, &capture_count); ovsize = 3*(capture_count+1); restart.code = erts_alloc(ERTS_ALC_T_RE_SUBJECT, code_size); - memcpy(restart.code, result, code_size); + sys_memcpy(restart.code, result, code_size); erts_pcre_free(result); erts_free(ERTS_ALC_T_RE_TMP_BUF, expr); /*unicode = (pflags & PARSE_FLAG_UNICODE) ? 1 : 0;*/ @@ -1252,7 +1252,7 @@ re_run(Process *p, Eterm arg1, Eterm arg2, Eterm arg3) BIF_ERROR(p, BADARG); } restart.code = erts_alloc(ERTS_ALC_T_RE_SUBJECT, code_size); - memcpy(restart.code, code_tmp, code_size); + sys_memcpy(restart.code, code_tmp, code_size); erts_free_aligned_binary_bytes(temp_alloc); } @@ -1364,7 +1364,7 @@ handle_iolist: RestartContext *restartp = ERTS_MAGIC_BIN_DATA(mbp); Eterm magic_ref; Eterm *hp; - memcpy(restartp,&restart,sizeof(RestartContext)); + sys_memcpy(restartp,&restart,sizeof(RestartContext)); BUMP_ALL_REDS(p); hp = HAlloc(p, ERTS_MAGIC_REF_THING_SIZE); magic_ref = erts_mk_magic_ref(&hp, &MSO(p), mbp); @@ -1517,7 +1517,7 @@ re_inspect_2(BIF_ALIST_2) last = NULL; name = nametable; for(i=0;i<top;++i) { - if (last == NULL || !has_dupnames || strcmp((char *) last+2, + if (last == NULL || !has_dupnames || sys_strcmp((char *) last+2, (char *) name+2)) { ++num_names; } @@ -1531,9 +1531,9 @@ re_inspect_2(BIF_ALIST_2) name = nametable; j = 0; for(i=0;i<top;++i) { - if (last == NULL || !has_dupnames || strcmp((char *) last+2, + if (last == NULL || !has_dupnames || sys_strcmp((char *) last+2, (char *) name+2)) { - tmp_vec[j++] = new_binary(BIF_P, (byte *) name+2, strlen((char *) name+2)); + tmp_vec[j++] = new_binary(BIF_P, (byte *) name+2, sys_strlen((char *) name+2)); } last = name; name += entrysize; diff --git a/erts/emulator/beam/erl_bif_unique.h b/erts/emulator/beam/erl_bif_unique.h index 9aa631fde9..6af9d4ac8c 100644 --- a/erts/emulator/beam/erl_bif_unique.h +++ b/erts/emulator/beam/erl_bif_unique.h @@ -307,7 +307,7 @@ erts_iref_storage_clean(ErtsIRefStorage *iref) if (iref->is_magic && erts_refc_dectest(&iref->u.mb->intern.refc, 0) == 0) erts_ref_bin_free(iref->u.mb); #ifdef DEBUG - memset((void *) iref, 0xf, sizeof(ErtsIRefStorage)); + sys_memset((void *) iref, 0xf, sizeof(ErtsIRefStorage)); #endif } @@ -342,7 +342,7 @@ erts_iref_storage_make_ref(ErtsIRefStorage *iref, #ifdef DEBUG if (clean_storage) - memset((void *) iref, 0xf, sizeof(ErtsIRefStorage)); + sys_memset((void *) iref, 0xf, sizeof(ErtsIRefStorage)); #endif return make_internal_ref(hp); diff --git a/erts/emulator/beam/erl_binary.h b/erts/emulator/beam/erl_binary.h index 05007e864e..46653a8580 100644 --- a/erts/emulator/beam/erl_binary.h +++ b/erts/emulator/beam/erl_binary.h @@ -363,8 +363,6 @@ erts_free_aligned_binary_bytes(byte* buf) # define CHICKEN_PAD (sizeof(void*) - 1) #endif -/* Caller must initialize 'refc' -*/ ERTS_GLB_INLINE Binary * erts_bin_drv_alloc_fnf(Uint size) { @@ -383,8 +381,6 @@ erts_bin_drv_alloc_fnf(Uint size) return res; } -/* Caller must initialize 'refc' -*/ ERTS_GLB_INLINE Binary * erts_bin_drv_alloc(Uint size) { @@ -401,9 +397,6 @@ erts_bin_drv_alloc(Uint size) return res; } - -/* Caller must initialize 'refc' -*/ ERTS_GLB_INLINE Binary * erts_bin_nrml_alloc(Uint size) { diff --git a/erts/emulator/beam/erl_db.c b/erts/emulator/beam/erl_db.c index 3ba0886464..a76d769283 100644 --- a/erts/emulator/beam/erl_db.c +++ b/erts/emulator/beam/erl_db.c @@ -1753,6 +1753,28 @@ BIF_RETTYPE ets_new_2(BIF_ALIST_2) BIF_RET(ret); } +/* +** Retrieves the tid() of a named ets table. +*/ +BIF_RETTYPE ets_whereis_1(BIF_ALIST_1) +{ + DbTable* tb; + Eterm res; + + if (is_not_atom(BIF_ARG_1)) { + BIF_ERROR(BIF_P, BADARG); + } + + if ((tb = db_get_table(BIF_P, BIF_ARG_1, DB_INFO, LCK_READ)) == NULL) { + BIF_RET(am_undefined); + } + + res = make_tid(BIF_P, tb); + db_unlock(tb, LCK_READ); + + BIF_RET(res); +} + /* ** The lookup BIF */ @@ -3126,7 +3148,8 @@ BIF_RETTYPE ets_info_1(BIF_ALIST_1) static Eterm fields[] = {am_protection, am_keypos, am_type, am_named_table, am_node, am_size, am_name, am_heir, am_owner, am_memory, am_compressed, am_write_concurrency, - am_read_concurrency}; + am_read_concurrency, + am_id}; Eterm results[sizeof(fields)/sizeof(Eterm)]; DbTable* tb; Eterm res; @@ -4016,7 +4039,10 @@ static Eterm table_info(Process* p, DbTable* tb, Eterm What) ret = is_table_named(tb) ? am_true : am_false; } else if (What == am_compressed) { ret = tb->common.compress ? am_true : am_false; + } else if (What == am_id) { + ret = make_tid(p, tb); } + /* * For debugging purposes */ @@ -4235,7 +4261,7 @@ erts_ets_colliding_names(Process* p, Eterm name, Uint cnt) while (index >= atom_table_size()) { char tmp[20]; erts_snprintf(tmp, sizeof(tmp), "am%x", atom_table_size()); - erts_atom_put((byte *) tmp, strlen(tmp), ERTS_ATOM_ENC_LATIN1, 1); + erts_atom_put((byte *) tmp, sys_strlen(tmp), ERTS_ATOM_ENC_LATIN1, 1); } list = CONS(hp, make_atom(index), list); hp += 2; diff --git a/erts/emulator/beam/erl_db_util.c b/erts/emulator/beam/erl_db_util.c index e017b9552b..956662aefa 100644 --- a/erts/emulator/beam/erl_db_util.c +++ b/erts/emulator/beam/erl_db_util.c @@ -91,7 +91,7 @@ do { \ (On).siz *= 2; \ (On).data \ = (((On).def == (On).data) \ - ? memcpy(erts_alloc(ERTS_ALC_T_DB_MC_STK, \ + ? sys_memcpy(erts_alloc(ERTS_ALC_T_DB_MC_STK, \ (On).siz*sizeof(*((On).data))), \ (On).def, \ DMC_DEFAULT_SIZE*sizeof(*((On).data))) \ @@ -1518,7 +1518,7 @@ restart: context.current_match < num_progs; ++context.current_match) { /* This loop is long, too long */ - memset(heap.vars, 0, heap.size * sizeof(*heap.vars)); + sys_memset(heap.vars, 0, heap.size * sizeof(*heap.vars)); t = context.matchexpr[context.current_match]; context.stack_used = 0; structure_checked = 0; @@ -2140,7 +2140,7 @@ restart: case matchEqFloat: if (!is_float(*ep)) FAIL(); - if (memcmp(float_val(*ep) + 1, pc, sizeof(double))) + if (sys_memcmp(float_val(*ep) + 1, pc, sizeof(double))) FAIL(); pc += TermWords(2); ++ep; @@ -2742,8 +2742,8 @@ Eterm db_format_dmc_err_info(Process *p, DMCErrInfo *ei) if (vnum >= 0) erts_snprintf(buff,sizeof(buff)+20,tmp->error_string, vnum); else - strcpy(buff,tmp->error_string); - sl = strlen(buff); + sys_strcpy(buff,tmp->error_string); + sl = sys_strlen(buff); shp = HAlloc(p, sl * 2 + 5); sev = (tmp->severity == dmcWarning) ? am_atom_put("warning",7) : @@ -5044,7 +5044,7 @@ static int match_compact(ErlHeapFragment *expr, DMCErrInfo *err_info) ASSERT(j < x); erts_snprintf(buff+1, sizeof(buff) - 1, "%u", (unsigned) j); /* Yes, writing directly into terms, they ARE off heap */ - *p = erts_atom_put((byte *) buff, strlen(buff), + *p = erts_atom_put((byte *) buff, sys_strlen(buff), ERTS_ATOM_ENC_LATIN1, 1); } ++p; @@ -5505,7 +5505,7 @@ void db_match_dis(Binary *bp) ++t; { double num; - memcpy(&num,t,sizeof(double)); + sys_memcpy(&num,t,sizeof(double)); t += TermWords(2); erts_printf("EqFloat\t%f\n", num); } diff --git a/erts/emulator/beam/erl_drv_thread.c b/erts/emulator/beam/erl_drv_thread.c index 71d4534ef9..4cf42fce57 100644 --- a/erts/emulator/beam/erl_drv_thread.c +++ b/erts/emulator/beam/erl_drv_thread.c @@ -439,7 +439,7 @@ erl_drv_tsd_key_create(char *name, ErlDrvTSDKey *key) name_copy = no_name; else { name_copy = erts_alloc_fnf(ERTS_ALC_T_DRV_TSD, - sizeof(char)*(strlen(name) + 1)); + sizeof(char)*(sys_strlen(name) + 1)); if (!name_copy) { *key = -1; return ENOMEM; @@ -750,7 +750,7 @@ erl_drv_steal_main_thread(char *name, + (name ? sys_strlen(name) + 1 : 0))); if (!dtid) return ENOMEM; - memset(dtid,0,sizeof(ErlDrvTid_)); + sys_memset(dtid,0,sizeof(ErlDrvTid_)); dtid->tid = (void * ) -1; dtid->drv_thr = 1; dtid->func = func; @@ -763,8 +763,8 @@ erl_drv_steal_main_thread(char *name, *tid = NULL; /* Ignore options and name... */ - memcpy(buff,&func,sizeof(void* (*)(void*))); - memcpy(buff + sizeof(void* (*)(void*)),&arg,sizeof(void *)); + sys_memcpy(buff,&func,sizeof(void* (*)(void*))); + sys_memcpy(buff + sizeof(void* (*)(void*)),&arg,sizeof(void *)); write(erts_darwin_main_thread_pipe[1],buff,buff_sz); return 0; } diff --git a/erts/emulator/beam/erl_gc.c b/erts/emulator/beam/erl_gc.c index 97a1ca915f..1c64644efc 100644 --- a/erts/emulator/beam/erl_gc.c +++ b/erts/emulator/beam/erl_gc.c @@ -116,6 +116,7 @@ typedef struct { static Uint setup_rootset(Process*, Eterm*, int, Rootset*); static void cleanup_rootset(Rootset *rootset); static Eterm *full_sweep_heaps(Process *p, + ErlHeapFragment *live_hf_end, int hibernate, Eterm *n_heap, Eterm* n_htop, char *oh, Uint oh_size, @@ -142,7 +143,7 @@ static Eterm* sweep_literal_area(Eterm* n_hp, Eterm* n_htop, static Eterm* sweep_literals_to_old_heap(Eterm* heap_ptr, Eterm* heap_end, Eterm* htop, char* src, Uint src_size); static Eterm* collect_live_heap_frags(Process* p, ErlHeapFragment *live_hf_end, - Eterm* heap, Eterm* htop, Eterm* objv, int nobj); + Eterm* htop); static int adjust_after_fullsweep(Process *p, int need, Eterm *objv, int nobj); static void shrink_new_heap(Process *p, Uint new_sz, Eterm *objv, int nobj); static void grow_new_heap(Process *p, Uint new_sz, Eterm* objv, int nobj); @@ -917,6 +918,7 @@ garbage_collect_hibernate(Process* p, int check_long_gc) htop = heap; htop = full_sweep_heaps(p, + ERTS_INVALID_HFRAG_PTR, 1, heap, htop, @@ -1161,7 +1163,7 @@ erts_garbage_collect_literals(Process* p, Eterm* literals, roots++; - while (g_sz--) { + for ( ; g_sz--; g_ptr++) { Eterm gval = *g_ptr; switch (primary_tag(gval)) { @@ -1170,26 +1172,21 @@ erts_garbage_collect_literals(Process* p, Eterm* literals, val = *ptr; if (IS_MOVED_BOXED(val)) { ASSERT(is_boxed(val)); - *g_ptr++ = val; + *g_ptr = val; } else if (ErtsInArea(ptr, area, area_size)) { - move_boxed(&ptr,val,&old_htop,g_ptr++); - } else { - g_ptr++; + move_boxed(ptr,val,&old_htop,g_ptr); } break; case TAG_PRIMARY_LIST: ptr = list_val(gval); val = *ptr; if (IS_MOVED_CONS(val)) { /* Moved */ - *g_ptr++ = ptr[1]; + *g_ptr = ptr[1]; } else if (ErtsInArea(ptr, area, area_size)) { - move_cons(&ptr,val,&old_htop,g_ptr++); - } else { - g_ptr++; - } + move_cons(ptr,val,&old_htop,g_ptr); + } break; default: - g_ptr++; break; } } @@ -1210,8 +1207,8 @@ erts_garbage_collect_literals(Process* p, Eterm* literals, p->old_htop = old_htop; /* - * Prepare to sweep binaries. Since all MSOs on the new heap - * must be come before MSOs on the old heap, find the end of + * Prepare to sweep off-heap objects. Since all MSOs on the new + * heap must be come before MSOs on the old heap, find the end of * current MSO list and use that as a starting point. */ @@ -1223,25 +1220,50 @@ erts_garbage_collect_literals(Process* p, Eterm* literals, } /* - * Sweep through all binaries in the temporary literal area. + * Sweep through all off-heap objects in the temporary literal area. */ while (oh) { if (IS_MOVED_BOXED(oh->thing_word)) { - Binary* bptr; struct erl_off_heap_header* ptr; - ptr = (struct erl_off_heap_header*) boxed_val(oh->thing_word); - ASSERT(thing_subtag(ptr->thing_word) == REFC_BINARY_SUBTAG); - bptr = ((ProcBin*)ptr)->val; - - /* - * This binary has been copied to the heap. + /* + * This off-heap object has been copied to the heap. * We must increment its reference count and * link it into the MSO list for the process. */ - erts_refc_inc(&bptr->intern.refc, 1); + ptr = (struct erl_off_heap_header*) boxed_val(oh->thing_word); + switch (thing_subtag(ptr->thing_word)) { + case REFC_BINARY_SUBTAG: + { + Binary* bptr = ((ProcBin*)ptr)->val; + erts_refc_inc(&bptr->intern.refc, 1); + break; + } + case FUN_SUBTAG: + { + ErlFunEntry* fe = ((ErlFunThing*)ptr)->fe; + erts_refc_inc(&fe->refc, 1); + break; + } + case REF_SUBTAG: + { + ErtsMagicBinary *bptr; + ASSERT(is_magic_ref_thing(ptr)); + bptr = ((ErtsMRefThing *) ptr)->mb; + erts_refc_inc(&bptr->intern.refc, 1); + break; + } + default: + { + ExternalThing *etp; + ASSERT(is_external_header(ptr->thing_word)); + etp = (ExternalThing *) ptr; + erts_refc_inc(&etp->node->refc, 1); + break; + } + } *prev = ptr; prev = &ptr->next; } @@ -1454,25 +1476,29 @@ do_minor(Process *p, ErlHeapFragment *live_hf_end, n_htop = n_heap = (Eterm*) ERTS_HEAP_ALLOC(ERTS_ALC_T_HEAP, sizeof(Eterm)*new_sz); + n = setup_rootset(p, objv, nobj, &rootset); + roots = rootset.roots; + + /* + * All allocations done. Start defile heap with move markers. + * A crash dump due to allocation failure above will see a healthy heap. + */ + if (live_hf_end != ERTS_INVALID_HFRAG_PTR) { /* * Move heap frags that we know are completely live * directly into the new young heap generation. */ - n_htop = collect_live_heap_frags(p, live_hf_end, n_heap, n_htop, - objv, nobj); + n_htop = collect_live_heap_frags(p, live_hf_end, n_htop); } - n = setup_rootset(p, objv, nobj, &rootset); - roots = rootset.roots; - GENSWEEP_NSTACK(p, old_htop, n_htop); while (n--) { Eterm* g_ptr = roots->v; Uint g_sz = roots->sz; roots++; - while (g_sz--) { + for ( ; g_sz--; g_ptr++) { gval = *g_ptr; switch (primary_tag(gval)) { @@ -1482,14 +1508,12 @@ do_minor(Process *p, ErlHeapFragment *live_hf_end, val = *ptr; if (IS_MOVED_BOXED(val)) { ASSERT(is_boxed(val)); - *g_ptr++ = val; + *g_ptr = val; } else if (ErtsInArea(ptr, mature, mature_size)) { - move_boxed(&ptr,val,&old_htop,g_ptr++); + move_boxed(ptr,val,&old_htop,g_ptr); } else if (ErtsInYoungGen(gval, ptr, oh, oh_size)) { - move_boxed(&ptr,val,&n_htop,g_ptr++); - } else { - g_ptr++; - } + move_boxed(ptr,val,&n_htop,g_ptr); + } break; } @@ -1497,19 +1521,15 @@ do_minor(Process *p, ErlHeapFragment *live_hf_end, ptr = list_val(gval); val = *ptr; if (IS_MOVED_CONS(val)) { /* Moved */ - *g_ptr++ = ptr[1]; + *g_ptr = ptr[1]; } else if (ErtsInArea(ptr, mature, mature_size)) { - move_cons(&ptr,val,&old_htop,g_ptr++); + move_cons(ptr,val,&old_htop,g_ptr); } else if (ErtsInYoungGen(gval, ptr, oh, oh_size)) { - move_cons(&ptr,val,&n_htop,g_ptr++); - } else { - g_ptr++; - } + move_cons(ptr,val,&n_htop,g_ptr); + } break; } - default: - g_ptr++; break; } } @@ -1543,9 +1563,9 @@ do_minor(Process *p, ErlHeapFragment *live_hf_end, ASSERT(is_boxed(val)); *n_hp++ = val; } else if (ErtsInArea(ptr, mature, mature_size)) { - move_boxed(&ptr,val,&old_htop,n_hp++); + move_boxed(ptr,val,&old_htop,n_hp++); } else if (ErtsInYoungGen(gval, ptr, oh, oh_size)) { - move_boxed(&ptr,val,&n_htop,n_hp++); + move_boxed(ptr,val,&n_htop,n_hp++); } else { n_hp++; } @@ -1557,9 +1577,9 @@ do_minor(Process *p, ErlHeapFragment *live_hf_end, if (IS_MOVED_CONS(val)) { *n_hp++ = ptr[1]; } else if (ErtsInArea(ptr, mature, mature_size)) { - move_cons(&ptr,val,&old_htop,n_hp++); + move_cons(ptr,val,&old_htop,n_hp++); } else if (ErtsInYoungGen(gval, ptr, oh, oh_size)) { - move_cons(&ptr,val,&n_htop,n_hp++); + move_cons(ptr,val,&n_htop,n_hp++); } else { n_hp++; } @@ -1579,10 +1599,10 @@ do_minor(Process *p, ErlHeapFragment *live_hf_end, *origptr = val; mb->base = binary_bytes(val); } else if (ErtsInArea(ptr, mature, mature_size)) { - move_boxed(&ptr,val,&old_htop,origptr); + move_boxed(ptr,val,&old_htop,origptr); mb->base = binary_bytes(mb->orig); } else if (ErtsInYoungGen(*origptr, ptr, oh, oh_size)) { - move_boxed(&ptr,val,&n_htop,origptr); + move_boxed(ptr,val,&n_htop,origptr); mb->base = binary_bytes(mb->orig); } } @@ -1708,16 +1728,8 @@ major_collection(Process* p, ErlHeapFragment *live_hf_end, n_htop = n_heap = (Eterm *) ERTS_HEAP_ALLOC(ERTS_ALC_T_HEAP, sizeof(Eterm)*new_sz); - if (live_hf_end != ERTS_INVALID_HFRAG_PTR) { - /* - * Move heap frags that we know are completely live - * directly into the heap. - */ - n_htop = collect_live_heap_frags(p, live_hf_end, n_heap, n_htop, - objv, nobj); - } - - n_htop = full_sweep_heaps(p, 0, n_heap, n_htop, oh, oh_size, objv, nobj); + n_htop = full_sweep_heaps(p, live_hf_end, 0, n_heap, n_htop, oh, oh_size, + objv, nobj); /* Move the stack to the end of the heap */ stk_sz = HEAP_END(p) - p->stop; @@ -1764,6 +1776,7 @@ major_collection(Process* p, ErlHeapFragment *live_hf_end, static Eterm * full_sweep_heaps(Process *p, + ErlHeapFragment *live_hf_end, int hibernate, Eterm *n_heap, Eterm* n_htop, char *oh, Uint oh_size, @@ -1780,6 +1793,19 @@ full_sweep_heaps(Process *p, n = setup_rootset(p, objv, nobj, &rootset); + /* + * All allocations done. Start defile heap with move markers. + * A crash dump due to allocation failure above will see a healthy heap. + */ + + if (live_hf_end != ERTS_INVALID_HFRAG_PTR) { + /* + * Move heap frags that we know are completely live + * directly into the heap. + */ + n_htop = collect_live_heap_frags(p, live_hf_end, n_htop); + } + #ifdef HIPE if (hibernate) hipe_empty_nstack(p); @@ -1793,7 +1819,7 @@ full_sweep_heaps(Process *p, Eterm g_sz = roots->sz; roots++; - while (g_sz--) { + for ( ; g_sz--; g_ptr++) { Eterm* ptr; Eterm val; Eterm gval = *g_ptr; @@ -1805,32 +1831,26 @@ full_sweep_heaps(Process *p, val = *ptr; if (IS_MOVED_BOXED(val)) { ASSERT(is_boxed(val)); - *g_ptr++ = val; + *g_ptr = val; } else if (!erts_is_literal(gval, ptr)) { - move_boxed(&ptr,val,&n_htop,g_ptr++); - } else { - g_ptr++; + move_boxed(ptr,val,&n_htop,g_ptr); } - continue; + break; } case TAG_PRIMARY_LIST: { ptr = list_val(gval); val = *ptr; if (IS_MOVED_CONS(val)) { - *g_ptr++ = ptr[1]; + *g_ptr = ptr[1]; } else if (!erts_is_literal(gval, ptr)) { - move_cons(&ptr,val,&n_htop,g_ptr++); - } else { - g_ptr++; + move_cons(ptr,val,&n_htop,g_ptr); } - continue; + break; } - default: { - g_ptr++; - continue; - } + default: + break; } } } @@ -2109,7 +2129,7 @@ sweep(Eterm *n_hp, Eterm *n_htop, ASSERT(is_boxed(val)); *n_hp++ = val; } else if (ERTS_IS_IN_SWEEP_AREA(gval, ptr)) { - move_boxed(&ptr,val,&n_htop,n_hp++); + move_boxed(ptr,val,&n_htop,n_hp++); } else { n_hp++; } @@ -2121,7 +2141,7 @@ sweep(Eterm *n_hp, Eterm *n_htop, if (IS_MOVED_CONS(val)) { *n_hp++ = ptr[1]; } else if (ERTS_IS_IN_SWEEP_AREA(gval, ptr)) { - move_cons(&ptr,val,&n_htop,n_hp++); + move_cons(ptr,val,&n_htop,n_hp++); } else { n_hp++; } @@ -2142,7 +2162,7 @@ sweep(Eterm *n_hp, Eterm *n_htop, *origptr = val; mb->base = binary_bytes(*origptr); } else if (ERTS_IS_IN_SWEEP_AREA(*origptr, ptr)) { - move_boxed(&ptr,val,&n_htop,origptr); + move_boxed(ptr,val,&n_htop,origptr); mb->base = binary_bytes(*origptr); } } @@ -2205,7 +2225,7 @@ sweep_literals_to_old_heap(Eterm* heap_ptr, Eterm* heap_end, Eterm* htop, ASSERT(is_boxed(val)); *heap_ptr++ = val; } else if (ErtsInArea(ptr, src, src_size)) { - move_boxed(&ptr,val,&htop,heap_ptr++); + move_boxed(ptr,val,&htop,heap_ptr++); } else { heap_ptr++; } @@ -2217,7 +2237,7 @@ sweep_literals_to_old_heap(Eterm* heap_ptr, Eterm* heap_end, Eterm* htop, if (IS_MOVED_CONS(val)) { *heap_ptr++ = ptr[1]; } else if (ErtsInArea(ptr, src, src_size)) { - move_cons(&ptr,val,&htop,heap_ptr++); + move_cons(ptr,val,&htop,heap_ptr++); } else { heap_ptr++; } @@ -2238,7 +2258,7 @@ sweep_literals_to_old_heap(Eterm* heap_ptr, Eterm* heap_end, Eterm* htop, *origptr = val; mb->base = binary_bytes(*origptr); } else if (ErtsInArea(ptr, src, src_size)) { - move_boxed(&ptr,val,&htop,origptr); + move_boxed(ptr,val,&htop,origptr); mb->base = binary_bytes(*origptr); } } @@ -2271,11 +2291,11 @@ move_one_area(Eterm* n_htop, char* src, Uint src_size) ASSERT(val != ERTS_HOLE_MARKER); if (is_header(val)) { ASSERT(ptr + header_arity(val) < end); - move_boxed(&ptr, val, &n_htop, &dummy_ref); + ptr = move_boxed(ptr, val, &n_htop, &dummy_ref); } else { /* must be a cons cell */ ASSERT(ptr+1 < end); - move_cons(&ptr, val, &n_htop, &dummy_ref); + move_cons(ptr, val, &n_htop, &dummy_ref); ptr += 2; } } @@ -2288,9 +2308,7 @@ move_one_area(Eterm* n_htop, char* src, Uint src_size) */ static Eterm* -collect_live_heap_frags(Process* p, ErlHeapFragment *live_hf_end, - Eterm* n_hstart, Eterm* n_htop, - Eterm* objv, int nobj) +collect_live_heap_frags(Process* p, ErlHeapFragment *live_hf_end, Eterm* n_htop) { ErlHeapFragment* qb; char* frag_begin; @@ -3256,8 +3274,7 @@ reply_gc_info(void *vgcirp) gcireq_free(vgcirp); } -void erts_sub_binary_to_heap_binary(Eterm **pp, Eterm **hpp, Eterm *orig) { - Eterm *ptr = *pp; +Eterm* erts_sub_binary_to_heap_binary(Eterm *ptr, Eterm **hpp, Eterm *orig) { Eterm *htop = *hpp; Eterm gval; ErlSubBin *sb = (ErlSubBin *)ptr; @@ -3285,7 +3302,7 @@ void erts_sub_binary_to_heap_binary(Eterm **pp, Eterm **hpp, Eterm *orig) { htop += heap_bin_size(sb->size); *hpp = htop; - *pp = ptr; + return ptr; } diff --git a/erts/emulator/beam/erl_gc.h b/erts/emulator/beam/erl_gc.h index 6a529b8443..dec0ab1143 100644 --- a/erts/emulator/beam/erl_gc.h +++ b/erts/emulator/beam/erl_gc.h @@ -33,14 +33,15 @@ #define IS_MOVED_BOXED(x) (!is_header((x))) #define IS_MOVED_CONS(x) (is_non_value((x))) -void erts_sub_binary_to_heap_binary(Eterm **pp, Eterm **hpp, Eterm *orig); +Eterm* erts_sub_binary_to_heap_binary(Eterm *ptr, Eterm **hpp, Eterm *orig); -ERTS_GLB_INLINE void move_cons(Eterm **pp, Eterm car, Eterm **hpp, Eterm *orig); +ERTS_GLB_INLINE void move_cons(Eterm *ERTS_RESTRICT ptr, Eterm car, Eterm **hpp, + Eterm *orig); #if ERTS_GLB_INLINE_INCL_FUNC_DEF -ERTS_GLB_INLINE void move_cons(Eterm **pp, Eterm car, Eterm **hpp, Eterm *orig) +ERTS_GLB_INLINE void move_cons(Eterm *ERTS_RESTRICT ptr, Eterm car, Eterm **hpp, + Eterm *orig) { - Eterm *ptr = *pp; - Eterm *htop = *hpp; + Eterm *ERTS_RESTRICT htop = *hpp; Eterm gval; htop[0] = car; /* copy car */ @@ -53,14 +54,15 @@ ERTS_GLB_INLINE void move_cons(Eterm **pp, Eterm car, Eterm **hpp, Eterm *orig) } #endif -ERTS_GLB_INLINE void move_boxed(Eterm **pp, Eterm hdr, Eterm **hpp, Eterm *orig); +ERTS_GLB_INLINE Eterm* move_boxed(Eterm *ERTS_RESTRICT ptr, Eterm hdr, Eterm **hpp, + Eterm *orig); #if ERTS_GLB_INLINE_INCL_FUNC_DEF -ERTS_GLB_INLINE void move_boxed(Eterm **pp, Eterm hdr, Eterm **hpp, Eterm *orig) +ERTS_GLB_INLINE Eterm* move_boxed(Eterm *ERTS_RESTRICT ptr, Eterm hdr, Eterm **hpp, + Eterm *orig) { Eterm gval; Sint nelts; - Eterm *ptr = *pp; - Eterm *htop = *hpp; + Eterm *ERTS_RESTRICT htop = *hpp; ASSERT(is_header(hdr)); nelts = header_arity(hdr); @@ -71,8 +73,7 @@ ERTS_GLB_INLINE void move_boxed(Eterm **pp, Eterm hdr, Eterm **hpp, Eterm *orig) /* convert sub-binary to heap-binary if applicable */ if (sb->bitsize == 0 && sb->bitoffs == 0 && sb->is_writable == 0 && sb->size <= sizeof(Eterm) * 3) { - erts_sub_binary_to_heap_binary(pp, hpp, orig); - return; + return erts_sub_binary_to_heap_binary(ptr, hpp, orig); } } nelts++; @@ -90,7 +91,7 @@ ERTS_GLB_INLINE void move_boxed(Eterm **pp, Eterm hdr, Eterm **hpp, Eterm *orig) while (nelts--) *htop++ = *ptr++; *hpp = htop; - *pp = ptr; + return ptr; } #endif diff --git a/erts/emulator/beam/erl_goodfit_alloc.c b/erts/emulator/beam/erl_goodfit_alloc.c index 50aa41b4d2..e3ba67f0af 100644 --- a/erts/emulator/beam/erl_goodfit_alloc.c +++ b/erts/emulator/beam/erl_goodfit_alloc.c @@ -170,6 +170,7 @@ static void unlink_free_block (Allctr_t *, Block_t *); static void update_last_aux_mbc (Allctr_t *, Carrier_t *); static Eterm info_options (Allctr_t *, char *, fmtfn_t *, void *, Uint **, Uint *); +static int gfalc_try_set_dyn_param(Allctr_t*, Eterm param, Uint value); static void init_atoms (void); #ifdef ERTS_ALLOC_UTIL_HARD_DEBUG @@ -250,6 +251,8 @@ erts_gfalc_start(GFAllctr_t *gfallctr, if (!erts_alcu_start(allctr, init)) return NULL; + allctr->try_set_dyn_param = gfalc_try_set_dyn_param; + if (allctr->min_block_size != MIN_BLK_SZ) return NULL; @@ -505,7 +508,7 @@ static struct { static void ERTS_INLINE atom_init(Eterm *atom, char *name) { - *atom = am_atom_put(name, strlen(name)); + *atom = am_atom_put(name, sys_strlen(name)); } #define AM_INIT(AM) atom_init(&am.AM, #AM) @@ -584,6 +587,15 @@ info_options(Allctr_t *allctr, return res; } +static int gfalc_try_set_dyn_param(Allctr_t* allctr, Eterm param, Uint value) +{ + if (param == am_sbct) { + /* Cannot change 'sbct' without rearranging buckets */ + return 0; + } + return erts_alcu_try_set_dyn_param(allctr, param, value); +} + /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *\ * NOTE: erts_gfalc_test() is only supposed to be used for testing. * * * diff --git a/erts/emulator/beam/erl_init.c b/erts/emulator/beam/erl_init.c index 6cef9bd0e3..8430a5559b 100644 --- a/erts/emulator/beam/erl_init.c +++ b/erts/emulator/beam/erl_init.c @@ -50,7 +50,7 @@ #define ERTS_WANT_TIMER_WHEEL_API #include "erl_time.h" #include "erl_check_io.h" - +#include "erl_osenv.h" #ifdef HIPE #include "hipe_mode_switch.h" /* for hipe_mode_switch_init() */ #include "hipe_signal.h" /* for hipe_signal_init() */ @@ -155,9 +155,6 @@ erts_atomic32_t erts_writing_erl_crash_dump; erts_tsd_key_t erts_is_crash_dumping_key; int erts_initialized = 0; - -int erts_use_sender_punish; - /* * Configurable parameters. */ @@ -241,7 +238,7 @@ progname(char *fullname) { int i; - i = strlen(fullname); + i = sys_strlen(fullname); while (i >= 0) { if ((fullname[i] != '/') && (fullname[i] != '\\')) i--; @@ -758,8 +755,6 @@ early_init(int *argc, char **argv) /* erts_initialized = 0; - erts_use_sender_punish = 1; - erts_pre_early_init_cpu_topology(&max_reader_groups, &ncpu, &ncpuonln, @@ -803,8 +798,9 @@ early_init(int *argc, char **argv) /* envbufsz = sizeof(envbuf); - /* erts_sys_getenv(_raw)() not initialized yet; need erts_sys_getenv__() */ - if (erts_sys_getenv__("ERL_THREAD_POOL_SIZE", envbuf, &envbufsz) == 0) + /* erts_osenv hasn't been initialized yet, so we need to fall back to + * erts_sys_explicit_host_getenv() */ + if (erts_sys_explicit_host_getenv("ERL_THREAD_POOL_SIZE", envbuf, &envbufsz) == 1) erts_async_max_threads = atoi(envbuf); else erts_async_max_threads = ERTS_DEFAULT_NO_ASYNC_THREADS; @@ -814,7 +810,7 @@ early_init(int *argc, char **argv) /* if (argc && argv) { int i = 1; while (i < *argc) { - if (strcmp(argv[i], "--") == 0) { /* end of emulator options */ + if (sys_strcmp(argv[i], "--") == 0) { /* end of emulator options */ i++; break; } @@ -896,7 +892,7 @@ early_init(int *argc, char **argv) /* else if (argv[i][2] == 'D') { char *arg; char *type = argv[i]+3; - if (strncmp(type, "Pcpu", 4) == 0) { + if (sys_strncmp(type, "Pcpu", 4) == 0) { int ptot, ponln; arg = get_arg(argv[i]+7, argv[i+1], &i); switch (sscanf(arg, "%d:%d", &ptot, &ponln)) { @@ -933,7 +929,7 @@ early_init(int *argc, char **argv) /* VERBOSE(DEBUG_SYSTEM, ("using %d:%d dirty CPU scheduler percentages\n", dirty_cpu_scheds_pctg, dirty_cpu_scheds_onln_pctg)); - } else if (strncmp(type, "cpu", 3) == 0) { + } else if (sys_strncmp(type, "cpu", 3) == 0) { int tot, onln; arg = get_arg(argv[i]+6, argv[i+1], &i); switch (sscanf(arg, "%d:%d", &tot, &onln)) { @@ -984,7 +980,7 @@ early_init(int *argc, char **argv) /* } VERBOSE(DEBUG_SYSTEM, ("using %d:%d dirty CPU scheduler(s)\n", tot, onln)); - } else if (strncmp(type, "io", 2) == 0) { + } else if (sys_strncmp(type, "io", 2) == 0) { arg = get_arg(argv[i]+5, argv[i+1], &i); dirty_io_scheds = atoi(arg); if (dirty_io_scheds < 0 || @@ -1210,20 +1206,20 @@ erl_start(int argc, char **argv) &time_warp_mode); envbufsz = sizeof(envbuf); - if (erts_sys_getenv_raw(ERL_MAX_ETS_TABLES_ENV, envbuf, &envbufsz) == 0) + if (erts_sys_explicit_8bit_getenv(ERL_MAX_ETS_TABLES_ENV, envbuf, &envbufsz) == 1) user_requested_db_max_tabs = atoi(envbuf); else user_requested_db_max_tabs = 0; envbufsz = sizeof(envbuf); - if (erts_sys_getenv_raw("ERL_FULLSWEEP_AFTER", envbuf, &envbufsz) == 0) { + if (erts_sys_explicit_8bit_getenv("ERL_FULLSWEEP_AFTER", envbuf, &envbufsz) == 1) { Uint16 max_gen_gcs = atoi(envbuf); erts_atomic32_set_nob(&erts_max_gen_gcs, (erts_aint32_t) max_gen_gcs); } envbufsz = sizeof(envbuf); - if (erts_sys_getenv_raw("ERL_MAX_PORTS", envbuf, &envbufsz) == 0) { + if (erts_sys_explicit_8bit_getenv("ERL_MAX_PORTS", envbuf, &envbufsz) == 1) { port_tab_sz = atoi(envbuf); port_tab_sz_ignore_files = 1; } @@ -1246,7 +1242,7 @@ erl_start(int argc, char **argv) if (argv[i][0] != '-') { erts_usage(); } - if (strcmp(argv[i], "--") == 0) { /* end of emulator options */ + if (sys_strcmp(argv[i], "--") == 0) { /* end of emulator options */ i++; break; } @@ -1274,12 +1270,12 @@ erl_start(int argc, char **argv) ("using display items %d\n",display_items)); break; case 'p': - if (!strncmp(argv[i],"-pc",3)) { + if (!sys_strncmp(argv[i],"-pc",3)) { int printable_chars = ERL_PRINTABLE_CHARACTERS_LATIN1; arg = get_arg(argv[i]+3, argv[i+1], &i); - if (!strcmp(arg,"unicode")) { + if (!sys_strcmp(arg,"unicode")) { printable_chars = ERL_PRINTABLE_CHARACTERS_UNICODE; - } else if (strcmp(arg,"latin1")) { + } else if (sys_strcmp(arg,"latin1")) { erts_fprintf(stderr, "bad range of printable " "characters: %s\n", arg); erts_usage(); @@ -1291,7 +1287,7 @@ erl_start(int argc, char **argv) erts_usage(); } case 'f': - if (!strncmp(argv[i],"-fn",3)) { + if (!sys_strncmp(argv[i],"-fn",3)) { int warning_type = ERL_FILENAME_WARNING_WARNING; arg = get_arg(argv[i]+3, argv[i+1], &i); switch (*arg) { @@ -1474,9 +1470,9 @@ erl_start(int argc, char **argv) } } else if (has_prefix("maxk", sub_param)) { arg = get_arg(sub_param+4, argv[i+1], &i); - if (strcmp(arg,"true") == 0) { + if (sys_strcmp(arg,"true") == 0) { H_MAX_FLAGS |= MAX_HEAP_SIZE_KILL; - } else if (strcmp(arg,"false") == 0) { + } else if (sys_strcmp(arg,"false") == 0) { H_MAX_FLAGS &= ~MAX_HEAP_SIZE_KILL; } else { erts_fprintf(stderr, "bad max heap kill %s\n", arg); @@ -1485,9 +1481,9 @@ erl_start(int argc, char **argv) VERBOSE(DEBUG_SYSTEM, ("using max heap kill %d\n", H_MAX_FLAGS)); } else if (has_prefix("maxel", sub_param)) { arg = get_arg(sub_param+5, argv[i+1], &i); - if (strcmp(arg,"true") == 0) { + if (sys_strcmp(arg,"true") == 0) { H_MAX_FLAGS |= MAX_HEAP_SIZE_LOG; - } else if (strcmp(arg,"false") == 0) { + } else if (sys_strcmp(arg,"false") == 0) { H_MAX_FLAGS &= ~MAX_HEAP_SIZE_LOG; } else { erts_fprintf(stderr, "bad max heap error logger %s\n", arg); @@ -1605,7 +1601,7 @@ erl_start(int argc, char **argv) case 'P': /* set maximum number of processes */ arg = get_arg(argv[i]+2, argv[i+1], &i); - if (strcmp(arg, "legacy") == 0) + if (sys_strcmp(arg, "legacy") == 0) legacy_proc_tab = 1; else { errno = 0; @@ -1621,7 +1617,7 @@ erl_start(int argc, char **argv) case 'Q': /* set maximum number of ports */ arg = get_arg(argv[i]+2, argv[i+1], &i); - if (strcmp(arg, "legacy") == 0) + if (sys_strcmp(arg, "legacy") == 0) legacy_port_tab = 1; else { errno = 0; @@ -1639,11 +1635,11 @@ erl_start(int argc, char **argv) case 'S' : /* Was handled in early_init() just read past it */ if (argv[i][2] == 'D') { char* type = argv[i]+3; - if (strcmp(type, "Pcpu") == 0) + if (sys_strcmp(type, "Pcpu") == 0) (void) get_arg(argv[i]+7, argv[i+1], &i); - if (strcmp(type, "cpu") == 0) + if (sys_strcmp(type, "cpu") == 0) (void) get_arg(argv[i]+6, argv[i+1], &i); - else if (strcmp(type, "io") == 0) + else if (sys_strcmp(type, "io") == 0) (void) get_arg(argv[i]+5, argv[i+1], &i); } else if (argv[i][2] == 'P') (void) get_arg(argv[i]+3, argv[i+1], &i); @@ -1750,6 +1746,7 @@ erl_start(int argc, char **argv) } else if (has_prefix("ecio", sub_param)) { /* ignore argument, eager check io no longer used */ + arg = get_arg(sub_param+4, argv[i+1], &i); } else if (has_prefix("pp", sub_param)) { arg = get_arg(sub_param+2, argv[i+1], &i); @@ -1764,8 +1761,6 @@ erl_start(int argc, char **argv) erts_usage(); } } - else if (sys_strcmp("nsp", sub_param) == 0) - erts_use_sender_punish = 0; else if (has_prefix("tbt", sub_param)) { arg = get_arg(sub_param+3, argv[i+1], &i); res = erts_init_scheduler_bind_type_string(arg); diff --git a/erts/emulator/beam/erl_instrument.c b/erts/emulator/beam/erl_instrument.c index 634509f880..2f70e7996e 100644 --- a/erts/emulator/beam/erl_instrument.c +++ b/erts/emulator/beam/erl_instrument.c @@ -103,7 +103,7 @@ static struct { static void ERTS_INLINE atom_init(Eterm *atom, const char *name) { - *atom = am_atom_put((char *) name, strlen(name)); + *atom = am_atom_put((char *) name, sys_strlen(name)); } #define AM_INIT(AM) atom_init(&am.AM, #AM) diff --git a/erts/emulator/beam/erl_io_queue.c b/erts/emulator/beam/erl_io_queue.c index 40d69ea6b0..d779d1031a 100644 --- a/erts/emulator/beam/erl_io_queue.c +++ b/erts/emulator/beam/erl_io_queue.c @@ -801,12 +801,11 @@ static Eterm iol2v_make_sub_bin(iol2v_state_t *state, Eterm bin_term, ERTS_GET_REAL_BIN(bin_term, orig_pb_term, byte_offset, bit_offset, bit_size); - (void)bit_offset; - (void)bit_size; + ASSERT(bit_size == 0); sb->thing_word = HEADER_SUB_BIN; + sb->bitoffs = bit_offset; sb->bitsize = 0; - sb->bitoffs = 0; sb->orig = orig_pb_term; sb->is_writable = 0; @@ -817,24 +816,15 @@ static Eterm iol2v_make_sub_bin(iol2v_state_t *state, Eterm bin_term, } static Eterm iol2v_promote_acc(iol2v_state_t *state) { - ProcBin *pb; - - state->acc = erts_bin_realloc(state->acc, state->acc_size); - - pb = (ProcBin*)HAlloc(state->process, PROC_BIN_SIZE); - pb->thing_word = HEADER_PROC_BIN; - pb->size = state->acc_size; - pb->val = state->acc; - pb->bytes = (byte*)(state->acc)->orig_bytes; - pb->flags = 0; - pb->next = MSO(state->process).first; - OH_OVERHEAD(&(MSO(state->process)), pb->size / sizeof(Eterm)); - MSO(state->process).first = (struct erl_off_heap_header*)pb; + Eterm bin; + bin = erts_build_proc_bin(&MSO(state->process), + HAlloc(state->process, PROC_BIN_SIZE), + erts_bin_realloc(state->acc, state->acc_size)); state->acc_size = 0; state->acc = NULL; - return make_binary(pb); + return bin; } /* Destructively enqueues a term to the result list, saving us the hassle of @@ -984,7 +974,7 @@ static int iol2v_append_binary(iol2v_state_t *state, Eterm bin_term) { parent_header = binary_val(parent_binary); binary_size = binary_size(bin_term); - if (bit_offset != 0 || bit_size != 0) { + if (bit_size != 0) { return 0; } else if (binary_size == 0) { state->bytereds_spent += 1; @@ -1026,8 +1016,16 @@ static int iol2v_append_binary(iol2v_state_t *state, Eterm bin_term) { * then just copy it into the accumulator. */ iol2v_expand_acc(state, binary_size); - sys_memcpy(&(state->acc)->orig_bytes[state->acc_size], - binary_data, binary_size); + if (ERTS_LIKELY(bit_offset == 0)) { + sys_memcpy(&(state->acc)->orig_bytes[state->acc_size], + binary_data, binary_size); + } else { + ASSERT(binary_size <= ERTS_UWORD_MAX / 8); + + erts_copy_bits(binary_data, bit_offset, 1, + (byte*)&(state->acc)->orig_bytes[state->acc_size], 0, 1, + binary_size * 8); + } state->acc_size += binary_size; } else { @@ -1038,8 +1036,16 @@ static int iol2v_append_binary(iol2v_state_t *state, Eterm bin_term) { iol2v_expand_acc(state, spill); - sys_memcpy(&(state->acc)->orig_bytes[state->acc_size], - binary_data, spill); + if (ERTS_LIKELY(bit_offset == 0)) { + sys_memcpy(&(state->acc)->orig_bytes[state->acc_size], + binary_data, spill); + } else { + ASSERT(binary_size <= ERTS_UWORD_MAX / 8); + + erts_copy_bits(binary_data, bit_offset, 1, + (byte*)&(state->acc)->orig_bytes[state->acc_size], 0, 1, + spill * 8); + } state->acc_size += spill; @@ -1188,7 +1194,10 @@ BIF_RETTYPE iolist_to_iovec_1(BIF_ALIST_1) { if (is_nil(BIF_ARG_1)) { BIF_RET(NIL); } else if (is_binary(BIF_ARG_1)) { - if (binary_size(BIF_ARG_1) != 0) { + if (binary_bitsize(BIF_ARG_1) != 0) { + ASSERT(!(BIF_P->flags & F_DISABLE_GC)); + BIF_ERROR(BIF_P, BADARG); + } else if (binary_size(BIF_ARG_1) != 0) { Eterm *hp = HAlloc(BIF_P, 2); BIF_RET(CONS(hp, BIF_ARG_1, NIL)); diff --git a/erts/emulator/beam/erl_lock_check.c b/erts/emulator/beam/erl_lock_check.c index 4cdef0200f..773b138d92 100644 --- a/erts/emulator/beam/erl_lock_check.c +++ b/erts/emulator/beam/erl_lock_check.c @@ -86,6 +86,10 @@ static erts_lc_lock_order_t erts_lock_order[] = { { "hipe_mfait_lock", NULL }, #endif { "nodes_monitors", NULL }, + { "meta_name_tab", "address" }, + { "db_tab", "address" }, + { "db_tab_fix", "address" }, + { "db_hash_slot", "address" }, { "resource_monitors", "address" }, { "driver_list", NULL }, { "proc_link", "pid" }, @@ -95,12 +99,8 @@ static erts_lc_lock_order_t erts_lock_order[] = { { "dist_entry_links", "address" }, { "code_write_permission", NULL }, { "purge_state", NULL }, - { "meta_name_tab", "address" }, - { "db_tab", "address" }, { "proc_status", "pid" }, { "proc_trace", "pid" }, - { "db_tab_fix", "address" }, - { "db_hash_slot", "address" }, { "node_table", NULL }, { "dist_table", NULL }, { "sys_tracers", NULL }, @@ -109,7 +109,6 @@ static erts_lc_lock_order_t erts_lock_order[] = { { "fun_tab", NULL }, { "environ", NULL }, { "release_literal_areas", NULL }, - { "efile_drv", "address" }, { "drv_ev_state_grow", NULL, }, { "drv_ev_state", "address" }, { "safe_hash", "address" }, @@ -170,9 +169,6 @@ static erts_lc_lock_order_t erts_lock_order[] = { { "save_ops_lock", NULL }, #endif #endif -#ifdef USE_VM_PROBES - { "efile_drv dtrace mutex", NULL }, -#endif { "mtrace_buf", NULL }, { "os_monotonic_time", NULL }, { "erts_alloc_hard_debug", NULL }, @@ -263,7 +259,7 @@ static ERTS_INLINE void lc_free(void *p) { erts_lc_free_block_t *fb = (erts_lc_free_block_t *) p; #ifdef DEBUG - memset((void *) p, 0xdf, sizeof(erts_lc_free_block_t)); + sys_memset((void *) p, 0xdf, sizeof(erts_lc_free_block_t)); #endif lc_lock(); fb->next = free_blocks; @@ -293,12 +289,12 @@ static void *lc_core_alloc(void) } for (i = 1; i < ERTS_LC_FB_CHUNK_SIZE - 1; i++) { #ifdef DEBUG - memset((void *) &fbs[i], 0xdf, sizeof(erts_lc_free_block_t)); + sys_memset((void *) &fbs[i], 0xdf, sizeof(erts_lc_free_block_t)); #endif fbs[i].next = &fbs[i+1]; } #ifdef DEBUG - memset((void *) &fbs[ERTS_LC_FB_CHUNK_SIZE-1], + sys_memset((void *) &fbs[ERTS_LC_FB_CHUNK_SIZE-1], 0xdf, sizeof(erts_lc_free_block_t)); #endif lc_lock(); @@ -699,7 +695,7 @@ erts_lc_get_lock_order_id(char *name) erts_fprintf(stderr, "Missing lock name\n"); else { for (i = 0; i < ERTS_LOCK_ORDER_SIZE; i++) - if (strcmp(erts_lock_order[i].name, name) == 0) + if (sys_strcmp(erts_lock_order[i].name, name) == 0) return i; erts_fprintf(stderr, "Lock name '%s' missing in lock order " @@ -1326,12 +1322,12 @@ erts_lc_init(void) static erts_lc_free_block_t fbs[ERTS_LC_FB_CHUNK_SIZE]; for (i = 0; i < ERTS_LC_FB_CHUNK_SIZE - 1; i++) { #ifdef DEBUG - memset((void *) &fbs[i], 0xdf, sizeof(erts_lc_free_block_t)); + sys_memset((void *) &fbs[i], 0xdf, sizeof(erts_lc_free_block_t)); #endif fbs[i].next = &fbs[i+1]; } #ifdef DEBUG - memset((void *) &fbs[ERTS_LC_FB_CHUNK_SIZE-1], + sys_memset((void *) &fbs[ERTS_LC_FB_CHUNK_SIZE-1], 0xdf, sizeof(erts_lc_free_block_t)); #endif fbs[ERTS_LC_FB_CHUNK_SIZE-1].next = NULL; diff --git a/erts/emulator/beam/erl_map.c b/erts/emulator/beam/erl_map.c index f0c54e05f7..8047a9567f 100644 --- a/erts/emulator/beam/erl_map.c +++ b/erts/emulator/beam/erl_map.c @@ -91,7 +91,6 @@ static BIF_RETTYPE hashmap_merge(Process *p, Eterm nodeA, Eterm nodeB, int swap_ static Export hashmap_merge_trap_export; static BIF_RETTYPE maps_merge_trap_1(BIF_ALIST_1); static Uint hashmap_subtree_size(Eterm node); -static Eterm hashmap_to_list(Process *p, Eterm map, Sint n); static Eterm hashmap_keys(Process *p, Eterm map); static Eterm hashmap_values(Process *p, Eterm map); static Eterm hashmap_delete(Process *p, Uint32 hx, Eterm key, Eterm node, Eterm *value); @@ -139,80 +138,6 @@ BIF_RETTYPE map_size_1(BIF_ALIST_1) { BIF_ERROR(BIF_P, BADMAP); } -/* maps:to_list/1 */ - -BIF_RETTYPE maps_to_list_1(BIF_ALIST_1) { - if (is_flatmap(BIF_ARG_1)) { - Uint n; - Eterm* hp; - Eterm *ks,*vs, res, tup; - flatmap_t *mp = (flatmap_t*)flatmap_val(BIF_ARG_1); - - ks = flatmap_get_keys(mp); - vs = flatmap_get_values(mp); - n = flatmap_get_size(mp); - hp = HAlloc(BIF_P, (2 + 3) * n); - res = NIL; - - while(n--) { - tup = TUPLE2(hp, ks[n], vs[n]); hp += 3; - res = CONS(hp, tup, res); hp += 2; - } - - BIF_RET(res); - } else if (is_hashmap(BIF_ARG_1)) { - return hashmap_to_list(BIF_P, BIF_ARG_1, -1); - } - - BIF_P->fvalue = BIF_ARG_1; - BIF_ERROR(BIF_P, BADMAP); -} - -/* erts_internal:maps_to_list/2 - * - * This function should be removed once iterators are in place. - * Never document it. - * Never encourage its usage. - * - * A negative value in ARG 2 means the entire map. - */ - -BIF_RETTYPE erts_internal_maps_to_list_2(BIF_ALIST_2) { - Sint m; - if (term_to_Sint(BIF_ARG_2, &m)) { - if (is_flatmap(BIF_ARG_1)) { - Uint n; - Eterm* hp; - Eterm *ks,*vs, res, tup; - flatmap_t *mp = (flatmap_t*)flatmap_val(BIF_ARG_1); - - ks = flatmap_get_keys(mp); - vs = flatmap_get_values(mp); - n = flatmap_get_size(mp); - - if (m >= 0) { - n = m < n ? m : n; - } - - hp = HAlloc(BIF_P, (2 + 3) * n); - res = NIL; - - while(n--) { - tup = TUPLE2(hp, ks[n], vs[n]); hp += 3; - res = CONS(hp, tup, res); hp += 2; - } - - BIF_RET(res); - } else if (is_hashmap(BIF_ARG_1)) { - return hashmap_to_list(BIF_P, BIF_ARG_1, m); - } - BIF_P->fvalue = BIF_ARG_1; - BIF_ERROR(BIF_P, BADMAP); - } - BIF_ERROR(BIF_P, BADARG); -} - - /* maps:find/2 * return value if key *matches* a key in the map */ @@ -1962,45 +1887,31 @@ BIF_RETTYPE maps_values_1(BIF_ALIST_1) { BIF_ERROR(BIF_P, BADMAP); } -static Eterm hashmap_to_list(Process *p, Eterm node, Sint m) { - DECLARE_WSTACK(stack); - Eterm *hp, *kv; - Eterm tup, res = NIL; - Uint n = hashmap_size(node); - - if (m >= 0) { - n = m < n ? m : n; - } - - hp = HAlloc(p, n * (2 + 3)); - hashmap_iterator_init(&stack, node, 0); - while (n--) { - kv = hashmap_iterator_next(&stack); - ASSERT(kv != NULL); - tup = TUPLE2(hp, CAR(kv), CDR(kv)); - hp += 3; - res = CONS(hp, tup, res); - hp += 2; - } - DESTROY_WSTACK(stack); - return res; -} - -void hashmap_iterator_init(ErtsWStack* s, Eterm node, int reverse) { - Eterm hdr = *hashmap_val(node); +static ERTS_INLINE +Uint hashmap_node_size(Eterm hdr, Eterm **nodep) +{ Uint sz; switch(hdr & _HEADER_MAP_SUBTAG_MASK) { case HAMT_SUBTAG_HEAD_ARRAY: sz = 16; + if (nodep) ++*nodep; break; case HAMT_SUBTAG_HEAD_BITMAP: + if (nodep) ++*nodep; case HAMT_SUBTAG_NODE_BITMAP: sz = hashmap_bitcount(MAP_HEADER_VAL(hdr)); + ASSERT(sz < 17); break; default: erts_exit(ERTS_ABORT_EXIT, "bad header"); } + return sz; +} + +void hashmap_iterator_init(ErtsWStack* s, Eterm node, int reverse) { + Eterm hdr = *hashmap_val(node); + Uint sz = hashmap_node_size(hdr, NULL); WSTACK_PUSH3((*s), (UWord)THE_NON_VALUE, /* end marker */ (UWord)(!reverse ? 0 : sz+1), @@ -2024,20 +1935,7 @@ Eterm* hashmap_iterator_next(ErtsWStack* s) { ptr = boxed_val(node); hdr = *ptr; ASSERT(is_header(hdr)); - switch(hdr & _HEADER_MAP_SUBTAG_MASK) { - case HAMT_SUBTAG_HEAD_ARRAY: - ptr++; - sz = 16; - break; - case HAMT_SUBTAG_HEAD_BITMAP: - ptr++; - case HAMT_SUBTAG_NODE_BITMAP: - sz = hashmap_bitcount(MAP_HEADER_VAL(hdr)); - ASSERT(sz < 17); - break; - default: - erts_exit(ERTS_ABORT_EXIT, "bad header"); - } + sz = hashmap_node_size(hdr, &ptr); idx++; @@ -2074,20 +1972,7 @@ Eterm* hashmap_iterator_prev(ErtsWStack* s) { ptr = boxed_val(node); hdr = *ptr; ASSERT(is_header(hdr)); - switch(hdr & _HEADER_MAP_SUBTAG_MASK) { - case HAMT_SUBTAG_HEAD_ARRAY: - ptr++; - sz = 16; - break; - case HAMT_SUBTAG_HEAD_BITMAP: - ptr++; - case HAMT_SUBTAG_NODE_BITMAP: - sz = hashmap_bitcount(MAP_HEADER_VAL(hdr)); - ASSERT(sz < 17); - break; - default: - erts_exit(ERTS_ERROR_EXIT, "bad header"); - } + sz = hashmap_node_size(hdr, &ptr); if (idx > sz) idx = sz; @@ -3061,6 +2946,363 @@ static Eterm hashmap_bld_tuple_uint(Uint **hpp, Uint *szp, Uint n, Uint nums[]) } +/** + * In hashmap the Path is a bit pattern that describes + * which slot we should traverse in each hashmap node. + * Since each hashmap node can only be up to 16 elements + * large we use 4 bits per level in the path. + * + * So a Path with value 0x110 will first get the 0:th + * slot in the head node, and then the 1:st slot in the + * resulting node and then finally the 1:st slot in the + * node beneath. If that slot is not a leaf, then the path + * continues down the 0:th slot until it finds a leaf. + * + * Once the leaf has been found, the return value is created + * by traversing the tree using the the stack that was built + * when searching for the first leaf to return. + * + * The index can become a bignum, which complicates the code + * a bit. However it should be very rare that this happens + * even on a 32bit system as you would need a tree of depth + * 7 or more. + * + * If the number of elements remaining in the map is greater + * than how many we want to return, we build a new Path, using + * the stack, that points to the next leaf. + * + * The third argument to this function controls how the data + * is returned. + * + * iterator: The key-value associations are to be used by + * maps:iterator. The return has this format: + * {K1,V1,{K2,V2,none | [Path | Map]}} + * this makes the maps:next function very simple + * and performant. + * + * list(): The key-value associations are to be used by + * maps:to_list. The return has this format: + * [Path, Map | [{K1,V1},{K2,V2} | BIF_ARG_3]] + * or if no more associations remain + * [{K1,V1},{K2,V2} | BIF_ARG_3] + */ + +#define PATH_ELEM_SIZE 4 +#define PATH_ELEM_MASK 0xf +#define PATH_ELEM(PATH) ((PATH) & PATH_ELEM_MASK) +#define PATH_ELEMS_PER_DIGIT (sizeof(ErtsDigit) * 8 / PATH_ELEM_SIZE) + +BIF_RETTYPE erts_internal_map_next_3(BIF_ALIST_3) { + + Eterm path, map; + enum { iterator, list } type; + + path = BIF_ARG_1; + map = BIF_ARG_2; + + if (!is_map(map)) + BIF_ERROR(BIF_P, BADARG); + + if (BIF_ARG_3 == am_iterator) { + type = iterator; + } else if (is_nil(BIF_ARG_3) || is_list(BIF_ARG_3)) { + type = list; + } else { + BIF_ERROR(BIF_P, BADARG); + } + + if (is_flatmap(map)) { + Uint n; + Eterm *ks,*vs, res, *hp; + flatmap_t *mp = (flatmap_t*)flatmap_val(map); + + ks = flatmap_get_keys(mp); + vs = flatmap_get_values(mp); + n = flatmap_get_size(mp); + + if (!is_small(BIF_ARG_1) || n < unsigned_val(BIF_ARG_1)) + BIF_ERROR(BIF_P, BADARG); + + if (type == iterator) { + hp = HAlloc(BIF_P, 4 * n); + res = am_none; + + while(n--) { + res = TUPLE3(hp, ks[n], vs[n], res); hp += 4; + } + } else { + hp = HAlloc(BIF_P, (2 + 3) * n); + res = BIF_ARG_3; + + while(n--) { + Eterm tup = TUPLE2(hp, ks[n], vs[n]); hp += 3; + res = CONS(hp, tup, res); hp += 2; + } + } + + BIF_RET(res); + } else { + Uint curr_path; + Uint path_length = 0; + Uint *path_rest = NULL; + int i, elems, orig_elems; + Eterm node = map, res, *path_ptr = NULL, *hp; + + /* A stack WSTACK is used when traversing the hashmap. + * It contains: node, idx, sz, ptr + * + * `node` is not really needed, but it is very nice to + * have when debugging. + * + * `idx` always points to the next un-explored entry in + * a node. If there are no more un-explored entries, + * `idx` is equal to `sz`. + * + * `sz` is the number of elements in the node. + * + * `ptr` is a pointer to where the elements of the node begins. + */ + DECLARE_WSTACK(stack); + + ASSERT(is_hashmap(node)); + +/* How many elements we return in one call depends on the number of reductions + * that the process has left to run. In debug we return fewer elements to test + * the Path implementation better. + * + * Also, when the path is 0 (i.e. for the first call) we limit the number of + * elements to MAP_SMALL_MAP_LIMIT in order to not use a huge amount of heap + * when only the first X associations in the hashmap was needed. + */ +#if defined(DEBUG) +#define FCALLS_ELEMS(BIF_P) ((BIF_P->fcalls / 4) & 0xF) +#else +#define FCALLS_ELEMS(BIF_P) (BIF_P->fcalls / 4) +#endif + + if (MAX(FCALLS_ELEMS(BIF_P), 1) < hashmap_size(map)) + elems = MAX(FCALLS_ELEMS(BIF_P), 1); + else + elems = hashmap_size(map); + +#undef FCALLS_ELEMS + + if (is_small(path)) { + curr_path = unsigned_val(path); + + if (curr_path == 0 && elems > MAP_SMALL_MAP_LIMIT) { + elems = MAP_SMALL_MAP_LIMIT; + } + } else if (is_big(path)) { + Eterm *big = big_val(path); + if (bignum_header_is_neg(*big)) + BIF_ERROR(BIF_P, BADARG); + path_length = BIG_ARITY(big) - 1; + curr_path = BIG_DIGIT(big, 0); + path_rest = BIG_V(big) + 1; + } else { + BIF_ERROR(BIF_P, BADARG); + } + + if (type == iterator) { + /* iterator uses the format {K, V, {K, V, {K, V, [Path | Map]}}}, + * so each element is 4 words large */ + hp = HAlloc(BIF_P, 4 * elems); + res = am_none; + } else { + /* list used the format [Path, Map, {K,V}, {K,V} | BIF_ARG_3], + * so each element is 2+3 words large */ + hp = HAlloc(BIF_P, (2 + 3) * elems); + res = BIF_ARG_3; + } + + orig_elems = elems; + + /* First we look for the leaf to start at using the + path given. While doing so, we push each map node + and the index onto the stack to use later. */ + for (i = 1; ; i++) { + Eterm *ptr = hashmap_val(node), + hdr = *ptr++; + Uint sz; + + sz = hashmap_node_size(hdr, &ptr); + + if (PATH_ELEM(curr_path) >= sz) + goto badarg; + + WSTACK_PUSH4(stack, node, PATH_ELEM(curr_path)+1, sz, (UWord)ptr); + + /* We have found a leaf, return it and the next X elements */ + if (is_list(ptr[PATH_ELEM(curr_path)])) { + Eterm *lst = list_val(ptr[PATH_ELEM(curr_path)]); + if (type == iterator) { + res = TUPLE3(hp, CAR(lst), CDR(lst), res); hp += 4; + /* Note where we should patch the Iterator is needed */ + path_ptr = hp-1; + } else { + Eterm tup = TUPLE2(hp, CAR(lst), CDR(lst)); hp += 3; + res = CONS(hp, tup, res); hp += 2; + } + elems--; + break; + } + + node = ptr[PATH_ELEM(curr_path)]; + + curr_path >>= PATH_ELEM_SIZE; + + if (i == PATH_ELEMS_PER_DIGIT) { + /* Switch to next bignum word if available, + otherwise just follow 0 path */ + i = 0; + if (path_length) { + curr_path = *path_rest; + path_length--; + path_rest++; + } else { + curr_path = 0; + } + } + } + + /* We traverse the hashmap and return at most `elems` elements */ + while(1) { + Eterm *ptr = (Eterm*)WSTACK_POP(stack); + Uint sz = (Uint)WSTACK_POP(stack); + Uint idx = (Uint)WSTACK_POP(stack); + Eterm node = (Eterm)WSTACK_POP(stack); + + while (idx < sz && elems != 0 && is_list(ptr[idx])) { + Eterm *lst = list_val(ptr[idx]); + if (type == iterator) { + res = TUPLE3(hp, CAR(lst), CDR(lst), res); hp += 4; + } else { + Eterm tup = TUPLE2(hp, CAR(lst), CDR(lst)); hp += 3; + res = CONS(hp, tup, res); hp += 2; + } + elems--; + idx++; + } + + if (elems == 0) { + if (idx < sz) { + /* There are more elements in this node to explore */ + WSTACK_PUSH4(stack, node, idx+1, sz, (UWord)ptr); + } else { + /* pop stack to find the next value */ + while (!WSTACK_ISEMPTY(stack)) { + Eterm *ptr = (Eterm*)WSTACK_POP(stack); + Uint sz = (Uint)WSTACK_POP(stack); + Uint idx = (Uint)WSTACK_POP(stack); + Eterm node = (Eterm)WSTACK_POP(stack); + if (idx < sz) { + WSTACK_PUSH4(stack, node, idx+1, sz, (UWord)ptr); + break; + } + } + } + break; + } else { + if (idx < sz) { + Eterm hdr; + /* Push next idx in current node */ + WSTACK_PUSH4(stack, node, idx+1, sz, (UWord)ptr); + + /* Push first idx in child node */ + node = ptr[idx]; + ptr = hashmap_val(ptr[idx]); + hdr = *ptr++; + sz = hashmap_node_size(hdr, &ptr); + WSTACK_PUSH4(stack, node, 0, sz, (UWord)ptr); + } + } + + /* There are no more element in the hashmap */ + if (WSTACK_ISEMPTY(stack)) { + break; + } + + } + + if (!WSTACK_ISEMPTY(stack)) { + Uint depth = WSTACK_COUNT(stack) / 4 + 1; + /* +1 because we already have the first element in curr_path */ + Eterm *path_digits = NULL; + Uint curr_path = 0; + + /* If the path cannot fit in a small, we allocate a bignum */ + if (depth >= PATH_ELEMS_PER_DIGIT) { + /* We need multiple ErtsDigit's to represent the path */ + int big_size = BIG_NEED_FOR_BITS(depth * PATH_ELEM_SIZE); + hp = HAlloc(BIF_P, big_size); + hp[0] = make_pos_bignum_header(big_size - BIG_NEED_SIZE(0)); + path_digits = hp + big_size - 1; + } + + + /* Pop the stack to create the complete path to the next leaf */ + while(!WSTACK_ISEMPTY(stack)) { + Uint idx; + + (void)WSTACK_POP(stack); + (void)WSTACK_POP(stack); + idx = (Uint)WSTACK_POP(stack)-1; + /* idx - 1 because idx in the stack is pointing to + the next element to fetch. */ + (void)WSTACK_POP(stack); + + depth--; + if (depth % PATH_ELEMS_PER_DIGIT == 0) { + /* Switch to next bignum element */ + path_digits[0] = curr_path; + path_digits--; + curr_path = 0; + } + + curr_path <<= PATH_ELEM_SIZE; + curr_path |= idx; + } + + if (path_digits) { + path_digits[0] = curr_path; + path = make_big(hp); + } else { + /* The Uint could be too large for a small */ + path = erts_make_integer(curr_path, BIF_P); + } + + if (type == iterator) { + hp = HAlloc(BIF_P, 2); + *path_ptr = CONS(hp, path, map); hp += 2; + } else { + hp = HAlloc(BIF_P, 4); + res = CONS(hp, map, res); hp += 2; + res = CONS(hp, path, res); hp += 2; + } + } else { + if (type == iterator) { + HRelease(BIF_P, hp + 4 * elems, hp); + } else { + HRelease(BIF_P, hp + (2+3) * elems, hp); + } + } + BIF_P->fcalls -= 4 * (orig_elems - elems); + DESTROY_WSTACK(stack); + BIF_RET(res); + + badarg: + if (type == iterator) { + HRelease(BIF_P, hp + 4 * elems, hp); + } else { + HRelease(BIF_P, hp + (2+3) * elems, hp); + } + BIF_P->fcalls -= 4 * (orig_elems - elems); + DESTROY_WSTACK(stack); + BIF_ERROR(BIF_P, BADARG); + } +} + /* implementation of builtin emulations */ #if !ERTS_AT_LEAST_GCC_VSN__(3, 4, 0) diff --git a/erts/emulator/beam/erl_map.h b/erts/emulator/beam/erl_map.h index c3ccf80b85..718d400e22 100644 --- a/erts/emulator/beam/erl_map.h +++ b/erts/emulator/beam/erl_map.h @@ -64,7 +64,6 @@ typedef struct flatmap_s { #define hashmap_shift_hash(Heap,Hx,Lvl,Key) \ (((++(Lvl)) & 7) ? (Hx) >> 4 : hashmap_make_hash(CONS(Heap, make_small((Lvl)>>3), Key))) - /* erl_term.h stuff */ #define flatmap_get_values(x) (((Eterm *)(x)) + sizeof(flatmap_t)/sizeof(Eterm)) #define flatmap_get_keys(x) (((Eterm *)tuple_val(((flatmap_t *)(x))->keys)) + 1) diff --git a/erts/emulator/beam/erl_message.c b/erts/emulator/beam/erl_message.c index abf194cf94..6f7c71ef98 100644 --- a/erts/emulator/beam/erl_message.c +++ b/erts/emulator/beam/erl_message.c @@ -616,7 +616,7 @@ erts_try_alloc_message_on_heap(Process *pp, } else { in_message_fragment: - if (!((*psp) & ERTS_PSFLG_ON_HEAP_MSGQ)) { + if ((*psp) & ERTS_PSFLG_OFF_HEAP_MSGQ) { mp = erts_alloc_message(sz, hpp); *ohpp = sz == 0 ? NULL : &mp->hfrag.off_heap; } @@ -1079,8 +1079,6 @@ erts_change_message_queue_management(Process *c_p, Eterm new_state) case am_on_heap: c_p->flags |= F_ON_HEAP_MSGQ; c_p->flags &= ~F_OFF_HEAP_MSGQ; - erts_atomic32_read_bor_nob(&c_p->state, - ERTS_PSFLG_ON_HEAP_MSGQ); /* * We are not allowed to clear ERTS_PSFLG_OFF_HEAP_MSGQ * if a off heap change is ongoing. It will be adjusted @@ -1106,8 +1104,6 @@ erts_change_message_queue_management(Process *c_p, Eterm new_state) break; case am_off_heap: c_p->flags &= ~F_ON_HEAP_MSGQ; - erts_atomic32_read_band_nob(&c_p->state, - ~ERTS_PSFLG_ON_HEAP_MSGQ); goto change_to_off_heap; default: res = THE_NON_VALUE; /* badarg */ diff --git a/erts/emulator/beam/erl_message.h b/erts/emulator/beam/erl_message.h index 9c8cf84e43..a14f4f51d8 100644 --- a/erts/emulator/beam/erl_message.h +++ b/erts/emulator/beam/erl_message.h @@ -167,10 +167,9 @@ typedef struct { Sint len; /* queue length */ /* - * The following two fields are used by the recv_mark/1 and + * The following field is used by the recv_mark/1 and * recv_set/1 instructions. */ - BeamInstr* mark; /* address to rec_loop/2 instruction */ ErtsMessage** saved_last; /* saved last pointer */ } ErlMessageQueue; @@ -236,12 +235,17 @@ typedef struct erl_trace_message_queue__ { (p)->msg.len--; \ if (__mp == NULL) \ (p)->msg.last = (p)->msg.save; \ - (p)->msg.mark = 0; \ } while(0) -/* Reset message save point (after receive match) */ -#define JOIN_MESSAGE(p) \ - (p)->msg.save = &(p)->msg.first +/* + * Reset message save point (after receive match). + * Also invalidate the saved position since it may no + * longer be safe to use. + */ +#define JOIN_MESSAGE(p) do { \ + (p)->msg.save = &(p)->msg.first; \ + (p)->msg.saved_last = 0; \ +} while(0) /* Save current message */ #define SAVE_MESSAGE(p) \ diff --git a/erts/emulator/beam/erl_msacc.c b/erts/emulator/beam/erl_msacc.c index d659842b7e..d13d6080e1 100644 --- a/erts/emulator/beam/erl_msacc.c +++ b/erts/emulator/beam/erl_msacc.c @@ -81,7 +81,7 @@ void erts_msacc_init(void) { sizeof(Eterm)*ERTS_MSACC_STATE_COUNT); for (i = 0; i < ERTS_MSACC_STATE_COUNT; i++) { erts_msacc_state_atoms[i] = am_atom_put(erts_msacc_states[i], - strlen(erts_msacc_states[i])); + sys_strlen(erts_msacc_states[i])); } } @@ -191,7 +191,7 @@ Eterm erts_msacc_gather_stats(ErtsMsAcc *msacc, ErtsHeapFactory *factory) { map->keys = key; hp[0] = state_map; hp[1] = msacc->id; - hp[2] = am_atom_put(msacc->type,strlen(msacc->type)); + hp[2] = am_atom_put(msacc->type,sys_strlen(msacc->type)); return make_flatmap(map); } diff --git a/erts/emulator/beam/erl_msacc.h b/erts/emulator/beam/erl_msacc.h index 2588dec903..895b1ae319 100644 --- a/erts/emulator/beam/erl_msacc.h +++ b/erts/emulator/beam/erl_msacc.h @@ -159,12 +159,12 @@ struct erl_msacc_t_ { #ifdef ERTS_ENABLE_MSACC -extern erts_tsd_key_t erts_msacc_key; +extern erts_tsd_key_t ERTS_WRITE_UNLIKELY(erts_msacc_key); #ifdef ERTS_MSACC_ALWAYS_ON #define erts_msacc_enabled 1 #else -extern int erts_msacc_enabled; +extern int ERTS_WRITE_UNLIKELY(erts_msacc_enabled); #endif #define ERTS_MSACC_TSD_GET() erts_tsd_get(erts_msacc_key) diff --git a/erts/emulator/beam/erl_mtrace.c b/erts/emulator/beam/erl_mtrace.c index f2a660f085..2807b443a1 100644 --- a/erts/emulator/beam/erl_mtrace.c +++ b/erts/emulator/beam/erl_mtrace.c @@ -314,7 +314,7 @@ disable_trace(int error, char *reason, int eno) erts_fprintf(stderr, "%s: %s\n", mt_dis, reason); else { eno_str = erl_errno_id(eno); - if (strcmp(eno_str, "unknown") == 0) + if (sys_strcmp(eno_str, "unknown") == 0) erts_fprintf(stderr, "%s: %s: %d\n", mt_dis, reason, eno); else erts_fprintf(stderr, "%s: %s: %s\n", mt_dis, reason, eno_str); @@ -401,9 +401,9 @@ write_trace_header(char *nodename, char *pid, char *hostname) PUT_UI32(tracep, ERTS_MT_MAJOR_VSN); PUT_UI32(tracep, ERTS_MT_MINOR_VSN); - n_len = strlen(nodename); - h_len = strlen(hostname); - p_len = strlen(pid); + n_len = sys_strlen(nodename); + h_len = sys_strlen(hostname); + p_len = sys_strlen(pid); hdr_prolog_len = (2*UI32_SZ + 3*UI16_SZ + 3*UI32_SZ @@ -436,15 +436,15 @@ write_trace_header(char *nodename, char *pid, char *hostname) PUT_UI32(tracep, start_time.usec); PUT_UI8(tracep, (byte) n_len); - memcpy((void *) tracep, (void *) nodename, n_len); + sys_memcpy((void *) tracep, (void *) nodename, n_len); tracep += n_len; PUT_UI8(tracep, (byte) h_len); - memcpy((void *) tracep, (void *) hostname, h_len); + sys_memcpy((void *) tracep, (void *) hostname, h_len); tracep += h_len; PUT_UI8(tracep, (byte) p_len); - memcpy((void *) tracep, (void *) pid, p_len); + sys_memcpy((void *) tracep, (void *) pid, p_len); tracep += p_len; ASSERT(startp + hdr_prolog_len == tracep); @@ -472,7 +472,7 @@ write_trace_header(char *nodename, char *pid, char *hostname) str = ERTS_ALC_A2AD(i); ASSERT(str); - str_len = strlen(str); + str_len = sys_strlen(str); if (str_len >= (1 << 8)) { disable_trace(1, "Excessively large allocator string", 0); return 0; @@ -493,7 +493,7 @@ write_trace_header(char *nodename, char *pid, char *hostname) PUT_UI16(tracep, aflags); PUT_UI16(tracep, (Uint16) i); PUT_UI8( tracep, (byte) str_len); - memcpy((void *) tracep, (void *) str, str_len); + sys_memcpy((void *) tracep, (void *) str, str_len); tracep += str_len; if (erts_allctrs_info[i].alloc_util) { PUT_UI8(tracep, 2); @@ -519,7 +519,7 @@ write_trace_header(char *nodename, char *pid, char *hostname) str = ERTS_ALC_N2TD(i); ASSERT(str); - str_len = strlen(str); + str_len = sys_strlen(str); if (str_len >= (1 << 8)) { disable_trace(1, "Excessively large type string", 0); return 0; @@ -543,7 +543,7 @@ write_trace_header(char *nodename, char *pid, char *hostname) PUT_UI16(tracep, nflags); PUT_UI16(tracep, (Uint16) i); PUT_UI8(tracep, (byte) str_len); - memcpy((void *) tracep, (void *) str, str_len); + sys_memcpy((void *) tracep, (void *) str, str_len); tracep += str_len; PUT_UI16(tracep, no); ASSERT(startp + entry_sz == tracep); diff --git a/erts/emulator/beam/erl_nif.c b/erts/emulator/beam/erl_nif.c index f7f12efe28..c60cc7fecf 100644 --- a/erts/emulator/beam/erl_nif.c +++ b/erts/emulator/beam/erl_nif.c @@ -237,9 +237,11 @@ static void cache_env(ErlNifEnv* env); static void full_flush_env(ErlNifEnv *env); static void flush_env(ErlNifEnv* env); -/* Temporary object header, auto-deallocated when NIF returns - * or when independent environment is cleared. - */ +/* Temporary object header, auto-deallocated when NIF returns or when + * independent environment is cleared. + * + * The payload can be accessed with &tmp_obj_ptr[1] but keep in mind that its + * first element must not require greater alignment than `next`. */ struct enif_tmp_obj_t { struct enif_tmp_obj_t* next; void (*dtor)(struct enif_tmp_obj_t*); @@ -256,6 +258,46 @@ static ERTS_INLINE void free_tmp_objs(ErlNifEnv* env) } } +/* Whether the given environment is bound to a process and will be cleaned up + * when the NIF returns. It's safe to use temp_alloc for objects in + * env->tmp_obj_list when this is true. */ +static ERTS_INLINE int is_proc_bound(ErlNifEnv *env) +{ + return env->mod_nif != NULL; +} + +/* Allocates and attaches an object to the given environment, running its + * destructor when the environment is cleared. To avoid temporary variables the + * address of the allocated object is returned instead of the enif_tmp_obj_t. + * + * The destructor *must* call `erts_free(tmp_obj->allocator, tmp_obj)` to free + * the object. If the destructor needs to refer to the allocated object its + * address will be &tmp_obj[1]. */ +static ERTS_INLINE void *alloc_tmp_obj(ErlNifEnv *env, size_t size, + void (*dtor)(struct enif_tmp_obj_t*)) { + struct enif_tmp_obj_t *tmp_obj; + ErtsAlcType_t allocator; + + allocator = is_proc_bound(env) ? ERTS_ALC_T_TMP : ERTS_ALC_T_NIF; + + tmp_obj = erts_alloc(allocator, sizeof(struct enif_tmp_obj_t) + MAX(1, size)); + + tmp_obj->next = env->tmp_obj_list; + tmp_obj->allocator = allocator; + tmp_obj->dtor = dtor; + + env->tmp_obj_list = tmp_obj; + + return (void*)&tmp_obj[1]; +} + +/* Generic destructor for objects allocated through alloc_tmp_obj that don't + * care about their payload. */ +static void tmp_alloc_dtor(struct enif_tmp_obj_t *tmp_obj) +{ + erts_free(tmp_obj->allocator, tmp_obj); +} + void erts_post_nif(ErlNifEnv* env) { erts_unblock_fpe(env->fpe_was_unmasked); @@ -446,6 +488,7 @@ static void cache_env(ErlNifEnv* env) env->hp_end = env->heap_frag->mem + env->heap_frag->alloc_size; } } + void* enif_priv_data(ErlNifEnv* env) { return env->mod_nif->priv_data; @@ -486,7 +529,7 @@ setup_nif_env(struct enif_msg_environment_t* msg_env, msg_env->env.tmp_obj_list = NULL; msg_env->env.proc = &msg_env->phony_proc; msg_env->env.exception_thrown = 0; - memset(&msg_env->phony_proc, 0, sizeof(Process)); + sys_memset(&msg_env->phony_proc, 0, sizeof(Process)); HEAP_START(&msg_env->phony_proc) = phony_heap; HEAP_TOP(&msg_env->phony_proc) = phony_heap; HEAP_LIMIT(&msg_env->phony_proc) = phony_heap; @@ -544,6 +587,9 @@ void enif_clear_env(ErlNifEnv* env) ASSERT(p == menv->env.proc); ASSERT(p->common.id == ERTS_INVALID_PID); ASSERT(MBUF(p) == menv->env.heap_frag); + + free_tmp_objs(env); + if (MBUF(p) != NULL) { erts_cleanup_offheap(&MSO(p)); clear_offheap(&MSO(p)); @@ -555,12 +601,11 @@ void enif_clear_env(ErlNifEnv* env) menv->env.hp = menv->env.hp_end = HEAP_TOP(p); ASSERT(!is_offheap(&MSO(p))); - free_tmp_objs(env); } #ifdef DEBUG static int enif_send_delay = 0; -#define ERTS_FORCE_ENIF_SEND_DELAY() (enif_send_delay++ % 2 == 0) +#define ERTS_FORCE_ENIF_SEND_DELAY() (enif_send_delay++ % 32 == 0) #else #ifdef ERTS_PROC_LOCK_OWN_IMPL #define ERTS_FORCE_ENIF_SEND_DELAY() 0 @@ -1017,11 +1062,6 @@ int enif_is_number(ErlNifEnv* env, ERL_NIF_TERM term) return is_number(term); } -static ERTS_INLINE int is_proc_bound(ErlNifEnv* env) -{ - return env->mod_nif != NULL; -} - static void aligned_binary_dtor(struct enif_tmp_obj_t* obj) { erts_free_aligned_binary_bytes_extra((byte*)obj, obj->allocator); @@ -1056,22 +1096,14 @@ int enif_inspect_binary(ErlNifEnv* env, Eterm bin_term, ErlNifBinary* bin) u.tmp->dtor = &aligned_binary_dtor; env->tmp_obj_list = u.tmp; } - bin->bin_term = bin_term; bin->size = binary_size(bin_term); bin->ref_bin = NULL; ADD_READONLY_CHECK(env, bin->data, bin->size); return 1; } -static void tmp_alloc_dtor(struct enif_tmp_obj_t* obj) -{ - erts_free(obj->allocator, obj); -} - int enif_inspect_iolist_as_binary(ErlNifEnv* env, Eterm term, ErlNifBinary* bin) { - struct enif_tmp_obj_t* tobj; - ErtsAlcType_t allocator; ErlDrvSizeT sz; if (is_binary(term)) { return enif_inspect_binary(env,term,bin); @@ -1079,7 +1111,6 @@ int enif_inspect_iolist_as_binary(ErlNifEnv* env, Eterm term, ErlNifBinary* bin) if (is_nil(term)) { bin->data = (unsigned char*) &bin->data; /* dummy non-NULL */ bin->size = 0; - bin->bin_term = THE_NON_VALUE; bin->ref_bin = NULL; return 1; } @@ -1087,16 +1118,8 @@ int enif_inspect_iolist_as_binary(ErlNifEnv* env, Eterm term, ErlNifBinary* bin) return 0; } - allocator = is_proc_bound(env) ? ERTS_ALC_T_TMP : ERTS_ALC_T_NIF; - tobj = erts_alloc(allocator, sz + sizeof(struct enif_tmp_obj_t)); - tobj->allocator = allocator; - tobj->next = env->tmp_obj_list; - tobj->dtor = &tmp_alloc_dtor; - env->tmp_obj_list = tobj; - - bin->data = (unsigned char*) &tobj[1]; + bin->data = alloc_tmp_obj(env, sz, &tmp_alloc_dtor); bin->size = sz; - bin->bin_term = THE_NON_VALUE; bin->ref_bin = NULL; erts_iolist_to_buf(term, (char*) bin->data, sz); ADD_READONLY_CHECK(env, bin->data, bin->size); @@ -1114,7 +1137,6 @@ int enif_alloc_binary(size_t size, ErlNifBinary* bin) bin->size = size; bin->data = (unsigned char*) refbin->orig_bytes; - bin->bin_term = THE_NON_VALUE; bin->ref_bin = refbin; return 1; } @@ -1148,12 +1170,10 @@ void enif_release_binary(ErlNifBinary* bin) { if (bin->ref_bin != NULL) { Binary* refbin = bin->ref_bin; - ASSERT(bin->bin_term == THE_NON_VALUE); erts_bin_release(refbin); } #ifdef DEBUG bin->data = NULL; - bin->bin_term = THE_NON_VALUE; bin->ref_bin = NULL; #endif } @@ -1201,11 +1221,12 @@ size_t enif_binary_to_term(ErlNifEnv *dst_env, Sint size; ErtsHeapFactory factory; byte *bp = (byte*) data; + Uint32 flags = 0; - ERTS_CT_ASSERT(ERL_NIF_BIN2TERM_SAFE == ERTS_DIST_EXT_BTT_SAFE); - - if (opts & ~ERL_NIF_BIN2TERM_SAFE) { - return 0; + switch ((Uint32)opts) { + case 0: break; + case ERL_NIF_BIN2TERM_SAFE: flags = ERTS_DIST_EXT_BTT_SAFE; break; + default: return 0; } if ((size = erts_decode_ext_size(bp, data_sz)) < 0) return 0; @@ -1217,7 +1238,7 @@ size_t enif_binary_to_term(ErlNifEnv *dst_env, erts_factory_dummy_init(&factory); } - *term = erts_decode_ext(&factory, &bp, (Uint32)opts); + *term = erts_decode_ext(&factory, &bp, flags); if (is_non_value(*term)) { return 0; @@ -1309,39 +1330,51 @@ int enif_get_string(ErlNifEnv *env, ERL_NIF_TERM list, char* buf, unsigned len, Eterm enif_make_binary(ErlNifEnv* env, ErlNifBinary* bin) { - if (bin->bin_term != THE_NON_VALUE) { - return bin->bin_term; - } - else if (bin->ref_bin != NULL) { - Binary* bptr = bin->ref_bin; - ProcBin* pb; - Eterm bin_term; - - /* !! Copy-paste from new_binary() !! */ - pb = (ProcBin *) alloc_heap(env, PROC_BIN_SIZE); - pb->thing_word = HEADER_PROC_BIN; - pb->size = bptr->orig_size; - pb->next = MSO(env->proc).first; - MSO(env->proc).first = (struct erl_off_heap_header*) pb; - pb->val = bptr; - pb->bytes = (byte*) bptr->orig_bytes; - pb->flags = 0; - - OH_OVERHEAD(&(MSO(env->proc)), pb->size / sizeof(Eterm)); - bin_term = make_binary(pb); - if (erts_refc_read(&bptr->intern.refc, 1) == 1) { - /* Total ownership transfer */ - bin->ref_bin = NULL; - bin->bin_term = bin_term; - } - return bin_term; - } - else { - flush_env(env); - bin->bin_term = new_binary(env->proc, bin->data, bin->size); - cache_env(env); - return bin->bin_term; + Eterm bin_term; + + if (bin->ref_bin != NULL) { + Binary* binary = bin->ref_bin; + + /* If the binary is smaller than the heap binary limit we'll return a + * heap binary to reduce the number of small refc binaries in the + * system. We can't simply release the refc binary right away however; + * the documentation states that the binary should be considered + * read-only from this point on, which implies that it should still be + * readable. + * + * We could keep it alive until we return by adding it to the temporary + * object list, but that requires an off-heap allocation which is + * potentially quite slow, so we create a dummy ProcBin instead and + * rely on the next minor GC to get rid of it. */ + if (bin->size <= ERL_ONHEAP_BIN_LIMIT) { + ErlHeapBin* hb; + + hb = (ErlHeapBin*)alloc_heap(env, heap_bin_size(bin->size)); + hb->thing_word = header_heap_bin(bin->size); + hb->size = bin->size; + + sys_memcpy(hb->data, bin->data, bin->size); + + erts_build_proc_bin(&MSO(env->proc), + alloc_heap(env, PROC_BIN_SIZE), + binary); + + bin_term = make_binary(hb); + } else { + bin_term = erts_build_proc_bin(&MSO(env->proc), + alloc_heap(env, PROC_BIN_SIZE), + binary); + } + + /* Our (possibly shared) ownership has been transferred to the term. */ + bin->ref_bin = NULL; + } else { + flush_env(env); + bin_term = new_binary(env->proc, bin->data, bin->size); + cache_env(env); } + + return bin_term; } Eterm enif_make_sub_binary(ErlNifEnv* env, ERL_NIF_TERM bin_term, @@ -3291,8 +3324,8 @@ typedef struct { Eterm sublist_start; Eterm sublist_end; - UWord offheap_size; - UWord onheap_size; + UWord referenced_size; + UWord copied_size; UWord iovec_len; } iovec_slice_t; @@ -3302,16 +3335,16 @@ static int examine_iovec_term(Eterm list, UWord max_length, iovec_slice_t *resul result->sublist_start = list; result->sublist_length = 0; - result->offheap_size = 0; - result->onheap_size = 0; + result->referenced_size = 0; + result->copied_size = 0; result->iovec_len = 0; lookahead = result->sublist_start; while (is_list(lookahead)) { - Eterm *binary_header, binary; + UWord byte_size; + Eterm binary; Eterm *cell; - UWord size; cell = list_val(lookahead); binary = CAR(cell); @@ -3320,39 +3353,42 @@ static int examine_iovec_term(Eterm list, UWord max_length, iovec_slice_t *resul return 0; } - size = binary_size(binary); - binary_header = binary_val(binary); + byte_size = binary_size(binary); + + if (byte_size > 0) { + int bit_offset, bit_size; + Eterm parent_binary; + UWord byte_offset; + + int requires_copying; - /* If we're a sub-binary we'll need to check our underlying binary to - * determine whether we're on-heap or not. */ - if(thing_subtag(*binary_header) == SUB_BINARY_SUBTAG) { - ErlSubBin *sb = (ErlSubBin*)binary_header; + ERTS_GET_REAL_BIN(binary, parent_binary, byte_offset, + bit_offset, bit_size); - /* Reject bitstrings */ - if((sb->bitoffs + sb->bitsize) > 0) { + (void)byte_offset; + + if (bit_size != 0) { return 0; } - ASSERT(size <= binary_size(sb->orig)); - binary_header = binary_val(sb->orig); - } - - if(thing_subtag(*binary_header) == HEAP_BINARY_SUBTAG) { - ASSERT(size <= ERL_ONHEAP_BIN_LIMIT); + /* If we're unaligned or an on-heap binary we'll need to copy + * ourselves over to a temporary buffer. */ + requires_copying = (bit_offset != 0) || + thing_subtag(*binary_val(parent_binary)) == HEAP_BINARY_SUBTAG; - result->iovec_len += 1; - result->onheap_size += size; - } else { - ASSERT(thing_subtag(*binary_header) == REFC_BINARY_SUBTAG); + if (requires_copying) { + result->copied_size += byte_size; + } else { + result->referenced_size += byte_size; + } - result->iovec_len += 1 + size / MAX_SYSIOVEC_IOVLEN; - result->offheap_size += size; + result->iovec_len += 1 + byte_size / MAX_SYSIOVEC_IOVLEN; } result->sublist_length += 1; lookahead = CDR(cell); - if(result->sublist_length >= max_length) { + if (result->sublist_length >= max_length) { break; } } @@ -3366,7 +3402,9 @@ static int examine_iovec_term(Eterm list, UWord max_length, iovec_slice_t *resul return 1; } -static void inspect_raw_binary_data(Eterm binary, ErlNifBinary *result) { +static void marshal_iovec_binary(Eterm binary, ErlNifBinary *copy_buffer, + UWord *copy_offset, ErlNifBinary *result) { + Eterm *parent_header; Eterm parent_binary; @@ -3377,14 +3415,19 @@ static void inspect_raw_binary_data(Eterm binary, ErlNifBinary *result) { ERTS_GET_REAL_BIN(binary, parent_binary, byte_offset, bit_offset, bit_size); + ASSERT(bit_size == 0); + parent_header = binary_val(parent_binary); result->size = binary_size(binary); - result->bin_term = binary; if (thing_subtag(*parent_header) == REFC_BINARY_SUBTAG) { ProcBin *pb = (ProcBin*)parent_header; + if (pb->flags & (PB_IS_WRITABLE | PB_ACTIVE_WRITER)) { + erts_emasculate_writable_binary(pb); + } + ASSERT(pb->val != NULL); ASSERT(byte_offset < pb->size); ASSERT(&pb->bytes[byte_offset] >= (byte*)(pb->val)->orig_bytes); @@ -3399,24 +3442,48 @@ static void inspect_raw_binary_data(Eterm binary, ErlNifBinary *result) { result->data = &((unsigned char*)&hb->data)[byte_offset]; result->ref_bin = NULL; } + + /* If this isn't an *aligned* refc binary, copy its contents to the buffer + * and reference that instead. */ + + if (result->ref_bin == NULL || bit_offset != 0) { + ASSERT(copy_buffer->ref_bin != NULL && copy_buffer->data != NULL); + ASSERT(result->size <= (copy_buffer->size - *copy_offset)); + + if (bit_offset == 0) { + sys_memcpy(©_buffer->data[*copy_offset], + result->data, result->size); + } else { + erts_copy_bits(result->data, bit_offset, 1, + (byte*)©_buffer->data[*copy_offset], 0, 1, + result->size * 8); + } + + result->data = ©_buffer->data[*copy_offset]; + result->ref_bin = copy_buffer->ref_bin; + + *copy_offset += result->size; + } } static int fill_iovec_with_slice(ErlNifEnv *env, iovec_slice_t *slice, ErlNifIOVec *iovec) { - UWord onheap_offset, iovec_idx; - ErlNifBinary onheap_data; + ErlNifBinary copy_buffer = {0}; + UWord copy_offset, iovec_idx; Eterm sublist_iterator; - /* Set up a common refc binary for all on-heap binaries. */ - if (slice->onheap_size > 0) { - if (!enif_alloc_binary(slice->onheap_size, &onheap_data)) { + /* Set up a common refc binary for all on-heap and unaligned binaries. */ + if (slice->copied_size > 0) { + if (!enif_alloc_binary(slice->copied_size, ©_buffer)) { return 0; } + + ASSERT(copy_buffer.ref_bin != NULL); } sublist_iterator = slice->sublist_start; - onheap_offset = 0; + copy_offset = 0; iovec_idx = 0; while (sublist_iterator != slice->sublist_end) { @@ -3424,27 +3491,13 @@ static int fill_iovec_with_slice(ErlNifEnv *env, Eterm *cell; cell = list_val(sublist_iterator); - inspect_raw_binary_data(CAR(cell), &raw_data); - - /* If this isn't a refc binary, copy its contents to the onheap buffer - * and reference that instead. */ - if (raw_data.ref_bin == NULL) { - ASSERT(onheap_offset < onheap_data.size); - ASSERT(slice->onheap_size > 0); - - sys_memcpy(&onheap_data.data[onheap_offset], - raw_data.data, raw_data.size); - - raw_data.data = &onheap_data.data[onheap_offset]; - raw_data.ref_bin = onheap_data.ref_bin; - } - - ASSERT(raw_data.ref_bin != NULL); + marshal_iovec_binary(CAR(cell), ©_buffer, ©_offset, &raw_data); while (raw_data.size > 0) { UWord chunk_len = MIN(raw_data.size, MAX_SYSIOVEC_IOVLEN); ASSERT(iovec_idx < iovec->iovcnt); + ASSERT(raw_data.ref_bin != NULL); iovec->iov[iovec_idx].iov_base = raw_data.data; iovec->iov[iovec_idx].iov_len = chunk_len; @@ -3469,16 +3522,18 @@ static int fill_iovec_with_slice(ErlNifEnv *env, erts_refc_inc(&refc_binary->intern.refc, 1); } - if (slice->onheap_size > 0) { + if (slice->copied_size > 0) { /* Transfer ownership to the iovec; we've taken references to it in * the above loop. */ - enif_release_binary(&onheap_data); + enif_release_binary(©_buffer); } } else { - if (slice->onheap_size > 0) { - /* Attach the binary to our environment and let the GC take care of - * it after returning. */ - enif_make_binary(env, &onheap_data); + if (slice->copied_size > 0) { + /* Attach the binary to our environment and let the next minor GC + * get rid of it. This is slightly faster than using the tmp object + * list since it avoids off-heap allocations. */ + erts_build_proc_bin(&MSO(env->proc), + alloc_heap(env, PROC_BIN_SIZE), copy_buffer.ref_bin); } } @@ -3504,19 +3559,14 @@ static int create_iovec_from_slice(ErlNifEnv *env, alloc_size = binv_offset; alloc_size += slice->iovec_len * sizeof(Binary*); - /* If we have an environment we'll attach the allocated data to it. The - * GC will take care of releasing it later on. */ + /* When the user passes an environment, we attach the iovec to it so + * the user won't have to bother managing it (similar to + * enif_inspect_binary). It'll disappear once the environment is + * cleaned up. */ if (env != NULL) { - ErlNifBinary gc_bin; - - if (!enif_alloc_binary(alloc_size, &gc_bin)) { - return 0; - } - - alloc_base = (char*)gc_bin.data; - enif_make_binary(env, &gc_bin); + alloc_base = alloc_tmp_obj(env, alloc_size, &tmp_alloc_dtor); } else { - alloc_base = enif_alloc(alloc_size); + alloc_base = erts_alloc(ERTS_ALC_T_NIF, alloc_size); } iovec = (ErlNifIOVec*)alloc_base; @@ -3525,12 +3575,12 @@ static int create_iovec_from_slice(ErlNifEnv *env, iovec->flags = 0; } - iovec->size = slice->offheap_size + slice->onheap_size; + iovec->size = slice->referenced_size + slice->copied_size; iovec->iovcnt = slice->iovec_len; if(!fill_iovec_with_slice(env, slice, iovec)) { if (env == NULL && !(iovec->flags & ERL_NIF_IOVEC_FLAGS_PREALLOC)) { - enif_free(iovec); + erts_free(ERTS_ALC_T_NIF, iovec); } return 0; @@ -3597,6 +3647,55 @@ int enif_ioq_deq(ErlNifIOQueue *q, size_t elems, size_t *size) return 1; } +int enif_ioq_peek_head(ErlNifEnv *env, ErlNifIOQueue *q, size_t *size, ERL_NIF_TERM *bin_term) { + SysIOVec *iov_entry; + Binary *ref_bin; + + if (q->size == 0) { + return 0; + } + + ASSERT(q->b_head != q->b_tail && q->v_head != q->v_tail); + + ref_bin = &q->b_head[0]->nif; + iov_entry = &q->v_head[0]; + + if (size != NULL) { + *size = iov_entry->iov_len; + } + + if (iov_entry->iov_len > ERL_ONHEAP_BIN_LIMIT) { + ProcBin *pb = (ProcBin*)alloc_heap(env, PROC_BIN_SIZE); + + pb->thing_word = HEADER_PROC_BIN; + pb->next = MSO(env->proc).first; + pb->val = ref_bin; + pb->flags = 0; + + ASSERT((byte*)iov_entry->iov_base >= (byte*)ref_bin->orig_bytes); + ASSERT(iov_entry->iov_len <= ref_bin->orig_size); + + pb->bytes = (byte*)iov_entry->iov_base; + pb->size = iov_entry->iov_len; + + MSO(env->proc).first = (struct erl_off_heap_header*) pb; + OH_OVERHEAD(&(MSO(env->proc)), pb->size / sizeof(Eterm)); + + erts_refc_inc(&ref_bin->intern.refc, 2); + *bin_term = make_binary(pb); + } else { + ErlHeapBin* hb = (ErlHeapBin*)alloc_heap(env, heap_bin_size(iov_entry->iov_len)); + + hb->thing_word = header_heap_bin(iov_entry->iov_len); + hb->size = iov_entry->iov_len; + + sys_memcpy(hb->data, iov_entry->iov_base, iov_entry->iov_len); + *bin_term = make_binary(hb); + } + + return 1; +} + SysIOVec *enif_ioq_peek(ErlNifIOQueue *q, int *iovlen) { return erts_ioq_peekq(q, iovlen); @@ -4171,34 +4270,31 @@ static unsigned calc_checksum(unsigned char* ptr, unsigned size); struct readonly_check_t { - struct enif_tmp_obj_t hdr; unsigned char* ptr; unsigned size; unsigned checksum; }; static void add_readonly_check(ErlNifEnv* env, unsigned char* ptr, unsigned sz) { - ErtsAlcType_t allocator = is_proc_bound(env) ? ERTS_ALC_T_TMP : ERTS_ALC_T_NIF; - struct readonly_check_t* obj = erts_alloc(allocator, - sizeof(struct readonly_check_t)); - obj->hdr.allocator = allocator; - obj->hdr.next = env->tmp_obj_list; - env->tmp_obj_list = &obj->hdr; - obj->hdr.dtor = &readonly_check_dtor; + struct readonly_check_t* obj; + + obj = alloc_tmp_obj(env, sizeof(struct readonly_check_t), + &readonly_check_dtor); + obj->ptr = ptr; obj->size = sz; - obj->checksum = calc_checksum(ptr, sz); + obj->checksum = calc_checksum(ptr, sz); } -static void readonly_check_dtor(struct enif_tmp_obj_t* o) +static void readonly_check_dtor(struct enif_tmp_obj_t* tmp_obj) { - struct readonly_check_t* obj = (struct readonly_check_t*) o; - unsigned chksum = calc_checksum(obj->ptr, obj->size); - if (chksum != obj->checksum) { + struct readonly_check_t* ro_check = (struct readonly_check_t*)&tmp_obj[1]; + unsigned chksum = calc_checksum(ro_check->ptr, ro_check->size); + if (chksum != ro_check->checksum) { fprintf(stderr, "\r\nReadonly data written by NIF, checksums differ" - " %x != %x\r\nABORTING\r\n", chksum, obj->checksum); + " %x != %x\r\nABORTING\r\n", chksum, ro_check->checksum); abort(); } - erts_free(obj->hdr.allocator, obj); + erts_free(tmp_obj->allocator, tmp_obj); } static unsigned calc_checksum(unsigned char* ptr, unsigned size) { @@ -4273,7 +4369,7 @@ static void get_string_maybe(ErlNifEnv *env, const ERL_NIF_TERM term, str_bin.size > bufsiz) { *ptr = NULL; } else { - memcpy(buf, (char *) str_bin.data, str_bin.size); + sys_memcpy(buf, (char *) str_bin.data, str_bin.size); buf[str_bin.size] = '\0'; *ptr = buf; } @@ -4290,7 +4386,7 @@ ERL_NIF_TERM erl_nif_user_trace_s1(ErlNifEnv* env, int argc, message_bin.size > MESSAGE_BUFSIZ) { return am_badarg; } - memcpy(messagebuf, (char *) message_bin.data, message_bin.size); + sys_memcpy(messagebuf, (char *) message_bin.data, message_bin.size); messagebuf[message_bin.size] = '\0'; DTRACE1(user_trace_s1, messagebuf); return am_true; diff --git a/erts/emulator/beam/erl_nif.h b/erts/emulator/beam/erl_nif.h index d195721054..a99b4db705 100644 --- a/erts/emulator/beam/erl_nif.h +++ b/erts/emulator/beam/erl_nif.h @@ -50,10 +50,12 @@ ** 2.9: 18.2 enif_getenv ** 2.10: Time API ** 2.11: 19.0 enif_snprintf -** 2.12: 20.0 add enif_queue +** 2.12: 20.0 add enif_select, enif_open_resource_type_x +** 2.13: 20.1 add enif_ioq +** 2.14: 21.0 add enif_ioq_peek_head */ #define ERL_NIF_MAJOR_VERSION 2 -#define ERL_NIF_MINOR_VERSION 12 +#define ERL_NIF_MINOR_VERSION 14 /* * The emulator will refuse to load a nif-lib with a major version @@ -135,8 +137,9 @@ typedef struct unsigned char* data; /* Internals (avert your eyes) */ - ERL_NIF_TERM bin_term; void* ref_bin; + /* for future additions to be ABI compatible (same struct size) */ + void* __spare__[2]; }ErlNifBinary; #if (defined(__WIN32__) || defined(_WIN32) || defined(_WIN32_)) diff --git a/erts/emulator/beam/erl_nif_api_funcs.h b/erts/emulator/beam/erl_nif_api_funcs.h index 9e573307d8..3750fd9b68 100644 --- a/erts/emulator/beam/erl_nif_api_funcs.h +++ b/erts/emulator/beam/erl_nif_api_funcs.h @@ -198,6 +198,7 @@ ERL_NIF_API_FUNC_DECL(SysIOVec*,enif_ioq_peek,(ErlNifIOQueue *q, int *iovlen)); ERL_NIF_API_FUNC_DECL(int,enif_inspect_iovec,(ErlNifEnv *env, size_t max_length, ERL_NIF_TERM iovec_term, ERL_NIF_TERM *tail, ErlNifIOVec **iovec)); ERL_NIF_API_FUNC_DECL(void,enif_free_iovec,(ErlNifIOVec *iov)); +ERL_NIF_API_FUNC_DECL(int,enif_ioq_peek_head,(ErlNifEnv *env, ErlNifIOQueue *q, size_t *size, ERL_NIF_TERM *head)); /* ** ADD NEW ENTRIES HERE (before this comment) !!! @@ -373,6 +374,7 @@ ERL_NIF_API_FUNC_DECL(void,enif_free_iovec,(ErlNifIOVec *iov)); # define enif_ioq_peek ERL_NIF_API_FUNC_MACRO(enif_ioq_peek) # define enif_inspect_iovec ERL_NIF_API_FUNC_MACRO(enif_inspect_iovec) # define enif_free_iovec ERL_NIF_API_FUNC_MACRO(enif_free_iovec) +# define enif_ioq_peek_head ERL_NIF_API_FUNC_MACRO(enif_ioq_peek_head) /* ** ADD NEW ENTRIES HERE (before this comment) diff --git a/erts/emulator/beam/erl_node_tables.c b/erts/emulator/beam/erl_node_tables.c index 0f3dfa797c..e8901a652f 100644 --- a/erts/emulator/beam/erl_node_tables.c +++ b/erts/emulator/beam/erl_node_tables.c @@ -39,9 +39,11 @@ erts_rwmtx_t erts_node_table_rwmtx; DistEntry *erts_hidden_dist_entries; DistEntry *erts_visible_dist_entries; +DistEntry *erts_pending_dist_entries; DistEntry *erts_not_connected_dist_entries; /* including erts_this_dist_entry */ Sint erts_no_of_hidden_dist_entries; Sint erts_no_of_visible_dist_entries; +Sint erts_no_of_pending_dist_entries; Sint erts_no_of_not_connected_dist_entries; /* including erts_this_dist_entry */ DistEntry *erts_this_dist_entry; @@ -101,7 +103,9 @@ void erts_ref_dist_entry(DistEntry *dep) { ASSERT(dep); - de_refc_inc(dep, 1); + if (de_refc_inc_read(dep, 1) == 1) { + de_refc_inc(dep, 2); /* Pending delete */ + } } void @@ -139,9 +143,7 @@ dist_table_cmp(void *dep1, void *dep2) static void* dist_table_alloc(void *dep_tmpl) { -#ifdef DEBUG erts_aint_t refc; -#endif Eterm sysname; Binary *bin; DistEntry *dep; @@ -158,13 +160,8 @@ dist_table_alloc(void *dep_tmpl) dist_entries++; -#ifdef DEBUG - refc = -#else - (void) -#endif - de_refc_dec_read(dep, -1); - ASSERT(refc == -1); + refc = de_refc_dec_read(dep, -1); + ASSERT(refc == -1); (void)refc; dep->prev = NULL; erts_rwmtx_init_opt(&dep->rwmtx, &rwmtx_opt, "dist_entry", sysname, @@ -173,7 +170,7 @@ dist_table_alloc(void *dep_tmpl) dep->cid = NIL; erts_atomic_init_nob(&dep->input_handler, (erts_aint_t) NIL); dep->connection_id = 0; - dep->status = 0; + dep->state = ERTS_DE_STATE_IDLE; dep->flags = 0; dep->version = 0; @@ -202,6 +199,7 @@ dist_table_alloc(void *dep_tmpl) erts_port_task_handle_init(&dep->dist_cmd); dep->send = NULL; dep->cache = NULL; + dep->transcode_ctx = NULL; /* Link in */ @@ -224,6 +222,8 @@ dist_table_free(void *vdep) { DistEntry *dep = (DistEntry *) vdep; + ASSERT(de_refc_read(dep, -1) == -1); + ASSERT(dep->state == ERTS_DE_STATE_IDLE); ASSERT(is_nil(dep->cid)); ASSERT(dep->nlinks == NULL); ASSERT(dep->node_links == NULL); @@ -384,56 +384,92 @@ erts_dhandle_to_dist_entry(Eterm dhandle) } Eterm -erts_make_dhandle(Process *c_p, DistEntry *dep) +erts_build_dhandle(Eterm **hpp, ErlOffHeap* ohp, DistEntry *dep) { - Binary *bin; - Eterm *hp; - - bin = ErtsDistEntry2Bin(dep); + Binary *bin = ErtsDistEntry2Bin(dep); ASSERT(bin); ASSERT(ERTS_MAGIC_BIN_DESTRUCTOR(bin) == erts_dist_entry_destructor); - hp = HAlloc(c_p, ERTS_MAGIC_REF_THING_SIZE); - return erts_mk_magic_ref(&hp, &c_p->off_heap, bin); + return erts_mk_magic_ref(hpp, ohp, bin); +} + +Eterm +erts_make_dhandle(Process *c_p, DistEntry *dep) +{ + Eterm *hp = HAlloc(c_p, ERTS_MAGIC_REF_THING_SIZE); + return erts_build_dhandle(&hp, &c_p->off_heap, dep); } -static void try_delete_dist_entry(void *vbin); +static void start_timer_delete_dist_entry(void *vdep); +static void prepare_try_delete_dist_entry(void *vdep); +static void try_delete_dist_entry(DistEntry*); + +static void schedule_delete_dist_entry(DistEntry* dep) +{ + /* + * Here we need thread progress to wait for other threads, that may have + * done lookup without refc++, to do either refc++ or drop their refs. + * + * Note that timeouts do not guarantee thread progress. + */ + erts_schedule_thr_prgr_later_op(start_timer_delete_dist_entry, + dep, &dep->later_op); +} static void -prepare_try_delete_dist_entry(void *vbin) +start_timer_delete_dist_entry(void *vdep) { - Binary *bin = (Binary *) vbin; - DistEntry *dep = ErtsBin2DistEntry(bin); - Uint size; + if (node_tab_delete_delay == 0) { + prepare_try_delete_dist_entry(vdep); + } + else { + ASSERT(node_tab_delete_delay > 0); + erts_start_timer_callback(node_tab_delete_delay, + prepare_try_delete_dist_entry, + vdep); + } +} + +static void +prepare_try_delete_dist_entry(void *vdep) +{ + DistEntry *dep = vdep; erts_aint_t refc; + /* + * Time has passed since we decremented refc to zero and DistEntry may + * have been revived. Do a fast check without table lock first. + */ refc = de_refc_read(dep, 0); - if (refc > 0) - return; - - size = ERTS_MAGIC_BIN_SIZE(sizeof(DistEntry)); - erts_schedule_thr_prgr_later_cleanup_op(try_delete_dist_entry, - vbin, &dep->later_op, size); + if (refc == 0) { + try_delete_dist_entry(dep); + } + else { + /* + * Someone has done lookup and done refc++ for us. + */ + refc = de_refc_dec_read(dep, 0); + if (refc == 0) + schedule_delete_dist_entry(dep); + } } -static void try_delete_dist_entry(void *vbin) +static void try_delete_dist_entry(DistEntry* dep) { - Binary *bin = (Binary *) vbin; - DistEntry *dep = ErtsBin2DistEntry(bin); erts_aint_t refc; erts_rwmtx_rwlock(&erts_dist_table_rwmtx); /* * Another thread might have looked up this dist entry after * we decided to delete it (refc became zero). If so, the other - * thread incremented refc twice. Once for the new reference - * and once for this thread. + * thread incremented refc one extra step for this thread. * - * If refc reach -1, no one has used the entry since we - * set up the timer. Delete the entry. + * If refc reach -1, no one has done lookup and no one can do lookup + * as we have table lock. Delete the entry. * - * If refc reach 0, the entry is currently not in use - * but has been used since we set up the timer. Set up a - * new timer. + * If refc reach 0, someone raced us and either + * (1) did lookup with own refc++ and already released it again + * (2) did lookup without own refc++ + * Schedule new delete operation. * * If refc > 0, the entry is in use. Keep the entry. */ @@ -443,12 +479,7 @@ static void try_delete_dist_entry(void *vbin) erts_rwmtx_rwunlock(&erts_dist_table_rwmtx); if (refc == 0) { - if (node_tab_delete_delay == 0) - prepare_try_delete_dist_entry(vbin); - else if (node_tab_delete_delay > 0) - erts_start_timer_callback(node_tab_delete_delay, - prepare_try_delete_dist_entry, - vbin); + schedule_delete_dist_entry(dep); } } @@ -462,12 +493,7 @@ int erts_dist_entry_destructor(Binary *bin) if (refc == -1) return 1; /* Allow deallocation of structure... */ - if (node_tab_delete_delay == 0) - prepare_try_delete_dist_entry((void *) bin); - else if (node_tab_delete_delay > 0) - erts_start_timer_callback(node_tab_delete_delay, - prepare_try_delete_dist_entry, - (void *) bin); + schedule_delete_dist_entry(dep); return 0; } @@ -498,12 +524,17 @@ erts_dist_table_size(void) i++; ASSERT(i == erts_no_of_hidden_dist_entries); i = 0; + for(dep = erts_pending_dist_entries; dep; dep = dep->next) + i++; + ASSERT(i == erts_no_of_pending_dist_entries); + i = 0; for(dep = erts_not_connected_dist_entries; dep; dep = dep->next) i++; ASSERT(i == erts_no_of_not_connected_dist_entries); ASSERT(dist_entries == (erts_no_of_visible_dist_entries + erts_no_of_hidden_dist_entries + + erts_no_of_pending_dist_entries + erts_no_of_not_connected_dist_entries)); #endif @@ -518,43 +549,46 @@ erts_dist_table_size(void) void erts_set_dist_entry_not_connected(DistEntry *dep) { + DistEntry** head; + ERTS_LC_ASSERT(erts_lc_is_de_rwlocked(dep)); erts_rwmtx_rwlock(&erts_dist_table_rwmtx); ASSERT(dep != erts_this_dist_entry); - ASSERT(is_internal_port(dep->cid) || is_internal_pid(dep->cid)); - - if(dep->flags & DFLAG_PUBLISHED) { - if(dep->prev) { - ASSERT(is_in_de_list(dep, erts_visible_dist_entries)); - dep->prev->next = dep->next; - } - else { - ASSERT(erts_visible_dist_entries == dep); - erts_visible_dist_entries = dep->next; - } - ASSERT(erts_no_of_visible_dist_entries > 0); - erts_no_of_visible_dist_entries--; + if (dep->state == ERTS_DE_STATE_PENDING) { + ASSERT(is_nil(dep->cid)); + ASSERT(erts_no_of_pending_dist_entries > 0); + erts_no_of_pending_dist_entries--; + head = &erts_pending_dist_entries; } else { - if(dep->prev) { - ASSERT(is_in_de_list(dep, erts_hidden_dist_entries)); - dep->prev->next = dep->next; - } - else { - ASSERT(erts_hidden_dist_entries == dep); - erts_hidden_dist_entries = dep->next; - } - - ASSERT(erts_no_of_hidden_dist_entries > 0); - erts_no_of_hidden_dist_entries--; + ASSERT(dep->state != ERTS_DE_STATE_IDLE); + ASSERT(is_internal_port(dep->cid) || is_internal_pid(dep->cid)); + if (dep->flags & DFLAG_PUBLISHED) { + ASSERT(erts_no_of_visible_dist_entries > 0); + erts_no_of_visible_dist_entries--; + head = &erts_visible_dist_entries; + } + else { + ASSERT(erts_no_of_hidden_dist_entries > 0); + erts_no_of_hidden_dist_entries--; + head = &erts_hidden_dist_entries; + } } + if(dep->prev) { + ASSERT(is_in_de_list(dep, *head)); + dep->prev->next = dep->next; + } + else { + ASSERT(*head == dep); + *head = dep->next; + } if(dep->next) dep->next->prev = dep->prev; - dep->status &= ~ERTS_DE_SFLG_CONNECTED; + dep->state = ERTS_DE_STATE_EXITING; dep->flags = 0; dep->prev = NULL; dep->cid = NIL; @@ -570,46 +604,86 @@ erts_set_dist_entry_not_connected(DistEntry *dep) } void +erts_set_dist_entry_pending(DistEntry *dep) +{ + ERTS_LC_ASSERT(erts_lc_is_de_rwlocked(dep)); + erts_rwmtx_rwlock(&erts_dist_table_rwmtx); + + ASSERT(dep != erts_this_dist_entry); + ASSERT(dep->state == ERTS_DE_STATE_IDLE); + ASSERT(is_nil(dep->cid)); + + if(dep->prev) { + ASSERT(is_in_de_list(dep, erts_not_connected_dist_entries)); + dep->prev->next = dep->next; + } + else { + ASSERT(dep == erts_not_connected_dist_entries); + erts_not_connected_dist_entries = dep->next; + } + + if(dep->next) + dep->next->prev = dep->prev; + + erts_no_of_not_connected_dist_entries--; + + dep->state = ERTS_DE_STATE_PENDING; + dep->flags = (DFLAG_DIST_MANDATORY | DFLAG_DIST_HOPEFULLY | DFLAG_NO_MAGIC); + dep->connection_id = (dep->connection_id + 1) & ERTS_DIST_CON_ID_MASK; + + dep->prev = NULL; + dep->next = erts_pending_dist_entries; + if(erts_pending_dist_entries) { + ASSERT(erts_pending_dist_entries->prev == NULL); + erts_pending_dist_entries->prev = dep; + } + erts_pending_dist_entries = dep; + erts_no_of_pending_dist_entries++; + erts_rwmtx_rwunlock(&erts_dist_table_rwmtx); +} + +void erts_set_dist_entry_connected(DistEntry *dep, Eterm cid, Uint flags) { + erts_aint32_t set_qflgs; + ERTS_LC_ASSERT(erts_lc_is_de_rwlocked(dep)); erts_rwmtx_rwlock(&erts_dist_table_rwmtx); ASSERT(dep != erts_this_dist_entry); ASSERT(is_nil(dep->cid)); + ASSERT(dep->state == ERTS_DE_STATE_PENDING); ASSERT(is_internal_port(cid) || is_internal_pid(cid)); if(dep->prev) { - ASSERT(is_in_de_list(dep, erts_not_connected_dist_entries)); + ASSERT(is_in_de_list(dep, erts_pending_dist_entries)); dep->prev->next = dep->next; } else { - ASSERT(erts_not_connected_dist_entries == dep); - erts_not_connected_dist_entries = dep->next; + ASSERT(erts_pending_dist_entries == dep); + erts_pending_dist_entries = dep->next; } if(dep->next) dep->next->prev = dep->prev; - ASSERT(erts_no_of_not_connected_dist_entries > 0); - erts_no_of_not_connected_dist_entries--; + ASSERT(erts_no_of_pending_dist_entries > 0); + erts_no_of_pending_dist_entries--; - dep->status |= ERTS_DE_SFLG_CONNECTED; - dep->flags = flags; + dep->state = ERTS_DE_STATE_CONNECTED; + dep->flags = flags & ~DFLAG_NO_MAGIC; dep->cid = cid; erts_atomic_set_nob(&dep->input_handler, (erts_aint_t) cid); - dep->connection_id++; - dep->connection_id &= ERTS_DIST_EXT_CON_ID_MASK; dep->prev = NULL; erts_atomic64_set_nob(&dep->in, 0); erts_atomic64_set_nob(&dep->out, 0); - erts_atomic32_set_nob(&dep->qflgs, - (is_internal_port(cid) - ? ERTS_DE_QFLG_PORT_CTRL - : ERTS_DE_QFLG_PROC_CTRL)); + set_qflgs = (is_internal_port(cid) ? + ERTS_DE_QFLG_PORT_CTRL : ERTS_DE_QFLG_PROC_CTRL); + erts_atomic32_read_bor_nob(&dep->qflgs, set_qflgs); + if(flags & DFLAG_PUBLISHED) { dep->next = erts_visible_dist_entries; if(erts_visible_dist_entries) { @@ -929,9 +1003,11 @@ void erts_init_node_tables(int dd_sec) erts_hidden_dist_entries = NULL; erts_visible_dist_entries = NULL; + erts_pending_dist_entries = NULL; erts_not_connected_dist_entries = NULL; erts_no_of_hidden_dist_entries = 0; erts_no_of_visible_dist_entries = 0; + erts_no_of_pending_dist_entries = 0; erts_no_of_not_connected_dist_entries = 0; node_tmpl.sysname = am_Noname; @@ -1057,6 +1133,7 @@ typedef struct { typedef struct dist_referrer_ { struct dist_referrer_ *next; int heap_ref; + int ets_ref; int node_ref; int ctrl_ref; int system_ref; @@ -1175,10 +1252,11 @@ insert_dist_referrer(ReferredDist *referred_dist, else { Uint *hp = &drp->id_heap[0]; ASSERT(is_tuple(id)); - drp->id = copy_struct(id, size_object(id), &hp, NULL); + drp->id = copy_struct(id, size_object(id), &hp, NULL); } drp->creation = creation; drp->heap_ref = 0; + drp->ets_ref = 0; drp->node_ref = 0; drp->ctrl_ref = 0; drp->system_ref = 0; @@ -1188,6 +1266,7 @@ insert_dist_referrer(ReferredDist *referred_dist, case NODE_REF: drp->node_ref++; break; case CTRL_REF: drp->ctrl_ref++; break; case HEAP_REF: drp->heap_ref++; break; + case ETS_REF: drp->ets_ref++; break; case SYSTEM_REF: drp->system_ref++; break; default: ASSERT(0); } @@ -1300,6 +1379,11 @@ insert_offheap2(ErlOffHeap *oh, void *arg) (((Bin)->intern.flags & BIN_FLAG_MAGIC) \ && ERTS_MAGIC_BIN_DESTRUCTOR((Bin)) == erts_dist_entry_destructor) +#define IsSendCtxBinary(Bin) \ + (((Bin)->intern.flags & BIN_FLAG_MAGIC) \ + && ERTS_MAGIC_BIN_DESTRUCTOR((Bin)) == erts_dsend_context_dtor) + + static void insert_offheap(ErlOffHeap *oh, int type, Eterm id) { @@ -1336,7 +1420,12 @@ insert_offheap(ErlOffHeap *oh, int type, Eterm id) inserted_bins = nib; UnUseTmpHeapNoproc(BIG_UINT_HEAP_SIZE); } - } + } + else if (IsSendCtxBinary(u.mref->mb)) { + ErtsSendContext* ctx = ERTS_MAGIC_BIN_DATA(u.mref->mb); + if (ctx->deref_dep) + insert_dist_entry(ctx->dep, type, id, 0); + } break; case REFC_BINARY_SUBTAG: case FUN_SUBTAG: @@ -1463,9 +1552,9 @@ insert_delayed_delete_node(void *state, } static void -insert_thr_prgr_delete_dist_entry(void *arg, ErtsThrPrgrVal thr_prgr, void *vbin) +insert_thr_prgr_delete_dist_entry(void *arg, ErtsThrPrgrVal thr_prgr, void *vdep) { - DistEntry *dep = ErtsBin2DistEntry(vbin); + DistEntry *dep = vdep; Eterm heap[3]; insert_dist_entry(dep, SYSTEM_REF, @@ -1476,9 +1565,9 @@ insert_thr_prgr_delete_dist_entry(void *arg, ErtsThrPrgrVal thr_prgr, void *vbin static void insert_delayed_delete_dist_entry(void *state, ErtsMonotonicTime timeout_pos, - void *vbin) + void *vdep) { - DistEntry *dep = ErtsBin2DistEntry(vbin); + DistEntry *dep = vdep; Eterm heap[3]; insert_dist_entry(dep, SYSTEM_REF, @@ -1520,7 +1609,7 @@ setup_reference_table(void) erts_debug_callback_timer_foreach(prepare_try_delete_dist_entry, insert_delayed_delete_dist_entry, NULL); - erts_debug_later_op_foreach(try_delete_dist_entry, + erts_debug_later_op_foreach(start_timer_delete_dist_entry, insert_thr_prgr_delete_dist_entry, NULL); @@ -1686,6 +1775,15 @@ setup_reference_table(void) insert_monitors(dep->monitors, dep->sysname); } + for(dep = erts_pending_dist_entries; dep; dep = dep->next) { + if(dep->nlinks) + insert_links2(dep->nlinks, dep->sysname); + if(dep->node_links) + insert_links(dep->node_links, dep->sysname); + if(dep->monitors) + insert_monitors(dep->monitors, dep->sysname); + } + /* Not connected dist entries should not have any links, but inspect them anyway */ for(dep = erts_not_connected_dist_entries; dep; dep = dep->next) { @@ -1855,6 +1953,10 @@ reference_table_term(Uint **hpp, ErlOffHeap *ohp, Uint *szp) tup = MK_2TUP(AM_heap, MK_UINT(drp->heap_ref)); drl = MK_CONS(tup, drl); } + if(drp->ets_ref) { + tup = MK_2TUP(AM_ets, MK_UINT(drp->ets_ref)); + drl = MK_CONS(tup, drl); + } if(drp->system_ref) { tup = MK_2TUP(AM_system, MK_UINT(drp->system_ref)); drl = MK_CONS(tup, drl); @@ -1871,12 +1973,17 @@ reference_table_term(Uint **hpp, ErlOffHeap *ohp, Uint *szp) else if (is_tuple(drp->id)) { Eterm *t; ASSERT(drp->system_ref && !drp->node_ref - && !drp->ctrl_ref && !drp->heap_ref); + && !drp->ctrl_ref && !drp->heap_ref && !drp->ets_ref); t = tuple_val(drp->id); ASSERT(2 == arityval(t[0])); tup = MK_2TUP(t[1], t[2]); } - else { + else if (drp->ets_ref) { + ASSERT(!drp->heap_ref && !drp->node_ref && + !drp->ctrl_ref && !drp->system_ref); + tup = MK_2TUP(AM_ets, drp->id); + } + else { ASSERT(!drp->ctrl_ref && drp->node_ref); ASSERT(is_atom(drp->id)); tup = MK_2TUP(drp->id, MK_UINT(drp->creation)); diff --git a/erts/emulator/beam/erl_node_tables.h b/erts/emulator/beam/erl_node_tables.h index ee8277b5ea..58279017c8 100644 --- a/erts/emulator/beam/erl_node_tables.h +++ b/erts/emulator/beam/erl_node_tables.h @@ -57,11 +57,12 @@ #define ERST_INTERNAL_CHANNEL_NO 0 -#define ERTS_DE_SFLG_CONNECTED (((Uint32) 1) << 0) -#define ERTS_DE_SFLG_EXITING (((Uint32) 1) << 1) - -#define ERTS_DE_SFLGS_ALL (ERTS_DE_SFLG_CONNECTED \ - | ERTS_DE_SFLG_EXITING) +enum dist_entry_state { + ERTS_DE_STATE_IDLE, + ERTS_DE_STATE_PENDING, + ERTS_DE_STATE_CONNECTED, + ERTS_DE_STATE_EXITING +}; #define ERTS_DE_QFLG_BUSY (((erts_aint32_t) 1) << 0) #define ERTS_DE_QFLG_EXIT (((erts_aint32_t) 1) << 1) @@ -85,10 +86,13 @@ typedef struct ErtsDistOutputBuf_ ErtsDistOutputBuf; struct ErtsDistOutputBuf_ { #ifdef DEBUG Uint dbg_pattern; + byte *alloc_endp; #endif ErtsDistOutputBuf *next; + Uint hopefull_flags; byte *extp; byte *ext_endp; + byte *msg_start; byte data[1]; }; @@ -109,8 +113,6 @@ struct ErtsProcList_; * unlock mutexes with higher numbers before mutexes with higher numbers. */ -struct erl_link; - typedef struct dist_entry_ { HashBucket hash_bucket; /* Hash bucket */ struct dist_entry_ *next; /* Next entry in dist_table (not sorted) */ @@ -123,7 +125,7 @@ typedef struct dist_entry_ { Eterm cid; /* connection handler (pid or port), NIL == free */ Uint32 connection_id; /* Connection id incremented on connect */ - Uint32 status; /* Slot status, like exiting reserved etc */ + enum dist_entry_state state; Uint32 flags; /* Distribution flags, like hidden, atom cache etc. */ unsigned long version; /* Protocol version */ @@ -159,6 +161,8 @@ typedef struct dist_entry_ { struct cache* cache; /* The atom cache */ ErtsThrPrgrLaterOp later_op; + + struct transcode_context* transcode_ctx; } DistEntry; typedef struct erl_node_ { @@ -177,9 +181,11 @@ extern erts_rwmtx_t erts_node_table_rwmtx; extern DistEntry *erts_hidden_dist_entries; extern DistEntry *erts_visible_dist_entries; +extern DistEntry *erts_pending_dist_entries; extern DistEntry *erts_not_connected_dist_entries; extern Sint erts_no_of_hidden_dist_entries; extern Sint erts_no_of_visible_dist_entries; +extern Sint erts_no_of_pending_dist_entries; extern Sint erts_no_of_not_connected_dist_entries; extern DistEntry *erts_this_dist_entry; @@ -195,6 +201,7 @@ void erts_schedule_delete_dist_entry(DistEntry *); Uint erts_dist_table_size(void); void erts_dist_table_info(fmtfn_t, void *); void erts_set_dist_entry_not_connected(DistEntry *); +void erts_set_dist_entry_pending(DistEntry *); void erts_set_dist_entry_connected(DistEntry *, Eterm, Uint); ErlNode *erts_find_or_insert_node(Eterm, Uint32); void erts_schedule_delete_node(ErlNode *); @@ -210,6 +217,7 @@ int erts_lc_is_de_rlocked(DistEntry *); #endif int erts_dist_entry_destructor(Binary *bin); DistEntry *erts_dhandle_to_dist_entry(Eterm dhandle); +Eterm erts_build_dhandle(Eterm **hpp, ErlOffHeap*, DistEntry*); Eterm erts_make_dhandle(Process *c_p, DistEntry *dep); void erts_ref_dist_entry(DistEntry *dep); void erts_deref_dist_entry(DistEntry *dep); diff --git a/erts/emulator/beam/erl_port.h b/erts/emulator/beam/erl_port.h index 9117eb1f72..0d148ee048 100644 --- a/erts/emulator/beam/erl_port.h +++ b/erts/emulator/beam/erl_port.h @@ -180,6 +180,7 @@ void erts_init_port_data(Port *); void erts_cleanup_port_data(Port *); Uint erts_port_data_size(Port *); ErlOffHeap *erts_port_data_offheap(Port *); +Eterm erts_port_data_read(Port* prt); #define ERTS_PORT_GET_CONNECTED(PRT) \ ((Eterm) erts_atomic_read_nob(&(PRT)->connected)) @@ -195,26 +196,52 @@ struct erl_drv_port_data_lock { Port *prt; }; +ERTS_GLB_INLINE void erts_init_runq_port(Port *prt, ErtsRunQueue *runq); +ERTS_GLB_INLINE void erts_set_runq_port(Port *prt, ErtsRunQueue *runq); +ERTS_GLB_INLINE ErtsRunQueue *erts_get_runq_port(Port *prt); ERTS_GLB_INLINE ErtsRunQueue *erts_port_runq(Port *prt); #if ERTS_GLB_INLINE_INCL_FUNC_DEF +ERTS_GLB_INLINE void +erts_init_runq_port(Port *prt, ErtsRunQueue *runq) +{ + if (!runq) + ERTS_INTERNAL_ERROR("Missing run-queue"); + erts_atomic_init_nob(&prt->run_queue, (erts_aint_t) runq); +} + +ERTS_GLB_INLINE void +erts_set_runq_port(Port *prt, ErtsRunQueue *runq) +{ + if (!runq) + ERTS_INTERNAL_ERROR("Missing run-queue"); + erts_atomic_set_nob(&prt->run_queue, (erts_aint_t) runq); +} + +ERTS_GLB_INLINE ErtsRunQueue * +erts_get_runq_port(Port *prt) +{ + ErtsRunQueue *runq; + runq = (ErtsRunQueue *) erts_atomic_read_nob(&prt->run_queue); + if (!runq) + ERTS_INTERNAL_ERROR("Missing run-queue"); + return runq; +} + + ERTS_GLB_INLINE ErtsRunQueue * erts_port_runq(Port *prt) { ErtsRunQueue *rq1, *rq2; - rq1 = (ErtsRunQueue *) erts_atomic_read_nob(&prt->run_queue); - if (!rq1) - return NULL; + rq1 = erts_get_runq_port(prt); while (1) { erts_runq_lock(rq1); - rq2 = (ErtsRunQueue *) erts_atomic_read_nob(&prt->run_queue); + rq2 = erts_get_runq_port(prt); if (rq1 == rq2) return rq1; erts_runq_unlock(rq1); rq1 = rq2; - if (!rq1) - return NULL; } } diff --git a/erts/emulator/beam/erl_port_task.c b/erts/emulator/beam/erl_port_task.c index a588477320..4a3671df0c 100644 --- a/erts/emulator/beam/erl_port_task.c +++ b/erts/emulator/beam/erl_port_task.c @@ -84,11 +84,10 @@ static void chk_task_queues(Port *pp, ErtsPortTask *execq, int processing_busy_q #define LTTNG_DRIVER(TRACEPOINT, PP) do {} while(0) #endif -#define ERTS_LC_VERIFY_RQ(RQ, PP) \ - do { \ +#define ERTS_LC_VERIFY_RQ(RQ, PP) \ + do { \ ERTS_LC_ASSERT(erts_lc_runq_is_locked(runq)); \ - ERTS_LC_ASSERT((RQ) == ((ErtsRunQueue *) \ - erts_atomic_read_nob(&(PP)->run_queue))); \ + ERTS_LC_ASSERT((RQ) == erts_get_runq_port((PP))); \ } while (0) #define ERTS_PT_STATE_SCHEDULED 0 @@ -1520,19 +1519,15 @@ erts_port_task_schedule(Eterm id, /* Enqueue port on run-queue */ runq = erts_port_runq(pp); - if (!runq) - ERTS_INTERNAL_ERROR("Missing run-queue"); xrunq = erts_check_emigration_need(runq, ERTS_PORT_PRIO_LEVEL); ERTS_LC_ASSERT(runq != xrunq); ERTS_LC_VERIFY_RQ(runq, pp); if (xrunq) { /* Emigrate port ... */ - erts_atomic_set_nob(&pp->run_queue, (erts_aint_t) xrunq); + erts_set_runq_port(pp, xrunq); erts_runq_unlock(runq); runq = erts_port_runq(pp); - if (!runq) - ERTS_INTERNAL_ERROR("Missing run-queue"); } enqueue_port(runq, pp); @@ -1593,8 +1588,6 @@ erts_port_task_free_port(Port *pp) ASSERT(!(erts_atomic32_read_nob(&pp->state) & ERTS_PORT_SFLGS_DEAD)); runq = erts_port_runq(pp); - if (!runq) - ERTS_INTERNAL_ERROR("Missing run-queue"); erts_port_task_sched_lock(&pp->sched); flags = erts_atomic32_read_bor_relb(&pp->sched.flags, ERTS_PTS_FLG_EXIT); @@ -1805,7 +1798,7 @@ erts_port_task_execute(ErtsRunQueue *runq, Port **curr_port_pp) erts_unblock_fpe(fpe_was_unmasked); ERTS_MSACC_POP_STATE_M(); - ASSERT(runq == (ErtsRunQueue *) erts_atomic_read_nob(&pp->run_queue)); + ASSERT(runq == erts_get_runq_port(pp)); active = finalize_exec(pp, &execq, processing_busy_q); @@ -1831,11 +1824,10 @@ erts_port_task_execute(ErtsRunQueue *runq, Port **curr_port_pp) } else { /* Emigrate port... */ - erts_atomic_set_nob(&pp->run_queue, (erts_aint_t) xrunq); + erts_set_runq_port(pp, xrunq); erts_runq_unlock(runq); xrunq = erts_port_runq(pp); - ASSERT(xrunq); enqueue_port(xrunq, pp); erts_runq_unlock(xrunq); erts_notify_inc_runq(xrunq); @@ -2069,7 +2061,7 @@ void erts_enqueue_port(ErtsRunQueue *rq, Port *pp) { ERTS_LC_ASSERT(erts_lc_runq_is_locked(rq)); - ASSERT(rq == (ErtsRunQueue *) erts_atomic_read_nob(&pp->run_queue)); + ASSERT(rq == erts_get_runq_port(pp)); ASSERT(erts_atomic32_read_nob(&pp->sched.flags) & ERTS_PTS_FLG_IN_RUNQ); enqueue_port(rq, pp); } @@ -2080,8 +2072,7 @@ erts_dequeue_port(ErtsRunQueue *rq) Port *pp; ERTS_LC_ASSERT(erts_lc_runq_is_locked(rq)); pp = pop_port(rq); - ASSERT(!pp - || rq == (ErtsRunQueue *) erts_atomic_read_nob(&pp->run_queue)); + ASSERT(!pp || rq == erts_get_runq_port(pp)); ASSERT(!pp || (erts_atomic32_read_nob(&pp->sched.flags) & ERTS_PTS_FLG_IN_RUNQ)); return pp; diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c index 0433b4ea17..d069f62b39 100644 --- a/erts/emulator/beam/erl_process.c +++ b/erts/emulator/beam/erl_process.c @@ -137,36 +137,6 @@ runq_got_work_to_execute(ErtsRunQueue *rq) return runq_got_work_to_execute_flags(ERTS_RUNQ_FLGS_GET_NOB(rq)); } -#undef RUNQ_READ_RQ -#undef RUNQ_SET_RQ -#define RUNQ_READ_RQ(X) ((ErtsRunQueue *) erts_atomic_read_nob((X))) -#define RUNQ_SET_RQ(X, RQ) erts_atomic_set_nob((X), (erts_aint_t) (RQ)) - -#ifdef DEBUG -# if defined(ARCH_64) -# define ERTS_DBG_SET_INVALID_RUNQP(RQP, N) \ - (RUNQ_SET_RQ((RQP), (0xdeadbeefdead0003LL | ((N) << 4))) -# define ERTS_DBG_VERIFY_VALID_RUNQP(RQP) \ -do { \ - ASSERT((RQP) != NULL); \ - ASSERT(((((Uint) (RQP)) & ((Uint) 0x3))) == ((Uint) 0)); \ - ASSERT((((Uint) (RQP)) & ~((Uint) 0xffff)) != ((Uint) 0xdeadbeefdead0000LL));\ -} while (0) -# else -# define ERTS_DBG_SET_INVALID_RUNQP(RQP, N) \ - (RUNQ_SET_RQ((RQP), (0xdead0003 | ((N) << 4)))) -# define ERTS_DBG_VERIFY_VALID_RUNQP(RQP) \ -do { \ - ASSERT((RQP) != NULL); \ - ASSERT(((((UWord) (RQP)) & ((UWord) 1))) == ((UWord) 0)); \ - ASSERT((((UWord) (RQP)) & ~((UWord) 0xffff)) != ((UWord) 0xdead0000)); \ -} while (0) -# endif -#else -# define ERTS_DBG_SET_INVALID_RUNQP(RQP, N) -# define ERTS_DBG_VERIFY_VALID_RUNQP(RQP) -#endif - const Process erts_invalid_process = {{ERTS_INVALID_PID}}; extern BeamInstr beam_apply[]; @@ -1538,6 +1508,20 @@ erts_proclist_destroy(ErtsProcList *plp) proclist_destroy(plp); } +void +erts_proclist_dump(fmtfn_t to, void *to_arg, ErtsProcList *plp) +{ + ErtsProcList *first = plp; + + while (plp) { + erts_print(to, to_arg, "%T", plp->pid); + plp = plp->next; + if (plp == first) + break; + } + erts_print(to, to_arg, "\n"); +} + void * erts_psd_set_init(Process *p, int ix, void *data) { @@ -3926,21 +3910,16 @@ immigrate(ErtsRunQueue *c_rq, ErtsMigrationPath *mp) Port *prt; prt = erts_dequeue_port(rq); if (prt) - RUNQ_SET_RQ(&prt->run_queue, c_rq); + erts_set_runq_port(prt, c_rq); erts_runq_unlock(rq); if (prt) { - /* port might terminate while we have no lock... */ rq = erts_port_runq(prt); - if (rq) { - if (rq != c_rq) - erts_exit(ERTS_ABORT_EXIT, - "%s:%d:%s(): Internal error", - __FILE__, __LINE__, __func__); - erts_enqueue_port(c_rq, prt); - if (!iflag) - return; /* done */ - erts_runq_unlock(c_rq); - } + if (rq != c_rq) + ERTS_INTERNAL_ERROR("Unexpected run-queue"); + erts_enqueue_port(c_rq, prt); + if (!iflag) + return; /* done */ + erts_runq_unlock(c_rq); } } else { @@ -3954,12 +3933,11 @@ immigrate(ErtsRunQueue *c_rq, ErtsMigrationPath *mp) while (proc) { erts_aint32_t state; state = erts_atomic32_read_acqb(&proc->state); - if (!(ERTS_PSFLG_BOUND & state) - && (prio == (int) ERTS_PSFLGS_GET_PRQ_PRIO(state))) { + if (prio == (int) ERTS_PSFLGS_GET_PRQ_PRIO(state) + && erts_try_change_runq_proc(proc, c_rq)) { ErtsRunQueueInfo *rqi = &rq->procs.prio_info[prio]; unqueue_process(rq, rpq, rqi, prio, prev_proc, proc); erts_runq_unlock(rq); - RUNQ_SET_RQ(&proc->run_queue, c_rq); rq_locked = 0; erts_runq_lock(c_rq); @@ -4140,21 +4118,13 @@ evacuate_run_queue(ErtsRunQueue *rq, while (prt) { ErtsRunQueue *prt_rq; prt = erts_dequeue_port(rq); - RUNQ_SET_RQ(&prt->run_queue, to_rq); + erts_set_runq_port(prt, to_rq); erts_runq_unlock(rq); - /* - * The port might terminate while - * we have no lock on it... - */ prt_rq = erts_port_runq(prt); - if (prt_rq) { - if (prt_rq != to_rq) - erts_exit(ERTS_ABORT_EXIT, - "%s:%d:%s() internal error\n", - __FILE__, __LINE__, __func__); - erts_enqueue_port(to_rq, prt); - erts_runq_unlock(to_rq); - } + if (prt_rq != to_rq) + ERTS_INTERNAL_ERROR("Unexpected run-queue"); + erts_enqueue_port(to_rq, prt); + erts_runq_unlock(to_rq); erts_runq_lock(rq); prt = rq->ports.start; } @@ -4165,8 +4135,6 @@ evacuate_run_queue(ErtsRunQueue *rq, for (prio_q = 0; prio_q < ERTS_NO_PROC_PRIO_QUEUES; prio_q++) { erts_aint32_t state; Process *proc; - int notify = 0; - to_rq = NULL; if (!mp->prio[prio_q].runq) return; @@ -4177,14 +4145,13 @@ evacuate_run_queue(ErtsRunQueue *rq, while (proc) { Process *real_proc; int prio; - erts_aint32_t max_qbit, qbit, real_state; + erts_aint32_t max_qbit, qbit; prio = ERTS_PSFLGS_GET_PRQ_PRIO(state); qbit = ((erts_aint32_t) 1) << prio; if (!(state & ERTS_PSFLG_PROXY)) { real_proc = proc; - real_state = state; } else { real_proc = erts_proc_lookup_raw(proc->common.id); @@ -4192,7 +4159,6 @@ evacuate_run_queue(ErtsRunQueue *rq, free_proxy_proc(proc); goto handle_next_proc; } - real_state = erts_atomic32_read_acqb(&real_proc->state); } max_qbit = (state >> ERTS_PSFLGS_IN_PRQ_MASK_OFFSET); @@ -4227,7 +4193,13 @@ evacuate_run_queue(ErtsRunQueue *rq, goto handle_next_proc; } - if (ERTS_PSFLG_BOUND & real_state) { + prio = (int) ERTS_PSFLGS_GET_PRQ_PRIO(state); + to_rq = mp->prio[prio].runq; + + if (!to_rq) + goto handle_next_proc; + + if (!erts_try_change_runq_proc(proc, to_rq)) { /* Bound processes get stuck here... */ proc->next = NULL; if (sbpp->last) @@ -4237,16 +4209,13 @@ evacuate_run_queue(ErtsRunQueue *rq, sbpp->last = proc; } else { - int prio = (int) ERTS_PSFLGS_GET_PRQ_PRIO(state); erts_runq_unlock(rq); - to_rq = mp->prio[prio].runq; - RUNQ_SET_RQ(&proc->run_queue, to_rq); - erts_runq_lock(to_rq); enqueue_process(to_rq, prio, proc); erts_runq_unlock(to_rq); - notify = 1; + + smp_notify_inc_runq(to_rq); erts_runq_lock(rq); } @@ -4254,8 +4223,7 @@ evacuate_run_queue(ErtsRunQueue *rq, handle_next_proc: proc = dequeue_process(rq, prio_q, &state); } - if (notify) - smp_notify_inc_runq(to_rq); + } } @@ -4309,14 +4277,13 @@ try_steal_task_from_victim(ErtsRunQueue *rq, int *rq_lockedp, ErtsRunQueue *vrq, proc = rpq->first; while (proc) { - erts_aint32_t state = erts_atomic32_read_acqb(&proc->state); - if (!(ERTS_PSFLG_BOUND & state)) { + if (erts_try_change_runq_proc(proc, rq)) { + erts_aint32_t state = erts_atomic32_read_acqb(&proc->state); /* Steal process */ int prio = (int) ERTS_PSFLGS_GET_PRQ_PRIO(state); ErtsRunQueueInfo *rqi = &vrq->procs.prio_info[prio]; unqueue_process(vrq, rpq, rqi, prio, prev_proc, proc); erts_runq_unlock(vrq); - RUNQ_SET_RQ(&proc->run_queue, rq); erts_runq_lock(rq); *rq_lockedp = 1; @@ -4341,26 +4308,14 @@ no_procs: if (vrq->ports.start) { ErtsRunQueue *prt_rq; Port *prt = erts_dequeue_port(vrq); - RUNQ_SET_RQ(&prt->run_queue, rq); + erts_set_runq_port(prt, rq); erts_runq_unlock(vrq); - - /* - * The port might terminate while - * we have no lock on it... - */ - prt_rq = erts_port_runq(prt); - if (!prt_rq) - return 0; - else { - if (prt_rq != rq) - erts_exit(ERTS_ABORT_EXIT, - "%s:%d:%s() internal error\n", - __FILE__, __LINE__, __func__); - *rq_lockedp = 1; - erts_enqueue_port(rq, prt); - return !0; - } + if (prt_rq != rq) + ERTS_INTERNAL_ERROR("Unexpected run-queue"); + *rq_lockedp = 1; + erts_enqueue_port(rq, prt); + return !0; } erts_runq_unlock(vrq); @@ -6116,7 +6071,8 @@ make_proxy_proc(Process *prev_proxy, Process *proc, erts_aint32_t prio) { erts_aint32_t state; Process *proxy; - ErtsRunQueue *rq = RUNQ_READ_RQ(&proc->run_queue); + int bound; + ErtsRunQueue *rq = erts_get_runq_proc(proc, &bound); state = (ERTS_PSFLG_PROXY | ERTS_PSFLG_IN_RUNQ @@ -6129,7 +6085,7 @@ make_proxy_proc(Process *prev_proxy, Process *proc, erts_aint32_t prio) proxy = prev_proxy; ASSERT(erts_atomic32_read_nob(&proxy->state) & ERTS_PSFLG_PROXY); erts_atomic32_set_nob(&proxy->state, state); - RUNQ_SET_RQ(&proc->run_queue, rq); + (void) erts_set_runq_proc(proc, rq, &bound); } else { proxy = erts_alloc(ERTS_ALC_T_PROC, sizeof(Process)); @@ -6142,8 +6098,7 @@ make_proxy_proc(Process *prev_proxy, Process *proc, erts_aint32_t prio) } #endif erts_atomic32_init_nob(&proxy->state, state); - erts_atomic_init_nob(&proxy->run_queue, - erts_atomic_read_nob(&proc->run_queue)); + erts_init_runq_proc(proc, rq, bound); } proxy->common.id = proc->common.id; @@ -6334,18 +6289,21 @@ select_enqueue_run_queue(int enqueue, int enq_prio, Process *p, erts_aint32_t st default: { ErtsRunQueue* runq; + int bound; ASSERT(enqueue == ERTS_ENQUEUE_NORMAL_QUEUE || enqueue == -ERTS_ENQUEUE_NORMAL_QUEUE); - runq = erts_get_runq_proc(p); + runq = erts_get_runq_proc(p, &bound); - if (!(ERTS_PSFLG_BOUND & state)) { + if (!bound) { ErtsRunQueue *new_runq = erts_check_emigration_need(runq, enq_prio); - if (new_runq) { - RUNQ_SET_RQ(&p->run_queue, new_runq); - runq = new_runq; - } + if (new_runq) { + if (erts_try_change_runq_proc(p, new_runq)) + runq = new_runq; + else + runq = erts_get_runq_proc(p, NULL); + } } ASSERT(runq); @@ -10824,7 +10782,7 @@ request_system_task(Process *c_p, Eterm requester, Eterm target, goto badarg; req_type = tp[1]; req_id = tp[2]; - req_id_sz = is_immed(req_id) ? req_id : size_object(req_id); + req_id_sz = is_immed(req_id) ? 0 : size_object(req_id); tot_sz = req_id_sz; for (i = 0; i < ERTS_MAX_PROC_SYS_TASK_ARGS; i++) { int tix = 3 + i; @@ -11462,6 +11420,7 @@ typedef struct { Process *proc; erts_aint32_t state; ErtsRunQueue *run_queue; + int bound; } ErtsEarlyProcInit; static void early_init_process_struct(void *varg, Eterm data) @@ -11472,10 +11431,9 @@ static void early_init_process_struct(void *varg, Eterm data) proc->common.id = make_internal_pid(data); erts_atomic32_init_nob(&proc->dirty_state, 0); proc->dirty_sys_tasks = NULL; + erts_init_runq_proc(proc, arg->run_queue, arg->bound); erts_atomic32_init_relb(&proc->state, arg->state); - RUNQ_SET_RQ(&proc->run_queue, arg->run_queue); - erts_proc_lock_init(proc); /* All locks locked */ } @@ -11484,7 +11442,7 @@ static void early_init_process_struct(void *varg, Eterm data) ** Allocate process and find out where to place next process. */ static Process* -alloc_process(ErtsRunQueue *rq, erts_aint32_t state) +alloc_process(ErtsRunQueue *rq, int bound, erts_aint32_t state) { ErtsEarlyProcInit init_arg; Process *p; @@ -11493,9 +11451,12 @@ alloc_process(ErtsRunQueue *rq, erts_aint32_t state) if (!p) return NULL; + ASSERT(rq); + init_arg.proc = (Process *) p; - init_arg.run_queue = rq; init_arg.state = state; + init_arg.run_queue = rq; + init_arg.bound = bound; ERTS_CT_ASSERT(offsetof(Process,common) == 0); @@ -11529,6 +11490,7 @@ erl_create_process(Process* parent, /* Parent of process (default group leader). Eterm args, /* Arguments for function (must be well-formed list). */ ErlSpawnOpts* so) /* Options for spawn. */ { + int bound = 0; Uint flags = 0; ErtsRunQueue *rq = NULL; Process *p; @@ -11565,7 +11527,7 @@ erl_create_process(Process* parent, /* Parent of process (default group leader). ASSERT(0 <= ix && ix < erts_no_run_queues); rq = ERTS_RUNQ_IX(ix); /* Unsupported feature... */ - state |= ERTS_PSFLG_BOUND; + bound = !0; } prio = (erts_aint32_t) so->priority; } @@ -11578,17 +11540,16 @@ erl_create_process(Process* parent, /* Parent of process (default group leader). flags |= F_OFF_HEAP_MSGQ; } else if (so->flags & SPO_ON_HEAP_MSGQ) { - state |= ERTS_PSFLG_ON_HEAP_MSGQ; flags |= F_ON_HEAP_MSGQ; } ASSERT((flags & F_ON_HEAP_MSGQ) || (flags & F_OFF_HEAP_MSGQ)); if (!rq) - rq = erts_get_runq_proc(parent); + rq = erts_get_runq_proc(parent, NULL); - p = alloc_process(rq, state); /* All proc locks are locked by this thread - on success */ + p = alloc_process(rq, bound, state); /* All proc locks are locked by this thread + on success */ if (!p) { erts_send_error_to_logger_str(parent->group_leader, "Too many processes\n"); @@ -11596,11 +11557,6 @@ erl_create_process(Process* parent, /* Parent of process (default group leader). goto error; } - ASSERT((erts_atomic32_read_nob(&p->state) - & ERTS_PSFLG_ON_HEAP_MSGQ) - || (erts_atomic32_read_nob(&p->state) - & ERTS_PSFLG_OFF_HEAP_MSGQ)); - #ifdef SHCOPY_SPAWN arg_size = copy_shared_calculate(args, &info); #else @@ -11974,7 +11930,7 @@ void erts_init_empty_process(Process *p) p->pending_exit.bp = NULL; erts_proc_lock_init(p); erts_proc_unlock(p, ERTS_PROC_LOCKS_ALL); - RUNQ_SET_RQ(&p->run_queue, ERTS_RUNQ_IX(0)); + erts_init_runq_proc(p, ERTS_RUNQ_IX(0), 0); #if !defined(NO_FPE_SIGNALS) || defined(HIPE) p->fp_exception = 0; @@ -12296,7 +12252,7 @@ save_pending_exiter(Process *p, ErtsProcList *plp) ERTS_LC_ASSERT(ERTS_PROC_LOCK_STATUS & erts_proc_lc_my_proc_locks(p)); - rq = RUNQ_READ_RQ(&p->run_queue); + rq = erts_get_runq_proc(p, NULL); ASSERT(rq && !ERTS_RUNQ_IX_IS_DIRTY(rq->ix)); if (!plp) @@ -12657,9 +12613,11 @@ static void doit_exit_monitor(ErtsMonitor *mon, void *vpcontext) erts_de_links_unlock(dep); if (rmon) { ErtsDSigData dsd; - int code = erts_dsig_prepare(&dsd, dep, NULL, - ERTS_DSP_NO_LOCK, 0); - if (code == ERTS_DSIG_PREP_CONNECTED) { + int code = erts_dsig_prepare(&dsd, dep, NULL, 0, + ERTS_DSP_NO_LOCK, 0, 0); + if (code == ERTS_DSIG_PREP_CONNECTED || + code == ERTS_DSIG_PREP_PENDING) { + code = erts_dsig_send_demonitor(&dsd, rmon->u.pid, mon->name, @@ -12703,9 +12661,11 @@ static void doit_exit_monitor(ErtsMonitor *mon, void *vpcontext) erts_de_links_unlock(dep); if (rmon) { ErtsDSigData dsd; - int code = erts_dsig_prepare(&dsd, dep, NULL, - ERTS_DSP_NO_LOCK, 0); - if (code == ERTS_DSIG_PREP_CONNECTED) { + int code = erts_dsig_prepare(&dsd, dep, NULL, 0, + ERTS_DSP_NO_LOCK, 0, 0); + if (code == ERTS_DSIG_PREP_CONNECTED || + code == ERTS_DSIG_PREP_PENDING) { + code = erts_dsig_send_demonitor(&dsd, rmon->u.pid, mon->u.pid, @@ -12762,8 +12722,8 @@ static void doit_exit_monitor(ErtsMonitor *mon, void *vpcontext) erts_de_links_unlock(dep); if (rmon) { ErtsDSigData dsd; - int code = erts_dsig_prepare(&dsd, dep, NULL, - ERTS_DSP_NO_LOCK, 0); + int code = erts_dsig_prepare(&dsd, dep, NULL, 0, + ERTS_DSP_NO_LOCK, 0, 0); if (code == ERTS_DSIG_PREP_CONNECTED) { code = erts_dsig_send_m_exit(&dsd, mon->u.pid, @@ -12885,14 +12845,18 @@ static void doit_exit_link(ErtsLink *lnk, void *vpcontext) int code; ErtsDistLinkData dld; erts_remove_dist_link(&dld, p->common.id, item, dep); - erts_proc_lock(p, ERTS_PROC_LOCK_MAIN); - code = erts_dsig_prepare(&dsd, dep, p, ERTS_DSP_NO_LOCK, 0); - if (code == ERTS_DSIG_PREP_CONNECTED) { - code = erts_dsig_send_exit_tt(&dsd, p->common.id, item, - reason, SEQ_TRACE_TOKEN(p)); - ASSERT(code == ERTS_DSIG_SEND_OK); - } - erts_proc_unlock(p, ERTS_PROC_LOCK_MAIN); + if (dld.d_lnk) { + erts_proc_lock(p, ERTS_PROC_LOCK_MAIN); + code = erts_dsig_prepare(&dsd, dep, p, 0, ERTS_DSP_NO_LOCK, 0, 0); + if (code == ERTS_DSIG_PREP_CONNECTED || + code == ERTS_DSIG_PREP_PENDING) { + + code = erts_dsig_send_exit_tt(&dsd, p->common.id, item, + reason, SEQ_TRACE_TOKEN(p)); + ASSERT(code == ERTS_DSIG_SEND_OK); + } + erts_proc_unlock(p, ERTS_PROC_LOCK_MAIN); + } erts_destroy_dist_link(&dld); } } @@ -13388,16 +13352,33 @@ stack_element_dump(fmtfn_t to, void *to_arg, Eterm* sp, int yreg) return yreg; } +static void print_current_process_info(fmtfn_t, void *to_arg, ErtsSchedulerData*); + /* * Print scheduler information */ void -erts_print_scheduler_info(fmtfn_t to, void *to_arg, ErtsSchedulerData *esdp) { +erts_print_scheduler_info(fmtfn_t to, void *to_arg, ErtsSchedulerData *esdp) +{ int i; erts_aint32_t flg; - Process *p; - erts_print(to, to_arg, "=scheduler:%u\n", esdp->no); + switch (esdp->type) { + case ERTS_SCHED_NORMAL: + erts_print(to, to_arg, "=scheduler:%u\n", esdp->no); + break; + case ERTS_SCHED_DIRTY_CPU: + erts_print(to, to_arg, "=dirty_cpu_scheduler:%u\n", + (esdp->dirty_no + erts_no_schedulers)); + break; + case ERTS_SCHED_DIRTY_IO: + erts_print(to, to_arg, "=dirty_io_scheduler:%u\n", + (esdp->dirty_no + erts_no_schedulers + erts_no_dirty_cpu_schedulers)); + break; + default: + erts_print(to, to_arg, "=unknown_scheduler_type:%u\n", esdp->type); + break; + } flg = erts_atomic32_read_dirty(&esdp->ssi->flags); erts_print(to, to_arg, "Scheduler Sleep Info Flags: "); @@ -13443,10 +13424,24 @@ erts_print_scheduler_info(fmtfn_t to, void *to_arg, ErtsSchedulerData *esdp) { } erts_print(to, to_arg, "\n"); - erts_print(to, to_arg, "Current Port: "); - if (esdp->current_port) - erts_print(to, to_arg, "%T", esdp->current_port->common.id); - erts_print(to, to_arg, "\n"); + if (esdp->type == ERTS_SCHED_NORMAL) { + erts_print(to, to_arg, "Current Port: "); + if (esdp->current_port) + erts_print(to, to_arg, "%T", esdp->current_port->common.id); + erts_print(to, to_arg, "\n"); + + erts_print_run_queue_info(to, to_arg, esdp->run_queue); + } + + /* This *MUST* to be the last information in scheduler block */ + print_current_process_info(to, to_arg, esdp); +} + +void erts_print_run_queue_info(fmtfn_t to, void *to_arg, + ErtsRunQueue *run_queue) +{ + erts_aint32_t flg; + int i; for (i = 0; i < ERTS_NO_PROC_PRIO_LEVELS; i++) { erts_print(to, to_arg, "Run Queue "); @@ -13468,12 +13463,12 @@ erts_print_scheduler_info(fmtfn_t to, void *to_arg, ErtsSchedulerData *esdp) { break; } erts_print(to, to_arg, "Length: %d\n", - erts_atomic32_read_dirty(&esdp->run_queue->procs.prio_info[i].len)); + erts_atomic32_read_dirty(&run_queue->procs.prio_info[i].len)); } erts_print(to, to_arg, "Run Queue Port Length: %d\n", - erts_atomic32_read_dirty(&esdp->run_queue->ports.info.len)); + erts_atomic32_read_dirty(&run_queue->ports.info.len)); - flg = erts_atomic32_read_dirty(&esdp->run_queue->flags); + flg = erts_atomic32_read_dirty(&run_queue->flags); erts_print(to, to_arg, "Run Queue Flags: "); for (i = 0; i < ERTS_RUNQ_FLG_MAX && flg; i++) { erts_aint32_t chk = (1 << i); @@ -13540,9 +13535,15 @@ erts_print_scheduler_info(fmtfn_t to, void *to_arg, ErtsSchedulerData *esdp) { } } erts_print(to, to_arg, "\n"); +} + + +static void print_current_process_info(fmtfn_t to, void *to_arg, + ErtsSchedulerData* esdp) +{ + Process *p = esdp->current_process; + erts_aint32_t flg; - /* This *MUST* to be the last information in scheduler block */ - p = esdp->current_process; erts_print(to, to_arg, "Current Process: "); if (esdp->current_process && !(ERTS_TRACE_FLAGS(p) & F_SENSITIVE)) { flg = erts_atomic32_read_dirty(&p->state); diff --git a/erts/emulator/beam/erl_process.h b/erts/emulator/beam/erl_process.h index 16b3526208..ebf990c837 100644 --- a/erts/emulator/beam/erl_process.h +++ b/erts/emulator/beam/erl_process.h @@ -105,13 +105,13 @@ struct saved_calls { }; extern Export exp_send, exp_receive, exp_timeout; -extern int erts_sched_compact_load; -extern int erts_sched_balance_util; -extern Uint erts_no_schedulers; -extern Uint erts_no_total_schedulers; -extern Uint erts_no_dirty_cpu_schedulers; -extern Uint erts_no_dirty_io_schedulers; -extern Uint erts_no_run_queues; +extern int ERTS_WRITE_UNLIKELY(erts_sched_compact_load); +extern int ERTS_WRITE_UNLIKELY(erts_sched_balance_util); +extern Uint ERTS_WRITE_UNLIKELY(erts_no_schedulers); +extern Uint ERTS_WRITE_UNLIKELY(erts_no_total_schedulers); +extern Uint ERTS_WRITE_UNLIKELY(erts_no_dirty_cpu_schedulers); +extern Uint ERTS_WRITE_UNLIKELY(erts_no_dirty_io_schedulers); +extern Uint ERTS_WRITE_UNLIKELY(erts_no_run_queues); extern int erts_sched_thread_suggested_stack_size; extern int erts_dcpu_sched_thread_suggested_stack_size; extern int erts_dio_sched_thread_suggested_stack_size; @@ -244,6 +244,9 @@ extern int erts_dio_sched_thread_suggested_stack_size; (erts_aint32_t) (MSK), \ (erts_aint32_t) (FLGS))) +#define ERTS_RUNQ_POINTER_MASK (~((erts_aint_t) 3)) +#define ERTS_RUNQ_BOUND_FLAG ((erts_aint_t) 1) + typedef enum { ERTS_SCHDLR_SSPND_DONE_MSCHED_BLOCKED, ERTS_SCHDLR_SSPND_DONE_NMSCHED_BLOCKED, @@ -522,7 +525,7 @@ typedef union { char align[ERTS_ALC_CACHE_LINE_ALIGN_SIZE(sizeof(ErtsRunQueue))]; } ErtsAlignedRunQueue; -extern ErtsAlignedRunQueue *erts_aligned_run_queues; +extern ErtsAlignedRunQueue * ERTS_WRITE_UNLIKELY(erts_aligned_run_queues); #define ERTS_PROC_REDUCTIONS_EXECUTED(SD, RQ, PRIO, REDS, AREDS)\ do { \ @@ -675,9 +678,9 @@ typedef union { char align[ERTS_ALC_CACHE_LINE_ALIGN_SIZE(sizeof(ErtsSchedulerData))]; } ErtsAlignedSchedulerData; -extern ErtsAlignedSchedulerData *erts_aligned_scheduler_data; -extern ErtsAlignedSchedulerData *erts_aligned_dirty_cpu_scheduler_data; -extern ErtsAlignedSchedulerData *erts_aligned_dirty_io_scheduler_data; +extern ErtsAlignedSchedulerData * ERTS_WRITE_UNLIKELY(erts_aligned_scheduler_data); +extern ErtsAlignedSchedulerData * ERTS_WRITE_UNLIKELY(erts_aligned_dirty_cpu_scheduler_data); +extern ErtsAlignedSchedulerData * ERTS_WRITE_UNLIKELY(erts_aligned_dirty_io_scheduler_data); #if defined(ERTS_ENABLE_LOCK_CHECK) @@ -1167,14 +1170,14 @@ void erts_check_for_holes(Process* p); #define ERTS_PSFLG_RUNNING ERTS_PSFLG_BIT(9) #define ERTS_PSFLG_SUSPENDED ERTS_PSFLG_BIT(10) #define ERTS_PSFLG_GC ERTS_PSFLG_BIT(11) -#define ERTS_PSFLG_BOUND ERTS_PSFLG_BIT(12) +/* #define ERTS_PSFLG_ ERTS_PSFLG_BIT(12) */ #define ERTS_PSFLG_TRAP_EXIT ERTS_PSFLG_BIT(13) #define ERTS_PSFLG_ACTIVE_SYS ERTS_PSFLG_BIT(14) #define ERTS_PSFLG_RUNNING_SYS ERTS_PSFLG_BIT(15) #define ERTS_PSFLG_PROXY ERTS_PSFLG_BIT(16) #define ERTS_PSFLG_DELAYED_SYS ERTS_PSFLG_BIT(17) #define ERTS_PSFLG_OFF_HEAP_MSGQ ERTS_PSFLG_BIT(18) -#define ERTS_PSFLG_ON_HEAP_MSGQ ERTS_PSFLG_BIT(19) +/* #define ERTS_PSFLG_ ERTS_PSFLG_BIT(19) */ #define ERTS_PSFLG_DIRTY_CPU_PROC ERTS_PSFLG_BIT(20) #define ERTS_PSFLG_DIRTY_IO_PROC ERTS_PSFLG_BIT(21) #define ERTS_PSFLG_DIRTY_ACTIVE_SYS ERTS_PSFLG_BIT(22) @@ -1269,7 +1272,7 @@ void erts_check_for_holes(Process* p); #define SPO_OFF_HEAP_MSGQ 16 #define SPO_ON_HEAP_MSGQ 32 -extern int erts_default_spo_flags; +extern int ERTS_WRITE_UNLIKELY(erts_default_spo_flags); /* * The following struct contains options for a process to be spawned. @@ -1325,10 +1328,10 @@ extern erts_rwmtx_t erts_cpu_bind_rwmtx; ** erts_system_monitor must be != NIL, to allow testing on just ** the erts_system_monitor_* variables. */ -extern Eterm erts_system_monitor; -extern Uint erts_system_monitor_long_gc; -extern Uint erts_system_monitor_long_schedule; -extern Uint erts_system_monitor_large_heap; +extern Eterm ERTS_WRITE_UNLIKELY(erts_system_monitor); +extern Uint ERTS_WRITE_UNLIKELY(erts_system_monitor_long_gc); +extern Uint ERTS_WRITE_UNLIKELY(erts_system_monitor_long_schedule); +extern Uint ERTS_WRITE_UNLIKELY(erts_system_monitor_large_heap); struct erts_system_monitor_flags_t { unsigned int busy_port : 1; unsigned int busy_dist_port : 1; @@ -1545,6 +1548,7 @@ Uint64 erts_step_proc_interval(void); ErtsProcList *erts_proclist_create(Process *); ErtsProcList *erts_proclist_copy(ErtsProcList *); void erts_proclist_destroy(ErtsProcList *); +void erts_proclist_dump(fmtfn_t to, void *to_arg, ErtsProcList*); ERTS_GLB_INLINE int erts_proclist_same(ErtsProcList *, Process *); ERTS_GLB_INLINE void erts_proclist_store_first(ErtsProcList **, ErtsProcList *); @@ -1791,6 +1795,7 @@ void erts_stack_dump(fmtfn_t to, void *to_arg, Process *); void erts_limited_stack_trace(fmtfn_t to, void *to_arg, Process *); void erts_program_counter_info(fmtfn_t to, void *to_arg, Process *); void erts_print_scheduler_info(fmtfn_t to, void *to_arg, ErtsSchedulerData *esdp); +void erts_print_run_queue_info(fmtfn_t, void *to_arg, ErtsRunQueue*); void erts_dump_extended_process_state(fmtfn_t to, void *to_arg, erts_aint32_t psflg); void erts_dump_process_state(fmtfn_t to, void *to_arg, erts_aint32_t psflg); @@ -2166,7 +2171,12 @@ ERTS_GLB_INLINE int erts_is_scheduler_bound(ErtsSchedulerData *esdp); ERTS_GLB_INLINE Process *erts_get_current_process(void); ERTS_GLB_INLINE Eterm erts_get_current_pid(void); ERTS_GLB_INLINE Uint erts_get_scheduler_id(void); -ERTS_GLB_INLINE ErtsRunQueue *erts_get_runq_proc(Process *p); +ERTS_GLB_INLINE void erts_init_runq_proc(Process *p, ErtsRunQueue *rq, int bnd); +ERTS_GLB_INLINE ErtsRunQueue *erts_set_runq_proc(Process *p, ErtsRunQueue *rq, int *boundp); +ERTS_GLB_INLINE int erts_try_change_runq_proc(Process *p, ErtsRunQueue *rq); +ERTS_GLB_INLINE ErtsRunQueue *erts_bind_runq_proc(Process *p, int bind); +ERTS_GLB_INLINE int erts_proc_runq_is_bound(Process *p); +ERTS_GLB_INLINE ErtsRunQueue *erts_get_runq_proc(Process *p, int *boundp); ERTS_GLB_INLINE ErtsRunQueue *erts_get_runq_current(ErtsSchedulerData *esdp); ERTS_GLB_INLINE void erts_runq_lock(ErtsRunQueue *rq); ERTS_GLB_INLINE int erts_runq_trylock(ErtsRunQueue *rq); @@ -2246,11 +2256,144 @@ Uint erts_get_scheduler_id(void) return esdp ? esdp->no : (Uint) 0; } +/** + * Init run-queue of process. + * + * @param p[in,out] Process + * @param rq[in] Run-queue that process will be assigned to + * @param bnd[in,out] If non-zero binds process to run-queue. + */ + +ERTS_GLB_INLINE void +erts_init_runq_proc(Process *p, ErtsRunQueue *rq, int bnd) +{ + erts_aint_t rqint = (erts_aint_t) rq; + if (bnd) + rqint |= ERTS_RUNQ_BOUND_FLAG; + erts_atomic_init_nob(&p->run_queue, rqint); +} + +/** + * Forcibly set run-queue of process. + * + * @param p[in,out] Process + * @param rq[in] Run-queue that process will be assigned to + * @param bndp[in,out] Pointer to integer. On input non-zero + * value causes the process to be bound to + * the run-queue. On output, indicating + * wether process previously was bound or + * not. + * @return Previous run-queue. + */ + +ERTS_GLB_INLINE ErtsRunQueue * +erts_set_runq_proc(Process *p, ErtsRunQueue *rq, int *bndp) +{ + erts_aint_t rqint = (erts_aint_t) rq; + ASSERT(bndp); + ASSERT(rq); + if (*bndp) + rqint |= ERTS_RUNQ_BOUND_FLAG; + rqint = erts_atomic_xchg_nob(&p->run_queue, rqint); + *bndp = (int) (rqint & ERTS_RUNQ_BOUND_FLAG); + return (ErtsRunQueue *) (rqint & ERTS_RUNQ_POINTER_MASK); +} + +/** + * Try to change run-queue assignment of a process. + * + * @param p[in,out] Process + * @param rq[int] Run-queue that process will be assigned to + * @return Non-zero if the run-queue assignment was + * successfully changed. + */ + +ERTS_GLB_INLINE int +erts_try_change_runq_proc(Process *p, ErtsRunQueue *rq) +{ + erts_aint_t old_rqint, new_rqint; + + new_rqint = (erts_aint_t) rq; + old_rqint = (erts_aint_t) erts_atomic_read_nob(&p->run_queue); + while (1) { + erts_aint_t act_rqint; + + if (old_rqint & ERTS_RUNQ_BOUND_FLAG) + return 0; + + act_rqint = erts_atomic_cmpxchg_nob(&p->run_queue, + new_rqint, + old_rqint); + if (act_rqint == old_rqint) + return !0; + } +} + +/** + * + * Bind or unbind process to/from currently used run-queue. + * + * @param p Process + * @param bind Bind if non-zero; otherwise unbind + * @return Pointer to previously bound run-queue, + * or NULL if previously unbound + */ + +ERTS_GLB_INLINE ErtsRunQueue * +erts_bind_runq_proc(Process *p, int bind) +{ + erts_aint_t rqint; + if (bind) + rqint = erts_atomic_read_bor_nob(&p->run_queue, + ERTS_RUNQ_BOUND_FLAG); + else + rqint = erts_atomic_read_band_nob(&p->run_queue, + ~ERTS_RUNQ_BOUND_FLAG); + if (rqint & ERTS_RUNQ_BOUND_FLAG) + return (ErtsRunQueue *) (rqint & ERTS_RUNQ_POINTER_MASK); + else + return NULL; +} + +/** + * Determine wether a process is bound to a run-queue or not. + * + * @return Returns a non-zero value if bound, + * and zero of not bound. + */ + +ERTS_GLB_INLINE int +erts_proc_runq_is_bound(Process *p) +{ + erts_aint_t rqint = erts_atomic_read_nob(&p->run_queue); + return (int) (rqint & ERTS_RUNQ_BOUND_FLAG); +} + +/** + * Set run-queue of process. + * + * @param p[in,out] Process + * @param bndp[out] Pointer to integer. If non-NULL pointer, + * the integer will be set to a non-zero + * value if the process is bound to the + * run-queue. + * @return Pointer to the normal run-queue that + * the process currently is assigend to. + * A process is always assigned to a + * normal run-queue. + */ + ERTS_GLB_INLINE ErtsRunQueue * -erts_get_runq_proc(Process *p) +erts_get_runq_proc(Process *p, int *bndp) { - ASSERT(ERTS_AINT_NULL != erts_atomic_read_nob(&p->run_queue)); - return (ErtsRunQueue *) erts_atomic_read_nob(&p->run_queue); + erts_aint_t rqint = erts_atomic_read_nob(&p->run_queue); + ErtsRunQueue *rq; + if (bndp) + *bndp = (int) (rqint & ERTS_RUNQ_BOUND_FLAG); + rqint &= ERTS_RUNQ_POINTER_MASK; + rq = (ErtsRunQueue *) rqint; + ASSERT(rq); + return rq; } ERTS_GLB_INLINE ErtsRunQueue * diff --git a/erts/emulator/beam/erl_process_dict.c b/erts/emulator/beam/erl_process_dict.c index 3c80f0e0f6..aee88841ae 100644 --- a/erts/emulator/beam/erl_process_dict.c +++ b/erts/emulator/beam/erl_process_dict.c @@ -79,6 +79,8 @@ /* Array access macro */ #define ARRAY_GET(PDict, Index) (ASSERT((Index) < (PDict)->arraySize), \ (PDict)->data[Index]) +#define ARRAY_GET_PTR(PDict, Index) (ASSERT((Index) < (PDict)->arraySize), \ + &(PDict)->data[Index]) #define ARRAY_PUT(PDict, Index, Val) (ASSERT((Index) < (PDict)->arraySize), \ (PDict)->data[Index] = (Val)) @@ -92,7 +94,7 @@ static void pd_hash_erase_all(Process *p); static Eterm pd_hash_get_with_hval(Process *p, Eterm bucket, Eterm id); static Eterm pd_hash_get_keys(Process *p, Eterm value); static Eterm pd_hash_get_all_keys(Process *p, ProcDict *pd); -static Eterm pd_hash_get_all(Process *p, ProcDict *pd); +static Eterm pd_hash_get_all(Process *p, ProcDict *pd, int keep_dict); static Eterm pd_hash_put(Process *p, Eterm id, Eterm value); static void shrink(Process *p, Eterm* ret); @@ -281,7 +283,7 @@ BIF_RETTYPE get_0(BIF_ALIST_0) { Eterm ret; PD_CHECK(BIF_P->dictionary); - ret = pd_hash_get_all(BIF_P, BIF_P->dictionary); + ret = pd_hash_get_all(BIF_P, BIF_P->dictionary, 1); PD_CHECK(BIF_P->dictionary); BIF_RET(ret); } @@ -329,7 +331,7 @@ BIF_RETTYPE erase_0(BIF_ALIST_0) { Eterm ret; PD_CHECK(BIF_P->dictionary); - ret = pd_hash_get_all(BIF_P, BIF_P->dictionary); + ret = pd_hash_get_all(BIF_P, BIF_P->dictionary, 0); pd_hash_erase_all(BIF_P); PD_CHECK(BIF_P->dictionary); BIF_RET(ret); @@ -541,29 +543,46 @@ static Eterm pd_hash_get_keys(Process *p, Eterm value) static Eterm -pd_hash_get_all(Process *p, ProcDict *pd) +pd_hash_get_all(Process *p, ProcDict *pd, int keep_dict) { Eterm* hp; + Eterm* tp; Eterm res = NIL; Eterm tmp, tmp2; unsigned int i; unsigned int num; + Uint need; if (pd == NULL) { return res; } num = HASH_RANGE(pd); - hp = HAlloc(p, pd->numElements * 2); - + + /* + * If this is not erase/0, then must copy all key-value tuples + * as they may be mutated by put/2. + */ + need = pd->numElements * (keep_dict ? 2+3 : 2); + hp = HAlloc(p, need); + for (i = 0; i < num; ++i) { tmp = ARRAY_GET(pd, i); if (is_boxed(tmp)) { - ASSERT(is_tuple(tmp)); + if (keep_dict) { + tp = tuple_val(tmp); + tmp = TUPLE2(hp, tp[1], tp[2]); + hp += 3; + } res = CONS(hp, tmp, res); hp += 2; } else if (is_list(tmp)) { while (tmp != NIL) { tmp2 = TCAR(tmp); + if (keep_dict) { + tp = tuple_val(tmp2); + tmp2 = TUPLE2(hp, tp[1], tp[2]); + hp += 3; + } res = CONS(hp, tmp2, res); hp += 2; tmp = TCDR(tmp); @@ -577,11 +596,14 @@ static Eterm pd_hash_put(Process *p, Eterm id, Eterm value) { unsigned int hval; Eterm *hp; + Eterm *tp; + Eterm *bucket; Eterm tpl; Eterm old; + Eterm old_val = am_undefined; Eterm tmp; int needed; - int i = 0; + int new_key = 1; #ifdef DEBUG Eterm *hp_limit; #endif @@ -595,7 +617,8 @@ static Eterm pd_hash_put(Process *p, Eterm id, Eterm value) p->dictionary->numElements = 0; } hval = pd_hash_value(p->dictionary, id); - old = ARRAY_GET(p->dictionary, hval); + bucket = ARRAY_GET_PTR(p->dictionary, hval); + old = *bucket; /* * Calculate the number of heap words needed and garbage @@ -603,32 +626,49 @@ static Eterm pd_hash_put(Process *p, Eterm id, Eterm value) */ needed = 3; /* {Key,Value} tuple */ if (is_boxed(old)) { - /* - * We don't want to compare keys twice, so we'll always - * reserve the space for two CONS cells. - */ - needed += 2+2; + ASSERT(is_tuple(old)); + tp = tuple_val(old); + if (EQ(tp[1], id)) { + old_val = tp[2]; + if (is_immed(value)) { + tp[2] = value; /* DESTRUCTIVE HEAP ASSIGNMENT */ + return old_val; + } + new_key = 0; + } + else { + needed += 2+2; + } } else if (is_list(old)) { - i = 0; - for (tmp = old; tmp != NIL && !EQ(tuple_val(TCAR(tmp))[1], id); tmp = TCDR(tmp)) { - ++i; - } - if (is_nil(tmp)) { - i = -1; - needed += 2; - } else { - needed += 2*(i+1); + Eterm* prev_cdr = bucket; + + needed += 2; + for (tmp = old; tmp != NIL; prev_cdr = &TCDR(tmp), tmp = *prev_cdr) { + tp = tuple_val(TCAR(tmp)); + if (EQ(tp[1], id)) { + old_val = tp[2]; + if (is_immed(value)) { + tp[2] = value; /* DESTRUCTIVE HEAP ASSIGNMENT */ + return old_val; + } + new_key = 0; + /* Unlink old {Key,Value} from list */ + *prev_cdr = TCDR(tmp); /* maybe DESTRUCTIVE HEAP ASSIGNMENT */ + break; + } } } if (HeapWordsLeft(p) < needed) { Eterm root[3]; root[0] = id; root[1] = value; - root[2] = old; + root[2] = old_val; erts_garbage_collect(p, needed, root, 3); id = root[0]; value = root[1]; - old = root[2]; + old_val = root[2]; + ASSERT(bucket == ARRAY_GET_PTR(p->dictionary, hval)); + old = *bucket; } #ifdef DEBUG hp_limit = p->htop + needed; @@ -644,66 +684,29 @@ static Eterm pd_hash_put(Process *p, Eterm id, Eterm value) * Update the dictionary. */ if (is_nil(old)) { - ARRAY_PUT(p->dictionary, hval, tpl); - ++(p->dictionary->numElements); + *bucket = tpl; } else if (is_boxed(old)) { ASSERT(is_tuple(old)); - if (EQ(tuple_val(old)[1],id)) { - ARRAY_PUT(p->dictionary, hval, tpl); - return tuple_val(old)[2]; + if (!new_key) { + ASSERT(EQ(tuple_val(old)[1],id)); + *bucket = tpl; + return old_val; } else { hp = HeapOnlyAlloc(p, 4); tmp = CONS(hp, old, NIL); hp += 2; - ++(p->dictionary->numElements); - ARRAY_PUT(p->dictionary, hval, CONS(hp, tpl, tmp)); + *bucket = CONS(hp, tpl, tmp); hp += 2; ASSERT(hp <= hp_limit); } } else if (is_list(old)) { - if (i == -1) { - /* - * New key. Simply prepend the tuple to the beginning of the list. - */ - hp = HeapOnlyAlloc(p, 2); - ARRAY_PUT(p->dictionary, hval, CONS(hp, tpl, old)); - hp += 2; - ASSERT(hp <= hp_limit); - ++(p->dictionary->numElements); - } else { - /* - * i = Number of CDRs to skip to reach the changed element in the list. - * - * Replace old value in list. To avoid pointers from the old generation - * to the new, we must rebuild the list from the beginning up to and - * including the changed element. - */ - Eterm nlist; - int j; - - hp = HeapOnlyAlloc(p, (i+1)*2); - - /* Find the list element to change. */ - for (j = 0, nlist = old; j < i; j++, nlist = TCDR(nlist)) { - ; - } - ASSERT(EQ(tuple_val(TCAR(nlist))[1], id)); - nlist = TCDR(nlist); /* Unchanged part of list. */ - - /* Rebuild list before the updated element. */ - for (tmp = old; i-- > 0; tmp = TCDR(tmp)) { - nlist = CONS(hp, TCAR(tmp), nlist); - hp += 2; - } - ASSERT(EQ(tuple_val(TCAR(tmp))[1], id)); - - /* Put the updated element first in the new list. */ - nlist = CONS(hp, tpl, nlist); - hp += 2; - ASSERT(hp <= hp_limit); - ARRAY_PUT(p->dictionary, hval, nlist); - return tuple_val(TCAR(tmp))[2]; - } + /* + * Simply prepend the tuple to the beginning of the list. + */ + hp = HeapOnlyAlloc(p, 2); + *bucket = CONS(hp, tpl, *bucket); + hp += 2; + ASSERT(hp <= hp_limit); } else { #ifdef DEBUG erts_fprintf(stderr, @@ -714,10 +717,13 @@ static Eterm pd_hash_put(Process *p, Eterm id, Eterm value) erts_exit(ERTS_ERROR_EXIT, "Damaged process dictionary found during put/2."); } + + p->dictionary->numElements += new_key; + if (HASH_RANGE(p->dictionary) <= p->dictionary->numElements) { grow(p); } - return am_undefined; + return old_val; } /* diff --git a/erts/emulator/beam/erl_process_dump.c b/erts/emulator/beam/erl_process_dump.c index 0b7f361622..05e7bcdea2 100644 --- a/erts/emulator/beam/erl_process_dump.c +++ b/erts/emulator/beam/erl_process_dump.c @@ -51,6 +51,8 @@ static void stack_trace_dump(fmtfn_t to, void *to_arg, Eterm* sp); static void print_function_from_pc(fmtfn_t to, void *to_arg, BeamInstr* x); static void heap_dump(fmtfn_t to, void *to_arg, Eterm x); static void dump_binaries(fmtfn_t to, void *to_arg, Binary* root); +void erts_print_base64(fmtfn_t to, void *to_arg, + byte* src, Uint size); static void dump_externally(fmtfn_t to, void *to_arg, Eterm term); static void mark_literal(Eterm* ptr); static void init_literal_areas(void); @@ -76,8 +78,17 @@ erts_deep_process_dump(fmtfn_t to, void *to_arg) Process *p = erts_pix2proc(i); if (p && p->i != ENULL) { erts_aint32_t state = erts_atomic32_read_acqb(&p->state); - if (!(state & (ERTS_PSFLG_EXITING|ERTS_PSFLG_GC))) - dump_process_info(to, to_arg, p); + if (state & ERTS_PSFLG_EXITING) + continue; + if (state & ERTS_PSFLG_GC) { + ErtsSchedulerData *sdp = erts_get_scheduler_data(); + if (!sdp || p != sdp->current_process) + continue; + + /* We want to dump the garbing process that caused the dump */ + } + + dump_process_info(to, to_arg, p); } } @@ -133,9 +144,12 @@ dump_process_info(fmtfn_t to, void *to_arg, Process *p) ErtsMessage* mp; int yreg = -1; + if (ERTS_TRACE_FLAGS(p) & F_SENSITIVE) + return; + ERTS_MSGQ_MV_INQ2PRIVQ(p); - if ((ERTS_TRACE_FLAGS(p) & F_SENSITIVE) == 0 && p->msg.first) { + if (p->msg.first) { erts_print(to, to_arg, "=proc_messages:%T\n", p->common.id); for (mp = p->msg.first; mp != NULL; mp = mp->next) { Eterm mesg = ERL_MESSAGE_TERM(mp); @@ -150,38 +164,34 @@ dump_process_info(fmtfn_t to, void *to_arg, Process *p) } } - if ((ERTS_TRACE_FLAGS(p) & F_SENSITIVE) == 0) { - if (p->dictionary) { - erts_print(to, to_arg, "=proc_dictionary:%T\n", p->common.id); - erts_deep_dictionary_dump(to, to_arg, - p->dictionary, dump_element_nl); - } + if (p->dictionary) { + erts_print(to, to_arg, "=proc_dictionary:%T\n", p->common.id); + erts_deep_dictionary_dump(to, to_arg, + p->dictionary, dump_element_nl); } - if ((ERTS_TRACE_FLAGS(p) & F_SENSITIVE) == 0) { - erts_print(to, to_arg, "=proc_stack:%T\n", p->common.id); - for (sp = p->stop; sp < STACK_START(p); sp++) { - yreg = stack_element_dump(to, to_arg, sp, yreg); - } + erts_print(to, to_arg, "=proc_stack:%T\n", p->common.id); + for (sp = p->stop; sp < STACK_START(p); sp++) { + yreg = stack_element_dump(to, to_arg, sp, yreg); + } - erts_print(to, to_arg, "=proc_heap:%T\n", p->common.id); - for (sp = p->stop; sp < STACK_START(p); sp++) { - Eterm term = *sp; - - if (!is_catch(term) && !is_CP(term)) { - heap_dump(to, to_arg, term); - } - } - for (mp = p->msg.first; mp != NULL; mp = mp->next) { - Eterm mesg = ERL_MESSAGE_TERM(mp); - if (is_value(mesg)) - heap_dump(to, to_arg, mesg); - mesg = ERL_MESSAGE_TOKEN(mp); - heap_dump(to, to_arg, mesg); - } - if (p->dictionary) { - erts_deep_dictionary_dump(to, to_arg, p->dictionary, heap_dump); - } + erts_print(to, to_arg, "=proc_heap:%T\n", p->common.id); + for (sp = p->stop; sp < STACK_START(p); sp++) { + Eterm term = *sp; + + if (!is_catch(term) && !is_CP(term)) { + heap_dump(to, to_arg, term); + } + } + for (mp = p->msg.first; mp != NULL; mp = mp->next) { + Eterm mesg = ERL_MESSAGE_TERM(mp); + if (is_value(mesg)) + heap_dump(to, to_arg, mesg); + mesg = ERL_MESSAGE_TOKEN(mp); + heap_dump(to, to_arg, mesg); + } + if (p->dictionary) { + erts_deep_dictionary_dump(to, to_arg, p->dictionary, heap_dump); } } @@ -193,6 +203,7 @@ dump_dist_ext(fmtfn_t to, void *to_arg, ErtsDistExternal *edep) else { byte *e; size_t sz; + if (!(edep->flags & ERTS_DIST_EXT_ATOM_TRANS_TAB)) erts_print(to, to_arg, "D0:"); else { @@ -210,12 +221,18 @@ dump_dist_ext(fmtfn_t to, void *to_arg, ErtsDistExternal *edep) else { ASSERT(*e == VERSION_MAGIC); } - erts_print(to, to_arg, "E%X:", sz); - if (edep->flags & ERTS_DIST_EXT_DFLAG_HDR) - erts_print(to, to_arg, "%02X", VERSION_MAGIC); - while (e < edep->ext_endp) - erts_print(to, to_arg, "%02X", *e++); + if (edep->flags & ERTS_DIST_EXT_DFLAG_HDR) { + byte sbuf[3]; + int i = 0; + + sbuf[i++] = VERSION_MAGIC; + while (i < sizeof(sbuf) && e < edep->ext_endp) { + sbuf[i++] = *e++; + } + erts_print_base64(to, to_arg, sbuf, i); + } + erts_print_base64(to, to_arg, e, edep->ext_endp - e); } } @@ -444,16 +461,13 @@ heap_dump(fmtfn_t to, void *to_arg, Eterm x) } else if (is_binary_header(hdr)) { Uint tag = thing_subtag(hdr); Uint size = binary_size(x); - Uint i; if (tag == HEAP_BINARY_SUBTAG) { byte* p; erts_print(to, to_arg, "Yh%X:", size); p = binary_bytes(x); - for (i = 0; i < size; i++) { - erts_print(to, to_arg, "%02X", p[i]); - } + erts_print_base64(to, to_arg, p, size); } else if (tag == REFC_BINARY_SUBTAG) { ProcBin* pb = (ProcBin *) binary_val(x); Binary* val = pb->val; @@ -596,16 +610,13 @@ static void dump_binaries(fmtfn_t to, void *to_arg, Binary* current) { while (current) { - long i; - long size = current->orig_size; + SWord size = current->orig_size; byte* bytes = (byte*) current->orig_bytes; erts_print(to, to_arg, "=binary:" PTR_FMT "\n", current); erts_print(to, to_arg, "%X:", size); - for (i = 0; i < size; i++) { - erts_print(to, to_arg, "%02X", bytes[i]); - } - erts_putc(to, to_arg, '\n'); + erts_print_base64(to, to_arg, bytes, size); + erts_putc(to, to_arg, '\n'); current = (Binary *) current->intern.flags; } } @@ -644,9 +655,7 @@ dump_externally(fmtfn_t to, void *to_arg, Eterm term) s = p = sbuf; erts_encode_ext(term, &p); erts_print(to, to_arg, "E%X:", p-s); - while (s < p) { - erts_print(to, to_arg, "%02X", *s++); - } + erts_print_base64(to, to_arg, sbuf, p-s); } /* @@ -802,16 +811,13 @@ dump_module_literals(fmtfn_t to, void *to_arg, ErtsLiteralArea* lit_area) } else if (is_binary_header(w)) { Uint tag = thing_subtag(w); Uint size = binary_size(term); - Uint i; if (tag == HEAP_BINARY_SUBTAG) { byte* p; erts_print(to, to_arg, "Yh%X:", size); p = binary_bytes(term); - for (i = 0; i < size; i++) { - erts_print(to, to_arg, "%02X", p[i]); - } + erts_print_base64(to, to_arg, p, size); } else if (tag == REFC_BINARY_SUBTAG) { ProcBin* pb = (ProcBin *) binary_val(term); Binary* val = pb->val; @@ -1003,8 +1009,6 @@ erts_dump_extended_process_state(fmtfn_t to, void *to_arg, erts_aint32_t psflg) erts_print(to, to_arg, "SUSPENDED"); break; case ERTS_PSFLG_GC: erts_print(to, to_arg, "GC"); break; - case ERTS_PSFLG_BOUND: - erts_print(to, to_arg, "BOUND"); break; case ERTS_PSFLG_TRAP_EXIT: erts_print(to, to_arg, "TRAP_EXIT"); break; case ERTS_PSFLG_ACTIVE_SYS: @@ -1017,8 +1021,6 @@ erts_dump_extended_process_state(fmtfn_t to, void *to_arg, erts_aint32_t psflg) erts_print(to, to_arg, "DELAYED_SYS"); break; case ERTS_PSFLG_OFF_HEAP_MSGQ: erts_print(to, to_arg, "OFF_HEAP_MSGQ"); break; - case ERTS_PSFLG_ON_HEAP_MSGQ: - erts_print(to, to_arg, "ON_HEAP_MSGQ"); break; case ERTS_PSFLG_DIRTY_CPU_PROC: erts_print(to, to_arg, "DIRTY_CPU_PROC"); break; case ERTS_PSFLG_DIRTY_IO_PROC: diff --git a/erts/emulator/beam/erl_term.h b/erts/emulator/beam/erl_term.h index 5ec6b6b44b..3dd1c2555c 100644 --- a/erts/emulator/beam/erl_term.h +++ b/erts/emulator/beam/erl_term.h @@ -55,9 +55,6 @@ struct erl_node_; /* Declared in erl_node_tables.h */ #if defined(ARCH_64) # define TAG_PTR_MASK__ 0x7 # if !defined(ERTS_HAVE_OS_PHYSICAL_MEMORY_RESERVATION) -# ifdef HIPE -# error Hipe on 64-bit needs a real mmap as it does not support the literal tag -# endif # define TAG_LITERAL_PTR 0x4 # else # undef TAG_LITERAL_PTR @@ -316,7 +313,8 @@ _ET_DECLARE_CHECKED(Uint,header_arity,Eterm) #define MAX_ARITYVAL ((((Uint)1) << 24) - 1) #define ERTS_MAX_TUPLE_SIZE MAX_ARITYVAL -#define make_arityval(sz) _make_header((sz),_TAG_HEADER_ARITYVAL) +#define make_arityval(sz) (ASSERT((sz) <= MAX_ARITYVAL), \ + _make_header((sz),_TAG_HEADER_ARITYVAL)) #define is_arity_value(x) (((x) & _TAG_HEADER_MASK) == _TAG_HEADER_ARITYVAL) #define is_sane_arity_value(x) ((((x) & _TAG_HEADER_MASK) == _TAG_HEADER_ARITYVAL) && \ (((x) >> _HEADER_ARITY_OFFS) <= MAX_ARITYVAL)) diff --git a/erts/emulator/beam/erl_thr_progress.h b/erts/emulator/beam/erl_thr_progress.h index fa936b5707..8c029bcf99 100644 --- a/erts/emulator/beam/erl_thr_progress.h +++ b/erts/emulator/beam/erl_thr_progress.h @@ -28,7 +28,7 @@ * Author: Rickard Green */ -#if !defined(ERL_THR_PROGRESS_H__TSD_TYPE__) +#ifndef ERL_THR_PROGRESS_H__TSD_TYPE__ #define ERL_THR_PROGRESS_H__TSD_TYPE__ #include "sys.h" diff --git a/erts/emulator/beam/erl_trace.c b/erts/emulator/beam/erl_trace.c index 4b996d8fc2..018894f685 100644 --- a/erts/emulator/beam/erl_trace.c +++ b/erts/emulator/beam/erl_trace.c @@ -2533,7 +2533,7 @@ load_tracer_nif(const ErtsTracer tracer) for(i = 0; i < num_of_funcs; i++) { for (j = 0; j < NIF_TRACER_TYPES; j++) { - if (strcmp(tracers[j].name, funcs[i].name) == 0 && tracers[j].arity == funcs[i].arity) { + if (sys_strcmp(tracers[j].name, funcs[i].name) == 0 && tracers[j].arity == funcs[i].arity) { tracers[j].cb = &(funcs[i]); break; } @@ -2783,24 +2783,28 @@ is_tracer_enabled(Process* c_p, ErtsProcLocks c_p_locks, ASSERT(0); } - /* Only remove tracer on self() and ports */ + /* Only remove tracer on (self() or ports) AND we are on a normal scheduler */ if (is_internal_port(t_p->id) || (c_p && c_p->common.id == t_p->id)) { + ErtsSchedulerData *esdp = erts_get_scheduler_data(); ErtsProcLocks c_p_xlocks = 0; - if (is_internal_pid(t_p->id)) { - ERTS_LC_ASSERT(erts_proc_lc_my_proc_locks(c_p) & ERTS_PROC_LOCK_MAIN); - if (c_p_locks != ERTS_PROC_LOCKS_ALL) { - c_p_xlocks = ~c_p_locks & ERTS_PROC_LOCKS_ALL; - if (erts_proc_trylock(c_p, c_p_xlocks) == EBUSY) { - erts_proc_unlock(c_p, c_p_locks & ~ERTS_PROC_LOCK_MAIN); - erts_proc_lock(c_p, ERTS_PROC_LOCKS_ALL_MINOR); + if (esdp && !ERTS_SCHEDULER_IS_DIRTY(esdp)) { + if (is_internal_pid(t_p->id)) { + ERTS_LC_ASSERT(erts_proc_lc_my_proc_locks(c_p) & ERTS_PROC_LOCK_MAIN); + if (c_p_locks != ERTS_PROC_LOCKS_ALL) { + c_p_xlocks = ~c_p_locks & ERTS_PROC_LOCKS_ALL; + if (erts_proc_trylock(c_p, c_p_xlocks) == EBUSY) { + erts_proc_unlock(c_p, c_p_locks & ~ERTS_PROC_LOCK_MAIN); + erts_proc_lock(c_p, ERTS_PROC_LOCKS_ALL_MINOR); + } } } - } - erts_tracer_replace(t_p, erts_tracer_nil); - t_p->trace_flags &= ~TRACEE_FLAGS; - if (c_p_xlocks) - erts_proc_unlock(c_p, c_p_xlocks); + erts_tracer_replace(t_p, erts_tracer_nil); + t_p->trace_flags &= ~TRACEE_FLAGS; + + if (c_p_xlocks) + erts_proc_unlock(c_p, c_p_xlocks); + } } return 0; @@ -3015,7 +3019,7 @@ static void *tracer_alloc_fun(void* tmpl) ErtsTracerNif *obj = erts_alloc(ERTS_ALC_T_TRACER_NIF, sizeof(ErtsTracerNif) + sizeof(ErtsThrPrgrLaterOp)); - memcpy(obj, tmpl, sizeof(*obj)); + sys_memcpy(obj, tmpl, sizeof(*obj)); return obj; } diff --git a/erts/emulator/beam/erl_unicode.c b/erts/emulator/beam/erl_unicode.c index b7a5c45fea..8673e029e6 100644 --- a/erts/emulator/beam/erl_unicode.c +++ b/erts/emulator/beam/erl_unicode.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2008-2017. All Rights Reserved. + * Copyright Ericsson AB 2008-2018. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -144,7 +144,7 @@ static Eterm make_magic_bin_for_restart(Process *p, RestartContext *rc) cleanup_restart_context_bin); RestartContext *restartp = ERTS_MAGIC_BIN_DATA(mbp); Eterm *hp; - memcpy(restartp,rc,sizeof(RestartContext)); + sys_memcpy(restartp,rc,sizeof(RestartContext)); hp = HAlloc(p, ERTS_MAGIC_REF_THING_SIZE); return erts_mk_magic_ref(&hp, &MSO(p), mbp); } @@ -254,12 +254,12 @@ static Uint copy_utf8_bin(byte *target, byte *source, Uint size, ASSERT(need > 0); ASSERT(from_source > 0); if (size < from_source) { - memcpy(leftover + (*num_leftovers), source, size); + sys_memcpy(leftover + (*num_leftovers), source, size); *num_leftovers += size; return 0; } /* leftover has room for four bytes (see bif) */ - memcpy(leftover + (*num_leftovers),source,from_source); + sys_memcpy(leftover + (*num_leftovers),source,from_source); c = copy_utf8_bin(target, leftover, need, NULL, NULL, &tmp_err_pos, characters); if (tmp_err_pos != 0) { *err_pos = source; @@ -291,7 +291,7 @@ static Uint copy_utf8_bin(byte *target, byte *source, Uint size, size -= 2; copied += 2; } else if (((*source) & ((byte) 0xF0)) == 0xE0) { if (leftover && size < 3) { - memcpy(leftover, source, (int) size); + sys_memcpy(leftover, source, (int) size); *num_leftovers = (int) size; break; } @@ -313,7 +313,7 @@ static Uint copy_utf8_bin(byte *target, byte *source, Uint size, size -= 3; copied += 3; } else if (((*source) & ((byte) 0xF8)) == 0xF0) { if (leftover && size < 4) { - memcpy(leftover, source, (int) size); + sys_memcpy(leftover, source, (int) size); *num_leftovers = (int) size; break; } @@ -1158,14 +1158,15 @@ static ERTS_INLINE int analyze_utf8(byte *source, Uint size, byte **err_pos, Uint *num_chars, int *left, Sint *num_latin1_chars, Uint max_chars) { + int res = ERTS_UTF8_OK; Uint latin1_count; int is_latin1; + Uint nchars = 0; *err_pos = source; if (num_latin1_chars) { is_latin1 = 1; latin1_count = 0; } - *num_chars = 0; while (size) { if (((*source) & ((byte) 0x80)) == 0) { source++; @@ -1174,11 +1175,13 @@ analyze_utf8(byte *source, Uint size, byte **err_pos, Uint *num_chars, int *left latin1_count++; } else if (((*source) & ((byte) 0xE0)) == 0xC0) { if (size < 2) { - return ERTS_UTF8_INCOMPLETE; + res = ERTS_UTF8_INCOMPLETE; + break; } if (((source[1] & ((byte) 0xC0)) != 0x80) || ((*source) < 0xC2) /* overlong */) { - return ERTS_UTF8_ERROR; + res = ERTS_UTF8_ERROR; + break; } if (num_latin1_chars) { latin1_count++; @@ -1189,16 +1192,19 @@ analyze_utf8(byte *source, Uint size, byte **err_pos, Uint *num_chars, int *left size -= 2; } else if (((*source) & ((byte) 0xF0)) == 0xE0) { if (size < 3) { - return ERTS_UTF8_INCOMPLETE; + res = ERTS_UTF8_INCOMPLETE; + break; } if (((source[1] & ((byte) 0xC0)) != 0x80) || ((source[2] & ((byte) 0xC0)) != 0x80) || (((*source) == 0xE0) && (source[1] < 0xA0)) /* overlong */ ) { - return ERTS_UTF8_ERROR; + res = ERTS_UTF8_ERROR; + break; } if ((((*source) & ((byte) 0xF)) == 0xD) && ((source[1] & 0x20) != 0)) { - return ERTS_UTF8_ERROR; + res = ERTS_UTF8_ERROR; + break; } source += 3; size -= 3; @@ -1206,37 +1212,47 @@ analyze_utf8(byte *source, Uint size, byte **err_pos, Uint *num_chars, int *left is_latin1 = 0; } else if (((*source) & ((byte) 0xF8)) == 0xF0) { if (size < 4) { - return ERTS_UTF8_INCOMPLETE; + res = ERTS_UTF8_INCOMPLETE; + break; } if (((source[1] & ((byte) 0xC0)) != 0x80) || ((source[2] & ((byte) 0xC0)) != 0x80) || ((source[3] & ((byte) 0xC0)) != 0x80) || (((*source) == 0xF0) && (source[1] < 0x90)) /* overlong */) { - return ERTS_UTF8_ERROR; + res = ERTS_UTF8_ERROR; + break; } if ((((*source) & ((byte)0x7)) > 0x4U) || ((((*source) & ((byte)0x7)) == 0x4U) && ((source[1] & ((byte)0x3F)) > 0xFU))) { - return ERTS_UTF8_ERROR; + res = ERTS_UTF8_ERROR; + break; } source += 4; size -= 4; if (num_latin1_chars) is_latin1 = 0; } else { - return ERTS_UTF8_ERROR; + res = ERTS_UTF8_ERROR; + break; } - ++(*num_chars); + ++nchars; *err_pos = source; - if (max_chars && size > 0 && *num_chars == max_chars) - return ERTS_UTF8_OK_MAX_CHARS; + if (max_chars && size > 0 && nchars == max_chars) { + res = ERTS_UTF8_OK_MAX_CHARS; + break; + } if (left && --(*left) <= 0 && size) { - return ERTS_UTF8_ANALYZE_MORE; + res = ERTS_UTF8_ANALYZE_MORE; + break; } } + + *num_chars = nchars; if (num_latin1_chars) *num_latin1_chars = is_latin1 ? latin1_count : -1; - return ERTS_UTF8_OK; + + return res; } int erts_analyze_utf8(byte *source, Uint size, @@ -1252,29 +1268,18 @@ int erts_analyze_utf8_x(byte *source, Uint size, return analyze_utf8(source, size, err_pos, num_chars, left, num_latin1_chars, max_chars); } -/* - * No errors should be able to occur - no overlongs, no malformed, no nothing - */ -static Eterm do_utf8_to_list(Process *p, Uint num, byte *bytes, Uint sz, - Uint left, - Uint *num_built, Uint *num_eaten, Eterm tail) +static ERTS_INLINE Eterm +make_list_from_utf8_buf(Eterm **hpp, Uint num, + byte *bytes, Uint sz, + Uint *num_built, Uint *num_eaten, + Eterm tail) { Eterm *hp; Eterm ret; + Uint left = num; byte *source, *ssource; Uint unipoint; - - ASSERT(num > 0); - if (left < num) { - if (left > 0) - num = left; - else - num = 1; - } - - *num_built = num; /* Always */ - - hp = HAlloc(p,num * 2); + hp = *hpp; ret = tail; source = bytes + sz; ssource = source; @@ -1302,20 +1307,97 @@ static Eterm do_utf8_to_list(Process *p, Uint num, byte *bytes, Uint sz, } ret = CONS(hp,make_small(unipoint),ret); hp += 2; - if (--num <= 0) { + if (--left <= 0) { break; } } + *hpp = hp; + *num_built = num; /* Always */ *num_eaten = (ssource - source); return ret; } +/* + * No errors should be able to occur - no overlongs, no malformed, no nothing + */ +static Eterm do_utf8_to_list(Process *p, Uint num, byte *bytes, Uint sz, + Uint left, + Uint *num_built, Uint *num_eaten, Eterm tail) +{ + Eterm *hp; + + ASSERT(num > 0); + if (left < num) { + if (left > 0) + num = left; + else + num = 1; + } + + hp = HAlloc(p,num * 2); + + return make_list_from_utf8_buf(&hp, num, bytes, sz, + num_built, num_eaten, + tail); +} Eterm erts_utf8_to_list(Process *p, Uint num, byte *bytes, Uint sz, Uint left, Uint *num_built, Uint *num_eaten, Eterm tail) { return do_utf8_to_list(p, num, bytes, sz, left, num_built, num_eaten, tail); } +Uint erts_atom_to_string_length(Eterm atom) +{ + Atom *ap; + + ASSERT(is_atom(atom)); + ap = atom_tab(atom_val(atom)); + + if (ap->latin1_chars >= 0) + return (Uint) ap->len; + else { + byte* err_pos; + Uint num_chars; +#ifdef DEBUG + int ares = +#endif + erts_analyze_utf8(ap->name, ap->len, &err_pos, &num_chars, NULL); + ASSERT(ares == ERTS_UTF8_OK); + + return num_chars; + } +} + +Eterm erts_atom_to_string(Eterm **hpp, Eterm atom) +{ + Atom *ap; + + ASSERT(is_atom(atom)); + ap = atom_tab(atom_val(atom)); + if (ap->latin1_chars >= 0) + return buf_to_intlist(hpp, (char*)ap->name, ap->len, NIL); + else { + Eterm res; + byte* err_pos; + Uint num_chars, num_built, num_eaten; +#ifdef DEBUG + Eterm *hp_start = *hpp; + int ares = +#endif + erts_analyze_utf8(ap->name, ap->len, &err_pos, &num_chars, NULL); + ASSERT(ares == ERTS_UTF8_OK); + + res = make_list_from_utf8_buf(hpp, num_chars, ap->name, ap->len, + &num_built, &num_eaten, NIL); + + ASSERT(num_built == num_chars); + ASSERT(num_eaten == ap->len); + ASSERT(*hpp - hp_start == 2*num_chars); + + return res; + } +} + static int is_candidate(Uint cp) { int index,pos; @@ -2026,7 +2108,7 @@ char *erts_convert_filename_to_encoding(Eterm name, char *statbuf, size_t statbu } else { name_buf = statbuf; } - memcpy(name_buf,bytes,size); + sys_memcpy(name_buf,bytes,size); name_buf[size]=0; } else { name_buf = erts_convert_filename_to_wchar(bytes, size, @@ -2083,18 +2165,9 @@ char* erts_convert_filename_to_wchar(byte* bytes, Uint size, return name_buf; } - -static int filename_len_16bit(byte *str) -{ - byte *p = str; - while(*p != '\0' || p[1] != '\0') { - p += 2; - } - return (p - str); -} -Eterm erts_convert_native_to_filename(Process *p, byte *bytes) +Eterm erts_convert_native_to_filename(Process *p, size_t size, byte *bytes) { - Uint size,num_chars; + Uint num_chars; Eterm *hp; byte *err_pos; Uint num_built; /* characters */ @@ -2108,7 +2181,6 @@ Eterm erts_convert_native_to_filename(Process *p, byte *bytes) case ERL_FILENAME_UTF8_MAC: mac = 1; case ERL_FILENAME_UTF8: - size = strlen((char *) bytes); if (size == 0) return NIL; if (erts_analyze_utf8(bytes,size,&err_pos,&num_chars,NULL) != ERTS_UTF8_OK) { @@ -2123,7 +2195,6 @@ Eterm erts_convert_native_to_filename(Process *p, byte *bytes) } return ret; case ERL_FILENAME_WIN_WCHAR: - size=filename_len_16bit(bytes); if ((size % 2) != 0) { /* Panic fixup to avoid crashing the emulator */ size--; hp = HAlloc(p, size+2); @@ -2146,7 +2217,6 @@ Eterm erts_convert_native_to_filename(Process *p, byte *bytes) goto noconvert; } noconvert: - size = strlen((char *) bytes); hp = HAlloc(p, 2 * size); return erts_bin_bytes_to_list(NIL, hp, bytes, size, 0); } @@ -2158,7 +2228,6 @@ Sint erts_native_filename_need(Eterm ioterm, int encoding) Eterm obj; DECLARE_ESTACK(stack); Sint need = 0; - int seen_null = 0; if (is_atom(ioterm)) { Atom* ap; @@ -2203,9 +2272,7 @@ Sint erts_native_filename_need(Eterm ioterm, int encoding) byte *name = ap->name; int len = ap->len; for (i = 0; i < len; i++) { - if (name[i] == 0) - seen_null = 1; - else if (seen_null) { + if (name[i] == 0) { need = -1; break; } @@ -2245,9 +2312,7 @@ L_Again: /* Restart with sublist, old listend was pushed on stack */ * Do not allow null in * the middle of filenames */ - if (x == 0) - seen_null = 1; - else if (seen_null) { + if (x == 0) { DESTROY_ESTACK(stack); return ((Sint) -1); } @@ -2580,7 +2645,6 @@ BIF_RETTYPE prim_file_internal_name2native_1(BIF_ALIST_1) BIF_ERROR(BIF_P,BADARG); } if (is_binary(BIF_ARG_1)) { - int seen_null = 0; byte *temp_alloc = NULL; byte *bytes; byte *err_pos; @@ -2597,8 +2661,6 @@ BIF_RETTYPE prim_file_internal_name2native_1(BIF_ALIST_1) for (i = 0; i < size; i++) { /* Don't allow null in the middle of filenames... */ if (bytes[i] == 0) - seen_null = 1; - else if (seen_null) goto bin_name_error; bin_p[i] = bytes[i]; } @@ -2617,8 +2679,6 @@ BIF_RETTYPE prim_file_internal_name2native_1(BIF_ALIST_1) while (size--) { /* Don't allow null in the middle of filenames... */ if (*bytes == 0) - seen_null = 1; - else if (seen_null) goto bin_name_error; *bin_p++ = *bytes++; *bin_p++ = 0; diff --git a/erts/emulator/beam/erl_unicode.h b/erts/emulator/beam/erl_unicode.h index e01eaa787e..31369fc8f9 100644 --- a/erts/emulator/beam/erl_unicode.h +++ b/erts/emulator/beam/erl_unicode.h @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2008-2016. All Rights Reserved. + * Copyright Ericsson AB 2008-2018. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -21,4 +21,7 @@ #ifndef _ERL_UNICODE_H #define _ERL_UNICODE_H +Uint erts_atom_to_string_length(Eterm atom); +Eterm erts_atom_to_string(Eterm **hpp, Eterm atom); + #endif /* _ERL_UNICODE_H */ diff --git a/erts/emulator/beam/erl_utils.h b/erts/emulator/beam/erl_utils.h index 44d8c85867..e4087e0ac8 100644 --- a/erts/emulator/beam/erl_utils.h +++ b/erts/emulator/beam/erl_utils.h @@ -91,7 +91,7 @@ Eterm erts_bld_tuple(Uint **hpp, Uint *szp, Uint arity, ...); #define erts_bld_tuple5(H,S,E1,E2,E3,E4,E5) erts_bld_tuple(H,S,5,E1,E2,E3,E4,E5) Eterm erts_bld_tuplev(Uint **hpp, Uint *szp, Uint arity, Eterm terms[]); Eterm erts_bld_string_n(Uint **hpp, Uint *szp, const char *str, Sint len); -#define erts_bld_string(hpp,szp,str) erts_bld_string_n(hpp,szp,str,strlen(str)) +#define erts_bld_string(hpp,szp,str) erts_bld_string_n(hpp,szp,str,sys_strlen(str)) Eterm erts_bld_list(Uint **hpp, Uint *szp, Sint length, Eterm terms[]); Eterm erts_bld_2tup_list(Uint **hpp, Uint *szp, Sint length, Eterm terms1[], Uint terms2[]); diff --git a/erts/emulator/beam/erl_vm.h b/erts/emulator/beam/erl_vm.h index 76980b5871..f8391fb665 100644 --- a/erts/emulator/beam/erl_vm.h +++ b/erts/emulator/beam/erl_vm.h @@ -69,7 +69,7 @@ # ifdef CHECK_FOR_HOLES # define INIT_HEAP_MEM(p,sz) erts_set_hole_marker(HEAP_TOP(p), (sz)) # else -# define INIT_HEAP_MEM(p,sz) memset(HEAP_TOP(p),0x01,(sz)*sizeof(Eterm*)) +# define INIT_HEAP_MEM(p,sz) sys_memset(HEAP_TOP(p),0x01,(sz)*sizeof(Eterm*)) # endif #else # define INIT_HEAP_MEM(p,sz) ((void)0) diff --git a/erts/emulator/beam/erlang_dtrace.d b/erts/emulator/beam/erlang_dtrace.d index 237889e0f5..d0b10e0306 100644 --- a/erts/emulator/beam/erlang_dtrace.d +++ b/erts/emulator/beam/erlang_dtrace.d @@ -634,72 +634,6 @@ provider erlang { */ probe aio_pool__get(char *, int); - /* Probes for efile_drv.c */ - - /** - * Entry into the efile_drv.c file I/O driver - * - * For a list of command numbers used by this driver, see the section - * "Guide to efile_drv.c probe arguments" in ../../../HOWTO/DTRACE.md. - * That section also contains explanation of the various integer and - * string arguments that may be present when any particular probe fires. - * - * NOTE: Not all Linux platforms (using SystemTap) can support - * arguments beyond arg9. - * - * - * TODO: Adding the port string, args[10], is a pain. Making that - * port string available to all the other efile_drv.c probes - * will be more pain. Is the pain worth it? If yes, then - * add them everywhere else and grit our teeth. If no, then - * rip it out. - * - * @param thread-id number of the scheduler Pthread arg0 - * @param tag number: {thread-id, tag} uniquely names a driver operation - * @param user-tag string arg2 - * @param command number arg3 - * @param string argument 1 arg4 - * @param string argument 2 arg5 - * @param integer argument 1 arg6 - * @param integer argument 2 arg7 - * @param integer argument 3 arg8 - * @param integer argument 4 arg9 - * @param port the port ID of the busy port args[10] - */ - probe efile_drv__entry(int, int, char *, int, char *, char *, - int64_t, int64_t, int64_t, int64_t, char *); - - /** - * Entry into the driver's internal work function. Computation here - * is performed by a async worker pool Pthread. - * - * @param thread-id number - * @param tag number - * @param command number - */ - probe efile_drv__int_entry(int, int, int); - - /** - * Return from the driver's internal work function. - * - * @param thread-id number - * @param tag number - * @param command number - */ - probe efile_drv__int_return(int, int, int); - - /** - * Return from the efile_drv.c file I/O driver - * - * @param thread-id number arg0 - * @param tag number arg1 - * @param user-tag string arg2 - * @param command number arg3 - * @param Success? 1 is success, 0 is failure arg4 - * @param If failure, the errno of the error. arg5 - */ - probe efile_drv__return(int, int, char *, int, int, int); - /* * The set of probes called by the erlang tracer nif backend. In order diff --git a/erts/emulator/beam/external.c b/erts/emulator/beam/external.c index 970158933f..fb42969a8f 100644 --- a/erts/emulator/beam/external.c +++ b/erts/emulator/beam/external.c @@ -122,8 +122,9 @@ static int encode_size_struct_int(struct TTBSizeContext_*, ErtsAtomCacheMap *acm static Export binary_to_term_trap_export; static BIF_RETTYPE binary_to_term_trap_1(BIF_ALIST_1); -static BIF_RETTYPE binary_to_term_int(Process* p, Uint32 flags, Eterm bin, Binary* context_b, - Export *bif, Eterm arg0, Eterm arg1); +static Sint transcode_dist_obuf(ErtsDistOutputBuf*, DistEntry*, Uint32 dflags, Sint reds); + + void erts_init_external(void) { erts_init_trap_export(&term_to_binary_trap_export, @@ -222,23 +223,10 @@ erts_destroy_atom_cache_map(ErtsAtomCacheMap *acmp) static ERTS_INLINE void insert_acache_map(ErtsAtomCacheMap *acmp, Eterm atom, Uint32 dflags) { - /* - * If the receiver do not understand utf8 atoms - * and this atom cannot be represented in latin1, - * we are not allowed to cache it. - * - * In this case all atoms are assumed to have - * latin1 encoding in the cache. By refusing it - * in the cache we will instead encode it using - * ATOM_UTF8_EXT/SMALL_ATOM_UTF8_EXT which the - * receiver do not recognize and tear down the - * connection. - */ - if (acmp && acmp->sz < ERTS_MAX_INTERNAL_ATOM_CACHE_ENTRIES - && ((dflags & DFLAG_UTF8_ATOMS) - || atom_tab(atom_val(atom))->latin1_chars >= 0)) { + if (acmp && acmp->sz < ERTS_MAX_INTERNAL_ATOM_CACHE_ENTRIES) { int ix; ASSERT(acmp->hdr_sz < 0); + ASSERT(dflags & DFLAG_UTF8_ATOMS); ix = atom2cix(atom); if (acmp->cache[ix].iix < 0) { acmp->cache[ix].iix = acmp->sz; @@ -258,9 +246,7 @@ get_iix_acache_map(ErtsAtomCacheMap *acmp, Eterm atom, Uint32 dflags) ASSERT(is_atom(atom)); ix = atom2cix(atom); if (acmp->cache[ix].iix < 0) { - ASSERT(acmp->sz == ERTS_MAX_INTERNAL_ATOM_CACHE_ENTRIES - || (!(dflags & DFLAG_UTF8_ATOMS) - && atom_tab(atom_val(atom))->latin1_chars < 0)); + ASSERT(acmp->sz == ERTS_MAX_INTERNAL_ATOM_CACHE_ENTRIES); return -1; } else { @@ -274,7 +260,6 @@ void erts_finalize_atom_cache_map(ErtsAtomCacheMap *acmp, Uint32 dflags) { if (acmp) { - int utf8_atoms = (int) (dflags & DFLAG_UTF8_ATOMS); int long_atoms = 0; /* !0 if one or more atoms are longer than 255. */ int i; int sz; @@ -285,6 +270,7 @@ erts_finalize_atom_cache_map(ErtsAtomCacheMap *acmp, Uint32 dflags) + 1 /* number of internal cache entries */ ; int min_sz; + ASSERT(dflags & DFLAG_UTF8_ATOMS); ASSERT(acmp->hdr_sz < 0); /* Make sure cache update instructions fit */ min_sz = fix_sz+(2+4)*acmp->sz; @@ -296,7 +282,7 @@ erts_finalize_atom_cache_map(ErtsAtomCacheMap *acmp, Uint32 dflags) atom = acmp->cache[acmp->cix[i]].atom; ASSERT(is_atom(atom)); a = atom_tab(atom_val(atom)); - len = (int) (utf8_atoms ? a->len : a->latin1_chars); + len = (int) a->len; ASSERT(len >= 0); if (!long_atoms && len > 255) long_atoms = 1; @@ -366,18 +352,77 @@ byte *erts_encode_ext_dist_header_setup(byte *ctl_ext, ErtsAtomCacheMap *acmp) } } -byte *erts_encode_ext_dist_header_finalize(byte *ext, ErtsAtomCache *cache, Uint32 dflags) + +#define PASS_THROUGH 'p' + +Sint erts_encode_ext_dist_header_finalize(ErtsDistOutputBuf* ob, + DistEntry* dep, + Uint32 dflags, + Sint reds) { byte *ip; byte instr_buf[(2+4)*ERTS_ATOM_CACHE_SIZE]; int ci, sz; byte dist_hdr_flags; int long_atoms; - int utf8_atoms = (int) (dflags & DFLAG_UTF8_ATOMS); - register byte *ep = ext; - ASSERT(ep[0] == VERSION_MAGIC); - if (ep[1] != DIST_HEADER) - return ext; + register byte *ep = ob->extp; + ASSERT(dflags & DFLAG_UTF8_ATOMS); + + /* + * The buffer can have different layouts at this point depending on + * what was known when encoded: + * + * Pending connection: CtrlTerm [, MsgTerm] + * With atom cache : VERSION_MAGIC, DIST_HEADER, ..., CtrlTerm [, MsgTerm] + * No atom cache : VERSION_MAGIC, CtrlTerm [, VERSION_MAGIC, MsgTerm] + */ + + if (ep[0] != VERSION_MAGIC || dep->transcode_ctx) { + /* + * Was encoded without atom cache toward pending connection. + */ + ASSERT(ep[0] == SMALL_TUPLE_EXT || ep[0] == LARGE_TUPLE_EXT); + + if (~dflags & (DFLAG_DIST_MONITOR | DFLAG_DIST_MONITOR_NAME) + && ep[0] == SMALL_TUPLE_EXT + && ep[1] == 4 + && ep[2] == SMALL_INTEGER_EXT + && (ep[3] == DOP_MONITOR_P || + ep[3] == DOP_MONITOR_P_EXIT || + ep[3] == DOP_DEMONITOR_P)) { + /* + * Receiver does not support process monitoring. + * Suppress monitor control msg (see erts_dsig_send_monitor) + * by converting it to an empty (tick) packet. + */ + ob->ext_endp = ob->extp; + return reds; + } + if (~dflags & (DFLAG_BIT_BINARIES | DFLAG_EXPORT_PTR_TAG + | DFLAG_DIST_HDR_ATOM_CACHE)) { + reds = transcode_dist_obuf(ob, dep, dflags, reds); + if (reds < 0) + return reds; + ep = ob->extp; + } + if (dflags & DFLAG_DIST_HDR_ATOM_CACHE) { + /* + * Encoding was done without atom caching but receiver expects + * a dist header, so we prepend an empty one. + */ + *--ep = 0; /* NumberOfAtomCacheRefs */ + *--ep = DIST_HEADER; + *--ep = VERSION_MAGIC; + } + goto done; + } + else if (ep[1] != DIST_HEADER) { + ASSERT(ep[1] == SMALL_TUPLE_EXT || ep[1] == LARGE_TUPLE_EXT); + ASSERT(!(dflags & DFLAG_DIST_HDR_ATOM_CACHE)); + /* Node without atom cache, 'pass through' needed */ + *--ep = PASS_THROUGH; + goto done; + } dist_hdr_flags = ep[2]; long_atoms = ERTS_DIST_HDR_LONG_ATOMS_FLG & ((int) dist_hdr_flags); @@ -405,6 +450,7 @@ byte *erts_encode_ext_dist_header_finalize(byte *ext, ErtsAtomCache *cache, Uint / sizeof(Uint32))+1]; register Uint32 flgs; int iix, flgs_bytes, flgs_buf_ix, used_half_bytes; + ErtsAtomCache* cache = dep->cache; #ifdef DEBUG int tot_used_half_bytes; #endif @@ -447,17 +493,9 @@ byte *erts_encode_ext_dist_header_finalize(byte *ext, ErtsAtomCache *cache, Uint Atom *a; cache->out_arr[cix] = atom; a = atom_tab(atom_val(atom)); - if (utf8_atoms) { - sz = a->len; - ep -= sz; - sys_memcpy((void *) ep, (void *) a->name, sz); - } - else { - ASSERT(0 <= a->latin1_chars && a->latin1_chars <= MAX_ATOM_CHARACTERS); - ep -= a->latin1_chars; - sz = erts_utf8_to_latin1(ep, a->name, a->len); - ASSERT(a->latin1_chars == sz); - } + sz = a->len; + ep -= sz; + sys_memcpy((void *) ep, (void *) a->name, sz); if (long_atoms) { ep -= 2; put_int16(sz, ep); @@ -504,12 +542,16 @@ byte *erts_encode_ext_dist_header_finalize(byte *ext, ErtsAtomCache *cache, Uint break; } } + reds -= 3; /*was ERTS_PORT_REDS_DIST_CMD_FINALIZE*/ } --ep; put_int8(ci, ep); *--ep = DIST_HEADER; *--ep = VERSION_MAGIC; - return ep; +done: + ob->extp = ep; + ASSERT(&ob->data[0] <= ob->extp && ob->extp < ob->ext_endp); + return reds < 0 ? 0 : reds; } int erts_encode_dist_ext_size(Eterm term, Uint32 flags, ErtsAtomCacheMap *acmp, @@ -520,7 +562,7 @@ int erts_encode_dist_ext_size(Eterm term, Uint32 flags, ErtsAtomCacheMap *acmp, return -1; } else { #ifndef ERTS_DEBUG_USE_DIST_SEP - if (!(flags & DFLAG_DIST_HDR_ATOM_CACHE)) + if (!(flags & (DFLAG_DIST_HDR_ATOM_CACHE | DFLAG_NO_MAGIC))) #endif sz++ /* VERSION_MAGIC */; @@ -536,7 +578,7 @@ int erts_encode_dist_ext_size_int(Eterm term, struct erts_dsig_send_context* ctx return -1; } else { #ifndef ERTS_DEBUG_USE_DIST_SEP - if (!(ctx->flags & DFLAG_DIST_HDR_ATOM_CACHE)) + if (!(ctx->flags & (DFLAG_DIST_HDR_ATOM_CACHE | DFLAG_NO_MAGIC))) #endif sz++ /* VERSION_MAGIC */; @@ -568,7 +610,7 @@ int erts_encode_dist_ext(Eterm term, byte **ext, Uint32 flags, ErtsAtomCacheMap { if (!ctx || !ctx->wstack.wstart) { #ifndef ERTS_DEBUG_USE_DIST_SEP - if (!(flags & DFLAG_DIST_HDR_ATOM_CACHE)) + if (!(flags & (DFLAG_DIST_HDR_ATOM_CACHE | DFLAG_NO_MAGIC))) #endif *(*ext)++ = VERSION_MAGIC; } @@ -643,7 +685,7 @@ erts_prepare_dist_ext(ErtsDistExternal *edep, #endif register byte *ep = ext; - int utf8_atoms = (int) (dep->flags & DFLAG_UTF8_ATOMS); + ASSERT(dep->flags & DFLAG_UTF8_ATOMS); edep->heap_size = -1; edep->ext_endp = ext+size; @@ -669,17 +711,16 @@ erts_prepare_dist_ext(ErtsDistExternal *edep, erts_de_rlock(dep); - if ((dep->status & (ERTS_DE_SFLG_EXITING|ERTS_DE_SFLG_CONNECTED)) - != ERTS_DE_SFLG_CONNECTED) { + if (dep->state != ERTS_DE_STATE_CONNECTED && + dep->state != ERTS_DE_STATE_PENDING) { erts_de_runlock(dep); return ERTS_PREP_DIST_EXT_CLOSED; } - if (dep->flags & DFLAG_DIST_HDR_ATOM_CACHE) edep->flags |= ERTS_DIST_EXT_DFLAG_HDR; *connection_id = dep->connection_id; - edep->flags |= (dep->connection_id & ERTS_DIST_EXT_CON_ID_MASK); + edep->connection_id = dep->connection_id; if (ep[1] != DIST_HEADER) { if (edep->flags & ERTS_DIST_EXT_DFLAG_HDR) @@ -806,9 +847,7 @@ erts_prepare_dist_ext(ErtsDistExternal *edep, CHKSIZE(len); atom = erts_atom_put((byte *) ep, len, - (utf8_atoms - ? ERTS_ATOM_ENC_UTF8 - : ERTS_ATOM_ENC_LATIN1), + ERTS_ATOM_ENC_UTF8, 0); if (is_non_value(atom)) ERTS_EXT_HDR_FAIL; @@ -895,7 +934,7 @@ bad_dist_ext(ErtsDistExternal *edep) erts_dsprintf(dsbufp, ", %d=%T", i, edep->attab.atom[i]); } erts_send_warning_to_logger_nogl(dsbufp); - erts_kill_dist_connection(dep, ERTS_DIST_EXT_CON_ID(edep)); + erts_kill_dist_connection(dep, edep->connection_id); } } @@ -1220,7 +1259,8 @@ typedef struct B2TContext_t { ErtsBinary2TermState b2ts; Uint32 flags; SWord reds; - Eterm trap_bin; + Uint used_bytes; /* In: boolean, Out: bytes */ + Eterm trap_bin; /* THE_NON_VALUE if not exported */ Export *bif; Eterm arg[2]; enum B2TState state; @@ -1314,6 +1354,11 @@ binary2term_prepare(ErtsBinary2TermState *state, byte *data, Sint data_size, ctx->u.uc.dbytes = state->extp; ctx->u.uc.dleft = dest_len; + if (ctx->used_bytes) { + ASSERT(ctx->used_bytes == 1); + /* to be subtracted by stream.avail_in when done */ + ctx->used_bytes = data_size; + } ctx->state = B2TUncompressChunk; *ctxp = ctx; } @@ -1416,13 +1461,15 @@ static int b2t_context_destructor(Binary *context_bin) return 1; } +static BIF_RETTYPE binary_to_term_int(Process*, Eterm bin, B2TContext*); + + static BIF_RETTYPE binary_to_term_trap_1(BIF_ALIST_1) { Binary *context_bin = erts_magic_ref2bin(BIF_ARG_1); ASSERT(ERTS_MAGIC_BIN_DESTRUCTOR(context_bin) == b2t_context_destructor); - return binary_to_term_int(BIF_P, 0, THE_NON_VALUE, context_bin, NULL, - THE_NON_VALUE, THE_NON_VALUE); + return binary_to_term_int(BIF_P, THE_NON_VALUE, ERTS_MAGIC_BIN_DATA(context_bin)); } @@ -1448,6 +1495,8 @@ static B2TContext* b2t_export_context(Process* p, B2TContext* src) b2t_context_destructor); B2TContext* ctx = ERTS_MAGIC_BIN_DATA(context_b); Eterm* hp; + + ASSERT(is_non_value(src->trap_bin)); sys_memcpy(ctx, src, sizeof(B2TContext)); if (ctx->state >= B2TDecode && ctx->u.dc.next == &src->u.dc.res) { ctx->u.dc.next = &ctx->u.dc.res; @@ -1457,8 +1506,7 @@ static B2TContext* b2t_export_context(Process* p, B2TContext* src) return ctx; } -static BIF_RETTYPE binary_to_term_int(Process* p, Uint32 flags, Eterm bin, Binary* context_b, - Export *bif_init, Eterm arg0, Eterm arg1) +static BIF_RETTYPE binary_to_term_int(Process* p, Eterm bin, B2TContext *ctx) { BIF_RETTYPE ret_val; #ifdef EXTREME_B2T_TRAPPING @@ -1466,25 +1514,17 @@ static BIF_RETTYPE binary_to_term_int(Process* p, Uint32 flags, Eterm bin, Binar #else SWord initial_reds = (Uint)(ERTS_BIF_REDS_LEFT(p) * B2T_BYTES_PER_REDUCTION); #endif - B2TContext c_buff; - B2TContext *ctx; int is_first_call; - if (context_b == NULL) { + if (is_value(bin)) { /* Setup enough to get started */ is_first_call = 1; - ctx = &c_buff; ctx->state = B2TPrepare; ctx->aligned_alloc = NULL; - ctx->flags = flags; - ctx->bif = bif_init; - ctx->arg[0] = arg0; - ctx->arg[1] = arg1; - IF_DEBUG(ctx->trap_bin = THE_NON_VALUE;) } else { - is_first_call = 0; - ctx = ERTS_MAGIC_BIN_DATA(context_b); + ASSERT(is_value(ctx->trap_bin)); ASSERT(ctx->state != B2TPrepare); + is_first_call = 0; } ctx->reds = initial_reds; @@ -1528,6 +1568,10 @@ static BIF_RETTYPE binary_to_term_int(Process* p, Uint32 flags, Eterm bin, Binar && zret == Z_STREAM_END && ctx->u.uc.dleft == 0) { ctx->reds -= chunk; + if (ctx->used_bytes) { + ASSERT(ctx->used_bytes > 5 + ctx->u.uc.stream.avail_in); + ctx->used_bytes -= ctx->u.uc.stream.avail_in; + } ctx->state = B2TSizeInit; } else { @@ -1546,11 +1590,11 @@ static BIF_RETTYPE binary_to_term_int(Process* p, Uint32 flags, Eterm bin, Binar break; case B2TDecodeInit: - if (ctx == &c_buff && ctx->b2ts.extsize > ctx->reds) { + if (is_non_value(ctx->trap_bin) && ctx->b2ts.extsize > ctx->reds) { /* dec_term will maybe trap, allocate space for magic bin before result term to make it easy to trim with HRelease. */ - ctx = b2t_export_context(p, &c_buff); + ctx = b2t_export_context(p, ctx); } ctx->u.dc.ep = ctx->b2ts.extp; ctx->u.dc.res = (Eterm) (UWord) NULL; @@ -1593,6 +1637,25 @@ static BIF_RETTYPE binary_to_term_int(Process* p, Uint32 flags, Eterm bin, Binar return ret_val; case B2TDone: + if (ctx->used_bytes) { + Eterm *hp; + Eterm used; + if (!ctx->b2ts.exttmp) { + ASSERT(ctx->used_bytes == 1); + ctx->used_bytes = (ctx->u.dc.ep - ctx->b2ts.extp + +1); /* VERSION_MAGIC */ + } + if (IS_USMALL(0, ctx->used_bytes)) { + hp = erts_produce_heap(&ctx->u.dc.factory, 3, 0); + used = make_small(ctx->used_bytes); + } + else { + hp = erts_produce_heap(&ctx->u.dc.factory, 3+BIG_UINT_HEAP_SIZE, 0); + used = uint_to_big(ctx->used_bytes, hp); + hp += BIG_UINT_HEAP_SIZE; + } + ctx->u.dc.res = TUPLE2(hp, ctx->u.dc.res, used); + } b2t_destroy_context(ctx); if (ctx->u.dc.factory.hp > ctx->u.dc.factory.hp_end) { @@ -1613,11 +1676,10 @@ static BIF_RETTYPE binary_to_term_int(Process* p, Uint32 flags, Eterm bin, Binar } }while (ctx->reds > 0 || ctx->state >= B2TDone); - if (ctx == &c_buff) { - ASSERT(ctx->trap_bin == THE_NON_VALUE); - ctx = b2t_export_context(p, &c_buff); + if (is_non_value(ctx->trap_bin)) { + ctx = b2t_export_context(p, ctx); + ASSERT(is_value(ctx->trap_bin)); } - ASSERT(ctx->trap_bin != THE_NON_VALUE); if (is_first_call) { erts_set_gc_state(p, 0); @@ -1634,23 +1696,35 @@ HIPE_WRAPPER_BIF_DISABLE_GC(binary_to_term, 1) BIF_RETTYPE binary_to_term_1(BIF_ALIST_1) { - return binary_to_term_int(BIF_P, 0, BIF_ARG_1, NULL, bif_export[BIF_binary_to_term_1], - BIF_ARG_1, THE_NON_VALUE); + B2TContext ctx; + + ctx.flags = 0; + ctx.used_bytes = 0; + ctx.trap_bin = THE_NON_VALUE; + ctx.bif = bif_export[BIF_binary_to_term_1]; + ctx.arg[0] = BIF_ARG_1; + ctx.arg[1] = THE_NON_VALUE; + return binary_to_term_int(BIF_P, BIF_ARG_1, &ctx); } HIPE_WRAPPER_BIF_DISABLE_GC(binary_to_term, 2) BIF_RETTYPE binary_to_term_2(BIF_ALIST_2) { + B2TContext ctx; Eterm opts; Eterm opt; - Uint32 flags = 0; + ctx.flags = 0; + ctx.used_bytes = 0; opts = BIF_ARG_2; while (is_list(opts)) { opt = CAR(list_val(opts)); if (opt == am_safe) { - flags |= ERTS_DIST_EXT_BTT_SAFE; + ctx.flags |= ERTS_DIST_EXT_BTT_SAFE; + } + else if (opt == am_used) { + ctx.used_bytes = 1; } else { goto error; @@ -1661,8 +1735,11 @@ BIF_RETTYPE binary_to_term_2(BIF_ALIST_2) if (is_not_nil(opts)) goto error; - return binary_to_term_int(BIF_P, flags, BIF_ARG_1, NULL, bif_export[BIF_binary_to_term_2], - BIF_ARG_1, BIF_ARG_2); + ctx.trap_bin = THE_NON_VALUE; + ctx.bif = bif_export[BIF_binary_to_term_2]; + ctx.arg[0] = BIF_ARG_1; + ctx.arg[1] = BIF_ARG_2; + return binary_to_term_int(BIF_P, BIF_ARG_1, &ctx); error: BIF_ERROR(BIF_P, BADARG); @@ -1876,7 +1953,7 @@ static Eterm erts_term_to_binary_int(Process* p, Eterm Term, int level, Uint fla context_b = erts_create_magic_binary(sizeof(TTBContext), \ ttb_context_destructor); \ context = ERTS_MAGIC_BIN_DATA(context_b); \ - memcpy(context,&c_buff,sizeof(TTBContext)); \ + sys_memcpy(context,&c_buff,sizeof(TTBContext)); \ } \ } while (0) @@ -1955,23 +2032,14 @@ static Eterm erts_term_to_binary_int(Process* p, Eterm Term, int level, Uint fla level = context->s.ec.level; BUMP_REDS(p, (initial_reds - reds) / TERM_TO_BINARY_LOOP_FACTOR); if (level == 0 || real_size < 6) { /* We are done */ - ProcBin* pb; return_normal: context->s.ec.result_bin = NULL; context->alive = 0; - pb = (ProcBin *) HAlloc(p, PROC_BIN_SIZE); - pb->thing_word = HEADER_PROC_BIN; - pb->size = real_size; - pb->next = MSO(p).first; - MSO(p).first = (struct erl_off_heap_header*)pb; - pb->val = result_bin; - pb->bytes = (byte*) result_bin->orig_bytes; - pb->flags = 0; - OH_OVERHEAD(&(MSO(p)), pb->size / sizeof(Eterm)); if (context_b && erts_refc_read(&context_b->intern.refc,0) == 0) { erts_bin_free(context_b); } - return make_binary(pb); + return erts_build_proc_bin(&MSO(p), HAlloc(p, PROC_BIN_SIZE), + result_bin); } /* Continue with compression... */ /* To make absolutely sure that zlib does not barf on a reallocated context, @@ -2028,16 +2096,7 @@ static Eterm erts_term_to_binary_int(Process* p, Eterm Term, int level, Uint fla result_bin = erts_bin_realloc(context->s.cc.destination_bin, context->s.cc.dest_len+6); context->s.cc.destination_bin = NULL; - pb = (ProcBin *) HAlloc(p, PROC_BIN_SIZE); - pb->thing_word = HEADER_PROC_BIN; - pb->size = context->s.cc.dest_len+6; - pb->next = MSO(p).first; - MSO(p).first = (struct erl_off_heap_header*)pb; - pb->val = result_bin; ASSERT(erts_refc_read(&result_bin->intern.refc, 1)); - pb->bytes = (byte*) result_bin->orig_bytes; - pb->flags = 0; - OH_OVERHEAD(&(MSO(p)), pb->size / sizeof(Eterm)); erts_bin_free(context->s.cc.result_bin); context->s.cc.result_bin = NULL; context->alive = 0; @@ -2045,7 +2104,9 @@ static Eterm erts_term_to_binary_int(Process* p, Eterm Term, int level, Uint fla if (context_b && erts_refc_read(&context_b->intern.refc,0) == 0) { erts_bin_free(context_b); } - return make_binary(pb); + return erts_build_proc_bin(&MSO(p), + HAlloc(p, PROC_BIN_SIZE), + result_bin); } default: /* Compression error, revert to uncompressed binary (still in context) */ @@ -2099,7 +2160,7 @@ enc_atom(ErtsAtomCacheMap *acmp, Eterm atom, byte *ep, Uint32 dflags) { int iix; int len; - int utf8_atoms = (int) (dflags & DFLAG_UTF8_ATOMS); + const int utf8_atoms = (int) (dflags & DFLAG_UTF8_ATOMS); ASSERT(is_atom(atom)); @@ -2494,8 +2555,6 @@ enc_term_int(TTBEncodeContext* ctx, ErtsAtomCacheMap *acmp, Eterm obj, byte* ep, break; } - L_jump_start: - if (ctx && --r <= 0) { *reds = 0; ctx->obj = obj; @@ -2503,6 +2562,8 @@ enc_term_int(TTBEncodeContext* ctx, ErtsAtomCacheMap *acmp, Eterm obj, byte* ep, WSTACK_SAVE(s, &ctx->wstack); return -1; } + + L_jump_start: switch(tag_val_def(obj)) { case NIL_DEF: *ep++ = NIL_EXT; @@ -2812,6 +2873,8 @@ enc_term_int(TTBEncodeContext* ctx, ErtsAtomCacheMap *acmp, Eterm obj, byte* ep, ep[j] = 0; /* Zero unused bits at end of binary */ data_dst = ep; ep += j + 1; + if (ctx) + ctx->hopefull_flags |= DFLAG_BIT_BINARIES; } else { /* * Bit-level binary, but the receiver doesn't support it. @@ -2847,6 +2910,8 @@ enc_term_int(TTBEncodeContext* ctx, ErtsAtomCacheMap *acmp, Eterm obj, byte* ep, ep = enc_atom(acmp, exp->info.mfa.function, ep, dflags); ep = enc_term(acmp, make_small(exp->info.mfa.arity), ep, dflags, off_heap); + if (ctx) + ctx->hopefull_flags |= DFLAG_EXPORT_PTR_TAG; } else { /* Tag, arity */ *ep++ = SMALL_TUPLE_EXT; @@ -2867,62 +2932,27 @@ enc_term_int(TTBEncodeContext* ctx, ErtsAtomCacheMap *acmp, Eterm obj, byte* ep, ErlFunThing* funp = (ErlFunThing *) fun_val(obj); int ei; - if ((dflags & DFLAG_NEW_FUN_TAGS) != 0) { - *ep++ = NEW_FUN_EXT; - WSTACK_PUSH2(s, ENC_PATCH_FUN_SIZE, - (UWord) ep); /* Position for patching in size */ - ep += 4; - *ep = funp->arity; - ep += 1; - sys_memcpy(ep, funp->fe->uniq, 16); - ep += 16; - put_int32(funp->fe->index, ep); - ep += 4; - put_int32(funp->num_free, ep); - ep += 4; - ep = enc_atom(acmp, funp->fe->module, ep, dflags); - ep = enc_term(acmp, make_small(funp->fe->old_index), ep, dflags, off_heap); - ep = enc_term(acmp, make_small(funp->fe->old_uniq), ep, dflags, off_heap); - ep = enc_pid(acmp, funp->creator, ep, dflags); - } else { - /* - * Communicating with an obsolete erl_interface or - * jinterface node. Convert the fun to a tuple to - * avoid crasching. - */ - - /* Tag, arity */ - *ep++ = SMALL_TUPLE_EXT; - put_int8(5, ep); - ep += 1; - - /* 'fun' */ - ep = enc_atom(acmp, am_fun, ep, dflags); - - /* Module name */ - ep = enc_atom(acmp, funp->fe->module, ep, dflags); - - /* Index, Uniq */ - *ep++ = INTEGER_EXT; - put_int32(funp->fe->old_index, ep); - ep += 4; - *ep++ = INTEGER_EXT; - put_int32(funp->fe->old_uniq, ep); - ep += 4; - - /* Environment sub-tuple arity */ - ASSERT(funp->num_free < MAX_ARG); - *ep++ = SMALL_TUPLE_EXT; - put_int8(funp->num_free, ep); - ep += 1; - } - for (ei = funp->num_free-1; ei > 0; ei--) { + ASSERT(dflags & DFLAG_NEW_FUN_TAGS); + *ep++ = NEW_FUN_EXT; + WSTACK_PUSH2(s, ENC_PATCH_FUN_SIZE, + (UWord) ep); /* Position for patching in size */ + ep += 4; + *ep = funp->arity; + ep += 1; + sys_memcpy(ep, funp->fe->uniq, 16); + ep += 16; + put_int32(funp->fe->index, ep); + ep += 4; + put_int32(funp->num_free, ep); + ep += 4; + ep = enc_atom(acmp, funp->fe->module, ep, dflags); + ep = enc_term(acmp, make_small(funp->fe->old_index), ep, dflags, off_heap); + ep = enc_term(acmp, make_small(funp->fe->old_uniq), ep, dflags, off_heap); + ep = enc_pid(acmp, funp->creator, ep, dflags); + + for (ei = funp->num_free-1; ei >= 0; ei--) { WSTACK_PUSH2(s, ENC_TERM, (UWord) funp->env[ei]); } - if (funp->num_free != 0) { - obj = funp->env[0]; - goto L_jump_start; - } } break; } @@ -3260,6 +3290,7 @@ dec_term_atom_common: n--; if (ctx) { if (reds < n) { + ASSERT(reds > 0); ctx->state = B2TDecodeList; ctx->u.dc.remaining_n = n - reds; n = reds; @@ -3541,18 +3572,9 @@ dec_term_atom_common: *objp = make_binary(hb); } else { Binary* dbin = erts_bin_nrml_alloc(n); - ProcBin* pb; - pb = (ProcBin *) hp; + + *objp = erts_build_proc_bin(factory->off_heap, hp, dbin); hp += PROC_BIN_SIZE; - pb->thing_word = HEADER_PROC_BIN; - pb->size = n; - pb->next = factory->off_heap->first; - factory->off_heap->first = (struct erl_off_heap_header*)pb; - OH_OVERHEAD(factory->off_heap, pb->size / sizeof(Eterm)); - pb->val = dbin; - pb->bytes = (byte*) dbin->orig_bytes; - pb->flags = 0; - *objp = make_binary(pb); if (ctx) { int n_limit = reds * B2T_MEMCPY_FACTOR; if (n > n_limit) { @@ -3592,18 +3614,9 @@ dec_term_atom_common: ep += n; } else { Binary* dbin = erts_bin_nrml_alloc(n); - ProcBin* pb; + Uint n_copy = n; - pb = (ProcBin *) hp; - pb->thing_word = HEADER_PROC_BIN; - pb->size = n; - pb->next = factory->off_heap->first; - factory->off_heap->first = (struct erl_off_heap_header*)pb; - OH_OVERHEAD(factory->off_heap, pb->size / sizeof(Eterm)); - pb->val = dbin; - pb->bytes = (byte*) dbin->orig_bytes; - pb->flags = 0; - bin = make_binary(pb); + bin = erts_build_proc_bin(factory->off_heap, hp, dbin); hp += PROC_BIN_SIZE; if (ctx) { int n_limit = reds * B2T_MEMCPY_FACTOR; @@ -3611,15 +3624,15 @@ dec_term_atom_common: ctx->state = B2TDecodeBinary; ctx->u.dc.remaining_n = n - n_limit; ctx->u.dc.remaining_bytes = dbin->orig_bytes + n_limit; - n = n_limit; + n_copy = n_limit; reds = 0; } - else + else { reds -= n / B2T_MEMCPY_FACTOR; + } } - sys_memcpy(dbin->orig_bytes, ep, n); - ep += n; - n = pb->size; + sys_memcpy(dbin->orig_bytes, ep, n_copy); + ep += n_copy; } if (bitsize == 8 || n == 0) { @@ -4000,6 +4013,7 @@ dec_term_atom_common: if (ctx) { ctx->state = B2TDone; ctx->reds = reds; + ctx->u.dc.ep = ep; } return ep; @@ -4269,24 +4283,12 @@ encode_size_struct_int(TTBSizeContext* ctx, ErtsAtomCacheMap *acmp, Eterm obj, { ErlFunThing* funp = (ErlFunThing *) fun_val(obj); - if ((dflags & DFLAG_NEW_FUN_TAGS) != 0) { - result += 20+1+1+4; /* New ID + Tag */ - result += 4; /* Length field (number of free variables */ - result += encode_size_struct2(acmp, funp->creator, dflags); - result += encode_size_struct2(acmp, funp->fe->module, dflags); - result += 2 * (1+4); /* Index, Uniq */ - } else { - /* - * Size when fun is mapped to a tuple. - */ - result += 1 + 1; /* Tuple tag, arity */ - result += 1 + 1 + 2 + - atom_tab(atom_val(am_fun))->len; /* 'fun' */ - result += 1 + 1 + 2 + - atom_tab(atom_val(funp->fe->module))->len; /* Module name */ - result += 2 * (1 + 4); /* Index + Uniq */ - result += 1 + (funp->num_free < 0x100 ? 1 : 4); - } + ASSERT(dflags & DFLAG_NEW_FUN_TAGS); + result += 20+1+1+4; /* New ID + Tag */ + result += 4; /* Length field (number of free variables */ + result += encode_size_struct2(acmp, funp->creator, dflags); + result += encode_size_struct2(acmp, funp->fe->module, dflags); + result += 2 * (1+4); /* Index, Uniq */ if (funp->num_free > 1) { WSTACK_PUSH2(s, (UWord) (funp->env + 1), (UWord) TERM_ARRAY_OP(funp->num_free-1)); @@ -4374,7 +4376,7 @@ decoded_size(byte *ep, byte* endp, int internal_tags, B2TContext* ctx) } } else - reds = 0; /* not used but compiler warns anyway */ + ERTS_UNDEF(reds, 0); heap_size = 0; terms = 1; @@ -4684,3 +4686,182 @@ error: #undef SKIP2 #undef CHKSIZE } + + +struct transcode_context { + enum { + TRANSCODE_DEC_MSG_SIZE, + TRANSCODE_DEC_MSG, + TRANSCODE_ENC_CTL, + TRANSCODE_ENC_MSG + }state; + Eterm ctl_term; + Eterm* ctl_heap; + ErtsHeapFactory ctl_factory; + Eterm* msg_heap; + B2TContext b2t; + TTBEncodeContext ttb; +#ifdef DEBUG + ErtsDistOutputBuf* dbg_ob; +#endif +}; + +void transcode_free_ctx(DistEntry* dep) +{ + struct transcode_context* ctx = dep->transcode_ctx; + + erts_factory_close(&ctx->ctl_factory); + erts_free(ERTS_ALC_T_DIST_TRANSCODE, ctx->ctl_heap); + + if (ctx->msg_heap) { + erts_factory_close(&ctx->b2t.u.dc.factory); + erts_free(ERTS_ALC_T_DIST_TRANSCODE, ctx->msg_heap); + } + erts_free(ERTS_ALC_T_DIST_TRANSCODE, ctx); + dep->transcode_ctx = NULL; +} + +Sint transcode_dist_obuf(ErtsDistOutputBuf* ob, + DistEntry* dep, + Uint32 dflags, + Sint reds) +{ + Sint hsz; + byte* decp; + const int have_msg = !!ob->msg_start; + int i; + struct transcode_context* ctx = dep->transcode_ctx; + + if (!ctx) { /* first call for 'ob' */ + ASSERT(!(ob->hopefull_flags & ~(Uint)(DFLAG_BIT_BINARIES | + DFLAG_EXPORT_PTR_TAG))); + if (~dflags & ob->hopefull_flags) { + /* + * Receiver does not support bitstrings and/or export funs + * and output buffer contains such message tags (hopefull_flags). + * Must transcode control and message terms to use tuple fallbacks. + */ + ctx = erts_alloc(ERTS_ALC_T_DIST_TRANSCODE, sizeof(struct transcode_context)); + dep->transcode_ctx = ctx; + #ifdef DEBUG + ctx->dbg_ob = ob; + #endif + + hsz = decoded_size(ob->extp, ob->ext_endp, 0, NULL); + ctx->ctl_heap = erts_alloc(ERTS_ALC_T_DIST_TRANSCODE, hsz*sizeof(Eterm)); + erts_factory_tmp_init(&ctx->ctl_factory, ctx->ctl_heap, hsz, ERTS_ALC_T_DIST_TRANSCODE); + ctx->msg_heap = NULL; + + decp = dec_term(NULL, &ctx->ctl_factory, ob->extp, &ctx->ctl_term, NULL); + if (have_msg) { + ASSERT(decp == ob->msg_start); (void)decp; + ctx->b2t.u.sc.ep = NULL; + ctx->b2t.state = B2TSize; + ctx->b2t.aligned_alloc = NULL; + ctx->b2t.b2ts.exttmp = 0; + ctx->state = TRANSCODE_DEC_MSG_SIZE; + } + else { + ASSERT(decp == ob->ext_endp); + ctx->state = TRANSCODE_ENC_CTL; + } + } + else if (!(dflags & DFLAG_DIST_HDR_ATOM_CACHE)) { + /* + * No need for full transcoding, but primitive receiver (erl_/jinterface) + * expects VERSION_MAGIC before both control and message terms. + */ + if (ob->msg_start) { + Sint ctl_bytes = ob->msg_start - ob->extp; + ASSERT(ob->extp < ob->msg_start && ob->msg_start < ob->ext_endp); + /* Move control term back 1 byte to make room */ + sys_memmove(ob->extp-1, ob->extp, ctl_bytes); + *--(ob->msg_start) = VERSION_MAGIC; + --(ob->extp); + reds -= ctl_bytes / (B2T_BYTES_PER_REDUCTION * B2T_MEMCPY_FACTOR); + } + *--(ob->extp) = VERSION_MAGIC; + goto done; + } + else + goto done; + } + else { /* continue after yield */ + ASSERT(ctx->dbg_ob == ob); + } + ctx->b2t.reds = reds * B2T_BYTES_PER_REDUCTION; + + switch (ctx->state) { + case TRANSCODE_DEC_MSG_SIZE: + hsz = decoded_size(ob->msg_start, ob->ext_endp, 0, &ctx->b2t); + if (ctx->b2t.state == B2TSize) { + return -1; + } + ASSERT(ctx->b2t.state == B2TDecodeInit); + ctx->msg_heap = erts_alloc(ERTS_ALC_T_DIST_TRANSCODE, hsz*sizeof(Eterm)); + ctx->b2t.u.dc.ep = ob->msg_start; + ctx->b2t.u.dc.res = (Eterm) NULL; + ctx->b2t.u.dc.next = &ctx->b2t.u.dc.res; + erts_factory_tmp_init(&ctx->b2t.u.dc.factory, + ctx->msg_heap, hsz, ERTS_ALC_T_DIST_TRANSCODE); + ctx->b2t.u.dc.flat_maps.wstart = NULL; + ctx->b2t.u.dc.hamt_array.pstart = NULL; + ctx->b2t.state = B2TDecode; + + ctx->state = TRANSCODE_DEC_MSG; + case TRANSCODE_DEC_MSG: + if (ctx->b2t.reds <= 0) + ctx->b2t.reds = 1; + decp = dec_term(NULL, NULL, NULL, NULL, &ctx->b2t); + if (ctx->b2t.state < B2TDone) { + return -1; + } + ASSERT(ctx->b2t.state == B2TDone); + ASSERT(decp && decp <= ob->ext_endp); + reds = ctx->b2t.reds / B2T_BYTES_PER_REDUCTION; + b2t_destroy_context(&ctx->b2t); + + ctx->state = TRANSCODE_ENC_CTL; + case TRANSCODE_ENC_CTL: + if (!(dflags & DFLAG_DIST_HDR_ATOM_CACHE)) { + ASSERT(!(dflags & DFLAG_NO_MAGIC)); + ob->extp -= 2; /* VERSION_MAGIC x 2 */ + } + ob->ext_endp = ob->extp; + i = erts_encode_dist_ext(ctx->ctl_term, &ob->ext_endp, dflags, + NULL, NULL, NULL); + ASSERT(i == 0); (void)i; + ASSERT(ob->ext_endp <= ob->alloc_endp); + + if (!have_msg) { + break; + } + ob->msg_start = ob->ext_endp; + ctx->ttb.wstack.wstart = NULL; + ctx->ttb.flags = dflags; + ctx->ttb.hopefull_flags = 0; + ctx->ttb.level = 0; + + ctx->state = TRANSCODE_ENC_MSG; + case TRANSCODE_ENC_MSG: + reds *= TERM_TO_BINARY_LOOP_FACTOR; + if (erts_encode_dist_ext(ctx->b2t.u.dc.res, &ob->ext_endp, dflags, NULL, + &ctx->ttb, &reds)) { + return -1; + } + reds /= TERM_TO_BINARY_LOOP_FACTOR; + + ASSERT(ob->ext_endp <= ob->alloc_endp); + ASSERT(!ctx->ttb.hopefull_flags); + } + transcode_free_ctx(dep); + +done: + if (!(dflags & DFLAG_DIST_HDR_ATOM_CACHE)) + *--(ob->extp) = PASS_THROUGH; + + if (reds < 0) + reds = 0; + + return reds; +} diff --git a/erts/emulator/beam/external.h b/erts/emulator/beam/external.h index 3c61d013da..f9f8abcc27 100644 --- a/erts/emulator/beam/external.h +++ b/erts/emulator/beam/external.h @@ -109,35 +109,26 @@ typedef struct { } ErtsAtomTranslationTable; /* - * These flags are tagged onto the high bits of a connection ID and stored in - * the ErtsDistExternal structure's flags field. They are used to indicate - * various bits of state necessary to decode binaries in a variety of - * scenarios. The mask ERTS_DIST_EXT_CON_ID_MASK is used later to separate the - * connection ID from the flags. Be careful to ensure that the mask does not - * overlap any of the bits used for flags, or ERTS will leak flags bits into - * connection IDs and leak connection ID bits into the flags. + * These flags are stored in the ErtsDistExternal structure's flags field. + * They are used to indicate various bits of state necessary to decode binaries + * in a variety of scenarios. */ -#define ERTS_DIST_EXT_DFLAG_HDR ((Uint32) 0x80000000) -#define ERTS_DIST_EXT_ATOM_TRANS_TAB ((Uint32) 0x40000000) -#define ERTS_DIST_EXT_BTT_SAFE ((Uint32) 0x20000000) -#define ERTS_DIST_EXT_CON_ID_MASK ((Uint32) 0x1fffffff) +#define ERTS_DIST_EXT_DFLAG_HDR ((Uint32) 0x1) +#define ERTS_DIST_EXT_ATOM_TRANS_TAB ((Uint32) 0x2) +#define ERTS_DIST_EXT_BTT_SAFE ((Uint32) 0x4) + +#define ERTS_DIST_CON_ID_MASK ((Uint32) 0x00ffffff) /* also in net_kernel.erl */ -#define ERTS_DIST_EXT_CON_ID(DIST_EXTP) \ - ((DIST_EXTP)->flags & ERTS_DIST_EXT_CON_ID_MASK) typedef struct { DistEntry *dep; byte *extp; byte *ext_endp; Sint heap_size; + Uint32 connection_id; Uint32 flags; ErtsAtomTranslationTable attab; } ErtsDistExternal; -typedef struct { - int have_header; - int cache_entries; -} ErtsDistHeaderPeek; - #define ERTS_DIST_EXT_SIZE(EDEP) \ (sizeof(ErtsDistExternal) \ - (((EDEP)->flags & ERTS_DIST_EXT_ATOM_TRANS_TAB) \ @@ -163,7 +154,7 @@ void erts_finalize_atom_cache_map(ErtsAtomCacheMap *, Uint32); Uint erts_encode_ext_dist_header_size(ErtsAtomCacheMap *); byte *erts_encode_ext_dist_header_setup(byte *, ErtsAtomCacheMap *); -byte *erts_encode_ext_dist_header_finalize(byte *, ErtsAtomCache *, Uint32); +Sint erts_encode_ext_dist_header_finalize(ErtsDistOutputBuf*, DistEntry *, Uint32 dflags, Sint reds); struct erts_dsig_send_context; int erts_encode_dist_ext_size(Eterm, Uint32, ErtsAtomCacheMap*, Uint* szp); int erts_encode_dist_ext_size_int(Eterm term, struct erts_dsig_send_context* ctx, Uint* szp); @@ -177,9 +168,6 @@ Uint erts_encode_ext_size_ets(Eterm); void erts_encode_ext(Eterm, byte **); byte* erts_encode_ext_ets(Eterm, byte *, struct erl_off_heap_header** ext_off_heap); -#ifdef ERTS_WANT_EXTERNAL_TAGS -ERTS_GLB_INLINE void erts_peek_dist_header(ErtsDistHeaderPeek *, byte *, Uint); -#endif ERTS_GLB_INLINE void erts_free_dist_ext_copy(ErtsDistExternal *); ERTS_GLB_INLINE void *erts_dist_ext_trailer(ErtsDistExternal *); ErtsDistExternal *erts_make_dist_ext_copy(ErtsDistExternal *, Uint); @@ -207,23 +195,9 @@ void erts_binary2term_abort(ErtsBinary2TermState *); Eterm erts_binary2term_create(ErtsBinary2TermState *, ErtsHeapFactory*); int erts_debug_max_atom_out_cache_index(void); int erts_debug_atom_to_out_cache_index(Eterm); - +void transcode_free_ctx(DistEntry* dep); #if ERTS_GLB_INLINE_INCL_FUNC_DEF -#ifdef ERTS_WANT_EXTERNAL_TAGS -ERTS_GLB_INLINE void -erts_peek_dist_header(ErtsDistHeaderPeek *dhpp, byte *ext, Uint sz) -{ - if (ext[0] == VERSION_MAGIC - || ext[1] != DIST_HEADER - || sz < (1+1+1)) - dhpp->have_header = 0; - else { - dhpp->have_header = 1; - dhpp->cache_entries = (int) get_int8(&ext[2]); - } -} -#endif ERTS_GLB_INLINE void erts_free_dist_ext_copy(ErtsDistExternal *edep) diff --git a/erts/emulator/beam/global.h b/erts/emulator/beam/global.h index 3dd3a60939..0f23027752 100644 --- a/erts/emulator/beam/global.h +++ b/erts/emulator/beam/global.h @@ -368,7 +368,7 @@ do {\ UWord _wsz = ESTACK_COUNT(s);\ (dst)->start = erts_alloc((s).alloc_type,\ DEF_ESTACK_SIZE * sizeof(Eterm));\ - memcpy((dst)->start, (s).start,_wsz*sizeof(Eterm));\ + sys_memcpy((dst)->start, (s).start,_wsz*sizeof(Eterm));\ (dst)->sp = (dst)->start + _wsz;\ (dst)->end = (dst)->start + DEF_ESTACK_SIZE;\ (dst)->edefault = NULL;\ @@ -536,7 +536,7 @@ do {\ UWord _wsz = WSTACK_COUNT(s);\ (dst)->wstart = erts_alloc(s.alloc_type,\ DEF_WSTACK_SIZE * sizeof(UWord));\ - memcpy((dst)->wstart, s.wstart,_wsz*sizeof(UWord));\ + sys_memcpy((dst)->wstart, s.wstart,_wsz*sizeof(UWord));\ (dst)->wsp = (dst)->wstart + _wsz;\ (dst)->wend = (dst)->wstart + DEF_WSTACK_SIZE;\ (dst)->wdefault = NULL;\ @@ -861,6 +861,7 @@ Eterm erts_new_heap_binary(Process *p, byte *buf, int len, byte** datap); Eterm erts_new_mso_binary(Process*, byte*, Uint); Eterm new_binary(Process*, byte*, Uint); Eterm erts_realloc_binary(Eterm bin, size_t size); +Eterm erts_build_proc_bin(ErlOffHeap*, Eterm*, Binary*); /* erl_bif_info.c */ @@ -947,8 +948,8 @@ void erts_update_ranges(BeamInstr* code, Uint size); void erts_remove_from_ranges(BeamInstr* code); UWord erts_ranges_sz(void); void erts_lookup_function_info(FunctionInfo* fi, BeamInstr* pc, int full_info); -ErtsLiteralArea** erts_dump_lit_areas; -Uint erts_dump_num_lit_areas; +extern ErtsLiteralArea** erts_dump_lit_areas; +extern Uint erts_dump_num_lit_areas; /* break.c */ void init_break_handler(void); @@ -958,6 +959,7 @@ void process_info(fmtfn_t, void *); void print_process_info(fmtfn_t, void *, Process*); void info(fmtfn_t, void *); void loaded(fmtfn_t, void *); +void erts_print_base64(fmtfn_t to, void *to_arg, byte* src, Uint size); /* sighandler sys.c */ int erts_set_signal(Eterm signal, Eterm type); @@ -1064,7 +1066,7 @@ Eterm copy_struct_x(Eterm, Uint, Eterm**, ErlOffHeap*, Uint*, erts_literal_area_ #define copy_struct_litopt(Obj,Sz,HPP,OH,LitArea) \ copy_struct_x(Obj,Sz,HPP,OH,NULL,LitArea) -Eterm copy_shallow(Eterm*, Uint, Eterm**, ErlOffHeap*); +Eterm copy_shallow(Eterm* ERTS_RESTRICT, Uint, Eterm**, ErlOffHeap*); void erts_move_multi_frags(Eterm** hpp, ErlOffHeap*, ErlHeapFragment* first, Eterm* refs, unsigned nrefs, int literals); @@ -1121,7 +1123,6 @@ extern int erts_no_line_info; extern Eterm erts_error_logger_warnings; extern int erts_initialized; extern int erts_compat_rel; -extern int erts_use_sender_punish; void erl_start(int, char**); void erts_usage(void); Eterm erts_preloaded(Process* p); @@ -1263,7 +1264,7 @@ char* erts_convert_filename_to_wchar(byte* bytes, Uint size, char *statbuf, size_t statbuf_size, ErtsAlcType_t alloc_type, Sint* used, Uint extra_wchars); -Eterm erts_convert_native_to_filename(Process *p, byte *bytes); +Eterm erts_convert_native_to_filename(Process *p, size_t size, byte *bytes); Eterm erts_utf8_to_list(Process *p, Uint num, byte *bytes, Uint sz, Uint left, Uint *num_built, Uint *num_eaten, Eterm tail); int erts_utf8_to_latin1(byte* dest, const byte* source, int slen); diff --git a/erts/emulator/beam/index.c b/erts/emulator/beam/index.c index 93d1111904..be1771b037 100644 --- a/erts/emulator/beam/index.c +++ b/erts/emulator/beam/index.c @@ -58,7 +58,7 @@ IndexTable* erts_index_init(ErtsAlcType_t type, IndexTable* t, char* name, int size, int limit, HashFunctions fun) { - Uint base_size = ((limit+INDEX_PAGE_SIZE-1)/INDEX_PAGE_SIZE)*sizeof(IndexSlot*); + Uint base_size = (((Uint)limit+INDEX_PAGE_SIZE-1)/INDEX_PAGE_SIZE)*sizeof(IndexSlot*); hash_init(type, &t->htable, name, 3*size/4, fun); t->size = 0; diff --git a/erts/emulator/beam/instrs.tab b/erts/emulator/beam/instrs.tab index c17d1a8f69..6a531fcc09 100644 --- a/erts/emulator/beam/instrs.tab +++ b/erts/emulator/beam/instrs.tab @@ -322,6 +322,16 @@ get_list(Src, Hd, Tl) { $Tl = tl; } +get_hd(Src, Hd) { + Eterm* tmp_ptr = list_val($Src); + $Hd = CAR(tmp_ptr); +} + +get_tl(Src, Tl) { + Eterm* tmp_ptr = list_val($Src); + $Tl = CDR(tmp_ptr); +} + i_get(Src, Dst) { $Dst = erts_pd_hash_get(c_p, $Src); } @@ -542,6 +552,13 @@ put_list(Hd, Tl, Dst) { HTOP += 2; } +update_list(Hd, Dst) { + HTOP[0] = $Hd; + HTOP[1] = $Dst; + $Dst = make_list(HTOP); + HTOP += 2; +} + i_put_tuple := i_put_tuple.make.fill; i_put_tuple.make(Dst) { @@ -924,3 +941,33 @@ i_raise() { //| -no_next } +build_stacktrace() { + SWAPOUT; + x(0) = build_stacktrace(c_p, x(0)); + SWAPIN; +} + +raw_raise() { + Eterm class = x(0); + Eterm value = x(1); + Eterm stacktrace = x(2); + + if (class == am_error) { + c_p->freason = EXC_ERROR & ~EXF_SAVETRACE; + c_p->fvalue = value; + c_p->ftrace = stacktrace; + goto find_func_info; + } else if (class == am_exit) { + c_p->freason = EXC_EXIT & ~EXF_SAVETRACE; + c_p->fvalue = value; + c_p->ftrace = stacktrace; + goto find_func_info; + } else if (class == am_throw) { + c_p->freason = EXC_THROWN & ~EXF_SAVETRACE; + c_p->fvalue = value; + c_p->ftrace = stacktrace; + goto find_func_info; + } else { + x(0) = am_badarg; + } +} diff --git a/erts/emulator/beam/io.c b/erts/emulator/beam/io.c index 85013af3ad..e4a5f2b6b6 100644 --- a/erts/emulator/beam/io.c +++ b/erts/emulator/beam/io.c @@ -298,7 +298,6 @@ static Port *create_port(char *name, erts_aint32_t state = ERTS_PORT_SFLG_CONNECTED; erts_aint32_t x_pts_flgs = 0; - ErtsRunQueue *runq; if (!driver_lock) { /* Align size for mutex following port struct */ port_size = size = ERTS_ALC_DATA_ALIGN_SIZE(sizeof(Port)); @@ -347,11 +346,16 @@ static Port *create_port(char *name, p += sizeof(erts_mtx_t); state |= ERTS_PORT_SFLG_PORT_SPECIFIC_LOCK; } - if (erts_get_scheduler_data()) - runq = erts_get_runq_current(NULL); - else - runq = ERTS_RUNQ_IX(0); - erts_atomic_set_nob(&prt->run_queue, (erts_aint_t) runq); + + { + ErtsRunQueue *runq; + ErtsSchedulerData *esdp = erts_get_scheduler_data(); + if (esdp) + runq = erts_get_runq_current(esdp); + else + runq = ERTS_RUNQ_IX(0); + erts_init_runq_port(prt, runq); + } prt->xports = NULL; @@ -584,7 +588,7 @@ erts_open_driver(erts_driver_t* driver, /* Pointer to driver. */ */ for (d = driver_list; d; d = d->next) { - if (strcmp(d->name, name) == 0 && + if (sys_strcmp(d->name, name) == 0 && erts_ddll_driver_ok(d->handle)) { driver = d; break; @@ -634,7 +638,7 @@ erts_open_driver(erts_driver_t* driver, /* Pointer to driver. */ trace_port_open(port, pid, erts_atom_put((byte *) port->name, - strlen(port->name), + sys_strlen(port->name), ERTS_ATOM_ENC_LATIN1, 1)); } @@ -2925,7 +2929,7 @@ erl_drv_init_ack(ErlDrvPort ix, ErlDrvData res) { break; case -2: { char *str = erl_errno_id(errno); - resp = erts_atom_put((byte *) str, strlen(str), + resp = erts_atom_put((byte *) str, sys_strlen(str), ERTS_ATOM_ENC_LATIN1, 1); break; } @@ -3384,24 +3388,11 @@ static void deliver_read_message(Port* prt, erts_aint32_t state, Eterm to, if ((state & ERTS_PORT_SFLG_BINARY_IO) == 0) { listp = buf_to_intlist(&hp, buf, len, listp); } else if (buf != NULL) { - ProcBin* pb; - Binary* bptr; - - bptr = erts_bin_nrml_alloc(len); + Binary* bptr = erts_bin_nrml_alloc(len); sys_memcpy(bptr->orig_bytes, buf, len); - pb = (ProcBin *) hp; - pb->thing_word = HEADER_PROC_BIN; - pb->size = len; - pb->next = ohp->first; - ohp->first = (struct erl_off_heap_header*)pb; - pb->val = bptr; - pb->bytes = (byte*) bptr->orig_bytes; - pb->flags = 0; + listp = erts_build_proc_bin(ohp, hp, bptr); hp += PROC_BIN_SIZE; - - OH_OVERHEAD(ohp, pb->size / sizeof(Eterm)); - listp = make_binary(pb); } /* Prepend the header */ @@ -3829,11 +3820,12 @@ static void sweep_one_link(ErtsLink *lnk, void *vpsc) ErtsDistLinkData dld; ErtsDSigData dsd; int code; - code = erts_dsig_prepare(&dsd, dep, NULL, ERTS_DSP_NO_LOCK, 0); + code = erts_dsig_prepare(&dsd, dep, NULL, 0, ERTS_DSP_NO_LOCK, 0, 0); switch (code) { case ERTS_DSIG_PREP_NOT_ALIVE: case ERTS_DSIG_PREP_NOT_CONNECTED: break; + case ERTS_DSIG_PREP_PENDING: case ERTS_DSIG_PREP_CONNECTED: erts_remove_dist_link(&dld, port_id, lnk->pid, dep); erts_destroy_dist_link(&dld); @@ -4203,17 +4195,9 @@ write_port_control_result(int control_flags, else { dbin = (ErlDrvBinary *) resp_bufp; if (dbin->orig_size > ERL_ONHEAP_BIN_LIMIT) { - ProcBin* pb = (ProcBin *) *hpp; + res = erts_build_proc_bin(ohp, *hpp, ErlDrvBinary2Binary(dbin)); *hpp += PROC_BIN_SIZE; - pb->thing_word = HEADER_PROC_BIN; - pb->size = dbin->orig_size; - pb->next = ohp->first; - ohp->first = (struct erl_off_heap_header *) pb; - pb->val = ErlDrvBinary2Binary(dbin); - pb->bytes = (byte*) dbin->orig_bytes; - pb->flags = 0; - OH_OVERHEAD(ohp, dbin->orig_size / sizeof(Eterm)); - return make_binary(pb); + return res; } resp_bufp = dbin->orig_bytes; resp_size = dbin->orig_size; @@ -5095,6 +5079,93 @@ static void prt_one_lnk(ErtsLink *lnk, void *vprtd) erts_print(prtd->to, prtd->arg, "%T", lnk->pid); } +static void dump_port_state(fmtfn_t to, void *arg, erts_aint32_t state) +{ + erts_aint32_t rest; + int unknown = 0; + char delim = ' '; + + erts_print(to, arg, "State:"); + + rest = state; + while (rest) { + erts_aint32_t chk = (rest ^ (rest-1)) & rest; /* lowest set bit */ + char* s; + + rest &= ~chk; + switch (chk) { + case ERTS_PORT_SFLG_CONNECTED: s = "CONNECTED"; break; + case ERTS_PORT_SFLG_EXITING: s = "EXITING"; break; + case ERTS_PORT_SFLG_DISTRIBUTION: s = "DISTR"; break; + case ERTS_PORT_SFLG_BINARY_IO: s = "BINARY_IO"; break; + case ERTS_PORT_SFLG_SOFT_EOF: s = "SOFT_EOF"; break; + case ERTS_PORT_SFLG_CLOSING: s = "CLOSING"; break; + case ERTS_PORT_SFLG_SEND_CLOSED: s = "SEND_CLOSED"; break; + case ERTS_PORT_SFLG_LINEBUF_IO: s = "LINEBUF_IO"; break; + case ERTS_PORT_SFLG_FREE: s = "FREE"; break; + case ERTS_PORT_SFLG_INITIALIZING: s = "INITIALIZING"; break; + case ERTS_PORT_SFLG_PORT_SPECIFIC_LOCK: s = "PORT_LOCK"; break; + case ERTS_PORT_SFLG_INVALID: s = "INVALID"; break; + case ERTS_PORT_SFLG_HALT: s = "HALT"; break; +#ifdef DEBUG + case ERTS_PORT_SFLG_PORT_DEBUG: s = "DEBUG"; break; +#endif + default: + unknown = 1; + continue; + } + erts_print(to, arg, "%c%s", delim, s); + delim = '|'; + } + if (unknown || !state) + erts_print(to, arg, "%c0x%x\n", delim, state); + else + erts_print(to, arg, "\n"); +} + +static void dump_port_task_flags(fmtfn_t to, void *arg, Port* p) +{ + erts_aint32_t flags = erts_atomic32_read_nob(&p->sched.flags); + erts_aint32_t unknown = 0; + char delim = ' '; + + if (!flags) + return; + + erts_print(to, arg, "Task Flags:"); + + while (flags) { + erts_aint32_t chk = (flags ^ (flags-1)) & flags; /* lowest set bit */ + char* s; + + flags &= ~chk; + switch (chk) { + case ERTS_PTS_FLG_IN_RUNQ: s = "IN_RUNQ"; break; + case ERTS_PTS_FLG_EXEC: s = "EXEC"; break; + case ERTS_PTS_FLG_HAVE_TASKS: s = "HAVE_TASKS"; break; + case ERTS_PTS_FLG_EXIT: s = "EXIT"; break; + case ERTS_PTS_FLG_BUSY_PORT: s = "BUSY_PORT"; break; + case ERTS_PTS_FLG_BUSY_PORT_Q: s = "BUSY_Q"; break; + case ERTS_PTS_FLG_CHK_UNSET_BUSY_PORT_Q: s = "CHK_UNSET_BUSY_Q"; break; + case ERTS_PTS_FLG_HAVE_BUSY_TASKS: s = "BUSY_TASKS"; break; + case ERTS_PTS_FLG_HAVE_NS_TASKS: s = "NS_TASKS"; break; + case ERTS_PTS_FLG_PARALLELISM: s = "PARALLELISM"; break; + case ERTS_PTS_FLG_FORCE_SCHED: s = "FORCE_SCHED"; break; + case ERTS_PTS_FLG_EXITING: s = "EXITING"; break; + case ERTS_PTS_FLG_EXEC_IMM: s = "EXEC_IMM"; break; + default: + unknown |= chk; + continue; + } + erts_print(to, arg, "%c%s", delim, s); + delim = '|'; + } + if (unknown) + erts_print(to, arg, "%cUNKNOWN(0x%x)\n", delim, unknown); + else + erts_print(to, arg, "\n"); +} + void print_port_info(Port *p, fmtfn_t to, void *arg) { @@ -5104,6 +5175,8 @@ print_port_info(Port *p, fmtfn_t to, void *arg) return; erts_print(to, arg, "=port:%T\n", p->common.id); + dump_port_state(to, arg, state); + dump_port_task_flags(to, arg, p); erts_print(to, arg, "Slot: %d\n", internal_port_index(p->common.id)); if (state & ERTS_PORT_SFLG_CONNECTED) { erts_print(to, arg, "Connected: %T", ERTS_PORT_GET_CONNECTED(p)); @@ -5126,6 +5199,10 @@ print_port_info(Port *p, fmtfn_t to, void *arg) erts_doforall_monitors(ERTS_P_MONITORS(p), &prt_one_monitor, &prtd); erts_print(to, arg, "\n"); } + if (p->suspended) { + erts_print(to, arg, "Suspended: "); + erts_proclist_dump(to, arg, p->suspended); + } if (p->common.u.alive.reg != NULL) erts_print(to, arg, "Registered as: %T\n", p->common.u.alive.reg->name); @@ -5143,6 +5220,14 @@ print_port_info(Port *p, fmtfn_t to, void *arg) } else { erts_print(to, arg, "Port controls linked-in driver: %s\n",p->name); } + erts_print(to, arg, "Input: %beu\n", p->bytes_in); + erts_print(to, arg, "Output: %beu\n", p->bytes_out); + erts_print(to, arg, "Queue: %beu\n", erts_ioq_size(&p->ioq)); + { + Eterm port_data = erts_port_data_read(p); + if (port_data != am_undefined) + erts_print(to, arg, "Port Data: %T\n", port_data); + } } void @@ -5977,21 +6062,12 @@ driver_deliver_term(Port *prt, Eterm to, ErlDrvTermData* data, int len) mess = make_binary(hbp); } else { - ProcBin* pbp; + Eterm* hp; Binary* bp = erts_bin_nrml_alloc(size); ASSERT(bufp); sys_memcpy((void *) bp->orig_bytes, (void *) bufp, size); - pbp = (ProcBin *) erts_produce_heap(&factory, - PROC_BIN_SIZE, HEAP_EXTRA); - pbp->thing_word = HEADER_PROC_BIN; - pbp->size = size; - pbp->next = factory.off_heap->first; - factory.off_heap->first = (struct erl_off_heap_header*)pbp; - pbp->val = bp; - pbp->bytes = (byte*) bp->orig_bytes; - pbp->flags = 0; - OH_OVERHEAD(factory.off_heap, pbp->size / sizeof(Eterm)); - mess = make_binary(pbp); + hp = erts_produce_heap(&factory, PROC_BIN_SIZE, HEAP_EXTRA); + mess = erts_build_proc_bin(factory.off_heap, hp, bp); } ptr += 2; break; @@ -7248,7 +7324,7 @@ int driver_failure_atom(ErlDrvPort ix, char* string) { return driver_failure_term(ix, erts_atom_put((byte *) string, - strlen(string), + sys_strlen(string), ERTS_ATOM_ENC_LATIN1, 1), 0); @@ -7714,13 +7790,27 @@ int null_func(void) int erl_drv_putenv(const char *key, char *value) { - return erts_sys_putenv_raw((char*)key, value); + switch (erts_sys_explicit_8bit_putenv((char*)key, value)) { + case -1: /* Insufficient buffer space */ + return 1; + case 1: /* Success */ + return 0; + default: /* Not found */ + return -1; + } } int erl_drv_getenv(const char *key, char *value, size_t *value_size) { - return erts_sys_getenv_raw((char*)key, value, value_size); + switch (erts_sys_explicit_8bit_getenv((char*)key, value, value_size)) { + case -1: /* Insufficient buffer space */ + return 1; + case 1: /* Success */ + return 0; + default: /* Not found */ + return -1; + } } /* get heart_port diff --git a/erts/emulator/beam/lttng-wrapper.h b/erts/emulator/beam/lttng-wrapper.h index 0bc75c1552..e7f2971bf7 100644 --- a/erts/emulator/beam/lttng-wrapper.h +++ b/erts/emulator/beam/lttng-wrapper.h @@ -61,13 +61,13 @@ #define lttng_proc_to_mfa_str(p, Name) \ do { \ if (ERTS_PROC_IS_EXITING((p))) { \ - strcpy(Name, "<exiting>"); \ + sys_strcpy(Name, "<exiting>"); \ } else { \ BeamInstr *_fptr = find_function_from_pc((p)->i); \ if (_fptr) { \ lttng_mfa_to_str(_fptr[0],_fptr[1],_fptr[2], Name); \ } else { \ - strcpy(Name, "<unknown>"); \ + sys_strcpy(Name, "<unknown>"); \ } \ } \ } while(0) diff --git a/erts/emulator/beam/macros.tab b/erts/emulator/beam/macros.tab index e0b5f56b53..494fe8961e 100644 --- a/erts/emulator/beam/macros.tab +++ b/erts/emulator/beam/macros.tab @@ -20,13 +20,12 @@ // // -// Use if there is a garbage collection before storing to a -// general destination (either X or Y register). +// Define a regular expression that will match instructions that +// perform GC. That will allow beam_makeops to check for instructions +// that don't use $REFRESH_GEN_DEST() when they should. // -REFRESH_GEN_DEST() { - dst_ptr = REG_TARGET_PTR(dst); -} +GC_REGEXP=erts_garbage_collect|erts_gc|GcBifFunction; // $Offset is relative to the start of the instruction (not to the // location of the failure label reference). Since combined diff --git a/erts/emulator/beam/map_instrs.tab b/erts/emulator/beam/map_instrs.tab index bbb2f49b66..c594a87298 100644 --- a/erts/emulator/beam/map_instrs.tab +++ b/erts/emulator/beam/map_instrs.tab @@ -31,7 +31,7 @@ new_map(Dst, Live, N) { Eterm res; HEAVY_SWAPOUT; - res = new_map(c_p, reg, $Live, $N, $NEXT_INSTRUCTION); + res = erts_gc_new_map(c_p, reg, $Live, $N, $NEXT_INSTRUCTION); HEAVY_SWAPIN; $REFRESH_GEN_DEST(); $Dst = res; @@ -44,7 +44,7 @@ i_new_small_map_lit(Dst, Live, Keys) { Eterm keys = $Keys; HEAVY_SWAPOUT; - res = new_small_map_lit(c_p, reg, keys, $Live, $NEXT_INSTRUCTION); + res = erts_gc_new_small_map_lit(c_p, reg, keys, $Live, $NEXT_INSTRUCTION); HEAVY_SWAPIN; $REFRESH_GEN_DEST(); $Dst = res; @@ -133,7 +133,7 @@ update_map_assoc(Src, Dst, Live, N) { reg[live] = $Src; HEAVY_SWAPOUT; - res = update_map_assoc(c_p, reg, live, $N, $NEXT_INSTRUCTION); + res = erts_gc_update_map_assoc(c_p, reg, live, $N, $NEXT_INSTRUCTION); HEAVY_SWAPIN; ASSERT(is_value(res)); $REFRESH_GEN_DEST(); @@ -147,7 +147,7 @@ update_map_exact(Fail, Src, Dst, Live, N) { reg[live] = $Src; HEAVY_SWAPOUT; - res = update_map_exact(c_p, reg, live, $N, $NEXT_INSTRUCTION); + res = erts_gc_update_map_exact(c_p, reg, live, $N, $NEXT_INSTRUCTION); HEAVY_SWAPIN; if (is_value(res)) { $REFRESH_GEN_DEST(); diff --git a/erts/emulator/beam/msg_instrs.tab b/erts/emulator/beam/msg_instrs.tab index 8055a8616f..d6d4d2fb49 100644 --- a/erts/emulator/beam/msg_instrs.tab +++ b/erts/emulator/beam/msg_instrs.tab @@ -43,27 +43,23 @@ // * // */ -recv_mark(Dest) { +i_recv_mark() { /* - * Save the current position in message buffer and the - * the label for the loop_rec/2 instruction for the - * the receive statement. + * Save the current position in message buffer. */ - $SET_REL_I(c_p->msg.mark, $Dest); c_p->msg.saved_last = c_p->msg.last; } i_recv_set() { /* - * If the mark is valid (points to the loop_rec/2 - * instruction that follows), we know that the saved - * position points to the first message that could - * possibly be matched out. + * If c_p->msg.saved_last is non-zero, it points to the first + * message that could possibly be matched out. * - * If the mark is invalid, we do nothing, meaning that - * we will look through all messages in the message queue. + * If c_p->msg.saved_last is zero, it means that it was invalidated + * because another receive was executed before this i_recv_set() + * instruction was reached. */ - if (c_p->msg.mark == (BeamInstr *) ($NEXT_INSTRUCTION)) { + if (c_p->msg.saved_last) { c_p->msg.save = c_p->msg.saved_last; } SET_I($NEXT_INSTRUCTION); @@ -131,6 +127,7 @@ i_loop_rec(Dest) { ASSERT(HTOP == c_p->htop && E == c_p->stop); /* TODO: Add DTrace probe for this bad message situation? */ UNLINK_MESSAGE(c_p, msgp); + c_p->msg.saved_last = 0; /* Better safe than sorry. */ msgp->next = NULL; erts_cleanup_messages(msgp); goto loop_rec__; diff --git a/erts/emulator/beam/ops.tab b/erts/emulator/beam/ops.tab index 7a2c39b3a8..77e375f2c0 100644 --- a/erts/emulator/beam/ops.tab +++ b/erts/emulator/beam/ops.tab @@ -182,6 +182,9 @@ get_list r x y get_list r y r get_list r x r +get_hd xy xy +get_tl xy xy + # Old-style catch. catch y f catch_end y @@ -503,6 +506,10 @@ i_put_tuple xy I # put_list Const=c n Dst => move Const x | put_list x n Dst +put_list Src Dst=x Dst => update_list Src Dst + +update_list xyc x + put_list x n x put_list y n x put_list x x x @@ -511,8 +518,6 @@ put_list y x x put_list y y x put_list x y x -put_list y x x - # put_list SrcReg Constant Dst put_list x c x @@ -527,8 +532,6 @@ put_list c y x # The following put_list instructions using x(0) are frequently used. -put_list y r r -put_list x r r put_list r n r put_list r n x put_list r x x @@ -539,6 +542,7 @@ put_list x x r put_list s s d %hot + # # Some more only used by the emulator # @@ -1567,8 +1571,20 @@ on_load # # R14A. # -recv_mark f +# Modified in OTP 21 because it turns out that we don't need the +# label after all. +# + +recv_mark f => i_recv_mark +i_recv_mark recv_set Fail | label Lbl | loop_rec Lf Reg => \ i_recv_set | label Lbl | loop_rec Lf Reg i_recv_set + +# +# OTP 21. +# + +build_stacktrace +raw_raise diff --git a/erts/emulator/beam/packet_parser.c b/erts/emulator/beam/packet_parser.c index f14910bc72..de1d481105 100644 --- a/erts/emulator/beam/packet_parser.c +++ b/erts/emulator/beam/packet_parser.c @@ -200,7 +200,7 @@ static int http_init(void) for (i = 0; i < HTTP_HDR_HASH_SIZE; i++) http_hdr_hash[i] = NULL; for (i = 0; http_hdr_strings[i] != NULL; i++) { - ASSERT(strlen(http_hdr_strings[i]) <= HTTP_MAX_NAME_LEN); + ASSERT(sys_strlen(http_hdr_strings[i]) <= HTTP_MAX_NAME_LEN); http_hdr_table[i].index = i; http_hash_insert(http_hdr_strings[i], &http_hdr_table[i], @@ -516,7 +516,7 @@ static http_atom_t* http_hash_lookup(const char* name, int len, while (ap != NULL) { if ((ap->h == h) && (ap->len == len) && - (strncmp(ap->name, name, len) == 0)) + (sys_strncmp(ap->name, name, len) == 0)) return ap; ap = ap->next; } @@ -656,7 +656,7 @@ int packet_parse_http(const char* buf, int len, int* statep, if (*statep == 0) { /* start-line = Request-Line | Status-Line */ - if (n >= 5 && (strncmp(buf, "HTTP/", 5) == 0)) { + if (n >= 5 && (sys_strncmp(buf, "HTTP/", 5) == 0)) { int major = 0; int minor = 0; int status = 0; @@ -750,7 +750,7 @@ int packet_parse_http(const char* buf, int len, int* statep, } if (n < 8) return -1; - if (strncmp(ptr, "HTTP/", 5) != 0) + if (sys_strncmp(ptr, "HTTP/", 5) != 0) return -1; ptr += 5; n -= 5; diff --git a/erts/emulator/beam/sys.h b/erts/emulator/beam/sys.h index bf7d310568..13ae80e4a5 100644 --- a/erts/emulator/beam/sys.h +++ b/erts/emulator/beam/sys.h @@ -636,6 +636,8 @@ typedef struct preload { */ typedef Eterm ErtsTracer; +#include "erl_osenv.h" + /* * This structure contains options to all built in drivers. * None of the drivers use all of the fields. @@ -651,8 +653,7 @@ typedef struct _SysDriverOpts { int hide_window; /* Hide this windows (Windows). */ int exit_status; /* Report exit status of subprocess. */ int overlapped_io; /* Only has effect on windows NT et al */ - char *envir; /* Environment of the port process, */ - /* in Windows format. */ + erts_osenv_t envir; /* Environment of the port process */ char **argv; /* Argument vector in Unix'ish format. */ char *wd; /* Working directory. */ unsigned spawn_type; /* Bitfield of ERTS_SPAWN_DRIVER | @@ -782,9 +783,6 @@ void set_break_quit(void (*)(void), void (*)(void)); void os_flavor(char*, unsigned); void os_version(int*, int*, int*); -void init_getenv_state(GETENV_STATE *); -char * getenv_string(GETENV_STATE *); -void fini_getenv_state(GETENV_STATE *); #define HAVE_ERTS_CHECK_IO_DEBUG typedef struct { @@ -805,20 +803,21 @@ int sys_double_to_chars_ext(double, char*, size_t, size_t); int sys_double_to_chars_fast(double, char*, int, int, int); void sys_get_pid(char *, size_t); -/* erts_sys_putenv() returns, 0 on success and a value != 0 on failure. */ -int erts_sys_putenv(char *key, char *value); -/* Simple variant used from drivers, raw eightbit interface */ -int erts_sys_putenv_raw(char *key, char *value); -/* erts_sys_getenv() returns 0 on success (length of value string in - *size), a value > 0 if value buffer is too small (*size is set to needed - size), and a value < 0 on failure. */ -int erts_sys_getenv(char *key, char *value, size_t *size); -/* Simple variant used from drivers, raw eightbit interface */ -int erts_sys_getenv_raw(char *key, char *value, size_t *size); -/* erts_sys_getenv__() is only allowed to be used in early init phase */ -int erts_sys_getenv__(char *key, char *value, size_t *size); -/* erst_sys_unsetenv() returns 0 on success and a value != 0 on failure. */ -int erts_sys_unsetenv(char *key); +/* erl_drv_get/putenv have been implicitly 8-bit for so long that we can't + * change them without breaking things on Windows. Their return values are + * identical to erts_osenv_get/putenv */ +int erts_sys_explicit_8bit_getenv(char *key, char *value, size_t *size); +int erts_sys_explicit_8bit_putenv(char *key, char *value); + +/* This is identical to erts_sys_explicit_8bit_getenv but falls down to the + * host OS implementation instead of erts_osenv. */ +int erts_sys_explicit_host_getenv(char *key, char *value, size_t *size); + +const erts_osenv_t *erts_sys_rlock_global_osenv(void); +void erts_sys_runlock_global_osenv(void); + +erts_osenv_t *erts_sys_rwlock_global_osenv(void); +void erts_sys_rwunlock_global_osenv(void); /* Easier to use, but not as efficient, environment functions */ char *erts_read_env(char *key); @@ -997,16 +996,36 @@ erts_refc_read(erts_refc_t *refcp, erts_aint_t min_val) #endif /* #if ERTS_GLB_INLINE_INCL_FUNC_DEF */ -#define sys_memcpy(s1,s2,n) memcpy(s1,s2,n) -#define sys_memmove(s1,s2,n) memmove(s1,s2,n) -#define sys_memcmp(s1,s2,n) memcmp(s1,s2,n) -#define sys_memset(s,c,n) memset(s,c,n) -#define sys_memzero(s, n) memset(s,'\0',n) -#define sys_strcmp(s1,s2) strcmp(s1,s2) -#define sys_strncmp(s1,s2,n) strncmp(s1,s2,n) -#define sys_strcpy(s1,s2) strcpy(s1,s2) -#define sys_strncpy(s1,s2,n) strncpy(s1,s2,n) -#define sys_strlen(s) strlen(s) +/* Thin wrappers around memcpy and friends, which should always be used in + * place of plain memcpy, memset, etc. + * + * Passing NULL to any of these functions is undefined behavior even though it + * may seemingly work when the length (if any) is zero; a compiler can take + * this as a hint that the passed operand may *never* be NULL and then optimize + * based on that information. + * + * (The weird casts in the assertions silence an "always evaluates to true" + * warning when an operand is the address of an lvalue) */ +#define sys_memcpy(s1,s2,n) \ + (ASSERT((void*)(s1) != NULL && (void*)(s2) != NULL), memcpy(s1,s2,n)) +#define sys_memmove(s1,s2,n) \ + (ASSERT((void*)(s1) != NULL && (void*)(s2) != NULL), memmove(s1,s2,n)) +#define sys_memcmp(s1,s2,n) \ + (ASSERT((void*)(s1) != NULL && (void*)(s2) != NULL), memcmp(s1,s2,n)) +#define sys_memset(s,c,n) \ + (ASSERT((void*)(s) != NULL), memset(s,c,n)) +#define sys_memzero(s, n) \ + (ASSERT((void*)(s) != NULL), memset(s,'\0',n)) +#define sys_strcmp(s1,s2) \ + (ASSERT((void*)(s1) != NULL && (void*)(s2) != NULL), strcmp(s1,s2)) +#define sys_strncmp(s1,s2,n) \ + (ASSERT((void*)(s1) != NULL && (void*)(s2) != NULL), strncmp(s1,s2,n)) +#define sys_strcpy(s1,s2) \ + (ASSERT((void*)(s1) != NULL && (void*)(s2) != NULL), strcpy(s1,s2)) +#define sys_strncpy(s1,s2,n) \ + (ASSERT((void*)(s1) != NULL && (void*)(s2) != NULL), strncpy(s1,s2,n)) +#define sys_strlen(s) \ + (ASSERT((void*)(s) != NULL), strlen(s)) /* define function symbols (needed in sys_drv_api) */ #define sys_fp_alloc sys_alloc diff --git a/erts/emulator/beam/utils.c b/erts/emulator/beam/utils.c index 993585be10..4bf60619ba 100644 --- a/erts/emulator/beam/utils.c +++ b/erts/emulator/beam/utils.c @@ -3152,6 +3152,9 @@ tailrecur_ne: int cmp; byte* a_ptr; byte* b_ptr; + if (eq_only && a_size != b_size) { + RETURN_NEQ(a_size - b_size); + } ERTS_GET_BINARY_BYTES(a, a_ptr, a_bitoffs, a_bitsize); ERTS_GET_BINARY_BYTES(b, b_ptr, b_bitoffs, b_bitsize); if ((a_bitsize | b_bitsize | a_bitoffs | b_bitoffs) == 0) { @@ -4360,15 +4363,20 @@ erts_read_env(char *key) char *value = erts_alloc(ERTS_ALC_T_TMP, value_len); int res; while (1) { - res = erts_sys_getenv_raw(key, value, &value_len); - if (res <= 0) - break; - value = erts_realloc(ERTS_ALC_T_TMP, value, value_len); + res = erts_sys_explicit_8bit_getenv(key, value, &value_len); + + if (res >= 0) { + break; + } + + value = erts_realloc(ERTS_ALC_T_TMP, value, value_len); } - if (res != 0) { - erts_free(ERTS_ALC_T_TMP, value); - return NULL; + + if (res != 1) { + erts_free(ERTS_ALC_T_TMP, value); + return NULL; } + return value; } diff --git a/erts/emulator/drivers/common/efile_drv.c b/erts/emulator/drivers/common/efile_drv.c deleted file mode 100644 index 4e1d2f0d7f..0000000000 --- a/erts/emulator/drivers/common/efile_drv.c +++ /dev/null @@ -1,4295 +0,0 @@ -/* - * %CopyrightBegin% - * - * Copyright Ericsson AB 1996-2017. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - * %CopyrightEnd% - */ -/* - * Purpose: Provides file and directory operations. - * - * This file is generic, and does the work of decoding the commands - * and encoding the responses. System-specific functions are found in - * the unix_efile.c and win_efile.c files. - */ - -/* Operations */ - -#define FILE_OPEN 1 /* Essential for startup */ -#define FILE_READ 2 -#define FILE_LSEEK 3 -#define FILE_WRITE 4 -#define FILE_FSTAT 5 /* Essential for startup */ -#define FILE_PWD 6 /* Essential for startup */ -#define FILE_READDIR 7 /* Essential for startup */ -#define FILE_CHDIR 8 -#define FILE_FSYNC 9 -#define FILE_MKDIR 10 -#define FILE_DELETE 11 -#define FILE_RENAME 12 -#define FILE_RMDIR 13 -#define FILE_TRUNCATE 14 -#define FILE_READ_FILE 15 /* Essential for startup */ -#define FILE_WRITE_INFO 16 -#define FILE_LSTAT 19 -#define FILE_READLINK 20 -#define FILE_LINK 21 -#define FILE_SYMLINK 22 -#define FILE_CLOSE 23 -#define FILE_PWRITEV 24 -#define FILE_PREADV 25 -#define FILE_SETOPT 26 -#define FILE_IPREAD 27 -#define FILE_ALTNAME 28 -#define FILE_READ_LINE 29 -#define FILE_FDATASYNC 30 -#define FILE_FADVISE 31 -#define FILE_SENDFILE 32 -#define FILE_FALLOCATE 33 -#define FILE_CLOSE_ON_PORT_EXIT 34 -/* Return codes */ - -#define FILE_RESP_OK 0 -#define FILE_RESP_ERROR 1 -#define FILE_RESP_DATA 2 -#define FILE_RESP_NUMBER 3 -#define FILE_RESP_INFO 4 -#define FILE_RESP_NUMERR 5 -#define FILE_RESP_LDATA 6 -#define FILE_RESP_N2DATA 7 -#define FILE_RESP_EOF 8 -#define FILE_RESP_FNAME 9 -#define FILE_RESP_ALL_DATA 10 -#define FILE_RESP_LFNAME 11 - -/* Options */ - -#define FILE_OPT_DELAYED_WRITE 0 -#define FILE_OPT_READ_AHEAD 1 - -/* IPREAD variants */ - -#define IPREAD_S32BU_P32BU 0 - -/* Limits */ - -#define FILE_SEGMENT_READ (256*1024) -#define FILE_SEGMENT_WRITE (256*1024) - -/* Internal */ - -/* Set to 1 to test having read_ahead implicitly for read_line */ -#define ALWAYS_READ_LINE_AHEAD 0 - - -/* Must not be possible to get from malloc()! */ -#define FILE_FD_INVALID ((Sint)(-1)) - -#ifdef HAVE_CONFIG_H -# include "config.h" -#endif - -#include <ctype.h> -#include <sys/types.h> -#include <stdlib.h> - -/* Need (NON)BLOCKING macros for sendfile */ -#ifndef WANT_NONBLOCKING -#define WANT_NONBLOCKING -#endif - -#include "sys.h" - -#include "erl_driver.h" -#include "erl_efile.h" -#include "erl_threads.h" -#include "gzio.h" -#include "dtrace-wrapper.h" - - -static ErlDrvSysInfo sys_info; - -/* For explanation of this var, see comment for same var in erl_async.c */ -static unsigned gcc_optimizer_hack = 0; - -#ifdef USE_VM_PROBES - -#define DTRACE_EFILE_BUFSIZ 128 - -#define DTRACE_INVOKE_SETUP(op) \ - do { DTRACE3(efile_drv_int_entry, d->sched_i1, d->sched_i2, op); } while (0) -#define DTRACE_INVOKE_SETUP_BY_NAME(op) \ - struct t_data *d = (struct t_data *) data ; \ - DTRACE_INVOKE_SETUP(op) -#define DTRACE_INVOKE_RETURN(op) \ - do { DTRACE3(efile_drv_int_return, d->sched_i1, d->sched_i2, \ - op); } while (0) ; gcc_optimizer_hack++ ; - -/* Assign human-friendlier id numbers to scheduler & I/O worker threads */ -int dt_driver_idnum = 0; -int dt_driver_io_worker_base = 5000; -erts_mtx_t dt_driver_mutex; -pthread_key_t dt_driver_key; - -typedef struct { - int thread_num; - Uint64 tag; -} dt_private; - -dt_private *get_dt_private(int); -#else /* USE_VM_PROBES */ -#define DTRACE_INVOKE_SETUP(op) do {} while (0) -#define DTRACE_INVOKE_SETUP_BY_NAME(op) do {} while (0) -#define DTRACE_INVOKE_RETURN(op) do {} while (0) -#endif /* USE_VM_PROBES */ - -/* #define TRACE 1 */ -#ifdef TRACE -# define TRACE_C(c) do { putchar(c); fflush(stdout); } while (0) -# define TRACE_S(s) do { fputs((s), stdout); fflush(stdout); } while (0) -# define TRACE_F(args) do { printf args ;fflush(stdout); } while (0) -#else -# define TRACE_C(c) ((void)(0)) -# define TRACE_S(s) ((void)(0)) -# define TRACE_F(args) ((void)(0)) -#endif - - -#define THRDS_AVAILABLE (sys_info.async_threads > 0) -#ifdef HARDDEBUG /* HARDDEBUG in io.c is expected too */ -#define TRACE_DRIVER fprintf(stderr, "Efile: ") -#else -#define TRACE_DRIVER -#endif -#define MUTEX_INIT(m, p) do { IF_THRDS { TRACE_DRIVER; (m = driver_pdl_create(p)); } } while (0) -#define MUTEX_LOCK(m) do { IF_THRDS { TRACE_DRIVER; driver_pdl_lock(m); } } while (0) -#define MUTEX_UNLOCK(m) do { IF_THRDS { TRACE_DRIVER; driver_pdl_unlock(m); } } while (0) -#define IF_THRDS if (THRDS_AVAILABLE) - - -#define SENDFILE_FLGS_USE_THREADS (1 << 0) -/** - * On DARWIN sendfile can deadlock with close if called in - * different threads. So until Apple fixes so that sendfile - * is not buggy we disable usage of the async pool for - * DARWIN. The testcase t_sendfile_crashduring reproduces - * this error when using +A 10 and enabling SENDFILE_FLGS_USE_THREADS. - */ -#if defined(__APPLE__) && defined(__MACH__) -#define USE_THRDS_FOR_SENDFILE(DATA) 0 -#else -#define USE_THRDS_FOR_SENDFILE(DATA) (DATA->flags & SENDFILE_FLGS_USE_THREADS) -#endif /* defined(__APPLE__) && defined(__MACH__) */ - - - -#if 0 -/* Experimental, for forcing all file operations to use the same thread. */ - static unsigned file_fixed_key = 1; -# define KEY(desc) (&file_fixed_key) -#else -# define KEY(desc) (&(desc)->key) -#endif - -#ifndef MAX -# define MAX(x, y) (((x) > (y)) ? (x) : (y)) -#endif - -#ifdef FILENAMES_16BIT -#ifdef USE_VM_PROBES -#error 16bit characters in filenames and dtrace in combination is not supported. -#endif -# define FILENAME_BYTELEN(Str) filename_len_16bit(Str) -# define FILENAME_COPY(To,From) filename_cpy_16bit((To),(From)) -# define FILENAME_CHARSIZE 2 - - static int filename_len_16bit(char *str) - { - char *p = str; - while(*p != '\0' || p[1] != '\0') { - p += 2; - } - return (p - str); - } - - static void filename_cpy_16bit(char *to, char *from) - { - while(*from != '\0' || from[1] != '\0') { - *to++ = *from++; - *to++ = *from++; - } - *to++ = *from++; - *to++ = *from++; - } - -#else -# define FILENAME_BYTELEN(Str) strlen(Str) -# define FILENAME_COPY(To,From) strcpy(To,From) -# define FILENAME_CHARSIZE 1 -#endif - -#if (MAXPATHLEN+1)*FILENAME_CHARSIZE+1 > BUFSIZ -# define RESBUFSIZE ((MAXPATHLEN+1)*FILENAME_CHARSIZE+1) -#else -# define RESBUFSIZE BUFSIZ -#endif - -#define READDIR_CHUNKS (5) - - - -#if ALWAYS_READ_LINE_AHEAD -#define DEFAULT_LINEBUF_SIZE 2048 -#else -#define DEFAULT_LINEBUF_SIZE 512 /* Small, it's usually discarded anyway */ -#endif - -typedef unsigned char uchar; - -static ErlDrvData file_start(ErlDrvPort port, char* command); -static int file_init(void); -static void file_stop(ErlDrvData); -static void file_output(ErlDrvData, char* buf, ErlDrvSizeT len); -static ErlDrvSSizeT file_control(ErlDrvData, unsigned int command, - char* buf, ErlDrvSizeT len, - char **rbuf, ErlDrvSizeT rlen); -static void file_timeout(ErlDrvData); -static void file_outputv(ErlDrvData, ErlIOVec*); -static void file_async_ready(ErlDrvData, ErlDrvThreadData); -static void file_flush(ErlDrvData); - -#ifdef HAVE_SENDFILE -static void file_ready_output(ErlDrvData data, ErlDrvEvent event); -static void file_stop_select(ErlDrvEvent event, void* _); -#endif /* HAVE_SENDFILE */ - - -enum e_timer {timer_idle, timer_again, timer_write}; -#ifdef HAVE_SENDFILE -enum e_sendfile {sending, not_sending}; -#define SENDFILE_USE_THREADS (1 << 0) -#endif /* HAVE_SENDFILE */ - -struct t_data; - -typedef struct { - SWord fd; - ErlDrvPort port; - unsigned int key; /* Async queue key */ - unsigned flags; /* Original flags from FILE_OPEN. */ - void (*invoke)(void *); - struct t_data *d; - void (*free)(void *); - struct t_data *cq_head; /* Queue of incoming commands */ - struct t_data *cq_tail; /* -""- */ - enum e_timer timer_state; -#ifdef HAVE_SENDFILE - enum e_sendfile sendfile_state; -#endif /* HAVE_SENDFILE */ - size_t read_bufsize; - ErlDrvBinary *read_binp; - size_t read_offset; - size_t read_size; - size_t write_bufsize; - unsigned long write_delay; - int write_error; - Efile_error write_errInfo; - ErlDrvPDL q_mtx; /* Mutex for the driver queue, known by the emulator. Also used for - mutual exclusion when accessing field(s) below. */ - size_t write_buffered; -#ifdef USE_VM_PROBES - int idnum; /* Unique ID # for this driver thread/desc */ - char port_str[DTRACE_TERM_BUF_SIZE]; -#endif -} file_descriptor; - - -static int reply_error(file_descriptor*, Efile_error* errInfo); - -struct erl_drv_entry efile_driver_entry = { - file_init, - file_start, - file_stop, - file_output, - NULL, -#ifdef HAVE_SENDFILE - file_ready_output, -#else - NULL, -#endif /* HAVE_SENDFILE */ - "efile", - NULL, - NULL, - file_control, - file_timeout, - file_outputv, - file_async_ready, - file_flush, - NULL, - NULL, - ERL_DRV_EXTENDED_MARKER, - ERL_DRV_EXTENDED_MAJOR_VERSION, - ERL_DRV_EXTENDED_MINOR_VERSION, - ERL_DRV_FLAG_USE_PORT_LOCKING, - NULL, - NULL, -#ifdef HAVE_SENDFILE - file_stop_select -#else - NULL -#endif /* HAVE_SENDFILE */ -}; - - - -static int thread_short_circuit; - -#define DRIVER_ASYNC(level, desc, f_invoke, data, f_free) \ -if (thread_short_circuit >= (level)) { \ - (*(f_invoke))(data); \ - file_async_ready((ErlDrvData)(desc), (data)); \ -} else { \ - driver_async((desc)->port, KEY(desc), (f_invoke), (data), (f_free)); \ -} - - - -struct t_pbuf_spec { - Sint64 offset; - size_t size; -}; - -struct t_pwritev { - ErlDrvPort port; - ErlDrvPDL q_mtx; - size_t size; - unsigned cnt; - unsigned n; - struct t_pbuf_spec specs[1]; -}; - -struct t_preadv { - ErlIOVec eiov; - unsigned n; - unsigned cnt; - size_t size; - Sint64 offsets[1]; -}; - -#define READDIR_BUFSIZE (8*1024)*READDIR_CHUNKS -#if READDIR_BUFSIZE < (1 + (2 + MAXPATHLEN)*FILENAME_CHARSIZE*READDIR_CHUNKS) -# undef READDIR_BUFSIZE -# define READDIR_BUFSIZE (1 + (2 + MAXPATHLEN)*FILENAME_CHARSIZE*READDIR_CHUNKS) -#endif - -struct t_readdir_buf { - struct t_readdir_buf *next; - size_t n; - char buf[READDIR_BUFSIZE]; -}; - -struct t_data -{ - struct t_data *next; - int command; - int level; - void (*invoke)(void *); - void (*free)(void *); - void *data_to_free; /* used by FILE_CLOSE_ON_PORT_EXIT only */ - int again; - int reply; -#ifdef USE_VM_PROBES - int sched_i1; - Uint64 sched_i2; - char sched_utag[DTRACE_EFILE_BUFSIZ+1]; -#endif - int result_ok; - Efile_error errInfo; - int flags; - SWord fd; - int is_fd_unused; - /**/ - Efile_info info; - EFILE_DIR_HANDLE dir_handle; /* Handle to open directory. */ - ErlDrvBinary *bin; - int drive; - size_t n; - /*off_t offset;*/ - /*size_t bytesRead; Bytes read from the file. */ - /**/ - union { - struct { - Sint64 offset; - int origin; - Sint64 location; - } lseek; - struct { - ErlDrvPort port; - ErlDrvPDL q_mtx; - size_t size; - size_t reply_size; - } writev; - struct t_pwritev pwritev; - struct t_preadv preadv; - struct { - ErlDrvBinary *binp; - size_t bin_offset; - size_t bin_size; - size_t size; - } read; - struct { - ErlDrvBinary *binp; /* in - out */ - size_t read_offset; /* in - out */ - size_t read_size; /* in - out */ - size_t nl_pos; /* out */ - short nl_skip; /* out, 0 or 1 */ -#if !ALWAYS_READ_LINE_AHEAD - short read_ahead; /* in, bool */ -#endif - } read_line; - struct { - ErlDrvBinary *binp; - int size; - int offset; - } read_file; - struct { - struct t_readdir_buf *first_buf; - struct t_readdir_buf *last_buf; - } read_dir; - struct { - Sint64 offset; - Sint64 length; - int advise; - } fadvise; -#ifdef HAVE_SENDFILE - struct { - ErlDrvPort port; - ErlDrvPDL q_mtx; - int out_fd; - off_t offset; - Uint64 nbytes; - Uint64 written; - } sendfile; -#endif /* HAVE_SENDFILE */ - struct { - Sint64 offset; - Sint64 length; - } fallocate; - } c; - char b[1]; -}; - -#define EF_ALLOC(S) driver_alloc((S)) -#define EF_REALLOC(P, S) driver_realloc((P), (S)) -#define EF_SAFE_ALLOC(S) ef_safe_alloc((S)) -#define EF_SAFE_REALLOC(P, S) ef_safe_realloc((P), (S)) -#define EF_FREE(P) do { if((P)) driver_free((P)); } while(0) - -static void *ef_safe_alloc(Uint s) -{ - void *p = EF_ALLOC(s); - if (!p) erts_exit(ERTS_ERROR_EXIT, "efile drv: Can't allocate %lu bytes of memory\n", (unsigned long)s); - return p; -} - -/********************************************************************* - * ErlIOVec manipulation functions. - */ - -/* char EV_CHAR_P(ErlIOVec *ev, int p, int q) */ -#define EV_CHAR_P(ev, p, q) \ - (((char *)(ev)->iov[q].iov_base) + (p)) - -/* int EV_GET_CHAR(ErlIOVec *ev, char *p, int *pp, int *qp) */ -#define EV_GET_CHAR(ev, p, pp, qp) efile_ev_get_char(ev, p ,pp, qp) -static int -efile_ev_get_char(ErlIOVec *ev, char *p, size_t *pp, size_t *qp) { - if (*pp + 1 <= ev->iov[*qp].iov_len) { - *p = *EV_CHAR_P(ev, *pp, *qp); - if (*pp + 1 < ev->iov[*qp].iov_len) - *pp += 1; - else { - *qp += 1; - *pp = 0; - } - return !0; - } - return 0; -} - -/* Uint32 EV_UINT32(ErlIOVec *ev, int p, int q)*/ -#define EV_UINT32(ev, p, q) \ - ((Uint32) ((unsigned char *)(ev)->iov[q].iov_base)[p]) - -/* int EV_GET_UINT32(ErlIOVec *ev, Uint32 *p, int *pp, int *qp) */ -#define EV_GET_UINT32(ev, p, pp, qp) efile_ev_get_uint32(ev, p, pp, qp) -static int -efile_ev_get_uint32(ErlIOVec *ev, Uint32 *p, size_t *pp, size_t *qp) { - if (*pp + 4 <= ev->iov[*qp].iov_len) { - *p = (EV_UINT32(ev, *pp, *qp) << 24) - | (EV_UINT32(ev, *pp + 1, *qp) << 16) - | (EV_UINT32(ev, *pp + 2, *qp) << 8) - | (EV_UINT32(ev, *pp + 3, *qp)); - if (*pp + 4 < ev->iov[*qp].iov_len) - *pp += 4; - else { - *qp += 1; - *pp = 0; - } - return !0; - } - return 0; -} - -/* Uint64 EV_UINT64(ErlIOVec *ev, int p, int q)*/ -#define EV_UINT64(ev, p, q) \ - ((Uint64) ((unsigned char *)(ev)->iov[q].iov_base)[p]) - -/* int EV_GET_UINT64(ErlIOVec *ev, Uint64 *p, int *pp, int *qp) */ -#define EV_GET_UINT64(ev, p, pp, qp) efile_ev_get_uint64(ev, p, pp, qp) -static int -efile_ev_get_uint64(ErlIOVec *ev, Uint64 *p, size_t *pp, size_t *qp) { - if (*pp + 8 <= ev->iov[*qp].iov_len) { - *p = (EV_UINT64(ev, *pp, *qp) << 56) - | (EV_UINT64(ev, *pp + 1, *qp) << 48) - | (EV_UINT64(ev, *pp + 2, *qp) << 40) - | (EV_UINT64(ev, *pp + 3, *qp) << 32) - | (EV_UINT64(ev, *pp + 4, *qp) << 24) - | (EV_UINT64(ev, *pp + 5, *qp) << 16) - | (EV_UINT64(ev, *pp + 6, *qp) << 8) - | (EV_UINT64(ev, *pp + 7, *qp)); - if (*pp + 8 < ev->iov[*qp].iov_len) - *pp += 8; - else { - *qp += 1; - *pp = 0; - } - return !0; - } - return 0; -} - -/* int EV_GET_SINT64(ErlIOVec *ev, Uint64 *p, int *pp, int *qp) */ -#define EV_GET_SINT64(ev, p, pp, qp) efile_ev_get_sint64(ev, p, pp, qp) -static int -efile_ev_get_sint64(ErlIOVec *ev, Sint64 *p, size_t *pp, size_t *qp) { - Uint64 *tmp = (Uint64*)p; - return EV_GET_UINT64(ev, tmp, pp, qp); -} - -#if 0 - -static void ev_clear(ErlIOVec *ev) { - ASSERT(ev); - ev->size = 0; - ev->vsize = 0; - ev->iov = NULL; - ev->binv = NULL; -} - -/* Assumes that ->iov and ->binv were allocated with sys_alloc(). - */ -static void ev_free(ErlIOVec *ev) { - if (! ev) { - return; - } - if (ev->vsize > 0) { - int i; - ASSERT(ev->iov); - ASSERT(ev->binv); - for (i = 0; i < ev->vsize; i++) { - if (ev->binv[i]) { - driver_free_binary(ev->binv[i]); - } - } - EF_FREE(ev->iov); - EF_FREE(ev->binv); - } -} - -/* Copy the contents from source to dest. - * Data in binaries is not copied, just the pointers; - * and refc is incremented. - */ -static ErlIOVec *ev_copy(ErlIOVec *dest, ErlIOVec *source) { - int *ip; - ASSERT(dest); - ASSERT(source); - if (source->vsize == 0) { - /* Empty source */ - ev_clear(dest); - return dest; - } - /* Allocate ->iov and ->binv */ - dest->iov = EF_ALLOC(sizeof(*dest->iov) * source->vsize); - if (! dest->iov) { - return NULL; - } - dest->binv = EF_ALLOC(sizeof(*dest->binv) * source->vsize); - if (! dest->binv) { - EF_FREE(dest->iov); - return NULL; - } - dest->size = source->size; - /* Copy one vector element at the time. - * Use *ip as an alias for dest->vsize to improve readabiliy. - * Keep dest consistent in every iteration by using - * dest->vsize==*ip as loop variable. - */ - for (ip = &dest->vsize, *ip = 0; *ip < source->vsize; (*ip)++) { - if (source->iov[*ip].iov_len == 0) { - /* Empty vector element */ - dest->iov[*ip].iov_len = 0; - dest->iov[*ip].iov_base = NULL; - dest->binv[*ip] = NULL; - } else { - /* Non empty vector element */ - if (source->binv[*ip]) { - /* Contents in binary - copy pointers and increment refc */ - dest->iov[*ip] = source->iov[*ip]; - dest->binv[*ip] = source->binv[*ip]; - driver_binary_inc_refc(source->binv[*ip]); - } else { - /* Contents not in binary - allocate new binary and copy data */ - if (! (dest->binv[*ip] = - driver_alloc_binary(source->iov[*ip].iov_len))) { - goto failed; - } - sys_memcpy(dest->binv[*ip]->orig_bytes, - source->iov[*ip].iov_base, - source->iov[*ip].iov_len); - dest->iov[*ip].iov_base = dest->binv[*ip]->orig_bytes; - dest->iov[*ip].iov_len = source->iov[*ip].iov_len; - } - } - } - return dest; - failed: - ev_free(dest); - return NULL; -} - -#endif - - - -/********************************************************************* - * Command queue functions - */ - -static void cq_enq(file_descriptor *desc, struct t_data *d) { - ASSERT(d); - if (desc->cq_head) { - ASSERT(desc->cq_tail); - ASSERT(!desc->cq_tail->next); - desc->cq_tail = desc->cq_tail->next = d; - } else { - ASSERT(desc->cq_tail == NULL); - desc->cq_head = desc->cq_tail = d; - } - d->next = NULL; -} - -static struct t_data *cq_deq(file_descriptor *desc) { - struct t_data *d = desc->cq_head; - ASSERT(d || (!d && !desc->cq_tail)); - if (d) { - ASSERT(!d->next || (d->next && desc->cq_tail != d)); - if ((desc->cq_head = d->next) == NULL) { - ASSERT(desc->cq_tail == d); - desc->cq_tail = NULL; - } - } - return d; -} - - -/********************************************************************* - * Driver entry point -> init - */ -static int -file_init(void) -{ - char buf[21]; /* enough to hold any 64-bit integer */ - size_t bufsz = sizeof(buf); - thread_short_circuit = (erl_drv_getenv("ERL_EFILE_THREAD_SHORT_CIRCUIT", - buf, - &bufsz) == 0 - ? atoi(buf) - : 0); - driver_system_info(&sys_info, sizeof(ErlDrvSysInfo)); - - /* run initiation of efile_driver if needed */ - efile_init(); - -#ifdef USE_VM_PROBES - erts_mtx_init(&dt_driver_mutex, "efile_drv dtrace mutex", NIL, - ERTS_LOCK_FLAGS_PROPERTY_STATIC | ERTS_LOCK_FLAGS_CATEGORY_IO); - pthread_key_create(&dt_driver_key, NULL); -#endif /* USE_VM_PROBES */ - - return 0; -} - - -/********************************************************************* - * Driver entry point -> start - */ -static ErlDrvData -file_start(ErlDrvPort port, char* command) - -{ - file_descriptor* desc; - - if ((desc = (file_descriptor*) EF_ALLOC(sizeof(file_descriptor))) - == NULL) { - errno = ENOMEM; - return ERL_DRV_ERROR_ERRNO; - } - desc->fd = FILE_FD_INVALID; - desc->port = port; - desc->key = driver_async_port_key(port); - desc->flags = 0; - desc->invoke = NULL; - desc->d = NULL; - desc->free = NULL; - desc->cq_head = NULL; - desc->cq_tail = NULL; - desc->timer_state = timer_idle; -#ifdef HAVE_SENDFILE - desc->sendfile_state = not_sending; -#endif - desc->read_bufsize = 0; - desc->read_binp = NULL; - desc->read_offset = 0; - desc->read_size = 0; - desc->write_delay = 0L; - desc->write_bufsize = 0; - desc->write_error = 0; - MUTEX_INIT(desc->q_mtx, port); /* Refc is one, referenced by emulator now */ - desc->write_buffered = 0; -#ifdef USE_VM_PROBES - dtrace_drvport_str(port, desc->port_str); - get_dt_private(0); /* throw away return value */ -#endif /* USE_VM_PROBES */ - return (ErlDrvData) desc; -} - -static void do_close(int flags, SWord fd) { - if (flags & EFILE_COMPRESSED) { - erts_gzclose((ErtsGzFile)(fd)); - } else { - efile_closefile((int) fd); - } -} - -static void invoke_close(void *data) -{ - struct t_data *d = (struct t_data *) data; - DTRACE_INVOKE_SETUP(FILE_CLOSE); - d->again = 0; - do_close(d->flags, d->fd); - DTRACE_INVOKE_RETURN(FILE_CLOSE); -} - -static void free_data(void *data) -{ - struct t_data *d = (struct t_data *) data; - - switch (d->command) { - case FILE_OPEN: - if (d->is_fd_unused && d->fd != FILE_FD_INVALID) { - /* This is OK to do in scheduler thread because there can be no async op - ongoing for this fd here, as we exited during async open. - Ideally, this close should happen in an async thread too, but that would - require a substantial rewrite, as we are here because of a dead port and - cannot schedule async jobs for that port any more... */ - do_close(d->flags, d->fd); - } - break; - case FILE_CLOSE_ON_PORT_EXIT: - EF_FREE(d->data_to_free); - break; - } - - EF_FREE(data); -} - - -/* - * Sends back an error reply to Erlang. - */ - -static void reply_posix_error(file_descriptor *desc, int posix_errno) { - char response[256]; /* Response buffer. */ - char* s; - char* t; - - /* - * Contents of buffer sent back: - * - * +-----------------------------------------+ - * | FILE_RESP_ERROR | Posix error id string | - * +-----------------------------------------+ - */ - - TRACE_C('E'); - - response[0] = FILE_RESP_ERROR; - for (s = erl_errno_id(posix_errno), t = response+1; *s; s++, t++) - *t = tolower(*s); - driver_output2(desc->port, response, t-response, NULL, 0); -} - -static void reply_Uint_posix_error(file_descriptor *desc, Uint num, - int posix_errno) { - char response[256]; /* Response buffer. */ - char* s; - char* t; - - /* - * Contents of buffer sent back: - * - * +----------------------------------------------------------------------+ - * | FILE_RESP_NUMERR | 64-bit number (big-endian) | Posix error id string | - * +----------------------------------------------------------------------+ - */ - - TRACE_C('N'); - - response[0] = FILE_RESP_NUMERR; -#if SIZEOF_VOID_P == 4 - put_int32(0, response+1); -#else - put_int32(num>>32, response+1); -#endif - put_int32((Uint32)num, response+1+4); - for (s = erl_errno_id(posix_errno), t = response+1+4+4; *s; s++, t++) - *t = tolower(*s); - driver_output2(desc->port, response, t-response, NULL, 0); -} - -#ifdef HAVE_SENDFILE -static void reply_string_error(file_descriptor *desc, char* str) { - char response[256]; /* Response buffer. */ - char* s; - char* t; - - response[0] = FILE_RESP_ERROR; - for (s = str, t = response+1; *s; s++, t++) - *t = tolower(*s); - driver_output2(desc->port, response, t-response, NULL, 0); -} -#endif - -static int reply_error(file_descriptor *desc, - Efile_error *errInfo) /* The error codes. */ -{ - reply_posix_error(desc, errInfo->posix_errno); - return 0; -} - -static int reply_Uint_error(file_descriptor *desc, Uint num, - Efile_error *errInfo) /* The error codes. */ -{ - reply_Uint_posix_error(desc, num, errInfo->posix_errno); - return 0; -} - -static int reply_ok(file_descriptor *desc) { - char c = FILE_RESP_OK; - - driver_output2(desc->port, &c, 1, NULL, 0); - return 0; -} - -static int reply(file_descriptor *desc, int ok, Efile_error *errInfo) { - if (!ok) { - reply_error(desc, errInfo); - } else { - TRACE_C('K'); - reply_ok(desc); - } - return 0; -} - -static int reply_Uint(file_descriptor *desc, Uint result) { - char tmp[1+4+4]; - - /* - * Contents of buffer sent back: - * - * +-----------------------------------------------+ - * | FILE_RESP_NUMBER | 64-bit number (big-endian) | - * +-----------------------------------------------+ - */ - - TRACE_C('R'); - - tmp[0] = FILE_RESP_NUMBER; -#if SIZEOF_VOID_P == 4 - put_int32(0, tmp+1); -#else - put_int32(result>>32, tmp+1); -#endif - put_int32((Uint32)result, tmp+1+4); - driver_output2(desc->port, tmp, sizeof(tmp), NULL, 0); - return 0; -} - -static int reply_Sint64(file_descriptor *desc, Sint64 result) { - char tmp[1+4+4]; - - /* - * Contents of buffer sent back: - * - * +-----------------------------------------------+ - * | FILE_RESP_NUMBER | 64-bit number (big-endian) | - * +-----------------------------------------------+ - */ - - TRACE_C('R'); - - tmp[0] = FILE_RESP_NUMBER; - put_int64(result, tmp+1); - driver_output2(desc->port, tmp, sizeof(tmp), NULL, 0); - return 0; -} - -#if 0 -static void reply_again(file_descriptor *desc) { - char tmp[1]; - tmp[0] = FILE_RESP_AGAIN; - driver_output2(desc->port, tmp, sizeof(tmp), NULL, 0); -} -#endif - -static void reply_ev(file_descriptor *desc, char response, ErlIOVec *ev) { - char tmp[1]; - /* Data arriving at the Erlang process: - * [Response, Binary0, Binary1, .... | BinaryN-1] - */ - tmp[0] = response; - driver_outputv(desc->port, tmp, sizeof(tmp), ev, 0); -} - -static void reply_data(file_descriptor *desc, - ErlDrvBinary *binp, size_t offset, size_t len) { - char header[1+4+4]; - /* Data arriving at the Erlang process: - * [?FILE_RESP_DATA, 64-bit length (big-endian) | Data] - */ - header[0] = FILE_RESP_DATA; -#if SIZEOF_SIZE_T == 4 - put_int32(0, header+1); -#else - put_int32(len>>32, header+1); -#endif - put_int32((Uint32)len, header+1+4); - driver_output_binary(desc->port, header, sizeof(header), - binp, offset, len); -} - -static void reply_buf(file_descriptor *desc, char *buf, size_t len) { - char header[1+4+4]; - /* Data arriving at the Erlang process: - * [?FILE_RESP_DATA, 64-bit length (big-endian) | Data] - */ - header[0] = FILE_RESP_DATA; -#if SIZEOF_SIZE_T == 4 - put_int32(0, header+1); -#else - put_int32(len>>32, header+1); -#endif - put_int32((Uint32)len, header+1+4); - driver_output2(desc->port, header, sizeof(header), buf, len); -} - -static int reply_eof(file_descriptor *desc) { - char c = FILE_RESP_EOF; - - driver_output2(desc->port, &c, 1, NULL, 0); - return 0; -} - -static void invoke_name(void *data, int (*f)(Efile_error *, char *)) -{ - struct t_data *d = (struct t_data *) data; - char *name = (char *) d->b; - - d->again = 0; - d->result_ok = (*f)(&d->errInfo, name); -} - -static void invoke_mkdir(void *data) -{ - DTRACE_INVOKE_SETUP_BY_NAME(FILE_MKDIR); - invoke_name(data, efile_mkdir); - DTRACE_INVOKE_RETURN(FILE_MKDIR); -} - -static void invoke_rmdir(void *data) -{ - DTRACE_INVOKE_SETUP_BY_NAME(FILE_RMDIR); - invoke_name(data, efile_rmdir); - DTRACE_INVOKE_RETURN(FILE_RMDIR); -} - -static void invoke_delete_file(void *data) -{ - DTRACE_INVOKE_SETUP_BY_NAME(FILE_DELETE); - invoke_name(data, efile_delete_file); - DTRACE_INVOKE_RETURN(FILE_DELETE); -} - -static void invoke_chdir(void *data) -{ - DTRACE_INVOKE_SETUP_BY_NAME(FILE_CHDIR); - invoke_name(data, efile_chdir); - DTRACE_INVOKE_RETURN(FILE_CHDIR); -} - -static void invoke_fdatasync(void *data) -{ - struct t_data *d = (struct t_data *) data; - int fd = (int) d->fd; - DTRACE_INVOKE_SETUP(FILE_FDATASYNC); - - d->again = 0; - d->result_ok = efile_fdatasync(&d->errInfo, fd); - DTRACE_INVOKE_RETURN(FILE_FDATASYNC); -} - -static void invoke_fsync(void *data) -{ - struct t_data *d = (struct t_data *) data; - int fd = (int) d->fd; - DTRACE_INVOKE_SETUP(FILE_FSYNC); - - d->again = 0; - d->result_ok = efile_fsync(&d->errInfo, fd); - DTRACE_INVOKE_RETURN(FILE_FSYNC); -} - -static void invoke_truncate(void *data) -{ - struct t_data *d = (struct t_data *) data; - int fd = (int) d->fd; - DTRACE_INVOKE_SETUP(FILE_TRUNCATE); - - d->again = 0; - d->result_ok = efile_truncate_file(&d->errInfo, &fd, d->flags); - DTRACE_INVOKE_RETURN(FILE_TRUNCATE); -} - -static void invoke_read(void *data) -{ - struct t_data *d = (struct t_data *) data; - int status, segment; - size_t size, read_size; - DTRACE_INVOKE_SETUP(FILE_READ); - - segment = d->again && d->c.read.bin_size >= 2*FILE_SEGMENT_READ; - if (segment) { - size = FILE_SEGMENT_READ; - } else { - size = d->c.read.bin_size; - } - read_size = size; - if (d->flags & EFILE_COMPRESSED) { - read_size = erts_gzread((ErtsGzFile)d->fd, - d->c.read.binp->orig_bytes + d->c.read.bin_offset, - size); - status = (read_size != (size_t) -1); - if (!status) { - d->errInfo.posix_errno = EIO; - } - } else { - status = efile_read(&d->errInfo, d->flags, (int) d->fd, - d->c.read.binp->orig_bytes + d->c.read.bin_offset, - size, - &read_size); - } - if ( (d->result_ok = status)) { - ASSERT(read_size <= size); - d->c.read.bin_offset += read_size; - if (read_size < size || !segment) { - d->c.read.bin_size = 0; - d->again = 0; - } else { - d->c.read.bin_size -= read_size; - } - } else { - d->again = 0; - } - DTRACE_INVOKE_RETURN(FILE_READ); -} - -static void free_read(void *data) -{ - struct t_data *d = (struct t_data *) data; - - driver_free_binary(d->c.read.binp); - EF_FREE(d); -} - -static void invoke_read_line(void *data) -{ - struct t_data *d = (struct t_data *) data; - int status; - size_t read_size = 0; - int local_loop = (d->again == 0); - DTRACE_INVOKE_SETUP(FILE_READ_LINE); - - do { - size_t size = (d->c.read_line.binp)->orig_size - - d->c.read_line.read_offset - d->c.read_line.read_size; - if (size == 0) { - /* Need more place */ - ErlDrvSizeT need = (d->c.read_line.read_size >= DEFAULT_LINEBUF_SIZE) ? - d->c.read_line.read_size + DEFAULT_LINEBUF_SIZE : DEFAULT_LINEBUF_SIZE; - ErlDrvBinary *newbin; -#if !ALWAYS_READ_LINE_AHEAD - /* Use read_ahead size if need does not exceed it */ - if (need < (d->c.read_line.binp)->orig_size && - d->c.read_line.read_ahead) - need = (d->c.read_line.binp)->orig_size; -#endif - newbin = driver_alloc_binary(need); - if (newbin == NULL) { - d->result_ok = 0; - d->errInfo.posix_errno = ENOMEM; - d->again = 0; - break; - } - memcpy(newbin->orig_bytes, (d->c.read_line.binp)->orig_bytes + d->c.read_line.read_offset, - d->c.read_line.read_size); - driver_free_binary(d->c.read_line.binp); - d->c.read_line.binp = newbin; - d->c.read_line.read_offset = 0; - size = need - d->c.read_line.read_size; - } - if (d->flags & EFILE_COMPRESSED) { - read_size = erts_gzread((ErtsGzFile)d->fd, - d->c.read_line.binp->orig_bytes + - d->c.read_line.read_offset + d->c.read_line.read_size, - size); - status = (read_size != (size_t) -1); - if (!status) { - d->errInfo.posix_errno = EIO; - } - } else { - status = efile_read(&d->errInfo, d->flags, (int) d->fd, - d->c.read_line.binp->orig_bytes + - d->c.read_line.read_offset + d->c.read_line.read_size, - size, - &read_size); - } - if ( (d->result_ok = status)) { - void *nl_ptr = memchr((d->c.read_line.binp)->orig_bytes + - d->c.read_line.read_offset + d->c.read_line.read_size,'\n',read_size); - ASSERT(read_size <= size); - d->c.read_line.read_size += read_size; - if (nl_ptr != NULL) { - /* If found, we're done */ - d->c.read_line.nl_pos = ((char *) nl_ptr) - - ((char *) ((d->c.read_line.binp)->orig_bytes)) + 1; - if (d->c.read_line.nl_pos > 1 && - *(((char *) nl_ptr) - 1) == '\r') { - --d->c.read_line.nl_pos; - *(((char *) nl_ptr) - 1) = '\n'; - d->c.read_line.nl_skip = 1; - } else { - d->c.read_line.nl_skip = 0; - } - d->again = 0; -#if !ALWAYS_READ_LINE_AHEAD - if (!(d->c.read_line.read_ahead)) { - /* Ouch! Undo buffering... */ - size_t too_much = d->c.read_line.read_size - d->c.read_line.nl_skip - - (d->c.read_line.nl_pos - d->c.read_line.read_offset); - d->c.read_line.read_size -= too_much; - ASSERT(d->c.read_line.read_size >= 0); - if (d->flags & EFILE_COMPRESSED) { - Sint64 location = erts_gzseek((ErtsGzFile)d->fd, - -((Sint64) too_much), EFILE_SEEK_CUR); - if (location == -1) { - d->result_ok = 0; - d->errInfo.posix_errno = errno; - } - } else { - Sint64 location; - d->result_ok = efile_seek(&d->errInfo, (int) d->fd, - -((Sint64) too_much), EFILE_SEEK_CUR, - &location); - } - } -#endif - break; - } else if (read_size == 0) { - d->c.read_line.nl_pos = - d->c.read_line.read_offset + d->c.read_line.read_size; - d->c.read_line.nl_skip = 0; - d->again = 0; - break; - } - } else { - d->again = 0; - break; - } - } while (local_loop); - DTRACE_INVOKE_RETURN(FILE_READ_LINE); -} - -static void free_read_line(void *data) -{ - struct t_data *d = (struct t_data *) data; - - driver_free_binary(d->c.read_line.binp); - EF_FREE(d); -} - -void read_file_zero_size(struct t_data* d); -#define ZERO_FILE_CHUNK (64 * 1024) - -/* [ERL-327] Some special files like /proc/... have reported size 0 */ -void read_file_zero_size(struct t_data* d) { - size_t total_read_size = 0; - size_t allocated_size = ZERO_FILE_CHUNK; /* allocd in invoke_read_file */ - for (;;) { - size_t read_result; - - /* Read until we hit EOF (read less than FILE_SEGMENT_READ) */ - d->result_ok = efile_read(&d->errInfo, - EFILE_MODE_READ, - (int) d->fd, - (d->c.read_file.binp->orig_bytes + - total_read_size), - ZERO_FILE_CHUNK, - &read_result); - if (!d->result_ok) { - break; - } - - total_read_size += read_result; - d->c.read_file.offset += read_result; - if (read_result < ZERO_FILE_CHUNK) { - break; - } - - /* Grow before the next read call */ - allocated_size = total_read_size + ZERO_FILE_CHUNK; - d->c.read_file.binp = driver_realloc_binary(d->c.read_file.binp, - allocated_size); - } - - /* Finalize the memory usage. Hopefully it was read fully on the first - * go, so the binary allocation overhead becomes: - * alloc ZERO_FILE_CHUNK (64kb) -> realloc real_size */ - if (allocated_size != total_read_size) { - d->c.read_file.binp = driver_realloc_binary(d->c.read_file.binp, - total_read_size); - } - d->again = 0; -} - -static void invoke_read_file(void *data) -{ - struct t_data *d = (struct t_data *) data; - size_t read_size; - int chop; - DTRACE_INVOKE_SETUP(FILE_READ_FILE); - - if (! d->c.read_file.binp) { /* First invocation only */ - int fd; - Sint64 size; - - if (! (d->result_ok = - efile_openfile(&d->errInfo, d->b, - EFILE_MODE_READ, &fd, &size))) { - goto done; - } - d->fd = fd; - d->c.read_file.size = (int) size; - - /* For zero sized files allocate a reasonable chunk to attempt reading - * anyway. Note: This will eat ZERO_FILE_CHUNK bytes for any 0 file - * and free them immediately after (if the file was empty). */ - ERTS_ASSERT(size >= 0); - d->c.read_file.binp = driver_alloc_binary(size != 0 ? (size_t)size - : ZERO_FILE_CHUNK); - - if (size < 0 || size != d->c.read_file.size || !d->c.read_file.binp) { - d->result_ok = 0; - d->errInfo.posix_errno = ENOMEM; - goto close; - } - d->c.read_file.offset = 0; - } - /* Invariant: d->c.read_file.size >= d->c.read_file.offset */ - - if (d->c.read_file.size == 0) { - read_file_zero_size(d); - goto close; - } - - read_size = (size_t) (d->c.read_file.size - d->c.read_file.offset); - if (! read_size) goto close; - chop = d->again && read_size >= FILE_SEGMENT_READ*2; - if (chop) read_size = FILE_SEGMENT_READ; - d->result_ok = - efile_read(&d->errInfo, - EFILE_MODE_READ, - (int) d->fd, - d->c.read_file.binp->orig_bytes + d->c.read_file.offset, - read_size, - &read_size); - if (d->result_ok) { - d->c.read_file.offset += read_size; - if (chop) goto chop_done; /* again */ - } - close: - efile_closefile((int) d->fd); - done: - d->again = 0; - chop_done: - DTRACE_INVOKE_RETURN(FILE_READ_FILE); -} - -static void free_read_file(void *data) -{ - struct t_data *d = (struct t_data *) data; - - if (d->c.read_file.binp) driver_free_binary(d->c.read_file.binp); - EF_FREE(d); -} - - - -static void invoke_preadv(void *data) -{ - struct t_data *d = (struct t_data *) data; - struct t_preadv *c = &d->c.preadv; - ErlIOVec *ev = &c->eiov; - size_t bytes_read_so_far = 0; - unsigned char *p = (unsigned char *)ev->iov[0].iov_base + 4+4+8*c->cnt; - DTRACE_INVOKE_SETUP(FILE_PREADV); - - while (c->cnt < c->n) { - size_t read_size = ev->iov[1 + c->cnt].iov_len - c->size; - size_t bytes_read = 0; - int chop = d->again - && bytes_read_so_far + read_size >= 2*FILE_SEGMENT_READ; - if (chop) { - ASSERT(bytes_read_so_far < FILE_SEGMENT_READ); - read_size = FILE_SEGMENT_READ + FILE_SEGMENT_READ/2 - - bytes_read_so_far; - } - if ( (d->result_ok - = efile_pread(&d->errInfo, - (int) d->fd, - c->offsets[c->cnt] + c->size, - ((char *)ev->iov[1 + c->cnt].iov_base) + c->size, - read_size, - &bytes_read))) { - bytes_read_so_far += bytes_read; - if (chop && bytes_read == read_size) { - c->size += bytes_read; - goto done; - } - ASSERT(bytes_read <= read_size); - ev->iov[1 + c->cnt].iov_len = bytes_read + c->size; - ev->size += bytes_read + c->size; - put_int64(bytes_read + c->size, p); p += 8; - c->size = 0; - c->cnt++; - if (d->again - && bytes_read_so_far >= FILE_SEGMENT_READ - && c->cnt < c->n) { - goto done; - } - } else { - /* In case of a read error, ev->size will not be correct, - * which does not matter since no read data is returned - * to Erlang. - */ - break; - } - } - d->again = 0; - done: - DTRACE_INVOKE_RETURN(FILE_PREADV); -} - -static void free_preadv(void *data) { - struct t_data *d = data; - int i; - ErlIOVec *ev = &d->c.preadv.eiov; - - for(i = 0; i < ev->vsize; i++) { - driver_free_binary(ev->binv[i]); - } - EF_FREE(d); -} - -static void invoke_ipread(void *data) -{ - struct t_data *d = data; - struct t_preadv *c = &d->c.preadv; - ErlIOVec *ev = &c->eiov; - size_t bytes_read = 0; - char buf[2*sizeof(Uint32)]; - Uint32 offset, size; - DTRACE_INVOKE_SETUP(FILE_IPREAD); - - /* Read indirection header */ - if (! efile_pread(&d->errInfo, (int) d->fd, c->offsets[0], - buf, sizeof(buf), &bytes_read)) { - goto error; - } - if (bytes_read != sizeof(buf)) goto done; /* eof */ - size = get_int32(buf); - offset = get_int32(buf+4); - if (size > c->size) goto done; /* eof */ - c->n = 1; - c->cnt = 0; - c->size = 0; - c->offsets[0] = offset; - if (! (ev->binv[0] = driver_alloc_binary(3*8))) { - d->errInfo.posix_errno = ENOMEM; - goto error; - } - ev->vsize = 1; - ev->iov[0].iov_len = 3*8; - ev->iov[0].iov_base = ev->binv[0]->orig_bytes; - ev->size = ev->iov[0].iov_len; - put_int64(offset, ev->iov[0].iov_base); - put_int64(size, ((char *)ev->iov[0].iov_base) + 2*8); - if (size == 0) { - put_int64(size, ((char *)ev->iov[0].iov_base) + 8); - goto done; - } - if (! (ev->binv[1] = driver_alloc_binary(size))) { - d->errInfo.posix_errno = ENOMEM; - goto error; - } - ev->vsize = 2; - ev->iov[1].iov_len = size; - ev->iov[1].iov_base = ev->binv[1]->orig_bytes; - /* Read data block */ - d->invoke = invoke_preadv; - invoke_preadv(data); - DTRACE_INVOKE_RETURN(FILE_IPREAD); - return; - error: - d->result_ok = 0; - d->again = 0; - DTRACE_INVOKE_RETURN(FILE_IPREAD); - return; - done: - d->result_ok = !0; - d->again = 0; - DTRACE_INVOKE_RETURN(FILE_IPREAD); -} - -/* invoke_writev and invoke_pwritev are the only thread functions that - * access non-thread data i.e the port queue and a mutex in the port - * structure that is used to lock the port queue. - * - * The port will normally not be terminated until the port queue is - * empty, but if the port is killed, i.e., exit(Port, kill) is called, - * it will terminate regardless of the port queue state. When the - * port is invalid driver_peekq() returns NULL and set the size to -1, - * and driver_sizeq() returns -1. - */ - -static void invoke_writev(void *data) { - struct t_data *d = (struct t_data *) data; - SysIOVec *iov0; - SysIOVec *iov; - int iovlen; - int iovcnt; - size_t size; - size_t p; - int segment; - DTRACE_INVOKE_SETUP(FILE_WRITE); - - segment = d->again && d->c.writev.size >= 2*FILE_SEGMENT_WRITE; - if (segment) { - size = FILE_SEGMENT_WRITE; - } else { - size = d->c.writev.size; - } - - /* Copy the io vector to avoid locking the port que while writing, - * also, both we and efile_writev might/will change the SysIOVec - * when segmenting or due to partial write and we do not want to - * tamper with the actual queue that we get from driver_peekq - */ - MUTEX_LOCK(d->c.writev.q_mtx); /* Lock before accessing the port queue */ - iov0 = driver_peekq(d->c.writev.port, &iovlen); - - /* Calculate iovcnt */ - for (p = 0, iovcnt = 0; - p < size && iovcnt < iovlen; - p += iov0[iovcnt++].iov_len) - ; - iov = EF_SAFE_ALLOC(sizeof(SysIOVec)*iovcnt); - memcpy(iov,iov0,iovcnt*sizeof(SysIOVec)); - MUTEX_UNLOCK(d->c.writev.q_mtx); - /* Let go of lock until we deque from original vector */ - - if (iovlen > 0) { - ASSERT(iov[iovcnt-1].iov_len > p - size); - iov[iovcnt-1].iov_len -= p - size; - if (d->flags & EFILE_COMPRESSED) { - int i, status = 1; - for (i = 0; i < iovcnt; i++) { - if (iov[i].iov_base && iov[i].iov_len > 0) { - /* Just in case, I do not know what gzwrite does - * with errno. - */ - errno = EINVAL; - status = erts_gzwrite((ErtsGzFile)d->fd, - iov[i].iov_base, - iov[i].iov_len) == iov[i].iov_len; - if (! status) { - d->errInfo.posix_errno = - d->errInfo.os_errno = errno; /* XXX Correct? */ - break; - } - } - } - d->result_ok = status; - } else { - d->result_ok = efile_writev(&d->errInfo, - d->flags, (int) d->fd, - iov, iovcnt); - } - } else if (iovlen == 0) { - d->result_ok = 1; - } - else { /* Port has terminated */ - d->result_ok = 0; - d->errInfo.posix_errno = d->errInfo.os_errno = EINVAL; - } - EF_FREE(iov); - - if (! d->result_ok) { - d->again = 0; - MUTEX_LOCK(d->c.writev.q_mtx); - driver_deq(d->c.writev.port, d->c.writev.size); - MUTEX_UNLOCK(d->c.writev.q_mtx); - } else { - if (! segment) { - d->again = 0; - } - d->c.writev.size -= size; - TRACE_F(("w%lu", (unsigned long)size)); - MUTEX_LOCK(d->c.writev.q_mtx); - driver_deq(d->c.writev.port, size); - MUTEX_UNLOCK(d->c.writev.q_mtx); - } - - - DTRACE_INVOKE_RETURN(FILE_WRITE); -} - -static void invoke_pwd(void *data) -{ - struct t_data *d = (struct t_data *) data; - DTRACE_INVOKE_SETUP(FILE_PWD); - - d->again = 0; - d->result_ok = efile_getdcwd(&d->errInfo,d->drive, d->b+1, - RESBUFSIZE-1); - DTRACE_INVOKE_RETURN(FILE_PWD); -} - -static void invoke_readlink(void *data) -{ - struct t_data *d = (struct t_data *) data; - char resbuf[RESBUFSIZE]; /* Result buffer. */ - DTRACE_INVOKE_SETUP(FILE_READLINK); - - d->again = 0; - d->result_ok = efile_readlink(&d->errInfo, d->b, resbuf+1, - RESBUFSIZE-1); - if (d->result_ok != 0) - FILENAME_COPY((char *) d->b + 1, resbuf+1); - DTRACE_INVOKE_RETURN(FILE_READLINK); -} - -static void invoke_altname(void *data) -{ - struct t_data *d = (struct t_data *) data; - char resbuf[RESBUFSIZE]; /* Result buffer. */ - DTRACE_INVOKE_SETUP(FILE_ALTNAME); - - d->again = 0; - d->result_ok = efile_altname(&d->errInfo, d->b, resbuf+1, - RESBUFSIZE-1); - if (d->result_ok != 0) - FILENAME_COPY((char *) d->b + 1, resbuf+1); - DTRACE_INVOKE_RETURN(FILE_ALTNAME); -} - -static void invoke_pwritev(void *data) { - struct t_data* const d = (struct t_data *) data; - struct t_pwritev * const c = &d->c.pwritev; - SysIOVec *iov0; - SysIOVec *iov; - int iovlen; - int iovcnt; - size_t p; - int segment; - size_t size, write_size, written; - DTRACE_INVOKE_SETUP(FILE_PWRITEV); - - segment = d->again && c->size >= 2*FILE_SEGMENT_WRITE; - if (segment) { - size = FILE_SEGMENT_WRITE; - } else { - size = c->size; - } - d->result_ok = !0; - p = 0; - /* Lock the queue just for a while, we don't want it locked during write */ - MUTEX_LOCK(c->q_mtx); - iov0 = driver_peekq(c->port, &iovlen); - iov = EF_SAFE_ALLOC(sizeof(SysIOVec)*iovlen); - memcpy(iov,iov0,sizeof(SysIOVec)*iovlen); - MUTEX_UNLOCK(c->q_mtx); - - if (iovlen < 0) - goto error; /* Port terminated */ - for (iovcnt = 0, written = 0; - c->cnt < c->n && iovcnt < iovlen && written < size; - c->cnt++) { - int chop; - write_size = c->specs[c->cnt].size; - if (iov[iovcnt].iov_len - p < write_size) { - goto error; - } - chop = segment && written + write_size >= 2*FILE_SEGMENT_WRITE; - if (chop) { - ASSERT(written < FILE_SEGMENT_WRITE); - write_size = FILE_SEGMENT_WRITE + FILE_SEGMENT_WRITE/2 - - written; - } - d->result_ok = efile_pwrite(&d->errInfo, (int) d->fd, - (char *)(iov[iovcnt].iov_base) + p, - write_size, - c->specs[c->cnt].offset); - if (! d->result_ok) { - d->again = 0; - goto deq_error; - } - written += write_size; - c->size -= write_size; - if (chop) { - c->specs[c->cnt].offset += write_size; - c->specs[c->cnt].size -= write_size; - /* Schedule out (d->again != 0) */ - break; - } - /* Move forward in buffer */ - p += write_size; - ASSERT(iov[iovcnt].iov_len >= p); - if (iov[iovcnt].iov_len == p) { - /* Move to next iov[], we trust that it is not a - * zero length vector, and thereby depend on that - * such are not queued. - */ - iovcnt++; p = 0; - } - } - if (! segment) { - if (c->cnt != c->n) { - /* Mismatch between number of - * pos/size specs vs number of queued buffers . - */ - error: - d->errInfo.posix_errno = EINVAL; - d->result_ok = 0; - d->again = 0; - deq_error: - MUTEX_LOCK(c->q_mtx); - driver_deq(c->port, c->size); - MUTEX_UNLOCK(c->q_mtx); - - goto done; - } else { - ASSERT(written == size); - d->again = 0; - } - } else { - ASSERT(written >= FILE_SEGMENT_WRITE); - } - - MUTEX_LOCK(c->q_mtx); - driver_deq(c->port, written); - MUTEX_UNLOCK(c->q_mtx); - done: - EF_FREE(iov); /* Free our copy of the vector, nothing to restore */ - - DTRACE_INVOKE_RETURN(FILE_PWRITEV); -} - -static void invoke_flstat(void *data) -{ - struct t_data *d = (struct t_data *) data; - - DTRACE3(efile_drv_int_entry, d->sched_i1, d->sched_i2, - d->command == FILE_LSTAT ? FILE_LSTAT : FILE_FSTAT); - d->again = 0; - d->result_ok = efile_fileinfo(&d->errInfo, &d->info, - d->b, d->command == FILE_LSTAT); - DTRACE3(efile_drv_int_entry, d->sched_i1, d->sched_i2, - d->command == FILE_LSTAT ? FILE_LSTAT : FILE_FSTAT); - gcc_optimizer_hack++; -} - -static void invoke_link(void *data) -{ - struct t_data *d = (struct t_data *) data; - char *name = d->b; - char *new_name; - DTRACE_INVOKE_SETUP(FILE_LINK); - - d->again = 0; - new_name = name+FILENAME_BYTELEN(name)+FILENAME_CHARSIZE; - d->result_ok = efile_link(&d->errInfo, name, new_name); - DTRACE_INVOKE_RETURN(FILE_LINK); -} - -static void invoke_symlink(void *data) -{ - struct t_data *d = (struct t_data *) data; - char *name = d->b; - char *new_name; - DTRACE_INVOKE_SETUP(FILE_SYMLINK); - - d->again = 0; - new_name = name+FILENAME_BYTELEN(name)+FILENAME_CHARSIZE; - d->result_ok = efile_symlink(&d->errInfo, name, new_name); - DTRACE_INVOKE_RETURN(FILE_SYMLINK); -} - -static void invoke_rename(void *data) -{ - struct t_data *d = (struct t_data *) data; - char *name = d->b; - char *new_name; - DTRACE_INVOKE_SETUP(FILE_RENAME); - - d->again = 0; - new_name = name+FILENAME_BYTELEN(name)+FILENAME_CHARSIZE; - d->result_ok = efile_rename(&d->errInfo, name, new_name); - DTRACE_INVOKE_RETURN(FILE_RENAME); -} - -static void invoke_write_info(void *data) -{ - struct t_data *d = (struct t_data *) data; - DTRACE_INVOKE_SETUP(FILE_WRITE_INFO); - - d->again = 0; - d->result_ok = efile_write_info(&d->errInfo, &d->info, d->b); - DTRACE_INVOKE_RETURN(FILE_WRITE_INFO); -} - -static void invoke_lseek(void *data) -{ - struct t_data *d = (struct t_data *) data; - int status; - DTRACE_INVOKE_SETUP(FILE_LSEEK); - - d->again = 0; - if (d->flags & EFILE_COMPRESSED) { - int offset = (int) d->c.lseek.offset; - - if (offset != d->c.lseek.offset) { - d->errInfo.posix_errno = EINVAL; - status = 0; - } else { - d->c.lseek.location = erts_gzseek((ErtsGzFile)d->fd, - offset, d->c.lseek.origin); - if (d->c.lseek.location == -1) { - d->errInfo.posix_errno = errno; - status = 0; - } else { - status = 1; - } - } - } else { - status = efile_seek(&d->errInfo, (int) d->fd, - d->c.lseek.offset, d->c.lseek.origin, - &d->c.lseek.location); - } - d->result_ok = status; - DTRACE_INVOKE_RETURN(FILE_LSEEK); -} - -static void invoke_readdir(void *data) -{ - struct t_data *d = (struct t_data *) data; - char *p = NULL; - size_t file_bs; - size_t n = 0, total = 0; - struct t_readdir_buf *b = NULL; - int res = 0; - DTRACE_INVOKE_SETUP(FILE_READDIR); - - d->again = 0; - d->errInfo.posix_errno = 0; - - do { - total = READDIR_BUFSIZE; - n = 1; - b = EF_SAFE_ALLOC(sizeof(struct t_readdir_buf)); - b->next = NULL; - - if (d->c.read_dir.last_buf) { - d->c.read_dir.last_buf->next = b; - } else { - d->c.read_dir.first_buf = b; - } - d->c.read_dir.last_buf = b; - - p = &b->buf[0]; - p[0] = FILE_RESP_LFNAME; - file_bs = READDIR_BUFSIZE - n; - - do { - res = efile_readdir(&d->errInfo, d->b, &d->dir_handle, p + n + 2, &file_bs); - - if (res) { - put_int16((Uint16)file_bs, p + n); - n += 2 + file_bs; - file_bs = READDIR_BUFSIZE - n; - } - } while( res && ((total - n - 2) >= MAXPATHLEN*FILENAME_CHARSIZE)); - - b->n = n; - } while(res); - - d->result_ok = (d->errInfo.posix_errno == 0); - DTRACE_INVOKE_RETURN(FILE_READDIR); -} - -static void invoke_open(void *data) -{ - struct t_data *d = (struct t_data *) data; - int status = 1; /* Status of open call. */ - DTRACE_INVOKE_SETUP(FILE_OPEN); - - d->again = 0; - if ((d->flags & EFILE_COMPRESSED) == 0) { - int fd; - status = efile_openfile(&d->errInfo, d->b, d->flags, &fd, NULL); - d->fd = fd; - } else { - char* mode = NULL; - - if (((d->flags & (EFILE_MODE_READ_WRITE)) == EFILE_MODE_READ_WRITE) || - (d->flags & EFILE_MODE_APPEND)) { - status = 0; - d->errInfo.posix_errno = EINVAL; - } else { - status = efile_may_openfile(&d->errInfo, d->b); - if (status || (d->errInfo.posix_errno != EISDIR)) { - mode = (d->flags & EFILE_MODE_READ) ? "rb" : "wb"; - d->fd = (SWord) erts_gzopen(d->b, mode); - if ((ErtsGzFile)d->fd) { - status = 1; - } else { - if (errno == 0) { - errno = ENOMEM; - } - d->errInfo.posix_errno = errno; - status = 0; - } - } - } - } - - d->result_ok = status; - if (!status) { - d->fd = FILE_FD_INVALID; - } - DTRACE_INVOKE_RETURN(FILE_OPEN); -} - -static void invoke_fadvise(void *data) -{ - struct t_data *d = (struct t_data *) data; - int fd = (int) d->fd; - off_t offset = (off_t) d->c.fadvise.offset; - off_t length = (off_t) d->c.fadvise.length; - int advise = (int) d->c.fadvise.advise; - DTRACE_INVOKE_SETUP(FILE_FADVISE); - - d->again = 0; - d->result_ok = efile_fadvise(&d->errInfo, fd, offset, length, advise); - DTRACE_INVOKE_RETURN(FILE_FADVISE); -} - -#ifdef HAVE_SENDFILE -static void invoke_sendfile(void *data) -{ - struct t_data *d = (struct t_data *)data; - int fd = d->fd; - int out_fd = (int)d->c.sendfile.out_fd; - Uint64 nbytes = d->c.sendfile.nbytes; - int result = 0; - d->again = 0; - - result = efile_sendfile(&d->errInfo, fd, out_fd, &d->c.sendfile.offset, &nbytes, NULL); - - d->c.sendfile.written += nbytes; - - if (result == 1 || (result == 0 && USE_THRDS_FOR_SENDFILE(d))) { - d->result_ok = 0; - } else if (result == 0 && (d->errInfo.posix_errno == EAGAIN - || d->errInfo.posix_errno == EINTR)) { - if ((d->c.sendfile.nbytes - nbytes) != 0) { - d->result_ok = 1; - if (d->c.sendfile.nbytes != 0) - d->c.sendfile.nbytes -= nbytes; - } else if (nbytes == 0 && d->c.sendfile.nbytes == 0) { - d->result_ok = 1; - } else - d->result_ok = 0; - } else { - d->result_ok = -1; - } -} - -static void free_sendfile(void *data) { - struct t_data *d = (struct t_data *)data; - if (USE_THRDS_FOR_SENDFILE(d)) { - SET_NONBLOCKING(d->c.sendfile.out_fd); - } else { - MUTEX_LOCK(d->c.sendfile.q_mtx); - driver_deq(d->c.sendfile.port,1); - MUTEX_UNLOCK(d->c.sendfile.q_mtx); - driver_select(d->c.sendfile.port, (ErlDrvEvent)(long)d->c.sendfile.out_fd, - ERL_DRV_USE_NO_CALLBACK|ERL_DRV_WRITE, 0); - } - EF_FREE(data); -} - -static void file_ready_output(ErlDrvData data, ErlDrvEvent event) -{ - file_descriptor* fd = (file_descriptor*) data; - - switch (fd->d->command) { - case FILE_SENDFILE: - driver_select(fd->d->c.sendfile.port, event, - (int)ERL_DRV_WRITE,(int) 0); - invoke_sendfile((void *)fd->d); - file_async_ready(data, (ErlDrvThreadData)fd->d); - break; - default: - break; - } -} - -static void file_stop_select(ErlDrvEvent event, void* _) -{ - -} - -static int flush_sendfile(file_descriptor *desc,void *_) { - if (desc->sendfile_state == sending) { - desc->d->result_ok = -1; - desc->d->errInfo.posix_errno = ECONNABORTED; - file_async_ready((ErlDrvData)desc,(ErlDrvThreadData)desc->d); - } - return 1; -} -#endif /* HAVE_SENDFILE */ - - -static void invoke_fallocate(void *data) -{ - struct t_data *d = (struct t_data *) data; - int fd = (int) d->fd; - Sint64 offset = d->c.fallocate.offset; - Sint64 length = d->c.fallocate.length; - - d->again = 0; - d->result_ok = efile_fallocate(&d->errInfo, fd, offset, length); -} - -static void free_readdir(void *data) -{ - struct t_data *d = (struct t_data *) data; - struct t_readdir_buf *b1 = d->c.read_dir.first_buf; - - while (b1) { - struct t_readdir_buf *b2 = b1; - b1 = b1->next; - EF_FREE(b2); - } - EF_FREE(d); -} - - - -static void try_free_read_bin(file_descriptor *desc) { - if ((desc->read_size == 0) - && (desc->read_offset >= desc->read_binp->orig_size)) { - ASSERT(desc->read_offset == desc->read_binp->orig_size); - driver_free_binary(desc->read_binp); - desc->read_binp = NULL; - desc->read_offset = 0; - desc->read_size = 0; - } -} - - - -static int try_again(file_descriptor *desc, struct t_data *d) { - if (! d->again) - return 0; - if (desc->timer_state != timer_idle) { - driver_cancel_timer(desc->port); - } - desc->timer_state = timer_again; - desc->invoke = d->invoke; - desc->d = d; - desc->free = d->free; - driver_set_timer(desc->port, 0L); - return !0; -} - - - -static void cq_execute(file_descriptor *desc) { - struct t_data *d; - register void *void_ptr; /* Soft cast variable */ - if (desc->timer_state == timer_again) - return; -#ifdef HAVE_SENDFILE - if (desc->sendfile_state == sending) - return; -#endif - if (! (d = cq_deq(desc))) - return; - TRACE_F(("x%i", (int) d->command)); - d->again = sys_info.async_threads == 0; - DRIVER_ASYNC(d->level, desc, d->invoke, void_ptr=d, d->free); -} - -static struct t_data *async_write(file_descriptor *desc, int *errp, - int reply, Uint32 reply_size -#ifdef USE_VM_PROBES - ,Sint64 *dt_i1, Sint64 *dt_i2, Sint64 *dt_i3 -#endif -) { - struct t_data *d; - if (! (d = EF_ALLOC(sizeof(struct t_data) - 1))) { - if (errp) *errp = ENOMEM; - return NULL; - } - TRACE_F(("w%lu", (unsigned long)desc->write_buffered)); - d->command = FILE_WRITE; - d->fd = desc->fd; - d->flags = desc->flags; - d->c.writev.port = desc->port; - d->c.writev.q_mtx = desc->q_mtx; - d->c.writev.size = desc->write_buffered; -#ifdef USE_VM_PROBES - if (dt_i1 != NULL) { - *dt_i1 = d->fd; - *dt_i2 = d->flags; - *dt_i3 = d->c.writev.size; - } -#endif - d->reply = reply; - d->c.writev.reply_size = reply_size; - d->invoke = invoke_writev; - d->free = free_data; - d->level = 1; - cq_enq(desc, d); - desc->write_buffered = 0; - return d; -} - -static int flush_write(file_descriptor *desc, int *errp -#ifdef USE_VM_PROBES - , dt_private *dt_priv, char *dt_utag -#endif -) { - int result = 0; -#ifdef USE_VM_PROBES - Sint64 dt_i1 = 0, dt_i2 = 0, dt_i3 = 0; -#endif - struct t_data *d = NULL; - - MUTEX_LOCK(desc->q_mtx); - if (desc->write_buffered > 0) { - if ((d = async_write(desc, errp, 0, 0 -#ifdef USE_VM_PROBES - ,&dt_i1, &dt_i2, &dt_i3 -#endif - )) == NULL) { - result = -1; - } - } - MUTEX_UNLOCK(desc->q_mtx); -#ifdef USE_VM_PROBES - if (d != NULL) { - d->sched_i1 = dt_priv->thread_num; - d->sched_i2 = dt_priv->tag; - d->sched_utag[0] = '\0'; - if (dt_utag != NULL) { - if (dt_utag[0] == '\0') { - dt_utag = NULL; - } else { - strncpy(d->sched_utag, dt_utag, sizeof(d->sched_utag) - 1); - d->sched_utag[sizeof(d->sched_utag) - 1] = '\0'; - } - } - DTRACE11(efile_drv_entry, dt_priv->thread_num, dt_priv->tag++, - dt_utag, FILE_WRITE, - NULL, NULL, dt_i1, dt_i2, dt_i3, 0, desc->port_str); - } -#endif /* USE_VM_PROBES */ - return result; -} - -static int check_write_error(file_descriptor *desc, int *errp) { - if (desc->write_error) { - if (errp) *errp = desc->write_errInfo.posix_errno; - desc->write_error = 0; - return -1; - } - return 0; -} - -static int flush_write_check_error(file_descriptor *desc, int *errp -#ifdef USE_VM_PROBES - , dt_private *dt_priv, char *dt_utag -#endif - ) { - int r; - if ( (r = flush_write(desc, errp -#ifdef USE_VM_PROBES - , dt_priv, dt_utag -#endif - )) != 0) { - check_write_error(desc, NULL); - return r; - } else { - return check_write_error(desc, errp); - } -} - -static struct t_data *async_lseek(file_descriptor *desc, int *errp, int reply, - Sint64 offset, int origin -#ifdef USE_VM_PROBES - , Sint64 *dt_i1, Sint64 *dt_i2, Sint64 *dt_i3 -#endif - ) { - struct t_data *d; - if (! (d = EF_ALLOC(sizeof(struct t_data)))) { - *errp = ENOMEM; - return NULL; - } - d->flags = desc->flags; - d->fd = desc->fd; - d->command = FILE_LSEEK; - d->reply = reply; - d->c.lseek.offset = offset; - d->c.lseek.origin = origin; -#ifdef USE_VM_PROBES - if (dt_i1 != NULL) { - *dt_i1 = d->fd; - *dt_i2 = d->c.lseek.offset; - *dt_i3 = d->c.lseek.origin; - } -#endif - d->invoke = invoke_lseek; - d->free = free_data; - d->level = 1; - cq_enq(desc, d); - return d; -} - -static void flush_read(file_descriptor *desc) { - desc->read_offset = 0; - desc->read_size = 0; - if (desc->read_binp) { - driver_free_binary(desc->read_binp); - desc->read_binp = NULL; - } -} - -static int lseek_flush_read(file_descriptor *desc, int *errp -#ifdef USE_VM_PROBES - ,dt_private *dt_priv, char *dt_utag -#endif - ) { - int r = 0; - size_t read_size = desc->read_size; -#ifdef USE_VM_PROBES - Sint64 dt_i1 = 0, dt_i2 = 0, dt_i3 = 0; -#endif - struct t_data *d; - - flush_read(desc); - if (read_size != 0) { - if ((d = async_lseek(desc, errp, 0, - -((ssize_t)read_size), EFILE_SEEK_CUR -#ifdef USE_VM_PROBES - , &dt_i1, &dt_i2, &dt_i3 -#endif - )) == NULL) { - r = -1; - } else { -#ifdef USE_VM_PROBES - d->sched_i1 = dt_priv->thread_num; - d->sched_i2 = dt_priv->tag; - d->sched_utag[0] = '\0'; - if (dt_utag != NULL) { - if (dt_utag[0] == '\0') { - dt_utag = NULL; - } else { - strncpy(d->sched_utag, dt_utag, sizeof(d->sched_utag) - 1); - d->sched_utag[sizeof(d->sched_utag) - 1] = '\0'; - } - } - DTRACE11(efile_drv_entry, dt_priv->thread_num, dt_priv->tag++, - dt_utag, FILE_LSEEK, - NULL, NULL, dt_i1, dt_i2, dt_i3, 0, desc->port_str); -#endif /* USE_VM_PROBES */ - } - } - return r; -} - - -/********************************************************************* - * Driver entry point -> stop - * The close has to be scheduled on async thread, so that currently active - * async operation does not suddenly have the ground disappearing under their feet... - */ -static void -file_stop(ErlDrvData e) -{ - file_descriptor* desc = (file_descriptor*)e; - - TRACE_C('p'); - - IF_THRDS { - flush_read(desc); - if (desc->fd != FILE_FD_INVALID) { - struct t_data *d = EF_SAFE_ALLOC(sizeof(struct t_data)); - d->command = FILE_CLOSE_ON_PORT_EXIT; - d->reply = !0; - d->fd = desc->fd; - d->flags = desc->flags; - d->invoke = invoke_close; - d->free = free_data; - d->level = 2; - d->data_to_free = (void *) desc; - cq_enq(desc, d); - desc->fd = FILE_FD_INVALID; - desc->flags = 0; - cq_execute(desc); - } else { - EF_FREE(desc); - } - } else { - if (desc->fd != FILE_FD_INVALID) { - do_close(desc->flags, desc->fd); - desc->fd = FILE_FD_INVALID; - desc->flags = 0; - } - if (desc->read_binp) { - driver_free_binary(desc->read_binp); - } - EF_FREE(desc); - } -} - -/********************************************************************* - * Driver entry point -> ready_async - */ -static void -file_async_ready(ErlDrvData e, ErlDrvThreadData data) -{ - file_descriptor *desc = (file_descriptor*)e; - struct t_data *d = (struct t_data *) data; - char header[5]; /* result code + count */ - char resbuf[RESBUFSIZE]; /* Result buffer. */ -#ifdef USE_VM_PROBES - int sched_i1 = d->sched_i1, sched_i2 = d->sched_i2, command = d->command, - result_ok = d->result_ok, - posix_errno = d->result_ok ? 0 : d->errInfo.posix_errno; - DTRACE_CHARBUF(sched_utag, DTRACE_EFILE_BUFSIZ+1); - - sched_utag[0] = '\0'; - if (DTRACE_ENABLED(efile_drv_return)) { - strncpy(sched_utag, d->sched_utag, DTRACE_EFILE_BUFSIZ); - sched_utag[DTRACE_EFILE_BUFSIZ] = '\0'; - } -#endif /* USE_VM_PROBES */ - - TRACE_C('r'); - - if (try_again(desc, d)) { - /* DTRACE TODO: what kind of probe makes sense here? */ - return; - } - - switch (d->command) - { - case FILE_READ: - if (!d->result_ok) { - reply_error(desc, &d->errInfo); - } else { - size_t available_bytes = - d->c.read.bin_offset + d->c.read.bin_size - desc->read_offset; - if (available_bytes < d->c.read.size) { - d->c.read.size = available_bytes; - } - TRACE_C('D'); - reply_data(desc, d->c.read.binp, - desc->read_offset, d->c.read.size); - desc->read_offset += d->c.read.size; - desc->read_size = - d->c.read.bin_offset + d->c.read.bin_size - desc->read_offset; - try_free_read_bin(desc); - } - free_read(data); - break; - case FILE_READ_LINE: - /* The read_line structure differs from the read structure. - The data->read_offset and d->c.read_line.read_offset are copies, as are - data->read_size and d->c.read_line.read_size - The read_line function does not kniow in advance how large the binary has to be, - why new allocation (but not reallocation of the old binary, for obvious reasons) - may happen in the worker thread. */ - if (!d->result_ok) { - reply_error(desc, &d->errInfo); - } else { - size_t len = d->c.read_line.nl_pos - d->c.read_line.read_offset; - TRACE_C('L'); - reply_data(desc, d->c.read_line.binp, - d->c.read_line.read_offset, len); - desc->read_offset = d->c.read_line.read_offset + d->c.read_line.nl_skip + len; - desc->read_size = - d->c.read_line.read_size - d->c.read_line.nl_skip - len; - if (desc->read_binp != d->c.read_line.binp) { /* New binary allocated */ - driver_free_binary(desc->read_binp); - desc->read_binp = d->c.read_line.binp; - driver_binary_inc_refc(desc->read_binp); - } -#if !ALWAYS_READ_LINE_AHEAD - ASSERT(desc->read_bufsize > 0 || desc->read_size == 0); - if (desc->read_bufsize == 0) { - desc->read_offset = desc->read_binp->orig_size; /* triggers cleanup */ - } -#endif - try_free_read_bin(desc); - } - free_read_line(data); - break; - case FILE_READ_FILE: - if (!d->result_ok) - reply_error(desc, &d->errInfo); - else { - header[0] = FILE_RESP_ALL_DATA; - TRACE_C('R'); - driver_output_binary(desc->port, header, 1, - d->c.read_file.binp, - 0, d->c.read_file.offset); - } - free_read_file(data); - break; - case FILE_WRITE: - if (d->reply) { - if (! d->result_ok) { - reply_error(desc, &d->errInfo); - } else { - reply_Uint(desc, d->c.writev.reply_size); - } - } else { - if (! d->result_ok) { - desc->write_error = !0; - desc->write_errInfo = d->errInfo; - } - } - free_data(data); - break; - case FILE_LSEEK: - if (d->reply) { - if (d->result_ok) - reply_Sint64(desc, d->c.lseek.location); - else - reply_error(desc, &d->errInfo); - } - free_data(data); - break; - case FILE_MKDIR: - case FILE_RMDIR: - case FILE_CHDIR: - case FILE_DELETE: - case FILE_FDATASYNC: - case FILE_FSYNC: - case FILE_TRUNCATE: - case FILE_LINK: - case FILE_SYMLINK: - case FILE_RENAME: - case FILE_WRITE_INFO: - case FILE_FADVISE: - case FILE_FALLOCATE: - reply(desc, d->result_ok, &d->errInfo); - free_data(data); - break; - case FILE_ALTNAME: - case FILE_PWD: - case FILE_READLINK: - { - int length; - char *resbuf = d->b; - - if (!d->result_ok) - reply_error(desc, &d->errInfo); - else { - resbuf[0] = FILE_RESP_FNAME; - length = 1+FILENAME_BYTELEN((char*) resbuf+1); - TRACE_C('R'); - driver_output2(desc->port, resbuf, 1, resbuf+1, length-1); - } - free_data(data); - break; - } - case FILE_OPEN: - if (!d->result_ok) { - reply_error(desc, &d->errInfo); - } else { - ASSERT(d->is_fd_unused); - desc->fd = d->fd; - desc->flags = d->flags; - d->is_fd_unused = 0; - reply_Uint(desc, d->fd); - } - free_data(data); - break; - case FILE_FSTAT: - case FILE_LSTAT: - { - if (d->result_ok) { - resbuf[0] = FILE_RESP_INFO; - - put_int32(d->info.size_high, &resbuf[1 + ( 0 * 4)]); - put_int32(d->info.size_low, &resbuf[1 + ( 1 * 4)]); - put_int32(d->info.type, &resbuf[1 + ( 2 * 4)]); - - /* Note 64 bit indexing in resbuf here */ - put_int64(d->info.accessTime, &resbuf[1 + ( 3 * 4)]); - put_int64(d->info.modifyTime, &resbuf[1 + ( 5 * 4)]); - put_int64(d->info.cTime, &resbuf[1 + ( 7 * 4)]); - - put_int32(d->info.mode, &resbuf[1 + ( 9 * 4)]); - put_int32(d->info.links, &resbuf[1 + (10 * 4)]); - put_int32(d->info.major_device, &resbuf[1 + (11 * 4)]); - put_int32(d->info.minor_device, &resbuf[1 + (12 * 4)]); - put_int32(d->info.inode, &resbuf[1 + (13 * 4)]); - put_int32(d->info.uid, &resbuf[1 + (14 * 4)]); - put_int32(d->info.gid, &resbuf[1 + (15 * 4)]); - put_int32(d->info.access, &resbuf[1 + (16 * 4)]); - -#define RESULT_SIZE (1 + (17 * 4)) - TRACE_C('R'); - driver_output2(desc->port, resbuf, RESULT_SIZE, NULL, 0); -#undef RESULT_SIZE - } else - reply_error(desc, &d->errInfo); - } - free_data(data); - break; - case FILE_READDIR: - if (!d->result_ok) { - reply_error(desc, &d->errInfo); - } else { - struct t_readdir_buf *b1 = d->c.read_dir.first_buf; - char op = FILE_RESP_LFNAME; - - TRACE_C('R'); - ASSERT(b1); - - while (b1) { - struct t_readdir_buf *b2 = b1; - char *p = &b1->buf[0]; - driver_output2(desc->port, p, 1, p + 1, b1->n - 1); - b1 = b1->next; - EF_FREE(b2); - } - driver_output2(desc->port, &op, 1, NULL, 0); - - d->c.read_dir.first_buf = NULL; - d->c.read_dir.last_buf = NULL; - } - free_readdir(data); - break; - case FILE_CLOSE: - if (d->reply) { - TRACE_C('K'); - reply_ok(desc); -#ifdef USE_VM_PROBES - result_ok = 1; -#endif - } - free_data(data); - break; - case FILE_PWRITEV: - if (!d->result_ok) { - reply_Uint_error(desc, d->c.pwritev.cnt, &d->errInfo); - } else { - reply_Uint(desc, d->c.pwritev.n); - } - free_data(data); - break; - case FILE_PREADV: - if (!d->result_ok) { - reply_error(desc, &d->errInfo); - } else { - reply_ev(desc, FILE_RESP_LDATA, &d->c.preadv.eiov); - } - free_preadv(data); - break; - case FILE_IPREAD: - if (!d->result_ok) { - reply_error(desc, &d->errInfo); - } else if (!d->c.preadv.eiov.vsize) { - reply_eof(desc); - } else { - reply_ev(desc, FILE_RESP_N2DATA, &d->c.preadv.eiov); - } - free_preadv(data); - break; -#ifdef HAVE_SENDFILE - case FILE_SENDFILE: - if (d->result_ok == -1) { - if (d->errInfo.posix_errno == ECONNRESET || - d->errInfo.posix_errno == ENOTCONN || - d->errInfo.posix_errno == EPIPE) - reply_string_error(desc,"closed"); - else - reply_error(desc, &d->errInfo); - desc->sendfile_state = not_sending; - free_sendfile(data); - } else if (d->result_ok == 0) { - reply_Sint64(desc, d->c.sendfile.written); - desc->sendfile_state = not_sending; - free_sendfile(data); - } else if (d->result_ok == 1) { /* If we are using select to send the rest of the data */ - desc->sendfile_state = sending; - desc->d = d; - driver_select(desc->port, (ErlDrvEvent)(long)d->c.sendfile.out_fd, - ERL_DRV_USE|ERL_DRV_WRITE, 1); - } - break; -#endif - case FILE_CLOSE_ON_PORT_EXIT: - /* See file_stop. However this is never invoked after the port is killed. */ - free_data(data); - desc = NULL; - /* This is it for this port, so just send dtrace and return, avoid doing anything to the freed data */ - DTRACE6(efile_drv_return, sched_i1, sched_i2, sched_utag, - command, result_ok, posix_errno); - return; - default: - abort(); - } - DTRACE6(efile_drv_return, sched_i1, sched_i2, sched_utag, - command, result_ok, posix_errno); - if (desc->write_buffered != 0 && desc->timer_state == timer_idle ) { - desc->timer_state = timer_write; - driver_set_timer(desc->port, desc->write_delay); - } - cq_execute(desc); - -} - - -/********************************************************************* - * Driver entry point -> output - */ -static void -file_output(ErlDrvData e, char* buf, ErlDrvSizeT count) -{ - file_descriptor* desc = (file_descriptor*)e; - Efile_error errInfo; /* The error codes for the last operation. */ - Sint fd; /* The file descriptor for this port, if any, - * -1 if none. - */ - char* name; /* Points to the filename in buf. */ - int command; - struct t_data *d = NULL; -#ifdef USE_VM_PROBES - char *dt_utag = NULL; - char *dt_s1 = NULL, *dt_s2 = NULL; - Sint64 dt_i1 = 0; - Sint64 dt_i2 = 0; - Sint64 dt_i3 = 0; - Sint64 dt_i4 = 0; - dt_private *dt_priv = get_dt_private(0); -#endif /* USE_VM_PROBES */ - - TRACE_C('o'); - - fd = desc->fd; - name = buf+1; - command = *(uchar*)buf++; - - switch(command) { - - case FILE_MKDIR: - { - d = EF_SAFE_ALLOC(sizeof(struct t_data) - 1 + FILENAME_BYTELEN(name) + FILENAME_CHARSIZE); - - FILENAME_COPY(d->b, name); -#ifdef USE_VM_PROBES - dt_s1 = d->b; - dt_utag = name + FILENAME_BYTELEN(name) + FILENAME_CHARSIZE; -#endif - d->command = command; - d->invoke = invoke_mkdir; - d->free = free_data; - d->level = 2; - goto done; - } - case FILE_RMDIR: - { - d = EF_SAFE_ALLOC(sizeof(struct t_data) - 1 + FILENAME_BYTELEN(name) + FILENAME_CHARSIZE); - - FILENAME_COPY(d->b, name); -#ifdef USE_VM_PROBES - dt_s1 = d->b; - dt_utag = name + FILENAME_BYTELEN(name) + FILENAME_CHARSIZE; -#endif - d->command = command; - d->invoke = invoke_rmdir; - d->free = free_data; - d->level = 2; - goto done; - } - case FILE_DELETE: - { - d = EF_SAFE_ALLOC(sizeof(struct t_data) - 1 + FILENAME_BYTELEN(name) + FILENAME_CHARSIZE); - - FILENAME_COPY(d->b, name); -#ifdef USE_VM_PROBES - dt_s1 = d->b; - dt_utag = name + FILENAME_BYTELEN(name) + FILENAME_CHARSIZE; -#endif - d->command = command; - d->invoke = invoke_delete_file; - d->free = free_data; - d->level = 2; - goto done; - } - case FILE_RENAME: - { - char* new_name; - int namelen = FILENAME_BYTELEN(name)+FILENAME_CHARSIZE; - new_name = name+namelen; - d = EF_SAFE_ALLOC(sizeof(struct t_data) - 1 - + namelen - + FILENAME_BYTELEN(new_name) + FILENAME_CHARSIZE); - - FILENAME_COPY(d->b, name); - FILENAME_COPY(d->b + namelen, new_name); -#ifdef USE_VM_PROBES - dt_s1 = d->b; - dt_s2 = d->b + namelen; - dt_utag = buf + namelen + FILENAME_BYTELEN(new_name) + FILENAME_CHARSIZE; -#endif - d->flags = desc->flags; - d->fd = fd; - d->command = command; - d->invoke = invoke_rename; - d->free = free_data; - d->level = 2; - goto done; - } - case FILE_CHDIR: - { - d = EF_SAFE_ALLOC(sizeof(struct t_data) - 1 + FILENAME_BYTELEN(name) + FILENAME_CHARSIZE); - - FILENAME_COPY(d->b, name); -#ifdef USE_VM_PROBES - dt_s1 = d->b; - dt_utag = name + FILENAME_BYTELEN(name) + FILENAME_CHARSIZE; -#endif - d->command = command; - d->invoke = invoke_chdir; - d->free = free_data; - d->level = 2; - goto done; - } - case FILE_PWD: - { - d = EF_SAFE_ALLOC(sizeof(struct t_data) - 1 + RESBUFSIZE + 1); - - d->drive = *(uchar*)buf; -#ifdef USE_VM_PROBES - dt_utag = buf + 1; -#endif - d->command = command; - d->invoke = invoke_pwd; - d->free = free_data; - d->level = 2; - goto done; - } - - case FILE_READDIR: - if (sys_info.async_threads > 0) - { - d = EF_SAFE_ALLOC(sizeof(struct t_data) - 1 + FILENAME_BYTELEN(name) + - FILENAME_CHARSIZE); - - FILENAME_COPY(d->b, name); -#ifdef USE_VM_PROBES - dt_s1 = d->b; - dt_utag = name + FILENAME_BYTELEN(name) + FILENAME_CHARSIZE; -#endif - d->dir_handle = NULL; - d->command = command; - d->invoke = invoke_readdir; - d->free = free_readdir; - d->level = 2; - d->c.read_dir.first_buf = NULL; - d->c.read_dir.last_buf = NULL; - goto done; - } - else - { - size_t resbufsize; - size_t n = 0, total = 0; - int res = 0; - char resbuf[READDIR_BUFSIZE]; - - EFILE_DIR_HANDLE dir_handle; /* Handle to open directory. */ - - total = READDIR_BUFSIZE; - errInfo.posix_errno = 0; - dir_handle = NULL; - resbuf[0] = FILE_RESP_LFNAME; - -#ifdef USE_VM_PROBES - dt_s1 = name; - dt_utag = name + FILENAME_BYTELEN(name) + FILENAME_CHARSIZE; -#endif - /* Fill the buffer with multiple directory listings before sending it to the - * receiving process. READDIR_CHUNKS is minimum number of files sent to the - * receiver. - * Format for each driver_output2: - * ------------------------------------ - * | Type | Len | Filename | ... - * | 1 byte | 2 bytes | Len bytes | ... - * ------------------------------------ - */ - - do { - n = 1; - resbufsize = READDIR_BUFSIZE - n; - - do { - res = efile_readdir(&errInfo, name, &dir_handle, resbuf + n + 2, &resbufsize); - - if (res) { - put_int16((Uint16)resbufsize, resbuf + n); - n += 2 + resbufsize; - resbufsize = READDIR_BUFSIZE - n; - } - } while( res && ((total - n - 2) >= MAXPATHLEN*FILENAME_CHARSIZE)); - - if (n > 1) { - driver_output2(desc->port, resbuf, 1, resbuf + 1, n - 1); - } - } while(res); - - if (errInfo.posix_errno != 0) { - reply_error(desc, &errInfo); - return; - } -#ifdef USE_VM_PROBES - if (dt_utag != NULL && dt_utag[0] == '\0') { - dt_utag = NULL; - } - - DTRACE11(efile_drv_entry, dt_priv->thread_num, dt_priv->tag, - dt_utag, command, name, dt_s2, - dt_i1, dt_i2, dt_i3, dt_i4, desc->port_str); - DTRACE6(efile_drv_return, dt_priv->thread_num, dt_priv->tag++, - dt_utag, command, 1, 0); -#endif - TRACE_C('R'); - driver_output2(desc->port, resbuf, 1, NULL, 0); - return; - } - case FILE_OPEN: - { - d = EF_SAFE_ALLOC(sizeof(struct t_data) - 1 + FILENAME_BYTELEN(buf+4) + - FILENAME_CHARSIZE); - - d->flags = get_int32((uchar*)buf); - name = buf+4; - FILENAME_COPY(d->b, name); -#ifdef USE_VM_PROBES - dt_i1 = d->flags; - dt_s1 = d->b; - dt_utag = name + FILENAME_BYTELEN(d->b) + FILENAME_CHARSIZE; -#endif - d->command = command; - d->invoke = invoke_open; - d->free = free_data; - d->level = 2; - d->is_fd_unused = 1; - goto done; - } - - case FILE_FDATASYNC: - { - d = EF_SAFE_ALLOC(sizeof(struct t_data)); - - d->fd = fd; -#ifdef USE_VM_PROBES - dt_utag = name; - dt_i1 = fd; -#endif - d->command = command; - d->invoke = invoke_fdatasync; - d->free = free_data; - d->level = 2; - goto done; - } - - case FILE_FSYNC: - { - d = EF_SAFE_ALLOC(sizeof(struct t_data)); - - d->fd = fd; -#ifdef USE_VM_PROBES - dt_utag = name; - dt_i1 = fd; -#endif - d->command = command; - d->invoke = invoke_fsync; - d->free = free_data; - d->level = 2; - goto done; - } - - - case FILE_FSTAT: - case FILE_LSTAT: - { - d = EF_SAFE_ALLOC(sizeof(struct t_data) - 1 + FILENAME_BYTELEN(name) + - FILENAME_CHARSIZE); - - FILENAME_COPY(d->b, name); - d->fd = fd; -#ifdef USE_VM_PROBES - dt_utag = name + FILENAME_BYTELEN(d->b) + FILENAME_CHARSIZE; - if (command == FILE_LSTAT) { - dt_s1 = d->b; - } else { - dt_i1 = fd; - } -#endif - d->command = command; - d->invoke = invoke_flstat; - d->free = free_data; - d->level = 2; - goto done; - } - - case FILE_TRUNCATE: - { - d = EF_SAFE_ALLOC(sizeof(struct t_data)); - - d->flags = desc->flags; - d->fd = fd; -#ifdef USE_VM_PROBES - dt_utag = name; - dt_i1 = fd; - dt_i2 = d->flags; -#endif - d->command = command; - d->invoke = invoke_truncate; - d->free = free_data; - d->level = 2; - goto done; - } - - case FILE_WRITE_INFO: - { - d = EF_SAFE_ALLOC(sizeof(struct t_data) - 1 - + FILENAME_BYTELEN(buf + 9*4) + FILENAME_CHARSIZE); - - d->info.mode = get_int32(buf + 0 * 4); - d->info.uid = get_int32(buf + 1 * 4); - d->info.gid = get_int32(buf + 2 * 4); - d->info.accessTime = get_int64(buf + 3 * 4); - d->info.modifyTime = get_int64(buf + 5 * 4); - d->info.cTime = get_int64(buf + 7 * 4); - - FILENAME_COPY(d->b, buf + 9*4); -#ifdef USE_VM_PROBES - dt_i1 = d->info.mode; - dt_i2 = d->info.uid; - dt_i3 = d->info.gid; - dt_s1 = d->b; - dt_utag = buf + 9 * 4 + FILENAME_BYTELEN(d->b) + FILENAME_CHARSIZE; -#endif - d->command = command; - d->invoke = invoke_write_info; - d->free = free_data; - d->level = 2; - goto done; - } - - case FILE_READLINK: - { - d = EF_SAFE_ALLOC(sizeof(struct t_data) - 1 + - MAX(RESBUFSIZE, (FILENAME_BYTELEN(name) + - FILENAME_CHARSIZE)) + 1); - FILENAME_COPY(d->b, name); -#ifdef USE_VM_PROBES - dt_s1 = d->b; - dt_utag = name + FILENAME_BYTELEN(d->b) + FILENAME_CHARSIZE; -#endif - d->command = command; - d->invoke = invoke_readlink; - d->free = free_data; - d->level = 2; - goto done; - } - - case FILE_ALTNAME: - { - d = EF_SAFE_ALLOC(sizeof(struct t_data) - 1 + - MAX(RESBUFSIZE, (FILENAME_BYTELEN(name) + - FILENAME_CHARSIZE)) + 1); - FILENAME_COPY(d->b, name); -#ifdef USE_VM_PROBES - dt_s1 = d->b; - dt_utag = name + FILENAME_BYTELEN(d->b) + FILENAME_CHARSIZE; -#endif - d->command = command; - d->invoke = invoke_altname; - d->free = free_data; - d->level = 2; - goto done; - } - - - case FILE_LINK: - { - char* new_name; - int namelen = FILENAME_BYTELEN(name) + FILENAME_CHARSIZE; - - new_name = name+namelen; - d = EF_SAFE_ALLOC(sizeof(struct t_data) - 1 - + namelen - + FILENAME_BYTELEN(new_name) + FILENAME_CHARSIZE); - - FILENAME_COPY(d->b, name); - FILENAME_COPY(d->b + namelen, new_name); -#ifdef USE_VM_PROBES - dt_s1 = d->b; - dt_s2 = d->b + namelen; - dt_utag = buf + namelen + FILENAME_BYTELEN(dt_s2) + FILENAME_CHARSIZE; -#endif - d->flags = desc->flags; - d->fd = fd; - d->command = command; - d->invoke = invoke_link; - d->free = free_data; - d->level = 2; - goto done; - } - - case FILE_SYMLINK: - { - char* new_name; - int namelen = FILENAME_BYTELEN(name) + FILENAME_CHARSIZE; - - new_name = name+namelen; - d = EF_SAFE_ALLOC(sizeof(struct t_data) - 1 - + namelen - + FILENAME_BYTELEN(new_name) + FILENAME_CHARSIZE); - - FILENAME_COPY(d->b, name); - FILENAME_COPY(d->b + namelen, new_name); -#ifdef USE_VM_PROBES - dt_s1 = d->b; - dt_s2 = d->b + namelen; - dt_utag = buf + namelen + FILENAME_BYTELEN(dt_s2) + FILENAME_CHARSIZE; -#endif - d->flags = desc->flags; - d->fd = fd; - d->command = command; - d->invoke = invoke_symlink; - d->free = free_data; - d->level = 2; - goto done; - } - - case FILE_FADVISE: - { - d = EF_SAFE_ALLOC(sizeof(struct t_data)); - - d->fd = fd; - d->command = command; - d->invoke = invoke_fadvise; - d->free = free_data; - d->level = 2; - d->c.fadvise.offset = get_int64((uchar*) buf); - d->c.fadvise.length = get_int64(((uchar*) buf) + sizeof(Sint64)); - d->c.fadvise.advise = get_int32(((uchar*) buf) + 2 * sizeof(Sint64)); -#ifdef USE_VM_PROBES - dt_i1 = d->fd; - dt_i2 = d->c.fadvise.offset; - dt_i3 = d->c.fadvise.length; - dt_i4 = d->c.fadvise.advise; - dt_utag = buf + 3 * sizeof(Sint64); -#endif - goto done; - } - - case FILE_FALLOCATE: - { - d = EF_SAFE_ALLOC(sizeof(struct t_data)); - - d->fd = fd; - d->command = command; - d->invoke = invoke_fallocate; - d->free = free_data; - d->level = 2; - d->c.fallocate.offset = get_int64((uchar*) buf); - d->c.fallocate.length = get_int64(((uchar*) buf) + sizeof(Sint64)); - goto done; - } - - } - - /* - * Ignore anything else -- let the caller hang. - */ - - return; - - done: - if (d) { -#ifdef USE_VM_PROBES - d->sched_i1 = dt_priv->thread_num; - d->sched_i2 = dt_priv->tag; - d->sched_utag[0] = '\0'; - if (dt_utag != NULL) { - if (dt_utag[0] == '\0') { - dt_utag = NULL; - } else { - strncpy(d->sched_utag, dt_utag, sizeof(d->sched_utag) - 1); - d->sched_utag[sizeof(d->sched_utag) - 1] = '\0'; - } - } - DTRACE11(efile_drv_entry, dt_priv->thread_num, dt_priv->tag++, - dt_utag, command, dt_s1, dt_s2, - dt_i1, dt_i2, dt_i3, dt_i4, desc->port_str); -#endif - cq_enq(desc, d); - } -} - -/********************************************************************* - * Driver entry point -> flush - */ -static void -file_flush(ErlDrvData e) { - file_descriptor *desc = (file_descriptor *)e; -#ifdef DEBUG - int r; -#endif -#ifdef USE_VM_PROBES - dt_private *dt_priv = get_dt_private(dt_driver_io_worker_base); -#endif - - TRACE_C('f'); - -#ifdef HAVE_SENDFILE - flush_sendfile(desc, NULL); -#endif - -#ifdef DEBUG - r = -#endif - flush_write(desc, NULL -#ifdef USE_VM_PROBES - , dt_priv, (desc->d == NULL) ? NULL : desc->d->sched_utag -#endif - ); - /* Only possible reason for bad return value is ENOMEM, and - * there is nobody to tell... - */ -#ifdef DEBUG - ASSERT(r == 0); -#endif - cq_execute(desc); -} - - - -/********************************************************************* - * Driver entry point -> control - * Only debug functionality... - */ -static ErlDrvSSizeT -file_control(ErlDrvData e, unsigned int command, - char* buf, ErlDrvSizeT len, char **rbuf, ErlDrvSizeT rlen) { - file_descriptor *desc = (file_descriptor *)e; - switch (command) { - case 'K' : - if (rlen < 4) { - *rbuf = EF_ALLOC(4); - } - (*rbuf)[0] = ((desc->key) >> 24) & 0xFF; - (*rbuf)[1] = ((desc->key) >> 16) & 0xFF; - (*rbuf)[2] = ((desc->key) >> 8) & 0xFF; - (*rbuf)[3] = (desc->key) & 0xFF; - return 4; - default: - return 0; - } -} - -/********************************************************************* - * Driver entry point -> timeout - */ -static void -file_timeout(ErlDrvData e) { - file_descriptor *desc = (file_descriptor *)e; - enum e_timer timer_state = desc->timer_state; -#ifdef USE_VM_PROBES - dt_private *dt_priv = get_dt_private(dt_driver_io_worker_base); -#endif - - TRACE_C('t'); - - desc->timer_state = timer_idle; - switch (timer_state) { - case timer_idle: - ASSERT(0); - break; - case timer_again: - ASSERT(desc->invoke); - ASSERT(desc->free); - driver_async(desc->port, KEY(desc), desc->invoke, desc->d, desc->free); - break; - case timer_write: { -#ifdef DEBUG - int r = -#endif - flush_write(desc, NULL -#ifdef USE_VM_PROBES - , dt_priv, (desc->d == NULL) ? NULL : desc->d->sched_utag -#endif - ); - /* Only possible reason for bad return value is ENOMEM, and - * there is nobody to tell... - */ - ASSERT(r == 0); - cq_execute(desc); - } break; - } /* case */ -} - - - -/********************************************************************* - * Driver entry point -> outputv - */ -static void -file_outputv(ErlDrvData e, ErlIOVec *ev) { - file_descriptor* desc = (file_descriptor*)e; - char command; - size_t p, q; - int err; - struct t_data *d = NULL; -#ifdef USE_VM_PROBES - Sint64 dt_i1 = 0, dt_i2 = 0, dt_i3 = 0; - Sint64 dt_i4 = 0; - char *dt_utag = NULL; - char *dt_s1 = NULL; - dt_private *dt_priv = get_dt_private(dt_driver_io_worker_base); -#endif - - TRACE_C('v'); - - p = 0; q = 1; - if (! EV_GET_CHAR(ev, &command, &p, &q)) { - /* Empty command */ - reply_posix_error(desc, EINVAL); - goto done; - } - /* 'command' contains the decoded command number, - * 'p' and 'q' point out the next byte in the command: - * ((char *)ev->iov[q].iov_base) + p; - */ - - TRACE_F(("%i", (int) command)); - - switch (command) { - - case FILE_CLOSE: { -#ifdef USE_VM_PROBES - dt_utag = EV_CHAR_P(ev, p, q); -#endif - flush_read(desc); - if (flush_write_check_error(desc, &err -#ifdef USE_VM_PROBES - , dt_priv, dt_utag -#endif - ) < 0) { - reply_posix_error(desc, err); - goto done; - } - if (desc->fd != FILE_FD_INVALID) { - if (! (d = EF_ALLOC(sizeof(struct t_data)))) { - reply_posix_error(desc, ENOMEM); - } else { - d->command = command; - d->reply = !0; - d->fd = desc->fd; - d->flags = desc->flags; -#ifdef USE_VM_PROBES - dt_i1 = d->fd; - dt_i2 = d->flags; -#endif - d->invoke = invoke_close; - d->free = free_data; - d->level = 2; - cq_enq(desc, d); - desc->fd = FILE_FD_INVALID; - desc->flags = 0; - } - } else { - reply_posix_error(desc, EBADF); - } - } goto done; - - case FILE_READ: { - Uint32 sizeH, sizeL; - size_t size, alloc_size; - - if (!EV_GET_UINT32(ev, &sizeH, &p, &q) - || !EV_GET_UINT32(ev, &sizeL, &p, &q)) { - /* Wrong buffer length to contain the read count */ - reply_posix_error(desc, EINVAL); - goto done; - } -#ifdef USE_VM_PROBES - dt_utag = EV_CHAR_P(ev, p, q); -#endif - if (flush_write_check_error(desc, &err -#ifdef USE_VM_PROBES - , dt_priv, dt_utag -#endif - ) < 0) { - reply_posix_error(desc, err); - goto done; - } -#if ALWAYS_READ_LINE_AHEAD - if (desc->read_bufsize == 0 && desc->read_binp != NULL && desc->read_size > 0) { - /* We have allocated a buffer for line mode but should not really have a - read-ahead buffer... */ - if (lseek_flush_read(desc, &err -#ifdef USE_VM_PROBES - , dt_priv, dt_utag -#endif - ) < 0) { - reply_posix_error(desc, err); - goto done; - } - } -#endif -#if SIZEOF_SIZE_T == 4 - if (sizeH != 0) { - reply_posix_error(desc, EINVAL); - goto done; - } - size = sizeL; -#else - size = ((size_t)sizeH << 32) | sizeL; -#endif - if ((desc->fd == FILE_FD_INVALID) - || (! (desc->flags & EFILE_MODE_READ)) ) { - reply_posix_error(desc, EBADF); - goto done; - } - if (size == 0) { - reply_buf(desc, &command, 0); - goto done; - } - if (desc->read_size >= size) { - /* We already have all data */ - TRACE_C('D'); - reply_data(desc, desc->read_binp, desc->read_offset, size); - desc->read_offset += size; - desc->read_size -= size; - try_free_read_bin(desc); - goto done; - } - /* We may have some of the data - */ - /* Justification for the following strange formula: - * If the read request is for such a large block as more than - * half the buffer size it may lead to a lot of unnecessary copying, - * since the tail of the old buffer is copied to the head of the - * new, and if the tail is almost half the buffer it is a lot - * to copy. Therefore allocate the exact amount needed in - * this case, giving no lingering tail. */ - alloc_size = - size > (desc->read_bufsize>>1) ? - size : desc->read_bufsize; - if (! desc->read_binp) { - /* Need to allocate a new binary for the result */ - if (! (desc->read_binp = driver_alloc_binary(alloc_size))) { - reply_posix_error(desc, ENOMEM); - goto done; - } - } else { - /* We already have a buffer */ - if (desc->read_binp->orig_size - desc->read_offset < size) { - /* Need to allocate a new binary for the result */ - ErlDrvBinary *binp; - if (! (binp = driver_alloc_binary(alloc_size))) { - reply_posix_error(desc, ENOMEM); - goto done; - } - /* Move data we already have to the new binary */ - sys_memcpy(binp->orig_bytes, - desc->read_binp->orig_bytes + desc->read_offset, - desc->read_size); - driver_free_binary(desc->read_binp); - desc->read_offset = 0; - desc->read_binp = binp; - } - } - if (! (d = EF_ALLOC(sizeof(struct t_data)))) { - reply_posix_error(desc, ENOMEM); - goto done; - } - d->command = command; - d->reply = !0; - d->fd = desc->fd; - d->flags = desc->flags; - d->c.read.binp = desc->read_binp; - d->c.read.bin_offset = desc->read_offset + desc->read_size; - d->c.read.bin_size = desc->read_binp->orig_size - d->c.read.bin_offset; - d->c.read.size = size; -#ifdef USE_VM_PROBES - dt_i1 = d->fd; - dt_i2 = d->flags; - dt_i3 = d->c.read.size; -#endif - driver_binary_inc_refc(d->c.read.binp); - d->invoke = invoke_read; - d->free = free_read; - d->level = 1; - cq_enq(desc, d); - } goto done; /* case FILE_READ: */ - - case FILE_READ_LINE: { - /* - * Icky little creature... We do mostly as ordinary file read, but with a few differences. - * 1) We have to scan for proper newline sequence if there is a buffer already, we cannot know - * in advance if the buffer contains a whole line without scanning. - * 2) We do not know how large the buffer needs to be in advance. We give a default buffer, - * but the worker may need to allocate a new one. Freeing the old and rereferencing a newly - * allocated binary + dealing with offsets and lengts are done in file_async ready - * for this OP. - */ -#ifdef USE_VM_PROBES - dt_utag = EV_CHAR_P(ev, p, q); -#endif - if (flush_write_check_error(desc, &err -#ifdef USE_VM_PROBES - , dt_priv, dt_utag -#endif - ) < 0) { - reply_posix_error(desc, err); - goto done; - } - if (ev->size != 1 -#ifdef USE_VM_PROBES - + FILENAME_BYTELEN(dt_utag) + FILENAME_CHARSIZE -#endif - ) { - /* Wrong command length */ - reply_posix_error(desc, EINVAL); - goto done; - } - if ((desc->fd == FILE_FD_INVALID) - || (! (desc->flags & EFILE_MODE_READ)) ) { - reply_posix_error(desc, EBADF); - goto done; - } - if (desc->read_size > 0) { - /* look for '\n' in what we'we already got */ - void *nl_ptr = memchr(desc->read_binp->orig_bytes + desc->read_offset,'\n',desc->read_size); - if (nl_ptr != NULL) { - /* If found, we're done */ - int skip = 0; - size_t size = ((char *) nl_ptr) - - ((char *) (desc->read_binp->orig_bytes + desc->read_offset)) + 1; - if (size > 1 && - *(((char *) nl_ptr) - 1) == '\r') { - *(((char *) nl_ptr) - 1) = '\n'; - skip = 1; - --size; - } - reply_data(desc, desc->read_binp, desc->read_offset, size); - desc->read_offset += (size + skip); - desc->read_size -= (size + skip); - try_free_read_bin(desc); - goto done; - } - } - /* Now, it's up to the thread to work out the need for more buffers and such, it's - no use doing it in this thread as we do not have the information required anyway. - Even a NULL buffer could be handled by the thread, but code is simplified by us - allocating it */ - if (! desc->read_binp) { - int alloc_size = (desc->read_bufsize > DEFAULT_LINEBUF_SIZE) ? desc->read_bufsize : - DEFAULT_LINEBUF_SIZE; - /* Allocate a new binary for the result */ - if (! (desc->read_binp = driver_alloc_binary(alloc_size))) { - reply_posix_error(desc, ENOMEM); - goto done; - } - } - if (! (d = EF_ALLOC(sizeof(struct t_data)))) { - reply_posix_error(desc, ENOMEM); - goto done; - } - - d->command = command; - d->reply = !0; - d->fd = desc->fd; - d->flags = desc->flags; - d->c.read_line.binp = desc->read_binp; - d->c.read_line.read_offset = desc->read_offset; - d->c.read_line.read_size = desc->read_size; -#ifdef USE_VM_PROBES - dt_i1 = d->fd; - dt_i2 = d->flags; - dt_i3 = d->c.read_line.read_offset; -#endif -#if !ALWAYS_READ_LINE_AHEAD - d->c.read_line.read_ahead = (desc->read_bufsize > 0); -#ifdef USE_VM_PROBES - dt_i4 = d->c.read_line.read_ahead; -#endif -#endif - driver_binary_inc_refc(d->c.read.binp); - d->invoke = invoke_read_line; - d->free = free_read_line; - d->level = 1; - cq_enq(desc, d); - } goto done; - case FILE_WRITE: { /* Dtrace: The dtrace user tag is not last in message, - but follows the message tag directly. - This is handled specially in prim_file.erl */ - ErlDrvSizeT skip = 1; - ErlDrvSizeT size = ev->size - skip; - -#ifdef USE_VM_PROBES - dt_utag = EV_CHAR_P(ev, p, q); - skip += FILENAME_BYTELEN(dt_utag) + FILENAME_CHARSIZE; - size = ev->size - skip; -#endif - if (lseek_flush_read(desc, &err -#ifdef USE_VM_PROBES - , dt_priv, dt_utag -#endif - ) < 0) { - reply_posix_error(desc, err); - goto done; - } - if (! (desc->flags & EFILE_MODE_WRITE)) { - reply_posix_error(desc, EBADF); - goto done; - } - if (size == 0) { - reply_Uint(desc, size); - goto done; - } - MUTEX_LOCK(desc->q_mtx); - if (driver_enqv(desc->port, ev, skip)) { - MUTEX_UNLOCK(desc->q_mtx); - reply_posix_error(desc, ENOMEM); - goto done; - } - desc->write_buffered += size; - if (desc->write_buffered < desc->write_bufsize) { - MUTEX_UNLOCK(desc->q_mtx); - reply_Uint(desc, size); - if (desc->timer_state == timer_idle) { - desc->timer_state = timer_write; - driver_set_timer(desc->port, desc->write_delay); - } - } else { - if ((d = async_write(desc, &err, !0, size -#ifdef USE_VM_PROBES - , &dt_i1, &dt_i2, &dt_i3 -#endif - )) == NULL) { - MUTEX_UNLOCK(desc->q_mtx); - reply_posix_error(desc, err); - goto done; - } else { - MUTEX_UNLOCK(desc->q_mtx); - } - } - } goto done; /* case FILE_WRITE */ - - case FILE_PWRITEV: { /* Dtrace: The dtrace user tag is not last in message, - but follows the message tag directly. - This is handled specially in prim_file.erl */ - Uint32 i, j, n; - size_t total; -#ifdef USE_VM_PROBES - char dt_tmp; - int dt_utag_bytes = 1; - - dt_utag = EV_CHAR_P(ev, p, q); - /* This will work for UTF-8, but not for UTF-16 - extra reminder here */ -#ifdef FILENAMES_16BIT -#error 16bit characters in filenames and dtrace in combination is not supported. -#endif - while (EV_GET_CHAR(ev, &dt_tmp, &p, &q) && dt_tmp != '\0') { - dt_utag_bytes++; - } -#endif - if (ev->size < 1+4 -#ifdef USE_VM_PROBES - + dt_utag_bytes -#endif - || !EV_GET_UINT32(ev, &n, &p, &q)) { - /* Buffer too short to contain even the number of pos/size specs */ - reply_Uint_posix_error(desc, 0, EINVAL); - goto done; - } - if (lseek_flush_read(desc, &err -#ifdef USE_VM_PROBES - , dt_priv, dt_utag -#endif - ) < 0) { - reply_Uint_posix_error(desc, 0, err); - goto done; - } - if (flush_write_check_error(desc, &err -#ifdef USE_VM_PROBES - , dt_priv, dt_utag -#endif - ) < 0) { - reply_Uint_posix_error(desc, 0, err); - goto done; - } - if (n == 0) { - /* Trivial case - nothing to write */ - if (ev->size != 1+4) { - reply_posix_error(desc, err); - } else { - reply_Uint(desc, 0); - } - goto done; - } - if (ev->size < 1+4+8*(2*n) -#ifdef USE_VM_PROBES - + dt_utag_bytes -#endif - ) { - /* Buffer too short to contain even the pos/size specs */ - reply_Uint_posix_error(desc, 0, EINVAL); - goto done; - } - d = EF_ALLOC(sizeof(struct t_data) - + (n * sizeof(struct t_pbuf_spec))); - if (! d) { - reply_Uint_posix_error(desc, 0, ENOMEM); - goto done; - } - d->command = command; - d->reply = !0; - d->fd = desc->fd; - d->flags = desc->flags; -#ifdef USE_VM_PROBES - dt_i1 = d->fd; - dt_i2 = d->flags; -#endif - d->c.pwritev.port = desc->port; - d->c.pwritev.q_mtx = desc->q_mtx; - d->c.pwritev.n = n; - d->c.pwritev.cnt = 0; - total = 0; - j = 0; - /* Create pos/size specs in the thread data structure - * for all non-zero size binaries. Calculate total size. - */ - for(i = 0; i < n; i++) { - Uint32 sizeH, sizeL; - size_t size; - if ( !EV_GET_SINT64(ev, &d->c.pwritev.specs[i].offset, &p, &q) - || !EV_GET_UINT32(ev, &sizeH, &p, &q) - || !EV_GET_UINT32(ev, &sizeL, &p, &q)) { - /* Misalignment in buffer */ - reply_Uint_posix_error(desc, 0, EINVAL); - EF_FREE(d); - goto done; - } -#if SIZEOF_SIZE_T == 4 - if (sizeH != 0) { - reply_Uint_posix_error(desc, 0, EINVAL); - EF_FREE(d); - goto done; - } - size = sizeL; -#else - size = ((size_t)sizeH<<32) | sizeL; -#endif - if (size > 0) { - total += size; - d->c.pwritev.specs[j].size = size; - j++; - } - } - d->c.pwritev.size = total; -#ifdef USE_VM_PROBES - dt_i3 = d->c.pwritev.size; -#endif - if (j == 0) { - /* Trivial case - nothing to write */ - EF_FREE(d); - reply_Uint(desc, 0); - } else { - ErlDrvSizeT skip = 1 + 4 + 8 * (2*n) -#ifdef USE_VM_PROBES - + dt_utag_bytes -#endif - ; - if (skip + total != ev->size) { - /* Actual amount of data does not match - * total of all pos/size specs - */ - EF_FREE(d); - reply_Uint_posix_error(desc, 0, EINVAL); - } else { - /* Enqueue the data */ - MUTEX_LOCK(desc->q_mtx); - driver_enqv(desc->port, ev, skip); - MUTEX_UNLOCK(desc->q_mtx); - /* Execute the command */ - d->invoke = invoke_pwritev; - d->free = free_data; - d->level = 1; - cq_enq(desc, d); - } - } - } goto done; /* case FILE_PWRITEV: */ - - case FILE_PREADV: { /* Dtrace: The dtrace user tag is not last in message, - but follows the message tag directly. - This is handled specially in prim_file.erl */ - register void * void_ptr; - Uint32 i, n; - ErlIOVec *res_ev; -#ifdef USE_VM_PROBES - char dt_tmp; - int dt_utag_bytes = 1; - /* This will work for UTF-8, but not for UTF-16 - extra reminder here */ -#ifdef FILENAMES_16BIT -#error 16bit characters in filenames and dtrace in combination is not supported. -#endif - dt_utag = EV_CHAR_P(ev, p, q); - while (EV_GET_CHAR(ev, &dt_tmp, &p, &q) && dt_tmp != '\0') { - dt_utag_bytes++; - } -#endif - if (lseek_flush_read(desc, &err -#ifdef USE_VM_PROBES - , dt_priv, dt_utag -#endif - ) < 0) { - reply_posix_error(desc, err); - goto done; - } - if (flush_write_check_error(desc, &err -#ifdef USE_VM_PROBES - , dt_priv, dt_utag -#endif - ) < 0) { - reply_posix_error(desc, err); - goto done; - } - if (ev->size < 1+8 -#ifdef USE_VM_PROBES - + dt_utag_bytes -#endif - || !EV_GET_UINT32(ev, &n, &p, &q) - || !EV_GET_UINT32(ev, &n, &p, &q)) { - /* Buffer too short to contain even the number of pos/size specs */ - reply_posix_error(desc, EINVAL); - goto done; - } - if (ev->size < 1+8+8*(2*n) -#ifdef USE_VM_PROBES - + dt_utag_bytes -#endif - ) { - /* Buffer wrong length to contain the pos/size specs */ - reply_posix_error(desc, EINVAL); - goto done; - } - /* Create the thread data structure with the contained ErlIOVec - * and corresponding binaries for the response - */ - d = EF_ALLOC(sizeof(*d) - + (n * sizeof(*d->c.preadv.offsets)) - + ((1+n) * (sizeof(*res_ev->iov) - + sizeof(*res_ev->binv)))); - if (! d) { - reply_posix_error(desc, ENOMEM); - goto done; - } - d->command = command; - d->reply = !0; - d->fd = desc->fd; - d->flags = desc->flags; -#ifdef USE_VM_PROBES - dt_i1 = d->fd; - dt_i2 = d->flags; -#endif - d->c.preadv.n = n; - d->c.preadv.cnt = 0; - d->c.preadv.size = 0; - res_ev = &d->c.preadv.eiov; - /* XXX possible alignment problems here for weird machines */ - res_ev->vsize = 1+d->c.preadv.n; - res_ev->iov = void_ptr = &d->c.preadv.offsets[d->c.preadv.n]; - res_ev->binv = void_ptr = &res_ev->iov[res_ev->vsize]; - /* Read in the pos/size specs and allocate binaries for the results */ - for (i = 1; i < 1+n; i++) { - Uint32 sizeH, sizeL; - size_t size; - if ( !EV_GET_SINT64(ev, &d->c.preadv.offsets[i-1], &p, &q) - || !EV_GET_UINT32(ev, &sizeH, &p, &q) - || !EV_GET_UINT32(ev, &sizeL, &p, &q)) { - reply_posix_error(desc, EINVAL); - break; - } -#if SIZEOF_SIZE_T == 4 - if (sizeH != 0) { - reply_posix_error(desc, EINVAL); - break; - } - size = sizeL; -#else - size = ((size_t)sizeH<<32) | sizeL; -#endif -#ifdef USE_VM_PROBES - dt_i3 += size; -#endif - if (! (res_ev->binv[i] = driver_alloc_binary(size))) { - reply_posix_error(desc, ENOMEM); - break; - } else { - res_ev->iov[i].iov_len = size; - res_ev->iov[i].iov_base = res_ev->binv[i]->orig_bytes; - } - } - if (i < 1+n) { - for (i--; i > 0; i--) { - driver_free_binary(res_ev->binv[i]); - } - EF_FREE(d); - goto done; - } - /* Allocate the header binary (index 0) */ - res_ev->binv[0] = driver_alloc_binary(4+4+8*n); - if (! res_ev->binv[0]) { - reply_posix_error(desc, ENOMEM); - for (i = 1; i < 1+n; i++) { - driver_free_binary(res_ev->binv[i]); - } - EF_FREE(d); - goto done; - } - res_ev->iov[0].iov_len = 4+4+8*n; - res_ev->iov[0].iov_base = res_ev->binv[0]->orig_bytes; - /* Fill in the number of buffers in the header */ - put_int32(0, res_ev->iov[0].iov_base); - put_int32(n, (char *)(res_ev->iov[0].iov_base) + 4); - /**/ - res_ev->size = res_ev->iov[0].iov_len; - if (n == 0) { - /* Trivial case - nothing to read */ - reply_ev(desc, FILE_RESP_LDATA, res_ev); - free_preadv(d); - goto done; - } else { - d->invoke = invoke_preadv; - d->free = free_preadv; - d->level = 1; - cq_enq(desc, d); - } - } goto done; /* case FILE_PREADV: */ - - case FILE_LSEEK: { - Sint64 offset; /* Offset for seek */ - Uint32 origin; /* Origin of seek. */ - - if (ev->size < 1+8+4 - || !EV_GET_SINT64(ev, &offset, &p, &q) - || !EV_GET_UINT32(ev, &origin, &p, &q)) { - /* Wrong length of buffer to contain offset and origin */ - reply_posix_error(desc, EINVAL); - goto done; - } -#ifdef USE_VM_PROBES - dt_utag = EV_CHAR_P(ev, p, q); -#endif - if (lseek_flush_read(desc, &err -#ifdef USE_VM_PROBES - , dt_priv, dt_utag -#endif - ) < 0) { - reply_posix_error(desc, err); - goto done; - } - if (flush_write_check_error(desc, &err -#ifdef USE_VM_PROBES - , dt_priv, dt_utag -#endif - ) < 0) { - reply_posix_error(desc, err); - goto done; - } - if ((d = async_lseek(desc, &err, !0, offset, origin -#ifdef USE_VM_PROBES - , &dt_i1, &dt_i2, &dt_i3 -#endif - )) == NULL) { - reply_posix_error(desc, err); - goto done; - } - } goto done; - - case FILE_READ_FILE: { - char *filename; - if (ev->size < 1+1) { - /* Buffer contains empty name */ - reply_posix_error(desc, ENOENT); - goto done; - } -#ifndef USE_VM_PROBES - /* In the dtrace case, the iov has an extra element, the dtrace utag - we will need - another test to see that - the filename is in a single buffer: */ - if (ev->size-1 != ev->iov[q].iov_len-p) { - /* Name not in one single buffer */ - reply_posix_error(desc, EINVAL); - goto done; - } -#else - if (((byte *)ev->iov[q].iov_base)[ev->iov[q].iov_len-1] != '\0') { - /* Name not in one single buffer */ - reply_posix_error(desc, EINVAL); - goto done; - } -#endif - filename = EV_CHAR_P(ev, p, q); - d = EF_ALLOC(sizeof(struct t_data) -1 + FILENAME_BYTELEN(filename) + FILENAME_CHARSIZE); - if (! d) { - reply_posix_error(desc, ENOMEM); - goto done; - } - d->command = command; - d->reply = !0; - /* Copy name */ - FILENAME_COPY(d->b, filename); -#ifdef USE_VM_PROBES - { - char dt_tmp; - - /* This will work for UTF-8, but not for UTF-16 - extra reminder here */ -#ifdef FILENAMES_16BIT -#error 16bit characters in filenames and dtrace in combination is not supported. -#endif - while (EV_GET_CHAR(ev, &dt_tmp, &p, &q) && dt_tmp != '\0') - ; - dt_s1 = d->b; - dt_utag = EV_CHAR_P(ev, p, q); - } -#endif - d->c.read_file.binp = NULL; - d->invoke = invoke_read_file; - d->free = free_read_file; - d->level = 2; - cq_enq(desc, d); - } goto done; - - case FILE_IPREAD: { - /* This operation cheets by using invoke_preadv() and free_preadv() - * plus its own invoke_ipread. Therefore the result format is - * a bit awkward - the header binary contains one extra 64 bit - * field that invoke_preadv() fortunately ignores, - * and the first 64 bit field does not contain the number of - * data binaries which invoke_preadv() also ignores. - */ - register void * void_ptr; - char mode; - Sint64 hdr_offset; - Uint32 max_size; - ErlIOVec *res_ev; - int vsize; - if (! EV_GET_CHAR(ev, &mode, &p, &q)) { - /* Empty command */ - reply_posix_error(desc, EINVAL); - goto done; - } - if (mode != IPREAD_S32BU_P32BU) { - reply_posix_error(desc, EINVAL); - goto done; - } - if (ev->size < 1+1+8+4 - || !EV_GET_SINT64(ev, &hdr_offset, &p, &q) - || !EV_GET_UINT32(ev, &max_size, &p, &q)) { - /* Buffer too short to contain - * the header offset and max size spec */ - reply_posix_error(desc, EINVAL); - goto done; - } -#ifdef USE_VM_PROBES - dt_utag = EV_CHAR_P(ev, p, q); -#endif - if (lseek_flush_read(desc, &err -#ifdef USE_VM_PROBES - , dt_priv, dt_utag -#endif - ) < 0) { - reply_posix_error(desc, err); - goto done; - } - if (flush_write_check_error(desc, &err -#ifdef USE_VM_PROBES - , dt_priv, dt_utag -#endif - ) < 0) { - reply_posix_error(desc, err); - goto done; - } - /* Create the thread data structure with the contained ErlIOVec - * and corresponding binaries for the response - */ - vsize = 2; - d = EF_ALLOC(sizeof(*d) + - vsize*(sizeof(*res_ev->iov) + sizeof(*res_ev->binv))); - if (! d) { - reply_posix_error(desc, ENOMEM); - goto done; - } - d->command = command; - d->reply = !0; - d->fd = desc->fd; - d->flags = desc->flags; - d->c.preadv.offsets[0] = hdr_offset; - d->c.preadv.size = max_size; -#ifdef USE_VM_PROBES - dt_i1 = d->fd; - dt_i2 = d->flags; - dt_i3 = d->c.preadv.offsets[0]; - dt_i4 = d->c.preadv.size; -#endif - res_ev = &d->c.preadv.eiov; - /* XXX possible alignment problems here for weird machines */ - res_ev->iov = void_ptr = d + 1; - res_ev->binv = void_ptr = res_ev->iov + vsize; - res_ev->size = 0; - res_ev->vsize = 0; - d->invoke = invoke_ipread; - d->free = free_preadv; - d->level = 1; - cq_enq(desc, d); - } goto done; /* case FILE_IPREAD: */ - - case FILE_SETOPT: { - char opt; - - if (ev->size < 1+1 - || !EV_GET_CHAR(ev, &opt, &p, &q)) { - /* Buffer too short to contain even the option type */ - reply_posix_error(desc, EINVAL); - goto done; - } -#ifdef USE_VM_PROBES - dt_i1 = opt; - dt_utag = EV_CHAR_P(ev, p, q); -#endif - switch (opt) { - case FILE_OPT_DELAYED_WRITE: { - Uint32 sizeH, sizeL, delayH, delayL; - if (ev->size != 1+1+4*sizeof(Uint32) -#ifdef USE_VM_PROBES - + FILENAME_BYTELEN(dt_utag) + FILENAME_CHARSIZE -#endif - || !EV_GET_UINT32(ev, &sizeH, &p, &q) - || !EV_GET_UINT32(ev, &sizeL, &p, &q) - || !EV_GET_UINT32(ev, &delayH, &p, &q) - || !EV_GET_UINT32(ev, &delayL, &p, &q)) { - /* Buffer has wrong length to contain the option values */ - reply_posix_error(desc, EINVAL); - goto done; - } -#if SIZEOF_SIZE_T == 4 - if (sizeH != 0) { - reply_posix_error(desc, EINVAL); - goto done; - } - desc->write_bufsize = sizeL; -#else - desc->write_bufsize = ((size_t)sizeH << 32) | sizeL; -#endif -#if SIZEOF_LONG == 4 - if (delayH != 0) { - reply_posix_error(desc, EINVAL); - goto done; - } - desc->write_delay = delayL; -#else - desc->write_delay = ((unsigned long)delayH << 32) | delayL; -#endif -#ifdef USE_VM_PROBES - dt_i2 = desc->write_delay; -#endif - TRACE_C('K'); - reply_ok(desc); - } goto done; - case FILE_OPT_READ_AHEAD: { - Uint32 sizeH, sizeL; - if (ev->size != 1+1+2*sizeof(Uint32) -#ifdef USE_VM_PROBES - + FILENAME_BYTELEN(dt_utag)+FILENAME_CHARSIZE -#endif - || !EV_GET_UINT32(ev, &sizeH, &p, &q) - || !EV_GET_UINT32(ev, &sizeL, &p, &q)) { - /* Buffer has wrong length to contain the option values */ - reply_posix_error(desc, EINVAL); - goto done; - } -#if SIZEOF_SIZE_T == 4 - if (sizeH != 0) { - reply_posix_error(desc, EINVAL); - goto done; - } - desc->read_bufsize = sizeL; -#else - desc->read_bufsize = ((size_t)sizeH << 32) | sizeL; -#endif -#ifdef USE_VM_PROBES - dt_i2 = desc->read_bufsize; -#endif - TRACE_C('K'); - reply_ok(desc); - } goto done; - default: - reply_posix_error(desc, EINVAL); - goto done; - } /* case FILE_OPT_DELAYED_WRITE: */ - } ASSERT(0); goto done; /* case FILE_SETOPT: */ - - case FILE_SENDFILE: { - -#ifdef HAVE_SENDFILE - struct t_data *d; - Uint32 out_fd, offsetH, offsetL, hd_len, tl_len; - Uint64 nbytes; - char flags; - - if (ev->size < 1 + 7 * sizeof(Uint32) + sizeof(char) - || !EV_GET_UINT32(ev, &out_fd, &p, &q) - || !EV_GET_CHAR(ev, &flags, &p, &q) - || !EV_GET_UINT32(ev, &offsetH, &p, &q) - || !EV_GET_UINT32(ev, &offsetL, &p, &q) - || !EV_GET_UINT64(ev, &nbytes, &p, &q) - || !EV_GET_UINT32(ev, &hd_len, &p, &q) - || !EV_GET_UINT32(ev, &tl_len, &p, &q)) { - /* Buffer has wrong length to contain all the needed values */ - reply_posix_error(desc, EINVAL); - goto done; - } - - if (hd_len != 0 || tl_len != 0) { - /* We do not allow header, trailers */ - reply_posix_error(desc, EINVAL); - goto done; - } - - - if (flags & SENDFILE_FLGS_USE_THREADS && !THRDS_AVAILABLE) { - /* We do not allow use_threads flag on a system where - no threads are available. */ - reply_posix_error(desc, EINVAL); - goto done; - } - - d = EF_SAFE_ALLOC(sizeof(struct t_data)); - d->fd = desc->fd; - d->command = command; - d->invoke = invoke_sendfile; - d->free = free_sendfile; - d->flags = flags; - d->level = 2; - - d->c.sendfile.out_fd = (int) out_fd; - d->c.sendfile.written = 0; - d->c.sendfile.port = desc->port; - d->c.sendfile.q_mtx = desc->q_mtx; - - #if SIZEOF_OFF_T == 4 - if (offsetH != 0) { - reply_posix_error(desc, EINVAL); - goto done; - } - d->c.sendfile.offset = (off_t) offsetL; - #else - d->c.sendfile.offset = ((off_t) offsetH << 32) | offsetL; - #endif - - d->c.sendfile.nbytes = nbytes; - - if (USE_THRDS_FOR_SENDFILE(d)) { - SET_BLOCKING(d->c.sendfile.out_fd); - } else { - /** - * Write a place holder to queue in order to force file_flush - * to be called before the driver is closed. - */ - char tmp[1] = ""; - MUTEX_LOCK(d->c.sendfile.q_mtx); - if (driver_enq(d->c.sendfile.port, tmp, 1)) { - MUTEX_UNLOCK(d->c.sendfile.q_mtx); - reply_posix_error(desc, ENOMEM); - goto done; - } - MUTEX_UNLOCK(d->c.sendfile.q_mtx); - } - - cq_enq(desc, d); -#else - reply_posix_error(desc, ENOTSUP); -#endif - goto done; - } /* case FILE_SENDFILE: */ - - } /* switch(command) */ - - if (lseek_flush_read(desc, &err -#ifdef USE_VM_PROBES - , dt_priv, dt_utag -#endif - ) < 0) { - reply_posix_error(desc, err); - goto done; - } - if (flush_write_check_error(desc, &err -#ifdef USE_VM_PROBES - , dt_priv, dt_utag -#endif - ) < 0) { - reply_posix_error(desc, err); - goto done; - } else { - /* Flatten buffer and send it to file_output(desc, buf, len) */ - int len = ev->size; - char *buf = EF_ALLOC(len); - if (! buf) { - reply_posix_error(desc, ENOMEM); - goto done; - } - driver_vec_to_buf(ev, buf, len); - file_output((ErlDrvData) desc, buf, len); - EF_FREE(buf); - goto done; - } - - done: - if (d != NULL) { -#ifdef USE_VM_PROBES - /* - * If d == NULL, then either: - * 1). There was an error of some sort, or - * 2). The command given to us is actually implemented - * by file_output() instead. - * - * Case #1 is probably a TODO item, perhaps? - * Case #2 we definitely don't want to activate a probe. - */ - d->sched_i1 = dt_priv->thread_num; - d->sched_i2 = dt_priv->tag; - d->sched_utag[0] = '\0'; - if (dt_utag != NULL) { - if (dt_utag[0] == '\0') { - dt_utag = NULL; - } else { - strncpy(d->sched_utag, dt_utag, sizeof(d->sched_utag) - 1); - d->sched_utag[sizeof(d->sched_utag) - 1] = '\0'; - } - } - DTRACE11(efile_drv_entry, dt_priv->thread_num, dt_priv->tag++, - dt_utag, command, dt_s1, NULL, dt_i1, dt_i2, dt_i3, dt_i4, - desc->port_str); -#endif - } - cq_execute(desc); -} - -#ifdef USE_VM_PROBES -dt_private * -get_dt_private(int base) -{ - dt_private *dt_priv = (dt_private *) pthread_getspecific(dt_driver_key); - - if (dt_priv == NULL) { - dt_priv = EF_SAFE_ALLOC(sizeof(dt_private)); - erts_mtx_lock(&dt_driver_mutex); - dt_priv->thread_num = (base + dt_driver_idnum++); - erts_mtx_unlock(&dt_driver_mutex); - dt_priv->tag = 0; - pthread_setspecific(dt_driver_key, dt_priv); - } - return dt_priv; -} -#endif /* USE_VM_PROBES */ diff --git a/erts/emulator/drivers/common/erl_efile.h b/erts/emulator/drivers/common/erl_efile.h deleted file mode 100644 index b7f063b4f2..0000000000 --- a/erts/emulator/drivers/common/erl_efile.h +++ /dev/null @@ -1,176 +0,0 @@ -/* - * %CopyrightBegin% - * - * Copyright Ericsson AB 1997-2016. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - * %CopyrightEnd% - */ -/* - * Defines the interfaces between the generic efile driver and its - * operating-system dependent helpers. - */ - -#include "sys.h" -#include "erl_driver.h" - -/* - * Open modes for efile_openfile(). - */ -#define EFILE_MODE_READ 1 -#define EFILE_MODE_WRITE 2 /* Implies truncating file when used alone. */ -#define EFILE_MODE_READ_WRITE 3 -#define EFILE_MODE_APPEND 4 -#define EFILE_COMPRESSED 8 -#define EFILE_MODE_EXCL 16 -#define EFILE_NO_TRUNCATE 32 /* Special for reopening on VxWorks */ -#define EFILE_MODE_SYNC 64 - -/* - * Seek modes for efile_seek(). - */ -#define EFILE_SEEK_SET 0 -#define EFILE_SEEK_CUR 1 -#define EFILE_SEEK_END 2 - -/* - * File types returned by efile_fileinfo(). - */ -#define FT_DEVICE 1 -#define FT_DIRECTORY 2 -#define FT_REGULAR 3 -#define FT_SYMLINK 4 -#define FT_OTHER 5 - -/* - * Access attributes returned by efile_fileinfo() (the bits can be ORed - * together). - */ -#define FA_NONE 0 -#define FA_WRITE 1 -#define FA_READ 2 - -/* Some OS'es (i.e. Windows) has filenames in wide charaqcters. That requires special handling */ -/* Note that we do *not* honor alignment in the communication to the OS specific driver, */ -/* which is not a problem on x86, but might be on other platforms. The OS specific efile */ -/* implementation is expected to align if needed */ -#ifdef __WIN32__ -#define FILENAMES_16BIT 1 -#endif - -/* We use sendfilev if it exist on solaris */ -#if !defined(HAVE_SENDFILE) && defined(HAVE_SENDFILEV) -#define HAVE_SENDFILE -#endif - -/* - * An handle to an open directory. To be cast to the correct type - * in the system-dependent directory functions. - */ - -typedef struct _Efile_Dir_Handle* EFILE_DIR_HANDLE; - -/* - * Error information from the last call. - */ -typedef struct _Efile_error { - int posix_errno; /* Posix error number, as in <errno.h>. */ - int os_errno; /* Os-dependent error number (not used). */ -} Efile_error; - -/* - * Describes what is returned by file:file_info/1. - */ - -typedef struct _Efile_info { - Uint32 size_low; /* Size of file, lower 32 bits.. */ - Uint32 size_high; /* Size of file, higher 32 bits. */ - Uint32 type; /* Type of file -- one of FT_*. */ - Uint32 access; /* Access to file -- one of FA_*. */ - Uint32 mode; /* Access permissions -- bit field. */ - Uint32 links; /* Number of links to file. */ - Uint32 major_device; /* Major device or file system. */ - Uint32 minor_device; /* Minor device (for devices). */ - Uint32 inode; /* Inode number. */ - Uint32 uid; /* User id of owner. */ - Uint32 gid; /* Group id of owner. */ - Sint64 accessTime; /* Last time the file was accessed. */ - Sint64 modifyTime; /* Last time the file was modified. */ - Sint64 cTime; /* Creation time (Windows) or last - * inode change (Unix). - */ -} Efile_info; - - -#ifdef HAVE_SENDFILE -/* - * Describes the structure of headers/trailers for sendfile - */ -struct t_sendfile_hdtl { - SysIOVec *headers; - int hdr_cnt; - SysIOVec *trailers; - int trl_cnt; -}; -#endif /* HAVE_SENDFILE */ - -/* - * Functions. - */ -int efile_init(void); -int efile_mkdir(Efile_error* errInfo, char* name); -int efile_rmdir(Efile_error* errInfo, char* name); -int efile_delete_file(Efile_error* errInfo, char* name); -int efile_rename(Efile_error* errInfo, char* src, char* dst); -int efile_chdir(Efile_error* errInfo, char* name); -int efile_getdcwd(Efile_error* errInfo, int drive, - char* buffer, size_t size); -int efile_readdir(Efile_error* errInfo, char* name, - EFILE_DIR_HANDLE* dir_handle, - char* buffer, size_t *size); -int efile_openfile(Efile_error* errInfo, char* name, int flags, - int* pfd, Sint64* pSize); -void efile_closefile(int fd); -int efile_fdatasync(Efile_error* errInfo, int fd); -int efile_fsync(Efile_error* errInfo, int fd); -int efile_fileinfo(Efile_error* errInfo, Efile_info* pInfo, - char *name, int info_for_link); -int efile_write_info(Efile_error* errInfo, Efile_info* pInfo, char *name); -int efile_write(Efile_error* errInfo, int flags, int fd, - char* buf, size_t count); -int efile_writev(Efile_error* errInfo, int flags, int fd, - SysIOVec* iov, int iovcnt); -int efile_read(Efile_error* errInfo, int flags, int fd, - char* buf, size_t count, size_t* pBytesRead); -int efile_seek(Efile_error* errInfo, int fd, - Sint64 offset, int origin, Sint64* new_location); -int efile_truncate_file(Efile_error* errInfo, int *fd, int flags); -int efile_pwrite(Efile_error* errInfo, int fd, - char* buf, size_t count, Sint64 offset); -int efile_pread(Efile_error* errInfo, int fd, - Sint64 offset, char* buf, size_t count, size_t* pBytesRead); -int efile_readlink(Efile_error* errInfo, char *name, - char* buffer, size_t size); -int efile_altname(Efile_error* errInfo, char *name, - char* buffer, size_t size); -int efile_link(Efile_error* errInfo, char* old, char* new); -int efile_symlink(Efile_error* errInfo, char* old, char* new); -int efile_may_openfile(Efile_error* errInfo, char *name); -int efile_fadvise(Efile_error* errInfo, int fd, Sint64 offset, Sint64 length, - int advise); -#ifdef HAVE_SENDFILE -int efile_sendfile(Efile_error* errInfo, int in_fd, int out_fd, - off_t *offset, Uint64 *nbytes, struct t_sendfile_hdtl *hdtl); -#endif /* HAVE_SENDFILE */ -int efile_fallocate(Efile_error* errInfo, int fd, Sint64 offset, Sint64 length); diff --git a/erts/emulator/drivers/common/gzio.c b/erts/emulator/drivers/common/gzio.c index f60c781894..86c3b07cea 100644 --- a/erts/emulator/drivers/common/gzio.c +++ b/erts/emulator/drivers/common/gzio.c @@ -19,726 +19,16 @@ #include <unistd.h> #endif #include <ctype.h> + #include "erl_driver.h" -#include "erl_efile.h" #include "sys.h" -#ifdef __WIN32__ -#ifndef HAVE_CONFLICTING_FREAD_DECLARATION -#define HAVE_CONFLICTING_FREAD_DECLARATION -#endif -#define FILENAMES_16BIT 1 -#endif - -#ifdef STDC -# define zstrerror(errnum) strerror(errnum) -#else -# define zstrerror(errnum) "" -#endif - #include "gzio_zutil.h" #include "erl_zlib.h" #include "gzio.h" -/********struct internal_state {int dummy;}; / * for buggy compilers */ - -#define Z_BUFSIZE 4096 - -#define ALLOC(size) driver_alloc(size) -#define TRYFREE(p) {if (p) driver_free(p);} - static int gz_magic[2] = {0x1f, 0x8b}; /* gzip magic header */ -/* gzip flag byte */ -#define ASCII_FLAG 0x01 /* bit 0 set: file probably ascii text */ -#define HEAD_CRC 0x02 /* bit 1 set: header CRC present */ -#define EXTRA_FIELD 0x04 /* bit 2 set: extra field present */ -#define ORIG_NAME 0x08 /* bit 3 set: original file name present */ -#define COMMENT 0x10 /* bit 4 set: file comment present */ -#define RESERVED 0xE0 /* bits 5..7: reserved */ - -typedef struct gz_stream { - z_stream stream; - int z_err; /* error code for last stream operation */ - int z_eof; /* set if end of input file */ -#ifdef UNIX - int file; /* .gz file descriptor */ -#else - FILE *file; /* .gz file */ -#endif - Byte *inbuf; /* input buffer */ - Byte *outbuf; /* output buffer */ - uLong crc; /* crc32 of uncompressed data */ - char *msg; /* error message */ - char *path; /* path name for debugging only */ - int transparent; /* 1 if input file is not a .gz file */ - char mode; /* 'w' or 'r' */ - int position; /* Position (for seek) */ - int (*destroy)(struct gz_stream*); /* Function to destroy - * this structure. */ -} gz_stream; - -local ErtsGzFile gz_open (const char *path, const char *mode); -local int get_byte (gz_stream *s); -local void check_header (gz_stream *s); -local int destroy (gz_stream *s); -local uLong getLong (gz_stream *s); - -#ifdef UNIX -/* - * In Solaris 8 and earlier, fopen() and its friends cannot handle - * file descriptors larger than 255. Therefore, we use read()/write() - * on all Unix systems. - */ -# define ERTS_GZWRITE(File, Buf, Count) write((File), (Buf), (Count)) -# define ERTS_GZREAD(File, Buf, Count) read((File), (Buf), (Count)) -#else -/* - * On all other operating systems, using fopen(), fread()/fwrite(), since - * there is not guaranteed to exist any read()/write() (not part of - * ANSI/ISO-C). - */ -# define ERTS_GZWRITE(File, Buf, Count) fwrite((Buf), 1, (Count), (File)) -# define ERTS_GZREAD(File, Buf, Count) fread((Buf), 1, (Count), (File)) -#endif - -/* - * Ripped from efile_drv.c - */ - -#ifdef FILENAMES_16BIT -# define FILENAME_BYTELEN(Str) filename_len_16bit(Str) -# define FILENAME_COPY(To,From) filename_cpy_16bit((To),(From)) -# define FILENAME_CHARSIZE 2 - - static int filename_len_16bit(const char *str) - { - const char *p = str; - while(*p != '\0' || p[1] != '\0') { - p += 2; - } - return (p - str); - } - - static void filename_cpy_16bit(char *to, const char *from) - { - while(*from != '\0' || from[1] != '\0') { - *to++ = *from++; - *to++ = *from++; - } - *to++ = *from++; - *to++ = *from++; - } - -#else -# define FILENAME_BYTELEN(Str) strlen(Str) -# define FILENAME_COPY(To,From) strcpy(To,From) -# define FILENAME_CHARSIZE 1 -#endif - -/* =========================================================================== - Opens a gzip (.gz) file for reading or writing. The mode parameter - is as in fopen ("rb" or "wb"). The file is given either by file descriptor - or path name (if fd == -1). - gz_open return NULL if the file could not be opened or if there was - insufficient memory to allocate the (de)compression state; errno - can be checked to distinguish the two cases (if errno is zero, the - zlib error is Z_MEM_ERROR). -*/ -local ErtsGzFile gz_open (path, mode) - const char *path; - const char *mode; -{ - int err; - int level = Z_DEFAULT_COMPRESSION; /* compression level */ - char *p = (char*)mode; - gz_stream *s; - char fmode[80]; /* copy of mode, without the compression level */ - char *m = fmode; - - if (!path || !mode) return Z_NULL; - - s = (gz_stream *)ALLOC(sizeof(gz_stream)); - if (!s) return Z_NULL; - - erl_zlib_alloc_init(&s->stream); - s->stream.next_in = s->inbuf = Z_NULL; - s->stream.next_out = s->outbuf = Z_NULL; - s->stream.avail_in = s->stream.avail_out = 0; -#ifdef UNIX - s->file = -1; -#else - s->file = NULL; -#endif - s->z_err = Z_OK; - s->z_eof = 0; - s->crc = crc32(0L, Z_NULL, 0); - s->msg = NULL; - s->transparent = 0; - s->position = 0; - s->destroy = destroy; - - s->path = (char*)ALLOC(FILENAME_BYTELEN(path)+FILENAME_CHARSIZE); - if (s->path == NULL) { - return s->destroy(s), (ErtsGzFile)Z_NULL; - } - FILENAME_COPY(s->path, path); /* do this early for debugging */ - - s->mode = '\0'; - do { - if (*p == 'r') - s->mode = 'r'; - if (*p == 'w' || *p == 'a') - s->mode = 'w'; - if (isdigit((int)*p)) { - level = *p - '0'; - } else { - *m++ = *p; /* Copy the mode */ - } - } while (*p++ && m < fmode + sizeof(fmode) - 1); - *m = '\0'; - if (s->mode == '\0') - return s->destroy(s), (ErtsGzFile)Z_NULL; - - if (s->mode == 'w') { - err = deflateInit2(&(s->stream), level, - Z_DEFLATED, MAX_WBITS+16, DEF_MEM_LEVEL, 0); - /* windowBits is passed < 0 to suppress zlib header */ - - s->stream.next_out = s->outbuf = (Byte*)ALLOC(Z_BUFSIZE); - - if (err != Z_OK || s->outbuf == Z_NULL) { - return s->destroy(s), (ErtsGzFile)Z_NULL; - } - } else { - /* - * It is tempting to use the built-in support in zlib - * for handling GZIP headers, but unfortunately it - * cannot handle multiple GZIP headers (which occur when - * several GZIP files have been concatenated). - */ - - err = inflateInit2(&(s->stream), -MAX_WBITS); - s->stream.next_in = s->inbuf = (Byte*)ALLOC(Z_BUFSIZE); - - if (err != Z_OK || s->inbuf == Z_NULL) { - return s->destroy(s), (ErtsGzFile)Z_NULL; - } - } - s->stream.avail_out = Z_BUFSIZE; - - errno = 0; -#if defined(FILENAMES_16BIT) - { - FILE* efile_wfopen(const WCHAR* name, const WCHAR* mode); - WCHAR wfmode[80]; - int i = 0; - int j; - for(j = 0; fmode[j] != '\0'; ++j) { - wfmode[i++] = (WCHAR) fmode[j]; - } - wfmode[i++] = L'\0'; - s->file = efile_wfopen((WCHAR *)path, wfmode); - if (s->file == NULL) { - return s->destroy(s), (ErtsGzFile)Z_NULL; - } - } -#elif defined(UNIX) - if (s->mode == 'r') { - s->file = open(path, O_RDONLY); - } else { - s->file = open(path, O_WRONLY | O_CREAT | O_TRUNC, 0666); - } - if (s->file == -1) { - return s->destroy(s), (ErtsGzFile)Z_NULL; - } -#else - s->file = fopen(path, fmode); - if (s->file == NULL) { - return s->destroy(s), (ErtsGzFile)Z_NULL; - } -#endif - if (s->mode == 'r') { - check_header(s); /* skip the .gz header */ - } - return (ErtsGzFile)s; -} - -/* =========================================================================== - Rewind a gzfile back to the beginning. -*/ - -local int gz_rewind (gz_stream *s) -{ - TRYFREE(s->msg); - -#ifdef UNIX - lseek(s->file, 0L, SEEK_SET); -#else - fseek(s->file, 0L, SEEK_SET); -#endif - inflateReset(&(s->stream)); - s->stream.next_in = Z_NULL; - s->stream.next_out = Z_NULL; - s->stream.avail_in = s->stream.avail_out = 0; - s->z_err = Z_OK; - s->z_eof = 0; - s->crc = crc32(0L, Z_NULL, 0); - s->msg = NULL; - s->position = 0; - s->stream.next_in = s->inbuf; - - s->stream.avail_out = Z_BUFSIZE; - - check_header(s); /* skip the .gz header */ - return 1; -} - -/* =========================================================================== - Opens a gzip (.gz) file for reading or writing. -*/ -ErtsGzFile erts_gzopen (path, mode) - const char *path; - const char *mode; -{ - return gz_open (path, mode); -} - - -/* =========================================================================== - Read a byte from a gz_stream; update next_in and avail_in. Return EOF - for end of file. - IN assertion: the stream s has been successfully opened for reading. -*/ -local int get_byte(s) - gz_stream *s; -{ - if (s->z_eof) return EOF; - if (s->stream.avail_in == 0) { -#ifdef UNIX - ssize_t res; - errno = 0; - res = ERTS_GZREAD(s->file, s->inbuf, Z_BUFSIZE); - if (res == 0) { - s->stream.avail_in = 0; - s->z_eof = 1; - return EOF; - } else if (res < 0) { - s->stream.avail_in = 0; - s->z_eof = 1; - s->z_err = Z_ERRNO; - return EOF; - } else { - s->stream.avail_in = (uInt) res; - } -#else - errno = 0; - s->stream.avail_in = ERTS_GZREAD(s->file, s->inbuf, Z_BUFSIZE); - if (s->stream.avail_in == 0) { - s->z_eof = 1; - if (s->file && ferror(s->file)) - s->z_err = Z_ERRNO; - return EOF; - } -#endif - s->stream.next_in = s->inbuf; - } - s->stream.avail_in--; - return *(s->stream.next_in)++; -} - -/* =========================================================================== - Check the gzip header of a gz_stream opened for reading. Set the stream - mode to transparent if the gzip magic header is not present; set s->err - to Z_DATA_ERROR if the magic header is present but the rest of the header - is incorrect. - IN assertion: the stream s has already been created sucessfully; - s->stream.avail_in is zero for the first time, but may be non-zero - for concatenated .gz files. -*/ -local void check_header(s) - gz_stream *s; -{ - int method; /* method byte */ - int flags; /* flags byte */ - uInt len; - int c; - - /* Check the gzip magic header */ - for (len = 0; len < 2; len++) { - c = get_byte(s); - if (c != gz_magic[len]) { - if (len != 0) s->stream.avail_in++, s->stream.next_in--; - if (c != EOF) { - s->stream.avail_in++, s->stream.next_in--; - s->transparent = 1; - } - s->z_err = s->stream.avail_in != 0 ? Z_OK : Z_STREAM_END; - return; - } - } - method = get_byte(s); - flags = get_byte(s); - if (method != Z_DEFLATED || (flags & RESERVED) != 0) { - s->z_err = Z_DATA_ERROR; - return; - } - - /* Discard time, xflags and OS code: */ - for (len = 0; len < 6; len++) (void)get_byte(s); - - if ((flags & EXTRA_FIELD) != 0) { /* skip the extra field */ - len = (uInt)get_byte(s); - len += ((uInt)get_byte(s))<<8; - /* len is garbage if EOF but the loop below will quit anyway */ - while (len-- != 0 && get_byte(s) != EOF) ; - } - if ((flags & ORIG_NAME) != 0) { /* skip the original file name */ - while ((c = get_byte(s)) != 0 && c != EOF) ; - } - if ((flags & COMMENT) != 0) { /* skip the .gz file comment */ - while ((c = get_byte(s)) != 0 && c != EOF) ; - } - if ((flags & HEAD_CRC) != 0) { /* skip the header crc */ - for (len = 0; len < 2; len++) (void)get_byte(s); - } - s->z_err = s->z_eof ? Z_DATA_ERROR : Z_OK; -} - - /* =========================================================================== - * Cleanup then free the given gz_stream. Return a zlib error code. - Try freeing in the reverse order of allocations. - */ -local int destroy (s) - gz_stream *s; -{ - int err = Z_OK; - - if (!s) return Z_STREAM_ERROR; - - TRYFREE(s->msg); - - if (s->stream.state != NULL) { - if (s->mode == 'w') { - err = deflateEnd(&(s->stream)); - } else if (s->mode == 'r') { - err = inflateEnd(&(s->stream)); - } - } -#ifdef UNIX - if (s->file != -1 && close(s->file)) { - err = Z_ERRNO; - } -#else - if (s->file != NULL && fclose(s->file)) { - err = Z_ERRNO; - } -#endif - if (s->z_err < 0) err = s->z_err; - - TRYFREE(s->inbuf); - TRYFREE(s->outbuf); - TRYFREE(s->path); - TRYFREE(s); - return err; -} - -/* =========================================================================== - Reads the given number of uncompressed bytes from the compressed file. - gzread returns the number of bytes actually read (0 for end of file). -*/ -int -erts_gzread(ErtsGzFile file, voidp buf, unsigned len) -{ - gz_stream *s = (gz_stream*)file; - Bytef *start = buf; /* starting point for crc computation */ - Byte *next_out; /* == stream.next_out but not forced far (for MSDOS) */ - - if (s == NULL || s->mode != 'r') return Z_STREAM_ERROR; - - if (s->z_err == Z_DATA_ERROR || s->z_err == Z_ERRNO) return -1; - if (s->z_err == Z_STREAM_END) return 0; /* EOF */ - - s->stream.next_out = next_out = buf; - s->stream.avail_out = len; - - while (s->stream.avail_out != 0) { - - if (s->transparent) { - /* Copy first the lookahead bytes: */ - uInt n = s->stream.avail_in; - if (n > s->stream.avail_out) n = s->stream.avail_out; - if (n > 0) { - zmemcpy(s->stream.next_out, s->stream.next_in, n); - next_out += n; - s->stream.next_out = next_out; - s->stream.next_in += n; - s->stream.avail_out -= n; - s->stream.avail_in -= n; - } - if (s->stream.avail_out > 0) { - s->stream.avail_out -= ERTS_GZREAD(s->file, next_out, - s->stream.avail_out); - } - len -= s->stream.avail_out; - s->stream.total_in += (uLong)len; - s->stream.total_out += (uLong)len; - if (len == 0) s->z_eof = 1; - s->position += (int)len; - return (int)len; - } - if (s->stream.avail_in == 0 && !s->z_eof) { -#ifdef UNIX - ssize_t res; - errno = 0; - res = ERTS_GZREAD(s->file, s->inbuf, Z_BUFSIZE); - if (res == 0) { - s->stream.avail_in = 0; - s->z_eof = 1; - return EOF; - } else if (res < 0) { - s->stream.avail_in = 0; - s->z_eof = 1; - s->z_err = Z_ERRNO; - return EOF; - } else { - s->stream.avail_in = (uInt) res; - } -#else - errno = 0; - s->stream.avail_in = ERTS_GZREAD(s->file, s->inbuf, Z_BUFSIZE); - if (s->stream.avail_in == 0) { - s->z_eof = 1; - if (s->file && ferror(s->file)) { - s->z_err = Z_ERRNO; - break; - } - } -#endif - s->stream.next_in = s->inbuf; - } - s->z_err = inflate(&(s->stream), Z_NO_FLUSH); - - if (s->z_err == Z_STREAM_END) { - /* Check CRC and original size */ - s->crc = crc32(s->crc, start, (uInt)(s->stream.next_out - start)); - start = s->stream.next_out; - - if (getLong(s) != s->crc) { - s->z_err = Z_DATA_ERROR; - } else { - (void)getLong(s); - /* The uncompressed length returned by above getlong() may - * be different from s->stream.total_out) in case of - * concatenated .gz files. Check for such files: - */ - check_header(s); - if (s->z_err == Z_OK) { - uLong total_in = s->stream.total_in; - uLong total_out = s->stream.total_out; - - inflateReset(&(s->stream)); - s->stream.total_in = total_in; - s->stream.total_out = total_out; - s->crc = crc32(0L, Z_NULL, 0); - } - } - } - if (s->z_err != Z_OK || s->z_eof) break; - } - s->crc = crc32(s->crc, start, (uInt)(s->stream.next_out - start)); - - s->position += (int)(len - s->stream.avail_out); - - return (int)(len - s->stream.avail_out); -} - -/* =========================================================================== - Writes the given number of uncompressed bytes into the compressed file. - gzwrite returns the number of bytes actually written (0 in case of error). -*/ -int -erts_gzwrite(ErtsGzFile file, voidp buf, unsigned len) -{ - gz_stream *s = (gz_stream*)file; - - if (s == NULL || s->mode != 'w') return Z_STREAM_ERROR; - - s->stream.next_in = buf; - s->stream.avail_in = len; - - while (s->stream.avail_in != 0) { - - if (s->stream.avail_out == 0) { - - s->stream.next_out = s->outbuf; - if (ERTS_GZWRITE(s->file, s->outbuf, Z_BUFSIZE) != Z_BUFSIZE) { - s->z_err = Z_ERRNO; - break; - } - s->stream.avail_out = Z_BUFSIZE; - } - s->z_err = deflate(&(s->stream), Z_NO_FLUSH); - if (s->z_err != Z_OK) break; - } - s->position += (int)(len - s->stream.avail_in); - return (int)(len - s->stream.avail_in); -} - -/* - * For use by Erlang file driver. - * - * XXX Limitations: - * - SEEK_END is not allowed (length of file is not known). - * - When writing, only forward seek is supported. - */ - -int -erts_gzseek(ErtsGzFile file, int offset, int whence) -{ - int pos; - gz_stream* s = (gz_stream *) file; - - switch (whence) { - case EFILE_SEEK_SET: whence = SEEK_SET; break; - case EFILE_SEEK_CUR: whence = SEEK_CUR; break; - case EFILE_SEEK_END: whence = SEEK_END; break; - default: - errno = EINVAL; - return -1; - } - - if (s == NULL) { - errno = EINVAL; - return -1; - } - if (s->z_err == Z_DATA_ERROR || s->z_err == Z_ERRNO) { - errno = EIO; - return -1; - } - - switch (whence) { - case SEEK_SET: pos = offset; break; - case SEEK_CUR: pos = s->position+offset; break; - case SEEK_END: - default: - errno = EINVAL; return -1; - } - - if (pos == s->position) { - return pos; - } - - if (pos < s->position) { - if (s->mode == 'w') { - errno = EINVAL; - return -1; - } - gz_rewind(s); - } - - while (s->position < pos) { - char buf[512]; - int n; - int save_pos = s->position; - - n = pos - s->position; - if (n > sizeof(buf)) - n = sizeof(buf); - - if (s->mode == 'r') { - erts_gzread(file, buf, n); - } else { - memset(buf, '\0', n); - erts_gzwrite(file, buf, n); - } - if (save_pos == s->position) break; - } - - return s->position; -} - -/* =========================================================================== - Flushes all pending output into the compressed file. The parameter - flush is as in the deflate() function. - gzflush should be called only when strictly necessary because it can - degrade compression. -*/ -int -erts_gzflush(ErtsGzFile file, int flush) -{ - uInt len; - int done = 0; - gz_stream *s = (gz_stream*)file; - - if (s == NULL || s->mode != 'w') return Z_STREAM_ERROR; - - s->stream.avail_in = 0; /* should be zero already anyway */ - - for (;;) { - len = Z_BUFSIZE - s->stream.avail_out; - - if (len != 0) { - if ((uInt)ERTS_GZWRITE(s->file, s->outbuf, len) != len) { - s->z_err = Z_ERRNO; - return Z_ERRNO; - } - s->stream.next_out = s->outbuf; - s->stream.avail_out = Z_BUFSIZE; - } - if (done) break; - s->z_err = deflate(&(s->stream), flush); - - /* deflate has finished flushing only when it hasn't used up - * all the available space in the output buffer: - */ - done = (s->stream.avail_out != 0 || s->z_err == Z_STREAM_END); - - if (s->z_err != Z_OK && s->z_err != Z_STREAM_END) break; - } -#ifndef UNIX - fflush(s->file); -#endif - return s->z_err == Z_STREAM_END ? Z_OK : s->z_err; -} - -/* =========================================================================== - Reads a long in LSB order from the given gz_stream. Sets -*/ -local uLong getLong (s) - gz_stream *s; -{ - uLong x = (uLong)get_byte(s); - int c; - - x += ((uLong)get_byte(s))<<8; - x += ((uLong)get_byte(s))<<16; - c = get_byte(s); - if (c == EOF) s->z_err = Z_DATA_ERROR; - x += ((uLong)c)<<24; - return x; -} - -/* =========================================================================== - Flushes all pending output if necessary, closes the compressed file - and deallocates all the (de)compression state. -*/ -int -erts_gzclose(ErtsGzFile file) -{ - int err; - gz_stream *s = (gz_stream*)file; - - if (s == NULL) return Z_STREAM_ERROR; - - if (s->mode == 'w') { - err = erts_gzflush (file, Z_FINISH); - if (err != Z_OK) return s->destroy(s); - } - return s->destroy(s); -} - - /* =========================================================================== Uncompresses the buffer given and returns a pointer to a binary. If the buffer was not compressed with gzip, the buffer contents diff --git a/erts/emulator/drivers/common/gzio.h b/erts/emulator/drivers/common/gzio.h index ee0ebe7bd8..e331b5208b 100644 --- a/erts/emulator/drivers/common/gzio.h +++ b/erts/emulator/drivers/common/gzio.h @@ -20,13 +20,5 @@ #include "zlib.h" -typedef struct erts_gzFile* ErtsGzFile; - -ErtsGzFile erts_gzopen (const char *path, const char *mode); -int erts_gzread(ErtsGzFile file, voidp buf, unsigned len); -int erts_gzwrite(ErtsGzFile file, voidp buf, unsigned len); -int erts_gzseek(ErtsGzFile, int, int); -int erts_gzflush(ErtsGzFile file, int flush); -int erts_gzclose(ErtsGzFile file); ErlDrvBinary* erts_gzinflate_buffer(char*, uLong); ErlDrvBinary* erts_gzdeflate_buffer(char*, uLong); diff --git a/erts/emulator/drivers/common/inet_drv.c b/erts/emulator/drivers/common/inet_drv.c index 95d61fcc5d..4294fb4f46 100644 --- a/erts/emulator/drivers/common/inet_drv.c +++ b/erts/emulator/drivers/common/inet_drv.c @@ -63,6 +63,20 @@ #include <sys/un.h> #endif +#ifdef HAVE_SENDFILE +#if defined(__linux__) || (defined(__sun) && defined(__SVR4)) + #include <sys/sendfile.h> +#elif defined(__FreeBSD__) || defined(__DragonFly__) + /* Need to define __BSD_VISIBLE in order to expose prototype of sendfile */ + #define __BSD_VISIBLE 1 + #include <sys/socket.h> +#endif +#endif + +#if defined(__APPLE__) && defined(__MACH__) && !defined(__DARWIN__) + #define __DARWIN__ 1 +#endif + /* All platforms fail on malloc errors. */ #define FATAL_MALLOC @@ -701,6 +715,7 @@ static size_t my_strnlen(const char *s, size_t maxlen) #define TCP_REQ_RECV 42 #define TCP_REQ_UNRECV 43 #define TCP_REQ_SHUTDOWN 44 +#define TCP_REQ_SENDFILE 45 /* UDP and SCTP requests */ #define PACKET_REQ_RECV 60 /* Common for UDP and SCTP */ /* #define SCTP_REQ_LISTEN 61 MERGED Different from TCP; not for UDP */ @@ -723,6 +738,7 @@ static size_t my_strnlen(const char *s, size_t maxlen) #define TCP_ADDF_DELAYED_ECONNRESET 128 /* An ECONNRESET error occurred on send or shutdown */ #define TCP_ADDF_SHUTDOWN_WR_DONE 256 /* A shutdown(sock, SHUT_WR) or SHUT_RDWR was made */ #define TCP_ADDF_LINGER_ZERO 512 /* Discard driver queue on port close */ +#define TCP_ADDF_SENDFILE 1024 /* Send from an fd instead of the driver queue */ /* *_REQ_* replies */ #define INET_REP_ERROR 0 @@ -1235,6 +1251,21 @@ typedef struct { inet_async_multi_op *multi_first;/* NULL == no multi-accept-queue, op is in ordinary queue */ inet_async_multi_op *multi_last; MultiTimerData *mtd; /* Timer structures for multiple accept */ +#ifdef HAVE_SENDFILE + struct { + ErlDrvSizeT ioq_skip; /* The number of bytes in the queue at the time + * sendfile was issued, which must be sent + * before issuing the sendfile call itself. */ + int dup_file_fd; /* The file handle to send from; this is + * duplicated when sendfile is issued to + * reduce (but not eliminate) the impact of a + * nasty race, so we have to remember to close + * it. */ + Uint64 bytes_sent; + Uint64 offset; + Uint64 length; + } sendfile; +#endif } tcp_descriptor; /* send function */ @@ -1245,6 +1276,8 @@ static int tcp_deliver(tcp_descriptor* desc, int len); static int tcp_shutdown_error(tcp_descriptor* desc, int err); +static int tcp_inet_sendfile(tcp_descriptor* desc); + static int tcp_inet_output(tcp_descriptor* desc, HANDLE event); static int tcp_inet_input(tcp_descriptor* desc, HANDLE event); @@ -1329,6 +1362,9 @@ static ErlDrvTermData am_ipv6_v6only; static ErlDrvTermData am_netns; static ErlDrvTermData am_bind_to_device; #endif +#ifdef HAVE_SENDFILE +static ErlDrvTermData am_sendfile; +#endif static char str_eafnosupport[] = "eafnosupport"; static char str_einval[] = "einval"; @@ -3875,6 +3911,10 @@ static int inet_init() INIT_ATOM(https); INIT_ATOM(scheme); +#ifdef HAVE_SENDFILE + INIT_ATOM(sendfile); +#endif + /* add TCP, UDP and SCTP drivers */ add_driver_entry(&tcp_inet_driver_entry); #ifdef HAVE_UDP @@ -9270,6 +9310,13 @@ static void tcp_inet_stop(ErlDrvData e) * will be freed through tcp_inet_stop later on. */ static void tcp_desc_close(tcp_descriptor* desc) { +#ifdef HAVE_SENDFILE + if(desc->tcp_add_flags & TCP_ADDF_SENDFILE) { + desc->tcp_add_flags &= ~TCP_ADDF_SENDFILE; + close(desc->sendfile.dup_file_fd); + } +#endif + tcp_clear_input(desc); tcp_clear_output(desc); @@ -9608,6 +9655,60 @@ static ErlDrvSSizeT tcp_inet_ctl(ErlDrvData e, unsigned int cmd, return ctl_reply(INET_REP_OK, NULL, 0, rbuf, rsize); } } + + case TCP_REQ_SENDFILE: { +#ifdef HAVE_SENDFILE + const ErlDrvSizeT required_len = + sizeof(desc->sendfile.dup_file_fd) + + sizeof(Uint64) * 2; + + int raw_file_fd; + + DEBUGF(("tcp_inet_ctl(%ld): SENDFILE\r\n", (long)desc->inet.port)); + + if (len != required_len) { + return ctl_error(EINVAL, rbuf, rsize); + } else if (!IS_CONNECTED(INETP(desc))) { + return ctl_error(ENOTCONN, rbuf, rsize); + } + + sys_memcpy(&raw_file_fd, buf, sizeof(raw_file_fd)); + buf += sizeof(raw_file_fd); + + desc->sendfile.dup_file_fd = dup(raw_file_fd); + + if(desc->sendfile.dup_file_fd == -1) { + return ctl_error(errno, rbuf, rsize); + } + + desc->sendfile.offset = get_int64(buf); + buf += sizeof(Uint64); + + desc->sendfile.length = get_int64(buf); + buf += sizeof(Uint64); + + ASSERT(desc->sendfile.offset >= 0); + ASSERT(desc->sendfile.length >= 0); + + desc->sendfile.ioq_skip = driver_sizeq(desc->inet.port); + desc->sendfile.bytes_sent = 0; + + desc->inet.caller = driver_caller(desc->inet.port); + desc->tcp_add_flags |= TCP_ADDF_SENDFILE; + + /* See if we can finish sending without selecting & rescheduling. */ + tcp_inet_sendfile(desc); + + if(desc->sendfile.length > 0) { + sock_select(INETP(desc), FD_WRITE, 1); + } + + return ctl_reply(INET_REP_OK, NULL, 0, rbuf, rsize); +#else + return ctl_error(ENOTSUP, rbuf, rsize); +#endif + } + default: DEBUGF(("tcp_inet_ctl(%ld): %u\r\n", (long)desc->inet.port, cmd)); return inet_ctl(INETP(desc), cmd, buf, len, rbuf, rsize); @@ -9747,12 +9848,27 @@ static void tcp_inet_commandv(ErlDrvData e, ErlIOVec* ev) static void tcp_inet_flush(ErlDrvData e) { tcp_descriptor* desc = (tcp_descriptor*)e; - if (!(desc->inet.event_mask & FD_WRITE)) { - /* Discard send queue to avoid hanging port (OTP-7615) */ - tcp_clear_output(desc); + int discard_output; + + /* Discard send queue to avoid hanging port (OTP-7615) */ + discard_output = !(desc->inet.event_mask & FD_WRITE); + + discard_output |= desc->tcp_add_flags & TCP_ADDF_LINGER_ZERO; + +#ifdef HAVE_SENDFILE + /* The old file driver aborted when it was stopped during sendfile, so + * we'll clear the flag and discard all output. */ + if(desc->tcp_add_flags & TCP_ADDF_SENDFILE) { + desc->tcp_add_flags &= ~TCP_ADDF_SENDFILE; + close(desc->sendfile.dup_file_fd); + + discard_output = 1; + } +#endif + + if (discard_output) { + tcp_clear_output(desc); } - if (desc->tcp_add_flags & TCP_ADDF_LINGER_ZERO) - tcp_clear_output(desc); } static void tcp_inet_process_exit(ErlDrvData e, ErlDrvMonitor *monitorp) @@ -10647,7 +10763,9 @@ static int tcp_sendv(tcp_descriptor* desc, ErlIOVec* ev) ev->size += h_len; } - if ((sz = driver_sizeq(ix)) > 0) { + sz = driver_sizeq(ix); + + if ((desc->tcp_add_flags & TCP_ADDF_SENDFILE) || sz > 0) { driver_enqv(ix, ev, 0); if (sz+ev->size >= desc->high) { DEBUGF(("tcp_sendv(%ld): s=%d, sender forced busy\r\n", @@ -10741,8 +10859,9 @@ static int tcp_send(tcp_descriptor* desc, char* ptr, ErlDrvSizeT len) inet_output_count(INETP(desc), len+h_len); + sz = driver_sizeq(ix); - if ((sz = driver_sizeq(ix)) > 0) { + if ((desc->tcp_add_flags & TCP_ADDF_SENDFILE) || sz > 0) { if (h_len > 0) driver_enq(ix, buf, h_len); driver_enq(ix, ptr, len); @@ -10832,6 +10951,246 @@ static void tcp_inet_drv_input(ErlDrvData data, ErlDrvEvent event) (void)tcp_inet_input((tcp_descriptor*)data, (HANDLE)event); } +#ifdef HAVE_SENDFILE +static int tcp_sendfile_completed(tcp_descriptor* desc) { + ErlDrvTermData spec[LOAD_PORT_CNT + LOAD_TUPLE_CNT * 2 + + LOAD_ATOM_CNT * 2 + LOAD_UINT_CNT * 2]; + Uint32 sent_low, sent_high; + int i; + + desc->tcp_add_flags &= ~TCP_ADDF_SENDFILE; + close(desc->sendfile.dup_file_fd); + + /* While we flushed the output queue prior to sending the file, we've + * deferred clearing busy status until now as there's no point in doing so + * while we still have a file to send. + * + * The watermark is checked since more data may have been added while we + * were sending the file. */ + + if (driver_sizeq(desc->inet.port) <= desc->low) { + if (IS_BUSY(INETP(desc))) { + desc->inet.caller = desc->inet.busy_caller; + desc->inet.state &= ~INET_F_BUSY; + + set_busy_port(desc->inet.port, 0); + + /* if we have a timer then cancel and send ok to client */ + if (desc->busy_on_send) { + driver_cancel_timer(desc->inet.port); + desc->busy_on_send = 0; + } + + inet_reply_ok(INETP(desc)); + } + } + + if (driver_sizeq(desc->inet.port) == 0) { + sock_select(INETP(desc), FD_WRITE, 0); + send_empty_out_q_msgs(INETP(desc)); + + if (desc->tcp_add_flags & TCP_ADDF_PENDING_SHUTDOWN) { + tcp_shutdown_async(desc); + } + } + + sent_low = ((Uint64)desc->sendfile.bytes_sent >> 0) & 0xFFFFFFFF; + sent_high = ((Uint64)desc->sendfile.bytes_sent >> 32) & 0xFFFFFFFF; + + i = LOAD_ATOM(spec, 0, am_sendfile); + i = LOAD_PORT(spec, i, desc->inet.dport); + i = LOAD_ATOM(spec, i, am_ok); + i = LOAD_UINT(spec, i, sent_low); + i = LOAD_UINT(spec, i, sent_high); + i = LOAD_TUPLE(spec, i, 3); + i = LOAD_TUPLE(spec, i, 3); + + ASSERT(i == sizeof(spec)/sizeof(*spec)); + + return erl_drv_output_term(desc->inet.dport, spec, i); +} + +static int tcp_sendfile_aborted(tcp_descriptor* desc, int socket_error) { + ErlDrvTermData spec[LOAD_PORT_CNT + LOAD_TUPLE_CNT * 2 + LOAD_ATOM_CNT * 3]; + int i; + + /* We don't clean up sendfile state here, as that's done in tcp_desc_close + * following normal error handling. All we do here is report the failure. */ + + i = LOAD_ATOM(spec, 0, am_sendfile); + i = LOAD_PORT(spec, i, desc->inet.dport); + i = LOAD_ATOM(spec, i, am_error); + + switch (socket_error) { + case ECONNRESET: + case ENOTCONN: + case EPIPE: + i = LOAD_ATOM(spec, i, am_closed); + break; + default: + i = LOAD_ATOM(spec, i, error_atom(socket_error)); + } + + i = LOAD_TUPLE(spec, i, 2); + i = LOAD_TUPLE(spec, i, 3); + + ASSERT(i == sizeof(spec)/sizeof(*spec)); + + return erl_drv_output_term(desc->inet.dport, spec, i); +} + +static int tcp_inet_sendfile(tcp_descriptor* desc) { + ErlDrvPort ix = desc->inet.port; + int result = 0; + ssize_t n; + + DEBUGF(("tcp_inet_sendfile(%ld) {s=%d\r\n", (long)ix, desc->inet.s)); + + /* If there was any data in the queue by the time sendfile was issued, + * we'll need to skip it first. Note that we don't clear busy status until + * we're finished sending the file. */ + while (desc->sendfile.ioq_skip > 0) { + ssize_t bytes_to_send; + SysIOVec* iov; + int vsize; + + ASSERT(driver_sizeq(ix) >= desc->sendfile.ioq_skip); + + if ((iov = driver_peekq(ix, &vsize)) == NULL) { + ERTS_INTERNAL_ERROR("ioq empty when sendfile.ioq_skip > 0"); + } + + bytes_to_send = MIN(desc->sendfile.ioq_skip, iov[0].iov_len); + n = sock_send(desc->inet.s, iov[0].iov_base, bytes_to_send, 0); + + if (!IS_SOCKET_ERROR(n)) { + desc->sendfile.ioq_skip -= n; + driver_deq(ix, n); + } else if (sock_errno() == ERRNO_BLOCK) { +#ifdef __WIN32__ + desc->inet.send_would_block = 1; +#endif + goto done; + } else if (sock_errno() != EINTR) { + goto socket_error; + } + } + + while (desc->sendfile.length > 0) { + /* For some reason the maximum ssize_t cannot be used as the max size. + * 1GB seems to work on all platforms */ + const Sint64 SENDFILE_CHUNK_SIZE = ((1UL << 30) - 1); + + ssize_t bytes_to_send = MIN(SENDFILE_CHUNK_SIZE, desc->sendfile.length); + off_t offset = desc->sendfile.offset; + +#if defined(__linux__) + n = sendfile(desc->inet.s, desc->sendfile.dup_file_fd, &offset, + bytes_to_send); +#elif defined(__FreeBSD__) || defined(__DragonFly__) || defined(__DARWIN__) + { + off_t bytes_sent; + int error; + + #if defined(__DARWIN__) + bytes_sent = bytes_to_send; + + error = sendfile(desc->sendfile.dup_file_fd, desc->inet.s, offset, + &bytes_sent, NULL, 0); + n = bytes_sent; + #else + error = sendfile(desc->sendfile.dup_file_fd, desc->inet.s, offset, + bytes_to_send, NULL, &bytes_sent, 0); + n = bytes_sent; + #endif + + if(error < 0) { + /* EAGAIN/EINTR report partial success by setting bytes_sent, + * so we have to skip error handling if nonzero, and skip EOF + * handling if zero, as it's possible that we didn't manage to + * send anything at all before being interrupted by a + * signal. */ + if((errno != EAGAIN && errno != EINTR) || bytes_sent == 0) { + n = -1; + } + } + } +#elif defined(__sun) && defined(__SVR4) && defined(HAVE_SENDFILEV) + { + sendfilevec_t sfvec[1]; + size_t bytes_sent; + ssize_t error; + + sfvec[0].sfv_fd = desc->sendfile.dup_file_fd; + sfvec[0].sfv_len = bytes_to_send; + sfvec[0].sfv_off = offset; + sfvec[0].sfv_flag = 0; + + error = sendfilev(desc->inet.s, sfvec, 1, &bytes_sent); + n = bytes_sent; + + if(error < 0) { + if(errno == EINVAL) { + /* On some solaris versions (I've seen it on SunOS 5.10), + * using a sfv_len larger than the filesize will result in + * a (-1 && errno == EINVAL). We translate this to a + * successful send of the data.*/ + } else { + /* EAGAIN/EINTR behavior is identical to *BSD. */ + if((errno != EAGAIN && errno != EINTR) || bytes_sent == 0) { + n = -1; + } + } + } + } +#else + #error "Unsupported sendfile syscall; update configure test." +#endif + + if (n > 0) { + desc->sendfile.bytes_sent += n; + desc->sendfile.offset += n; + desc->sendfile.length -= n; + } else if (n == 0) { + /* EOF. */ + desc->sendfile.length = 0; + break; + } else if (IS_SOCKET_ERROR(n) && sock_errno() != EINTR) { + if (sock_errno() != ERRNO_BLOCK) { + goto socket_error; + } + +#ifdef __WIN32__ + desc->inet.send_would_block = 1; +#endif + break; + } + } + + if (desc->sendfile.length == 0) { + tcp_sendfile_completed(desc); + } + + goto done; + +socket_error: { + int socket_errno = sock_errno(); + + DEBUGF(("tcp_inet_sendfile(%ld): send errno = %d (errno %d)\r\n", + (long)desc->inet.port, socket_errno, errno)); + + result = tcp_send_error(desc, socket_errno); + tcp_sendfile_aborted(desc, socket_errno); + + goto done; + } + +done: + DEBUGF(("tcp_inet_sendfile(%ld) }\r\n", (long)desc->inet.port)); + return result; +} +#endif /* HAVE_SENDFILE */ + /* socket ready for ouput: ** 1. INET_STATE_CONNECTING => non block connect ? ** 2. INET_STATE_CONNECTED => write output @@ -10892,7 +11251,14 @@ static int tcp_inet_output(tcp_descriptor* desc, HANDLE event) async_ok(INETP(desc)); } else if (IS_CONNECTED(INETP(desc))) { - for (;;) { + +#ifdef HAVE_SENDFILE + if(desc->tcp_add_flags & TCP_ADDF_SENDFILE) { + return tcp_inet_sendfile(desc); + } +#endif + + for (;;) { int vsize; ssize_t n; SysIOVec* iov; diff --git a/erts/emulator/drivers/unix/unix_efile.c b/erts/emulator/drivers/unix/unix_efile.c deleted file mode 100644 index f8341f788a..0000000000 --- a/erts/emulator/drivers/unix/unix_efile.c +++ /dev/null @@ -1,1102 +0,0 @@ -/* - * %CopyrightBegin% - * - * Copyright Ericsson AB 1997-2017. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - * %CopyrightEnd% - */ -/* - * Purpose: Provides file and directory operations for Unix. - */ -#ifdef HAVE_CONFIG_H -# include "config.h" -#endif -#if defined(HAVE_POSIX_FALLOCATE) && !defined(__sun) && !defined(__sun__) -#define _XOPEN_SOURCE 600 -#endif -#if !defined(_GNU_SOURCE) && defined(HAVE_LINUX_FALLOC_H) -#define _GNU_SOURCE -#endif -#include "sys.h" -#include "erl_driver.h" -#include "erl_efile.h" -#include <utime.h> -#ifdef HAVE_UNISTD_H -#include <unistd.h> -#endif -#ifdef HAVE_SYS_UIO_H -#include <sys/types.h> -#include <sys/uio.h> -#if defined(HAVE_SENDFILE) && (defined(__FreeBSD__) || defined(__DragonFly__)) -/* Need to define __BSD_VISIBLE in order to expose prototype of sendfile */ -#define __BSD_VISIBLE 1 -#include <sys/socket.h> -#endif -#endif -#if defined(HAVE_SENDFILE) && (defined(__linux__) || (defined(__sun) && defined(__SVR4))) -#include <sys/sendfile.h> -#endif - -#if defined(__APPLE__) && defined(__MACH__) && !defined(__DARWIN__) -#define __DARWIN__ 1 -#endif - -#if defined(__DARWIN__) || defined(HAVE_LINUX_FALLOC_H) || defined(HAVE_POSIX_FALLOCATE) -#include <fcntl.h> -#endif - -#ifdef HAVE_LINUX_FALLOC_H -#include <linux/falloc.h> -#endif - -#ifdef SUNOS4 -# define getcwd(buf, size) getwd(buf) -#endif - -/* Find a definition of MAXIOV, that is used in the code later. */ -#if defined IOV_MAX -#define MAXIOV IOV_MAX -#elif defined UIO_MAXIOV -#define MAXIOV UIO_MAXIOV -#else -#define MAXIOV 16 -#endif - - -/* - * Macros for testing file types. - */ - -#define ISDIR(st) (S_ISDIR((st).st_mode)) -#define ISREG(st) (S_ISREG((st).st_mode)) -#define ISDEV(st) (S_ISCHR((st).st_mode) || S_ISBLK((st).st_mode)) -#define ISLNK(st) (S_ISLNK((st).st_mode)) -#ifdef NO_UMASK -#define FILE_MODE 0644 -#define DIR_MODE 0755 -#else -#define FILE_MODE 0666 -#define DIR_MODE 0777 -#endif - -#define IS_DOT_OR_DOTDOT(s) \ - (s[0] == '.' && (s[1] == '\0' || (s[1] == '.' && s[2] == '\0'))) - -static int check_error(int result, Efile_error* errInfo); - -static int -check_error(int result, Efile_error *errInfo) -{ - if (result < 0) { - errInfo->posix_errno = errInfo->os_errno = errno; - return 0; - } - return 1; -} - -int -efile_init() { - return 1; -} - -int -efile_mkdir(Efile_error* errInfo, /* Where to return error codes. */ - char* name) /* Name of directory to create. */ -{ -#ifdef NO_MKDIR_MODE - return check_error(mkdir(name), errInfo); -#else - return check_error(mkdir(name, DIR_MODE), errInfo); -#endif -} - -int -efile_rmdir(Efile_error* errInfo, /* Where to return error codes. */ - char* name) /* Name of directory to delete. */ -{ - if (rmdir(name) == 0) { - return 1; - } - if (errno == ENOTEMPTY) { - errno = EEXIST; - } - if (errno == EEXIST) { - int saved_errno = errno; - struct stat file_stat; - struct stat cwd_stat; - - /* - * The error code might be wrong if this is the current directory. - */ - - if (stat(name, &file_stat) == 0 && stat(".", &cwd_stat) == 0 && - file_stat.st_ino == cwd_stat.st_ino && - file_stat.st_dev == cwd_stat.st_dev) { - saved_errno = EINVAL; - } - errno = saved_errno; - } - return check_error(-1, errInfo); -} - -int -efile_delete_file(Efile_error* errInfo, /* Where to return error codes. */ - char* name) /* Name of file to delete. */ -{ - if (unlink(name) == 0) { - return 1; - } - if (errno == EISDIR) { /* Linux sets the wrong error code. */ - errno = EPERM; - } - return check_error(-1, errInfo); -} - -/* - *--------------------------------------------------------------------------- - * - * Changes the name of an existing file or directory, from src to dst. - * If src and dst refer to the same file or directory, does nothing - * and returns success. Otherwise if dst already exists, it will be - * deleted and replaced by src subject to the following conditions: - * If src is a directory, dst may be an empty directory. - * If src is a file, dst may be a file. - * In any other situation where dst already exists, the rename will - * fail. - * - * Results: - * If the directory was successfully created, returns 1. - * Otherwise the return value is 0 and errno is set to - * indicate the error. Some possible values for errno are: - * - * EACCES: src or dst parent directory can't be read and/or written. - * EEXIST: dst is a non-empty directory. - * EINVAL: src is a root directory or dst is a subdirectory of src. - * EISDIR: dst is a directory, but src is not. - * ENOENT: src doesn't exist, or src or dst is "". - * ENOTDIR: src is a directory, but dst is not. - * EXDEV: src and dst are on different filesystems. - * - * Side effects: - * The implementation of rename may allow cross-filesystem renames, - * but the caller should be prepared to emulate it with copy and - * delete if errno is EXDEV. - * - *--------------------------------------------------------------------------- - */ - -int -efile_rename(Efile_error* errInfo, /* Where to return error codes. */ - char* src, /* Original name. */ - char* dst) /* New name. */ -{ - if (rename(src, dst) == 0) { - return 1; - } - if (errno == ENOTEMPTY) { - errno = EEXIST; - } -#if defined (sparc) - /* - * SunOS 4.1.4 reports overwriting a non-empty directory with a - * directory as EINVAL instead of EEXIST (first rule out the correct - * EINVAL result code for moving a directory into itself). Must be - * conditionally compiled because realpath() is only defined on SunOS. - */ - - if (errno == EINVAL) { - char srcPath[MAXPATHLEN], dstPath[MAXPATHLEN]; - DIR *dirPtr; - struct dirent *dirEntPtr; - -#ifdef PURIFY - memset(srcPath, '\0', sizeof(srcPath)); - memset(dstPath, '\0', sizeof(dstPath)); -#endif - - if ((realpath(src, srcPath) != NULL) - && (realpath(dst, dstPath) != NULL) - && (strncmp(srcPath, dstPath, strlen(srcPath)) != 0)) { - dirPtr = opendir(dst); - if (dirPtr != NULL) { - while ((dirEntPtr = readdir(dirPtr)) != NULL) { - if ((strcmp(dirEntPtr->d_name, ".") != 0) && - (strcmp(dirEntPtr->d_name, "..") != 0)) { - errno = EEXIST; - closedir(dirPtr); - return check_error(-1, errInfo); - } - } - closedir(dirPtr); - } - } - errno = EINVAL; - } -#endif /* sparc */ - - if (strcmp(src, "/") == 0) { - /* - * Alpha reports renaming / as EBUSY and Linux reports it as EACCES, - * instead of EINVAL. - */ - - errno = EINVAL; - } - - /* - * DEC Alpha OSF1 V3.0 returns EACCES when attempting to move a - * file across filesystems and the parent directory of that file is - * not writable. Most other systems return EXDEV. Does nothing to - * correct this behavior. - */ - - return check_error(-1, errInfo); -} - -int -efile_chdir(Efile_error* errInfo, /* Where to return error codes. */ - char* name) /* Name of directory to make current. */ -{ - return check_error(chdir(name), errInfo); -} - - -int -efile_getdcwd(Efile_error* errInfo, /* Where to return error codes. */ - int drive, /* 0 - current, 1 - A, 2 - B etc. */ - char* buffer, /* Where to return the current - directory. */ - size_t size) /* Size of buffer. */ -{ - if (drive == 0) { - if (getcwd(buffer, size) == NULL) - return check_error(-1, errInfo); - -#ifdef SIMSPARCSOLARIS - /* We get "host:" prepended to the dirname - remove!. */ - { - int i = 0; - int j = 0; - while ((buffer[i] != ':') && (buffer[i] != '\0')) i++; - if (buffer[i] == ':') { - i++; - while ((buffer[j++] = buffer[i++]) != '\0'); - } - } -#endif - return 1; - } - - /* - * Drives other than 0 is not supported on Unix. - */ - - errno = ENOTSUP; - return check_error(-1, errInfo); -} - -int -efile_readdir(Efile_error* errInfo, /* Where to return error codes. */ - char* name, /* Name of directory to open. */ - EFILE_DIR_HANDLE* p_dir_handle, /* Pointer to directory - handle of - open directory.*/ - char* buffer, /* Pointer to buffer for - one filename. */ - size_t *size) /* in-out Size of buffer, length - of name. */ -{ - DIR *dp; /* Pointer to directory structure. */ - struct dirent* dirp; /* Pointer to directory entry. */ - - /* - * If this is the first call, we must open the directory. - */ - - if (*p_dir_handle == NULL) { - dp = opendir(name); - if (dp == NULL) - return check_error(-1, errInfo); - *p_dir_handle = (EFILE_DIR_HANDLE) dp; - } - - /* - * Retrieve the name of the next file using the directory handle. - */ - - dp = *((DIR **)((void *)p_dir_handle)); - for (;;) { - dirp = readdir(dp); - if (dirp == NULL) { - closedir(dp); - return 0; - } - if (IS_DOT_OR_DOTDOT(dirp->d_name)) - continue; - buffer[0] = '\0'; - strncat(buffer, dirp->d_name, (*size)-1); - *size = strlen(dirp->d_name); - return 1; - } -} - -int -efile_openfile(Efile_error* errInfo, /* Where to return error codes. */ - char* name, /* Name of directory to open. */ - int flags, /* Flags to user for opening. */ - int* pfd, /* Where to store the file - descriptor. */ - Sint64 *pSize) /* Where to store the size of the - file. */ -{ - struct stat statbuf; - int fd; - int mode; /* Open mode. */ - - switch (flags & (EFILE_MODE_READ|EFILE_MODE_WRITE)) { - case EFILE_MODE_READ: - mode = O_RDONLY; - break; - case EFILE_MODE_WRITE: - if (flags & EFILE_NO_TRUNCATE) - mode = O_WRONLY | O_CREAT; - else - mode = O_WRONLY | O_CREAT | O_TRUNC; - break; - case EFILE_MODE_READ_WRITE: - mode = O_RDWR | O_CREAT; - break; - default: - errno = EINVAL; - return check_error(-1, errInfo); - } - - if (flags & EFILE_MODE_APPEND) { - mode &= ~O_TRUNC; - mode |= O_APPEND; - } - if (flags & EFILE_MODE_EXCL) { - mode |= O_EXCL; - } - if (flags & EFILE_MODE_SYNC) { -#ifdef O_SYNC - mode |= O_SYNC; -#else - errno = ENOTSUP; - return check_error(-1, errInfo); -#endif - } - -#ifdef HAVE_FSTAT - while (((fd = open(name, mode, FILE_MODE)) < 0) && (errno == EINTR)); - if (!check_error(fd, errInfo)) return 0; -#endif - - if ( -#ifdef HAVE_FSTAT - fstat(fd, &statbuf) < 0 -#else - stat(name, &statbuf) < 0 -#endif - ) { - /* statbuf is undefined: if the caller depends on it, - i.e. invoke_read_file(), fail the call immediately */ - if (pSize && flags == EFILE_MODE_READ) { - check_error(-1, errInfo); -#ifdef HAVE_FSTAT - efile_closefile(fd); -#endif - return 0; - } - } - else if (! ISREG(statbuf)) { - struct stat nullstatbuf; - /* - * For UNIX only, here is some ugly code to allow - * /dev/null to be opened as a file. - */ - if ( (stat("/dev/null", &nullstatbuf) < 0) - || (statbuf.st_ino != nullstatbuf.st_ino) - || (statbuf.st_dev != nullstatbuf.st_dev) ) { -#ifdef HAVE_FSTAT - efile_closefile(fd); -#endif - errno = EISDIR; - return check_error(-1, errInfo); - } - } - -#ifndef HAVE_FSTAT - while (((fd = open(name, mode, FILE_MODE)) < 0) && (errno == EINTR)); - if (!check_error(fd, errInfo)) return 0; -#endif - - *pfd = fd; - if (pSize) *pSize = statbuf.st_size; - return 1; -} - -int -efile_may_openfile(Efile_error* errInfo, char *name) { - struct stat statbuf; /* Information about the file */ - int result; - - result = stat(name, &statbuf); - if (!check_error(result, errInfo)) - return 0; - if (!ISREG(statbuf)) { - errno = EISDIR; - return check_error(-1, errInfo); - } - return 1; -} - -void -efile_closefile(int fd) -{ - while((close(fd) < 0) && (errno == EINTR)); -} - -int -efile_fdatasync(Efile_error *errInfo, /* Where to return error codes. */ - int fd) /* File descriptor for file to sync data. */ -{ -#if defined(HAVE_FDATASYNC) && !defined(__DARWIN__) - return check_error(fdatasync(fd), errInfo); -#else - return efile_fsync(errInfo, fd); -#endif -} - -int -efile_fsync(Efile_error *errInfo, /* Where to return error codes. */ - int fd) /* File descriptor for file to sync. */ -{ -#ifdef NO_FSYNC - undefined fsync /* XXX: Really? */ -#else -#if defined(__DARWIN__) && defined(F_FULLFSYNC) - return check_error(fcntl(fd, F_FULLFSYNC), errInfo); -#else - return check_error(fsync(fd), errInfo); -#endif /* __DARWIN__ */ -#endif /* NO_FSYNC */ -} - -int -efile_fileinfo(Efile_error* errInfo, Efile_info* pInfo, - char* name, int info_for_link) -{ - struct stat statbuf; /* Information about the file */ - int result; - - if (info_for_link) { - result = lstat(name, &statbuf); - } else { - result = stat(name, &statbuf); - } - if (!check_error(result, errInfo)) { - return 0; - } - -#if SIZEOF_OFF_T == 4 - pInfo->size_high = 0; -#else - pInfo->size_high = (Uint32)(statbuf.st_size >> 32); -#endif - pInfo->size_low = (Uint32)statbuf.st_size; - -#ifdef NO_ACCESS - /* Just look at read/write access for owner. */ - - pInfo->access = ((statbuf.st_mode >> 6) & 07) >> 1; - -#else - pInfo->access = FA_NONE; - if (access(name, R_OK) == 0) - pInfo->access |= FA_READ; - if (access(name, W_OK) == 0) - pInfo->access |= FA_WRITE; - -#endif - - if (ISDEV(statbuf)) - pInfo->type = FT_DEVICE; - else if (ISDIR(statbuf)) - pInfo->type = FT_DIRECTORY; - else if (ISREG(statbuf)) - pInfo->type = FT_REGULAR; - else if (ISLNK(statbuf)) - pInfo->type = FT_SYMLINK; - else - pInfo->type = FT_OTHER; - - pInfo->accessTime = (Sint64)statbuf.st_atime; - pInfo->modifyTime = (Sint64)statbuf.st_mtime; - pInfo->cTime = (Sint64)statbuf.st_ctime; - - pInfo->mode = statbuf.st_mode; - pInfo->links = statbuf.st_nlink; - pInfo->major_device = statbuf.st_dev; - pInfo->minor_device = statbuf.st_rdev; - pInfo->inode = statbuf.st_ino; - pInfo->uid = statbuf.st_uid; - pInfo->gid = statbuf.st_gid; - - return 1; -} - -int -efile_write_info(Efile_error *errInfo, Efile_info *pInfo, char *name) -{ - struct utimbuf tval; - - /* - * On some systems chown will always fail for a non-root user unless - * POSIX_CHOWN_RESTRICTED is not set. Others will succeed as long as - * you don't try to chown a file to someone besides youself. - */ - - if (chown(name, pInfo->uid, pInfo->gid) && errno != EPERM) { - return check_error(-1, errInfo); - } - - if (pInfo->mode != -1) { - mode_t newMode = pInfo->mode & (S_ISUID | S_ISGID | - S_IRWXU | S_IRWXG | S_IRWXO); - if (chmod(name, newMode)) { - newMode &= ~(S_ISUID | S_ISGID); - if (chmod(name, newMode)) { - return check_error(-1, errInfo); - } - } - } - - tval.actime = (time_t)pInfo->accessTime; - tval.modtime = (time_t)pInfo->modifyTime; - - return check_error(utime(name, &tval), errInfo); -} - - -int -efile_write(Efile_error* errInfo, /* Where to return error codes. */ - int flags, /* Flags given when file was - opened. */ - int fd, /* File descriptor to write to. */ - char* buf, /* Buffer to write. */ - size_t count) /* Number of bytes to write. */ -{ - ssize_t written; /* Bytes written in last operation. */ - - while (count > 0) { - if ((written = write(fd, buf, count)) < 0) { - if (errno != EINTR) - return check_error(-1, errInfo); - else - written = 0; - } - ASSERT(written <= count); - buf += written; - count -= written; - } - return 1; -} - -int -efile_writev(Efile_error* errInfo, /* Where to return error codes */ - int flags, /* Flags given when file was - * opened */ - int fd, /* File descriptor to write to */ - SysIOVec* iov, /* Vector of buffer structs. - * The structs may be changed i.e. - * due to incomplete writes */ - int iovcnt) /* Number of structs in vector */ -{ - int cnt = 0; /* Buffers so far written */ - - ASSERT(iovcnt >= 0); - - while (cnt < iovcnt) { - if ((! iov[cnt].iov_base) || (iov[cnt].iov_len <= 0)) { - /* Empty buffer - skip */ - cnt++; - } else { /* Non-empty buffer */ - ssize_t w; /* Bytes written in this call */ -#ifdef HAVE_WRITEV - int b = iovcnt - cnt; /* Buffers to write */ - /* Use as many buffers as MAXIOV allows */ - if (b > MAXIOV) - b = MAXIOV; - if (b > 1) { - do { - w = writev(fd, &iov[cnt], b); - } while (w < 0 && errno == EINTR); - if (w < 0 && errno == EINVAL) { - goto single_write; - } - } else - single_write: - /* Degenerated io vector - use regular write */ -#endif - { - do { - size_t iov_len = iov[cnt].iov_len; - size_t limit = 1024*1024*1024; /* 1GB */ - if (iov_len > limit) { - iov_len = limit; - } - w = write(fd, iov[cnt].iov_base, iov_len); - } while (w < 0 && errno == EINTR); - ASSERT(w <= iov[cnt].iov_len || - (w == -1 && errno != EINTR)); - } - if (w < 0) return check_error(-1, errInfo); - /* Move forward to next buffer to write */ - for (; cnt < iovcnt && w > 0; cnt++) { - if (iov[cnt].iov_base && iov[cnt].iov_len > 0) { - if (w < iov[cnt].iov_len) { - /* Adjust the buffer for next write */ - iov[cnt].iov_len -= w; - iov[cnt].iov_base = ((char *)iov[cnt].iov_base) + w; - w = 0; - break; - } else { - w -= iov[cnt].iov_len; - } - } - } - ASSERT(w == 0); - } /* else Non-empty buffer */ - } /* while (cnt< iovcnt) */ - return 1; -} - -int -efile_read(Efile_error* errInfo, /* Where to return error codes. */ - int flags, /* Flags given when file was opened. */ - int fd, /* File descriptor to read from. */ - char* buf, /* Buffer to read into. */ - size_t count, /* Number of bytes to read. */ - size_t *pBytesRead) /* Where to return number of - bytes read. */ -{ - ssize_t n; - - for (;;) { - if ((n = read(fd, buf, count)) >= 0) - break; - else if (errno != EINTR) - return check_error(-1, errInfo); - } - *pBytesRead = (size_t) n; - return 1; -} - - -/* pread() and pwrite() */ -/* Some unix systems, notably Solaris has these syscalls */ -/* It is especially nice for i.e. the dets module to have support */ -/* for this, even if the underlying OS dosn't support it, it is */ -/* reasonably easy to work around by first calling seek, and then */ -/* calling read(). */ -/* This later strategy however changes the file pointer, which pread() */ -/* does not do. We choose to ignore this and say that the location */ -/* of the file pointer is undefined after a call to any of the p functions*/ - - -int -efile_pread(Efile_error* errInfo, /* Where to return error codes. */ - int fd, /* File descriptor to read from. */ - Sint64 offset, /* Offset in bytes from BOF. */ - char* buf, /* Buffer to read into. */ - size_t count, /* Number of bytes to read. */ - size_t *pBytesRead) /* Where to return - number of bytes read. */ -{ -#if defined(HAVE_PREAD) && defined(HAVE_PWRITE) - ssize_t n; - off_t off = (off_t) offset; - if (off != offset) { - errno = EINVAL; - return check_error(-1, errInfo); - } - for (;;) { - if ((n = pread(fd, buf, count, offset)) >= 0) - break; - else if (errno != EINTR) - return check_error(-1, errInfo); - } - *pBytesRead = (size_t) n; - return 1; -#else - { - int res = efile_seek(errInfo, fd, offset, EFILE_SEEK_SET, NULL); - if (res) { - return efile_read(errInfo, 0, fd, buf, count, pBytesRead); - } else { - return res; - } - } -#endif -} - - - -int -efile_pwrite(Efile_error* errInfo, /* Where to return error codes. */ - int fd, /* File descriptor to write to. */ - char* buf, /* Buffer to write. */ - size_t count, /* Number of bytes to write. */ - Sint64 offset) /* where to write it */ -{ -#if defined(HAVE_PREAD) && defined(HAVE_PWRITE) - ssize_t written; /* Bytes written in last operation. */ - off_t off = (off_t) offset; - if (off != offset) { - errno = EINVAL; - return check_error(-1, errInfo); - } - - while (count > 0) { - if ((written = pwrite(fd, buf, count, offset)) < 0) { - if (errno != EINTR) - return check_error(-1, errInfo); - else - written = 0; - } - ASSERT(written <= count); - buf += written; - count -= written; - offset += written; - } - return 1; -#else /* For unix systems that don't support pread() and pwrite() */ - { - int res = efile_seek(errInfo, fd, offset, EFILE_SEEK_SET, NULL); - - if (res) { - return efile_write(errInfo, 0, fd, buf, count); - } else { - return res; - } - } -#endif -} - - -int -efile_seek(Efile_error* errInfo, /* Where to return error codes. */ - int fd, /* File descriptor to do the seek on. */ - Sint64 offset, /* Offset in bytes from the given - origin. */ - int origin, /* Origin of seek (SEEK_SET, SEEK_CUR, - SEEK_END). */ - Sint64 *new_location) /* Resulting new location in file. */ -{ - off_t off, result; - - switch (origin) { - case EFILE_SEEK_SET: origin = SEEK_SET; break; - case EFILE_SEEK_CUR: origin = SEEK_CUR; break; - case EFILE_SEEK_END: origin = SEEK_END; break; - default: - errno = EINVAL; - return check_error(-1, errInfo); - } - off = (off_t) offset; - if (off != offset) { - errno = EINVAL; - return check_error(-1, errInfo); - } - - errno = 0; - result = lseek(fd, off, origin); - - /* - * Note that the man page for lseek (on SunOs 5) says: - * - * "if fildes is a remote file descriptor and offset is - * negative, lseek() returns the file pointer even if it is - * negative." - */ - - if (result < 0 && errno == 0) - errno = EINVAL; - if (result < 0) - return check_error(-1, errInfo); - if (new_location) { - *new_location = result; - } - return 1; -} - - -int -efile_truncate_file(Efile_error* errInfo, int *fd, int flags) -{ -#ifndef NO_FTRUNCATE - off_t offset; - - return check_error((offset = lseek(*fd, 0, 1)) >= 0 && - ftruncate(*fd, offset) == 0 ? 1 : -1, - errInfo); -#else - return 1; -#endif -} - -int -efile_readlink(Efile_error* errInfo, char* name, char* buffer, size_t size) -{ - int len; - ASSERT(size > 0); - len = readlink(name, buffer, size-1); - if (len == -1) { - return check_error(-1, errInfo); - } - buffer[len] = '\0'; - return 1; -} - -int -efile_altname(Efile_error* errInfo, char* name, char* buffer, size_t size) -{ - errno = ENOTSUP; - return check_error(-1, errInfo); -} - -int -efile_link(Efile_error* errInfo, char* old, char* new) -{ - return check_error(link(old, new), errInfo); -} - -int -efile_symlink(Efile_error* errInfo, char* old, char* new) -{ - return check_error(symlink(old, new), errInfo); -} - -int -efile_fadvise(Efile_error* errInfo, int fd, Sint64 offset, - Sint64 length, int advise) -{ -#ifdef HAVE_POSIX_FADVISE - return check_error(posix_fadvise(fd, offset, length, advise), errInfo); -#else - return check_error(0, errInfo); -#endif -} - -#ifdef HAVE_SENDFILE -/* For some reason the maximum size_t cannot be used as the max size - 3GB seems to work on all platforms */ -#define SENDFILE_CHUNK_SIZE ((1UL << 30) -1) - -/* - * sendfile: The implementation of the sendfile system call varies - * a lot on different *nix platforms so to make the api similar in all - * we have to emulate some things in linux and play with variables on - * bsd/darwin. - * - * All of the calls will split a command which tries to send more than - * SENDFILE_CHUNK_SIZE of data at once. - * - * On platforms where *nbytes of 0 does not mean the entire file, this is - * simulated. - * - * It could be possible to implement header/trailer in sendfile. Though - * you would have to emulate it in linux and on BSD/Darwin some complex - * calculations have to be made when using a non blocking socket to figure - * out how much of the header/file/trailer was sent in each command. - * - * The semantics of the API is this: - * Return value: 1 if all data was sent and the function does not need to - * be called again. 0 if an error occures OR if there is more data which - * has to be sent (EAGAIN or EINTR will be set appropriately) - * - * The amount of data written in a call is returned through nbytes. - * - */ - -int -efile_sendfile(Efile_error* errInfo, int in_fd, int out_fd, - off_t *offset, Uint64 *nbytes, struct t_sendfile_hdtl* hdtl) -{ - Uint64 written = 0; -#if defined(__linux__) - ssize_t retval; - do { - /* check if *nbytes is 0 or greater than chunk size */ - if (*nbytes == 0 || *nbytes > SENDFILE_CHUNK_SIZE) - retval = sendfile(out_fd, in_fd, offset, SENDFILE_CHUNK_SIZE); - else - retval = sendfile(out_fd, in_fd, offset, *nbytes); - if (retval > 0) { - written += retval; - *nbytes -= retval; - } - } while (retval == SENDFILE_CHUNK_SIZE); - if (written != 0) { - /* -1 is not returned by the linux API so we have to simulate it */ - retval = -1; - errno = EAGAIN; - } -#elif defined(__sun) && defined(__SVR4) && defined(HAVE_SENDFILEV) - ssize_t retval; - size_t len; - sendfilevec_t fdrec; - fdrec.sfv_fd = in_fd; - fdrec.sfv_flag = 0; - do { - fdrec.sfv_off = *offset; - len = 0; - /* check if *nbytes is 0 or greater than chunk size */ - if (*nbytes == 0 || *nbytes > SENDFILE_CHUNK_SIZE) - fdrec.sfv_len = SENDFILE_CHUNK_SIZE; - else - fdrec.sfv_len = *nbytes; - - retval = sendfilev(out_fd, &fdrec, 1, &len); - - if (retval == -1 && errno == EINVAL) { - /* On some solaris versions (I've seen it on SunOS 5.10), - using a sfv_len larger then a filesize will result in - a -1 && errno == EINVAL return. We translate this so - a successful send of the data.*/ - retval = len; - } - - if (retval != -1 || errno == EAGAIN || errno == EINTR) { - *offset += len; - *nbytes -= len; - written += len; - } - } while (len == SENDFILE_CHUNK_SIZE); -#elif defined(__DARWIN__) - int retval; - off_t len; - do { - /* check if *nbytes is 0 or greater than chunk size */ - if(*nbytes > SENDFILE_CHUNK_SIZE) - len = SENDFILE_CHUNK_SIZE; - else - len = *nbytes; - retval = sendfile(in_fd, out_fd, *offset, &len, NULL, 0); - if (retval != -1 || errno == EAGAIN || errno == EINTR) { - *offset += len; - *nbytes -= len; - written += len; - } - } while (len == SENDFILE_CHUNK_SIZE); -#elif defined(__FreeBSD__) || defined(__DragonFly__) - off_t len; - int retval; - do { - if (*nbytes > SENDFILE_CHUNK_SIZE) - retval = sendfile(in_fd, out_fd, *offset, SENDFILE_CHUNK_SIZE, - NULL, &len, 0); - else - retval = sendfile(in_fd, out_fd, *offset, *nbytes, NULL, &len, 0); - if (retval != -1 || errno == EAGAIN || errno == EINTR) { - *offset += len; - *nbytes -= len; - written += len; - } - } while(len == SENDFILE_CHUNK_SIZE); -#endif - *nbytes = written; - return check_error(retval, errInfo); -} -#endif /* HAVE_SENDFILE */ - -#ifdef HAVE_POSIX_FALLOCATE -static int -call_posix_fallocate(int fd, Sint64 offset, Sint64 length) -{ - int ret; - - /* - * On Linux and Solaris for example, posix_fallocate() returns - * a positive error number on error and it does not set errno. - * On FreeBSD however (9.0 at least), it returns -1 on error - * and it sets errno. - */ - do { - ret = posix_fallocate(fd, (off_t) offset, (off_t) length); - if (ret > 0) { - errno = ret; - ret = -1; - } - } while (ret != 0 && errno == EINTR); - - return ret; -} -#endif /* HAVE_POSIX_FALLOCATE */ - -int -efile_fallocate(Efile_error* errInfo, int fd, Sint64 offset, Sint64 length) -{ -#if defined HAVE_FALLOCATE - /* Linux specific, more efficient than posix_fallocate. */ - int ret; - - do { - ret = fallocate(fd, FALLOC_FL_KEEP_SIZE, (off_t) offset, (off_t) length); - } while (ret != 0 && errno == EINTR); - -#if defined HAVE_POSIX_FALLOCATE - /* Fallback to posix_fallocate if available. */ - if (ret != 0) { - ret = call_posix_fallocate(fd, offset, length); - } -#endif - - return check_error(ret, errInfo); -#elif defined F_PREALLOCATE - /* Mac OS X specific, equivalent to posix_fallocate. */ - int ret; - fstore_t fs; - - memset(&fs, 0, sizeof(fs)); - fs.fst_flags = F_ALLOCATECONTIG; - fs.fst_posmode = F_VOLPOSMODE; - fs.fst_offset = (off_t) offset; - fs.fst_length = (off_t) length; - - ret = fcntl(fd, F_PREALLOCATE, &fs); - - if (-1 == ret) { - fs.fst_flags = F_ALLOCATEALL; - ret = fcntl(fd, F_PREALLOCATE, &fs); - -#if defined HAVE_POSIX_FALLOCATE - /* Fallback to posix_fallocate if available. */ - if (-1 == ret) { - ret = call_posix_fallocate(fd, offset, length); - } -#endif - } - - return check_error(ret, errInfo); -#elif defined HAVE_POSIX_FALLOCATE - /* Other Unixes, use posix_fallocate if available. */ - return check_error(call_posix_fallocate(fd, offset, length), errInfo); -#else - errno = ENOTSUP; - return check_error(-1, errInfo); -#endif -} diff --git a/erts/emulator/drivers/win32/win_efile.c b/erts/emulator/drivers/win32/win_efile.c deleted file mode 100644 index 2d366b5833..0000000000 --- a/erts/emulator/drivers/win32/win_efile.c +++ /dev/null @@ -1,2058 +0,0 @@ -/* - * %CopyrightBegin% - * - * Copyright Ericsson AB 1997-2016. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - * %CopyrightEnd% - */ -/* - * Purpose: Provides file and directory operations for Windows. - */ - -#include <windows.h> -#ifdef HAVE_CONFIG_H -# include "config.h" -#endif -#include "sys.h" -#include <ctype.h> -#include <wchar.h> -#include "erl_efile.h" - -#define DBG_TRACE_MASK 0 -/* 1 = file name ops - * 2 = file descr ops - * 4 = errors - * 8 = path name conversion - */ -#if !DBG_TRACE_MASK -# define DBG_TRACE(M,S) -# define DBG_TRACE1(M,FMT,A) -# define DBG_TRACE2(M,FMT,A,B) -#else -# define DBG_TRACE(M,S) do { if ((M)&DBG_TRACE_MASK) fwprintf(stderr, L"DBG_TRACE %d: %s\r\n", __LINE__, (WCHAR*)(S)); }while(0) -# define DBG_TRACE1(M,FMT,A) do { if ((M)&DBG_TRACE_MASK) fwprintf(stderr, L"DBG_TRACE %d: " L##FMT L"\r\n", __LINE__, (A)); }while(0) -# define DBG_TRACE2(M,FMT,A,B) do { if ((M)&DBG_TRACE_MASK) fwprintf(stderr, L"DBG_TRACE %d: " L##FMT L"\r\n", __LINE__, (A), (B)); }while(0) -#endif - -/* - * Microsoft-specific function to map a WIN32 error code to a Posix errno. - */ - -#define ISSLASH(a) ((a) == L'\\' || (a) == L'/') -#define ISDIR(st) (((st).st_mode&S_IFMT) == S_IFDIR) -#define ISREG(st) (((st).st_mode&S_IFMT) == S_IFREG) - -#define IS_DOT_OR_DOTDOT(s) \ - ((s)[0] == L'.' && ((s)[1] == L'\0' || ((s)[1] == L'.' && (s)[2] == L'\0'))) - -#define FILE_SHARE_FLAGS (FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE) - -#ifndef INVALID_FILE_ATTRIBUTES -#define INVALID_FILE_ATTRIBUTES ((DWORD) 0xFFFFFFFF) -#endif - -#define TICKS_PER_SECOND (10000000ULL) -#define EPOCH_DIFFERENCE (11644473600LL) - -#define FILETIME_TO_EPOCH(epoch, ft) \ - do { \ - ULARGE_INTEGER ull; \ - ull.LowPart = (ft).dwLowDateTime; \ - ull.HighPart = (ft).dwHighDateTime; \ - (epoch) = ((ull.QuadPart / TICKS_PER_SECOND) - EPOCH_DIFFERENCE); \ - } while(0) - -#define EPOCH_TO_FILETIME(ft, epoch) \ - do { \ - ULARGE_INTEGER ull; \ - ull.QuadPart = (((epoch) + EPOCH_DIFFERENCE) * TICKS_PER_SECOND); \ - (ft).dwLowDateTime = ull.LowPart; \ - (ft).dwHighDateTime = ull.HighPart; \ - } while(0) - - -static int check_error(int result, Efile_error* errInfo); -static int set_error(Efile_error* errInfo); -static int set_os_errno(Efile_error* errInfo, DWORD os_errno); -static int is_root_unc_name(const WCHAR *path); -static int extract_root(WCHAR *name); -static unsigned short dos_to_posix_mode(int attr, const WCHAR *name); - - -struct wpath_tmp_buffer { - struct wpath_tmp_buffer* next; - WCHAR buffer[1]; -}; - -typedef struct { - Efile_error* errInfo; - struct wpath_tmp_buffer* buf_list; -}Efile_call_state; - -static void call_state_init(Efile_call_state* state, Efile_error* errInfo) -{ - state->errInfo = errInfo; - state->buf_list = NULL; -} -static WCHAR* wpath_tmp_alloc(Efile_call_state* state, size_t len) -{ - size_t sz = offsetof(struct wpath_tmp_buffer, buffer) - + (len+1)*sizeof(WCHAR); - struct wpath_tmp_buffer* p = driver_alloc(sz); - p->next = state->buf_list; - state->buf_list = p; - return p->buffer; -} -static void call_state_free(Efile_call_state* state) -{ - while(state->buf_list) { - struct wpath_tmp_buffer* next = state->buf_list->next; - driver_free(state->buf_list); - state->buf_list = next; - } -} -static WCHAR* get_cwd_wpath_tmp(Efile_call_state* state) -{ - WCHAR dummy; - DWORD size = GetCurrentDirectoryW(0, &dummy); - WCHAR* ret = NULL; - - if (size) { - ret = wpath_tmp_alloc(state, size); - if (!GetCurrentDirectoryW(size, ret)) { - ret = NULL; - } - } - return ret; -} -static WCHAR* get_full_wpath_tmp(Efile_call_state* state, - const WCHAR* file, - WCHAR** file_part, - DWORD extra) -{ - WCHAR dummy; - DWORD size = GetFullPathNameW(file, 0, &dummy, NULL); - WCHAR* ret = NULL; - - if (size) { - int ok; - ret = wpath_tmp_alloc(state, size + extra); - if (file_part) { - ok = (GetFullPathNameW(file, size, ret, file_part) != 0); - } - else { - ok = (_wfullpath(ret, file, size) != NULL); - } - if (!ok) { - ret = NULL; - } - } - return ret; -} - -static void ensure_wpath_max(Efile_call_state* state, WCHAR** pathp, size_t max); -static int do_rmdir(Efile_call_state*, char* name); -static int do_rename(Efile_call_state*, char* src, char* dst); -static int do_readdir(Efile_call_state*, char* name, EFILE_DIR_HANDLE*, char* buffer, size_t *size); -static int do_fileinfo(Efile_call_state*, Efile_info*, char* orig_name, int info_for_link); -static char* do_readlink(Efile_call_state*, char* name, char* buffer, size_t size); -static int do_altname(Efile_call_state*, char* orig_name, char* buffer, size_t size); - - -static int errno_map(DWORD last_error) { - - switch (last_error) { - case ERROR_SUCCESS: - return 0; - case ERROR_INVALID_FUNCTION: - case ERROR_INVALID_DATA: - case ERROR_INVALID_PARAMETER: - case ERROR_INVALID_TARGET_HANDLE: - case ERROR_INVALID_CATEGORY: - case ERROR_NEGATIVE_SEEK: - return EINVAL; - case ERROR_DIR_NOT_EMPTY: - return EEXIST; - case ERROR_BAD_FORMAT: - return ENOEXEC; - case ERROR_PATH_NOT_FOUND: - case ERROR_FILE_NOT_FOUND: - case ERROR_NO_MORE_FILES: - return ENOENT; - case ERROR_TOO_MANY_OPEN_FILES: - return EMFILE; - case ERROR_ACCESS_DENIED: - case ERROR_INVALID_ACCESS: - case ERROR_CURRENT_DIRECTORY: - case ERROR_SHARING_VIOLATION: - case ERROR_LOCK_VIOLATION: - case ERROR_INVALID_PASSWORD: - case ERROR_DRIVE_LOCKED: - return EACCES; - case ERROR_INVALID_HANDLE: - return EBADF; - case ERROR_NOT_ENOUGH_MEMORY: - case ERROR_OUTOFMEMORY: - case ERROR_OUT_OF_STRUCTURES: - return ENOMEM; - case ERROR_INVALID_DRIVE: - case ERROR_BAD_UNIT: - case ERROR_NOT_READY: - case ERROR_REM_NOT_LIST: - case ERROR_DUP_NAME: - case ERROR_BAD_NETPATH: - case ERROR_NETWORK_BUSY: - case ERROR_DEV_NOT_EXIST: - case ERROR_BAD_NET_NAME: - return ENXIO; - case ERROR_NOT_SAME_DEVICE: - return EXDEV; - case ERROR_WRITE_PROTECT: - return EROFS; - case ERROR_BAD_LENGTH: - case ERROR_BUFFER_OVERFLOW: - return E2BIG; - case ERROR_SEEK: - case ERROR_SECTOR_NOT_FOUND: - return ESPIPE; - case ERROR_NOT_DOS_DISK: - return ENODEV; - case ERROR_GEN_FAILURE: - return ENODEV; - case ERROR_SHARING_BUFFER_EXCEEDED: - case ERROR_NO_MORE_SEARCH_HANDLES: - return EMFILE; - case ERROR_HANDLE_EOF: - case ERROR_BROKEN_PIPE: - return EPIPE; - case ERROR_HANDLE_DISK_FULL: - case ERROR_DISK_FULL: - return ENOSPC; - case ERROR_NOT_SUPPORTED: - return ENOTSUP; - case ERROR_FILE_EXISTS: - case ERROR_ALREADY_EXISTS: - case ERROR_CANNOT_MAKE: - return EEXIST; - case ERROR_ALREADY_ASSIGNED: - return EBUSY; - case ERROR_NO_PROC_SLOTS: - return EAGAIN; - case ERROR_CANT_RESOLVE_FILENAME: - return EMLINK; - case ERROR_PRIVILEGE_NOT_HELD: - return EPERM; - case ERROR_ARENA_TRASHED: - case ERROR_INVALID_BLOCK: - case ERROR_BAD_ENVIRONMENT: - case ERROR_BAD_COMMAND: - case ERROR_CRC: - case ERROR_OUT_OF_PAPER: - case ERROR_READ_FAULT: - case ERROR_WRITE_FAULT: - case ERROR_WRONG_DISK: - case ERROR_NET_WRITE_FAULT: - return EIO; - default: /* not to do with files I expect. */ - return EIO; - } -} - -static int -check_error(int result, Efile_error* errInfo) -{ - if (result < 0) { - errInfo->posix_errno = errno; - errInfo->os_errno = GetLastError(); - DBG_TRACE2(4, "ERROR os_error=%d errno=%d @@@@@@@@@@@@@@@@@@@@@@@@@@@@", - errInfo->os_errno, errInfo->posix_errno); - return 0; - } - return 1; -} - -static void -save_last_error(Efile_error* errInfo) -{ - errInfo->posix_errno = errno; - errInfo->os_errno = GetLastError(); - DBG_TRACE2(4, "ERROR os_error=%d errno=%d $$$$$$$$$$$$$$$$$$$$$$$$$$$$$", - errInfo->os_errno, errInfo->posix_errno); -} - - -/* - * Fills the provided error information structure with information - * with the error code given by GetLastError() and its corresponding - * Posix error number. - * - * Returns 0. - */ - -static int -set_error(Efile_error* errInfo) -{ - set_os_errno(errInfo, GetLastError()); - return 0; -} - - -static int -set_os_errno(Efile_error* errInfo, DWORD os_errno) -{ - errInfo->os_errno = os_errno; - errInfo->posix_errno = errno_map(os_errno); - DBG_TRACE2(4, "ERROR os_error=%d errno=%d ############################", - errInfo->os_errno, errInfo->posix_errno); - return 0; -} - -int -efile_init() { - return 1; -} - -/* - * A writev with Unix semantics, but with Windows arguments - */ -static int -win_writev(Efile_error* errInfo, - HANDLE fd, /* handle to file */ - FILE_SEGMENT_ELEMENT iov[], /* array of buffer pointers */ - DWORD *size) /* number of bytes to write */ -{ - OVERLAPPED ov; - ov.Offset = 0L; - ov.OffsetHigh = 0L; - ov.hEvent = CreateEvent(NULL, TRUE, FALSE, NULL); - if (ov.hEvent == NULL) - return set_error(errInfo); - if (! write_file_gather(fd, iov, *size, NULL, &ov)) - return set_error(errInfo); - if (WaitForSingleObject(ov.hEvent, INFINITE) != WAIT_OBJECT_0) - return set_error(errInfo); - if (! GetOverlappedResult(fd, &ov, size, FALSE)) - return set_error(errInfo); - return 1; -} - - -/* Check '*pathp' and convert it if needed to something that windows will accept. - * Typically use UNC path with \\?\ prefix if absolute path is longer than 260. - */ -static void ensure_wpath(Efile_call_state* state, WCHAR** pathp) -{ - ensure_wpath_max(state, pathp, MAX_PATH); -} - -static void ensure_wpath_max(Efile_call_state* state, WCHAR** pathp, size_t max) -{ - WCHAR* path = *pathp; - WCHAR* p; - size_t len = wcslen(path); - int unc_fixup = 0; - - if (path[0] == 0) { - DBG_TRACE(8, L"Let empty path pass through"); - return; - } - - DBG_TRACE1(8,"IN: %s", path); - - if (path[1] == L':' && ISSLASH(path[2])) { /* absolute path */ - if (len >= max) { - WCHAR *src, *dst; - - *pathp = wpath_tmp_alloc(state, 4+len+1); - dst = *pathp; - wcscpy(dst, L"\\\\?\\"); - for (src=path,dst+=4; *src; src++) { - if (*src == L'/') { - if (dst[-1] != L'\\') { - *dst++ = L'\\'; - } - /*else ignore redundant slashes */ - } - else - *dst++ = *src; - } - *dst = 0; - unc_fixup = 1; - } - } - else if (!(ISSLASH(path[0]) && ISSLASH(path[1]))) { /* relative path */ - DWORD cwdLen = GetCurrentDirectoryW(0, NULL); - DWORD absLen = cwdLen + 1 + len; - if (absLen >= max) { - WCHAR *fullPath = wpath_tmp_alloc(state, 4+4+absLen); - DWORD fullLen; - - fullLen = GetFullPathNameW(path, 4 + absLen, fullPath+4, NULL); - if (fullLen >= 4+absLen) { - *pathp = path; - DBG_TRACE2(8,"ensure_wpath FAILED absLen=%u %s", (int)absLen, path); - return; - } - /* GetFullPathNameW can return paths longer than MAX_PATH without the \\?\ prefix. - * At least seen on Windows 7. Go figure... - */ - if (fullLen >= max && wcsncmp(fullPath+4, L"\\\\?\\", 4) != 0) { - wcsncpy(fullPath, L"\\\\?\\", 4); - *pathp = fullPath; - } - else { - *pathp = fullPath + 4; - } - } - } - - if (unc_fixup) { - WCHAR* endp; - - p = *pathp; - len = wcslen(p); - endp = p + len; - if (len > 4) { - p += 4; - while (*p) { - if (p[0] == L'\\' && p[1] == L'.') { - if (p[2] == L'\\' || !p[2]) { /* single dot */ - wmemmove(p, p+2, (&endp[1] - &p[2])); - endp -= 2; - } - else if (p[2] == L'.' && (p[3] == L'\\' || !p[3])) { /* double dot */ - WCHAR* r; - for (r=p-1; *r == L'\\'; --r) - /*skip redundant slashes*/; - for (; *r != L'\\'; --r) - /*find start of prev directory*/; - if (r < *pathp + 6) - break; - wmemmove(r, p+3, (&endp[1] - &p[3])); - p = r; - } - else p += 3; - } - else ++p; - } - } - } - DBG_TRACE1(8,"OUT: %s", *pathp); -} - -int -efile_mkdir(Efile_error* errInfo, /* Where to return error codes. */ - char* name) /* Name of directory to create. */ -{ - Efile_call_state state; - WCHAR* wname = (WCHAR*)name; - int ret; - - DBG_TRACE(1, name); - call_state_init(&state, errInfo); - ensure_wpath_max(&state, &wname, 248); /* Yes, 248 limit for normal paths */ - - ret = (int) CreateDirectoryW(wname, NULL); - if (!ret) - set_error(errInfo); - - call_state_free(&state); - return ret; -} - -int -efile_rmdir(Efile_error* errInfo, /* Where to return error codes. */ - char* name) /* Name of directory to delete. */ -{ - Efile_call_state state; - int ret; - - DBG_TRACE(1, name); - call_state_init(&state, errInfo); - ret = do_rmdir(&state, name); - call_state_free(&state); - return ret; -} - -static int do_rmdir(Efile_call_state* state, char* name) -{ - OSVERSIONINFO os; - DWORD attr; - WCHAR *wname = (WCHAR *) name; - WCHAR *buffer = NULL; - - ensure_wpath(state, &wname); - - if (RemoveDirectoryW(wname) != FALSE) { - return 1; - } - errno = errno_map(GetLastError()); - if (errno == EACCES) { - attr = GetFileAttributesW(wname); - if (attr != (DWORD) -1) { - if ((attr & FILE_ATTRIBUTE_DIRECTORY) == 0) { - /* - * Windows 95 reports calling RemoveDirectory on a file as an - * EACCES, not an ENOTDIR. - */ - - errno = ENOTDIR; - goto end; - } - - /* - * Windows 95 reports removing a non-empty directory as - * an EACCES, not an EEXIST. If the directory is not empty, - * change errno so caller knows what's going on. - */ - - os.dwOSVersionInfoSize = sizeof(os); - GetVersionEx(&os); - if (os.dwPlatformId == VER_PLATFORM_WIN32_WINDOWS) { - HANDLE handle; - WIN32_FIND_DATAW data; - int len = wcslen(wname); - - buffer = wpath_tmp_alloc(state, len + 4); - wcscpy(buffer, wname); - if (buffer[0] && buffer[len-1] != L'\\' && buffer[len-1] != L'/') { - wcscat(buffer, L"\\"); - } - wcscat(buffer, L"*.*"); - handle = FindFirstFileW(buffer, &data); - if (handle != INVALID_HANDLE_VALUE) { - while (1) { - if ((wcscmp(data.cFileName, L".") != 0) - && (wcscmp(data.cFileName, L"..") != 0)) { - /* - * Found something in this directory. - */ - - errno = EEXIST; - break; - } - if (FindNextFileW(handle, &data) == FALSE) { - break; - } - } - FindClose(handle); - } - } - } - } - - if (errno == ENOTEMPTY) { - /* - * Posix allows both EEXIST or ENOTEMPTY, but we'll always - * return EEXIST to allow easy matching in Erlang code. - */ - - errno = EEXIST; - } - - end: - save_last_error(state->errInfo); - return 0; -} - -int -efile_delete_file(Efile_error* errInfo, /* Where to return error codes. */ - char* name) /* Name of file to delete. */ -{ - Efile_call_state state; - int ret; - DBG_TRACE(1, name); - call_state_init(&state, errInfo); - ret = do_delete_file(&state, name); - call_state_free(&state); - return ret; -} - -static int do_delete_file(Efile_call_state* state, char* name) -{ - DWORD attr; - WCHAR *wname = (WCHAR *) name; - - ensure_wpath(state, &wname); - - if (DeleteFileW(wname) != FALSE) { - return 1; - } - - errno = errno_map(GetLastError()); - if (errno == EACCES) { - attr = GetFileAttributesW(wname); - if (attr != (DWORD) -1) { - if (attr & FILE_ATTRIBUTE_DIRECTORY) { - /* - * Windows NT reports removing a directory as EACCES instead - * of EPERM. - */ - - errno = EPERM; - } - } - } else if (errno == ENOENT) { - attr = GetFileAttributesW(wname); - if (attr != (DWORD) -1) { - if (attr & FILE_ATTRIBUTE_DIRECTORY) { - /* - * Windows 95 reports removing a directory as ENOENT instead - * of EPERM. - */ - - errno = EPERM; - } - } - } else if (errno == EINVAL) { - /* - * Windows NT reports removing a char device as EINVAL instead of - * EACCES. - */ - - errno = EACCES; - } - - return check_error(-1, state->errInfo); -} - -/* - *--------------------------------------------------------------------------- - * - * Changes the name of an existing file or directory, from src to dst. - * If src and dst refer to the same file or directory, does nothing - * and returns success. Otherwise if dst already exists, it will be - * deleted and replaced by src subject to the following conditions: - * If src is a directory, dst may be an empty directory. - * If src is a file, dst may be a file. - * In any other situation where dst already exists, the rename will - * fail. - * - * Some possible error codes: - * - * EACCES: src or dst parent directory can't be read and/or written. - * EEXIST: dst is a non-empty directory. - * EINVAL: src is a root directory or dst is a subdirectory of src. - * EISDIR: dst is a directory, but src is not. - * ENOENT: src doesn't exist, or src or dst is "". - * ENOTDIR: src is a directory, but dst is not. - * EXDEV: src and dst are on different filesystems. - * - * Side effects: - * The implementation of rename may allow cross-filesystem renames, - * but the caller should be prepared to emulate it with copy and - * delete if errno is EXDEV. - * - *--------------------------------------------------------------------------- - */ - -int -efile_rename(Efile_error* errInfo, char* src, char* dst) -{ - Efile_call_state state; - int ret; - DBG_TRACE(1, src); - call_state_init(&state, errInfo); - ret = do_rename(&state, src, dst); - call_state_free(&state); - return ret; -} - -static int -do_rename(Efile_call_state* state, - char* src, /* Original name. */ - char* dst) /* New name. */ -{ - DWORD srcAttr, dstAttr; - WCHAR *wsrc = (WCHAR *) src; - WCHAR *wdst = (WCHAR *) dst; - - ensure_wpath(state, &wsrc); - ensure_wpath(state, &wdst); - - if (MoveFileW(wsrc, wdst) != FALSE) { - return 1; - } - - errno = errno_map(GetLastError()); - srcAttr = GetFileAttributesW(wsrc); - dstAttr = GetFileAttributesW(wdst); - if (srcAttr == (DWORD) -1) { - srcAttr = 0; - } - if (dstAttr == (DWORD) -1) { - dstAttr = 0; - } - - if (errno == EBADF) { - errno = EACCES; - return check_error(-1, state->errInfo); - } - if (errno == EACCES) { - decode: - if (srcAttr & FILE_ATTRIBUTE_DIRECTORY) { - WCHAR *srcPath, *dstPath; - WCHAR *srcRest, *dstRest; - int size; - - srcPath = get_full_wpath_tmp(state, wsrc, &srcRest, 0); - if (!srcPath) { - save_last_error(state->errInfo); - return 0; - } - - dstPath = get_full_wpath_tmp(state, wdst, &dstRest, 0); - if (!dstPath) { - save_last_error(state->errInfo); - return 0; - } - - if (srcRest == NULL) { - srcRest = srcPath + wcslen(srcPath); - } - if (_wcsnicmp(srcPath, dstPath, srcRest - srcPath) == 0) { - /* - * Trying to move a directory into itself. - */ - - errno = EINVAL; - } - if (extract_root(srcPath)) { - /* - * Attempt to move a root directory. Never allowed. - */ - errno = EINVAL; - } - - (void) extract_root(dstPath); - if (dstPath[0] == L'\0') { - /* - * The filename was invalid. (Don't know why, - * but play it safe.) - */ - errno = EINVAL; - } - if (_wcsicmp(srcPath, dstPath) != 0) { - /* - * If src is a directory and dst filesystem != src - * filesystem, errno should be EXDEV. It is very - * important to get this behavior, so that the caller - * can respond to a cross filesystem rename by - * simulating it with copy and delete. The MoveFile - * system call already handles the case of moving a - * *file* between filesystems. - */ - - errno = EXDEV; - } - } - - /* - * Other types of access failure is that dst is a read-only - * filesystem, that an open file referred to src or dest, or that - * src or dest specified the current working directory on the - * current filesystem. EACCES is returned for those cases. - */ - - } else if (errno == EEXIST) { - /* - * Reports EEXIST any time the target already exists. If it makes - * sense, remove the old file and try renaming again. - */ - - if (srcAttr & FILE_ATTRIBUTE_DIRECTORY) { - if (dstAttr & FILE_ATTRIBUTE_DIRECTORY) { - /* - * Overwrite empty dst directory with src directory. The - * following call will remove an empty directory. If it - * fails, it's because it wasn't empty. - */ - - if (RemoveDirectoryW(wdst)) { - /* - * Now that that empty directory is gone, we can try - * renaming again. If that fails, we'll put this empty - * directory back, for completeness. - */ - - if (MoveFileW(wsrc, wdst) != FALSE) { - return 1; - } - - /* - * Some new error has occurred. Don't know what it - * could be, but report this one. - */ - - errno = errno_map(GetLastError()); - CreateDirectoryW(wdst, NULL); - SetFileAttributesW(wdst, dstAttr); - if (errno == EACCES) { - /* - * Decode the EACCES to a more meaningful error. - */ - - goto decode; - } - } - } else { /* (dstAttr & FILE_ATTRIBUTE_DIRECTORY) == 0 */ - errno = ENOTDIR; - } - } else { /* (srcAttr & FILE_ATTRIBUTE_DIRECTORY) == 0 */ - if (dstAttr & FILE_ATTRIBUTE_DIRECTORY) { - errno = EISDIR; - } else { - /* - * Overwrite existing file by: - * - * 1. Rename existing file to temp name. - * 2. Rename old file to new name. - * 3. If success, delete temp file. If failure, - * put temp file back to old name. - */ - - WCHAR *tempName; - int result; - WCHAR *rest; - - tempName = get_full_wpath_tmp(state, wdst, &rest, 14); - if (!tempName || !rest) { - save_last_error(state->errInfo); - return 0; - } - - *rest = L'\0'; - result = -1; - if (GetTempFileNameW(tempName, L"erlr", 0, tempName) != 0) { - /* - * Strictly speaking, need the following DeleteFile and - * MoveFile to be joined as an atomic operation so no - * other app comes along in the meantime and creates the - * same temp file. - */ - - DeleteFileW(tempName); - if (MoveFileW(wdst, tempName) != FALSE) { - if (MoveFileW(wsrc, wdst) != FALSE) { - SetFileAttributesW(tempName, FILE_ATTRIBUTE_NORMAL); - DeleteFileW(tempName); - return 1; - } else { - DeleteFileW(wdst); - MoveFileW(tempName, wdst); - } - } - - /* - * Can't backup dst file or move src file. Return that - * error. Could happen if an open file refers to dst. - */ - - errno = errno_map(GetLastError()); - if (errno == EACCES) { - /* - * Decode the EACCES to a more meaningful error. - */ - goto decode; - } - } - return result; - } - } - } - return check_error(-1, state->errInfo); -} - -int -efile_chdir(Efile_error* errInfo, /* Where to return error codes. */ - char* name) /* Name of directory to make current. */ -{
- /* We don't even try to handle long paths here
- * as current working directory is always limited to MAX_PATH
- * even if we use UNC paths and SetCurrentDirectoryW()
- */
- int success = check_error(_wchdir((WCHAR *) name), errInfo);
- if (!success && errInfo->posix_errno == EINVAL)
- /* POSIXification of errno */
- errInfo->posix_errno = ENOENT; - return success; -} - -int -efile_getdcwd(Efile_error* errInfo, /* Where to return error codes. */ - int drive, /* 0 - current, 1 - A, 2 - B etc. */ - char* buffer, /* Where to return the current directory. */ - size_t size) /* Size of buffer. */ -{ - WCHAR *wbuffer = (WCHAR *) buffer; - size_t wbuffer_size = size / 2; - DBG_TRACE(1, L"#getdcwd#"); - if (_wgetdcwd(drive, wbuffer, wbuffer_size) == NULL) { - return check_error(-1, errInfo); - } - DBG_TRACE1(8, "getdcwd OS=%s", wbuffer); - if (wcsncmp(wbuffer, L"\\\\?\\", 4) == 0) { - wmemmove(wbuffer, wbuffer+4, wcslen(wbuffer+4)+1); - } - for ( ; *wbuffer; wbuffer++) - if (*wbuffer == L'\\') - *wbuffer = L'/'; - DBG_TRACE1(8, "getdcwd ERLANG=%s", (WCHAR*)buffer); - return 1; -} - -int -efile_readdir(Efile_error* errInfo, char* name, EFILE_DIR_HANDLE* dir_handle, - char* buffer, size_t *size) -{ - Efile_call_state state; - int ret; - DBG_TRACE(dir_handle?2:1, name); - call_state_init(&state, errInfo); - ret = do_readdir(&state, name, dir_handle, buffer, size); - call_state_free(&state); - return ret; -} - -static int do_readdir(Efile_call_state* state, - char* name, /* Name of directory to list */ - EFILE_DIR_HANDLE* dir_handle, /* Handle of opened directory or NULL */ - char* buffer, /* Buffer to put one filename in */ - size_t *size) /* in-out size of buffer/size of filename excluding zero - termination in bytes*/ -{ - HANDLE dir; /* Handle to directory. */ - WIN32_FIND_DATAW findData; /* Data found by FindFirstFile() or FindNext(). */ - /* Alignment is not honored, this works on x86 because of alignment fixup by processor. - Not perfect, but faster than alinging by hand (really) */ - WCHAR *wbuffer = (WCHAR *) buffer; - - /* - * First time we must setup everything. - */ - - if (*dir_handle == NULL) { - WCHAR *wname = (WCHAR *) name; - WCHAR* wildcard; - int length; - WCHAR* s; - - ensure_wpath_max(state, &wname, MAX_PATH-2); - length = wcslen(wname); - - wildcard = wpath_tmp_alloc(state, length+3); - - wcscpy(wildcard, wname); - s = wildcard+length-1; - if (*s != L'/' && *s != L'\\') - *++s = L'\\'; - *++s = L'*'; - *++s = L'\0'; - DEBUGF(("Reading %ws\n", wildcard)); - dir = FindFirstFileW(wildcard, &findData); - if (dir == INVALID_HANDLE_VALUE) { - set_error(state->errInfo); - return 0; - } - *dir_handle = (EFILE_DIR_HANDLE) dir; - - if (!IS_DOT_OR_DOTDOT(findData.cFileName)) { - wcscpy(wbuffer, findData.cFileName); - *size = wcslen(wbuffer)*2; - return 1; - } - } - - /* - * Retrieve the name of the next file using the directory handle. - */ - - dir = (HANDLE) *dir_handle; - - for (;;) { - if (FindNextFileW(dir, &findData)) { - if (IS_DOT_OR_DOTDOT(findData.cFileName)) - continue; - wcscpy(wbuffer, findData.cFileName); - *size = wcslen(wbuffer)*2; - return 1; - } - - if (GetLastError() == ERROR_NO_MORE_FILES) { - state->errInfo->posix_errno = state->errInfo->os_errno = 0; - } - else { - set_error(state->errInfo); - } - FindClose(dir); - return 0; - } -} - -int -efile_openfile(Efile_error* errInfo, char* name, int flags, int* pfd, Sint64* pSize) -{ - Efile_call_state state; - int ret; - DBG_TRACE1(1, "openfile(%s)", name); - call_state_init(&state, errInfo); - ret = do_openfile(&state, name, flags, pfd, pSize); - call_state_free(&state); - return ret; -} - -static -int do_openfile(Efile_call_state* state, /* Where to return error codes. */ - char* name, /* Name of directory to open. */ - int flags, /* Flags to use for opening. */ - int* pfd, /* Where to store the file descriptor. */ - Sint64* pSize) /* Where to store the size of the file. */ -{ - Efile_error* errInfo = state->errInfo; - BY_HANDLE_FILE_INFORMATION fileInfo; /* File information from a handle. */ - HANDLE fd; /* Handle to open file. */ - DWORD access; /* Access mode: GENERIC_READ, GENERIC_WRITE. */ - DWORD crFlags; - DWORD flagsAndAttrs = FILE_ATTRIBUTE_NORMAL; - WCHAR *wname = (WCHAR *) name; - - switch (flags & (EFILE_MODE_READ|EFILE_MODE_WRITE)) { - case EFILE_MODE_READ: - access = GENERIC_READ; - crFlags = OPEN_EXISTING; - break; - case EFILE_MODE_WRITE: - access = GENERIC_WRITE; - crFlags = CREATE_ALWAYS; - break; - case EFILE_MODE_READ_WRITE: - access = GENERIC_READ|GENERIC_WRITE; - crFlags = OPEN_ALWAYS; - break; - default: - errno = EINVAL; - check_error(-1, errInfo); - return 0; - } - - if (flags & EFILE_MODE_SYNC) { - flagsAndAttrs = FILE_FLAG_WRITE_THROUGH; - } - - if (flags & EFILE_MODE_APPEND) { - crFlags = OPEN_ALWAYS; - } - if (flags & EFILE_MODE_EXCL) { - crFlags = CREATE_NEW; - } - ensure_wpath(state, &wname); - fd = CreateFileW(wname, access, - FILE_SHARE_FLAGS, - NULL, crFlags, flagsAndAttrs, NULL); - - /* - * Check for errors. - */ - - if (fd == INVALID_HANDLE_VALUE) { - DWORD attr; - - set_error(errInfo); - - /* - * If the error is EACESS, the reason could be that we tried to - * open a directory. In that case, we'll change the error code - * to EISDIR. - */ - if (errInfo->posix_errno && - (attr = GetFileAttributesW(wname)) != INVALID_FILE_ATTRIBUTES && - (attr & FILE_ATTRIBUTE_DIRECTORY)) { - errInfo->posix_errno = EISDIR; - } - return 0; - } - - /* - * Get and return the length of the open file. - */ - - if (!GetFileInformationByHandle(fd, &fileInfo)) - return set_error(errInfo); - *pfd = (int) fd; - if (pSize) { - *pSize = (Sint64) - (((Uint64)fileInfo.nFileSizeHigh << 32) | - (Uint64)fileInfo.nFileSizeLow); - } - return 1; -} - -int -efile_may_openfile(Efile_error* errInfo, char *name) -{ - Efile_call_state state; - WCHAR *wname = (WCHAR *) name; - DWORD attr; - int ret; - - DBG_TRACE(1, name); - call_state_init(&state, errInfo); - ensure_wpath(&state, &wname); - if ((attr = GetFileAttributesW(wname)) == INVALID_FILE_ATTRIBUTES) { - errno = ENOENT; - ret = check_error(-1, errInfo); - } - else if (attr & FILE_ATTRIBUTE_DIRECTORY) { - errno = EISDIR; - ret = check_error(-1, errInfo); - } - else ret = 1; - - call_state_free(&state); - return ret; -} - -void -efile_closefile(fd) -int fd; /* File descriptor for file to close. */ -{ - DBG_TRACE(2, L""); - CloseHandle((HANDLE) fd); -} - -FILE* efile_wfopen(const WCHAR* name, const WCHAR* mode) -{ - Efile_call_state state; - Efile_error dummy; - FILE* f; - call_state_init(&state, &dummy); - ensure_wpath(&state, (WCHAR**)&name); - f = _wfopen(name, mode); - call_state_free(&state); - return f; -} - -int -efile_fdatasync(errInfo, fd) -Efile_error* errInfo; /* Where to return error codes. */ -int fd; /* File descriptor for file to sync. */ -{ - DBG_TRACE(2, L""); - /* Not available in Windows, just call regular fsync */ - return efile_fsync(errInfo, fd); -} - -int -efile_fsync(errInfo, fd) -Efile_error* errInfo; /* Where to return error codes. */ -int fd; /* File descriptor for file to sync. */ -{ - DBG_TRACE(2, L""); - if (!FlushFileBuffers((HANDLE) fd)) { - return check_error(-1, errInfo); - } - return 1; -} - -int -efile_fileinfo(Efile_error* errInfo, Efile_info* pInfo, - char* orig_name, int info_for_link) -{ - Efile_call_state state; - int ret; - DBG_TRACE(1, L""); - call_state_init(&state, errInfo); - ret = do_fileinfo(&state, pInfo, orig_name, info_for_link); - call_state_free(&state); - return ret; -} - -static int -do_fileinfo(Efile_call_state* state, Efile_info* pInfo, - char* orig_name, int info_for_link) -{ - Efile_error* errInfo = state->errInfo; - HANDLE findhandle; /* Handle returned by FindFirstFile(). */ - WIN32_FIND_DATAW findbuf; /* Data return by FindFirstFile(). */ - WCHAR* name = NULL; - WCHAR* win_path; - int name_len; - int drive; /* Drive for filename (1 = A:, 2 = B: etc). */ - WCHAR *worig_name = (WCHAR *) orig_name; - - ensure_wpath(state, &worig_name); - /* Don't allow wildcards to be interpreted by system */ - - - /* - * Move the name to a buffer and make sure to remove a trailing - * slash, because it causes FindFirstFile() to fail on Win95. - */ - - name_len = wcslen(worig_name); - - name = wpath_tmp_alloc(state, name_len+1); - wcscpy(name, worig_name); - if (name_len > 2 && ISSLASH(name[name_len-1]) && - name[name_len-2] != L':') { - name[name_len-1] = L'\0'; - } - - win_path = name; - if (wcsncmp(name, L"\\\\?\\", 4) == 0) { - win_path += 4; - } - - if (wcspbrk(win_path, L"?*")) { - enoent: - errInfo->posix_errno = ENOENT; - errInfo->os_errno = ERROR_FILE_NOT_FOUND; - return 0; - } - - /* Try to get disk from name. If none, get current disk. */ - - if (win_path[1] != L':') { - WCHAR* cwd_path = get_cwd_wpath_tmp(state); - drive = 0; - if (cwd_path[1] == L':') { - drive = towlower(cwd_path[0]) - L'a' + 1; - } - } else if (*win_path && win_path[2] == L'\0') { - /* - * X: and nothing more is an error. - */ - errInfo->posix_errno = ENOENT; - errInfo->os_errno = ERROR_FILE_NOT_FOUND; - return 0; - } else { - drive = towlower(*win_path) - L'a' + 1; - } - - findhandle = FindFirstFileW(name, &findbuf); - if (findhandle == INVALID_HANDLE_VALUE) { - WCHAR* path = NULL; - - if (!(wcspbrk(name, L"./\\") && - (path = get_full_wpath_tmp(state, name, NULL, 0)) && - /* root dir. ('C:\') or UNC root dir. ('\\server\share\') */ - ((wcslen(path) == 3) || is_root_unc_name(path)) && - (GetDriveTypeW(path) > 1) ) ) { - - errInfo->posix_errno = ENOENT; - errInfo->os_errno = ERROR_FILE_NOT_FOUND; - return 0; - } - - /* - * Root directories (such as C:\ or \\server\share\ are fabricated. - */ - - findbuf.dwFileAttributes = FILE_ATTRIBUTE_DIRECTORY; - findbuf.nFileSizeHigh = 0; - findbuf.nFileSizeLow = 0; - findbuf.cFileName[0] = L'\0'; - - pInfo->links = 1; - pInfo->cTime = pInfo->accessTime = pInfo->modifyTime = 0; - } else { - SYSTEMTIME SystemTime; - FILETIME LocalFTime; - - /*first check if we are a symlink */ - if (!info_for_link && (findbuf.dwFileAttributes & - FILE_ATTRIBUTE_REPARSE_POINT)){ - /* - * given that we know this is a symlink, - we should be able to find its target */ - WCHAR* target_name = (WCHAR*) do_readlink(state, (char *) name, NULL, 0); - if (target_name) { - FindClose(findhandle); - return do_fileinfo(state, pInfo, - (char *) target_name, info_for_link); - } - } - - /* number of links: */ - { - HANDLE handle; /* Handle returned by CreateFile() */ - BY_HANDLE_FILE_INFORMATION fileInfo; /* from CreateFile() */ - - /* We initialise nNumberOfLinks as GetFileInformationByHandle - does not always initialise this field */ - fileInfo.nNumberOfLinks = 1; - if (handle = CreateFileW(name, GENERIC_READ, FILE_SHARE_FLAGS, NULL, - OPEN_EXISTING, 0, NULL)) { - GetFileInformationByHandle(handle, &fileInfo); - pInfo->links = fileInfo.nNumberOfLinks; - CloseHandle(handle); - } else { - pInfo->links = 1; - } - } - - FILETIME_TO_EPOCH(pInfo->modifyTime, findbuf.ftLastWriteTime); - - if (findbuf.ftLastAccessTime.dwLowDateTime == 0 && - findbuf.ftLastAccessTime.dwHighDateTime == 0) { - pInfo->accessTime = pInfo->modifyTime; - } else { - FILETIME_TO_EPOCH(pInfo->accessTime, findbuf.ftLastAccessTime); - } - - if (findbuf.ftCreationTime.dwLowDateTime == 0 && - findbuf.ftCreationTime.dwHighDateTime == 0) { - pInfo->cTime = pInfo->modifyTime; - } else { - FILETIME_TO_EPOCH(pInfo->cTime ,findbuf.ftCreationTime); - } - FindClose(findhandle); - } - - pInfo->size_low = findbuf.nFileSizeLow; - pInfo->size_high = findbuf.nFileSizeHigh; - - if (info_for_link && (findbuf.dwFileAttributes & - FILE_ATTRIBUTE_REPARSE_POINT)) - pInfo->type = FT_SYMLINK; - else if (findbuf.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY) - pInfo->type = FT_DIRECTORY; - else - pInfo->type = FT_REGULAR; - - if (findbuf.dwFileAttributes & FILE_ATTRIBUTE_READONLY) - pInfo->access = FA_READ; - else - pInfo->access = FA_READ|FA_WRITE; - - pInfo->mode = dos_to_posix_mode(findbuf.dwFileAttributes, name); - pInfo->major_device = drive; - pInfo->minor_device = 0; - pInfo->inode = 0; - pInfo->uid = 0; - pInfo->gid = 0; - - return 1; -} - -int -efile_write_info(Efile_error* errInfo, - Efile_info* pInfo, - char* name) -{ - Efile_call_state state; - int ret; - call_state_init(&state, errInfo); - ret = do_write_info(&state, pInfo, name); - call_state_free(&state); - return ret; -} - -static int -do_write_info(Efile_call_state* state, - Efile_info* pInfo, - char* name) -{ - Efile_error* errInfo = state->errInfo; - SYSTEMTIME timebuf; - FILETIME ModifyFileTime; - FILETIME AccessFileTime; - FILETIME CreationFileTime; - HANDLE fd; - DWORD attr; - DWORD tempAttr; - WCHAR *wname = (WCHAR *) name; - - DBG_TRACE(1, name); - - ensure_wpath(state, &wname); - - /* - * Get the attributes for the file. - */ - - tempAttr = attr = GetFileAttributesW(wname); - if (attr == 0xffffffff) { - return set_error(errInfo); - } - if (pInfo->mode != -1) { - if (pInfo->mode & _S_IWRITE) { - /* clear read only bit */ - attr &= ~FILE_ATTRIBUTE_READONLY; - } else { - /* set read only bit */ - attr |= FILE_ATTRIBUTE_READONLY; - } - } - - /* - * Construct all file times. - */ - - EPOCH_TO_FILETIME(ModifyFileTime, pInfo->modifyTime); - EPOCH_TO_FILETIME(AccessFileTime, pInfo->accessTime); - EPOCH_TO_FILETIME(CreationFileTime, pInfo->cTime); - - /* - * If necessary, set the file times. - */ - - /* - * If the has read only access, we must temporarily turn on - * write access (this is necessary for native filesystems, - * but not for NFS filesystems). - */ - - if (tempAttr & FILE_ATTRIBUTE_READONLY) { - tempAttr &= ~FILE_ATTRIBUTE_READONLY; - if (!SetFileAttributesW(wname, tempAttr)) { - return set_error(errInfo); - } - } - - fd = CreateFileW(wname, GENERIC_READ|GENERIC_WRITE, - FILE_SHARE_FLAGS, - NULL, OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL, NULL); - if (fd != INVALID_HANDLE_VALUE) { - BOOL result = SetFileTime(fd, &CreationFileTime, &AccessFileTime, &ModifyFileTime); - if (!result) { - return set_error(errInfo); - } - CloseHandle(fd); - } - - /* - * If the file doesn't have the correct attributes, set them now. - * (It could have been done before setting the file times, above). - */ - - if (tempAttr != attr) { - if (!SetFileAttributesW(wname, attr)) { - return set_error(errInfo); - } - } - return 1; -} - - -int -efile_pwrite(errInfo, fd, buf, count, offset) -Efile_error* errInfo; /* Where to return error codes. */ -int fd; /* File descriptor to write to. */ -char* buf; /* Buffer to write. */ -size_t count; /* Number of bytes to write. */ -Sint64 offset; /* where to write it */ -{ - int res; - DBG_TRACE(2, L""); - res = efile_seek(errInfo, fd, offset, EFILE_SEEK_SET, NULL); - if (res) { - return efile_write(errInfo, EFILE_MODE_WRITE, fd, buf, count); - } else { - return res; - } -} - -/* position and read/write as a single atomic op */ -int -efile_pread(errInfo, fd, offset, buf, count, pBytesRead) -Efile_error* errInfo; /* Where to return error codes. */ -int fd; /* File descriptor to read from. */ -Sint64 offset; /* Offset in bytes from BOF. */ -char* buf; /* Buffer to read into. */ -size_t count; /* Number of bytes to read. */ -size_t* pBytesRead; /* Where to return number of bytes read. */ -{ - int res; - DBG_TRACE(2, L""); - res = efile_seek(errInfo, fd, offset, EFILE_SEEK_SET, NULL); - if (res) { - return efile_read(errInfo, EFILE_MODE_READ, fd, buf, count, pBytesRead); - } else { - return res; - } -} - - - -int -efile_write(errInfo, flags, fd, buf, count) -Efile_error* errInfo; /* Where to return error codes. */ -int flags; /* Flags given when file was opened. */ -int fd; /* File descriptor to write to. */ -char* buf; /* Buffer to write. */ -size_t count; /* Number of bytes to write. */ -{ - DWORD written; /* Bytes written in last operation. */ - OVERLAPPED overlapped; - OVERLAPPED* pOverlapped = NULL; - - DBG_TRACE(2, L""); - if (flags & EFILE_MODE_APPEND) { - memset(&overlapped, 0, sizeof(overlapped)); - overlapped.Offset = 0xffffffff; - overlapped.OffsetHigh = 0xffffffff; - pOverlapped = &overlapped; - } - while (count > 0) { - if (!WriteFile((HANDLE) fd, buf, count, &written, pOverlapped)) - return set_error(errInfo); - buf += written; - count -= written; - } - return 1; -} - -int -efile_writev(Efile_error* errInfo, /* Where to return error codes */ - int flags, /* Flags given when file was - * opened */ - int fd, /* File descriptor to write to */ - SysIOVec* iov, /* Vector of buffer structs. - * The structs are unchanged - * after the call */ - int iovcnt) /* Number of structs in vector */ -{ - int cnt; /* Buffers so far written */ - OVERLAPPED overlapped; - OVERLAPPED* pOverlapped = NULL; - - DBG_TRACE(2, L""); - ASSERT(iovcnt >= 0); - - if (flags & EFILE_MODE_APPEND) { - memset(&overlapped, 0, sizeof(overlapped)); - overlapped.Offset = 0xffffffff; - overlapped.OffsetHigh = 0xffffffff; - pOverlapped = &overlapped; - } - for (cnt = 0; cnt < iovcnt; cnt++) { - if (iov[cnt].iov_base && iov[cnt].iov_len > 0) { - /* Non-empty buffer */ - int p; /* Position in buffer */ - int w = iov[cnt].iov_len;/* Bytes written in this call */ - for (p = 0; p < iov[cnt].iov_len; p += w) { - if (!WriteFile((HANDLE) fd, - iov[cnt].iov_base + p, - iov[cnt].iov_len - p, - &w, - pOverlapped)) - return set_error(errInfo); - } - } - } - return 1; -} - -int -efile_read(errInfo, flags, fd, buf, count, pBytesRead) -Efile_error* errInfo; /* Where to return error codes. */ -int flags; /* Flags given when file was opened. */ -int fd; /* File descriptor to read from. */ -char* buf; /* Buffer to read into. */ -size_t count; /* Number of bytes to read. */ -size_t* pBytesRead; /* Where to return number of bytes read. */ -{ - DWORD nbytes = 0; - - DBG_TRACE(2, L""); - if (!ReadFile((HANDLE) fd, buf, count, &nbytes, NULL)) - return set_error(errInfo); - - *pBytesRead = nbytes; - return 1; -} - -int -efile_seek(errInfo, fd, offset, origin, new_location) -Efile_error* errInfo; /* Where to return error codes. */ -int fd; /* File descriptor to do the seek on. */ -Sint64 offset; /* Offset in bytes from the given origin. */ -int origin; /* Origin of seek (SEEK_SET, SEEK_CUR, - * SEEK_END). - */ -Sint64* new_location; /* Resulting new location in file. */ -{ - LARGE_INTEGER off, new_loc; - - DBG_TRACE(2, L""); - switch (origin) { - case EFILE_SEEK_SET: origin = FILE_BEGIN; break; - case EFILE_SEEK_CUR: origin = FILE_CURRENT; break; - case EFILE_SEEK_END: origin = FILE_END; break; - default: - errno = EINVAL; - check_error(-1, errInfo); - break; - } - - off.QuadPart = offset; - if (! SetFilePointerEx((HANDLE) fd, off, - new_location ? &new_loc : NULL, origin)) { - return set_error(errInfo); - } - if (new_location) { - *new_location = new_loc.QuadPart; - DEBUGF(("efile_seek(offset=%ld, origin=%d) -> %ld\n", - (long) offset, origin, (long) *new_location)); - } else { - DEBUGF(("efile_seek(offset=%ld, origin=%d)\n", (long) offset, origin)); - } - return 1; -} - -int -efile_truncate_file(errInfo, fd, flags) -Efile_error* errInfo; /* Where to return error codes. */ -int *fd; /* File descriptor for file to truncate. */ -int flags; -{ - DBG_TRACE(2, L""); - if (!SetEndOfFile((HANDLE) (*fd))) - return set_error(errInfo); - return 1; -} - - -/* - * is_root_unc_name - returns TRUE if the argument is a UNC name specifying - * a root share. That is, if it is of the form \\server\share\. - * This routine will also return true if the argument is of the - * form \\server\share (no trailing slash) but Win32 currently - * does not like that form. - * - * Forward slashes ('/') may be used instead of backslashes ('\'). - */ - -static int -is_root_unc_name(const WCHAR *path) -{ - /* - * If a root UNC name, path will start with 2 (but not 3) slashes - */ - - if ((wcslen(path) >= 5) /* minimum string is "//x/y" */ - && ISSLASH(path[0]) && ISSLASH(path[1])) - { - const WCHAR *p = path + 2; - - /* - * find the slash between the server name and share name - */ - while ( * ++ p ) - if ( ISSLASH(*p) ) - break ; - - if ( *p && p[1] ) - { - /* - * is there a further slash? - */ - while ( * ++ p ) - if ( ISSLASH(*p) ) - break ; - - /* - * just final slash (or no final slash) - */ - if ( !*p || !p[1]) - return 1; - } - } - - return 0 ; -} - -/* - * Extracts the root part of an absolute filename (by modifying the string - * pointed to by the name argument). The name can start - * with either a driver letter (for example, C:\), or a UNC name - * (for example, \\guinness\bjorn). - * - * If the name is invalid, the buffer will be modified to point to - * an empty string. - * - * Returns: 1 if the name consists of just the root part, 0 if - * the name was longer. - */ - -static int -extract_root(WCHAR* name) -{ - int len = wcslen(name); - - if (iswalpha(name[0]) && name[1] == L':' && ISSLASH(name[2])) { - WCHAR c = name[3]; - name[3] = L'\0'; - return c == L'\0'; - } else if (len < 5 || !ISSLASH(name[0]) || !ISSLASH(name[1])) { - goto error; - } else { /* Try to find the end of the UNC name. */ - WCHAR* p; - WCHAR c; - - /* - * Find the slash between the server name and share name. - */ - - for (p = name + 2; *p; p++) - if (ISSLASH(*p)) - break; - if (*p == L'\0') - goto error; - - /* - * Find the slash after the share name. - */ - - for (p++; *p; p++) - if (ISSLASH(*p)) - break; - c = *p; - *p = L'\0'; - return c == L'\0' || p[1] == L'\0'; - } - - error: - *name = L'\0'; - return 1; -} - -static unsigned short -dos_to_posix_mode(int attr, const WCHAR *name) -{ - register unsigned short uxmode; - unsigned dosmode; - register const WCHAR *p; - - dosmode = attr & 0xff; - if ((p = name)[1] == L':') - p += 2; - - /* check to see if this is a directory - note we must make a special - * check for the root, which DOS thinks is not a directory - */ - - uxmode = (unsigned short) - (((ISSLASH(*p) && !p[1]) || (dosmode & FILE_ATTRIBUTE_DIRECTORY) || - *p == L'\0') ? _S_IFDIR|_S_IEXEC : _S_IFREG); - - /* If attribute byte does not have read-only bit, it is read-write */ - - uxmode |= (dosmode & FILE_ATTRIBUTE_READONLY) ? - _S_IREAD : (_S_IREAD|_S_IWRITE); - - /* see if file appears to be executable - check extension of name */ - - if (p = wcsrchr(name, L'.')) { - if (!_wcsicmp(p, L".exe") || - !_wcsicmp(p, L".cmd") || - !_wcsicmp(p, L".bat") || - !_wcsicmp(p, L".com")) - uxmode |= _S_IEXEC; - } - - /* propagate user read/write/execute bits to group/other fields */ - - uxmode |= (uxmode & 0700) >> 3; - uxmode |= (uxmode & 0700) >> 6; - - return uxmode; -} - - -int -efile_readlink(Efile_error* errInfo, char* name, char* buffer, size_t size) -{ - Efile_call_state state; - int ret; - DBG_TRACE(1, name); - call_state_init(&state, errInfo); - ret = !!do_readlink(&state, name, buffer, size); - call_state_free(&state); - return ret; -} - -/* If buffer==0, return buffer allocated by wpath_tmp_allocate -*/ -static char* -do_readlink(Efile_call_state* state, char* name, char* buffer, size_t size) -{ - /* - * load dll and see if we have CreateSymbolicLink at runtime: - * (Vista only) - */ - HINSTANCE hModule = NULL; - WCHAR *wname = (WCHAR *) name; - WCHAR *wbuffer = (WCHAR *) buffer; - DWORD wsize = size / sizeof(WCHAR); - char* ret = NULL; - - if ((hModule = LoadLibrary("kernel32.dll")) != NULL) { - typedef DWORD (WINAPI * GETFINALPATHNAMEBYHANDLEPTR)( - HANDLE hFile, - LPCWSTR lpFilePath, - DWORD cchFilePath, - DWORD dwFlags); - - GETFINALPATHNAMEBYHANDLEPTR pGetFinalPathNameByHandle = - (GETFINALPATHNAMEBYHANDLEPTR)GetProcAddress(hModule, "GetFinalPathNameByHandleW"); - - if (pGetFinalPathNameByHandle != NULL) { - DWORD fileAttributes; - ensure_wpath(state, &wname); - /* first check if file is a symlink; {error, einval} otherwise */ - fileAttributes = GetFileAttributesW(wname); - if ((fileAttributes & FILE_ATTRIBUTE_REPARSE_POINT)) { - DWORD success = 0; - HANDLE h = CreateFileW(wname, GENERIC_READ, FILE_SHARE_FLAGS, NULL, OPEN_EXISTING, FILE_FLAG_BACKUP_SEMANTICS, NULL); - int len; - if(h != INVALID_HANDLE_VALUE) { - if (!wbuffer) { /* dynamic allocation */ - WCHAR dummy; - wsize = pGetFinalPathNameByHandle(h, &dummy, 0, 0); - if (wsize) { - wbuffer = wpath_tmp_alloc(state, wsize); - } - } - if (wbuffer - && (success = pGetFinalPathNameByHandle(h, wbuffer, wsize, 0)) - && success < wsize) { - WCHAR* wp; - - /* GetFinalPathNameByHandle prepends path with "\\?\": */ - len = wcslen(wbuffer); - wmemmove(wbuffer,wbuffer+4,len-3); - if (len - 4 >= 2 && wbuffer[1] == L':' && wbuffer[0] >= L'A' && - wbuffer[0] <= L'Z') { - wbuffer[0] = wbuffer[0] + L'a' - L'A'; - } - - for (wp=wbuffer ; *wp; wp++) - if (*wp == L'\\') - *wp = L'/'; - } - CloseHandle(h); - } - if (success) { - ret = (char*) wbuffer; - } else { - set_error(state->errInfo); - } - } else { - errno = EINVAL; - save_last_error(state->errInfo); - } - goto done; - } - } - errno = ENOTSUP; - save_last_error(state->errInfo); - -done: - if (hModule) - FreeLibrary(hModule); - return ret; -} - - -int -efile_altname(Efile_error* errInfo, char* orig_name, char* buffer, size_t size) -{ - Efile_call_state state; - int ret; - DBG_TRACE(1, orig_name); - call_state_init(&state, errInfo); - ret = do_altname(&state, orig_name, buffer, size); - call_state_free(&state); - return ret; -} - -static int -do_altname(Efile_call_state* state, char* orig_name, char* buffer, size_t size) -{ - WIN32_FIND_DATAW wfd; - HANDLE fh; - WCHAR* name; - int name_len; - WCHAR* full_path = NULL; - WCHAR *worig_name = (WCHAR *) orig_name; - WCHAR *wbuffer = (WCHAR *) buffer; - int drive; /* Drive for filename (1 = A:, 2 = B: etc). */ - - /* Don't allow wildcards to be interpreted by system */ - - if (wcspbrk(worig_name, L"?*")) { - enoent: - state->errInfo->posix_errno = ENOENT; - state->errInfo->os_errno = ERROR_FILE_NOT_FOUND; - return 0; - } - - /* - * Move the name to a buffer and make sure to remove a trailing - * slash, because it causes FindFirstFile() to fail on Win95. - */ - ensure_wpath(state, &worig_name); - name_len = wcslen(worig_name); - - name = wpath_tmp_alloc(state, name_len + 1); - wcscpy(name, worig_name); - if (name_len > 2 && ISSLASH(name[name_len-1]) && - name[name_len-2] != L':') { - name[name_len-1] = L'\0'; - } - - /* Try to get disk from name. If none, get current disk. */ - - if (name[1] != L':') { - WCHAR* cwd_path = get_cwd_wpath_tmp(state); - drive = 0; - if (cwd_path[1] == L':') { - drive = towlower(cwd_path[0]) - L'a' + 1; - } - } else if (*name && name[2] == L'\0') { - /* - * X: and nothing more is an error. - */ - goto enoent; - } else { - drive = towlower(*name) - L'a' + 1; - } - fh = FindFirstFileW(name,&wfd); - if (fh == INVALID_HANDLE_VALUE) { - DWORD fff_error = GetLastError(); - if (!(wcspbrk(name, L"./\\") && - (full_path = get_full_wpath_tmp(state, name, NULL, 0)) && - /* root dir. ('C:\') or UNC root dir. ('\\server\share\') */ - ((wcslen(full_path) == 3) || is_root_unc_name(full_path)) && - (GetDriveTypeW(full_path) > 1) ) ) { - - set_os_errno(state->errInfo, fff_error); - return 0; - } - /* - * Root directories (such as C:\ or \\server\share\ are fabricated. - */ - wcscpy(wbuffer,name); - return 1; - } - - wcscpy(wbuffer,wfd.cAlternateFileName); - if (!*wbuffer) { - wcscpy(wbuffer,wfd.cFileName); - } - FindClose(fh); - return 1; -} - - -int -efile_link(Efile_error* errInfo, char* old, char* new) -{ - Efile_call_state state; - WCHAR *wold = (WCHAR *) old; - WCHAR *wnew = (WCHAR *) new; - int ret; - DBG_TRACE(1, old); - call_state_init(&state, errInfo); - ensure_wpath(&state, &wold); - ensure_wpath(&state, &wnew); - if(!CreateHardLinkW(wnew, wold, NULL)) { - ret = set_error(errInfo); - } - else ret =1; - call_state_free(&state); - return ret; -} - -int -efile_symlink(Efile_error* errInfo, char* old, char* new) -{ - Efile_call_state state; - int ret; - DBG_TRACE2(1, "symlink(%s <- %s)", old, new); - call_state_init(&state, errInfo); - ret = do_symlink(&state, old, new); - call_state_free(&state); - return ret; -} - -static int -do_symlink(Efile_call_state* state, char* old, char* new) -{ - /* - * Load dll and see if we have CreateSymbolicLink at runtime: - * (Vista only) - */ - HINSTANCE hModule = NULL; - WCHAR *wold = (WCHAR *) old; - WCHAR *wnew = (WCHAR *) new; - - DBG_TRACE(1, old); - if ((hModule = LoadLibrary("kernel32.dll")) != NULL) { - typedef BOOLEAN (WINAPI * CREATESYMBOLICLINKFUNCPTR) ( - LPCWSTR lpSymlinkFileName, - LPCWSTR lpTargetFileName, - DWORD dwFlags); - - CREATESYMBOLICLINKFUNCPTR pCreateSymbolicLink = - (CREATESYMBOLICLINKFUNCPTR) GetProcAddress(hModule, - "CreateSymbolicLinkW"); - /* A for MBCS, W for UNICODE... char* above implies 'W'! */ - if (pCreateSymbolicLink != NULL) { - ensure_wpath(state, &wold); - ensure_wpath(state, &wnew); - { - DWORD attr = GetFileAttributesW(wold); - int flag = (attr != INVALID_FILE_ATTRIBUTES && - attr & FILE_ATTRIBUTE_DIRECTORY) ? 1 : 0; - /* SYMBOLIC_LINK_FLAG_DIRECTORY = 1 */ - BOOLEAN success = pCreateSymbolicLink(wnew, wold, flag); - FreeLibrary(hModule); - - if (success) { - return 1; - } else { - return set_error(state->errInfo); - } - } - } else - FreeLibrary(hModule); - } - errno = ENOTSUP; - return check_error(-1, state->errInfo); -} - -int -efile_fadvise(Efile_error* errInfo, int fd, Sint64 offset, - Sint64 length, int advise) -{ - DBG_TRACE(2, L""); - /* posix_fadvise is not available on Windows, do nothing */ - errno = ERROR_SUCCESS; - return check_error(0, errInfo); -} - -int -efile_fallocate(Efile_error* errInfo, int fd, Sint64 offset, Sint64 length) -{ - DBG_TRACE(2, L""); - /* No file preallocation method available in Windows. */ - errno = errno_map(ERROR_NOT_SUPPORTED); - SetLastError(ERROR_NOT_SUPPORTED); - - return check_error(-1, errInfo); -} diff --git a/erts/emulator/hipe/hipe_amd64.c b/erts/emulator/hipe/hipe_amd64.c index e3cff4a4ba..f23f341e6d 100644 --- a/erts/emulator/hipe/hipe_amd64.c +++ b/erts/emulator/hipe/hipe_amd64.c @@ -28,6 +28,7 @@ #include "error.h" #include "bif.h" #include "big.h" /* term_to_Sint() */ +#include "erl_binary.h" #include "hipe_arch.h" #include "hipe_bif0.h" @@ -38,6 +39,8 @@ #undef ERL_FUN_SIZE #include "hipe_literals.h" +static void patch_trampoline(void *trampoline, void *destAddress); + const Uint sse2_fnegate_mask[2] = {0x8000000000000000,0}; void hipe_patch_load_fe(Uint64 *address, Uint64 value) @@ -52,9 +55,9 @@ int hipe_patch_insn(void *address, Uint64 value, Eterm type) switch (type) { case am_closure: case am_constant: + case am_c_const: *(Uint64*)address = value; break; - case am_c_const: case am_atom: /* check that value fits in an unsigned imm32 */ /* XXX: are we sure it's not really a signed imm32? */ @@ -71,14 +74,18 @@ int hipe_patch_insn(void *address, Uint64 value, Eterm type) int hipe_patch_call(void *callAddress, void *destAddress, void *trampoline) { - Sint rel32; + Sint64 destOffset = (Sint64)destAddress - (Sint64)callAddress - 4; - ASSERT(trampoline == NULL); + if ((destOffset < -0x80000000L) || (destOffset >= 0x80000000L)) { + destOffset = (Sint64)trampoline - (Sint64)callAddress - 4; - rel32 = (Sint)destAddress - (Sint)callAddress - 4; - if ((Sint)(Sint32)rel32 != rel32) - return -1; - *(Uint32*)callAddress = (Uint32)rel32; + if ((destOffset < -0x80000000L) || (destOffset >= 0x80000000L)) + return -1; + + patch_trampoline(trampoline, destAddress); + } + + *(Uint32*)callAddress = (Uint32)destOffset; hipe_flush_icache_word(callAddress); return 0; } @@ -96,12 +103,80 @@ static void *alloc_code(unsigned int alloc_bytes) return erts_alloc(ERTS_ALC_T_HIPE_EXEC, alloc_bytes); } +static int check_callees(Eterm callees) +{ + Eterm *tuple; + Uint arity; + Uint i; + + if (is_not_tuple(callees)) + return -1; + tuple = tuple_val(callees); + arity = arityval(tuple[0]); + for (i = 1; i <= arity; ++i) { + Eterm mfa = tuple[i]; + if (is_atom(mfa)) + continue; + if (is_not_tuple(mfa) || + tuple_val(mfa)[0] != make_arityval(3) || + is_not_atom(tuple_val(mfa)[1]) || + is_not_atom(tuple_val(mfa)[2]) || + is_not_small(tuple_val(mfa)[3]) || + unsigned_val(tuple_val(mfa)[3]) > 255) + return -1; + } + return arity; +} + +#define TRAMPOLINE_BYTES 12 + +static void generate_trampolines(unsigned char *address, + int nrcallees, Eterm callees, + unsigned char **trampvec) +{ + unsigned char *trampoline = address; + int i; + + for(i = 0; i < nrcallees; ++i) { + trampoline[0] = 0x48; /* movabsq $..., %rax; */ + trampoline[1] = 0xb8; + *(void**)(trampoline+2) = NULL; /* callee's address */ + trampoline[10] = 0xff; /* jmpq *%rax */ + trampoline[11] = 0xe0; + trampvec[i] = trampoline; + trampoline += TRAMPOLINE_BYTES; + } + hipe_flush_icache_range(address, nrcallees*TRAMPOLINE_BYTES); +} + +static void patch_trampoline(void *trampoline, void *destAddress) +{ + unsigned char *tp = (unsigned char*) trampoline; + + ASSERT(tp[0] == 0x48 && tp[1] == 0xb8); + + *(void**)(tp+2) = destAddress; /* callee's address */ + hipe_flush_icache_word(tp+2); +} + void *hipe_alloc_code(Uint nrbytes, Eterm callees, Eterm *trampolines, Process *p) { - if (is_not_nil(callees)) + int nrcallees; + Eterm trampvecbin; + unsigned char **trampvec; + unsigned char *address; + + nrcallees = check_callees(callees); + if (nrcallees < 0) return NULL; - *trampolines = NIL; - return alloc_code(nrbytes); + + trampvecbin = new_binary(p, NULL, nrcallees*sizeof(unsigned char*)); + trampvec = (unsigned char **)binary_bytes(trampvecbin); + + address = alloc_code(nrbytes + nrcallees*TRAMPOLINE_BYTES); + generate_trampolines(address + nrbytes, nrcallees, callees, trampvec); + *trampolines = trampvecbin; + return address; } void hipe_free_code(void* code, unsigned int bytes) @@ -129,10 +204,9 @@ void *hipe_make_native_stub(void *callee_exp, unsigned int beamArity) */ unsigned int codeSize; unsigned char *code, *codep; - unsigned int callEmuOffset; - codeSize = /* 23, 26, 29, or 32 bytes */ - 23 + /* 23 when all offsets are 8-bit */ + codeSize = /* 30, 33, 36, or 39 bytes */ + 30 + /* 30 when all offsets are 8-bit */ (P_CALLEE_EXP >= 128 ? 3 : 0) + ((P_CALLEE_EXP + 4) >= 128 ? 3 : 0) + (P_ARITY >= 128 ? 3 : 0); @@ -197,14 +271,15 @@ void *hipe_make_native_stub(void *callee_exp, unsigned int beamArity) codep[0] = beamArity; codep += 1; - /* jmp callemu; 5 bytes */ - callEmuOffset = (unsigned char*)nbif_callemu - (code + codeSize); - codep[0] = 0xe9; - codep[1] = callEmuOffset & 0xFF; - codep[2] = (callEmuOffset >> 8) & 0xFF; - codep[3] = (callEmuOffset >> 16) & 0xFF; - codep[4] = (callEmuOffset >> 24) & 0xFF; - codep += 5; + /* jmp callemu; 12 bytes */ + codep[0] = 0x48; + codep[1] = 0xb8; + codep += 2; + *(Uint64*)codep = (Uint64)nbif_callemu; + codep += 8; + codep[0] = 0xff; + codep[1] = 0xe0; + codep += 2; ASSERT(codep == code + codeSize); diff --git a/erts/emulator/hipe/hipe_amd64_bifs.m4 b/erts/emulator/hipe/hipe_amd64_bifs.m4 index b3c9a460bb..cf4c59c9af 100644 --- a/erts/emulator/hipe/hipe_amd64_bifs.m4 +++ b/erts/emulator/hipe/hipe_amd64_bifs.m4 @@ -462,6 +462,7 @@ ASYM($1): TYPE_FUNCTION(ASYM($1)) #endif') + /* * noproc_primop_interface_0(nbif_name, cbif_name) * noproc_primop_interface_1(nbif_name, cbif_name) diff --git a/erts/emulator/hipe/hipe_bif0.c b/erts/emulator/hipe/hipe_bif0.c index 380031bf13..a8be64e08d 100644 --- a/erts/emulator/hipe/hipe_bif0.c +++ b/erts/emulator/hipe/hipe_bif0.c @@ -1112,7 +1112,7 @@ static struct hipe_mfa_info* mod2mfa_put(struct hipe_mfa_info* mfa) struct hipe_ref { struct hipe_ref_head head; /* list of refs to same calleee */ void *address; -#if defined(__arm__) || defined(__powerpc__) || defined(__ppc__) || defined(__powerpc64__) +#if defined(__x86_64__) || defined(__arm__) || defined(__powerpc__) || defined(__ppc__) || defined(__powerpc64__) void *trampoline; #endif unsigned int flags; @@ -1543,7 +1543,7 @@ BIF_RETTYPE hipe_bifs_add_ref_2(BIF_ALIST_2) ref = erts_alloc(ERTS_ALC_T_HIPE_LL, sizeof(struct hipe_ref)); ref->address = address; -#if defined(__arm__) || defined(__powerpc__) || defined(__ppc__) || defined(__powerpc64__) +#if defined(__x86_64__) || defined(__arm__) || defined(__powerpc__) || defined(__ppc__) || defined(__powerpc64__) ref->trampoline = trampoline; #endif ref->flags = flags; @@ -1771,7 +1771,8 @@ void hipe_redirect_to_module(Module* modp) struct hipe_mfa_info *p; struct hipe_ref_head* refh; - ERTS_LC_ASSERT(erts_thr_progress_is_blocking()); + ERTS_LC_ASSERT(erts_thr_progress_is_blocking() || + erts_is_multi_scheduling_blocked()); for (p = mod2mfa_get(modp); p; p = p->next_in_mod) { if (p->new_address) { @@ -1818,7 +1819,7 @@ void hipe_redirect_to_module(Module* modp) if (ref->flags & REF_FLAG_IS_LOAD_MFA) res = hipe_patch_insn(ref->address, (Uint)p->remote_address, am_load_mfa); else { -#if defined(__arm__) || defined(__powerpc__) || defined(__ppc__) || defined(__powerpc64__) +#if defined(__x86_64__) || defined(__arm__) || defined(__powerpc__) || defined(__ppc__) || defined(__powerpc64__) void* trampoline = ref->trampoline; #else void* trampoline = NULL; diff --git a/erts/emulator/hipe/hipe_bif0.tab b/erts/emulator/hipe/hipe_bif0.tab index 4038ca7ef8..6728e20123 100644 --- a/erts/emulator/hipe/hipe_bif0.tab +++ b/erts/emulator/hipe/hipe_bif0.tab @@ -109,6 +109,7 @@ atom suspend_0 atom gc_1 atom hipe_apply atom rethrow +atom raw_raise atom find_na_or_make_stub atom nonclosure_address atom atomic_inc @@ -135,8 +136,8 @@ atom bs_utf16_size atom bs_put_utf16be atom bs_put_utf16le atom bs_get_utf16 -atom bs_validate_unicode atom bs_validate_unicode_retract atom emulate_fpe atom emasculate_binary atom is_divisible +atom is_unicode
\ No newline at end of file diff --git a/erts/emulator/hipe/hipe_bif2.c b/erts/emulator/hipe/hipe_bif2.c index 9ebbb22846..df377b2153 100644 --- a/erts/emulator/hipe/hipe_bif2.c +++ b/erts/emulator/hipe/hipe_bif2.c @@ -190,3 +190,8 @@ BIF_RETTYPE hipe_bifs_llvm_fix_pinned_regs_0(BIF_ALIST_0) { BIF_RET(am_ok); } + +BIF_RETTYPE hipe_bifs_build_stacktrace_1(BIF_ALIST_1) +{ + BIF_RET(build_stacktrace(BIF_P, BIF_ARG_1)); +} diff --git a/erts/emulator/hipe/hipe_bif2.tab b/erts/emulator/hipe/hipe_bif2.tab index bbcb577be0..c4da44606a 100644 --- a/erts/emulator/hipe/hipe_bif2.tab +++ b/erts/emulator/hipe/hipe_bif2.tab @@ -32,3 +32,4 @@ bif hipe_bifs:modeswitch_debug_on/0 bif hipe_bifs:modeswitch_debug_off/0 bif hipe_bifs:debug_native_called/2 bif hipe_bifs:llvm_fix_pinned_regs/0 +bif hipe_bifs:build_stacktrace/1 diff --git a/erts/emulator/hipe/hipe_bif_list.m4 b/erts/emulator/hipe/hipe_bif_list.m4 index bebe20a18e..33b3cc1ee5 100644 --- a/erts/emulator/hipe/hipe_bif_list.m4 +++ b/erts/emulator/hipe/hipe_bif_list.m4 @@ -186,6 +186,7 @@ gc_bif_interface_1(nbif_erase_1, erase_1) gc_bif_interface_1(nbif_erts_internal_garbage_collect_1, erts_internal_garbage_collect_1) gc_nofail_primop_interface_1(nbif_gc_1, hipe_gc) gc_bif_interface_2(nbif_put_2, put_2) +gc_bif_interface_2(nbif_hipe_bifs_build_stacktrace, hipe_bifs_build_stacktrace_1) /* * Debug BIFs that need read access to the full state. @@ -219,10 +220,12 @@ standard_bif_interface_1(nbif_bnot_1, bnot_1) standard_bif_interface_1(nbif_set_timeout, hipe_set_timeout) standard_bif_interface_1(nbif_conv_big_to_float, hipe_conv_big_to_float) standard_bif_interface_2(nbif_rethrow, hipe_rethrow) +standard_bif_interface_3(nbif_raw_raise, hipe_raw_raise) standard_bif_interface_3(nbif_find_na_or_make_stub, hipe_find_na_or_make_stub) standard_bif_interface_2(nbif_nonclosure_address, hipe_nonclosure_address) nocons_nofail_primop_interface_0(nbif_fclearerror_error, hipe_fclearerror_error) -standard_bif_interface_2(nbif_is_divisible, hipe_is_divisible) +noproc_primop_interface_2(nbif_is_divisible, hipe_is_divisible) +noproc_primop_interface_1(nbif_is_unicode, hipe_is_unicode) /* * Mbox primops with implicit P parameter. @@ -244,10 +247,9 @@ noproc_primop_interface_2(nbif_eq_2, eq) nofail_primop_interface_3(nbif_bs_get_integer_2, erts_bs_get_integer_2) nofail_primop_interface_3(nbif_bs_get_binary_2, erts_bs_get_binary_2) nofail_primop_interface_3(nbif_bs_get_float_2, erts_bs_get_float_2) -standard_bif_interface_3(nbif_bs_put_utf8, hipe_bs_put_utf8) +nocons_nofail_primop_interface_3(nbif_bs_put_utf8, hipe_bs_put_utf8) standard_bif_interface_3(nbif_bs_put_utf16be, hipe_bs_put_utf16be) standard_bif_interface_3(nbif_bs_put_utf16le, hipe_bs_put_utf16le) -standard_bif_interface_1(nbif_bs_validate_unicode, hipe_bs_validate_unicode) /* * Bit-syntax primops without any P parameter. diff --git a/erts/emulator/hipe/hipe_debug.c b/erts/emulator/hipe/hipe_debug.c index 222a11db3d..929b2a9432 100644 --- a/erts/emulator/hipe/hipe_debug.c +++ b/erts/emulator/hipe/hipe_debug.c @@ -63,12 +63,13 @@ static void print_beam_pc(BeamInstr *pc) printf("normal-process-exit"); } else { ErtsCodeMFA *cmfa = find_function_from_pc(pc); - if (cmfa) + if (cmfa) { + fflush(stdout); erts_printf("%T:%T/%bpu + 0x%bpx", cmfa->module, cmfa->function, cmfa->arity, pc - erts_codemfa_to_code(cmfa)); - else + } else printf("?"); } } @@ -116,6 +117,7 @@ static void print_stack(Eterm *sp, Eterm *end) printf(" | 0x%0*lx | 0x%0*lx | ", 2*(int)sizeof(long), (unsigned long)sp, 2*(int)sizeof(long), (unsigned long)val); + fflush(stdout); erts_printf("%.30T", val); printf("\r\n"); } @@ -126,7 +128,9 @@ static void print_stack(Eterm *sp, Eterm *end) void hipe_print_estack(Process *p) { - printf(" | BEAM STACK |\r\n"); + printf(" | %*s BEAM STACK %*s |\r\n", + 2*(int)sizeof(long)-3, "", + 2*(int)sizeof(long)-4, ""); print_stack(p->stop, STACK_START(p)); } @@ -135,7 +139,9 @@ static void print_heap(Eterm *pos, Eterm *end) printf("From: 0x%0*lx to 0x%0*lx\n\r", 2*(int)sizeof(long), (unsigned long)pos, 2*(int)sizeof(long), (unsigned long)end); - printf(" | H E A P |\r\n"); + printf(" | %*s H E A P %*s |\r\n", + 2*(int)sizeof(long)-1, "", + 2*(int)sizeof(long)-1, ""); printf(" | %*s | %*s |\r\n", 2+2*(int)sizeof(long), "Address", 2+2*(int)sizeof(long), "Contents"); @@ -158,8 +164,10 @@ static void print_heap(Eterm *pos, Eterm *end) ++pos; --ari; } - } else + } else { + fflush(stdout); erts_printf("%.30T", val); + } printf("\r\n"); } printf(" |%s|%s|\r\n", dashes, dashes); @@ -173,11 +181,15 @@ void hipe_print_heap(Process *p) void hipe_print_pcb(Process *p) { printf("P: 0x%0*lx\r\n", 2*(int)sizeof(long), (unsigned long)p); - printf("-----------------------------------------------\r\n"); - printf("Offset| Name | Value | *Value |\r\n"); + printf("%.*s\r\n", + 6+1+13+1+2*(int)sizeof(long)+4+1+2*(int)sizeof(long)+4+1, + "---------------------------------------------------------------"); + printf("Offset| Name | Value %*s | *Value %*s |\r\n", + 2*(int)sizeof(long)-4, "", + 2*(int)sizeof(long)-5, ""); #undef U #define U(n,x) \ - printf(" % 4d | %s | 0x%0*lx | |\r\n", (int)offsetof(Process,x), n, 2*(int)sizeof(long), (unsigned long)p->x) + printf(" % 4d | %s | 0x%0*lx | %*s |\r\n", (int)offsetof(Process,x), n, 2*(int)sizeof(long), (unsigned long)p->x, 2*(int)sizeof(long)+2, "") #undef P #define P(n,x) \ printf(" % 4d | %s | 0x%0*lx | 0x%0*lx |\r\n", (int)offsetof(Process,x), n, 2*(int)sizeof(long), (unsigned long)p->x, 2*(int)sizeof(long), p->x ? (unsigned long)*(p->x) : -1UL) @@ -241,5 +253,7 @@ void hipe_print_pcb(Process *p) #endif /* HIPE */ #undef U #undef P - printf("-----------------------------------------------\r\n"); + printf("%.*s\r\n", + 6+1+14+1+2*(int)sizeof(long)+4+1+2*(int)sizeof(long)+4+1, + "---------------------------------------------------------------"); } diff --git a/erts/emulator/hipe/hipe_gc.c b/erts/emulator/hipe/hipe_gc.c index 1a4a4c7952..aaedba1afd 100644 --- a/erts/emulator/hipe/hipe_gc.c +++ b/erts/emulator/hipe/hipe_gc.c @@ -91,7 +91,7 @@ Eterm *fullsweep_nstack(Process *p, Eterm *n_htop) ASSERT(is_boxed(val)); *nsp_i = val; } else if (!erts_is_literal(gval, ptr)) { - move_boxed(&ptr, val, &n_htop, nsp_i); + move_boxed(ptr, val, &n_htop, nsp_i); } } else if (is_list(gval)) { Eterm *ptr = list_val(gval); @@ -100,7 +100,7 @@ Eterm *fullsweep_nstack(Process *p, Eterm *n_htop) *nsp_i = ptr[1]; } else if (!erts_is_literal(gval, ptr)) { ASSERT(erts_dbg_within_proc(ptr, p, NULL)); - move_cons(&ptr, val, &n_htop, nsp_i); + move_cons(ptr, val, &n_htop, nsp_i); } } } @@ -206,10 +206,10 @@ void gensweep_nstack(Process *p, Eterm **ptr_old_htop, Eterm **ptr_n_htop) ASSERT(is_boxed(val)); *nsp_i = val; } else if (ErtsInArea(ptr, mature, mature_size)) { - move_boxed(&ptr, val, &old_htop, nsp_i); + move_boxed(ptr, val, &old_htop, nsp_i); } else if (ErtsInYoungGen(gval, ptr, oh, oh_size)) { ASSERT(erts_dbg_within_proc(ptr, p, NULL)); - move_boxed(&ptr, val, &n_htop, nsp_i); + move_boxed(ptr, val, &n_htop, nsp_i); } } else if (is_list(gval)) { Eterm *ptr = list_val(gval); @@ -217,10 +217,10 @@ void gensweep_nstack(Process *p, Eterm **ptr_old_htop, Eterm **ptr_n_htop) if (IS_MOVED_CONS(val)) { *nsp_i = ptr[1]; } else if (ErtsInArea(ptr, mature, mature_size)) { - move_cons(&ptr, val, &old_htop, nsp_i); + move_cons(ptr, val, &old_htop, nsp_i); } else if (ErtsInYoungGen(gval, ptr, oh, oh_size)) { ASSERT(erts_dbg_within_proc(ptr, p, NULL)); - move_cons(&ptr, val, &n_htop, nsp_i); + move_cons(ptr, val, &n_htop, nsp_i); } } } @@ -278,7 +278,7 @@ Eterm *sweep_literals_nstack(Process *p, Eterm *old_htop, char *area, ASSERT(is_boxed(val)); *nsp_i = val; } else if (ErtsInArea(ptr, area, area_size)) { - move_boxed(&ptr, val, &old_htop, nsp_i); + move_boxed(ptr, val, &old_htop, nsp_i); } } else if (is_list(gval)) { Eterm *ptr = list_val(gval); @@ -286,7 +286,7 @@ Eterm *sweep_literals_nstack(Process *p, Eterm *old_htop, char *area, if (IS_MOVED_CONS(val)) { *nsp_i = ptr[1]; } else if (ErtsInArea(ptr, area, area_size)) { - move_cons(&ptr, val, &old_htop, nsp_i); + move_cons(ptr, val, &old_htop, nsp_i); } } } diff --git a/erts/emulator/hipe/hipe_mkliterals.c b/erts/emulator/hipe/hipe_mkliterals.c index 6ea120c65c..3323e8640b 100644 --- a/erts/emulator/hipe/hipe_mkliterals.c +++ b/erts/emulator/hipe/hipe_mkliterals.c @@ -462,7 +462,15 @@ static const struct rts_param rts_params[] = { 0 #endif }, - /* This parameter is always defined, but its value depends on ERTS_SMP. */ + /* This flag is always defined, but its value is configuration-dependent. */ + { 17, "ERTS_USE_LITERAL_TAG", + 1, +#if defined(TAG_LITERAL_PTR) + 1 +#else + 0 +#endif + }, { 19, "MSG_MESSAGE", 1, offsetof(struct erl_mesg, m[0]) }, @@ -522,6 +530,9 @@ static const struct rts_param rts_params[] = { 1, offsetof(struct process, hipe.gc_is_unsafe) #endif }, + + { 54, "P_MSG_LAST", 1, offsetof(struct process, msg.last) }, + { 55, "P_MSG_SAVED_LAST", 1, offsetof(struct process, msg.saved_last) }, }; #define NR_PARAMS ARRAY_SIZE(rts_params) @@ -529,6 +540,11 @@ static const struct rts_param rts_params[] = { static unsigned int literals_crc; static unsigned int system_crc; +/* + * Change this version value to detect incompatible changes in primop interface. + */ +#define PRIMOP_ABI_VSN 0x090300 /* erts-9.3 */ + static void compute_crc(void) { unsigned int crc_value; @@ -544,6 +560,8 @@ static void compute_crc(void) for (i = 0; i < NR_PARAMS; ++i) if (rts_params[i].is_defined) crc_value = crc_update_int(crc_value, &rts_params[i].value); + + crc_value ^= PRIMOP_ABI_VSN; crc_value &= 0x07FFFFFF; system_crc = crc_value; } diff --git a/erts/emulator/hipe/hipe_native_bif.c b/erts/emulator/hipe/hipe_native_bif.c index 23f64a6991..498b43ac6b 100644 --- a/erts/emulator/hipe/hipe_native_bif.c +++ b/erts/emulator/hipe/hipe_native_bif.c @@ -254,6 +254,8 @@ void hipe_handle_exception(Process *c_p) /* Synthesized to avoid having to generate code for it. */ c_p->def_arg_reg[0] = exception_tag[GET_EXC_CLASS(c_p->freason)]; + c_p->msg.saved_last = 0; /* No longer safe to use this position */ + hipe_find_handler(c_p); } @@ -312,6 +314,32 @@ BIF_RETTYPE nbif_impl_hipe_rethrow(NBIF_ALIST_2) } } +/* Called via standard_bif_interface_3 */ +BIF_RETTYPE nbif_impl_hipe_raw_raise(NBIF_ALIST_3) +{ + Process *c_p = BIF_P; + Eterm class = BIF_ARG_1; + Eterm value = BIF_ARG_2; + Eterm stacktrace = BIF_ARG_3; + Eterm reason; + + if (class == am_error) { + c_p->fvalue = value; + reason = EXC_ERROR; + } else if (class == am_exit) { + c_p->fvalue = value; + reason = EXC_EXIT; + } else if (class == am_throw) { + c_p->fvalue = value; + reason = EXC_THROWN; + } else { + return am_badarg; + } + reason &= ~EXF_SAVETRACE; + c_p->ftrace = stacktrace; + BIF_ERROR(c_p, reason); +} + /* * Support for compiled binary syntax operations. */ @@ -394,12 +422,8 @@ Eterm hipe_bs_utf8_size(Eterm arg) return make_small(4); } -BIF_RETTYPE nbif_impl_hipe_bs_put_utf8(NBIF_ALIST_3) +Eterm hipe_bs_put_utf8(Process* p, Eterm arg, byte* base, Uint offset) { - Process* p = BIF_P; - Eterm arg = BIF_ARG_1; - byte* base = (byte*) BIF_ARG_2; - Uint offset = (Uint) BIF_ARG_3; byte *save_bin_buf; Uint save_bin_offset; int res; @@ -415,7 +439,8 @@ BIF_RETTYPE nbif_impl_hipe_bs_put_utf8(NBIF_ALIST_3) erts_current_bin = save_bin_buf; erts_bin_offset = save_bin_offset; if (res == 0) - BIF_ERROR(p, BADARG); + return 0; + ASSERT(new_offset != 0); return new_offset; } @@ -482,15 +507,12 @@ static int validate_unicode(Eterm arg) return 1; } -BIF_RETTYPE nbif_impl_hipe_bs_validate_unicode(NBIF_ALIST_1) +Uint hipe_is_unicode(Eterm arg) { - Process *p = BIF_P; - Eterm arg = BIF_ARG_1; - if (!validate_unicode(arg)) - BIF_ERROR(p, BADARG); - return NIL; + return (Uint) validate_unicode(arg); } + int hipe_bs_validate_unicode_retract(ErlBinMatchBuffer* mb, Eterm arg) { if (!validate_unicode(arg)) { @@ -500,16 +522,12 @@ int hipe_bs_validate_unicode_retract(ErlBinMatchBuffer* mb, Eterm arg) return 1; } -/* Called via standard_bif_interface_2 */ -BIF_RETTYPE nbif_impl_hipe_is_divisible(NBIF_ALIST_2) +Uint hipe_is_divisible(Uint dividend, Uint divisor) { - /* Arguments are Eterm-sized unsigned integers */ - Uint dividend = BIF_ARG_1; - Uint divisor = BIF_ARG_2; if (dividend % divisor) { - BIF_ERROR(BIF_P, BADARG); + return 0; } else { - return NIL; + return 1; } } diff --git a/erts/emulator/hipe/hipe_native_bif.h b/erts/emulator/hipe/hipe_native_bif.h index cbc7ab8dc6..ba42b126be 100644 --- a/erts/emulator/hipe/hipe_native_bif.h +++ b/erts/emulator/hipe/hipe_native_bif.h @@ -36,6 +36,7 @@ AEXTERN(int,nbif_suspend_msg,(void)); AEXTERN(int,nbif_suspend_msg_timeout,(void)); AEXTERN(Eterm,nbif_rethrow,(Process*, Eterm, Eterm)); +AEXTERN(Eterm,nbif_raw_raise,(Process*, Eterm, Eterm, Eterm)); AEXTERN(Eterm,nbif_set_timeout,(Process*, Eterm)); AEXTERN(Eterm,nbif_gc_1,(void)); @@ -66,9 +67,9 @@ AEXTERN(Eterm,nbif_bs_utf16_size,(Eterm)); AEXTERN(Eterm,nbif_bs_put_utf16be,(Process*,Eterm,byte*,unsigned int)); AEXTERN(Eterm,nbif_bs_put_utf16le,(Process*,Eterm,byte*,unsigned int)); AEXTERN(Eterm,nbif_bs_get_utf16,(void)); -AEXTERN(Eterm,nbif_bs_validate_unicode,(Process*,Eterm)); +AEXTERN(Uint,nbif_is_unicode,(Eterm)); AEXTERN(Eterm,nbif_bs_validate_unicode_retract,(void)); -AEXTERN(void,nbif_is_divisible,(Process*,Uint,Uint)); +AEXTERN(Uint,nbif_is_divisible,(Uint,Uint)); AEXTERN(void,nbif_select_msg,(Process*)); AEXTERN(Eterm,nbif_cmp_2,(void)); @@ -82,19 +83,20 @@ void hipe_gc(Process*, Eterm); BIF_RETTYPE nbif_impl_hipe_set_timeout(NBIF_ALIST_1); void hipe_handle_exception(Process*); BIF_RETTYPE nbif_impl_hipe_rethrow(NBIF_ALIST_2); +BIF_RETTYPE nbif_impl_hipe_raw_raise(NBIF_ALIST_3); char *hipe_bs_allocate(int); Binary *hipe_bs_reallocate(Binary*, int); int hipe_bs_put_small_float(Process*, Eterm, Uint, byte*, unsigned, unsigned); void hipe_bs_put_bits(Eterm, Uint, byte*, unsigned, unsigned); Eterm hipe_bs_utf8_size(Eterm); -BIF_RETTYPE nbif_impl_hipe_bs_put_utf8(NBIF_ALIST_3); +Eterm hipe_bs_put_utf8(Process*, Eterm arg, byte* base, Uint offset); Eterm hipe_bs_utf16_size(Eterm); BIF_RETTYPE nbif_impl_hipe_bs_put_utf16be(NBIF_ALIST_3); BIF_RETTYPE nbif_impl_hipe_bs_put_utf16le(NBIF_ALIST_3); -BIF_RETTYPE nbif_impl_hipe_bs_validate_unicode(NBIF_ALIST_1); +Uint hipe_is_unicode(Eterm); struct erl_bin_match_buffer; int hipe_bs_validate_unicode_retract(struct erl_bin_match_buffer*, Eterm); -BIF_RETTYPE nbif_impl_hipe_is_divisible(NBIF_ALIST_2); +Uint hipe_is_divisible(Uint, Uint); #ifdef NO_FPE_SIGNALS AEXTERN(void,nbif_emulate_fpe,(Process*)); @@ -104,6 +106,9 @@ void hipe_emulate_fpe(Process*); AEXTERN(void,nbif_emasculate_binary,(Eterm)); void hipe_emasculate_binary(Eterm); +AEXTERN(BIF_RETTYPE,nbif_hipe_bifs_build_stacktrace,(Process*,Eterm)); +BIF_RETTYPE hipe_bifs_build_stacktrace_1(BIF_ALIST_1); + /* * Stuff that is different in SMP and non-SMP. */ diff --git a/erts/emulator/hipe/hipe_primops.h b/erts/emulator/hipe/hipe_primops.h index 6aac5e6205..c5f10672f3 100644 --- a/erts/emulator/hipe/hipe_primops.h +++ b/erts/emulator/hipe/hipe_primops.h @@ -46,6 +46,7 @@ PRIMOP_LIST(am_clear_timeout, &nbif_clear_timeout) PRIMOP_LIST(am_select_msg, &nbif_select_msg) PRIMOP_LIST(am_set_timeout, &nbif_set_timeout) PRIMOP_LIST(am_rethrow, &nbif_rethrow) +PRIMOP_LIST(am_raw_raise, &nbif_raw_raise) PRIMOP_LIST(am_bs_get_integer_2, &nbif_bs_get_integer_2) @@ -63,7 +64,7 @@ PRIMOP_LIST(am_bs_utf16_size, &nbif_bs_utf16_size) PRIMOP_LIST(am_bs_put_utf16be, &nbif_bs_put_utf16be) PRIMOP_LIST(am_bs_put_utf16le, &nbif_bs_put_utf16le) PRIMOP_LIST(am_bs_get_utf16, &nbif_bs_get_utf16) -PRIMOP_LIST(am_bs_validate_unicode, &nbif_bs_validate_unicode) +PRIMOP_LIST(am_is_unicode, &nbif_is_unicode) PRIMOP_LIST(am_bs_validate_unicode_retract, &nbif_bs_validate_unicode_retract) PRIMOP_LIST(am_is_divisible, &nbif_is_divisible) @@ -83,6 +84,7 @@ PRIMOP_LIST(am_emulate_fpe, &nbif_emulate_fpe) #endif PRIMOP_LIST(am_emasculate_binary, &nbif_emasculate_binary) PRIMOP_LIST(am_debug_native_called, &nbif_hipe_bifs_debug_native_called) +PRIMOP_LIST(am_build_stacktrace, &nbif_hipe_bifs_build_stacktrace) #if defined(__sparc__) #include "hipe_sparc_primops.h" diff --git a/erts/emulator/hipe/hipe_risc_stack.c b/erts/emulator/hipe/hipe_risc_stack.c index 4001bedeb6..bb93a918a2 100644 --- a/erts/emulator/hipe/hipe_risc_stack.c +++ b/erts/emulator/hipe/hipe_risc_stack.c @@ -47,8 +47,10 @@ static void print_slot(Eterm *sp, unsigned int live) printf(" | 0x%0*lx | 0x%0*lx | ", 2*(int)sizeof(long), (unsigned long)sp, 2*(int)sizeof(long), val); - if (live) + if (live) { + fflush(stdout); erts_printf("%.30T", val); + } printf("\r\n"); } @@ -68,7 +70,9 @@ void hipe_print_nstack(Process *p) [0 ... 2*sizeof(long)+3] = '-' }; - printf(" | NATIVE STACK |\r\n"); + printf(" | %*s NATIVE STACK %*s |\r\n", + 2*(int)sizeof(long)-5, "", + 2*(int)sizeof(long)-4, ""); printf(" |%s|%s|\r\n", dashes, dashes); printf(" | %*s | 0x%0*lx |\r\n", 2+2*(int)sizeof(long), "heap", diff --git a/erts/emulator/hipe/hipe_x86_stack.c b/erts/emulator/hipe/hipe_x86_stack.c index 31582b3a2e..615e07917a 100644 --- a/erts/emulator/hipe/hipe_x86_stack.c +++ b/erts/emulator/hipe/hipe_x86_stack.c @@ -43,8 +43,10 @@ static void print_slot(Eterm *sp, unsigned int live) printf(" | 0x%0*lx | 0x%0*lx | ", 2*(int)sizeof(long), (unsigned long)sp, 2*(int)sizeof(long), val); - if (live) + if (live) { + fflush(stdout); erts_printf("%.30T", val); + } printf("\r\n"); } @@ -74,7 +76,9 @@ void hipe_print_nstack(Process *p) sdesc0.livebits[0] = ~1; sdesc = &sdesc0; - printf(" | NATIVE STACK |\r\n"); + printf(" | %*s NATIVE STACK %*s |\r\n", + 2*(int)sizeof(long)-5, "", + 2*(int)sizeof(long)-4, ""); printf(" |%s|%s|\r\n", dashes, dashes); printf(" | %*s | 0x%0*lx |\r\n", 2+2*(int)sizeof(long), "heap", diff --git a/erts/emulator/internal_doc/CarrierMigration.md b/erts/emulator/internal_doc/CarrierMigration.md index 2a9594db25..3a796d11b7 100644 --- a/erts/emulator/internal_doc/CarrierMigration.md +++ b/erts/emulator/internal_doc/CarrierMigration.md @@ -3,17 +3,17 @@ Carrier Migration The ERTS memory allocators manage memory blocks in two types of raw memory chunks. We call these chunks of raw memory -*carriers*. Singleblock carriers which only contain one large block, -and multiblock carriers which contain multiple blocks. A carrier is +*carriers*. Single-block carriers which only contain one large block, +and multi-block carriers which contain multiple blocks. A carrier is typically created using `mmap()` on unix systems. However, how a carrier is created is of minor importance. An allocator instance -typically manages a mixture of single- and multiblock carriers. +typically manages a mixture of single- and multi-block carriers. Problem ------- When a carrier is empty, i.e. contains only one large free block, it -is deallocated. Since multiblock carriers can contain both allocated +is deallocated. Since multi-block carriers can contain both allocated blocks and free blocks at the same time, an allocator instance might be stuck with a large amount of poorly utilized carriers if the memory load decreases. After a peak in memory usage it is expected that not @@ -23,9 +23,9 @@ can usually be reused if the memory load increases again. However, since each scheduler thread manages its own set of allocator instances, and memory load is not necessarily correlated to CPU load, we might get into a situation where there are lots of poorly utilized -multiblock carriers on some allocator instances while we need to -allocate new multiblock carriers on other allocator instances. In -scenarios like this, the demand for multiblock carriers in the system +multi-block carriers on some allocator instances while we need to +allocate new multi-block carriers on other allocator instances. In +scenarios like this, the demand for multi-block carriers in the system might increase at the same time as the actual memory demand in the system has decreased which is both unwanted and quite unexpected for the end user. @@ -34,7 +34,7 @@ Solution -------- In order to prevent scenarios like this we've implemented support for -migration of multiblock carriers between allocator instances of the +migration of multi-block carriers between allocator instances of the same type. ### Management of Free Blocks ### @@ -44,7 +44,7 @@ and add it to another we need to be able to move references to the free blocks of the carrier between the allocator instances. The allocator instance specific data structure referring to the free blocks it manages often refers to the same carrier from multiple -places. For example, when the address order bestfit strategy is used +places. For example, when the address order best-fit strategy is used this data structure is a binary search tree spanning all carriers that the allocator instance manages. Free blocks in one specific carrier can be referred to from potentially every other carrier that is @@ -135,7 +135,7 @@ carriers between scheduler specific allocator instances of the same allocator type. Each allocator instance keeps track of the current utilization of its -multiblock carriers. When the total utilization falls below the "abandon +multi-block carriers. When the total utilization falls below the "abandon carrier utilization limit" it starts to inspect the utilization of the current carrier when deallocations are made. If also the utilization of the carrier falls below the "abandon carrier utilization limit" it @@ -144,31 +144,45 @@ and inserts the carrier into the pool. Since the carrier has been unlinked from the data structure of available free blocks, no more allocations will be made in the -carrier. The allocator instance putting the carrier into the pool, -however, still has the responsibility of performing deallocations in -it while it remains in the pool. The allocator instance with this -deallocation responsibility is here called the **employer**. - -Each carrier has a flag field containing information about the -employing allocator instance, a flag indicating if the carrier is in -the pool or not, and a flag indicating if it is busy or not. When the -carrier is in the pool, the employing allocator instance needs to mark it -as busy while operating on it. If another thread inspects it in order -to try to fetch it from the pool, it will skip it if it is busy. When -fetching the carrier from the pool, employment will change and further +carrier. + +The allocator instance that created a carrier is called its **owner**. +Ownership never changes. + +The allocator instance that has the responsibility to perform deallocations in a +carrier is called its **employer**. The employer may also perform allocations if +the carrier is not in the pool. Employment may change when a carrier is fetched from +or inserted into the pool. + +Deallocations in a carrier, while it remains in the pool, is always performed +the owner. That is, all pooled carriers are employed by their owners. + +Each carrier has an atomic word containing a pointer to the employing allocator +instance and three bit flags; IN_POOL, BUSY and HOMECOMING. + +When fetching a carrier from the pool, employment may change and further deallocations in the carrier will be redirected to the new employer using the delayed dealloc functionality. -If a carrier in the pool becomes empty, it will be withdrawn from the -pool. All carriers that become empty are also always passed to its -**owning** allocator instance for deallocation using the delayed -dealloc functionality. Since carriers this way always will be -deallocated by the owner that allocated the carrier, the +When a foreign allocator instance abandons a carrier back into the pool, it will +also pass it back to its **owner** using the delayed dealloc queue. When doing +this it will set the HOMECOMING bit flag to mark it as "enqueued". The owner +will later clear the HOMECOMING bit when the carrier is dequeued. This mechanism +prevents a carrier from being enqueued again before it has been dequeued. + +When a carrier becomes empty, it will be deallocated. Carrier deallocation is +always done by the owner that allocated the carrier. By doing this, the underlying functionality of allocating and deallocating carriers can remain simple and doesn't have to bother about multiple threads. In a NUMA system we will also not mix carriers originating from multiple NUMA nodes. +If a carrier in the pool becomes empty, it will be withdrawn from the +pool and be deallocated by the owner which already employs it. + +If a carrier employed by a foreign allocator becomes empty, it will be passed +back to the owner for deallocation using the delayed dealloc functionality. + In short: * The allocator instance that created a carrier **owns** it. @@ -177,34 +191,31 @@ In short: * The allocator instance that uses a carrier **employs** it. * An **employer** can abandon a carrier into the pool. * Pooled carriers are not allocated from. -* Deallocation in a pooled carrier is still performed by its **employer**. -* **Employment** can only change when a carrier is fetched from the pool. +* Pooled carriers are always **employed** by their **owner**. +* **Employment** can only change from **owner** to a foreign allocator + when a carrier is fetched from the pool. + ### Searching the pool ### +When an allocator instance needs more carrier space, it inspects the pool. If no +carrier could be fetched from the pool, it will allocate a new +carrier. Regardless of where the allocator instance gets the carrier from, it +just links in the carrier into its data structure of free blocks. + To harbor real time characteristics, searching the pool is limited. We only inspect a limited number of carriers. If none of those carriers had a free block large enough to satisfy the allocation -request, the search will fail. A carrier in the pool can also be busy +request, the search will fail. A carrier in the pool can also be BUSY if another thread is currently doing block deallocation work on the -carrier. A busy carrier will also be skipped by the search as it can +carrier. A BUSY carrier will also be skipped by the search as it can not satisfy the request. The pool is lock-free and we do not want to block, waiting for the other thread to finish. -#### Before OTP 17.4 #### +### The bad cluster problem ### -When an allocator instance needs more carrier space, it always begins -by inspecting its own carriers that are waiting for thread progress -before they can be deallocated. If no such carrier could be found, it -then inspects the pool. If no carrier could be fetched from the pool, -it will allocate a new carrier. Regardless of where the allocator -instance gets the carrier from it the just links in the carrier into -its data structure of free blocks. - -#### After OTP 17.4 #### - -The old search algorithm had a problem as the search always started at -the same position in the pool, the sentinel. This could lead to +Before OTP-17.4 the search algorithm had a problem as the search always started +at the same position in the pool, the sentinel. This could lead to contention from concurrent searching processes. But even worse, it could lead to a "bad" state when searches fail with a high rate leading to new carriers instead being allocated. These new carriers @@ -236,26 +247,27 @@ The result is that we prefer carriers created by the thread itself, which is good for NUMA performance. And we get more entry points when searching the pool, which will ease contention and clustering. +### Our own pooled tree ### + To do the first search among own carriers, every allocator instance -has two new lists: `pooled_list` and `traitor_list`. These lists are only -accessed by the allocator itself and they only contain the allocator's -own carriers. When an owned carrier is abandoned and put in the -pool, it is also linked into `pooled_list`. When we search our -`pooled_list` and find a carrier that is no longer in the pool, we -move that carrier from `pooled_list` to `traitor_list` as it is now -employed by another allocator. If searching `pooled_list` fails, we -also do a limited search of `traitor_list`. When finding an abandoned -carrier in `traitor_list` it is either employed or moved back to -`pooled_list` if it could not satisfy the allocation request. - -When searching `pooled_list` and `traitor_list` we always start at the -point where the last search ended. This to avoid clustering -problems and increase the probability to find a "good" carrier. As -`pooled_list` and `traitor_list` are only accessed by the owning -allocator instance, they need no thread synchronization at all. +has a `pooled_tree` of carriers. This tree is only accessed by the allocator +itself and can only contain its own carriers. When a carrier is +abandoned and put in the pool, it is also inserted into `pooled_tree`. This is +either done direct, if the carrier was already employed by its owner, or by +first passing it back to the owner via the delayed dealloc queue. + +When we search our `pooled_tree` and find a carrier that is no longer in the +pool, we remove that carrier from `pooled_tree` and mark it as TRAITOR, as it is +now employed by a foreign allocator. We will not find any carriers in +`pooled_tree` that are marked as BUSY by other threads. + +If no carrier in `pooled_tree` had a large enough free block, we search it again +to find any carrier that may act as an entry point into the shared list of all +pooled carriers. This in order to, if possible, avoid starting at the sentinel +and thereby ease the "bad clustering" problem. Furthermore, the search for own carriers that are scheduled -for deallocation is now done as the last search option. The idea is +for deallocation is done as the last search option. The idea is that it is better to reuse a poorly utilized carrier than to resurrect an empty carrier that was just about to be released back to the OS. @@ -271,14 +283,14 @@ load did not. When using the `aoffcaobf` or `aoff` strategies compared to `gf` or `bf`, we loose some performance since we get more modifications in the data structure of free blocks. This performance penalty is however -reduced using the `aoffcbf` strategy. A tradeoff between memory +reduced using the `aoffcbf` strategy. A trade off between memory consumption and performance is however inevitable, and it is up to the user to decide what is most important. Further work ------------ -It would be quite easy to extend this to allow migration of multiblock +It would be quite easy to extend this to allow migration of multi-block carriers between all allocator types. More or less the only obstacle is maintenance of the statistics information. diff --git a/erts/emulator/internal_doc/beam_makeops.md b/erts/emulator/internal_doc/beam_makeops.md new file mode 100644 index 0000000000..1da8d2ab05 --- /dev/null +++ b/erts/emulator/internal_doc/beam_makeops.md @@ -0,0 +1,1846 @@ +The beam\_makeops script +======================= + +This document describes the **beam\_makeops** script. + +Introduction +------------ + +The **beam\_makeops** Perl script is used at build-time by both the +compiler and runtime system. Given a number of input files (all with +the extension `.tab`), it will generate source files used by the +Erlang compiler and by the runtime system to load and execute BEAM +instructions. + +Essentially those `.tab` files define: + +* External generic BEAM instructions. They are the instructions that +are known to both the compiler and the runtime system. Generic +instructions are stable between releases. New generic instructions +with high numbers than previous instructions can be added in major +releases. The OTP 20 release has 159 external generic instructions. + +* Internal generic instructions. They are known only to the runtime +system and can be changed at any time without compatibility issues. +They are created by transformation rules (described next). + +* Rules for transforming one or more generic instructions to other +generic instructions. The transformation rules allow combining, +splitting, and removal of instructions, as well as shuffling operands. +Because of the transformation rules, the runtime can have many +internal generic instructions that are only known to runtime system. + +* Specific BEAM instructions. The specific instructions are the +instructions that are actually executed by the runtime system. They +can be changed at any time without causing compatibility issues. +The loader translates generic instructions to specific instructions. +In general, for each generic instruction, there exists a family of +specific instructions. The OTP 20 release has 389 specific +instructions. + +* The implementation of specific instructions. + +Generic instructions have typed operands. Here are a few examples of +operands for `move/2`: + + {move,{atom,id},{x,5}}. + {move,{x,3},{x,0}}. + {move,{x,2},{y,1}}. + +When those instructions are loaded, the loader rewrites them +to specific instructions: + + move_cx id 5 + move_xx 3 0 + move_xy 2 1 + +Corresponding to each generic instruction, there is a family of +specific instructions. The types that an instance of a specific +instruction can handle are encoded in the instruction names. For +example, `move_xy` takes an X register number as the first operand and +a Y register number as the second operand. `move_cx` takes a tagged +Erlang term as the first operand and an X register number as the +second operand. + +An example: the move instruction +-------------------------------- + +Using the `move` instruction as an example, we will give a quick +tour to show the main features of **beam\_makeops**. + +In the `compiler` application, in the file `genop.tab`, there is the +following line: + + 64: move/2 + +This is a definition of an external generic BEAM instruction. Most +importantly it specifices that the opcode is 64. It also defines that +it has two operands. The BEAM assembler will use the opcode when +creating `.beam` files. The compiler does not really need the arity, +but it will use it as an internal sanity check when assembling the +BEAM code. + +Let's have a look at `ops.tab` in `erts/emulator/beam`, where the +specific `move` instructions are defined. Here are a few of them: + + move x x + move x y + move c x + +Each specific instructions is defined by following the name of the +instruction with the types for each operand. An operand type is a +single letter. For example, `x` means an X register, `y` +means a Y register, and `c` is a "constant" (a tagged term such as +an integer, an atom, or a literal). + +Now let's look at the implementation of the `move` instruction. There +are multiple files containing implementations of instructions in the +`erts/emulator/beam` directory. The `move` instruction is defined in +`instrs.tab`. It looks like this: + + move(Src, Dst) { + $Dst = $Src; + } + +The implementation for an instruction largely follows the C syntax, +except that the variables in the function head don't have any types. +The `$` before an identifier denotes a macro expansion. Thus, +`$Src` will expand to the code to pick up the source operand for +the instruction and `$Dst` to the code for the destination register. + +We will look at the code for each specific instruction in turn. To +make the code easier to understand, let's first look at the memory +layout for the instruction `{move,{atom,id},{x,5}}`: + + +--------------------+--------------------+ + I -> | 40 | &&lb_move_cx | + +--------------------+--------------------+ + | Tagged atom 'id' | + +--------------------+--------------------+ + +This example and all other examples in the document assumes a 64-bit +archictecture, and furthermore that pointers to C code fit in 32 bits. + +`I` in the BEAM virtual machine is the instruction pointer. When BEAM +executes an instruction, `I` points to the first word of the +instruction. + +`&&lb_move_cx` is the address to C code that implements `move_cx`. It +is stored in the lower 32 bits of the word. In the upper 32 bits is +the byte offset to the X register; the register number 5 has been +multiplied by the word size size 8. + +In the next word the tagged atom `id` is stored. + +With that background, we can look at the generated code for `move_cx` +in `beam_hot.h`: + + OpCase(move_cx): + { + BeamInstr next_pf = BeamCodeAddr(I[2]); + xb(BeamExtraData(I[0])) = I[1]; + I += 2; + ASSERT(VALID_INSTR(next_pf)); + GotoPF(next_pf); + } + +We will go through each line in turn. + +* `OpCase(move_cx):` defines a label for the instruction. The +`OpCase()` macro is defined in `beam_emu.c`. It will expand this line +to `lb_move_cx:`. + +* `BeamInstr next_pf = BeamCodeAddr(I[2]);` fetches the pointer to +code for the next instruction to be executed. The `BeamCodeAddr()` +macro extracts the pointer from the lower 32 bits of the instruction +word. + +* `xb(BeamExtraData(I[0])) = I[1];` is the expansion of `$Dst = $Src`. +`BeamExtraData()` is a macro that will extract the upper 32 bits from +the instruction word. In this example, it will return 40 which is the +byte offset for X register 5. The `xb()` macro will cast a byte +pointer to an `Eterm` pointer and dereference it. The `I[1]` on +the right side of the `=` fetches an Erlang term (the atom `id` in +this case). + +* `I += 2` advances the instruction pointer to the next +instruction. + +* In a debug-compiled emulator, `ASSERT(VALID_INSTR(next_pf));` makes +sure that `next_pf` is a valid instruction (that is, that it points +within the `process_main()` function in `beam_emu.c`). + +* `GotoPF(next_pf);` transfers control to the next instruction. + +Now let's look at the implementation of `move_xx`: + + OpCase(move_xx): + { + Eterm tmp_packed1 = BeamExtraData(I[0]); + BeamInstr next_pf = BeamCodeAddr(I[1]); + xb((tmp_packed1>>BEAM_TIGHT_SHIFT)) = xb(tmp_packed1&BEAM_TIGHT_MASK); + I += 1; + ASSERT(VALID_INSTR(next_pf)); + GotoPF(next_pf); + } + +We will go through the lines that are new or have changed compared to +`move_cx`. + +* `Eterm tmp_packed1 = BeamExtraData(I[0]);` picks up both X register +numbers packed into the upper 32 bits of the instruction word. + +* `BeamInstr next_pf = BeamCodeAddr(I[1]);` pre-fetches the address of +the next instruction. Note that because both X registers operands fits +into the instruction word, the next instruction is in the very next +word. + +* `xb((tmp_packed1>>BEAM_TIGHT_SHIFT)) = xb(tmp_packed1&BEAM_TIGHT_MASK);` +copies the source to the destination. (For a 64-bit architecture, +`BEAM_TIGHT_SHIFT` is 16 and `BEAM_TIGHT_MASK` is `0xFFFF`.) + +* `I += 1;` advances the instruction pointer to the next instruction. + +`move_xy` is almost identical to `move_xx`. The only difference is +the use of the `yb()` macro instead of `xb()` to reference the +destination register: + + OpCase(move_xy): + { + Eterm tmp_packed1 = BeamExtraData(I[0]); + BeamInstr next_pf = BeamCodeAddr(I[1]); + yb((tmp_packed1>>BEAM_TIGHT_SHIFT)) = xb(tmp_packed1&BEAM_TIGHT_MASK); + I += 1; + ASSERT(VALID_INSTR(next_pf)); + GotoPF(next_pf); + } + +### Transformation rules ### + +Next let's look at how we can do some optimizations using transformation +rules. For simple instructions such as `move/2`, the instruction dispatch +overhead can be substantial. A simple optimization is to combine common +instructions sequences to a single instruction. One such common sequence +is multiple `move` instructions moving X registers to Y registers. + +Using the following rule we can combine two `move` instructions +to a `move2` instruction: + + move X1=x Y1=y | move X2=x Y2=y => move2 X1 Y1 X2 Y2 + +The left side of the arrow (`=>`) is a pattern. If the pattern +matches, the matching instructions will be replaced by the +instructions on the right side. Variables in a pattern must start +with an uppercase letter just as in Erlang. A pattern variable may be +followed `=` and one or more type letters to constrain the match to +one of those types. The variables that are bound on the left side can +be used on the right side. + +We will also need to define a specific instruction and an implementation: + + # In ops.tab + move2 x y x y + + // In instrs.tab + move2(S1, D1, S2, D2) { + Eterm V1, V2; + V1 = $S1; + V2 = $S2; + $D1 = V1; + $D2 = V2; + } + +When the loader has found a match and replaced the matched instructions, +it will match the new instructions against the transformation rules. +Because of that, we can define the rule for a `move3/6` instruction +as follows: + + move2 X1=x Y1=y X2=x Y2=y | move X3=x Y3=y => \ + move3 X1 Y1 X2 Y2 X3 Y3 + +(A `\` before a newline can be used to break a long line for readability.) + +It would also be possible to define it like this: + + move X1=x Y1=y | move X2=x Y2=y | move X3=x Y3=y => \ + move3 X1 Y1 X2 Y2 X3 Y3 + +but in that case it must be defined before the rule for `move2/4` +because the first matching rule will be applied. + +One must be careful not to create infinite loops. For example, if we +for some reason would want to reverse the operand order for the `move` +instruction, we must not do like this: + + move Src Dst => move Dst Src + +The loader would swap the operands forever. To avoid the loop, we must +rename the instruction. For example: + + move Src Dst => assign Dst Src + +This concludes the quick tour of the features of **beam\_makeops**. + +Short overview of instruction loading +------------------------------------- + +To give some background to the rest of this document, here follows a +quick overview of how instructions are loaded. + +* The loader reads and decodes one instruction at a time from the BEAM +code and creates a generic instruction. Many transformation rules +must look at multiple instructions, so the loader will +keep multiple generic instructions in a linked list. + +* The loader tries to apply transformation rules against the +generic instructions in the linked list. If a rule matches, the +matched instructions will be removed and replaced with new +generic instructions constructed from the right side of the +transformation. + +* If a transformation rule matched, the loader applies the +transformation rules again. + +* If no transformation rule match, the loader will begin rewriting +the first of generic instructions to a specific instruction. + +* First the loader will search for a specific operation where the +types for all operands match the type for the generic instruction. +The first matching instruction will be selected. **beam\_makeops** +has ordered the specific instructions so that instructions with more +specific operands comes before instructions with less specific +operands. For example, `move_nx` is more specific than `move_cx`. If +the first operand is `[]` (NIL), `move_nx` will be selected. + +* Given the opcode for the selected specific instruction, the loader +looks up the pointer to the C code for the instruction and stores +in the code area for the module being loaded. + +* The loader translates each operand to a machine word and stores it +in the code area. The operand type for the selected specific +instruction guides the translation. For example, if the type is `e`, +the value of the operand is an index into an arry of external +functions and will be translated to a pointer to the export entry for +the function to call. If the type is `x`, the number of the X +register will be multiplied by the word size to produce a byte offset. + +* The loader runs the packing engine to pack multiple operands into a +single word. The packing engine is controlled by a small program, +which is a string where each character is an instruction. For +example, the code to pack the operands for `move_xy` is `"22#"` (on a +64-bit machine). That program will pack the byte offsets for both +registers into the same word as the pointer to C code. + +Running beam_makeops +-------------------- + +**beam\_makeops** is found in `$ERL_TOP/erts/emulator/utils`. Options +start with a hyphen (`-`). The options are followed by the name of +the input files. By convention, all input files have the extension +`.tab`, but is not enforced by **beam\_makeops**. + +### The -outdir option ### + +The option `-outdir Directory` specifies the output directory for +the generated files. Default is the current working directory. + +### Running beam_makeops for the compiler ### + +Give the option `-compiler` to produce output files for the compiler. +The following files will be written to the output directory: + +* `beam_opcodes.erl` - Used primarily by `beam_asm` and `beam_diasm`. + +* `beam_opcode.hrl` - Used by `beam_asm`. It contains tag definitions +used for encoding instruction operands. + +The input file should only contain the definition of BEAM_FORMAT_NUMBER +and external generic instructions. (Everything else would be ignored.) + +### Running beam_makeops for the emulator ### + +Give the option `-emulator` to produce output files for the emulator. +The following output files will be generated in the output directory. + +* `beam_hot.h`, `beam_warm.h`, `beam_cold.`h - Implementation of +instructions. Included inside the `process_main()` function in +`beam_emu.c`. + +* `beam_opcodes.c` - Defines static data used by the loader +(`beam_load.c`). Data about generic instructions, specific +instructions (including how to pack their operands), and +transformation rules are all part of this file. + +* `beam_opcodes.h` - Miscellanous preprocessor definitions, mainly +used by `beam_load.c` but also by `beam_{hot,warm,cold}.h`. + +* `beam_pred_funcs.h` - Included by `beam_load.c`. Contains defines +needed to call guard constraints in transformation rules. + +* `beam_tr_funcs.h` - Included by `beam_load.c`. Contains defines +needed to call a C function to the right of a transformation rule. + +The following options can be given: + +* `wordsize 32|64` - Defines the word size. Default is 32. + +* `code-model Model` - The code model as given to `-mcmodel` option +for GCC. Default is `unknown`. If the code model is `small` (and +the word size is 64 bits), **beam\_makeops** will pack operands +into the upper 32 bits of the instruction word. + +* `DSymbol=0|1` - Defines the value for a symbol. The symbol can be +used in `%if` and `%unless` directives. + +Syntax of .tab files +-------------------- + +### Comments ### + +Any line starting with `#` is a comment and is ignored. + +A line with `//` is also a comment. It is recommended to only +use this style of comments in files that define implementations of +instructions. + +A long line can be broken into shorter lines by a placing a`\` before +the newline. + +### Variable definitions ### + +A variable definition binds a variable to a Perl variable. It is only +meaningful to add a new definition if **beam\_makeops** is updated +at the same time to use the variable. A variable definition looks this: + +*name*=*value*[;] + +where *name* is the name of a Perl variable in **beam\_makeops**, +and *value* is the value to be given to the variable. The line +can optionally end with a `;` (to avoid messing up the +C indentation mode in Emacs). + +Here follows a description of the variables that are defined. + +#### BEAM\_FORMAT\_NUMBER #### + +`genop.tab` has the following definition: + + BEAM_FORMAT_NUMBER=0 + +It defines the version of the instruction set (which will be +included in the code header in the BEAM code). Theoretically, +the version could be bumped, and all instructions changed. +In practice, we would have two support two instruction sets +in the runtime system for at least two releases, so it will +probably never happen in practice. + +#### GC\_REGEXP #### + +In `macros.tab`, there is a definition of `GC_REGEXP`. +It will be described in [a later section](#the-gc_regexp-definition). + +### Directives ### + +There are directives to classify specific instructions depending +on how frequently used they are: + +* `%hot` - Implementation will be placed in `beam_hot.h`. Frequently +executed instructions. + +* `%warm` - Implementation will be placed in `beam_warm.h`. Binary +syntax instructions. + +* `%cold` - Implementation will be placed in `beam_cold.h`. Trace +instructions and infrequently used instructions. + +Default is `%hot`. The directives will be applied to declarations +of the specific instruction that follow. Here is an example: + + %cold + is_number f? xy + %hot + +#### Conditional compilation directives #### + +The `%if` directive includes a range of lines if a condition is +true. For example: + + %if ARCH_64 + i_bs_get_integer_32 x f? x + %endif + +The specific instruction `i_bs_get_integer_32` will only be defined +on a 64-bit machine. + +The condition can be inverted by using `%unless` instead of `%if`: + + %unless NO_FPE_SIGNALS + fcheckerror p => i_fcheckerror + i_fcheckerror + fclearerror + %endif + +It is also possible to add an `%else` clause: + + %if ARCH_64 + BS_SAFE_MUL(A, B, Fail, Dst) { + Uint64 res = ($A) * ($B); + if (res / $B != $A) { + $Fail; + } + $Dst = res; + } + %else + BS_SAFE_MUL(A, B, Fail, Dst) { + Uint64 res = (Uint64)($A) * (Uint64)($B); + if ((res >> (8*sizeof(Uint))) != 0) { + $Fail; + } + $Dst = res; + } + %endif + +#### Symbols that are defined in directives #### + +The following symbols are always defined. + +* `ARCH_64` - is 1 for a 64-bit machine, and 0 otherwise. +* `ARCH_32` - is 1 for 32-bit machine, and 1 otherwise. + +The `Makefile` for building the emulator currently defines the +following symbols by using the `-D` option on the command line for +**beam\_makeops**. + +* `NO_FPE_SIGNALS` - 1 if FPE signals are not enable in runtime system, +0 otherwise. +* `USE_VM_PROBES` - 1 if the runtime system is compiled to use VM probes (support for dtrace or systemtap), 0 otherwise. + +### Defining external generic instructions ### + +External generic BEAM instructions are known to both the compiler and +the runtime system. They remain stable between releases. A new major +release may add more external generic instructions, but must not change +the semantics for a previously defined instruction. + +The syntax for an external generic instruction is as follows: + +*opcode*: [-]*name*/*arity* + +*opcode* is an integer greater than or equal to 1. + +*name* is an identifier starting with a lowercase letter. *arity* is +an integer denoting the number of operands. + +*name* can optionally be preceded by `-` to indicate that it has been +obsoleted. The compiler is not allowed to generate BEAM files that +use obsolete instructions and the loader will refuse to load BEAM +files that use obsolete instructions. + +It only makes sense to define external generic instructions in the +file `genop.tab` in `lib/compiler/src`, because the compiler must +know about them in order to use them. + +New instructions must be added at the end of the file, with higher +numbers than the previous instructions. + +### Defining internal generic instructions ### + +Internal generic instructions are known only to the runtime +system and can be changed at any time without compatibility issues. + +There are two ways to define internal generic instructions: + +* Implicitly when a specific instruction is defined. This is by far +the most common way. Whenever a specific instruction is created, +**beam\_makeops** automatically creates an internal generic instruction +if it does not previously exist. + +* Explicitly. This is necessary only when a generic instruction does +not have any corresponding specific instruction. + +The syntax for an internal generic instruction is as follows: + +*name*/*arity* + +*name* is an identifier starting with a lowercase letter. *arity* is +an integer denoting the number of operands. + +### About generic instructions in general ### + +Each generic instruction has an opcode. The opcode is an integer, +greater than or equal to 1. For an external generic instruction, it +must be explicitly given `genop.tab`, while internal generic +instructions are automatically numbered by **beam\_makeops**. + +The identity of a generic instruction is its name combined with its +arity. That means that it is allowed to define two distinct generic +instructions having the same name but with different arities. For +example: + + move_window/5 + move_window/6 + +Each operand of a generic instruction is tagged with its type. A generic +instruction can have one of the following types: + +* `x` - X register. + +* `y` - Y register. + +* `l` - Floating point register number. + +* `i` - Tagged literal integer. + +* `a` - Tagged literal atom. + +* `n` - NIL (`[]`, the empty list). + +* `q` - Literal that don't fit in a word, that is an object stored on +the heap such as a list or tuple. Any heap object type is supported, +even types that don't have real literals such as external references. + +* `f` - Non-zero failure label. + +* `p` - Zero failure label. + +* `u` - Untagged integer that fits in a machine word. It is used for many +different purposes, such as the number of live registers in `test_heap/2`, +as a reference to the export for `call_ext/2`, and as the flags operand for +binary syntax instructions. When the generic instruction is translated to a +specific instruction, the type for the operand in the specific operation will +tell the loader how to treat the operand. + +* `o` - Overflow. If the value for an `u` operand does not fit in a machine +word, the type of the operand will be changed to `o` (with no associated +value). Currently only used internally in the loader in the guard constraint +function `binary_too_big()`. + +* `v` - Arity value. Only used internally in the loader. + + +### Defining specific instructions ### + +The specific instructions are known only to the runtime system and +are the instructions that are actually executed. They can be changed +at any time without causing compatibility issues. + +A specific instruction can have at most 6 operands. + +A specific instruction is defined by first giving its name followed by +the types for each operand. For example: + + move x y + +Internally, for example in the generated code and in the output from +the BEAM disassembler, the instruction `move x y` will be called `move_xy`. + +The name for a specific instruction is an identifier starting with a +lowercase letter. A type is an lowercase or uppercase letter. + +All specific instructions with a given name must have the same number +of operands. That is, the following is **not** allowed: + + move x x + move x y x y + +Here follows the type letters that more or less directly corresponds +to the types for generic instructions. + +* `x` - X register. Will be loaded as a byte offset to the X register +relative to the base of X register array. (Can be packed with other +operands.) + +* `y` - Y register. Will be loaded as a byte offset to the Y register +relative to the stack frame. (Can be packed with other operands.) + +* `r` - X register 0. An implicit operand that will not be stored in +the loaded code. + +* `l` - Floating point register number. (Can be packed with other +operands.) + +* `i` - Tagged literal integer (a SMALL that will fit in one word). + +* `a` - Tagged atom. + +* `n` - NIL or the empty list. (Will not be stored in the loaded code.) + +* `q` - Tagged CONS or BOXED pointer. That is, a term such as a list +or tuple. Any heap object type is supported, even types that don't +have real literals such as external references. + +* `f` - Failure label (non-zero). The target for a branch +or call instruction. + +* `p` - The 0 failure label, meaning that an exception should be raised +if the instruction fails. (Will not be stored in the loaded code.) + +* `c` - Any literal term; that is, immediate literals such as SMALL, +and CONS or BOXED pointers to literals. (Can be used where the +operand in the generic instruction has one of the types `i`, `a`, `n`, +or `q`.) + +The types that follow do a type test of the operand at runtime; thus, +they are generally more expensive in terms of runtime than the types +described earlier. However, those operand types are needed to avoid a +combinatorial explosion in the number of specific instructions and +overall code size of `process_main()`. + +* `s` - Tagged source: X register, Y register, or a literal term. The +tag will be tested at runtime to retrieve the value from an X +register, a Y register, or simply use the value as a tagged Erlang +term. (Implementation note: An X register is tagged as a pid, and a Y +register as a port. Therefore the literal term must not contain a +port or pid.) + +* `S` - Tagged source register (X or Y). The tag will be tested at +runtime to retrieve the value from an X register or a Y register. Slighly +cheaper than `s`. + +* `d` - Tagged destination register (X or Y). The tag will be tested +at runtime to set up a pointer to the destination register. If the +instrution performs a garbarge collection, it must use the +`$REFRESH_GEN_DEST()` macro to refresh the pointer before storing to +it (there are more details about that in a later section). + +* `j` - A failure label (combination of `f` and `p`). If the branch target 0, +an exception will be raised if instruction fails, otherwise control will be +transfered to the target address. + +The types that follows are all applied to an operand that has the `u` +type. + +* `t` - An untagged integer that will fit in 12 bits (0-4096). It can be +packed with other operands in a word. Most often used as the number +of live registers in instructions such as `test_heap`. + +* `I` - An untagged integer that will fit in 32 bits. It can be +packed with other operands in a word on a 64-bit system. + +* `W` - Untagged integer or pointer. Not possible to pack with other +operands. + +* `e` - Pointer to an export entry. Use by call instructions that call +other modules, such as `call_ext`. + +* `L` - A label. Only used by the `label/1` instruction. + +* `b` - Pointer to BIF. Used by instructions that BIFs, such as +`call_bif`. + +* `A` - A tagged arityvalue. Used in instructions that test the arity +of a tuple. + +* `P` - A byte offset into a tuple. + +* `Q` - A byte offset into the stack. Used for updating the frame +pointer register. Can be packed with other operands. + +When the loader translates a generic instruction a specific +instruction, it will choose the most specific instruction that will +fit the types. Consider the following two instructions: + + move c x + move n x + +The `c` operand can encode any literal value, including NIL. The +`n` operand only works for NIL. If we have the generic instruction +`{move,nil,{x,1}}`, the loader will translate it to `move_nx 1` +because `move n x` is more specific. `move_nx` could be slightly +faster or smaller (depending on the architecture), because the `[]` +is not stored explicitly as an operand. + +#### Syntactic sugar for specific instructions #### + +It is possible to specify more than one type letter for each operand. +Here is an example: + + move cxy xy + +This is syntactic sugar for: + + move c x + move c y + move x x + move x y + move y x + move y y + +Note the difference between `move c xy` and `move c d`. Note that `move c xy` +is equivalent to the following two definitions: + + move c x + move c y + +On the other hand, `move c d` is a single instruction. At runtime, +the `d` operand will be tested to see whether it refers to an X +register or a Y register, and a pointer to the register will be set +up. + +#### The '?' type modifier #### + +The character `?` can be added to the end of an operand to indicate +that the operand will not be used every time the instruction is executed. +For example: + + allocate_heap t I t? + is_eq_exact f? x xy + +In `allocate_heap`, the last operand is the number of live registers. +It will only be used if there is not enough heap space and a garbage +collection must be performed. + +In `is_eq_exact`, the failure address (the first operand) will only be +used if the two register operands are not equal. + +Knowing that an operand is not always used can improve how packing +is done for some instructions. + +For the `allocate_heap` instruction, without the `?` the packing would +be done like this: + + +--------------------+--------------------+ + I -> | Stack needed | &&lb_allocate_heap + + +--------------------+--------------------+ + | Heap needed | Live registers + + +--------------------+--------------------+ + +"Stack needed" and "Heap needed" are always used, but they are in +different words. Thus, at runtime the `allocate_heap` instruction +must read both words from memory even though it will not always use +"Live registers". + +With the `?`, the operands will be packed like this: + + +--------------------+--------------------+ + I -> | Live registers | &&lb_allocate_heap + + +--------------------+--------------------+ + | Heap needed | Stack needed + + +--------------------+--------------------+ + +Now "Stack needed" and "Heap needed" are in the same word. + +### Defining transformation rules ### + +Transformation rules are used to rewrite generic instructions to other +generic instructions. The transformations rules are applied +repeatedly until no rule match. At that point, the first instruction +in the resulting instruction sequence will be converted to a specific +instruction and added to the code for the module being loaded. Then +the transformation rules for the remaining instructions are run in the +same way. + +A rule is recognized by its right-pointer arrow: `=>`. To the left of +the arrow is one or more instruction patterns, separated by `|`. To +the right of the arrow is zero or more instructions, separated by `|`. +If the instructions from the BEAM code matches the instruction +patterns on the left side, they will be replaced with instructions on +the right side (or removed if there are no instructions on the right). + +#### Defining instruction patterns #### + +We will start looking at the patterns on the left side of the arrow. + +A pattern for an instruction consists of its name, followed by a pattern +for each of its operands. The operand patterns are separated by spaces. + +The simplest possible pattern is a variable. Just like in Erlang, +a variable must begin with an uppercase letter. If the same variable is +used in multiple operands, the pattern will only match if the operands +are equal. For example: + + move Same Same => + +This pattern will match if the operands for `move` are the same. If +the pattern match, the instruction will be removed. (That used to be an +actual rule a long time ago when the compiler would occasionally produce +instructions such as `{move,{x,2},{x,2}}`.) + +Variables that have been bound on the left side can be used on the +right side. For example, this rule will rewrite all `move` instructions +to `assign` instructions with the operands swapped: + + move Src Dst => assign Dst Src + +If we only want to match operands of a certain type, we can +use a type constraint. A type constraint consists of one or more +lowercase letters, each specifying a type. For example: + + is_integer Fail an => jump Fail + +The second operand pattern, `an`, will match if the second operand is +either an atom or NIL (the empty list). In case of a match, the +`is_integer/2` instruction will be replaced with a `jump/1` +instruction. + +An operand pattern can bind a variable and constrain the type at the +same time by following the variable with a `=` and the constraint. +For example: + + is_eq_exact Fail=f R=xy C=q => i_is_eq_exact_literal Fail R C + +Here the `is_eq_exact` instruction is replaced with a specialized instruction +that only compares literals, but only if the first operand is a register and +the second operand is a literal. + +#### Further constraining patterns #### + +In addition to specifying a type letter, the actual value for the type can +be specified. For example: + + move C=c x==1 => move_x1 C + +Here the second operand of `move` is constrained to be X register 1. + +When specifying an atom constraint, the atom is written as it would be +in the C source code. That is, it needs an `am_` prefix, and it must +be listed in `atom.names`. For example: + + is_boolean Fail=f a==am_true => + is_boolean Fail=f a==am_false => + +There are several constraints available for testing whether a call is to a BIF +or a function. + +The constraint `u$is_bif` will test whether the given operand refers to a BIF. +For example: + + call_ext u Bif=u$is_bif => call_bif Bif + call_ext u Func => i_call_ext Func + +The `call_ext` instruction can be used to call functions written in +Erlang as well as BIFs (or more properly called SNIFs). The +`u$is_bif` constraint will match if the operand refers to a BIF (that +is, if it is listed in the file `bif.tab`). Note that `u$is_bif` +should only be applied to operands that are known to contain an index +to the import table chunk in the BEAM file (such operands have the +type `b` or `e` in the corresponding specific instruction). If +applied to other `u` operands, it will at best return a nonsense +result. + +The `u$is_not_bif` constraint matches if the operand does not refer to +a BIF (not listed in `bif.tab`). For example: + + move S X0=x==0 | line Loc | call_ext_last Ar Func=u$is_not_bif D => \ + move S X0 | call_ext_last Ar Func D + +The `u$bif:Module:Name/Arity` constraint tests whether the given +operand refers to a specific BIF. Note that `Module:Name/Arity` +**must** be an existing BIF defined in `bif.tab`, or there will +be a compilation error. It is useful when a call to a specific BIF +should be replaced with an instruction as in this example: + + gc_bif2 Fail Live u$bif:erlang:splus/2 S1 S2 Dst => \ + gen_plus Fail Live S1 S2 Dst + +Here the call to the GC BIF `'+'/2` will be replaced with the instruction +`gen_plus/5`. Note that the same name as used in the C source code must be +used for the BIF, which in this case is `splus`. It is defined like this +in `bit.tab`: + + ubif erlang:'+'/2 splus_2 + +The `u$func:Module:Name/Arity` will test whether the given operand is a +a specific function. Here is an example: + + bif1 Fail u$func:erlang:is_constant/1 Src Dst => too_old_compiler + +`is_constant/1` used to be a BIF a long time ago. The transformation +replaces the call with the `too_old_compiler` instruction which will produce +a nicer error message than the default error would be for a missing guard BIF. + +#### Type constraints allowed in patterns #### + +Here are all type letters that are allowed on the left side of a transformation +rule. + +* `u` - An untagged integer that fits in a machine word. + +* `x` - X register. + +* `y` - Y register. + +* `l` - Floating point register number. + +* `i` - Tagged literal integer. + +* `a` - Tagged literal atom. + +* `n` - NIL (`[]`, the empty list). + +* `q` - Literals that don't fit in a word, such as list or tuples. + +* `f` - Non-zero failure label. + +* `p` - The zero failure label. + +* `j` - Any label. Equivalent to `fp`. + +* `c` - Any literal term. Equivalent to `ainq`. + +* `s` - X register, Y register, or any literal term. Equivalent to `xyc`. + +* `d` - X or Y register. Equivalent to `xy`. (In a pattern `d` will +match both source and destination registers. As an operand in a specific +instruction, it must only be used for a destination register.) + +* `o` - Overflow. An untagged integer that does not fit in a machine word. + +#### Guard constraints #### + +If the constraints described so far is not enough, additional +constraints can be written in C in `beam_load.c` and be called as a +guard function on the left side of the transformation. If the guard +function returns a non-zero value, the matching of the rule will +continue, otherwise the match will fail. For example: + + ensure_map Lit=q | literal_is_map(Lit) => + +The guard test `literal_is_map/1` tests whether the given literal is a map. +If the literal is a map, the instruction is unnecessary and can be removed. + +It is outside the scope for this document to describe in detail how such +guard functions are written, but for the curious here is the implementation +of `literal_is_map()`: + + static int + literal_is_map(LoaderState* stp, GenOpArg Lit) + { + Eterm term; + + ASSERT(Lit.type == TAG_q); + term = stp->literals[Lit.val].term; + return is_map(term); + } + +#### Handling instruction with variable number of operands #### + +Some instructions, such as `select_val/3`, essentially has a variable +number of operands. Such instructions have a `{list,[...]}` operand +as their last operand in the BEAM assembly code. For example: + + {select_val,{x,0}, + {f,1}, + {list,[{atom,b},{f,4},{atom,a},{f,5}]}}. + +The loader will convert a `{list,[...]}` operand to an `u` operand whose +value is the number of elements in the list, followed by each element in +the list. The instruction above would be translated to the following +generic instruction: + + {select_val,{x,0},{f,1},{u,4},{atom,b},{f,4},{atom,a},{f,5}} + +To match a variable number of arguments we need to use the special +operand type `*` like this: + + select_val Src=aiq Fail=f Size=u List=* => \ + i_const_select_val Src Fail Size List + +This transformation renames a `select_val/3` instruction +with a constant source operand to `i_const_select_val/3`. + +#### Constructing new instructions on the right side #### + +The most common operand on the right side is a variable that was bound while +matching the left side. For example: + + trim N Remaining => i_trim N + +An operand can also be a type letter to construct an operand of that type. +Each type has a default value. For example, the type `x` has the default +value 1023, which is the highest X register. That makes `x` on the right +side a convenient shortcut for a temporary X register. For example: + + is_number Fail Literal=q => move Literal x | is_number Fail x + +If the second operand for `is_number/2` is a literal, it will be moved to +X register 1023. Then `is_number/2` will test whether the value stored in +X register 1023 is a number. + +This kind of transformation is useful when it is rare that an operand can +be anything else but a register. In the case of `is_number/2`, the second +operand is always a register unless the compiler optimizations have been +disabled. + +If the default value is not suitable, the type letter can be followed +by `=` and a value. Most types take an integer value. The value for +an atom is written the same way as in the C source code. For example, +the atom `false` is written as `am_false`. The atom must be listed in +`atom.names`. + +Here is an example showing how values can be specified: + + bs_put_utf32 Fail=j Flags=u Src=s => \ + i_bs_validate_unicode Fail Src | \ + bs_put_integer Fail i=32 u=1 Flags Src + +#### Type letters on the right side #### + +Here follows all types that are allowed to be used in operands for +instructions being constructed on the right side of a transformation +rule. + +* `u` - Construct an untagged integer. The default value is 0. + +* `x` - X register. The default value is 1023. That makes `x` convenient to +use as a temporary X register. + +* `y` - Y register. The default value is 0. + +* `l` - Foating point register number. The default value is 0. + +* `i` - Tagged literal integer. The default value is 0. + +* `a` - Tagged atom. The default value is the empty atom (`am_Empty`). + +* `n` - NIL (`[]`, the empty list). + +#### Function call on the right side #### + +Transformations that are not possible to describe with the rule +language as described here can be written as a C function in +`beam_load.c` and called from the right side of a transformation. The +left side of the transformation will perform the match and bind +operands to variables. The variables can then be passed to a +generator function on the right side. For example: + + bif2 Fail=j u$bif:erlang:element/2 Index=s Tuple=xy Dst=d => \ + gen_element(Jump, Index, Tuple, Dst) + +This transformation rule matches a call to the BIF `element/2`. +The operands will be captured and the function `gen_element()` will +be called. + +`gen_element()` will produce one of two instructions depending +on `Index`. If `Index` is an integer in the range from 1 up to +the maximum tuple size, the instruction `i_fast_element/2` will +be produced, otherwise the instruction `i_element/4` will be +produced. The corresponding specific instructions are: + + i_fast_element xy j? I d + i_element xy j? s d + +The `i_fast_element/2` instruction is faster because the tuple is +already an untagged integer. It also knows that the index is at least +1, so it does not have to test for that. The `i_element/4` +instruction will have to fetch the index from a register, test that it +is an integer, and untag the integer. + +It is outside the scope of this document to describe in detail how +generator functions are written, but for the curious, here is the +implementation of `gen_element()`: + + static GenOp* + gen_element(LoaderState* stp, GenOpArg Fail, + GenOpArg Index, GenOpArg Tuple, GenOpArg Dst) + { + GenOp* op; + + NEW_GENOP(stp, op); + op->arity = 4; + op->next = NULL; + + if (Index.type == TAG_i && Index.val > 0 && + Index.val <= ERTS_MAX_TUPLE_SIZE && + (Tuple.type == TAG_x || Tuple.type == TAG_y)) { + op->op = genop_i_fast_element_4; + op->a[0] = Tuple; + op->a[1] = Fail; + op->a[2].type = TAG_u; + op->a[2].val = Index.val; + op->a[3] = Dst; + } else { + op->op = genop_i_element_4; + op->a[0] = Tuple; + op->a[1] = Fail; + op->a[2] = Index; + op->a[3] = Dst; + } + + return op; + } +} + +### Defining the implementation ### + +The actual implementation of instructions are also defined in `.tab` +files processed by **beam\_makeops**. For practical reasons, +instruction definitions are stored in several files, at the time of +writing in the following files: + + bif_instrs.tab + arith_instrs.tab + bs_instrs.tab + float_instrs.tab + instrs.tab + map_instrs.tab + msg_instrs.tab + select_instrs.tab + trace_instrs.tab + +There is also a file that only contains macro definitions: + + macros.tab + +The syntax of each file is similar to C code. In fact, most of +the contents *is* C code, interspersed with macro invocations. + +To allow Emacs to auto-indent the code, each file starts with the +following line: + + // -*- c -*- + +To avoid messing up the indentation, all comments are written +as C++ style comments (`//`) instead of `#`. Note that a comment +must start at the beginning of a line. + +The meat of an instruction definition file are macro definitions. +We have seen this macro definition before: + + move(Src, Dst) { + $Dst = $Src; + } + +A macro definitions must start at the beginning of the line (no spaces +allowed), the opening curly bracket must be on the same line, and the +finishing curly bracket must be at the beginning of a line. It is +recommended that the macro body is properly indented. + +As a convention, the macro arguments in the head all start with an +uppercase letter. In the body, the macro arguments can be expanded +by preceding them with `$`. + +A macro definition whose name and arity matches a family of +specific instructions is assumed to be the implementation of that +instruction. + +A macro can also be invoked from within another macro. For example, +`move_deallocate_return/2` avoids repeating code by invoking +`$deallocate_return()` as a macro: + + move_deallocate_return(Src, Deallocate) { + x(0) = $Src; + $deallocate_return($Deallocate); + } + +Here is the definition of `deallocate_return/1`: + + deallocate_return(Deallocate) { + //| -no_next + int words_to_pop = $Deallocate; + SET_I((BeamInstr *) cp_val(*E)); + E = ADD_BYTE_OFFSET(E, words_to_pop); + CHECK_TERM(x(0)); + DispatchReturn; + } + +The expanded code for `move_deallocate_return` will look this: + + OpCase(move_deallocate_return_cQ): + { + x(0) = I[1]; + do { + int words_to_pop = Qb(BeamExtraData(I[0])); + SET_I((BeamInstr *) cp_val(*E)); + E = ADD_BYTE_OFFSET(E, words_to_pop); + CHECK_TERM(x(0)); + DispatchReturn; + } while (0); + } + +When expanding macros, **beam\_makeops** wraps the expansion in a +`do`/`while` wrapper unless **beam\_makeops** can clearly see that no +wrapper is needed. In this case, the wrapper is needed. + +Note that arguments for macros cannot be complex expressions, because +the arguments are split on `,`. For example, the following would +not work because **beam\_makeops** would split the expression into +two arguments: + + $deallocate_return(get_deallocation(y, $Deallocate)); + +#### Code generation directives #### + +Within macro definitions, `//` comments are in general not treated +specially. They will be copied to the file with the generated code +along with the rest of code in the body. + +However, there is an exception. Within a macro definition, a line that +starts with whitespace followed by `//|` is treated specially. The +rest of the line is assumed to contain directives to control code +generation. + +Currently, two code generation directives are recognized: + +* `-no_prefetch` +* `-no_next` + +##### The -no_prefetch directive ##### + +To see what `-no_prefetch` does, let's first look at the default code +generation. Here is the code generated for `move_cx`: + + OpCase(move_cx): + { + BeamInstr next_pf = BeamCodeAddr(I[2]); + xb(BeamExtraData(I[0])) = I[1]; + I += 2; + ASSERT(VALID_INSTR(next_pf)); + GotoPF(next_pf); + } + +Note that the very first thing done is to fetch the address to the +next instruction. The reason is that it usually improves performance. + +Just as a demonstration, we can add a `-no_prefetch` directive to +the `move/2` instruction: + + move(Src, Dst) { + //| -no_prefetch + $Dst = $Src; + } + +We can see that the prefetch is no longer done: + + OpCase(move_cx): + { + xb(BeamExtraData(I[0])) = I[1]; + I += 2; + ASSERT(VALID_INSTR(*I)); + Goto(*I); + } + +When would we want to turn off the prefetch in practice? + +In instructions that will not always execute the next instruction. +For example: + + is_atom(Fail, Src) { + if (is_not_atom($Src)) { + $FAIL($Fail); + } + } + + // From macros.tab + FAIL(Fail) { + //| -no_prefetch + $SET_I_REL($Fail); + Goto(*I); + } + +`is_atom/2` may either execute the next instruction (if the second +operand is an atom) or branch to the failure label. + +The generated code looks like this: + + OpCase(is_atom_fx): + { + if (is_not_atom(xb(I[1]))) { + ASSERT(VALID_INSTR(*(I + (fb(BeamExtraData(I[0]))) + 0))); + I += fb(BeamExtraData(I[0])) + 0;; + Goto(*I);; + } + I += 2; + ASSERT(VALID_INSTR(*I)); + Goto(*I); + } + +##### The -no_next directive ##### + +Next we will look at when the `-no_next` directive can be used. Here +is the `jump/1` instruction: + + jump(Fail) { + $JUMP($Fail); + } + + // From macros.tab + JUMP(Fail) { + //| -no_next + $SET_I_REL($Fail); + Goto(*I); + } + +The generated code looks like this: + + OpCase(jump_f): + { + ASSERT(VALID_INSTR(*(I + (fb(BeamExtraData(I[0]))) + 0))); + I += fb(BeamExtraData(I[0])) + 0;; + Goto(*I);; + } + +If we remove the `-no_next` directive, the code would look like this: + + OpCase(jump_f): + { + BeamInstr next_pf = BeamCodeAddr(I[1]); + ASSERT(VALID_INSTR(*(I + (fb(BeamExtraData(I[0]))) + 0))); + I += fb(BeamExtraData(I[0])) + 0;; + Goto(*I);; + I += 1; + ASSERT(VALID_INSTR(next_pf)); + GotoPF(next_pf); + } + +In the end, the C compiler will probably optimize this code to the +same native code as the first version, but the first version is certainly +much easier to read for human readers. + +#### Macros in the macros.tab file #### + +The file `macros.tab` contains many useful macros. When implementing +new instructions it is good practice to look through `macros.tab` to +see if any of existing macros can be used rather than re-inventing +the wheel. + +We will describe a few of the most useful macros here. + +##### The GC_REGEXP definition ##### + +The following line defines a regular expression that will recognize +a call to a function that does a garbage collection: + + GC_REGEXP=erts_garbage_collect|erts_gc|GcBifFunction; + +The purpose is that **beam\_makeops** can verify that an instruction +that does a garbage collection and has an `d` operand uses the +`$REFRESH_GEN_DEST()` macro. + +If you need to define a new function that does garbage collection, +you should give it the prefix `erts_gc_`. If that is not possible +you should update the regular expression so that it will match your +new function. + +##### FAIL(Fail) ##### + +Branch to `$Fail`. Will suppress prefetch (`-no_prefetch`). Typical use: + + is_nonempty_list(Fail, Src) { + if (is_not_list($Src)) { + $FAIL($Fail); + } + } + +##### JUMP(Fail) ##### + +Branch to `$Fail`. Suppresses generation of dispatch of the next +instruction (`-no_next`). Typical use: + + jump(Fail) { + $JUMP($Fail); + } + +##### GC_TEST(NeedStack, NeedHeap, Live) ##### + +`$GC_TEST(NeedStack, NeedHeap, Live)` tests that given amount of +stack space and heap space is available. If not it will do a +garbage collection. Typical use: + + test_heap(Nh, Live) { + $GC_TEST(0, $Nh, $Live); + } + +##### AH(NeedStack, NeedHeap, Live) ##### + +`AH(NeedStack, NeedHeap, Live)` allocates a stack frame and +optionally additional heap space. + +#### Pre-defined macros and variables #### + +**beam\_makeops** defines several built-in macros and pre-bound variables. + +##### The NEXT_INSTRUCTION pre-bound variable ##### + +The NEXT_INSTRUCTION is a pre-bound variable that is available in +all instructions. It expands to the address of the next instruction. + +Here is an example: + + i_call(CallDest) { + SET_CP(c_p, $NEXT_INSTRUCTION); + $DISPATCH_REL($CallDest); + } + +When calling a function, the return address is first stored in `c_p->cp` +(using the `SET_CP()` macro defined in `beam_emu.c`), and then control is +transferred to the callee. Here is the generated code: + + OpCase(i_call_f): + { + SET_CP(c_p, I+1); + ASSERT(VALID_INSTR(*(I + (fb(BeamExtraData(I[0]))) + 0))); + I += fb(BeamExtraData(I[0])) + 0;; + DTRACE_LOCAL_CALL(c_p, erts_code_to_codemfa(I)); + Dispatch();; + } + +We can see that that `$NEXT_INSTRUCTION` has been expanded to `I+1`. +That makes sense since the size of the `i_call_f/1` instruction is +one word. + +##### The IP_ADJUSTMENT pre-bound variable ##### + +`$IP_ADJUSTMENT` is usually 0. In a few combined instructions +(described below) it can be non-zero. It is used like this +in `macros.tab`: + + SET_I_REL(Offset) { + ASSERT(VALID_INSTR(*(I + ($Offset) + $IP_ADJUSTMENT))); + I += $Offset + $IP_ADJUSTMENT; + } + +Avoid using `IP_ADJUSTMENT` directly. Use `SET_I_REL()` or +one of the macros that invoke such as `FAIL()` or `JUMP()` +defined in `macros.tab`. + +#### Pre-defined macro functions #### + +##### The IF() macro ##### + +`$IF(Expr, IfTrue, IfFalse)` evaluates `Expr`, which must be a valid +Perl expression (which for simple numeric expressions have the same +syntax as C). If `Expr` evaluates to 0, the entire `IF()` expression will be +replaced with `IfFalse`, otherwise it will be replaced with `IfTrue`. + +See the description of `OPERAND_POSITION()` for an example. + +##### The OPERAND\_POSITION() macro ##### + +`$OPERAND_POSITION(Expr)` returns the position for `Expr`, if +`Expr` is an operand that is not packed. The first operand is +at position 1. + +Returns 0 otherwise. + +This macro could be used like this in order to share code: + + FAIL(Fail) { + //| -no_prefetch + $IF($OPERAND_POSITION($Fail) == 1 && $IP_ADJUSTMENT == 0, + goto common_jump, + $DO_JUMP($Fail)); + } + + DO_JUMP(Fail) { + $SET_I_REL($Fail); + Goto(*I)); + } + + // In beam_emu.c: + common_jump: + I += I[1]; + Goto(*I)); + + +#### The $REFRESH\_GEN\_DEST() macro #### + +When a specific instruction has a `d` operand, early during execution +of the instruction, a pointer will be initialized to point to the X or +Y register in question. + +If there is a garbage collection before the result is stored, +the stack will move and if the `d` operand refered to a Y +register, the pointer will no longer be valid. (Y registers are +stored on the stack.) + +In those circumstances, `$REFRESH_GEN_DEST()` must be invoked +to set up the pointer again. **beam\_makeops** will notice +if there is a call to a function that does a garbage collection and +`$REFRESH_GEN_DEST()` is not called. + +Here is a complete example. The `new_map` instruction is defined +like this: + + new_map d t I + +It is implemented like this: + + new_map(Dst, Live, N) { + Eterm res; + + HEAVY_SWAPOUT; + res = erts_gc_new_map(c_p, reg, $Live, $N, $NEXT_INSTRUCTION); + HEAVY_SWAPIN; + $REFRESH_GEN_DEST(); + $Dst = res; + $NEXT($NEXT_INSTRUCTION+$N); + } + +If we have forgotten the `$REFRESH_GEN_DEST()` there would be a message +similar to this: + + pointer to destination register is invalid after GC -- use $REFRESH_GEN_DEST() + ... from the body of new_map at beam/map_instrs.tab(30) + +#### Combined instructions #### + +**Problem**: For frequently executed instructions we want to use +"fast" operands types such as `x` and `y`, as opposed to `s` or `S`. +To avoid an explosion in code size, we want to share most of the +implementation between the instructions. Here are the specific +instructions for `i_increment/5`: + + i_increment r W t d + i_increment x W t d + i_increment y W t d + +The `i_increment` instruction is implemented like this: + + i_increment(Source, IncrementVal, Live, Dst) { + Eterm increment_reg_source = $Source; + Eterm increment_val = $IncrementVal; + Uint live; + Eterm result; + + if (ERTS_LIKELY(is_small(increment_reg_val))) { + Sint i = signed_val(increment_reg_val) + increment_val; + if (ERTS_LIKELY(IS_SSMALL(i))) { + $Dst = make_small(i); + $NEXT0(); + } + } + live = $Live; + HEAVY_SWAPOUT; + reg[live] = increment_reg_val; + reg[live+1] = make_small(increment_val); + result = erts_gc_mixed_plus(c_p, reg, live); + HEAVY_SWAPIN; + ERTS_HOLE_CHECK(c_p); + if (ERTS_LIKELY(is_value(result))) { + $REFRESH_GEN_DEST(); + $Dst = result; + $NEXT0(); + } + ASSERT(c_p->freason != BADMATCH || is_value(c_p->fvalue)); + goto find_func_info; + } + +There will be three almost identical copies of the code. Given the +size of the code, that could be too high cost to pay. + +To avoid the three copies of the code, we could use only one specific +instruction: + + i_increment S W t d + +(The same implementation as above will work.) + +That reduces the code size, but is slower because `S` means that +there will be extra code to test whether the operand refers to an X +register or a Y register. + +**Solution**: We can use "combined instructions". Combined +instructions are combined from instruction fragments. The +bulk of the code can be shared. + +Here we will show how `i_increment` can be implemented as a combined +instruction. We will show each individual fragment first, and then +show how to connect them together. First we will need a variable that +we can store the value fetched from the register in: + + increment.head() { + Eterm increment_reg_val; + } + +The name `increment` is the name of the group that the fragment +belongs to. Note that it does not need to have the same +name as the instruction. The group name is followed by `.` and +the name of the fragment. The name `head` is pre-defined. +The code in it will be placed at the beginning of a block, so +that all fragments in the group can access it. + +Next we define the fragment that will pick up the value from the +register from the first operand: + + increment.fetch(Src) { + increment_reg_val = $Src; + } + +We call this fragment `fetch`. This fragment will be duplicated three +times, one for each value of the first operand (`r`, `x`, and `y`). + +Next we define the main part of the code that do the actual incrementing. + + increment.execute(IncrementVal, Live, Dst) { + Eterm increment_val = $IncrementVal; + Uint live; + Eterm result; + + if (ERTS_LIKELY(is_small(increment_reg_val))) { + Sint i = signed_val(increment_reg_val) + increment_val; + if (ERTS_LIKELY(IS_SSMALL(i))) { + $Dst = make_small(i); + $NEXT0(); + } + } + live = $Live; + HEAVY_SWAPOUT; + reg[live] = increment_reg_val; + reg[live+1] = make_small(increment_val); + result = erts_gc_mixed_plus(c_p, reg, live); + HEAVY_SWAPIN; + ERTS_HOLE_CHECK(c_p); + if (ERTS_LIKELY(is_value(result))) { + $REFRESH_GEN_DEST(); + $Dst = result; + $NEXT0(); + } + ASSERT(c_p->freason != BADMATCH || is_value(c_p->fvalue)); + goto find_func_info; + } + +We call this fragment `execute`. It will handle the three remaining +operands (`W t d`). There will only be one copy of this fragment. + +Now that we have defined the fragments, we need to inform +**beam\_makeops** how they should be connected: + + i_increment := increment.fetch.execute; + +To the left of the `:=` is the name of the specific instruction that +should be implemented by the fragments, in this case `i_increment`. +To the right of `:=` is the name of the group with the fragments, +followed by a `.`. Then the name of the fragments in the group are +listed in the order they should be executed. Note that the `head` +fragment is not listed. + +The line ends in `;` (to avoid messing up the indentation in Emacs). + +(Note that in practice the `:=` line is usually placed before the +fragments.) + +The generated code looks like this: + + { + Eterm increment_reg_val; + OpCase(i_increment_rWtd): + { + increment_reg_val = r(0); + } + goto increment__execute; + + OpCase(i_increment_xWtd): + { + increment_reg_val = xb(BeamExtraData(I[0])); + } + goto increment__execute; + + OpCase(i_increment_yWtd): + { + increment_reg_val = yb(BeamExtraData(I[0])); + } + goto increment__execute; + + increment__execute: + { + // Here follows the code from increment.execute() + . + . + . + } + +##### Some notes about combined instructions ##### + +The operands that are different must be at +the beginning of the instruction. All operands in the last +fragment must have the same operands in all variants of +the specific instruction. + +As an example, the following specific instructions cannot be +implemented as a combined instruction: + + i_times j? t x x d + i_times j? t x y d + i_times j? t s s d + +We would have to change the order of the operands so that the +two operands that are different are placed first: + + i_times x x j? t d + i_times x y j? t d + i_times s s j? t d + +We can then define: + + i_times := times.fetch.execute; + + times.head { + Eterm op1, op2; + } + + times.fetch(Src1, Src2) { + op1 = $Src1; + op2 = $Src2; + } + + times.execute(Fail, Live, Dst) { + // Multiply op1 and op2. + . + . + . + } + +Several instructions can share a group. As an example, the following +instructions have different names, but in the end they all create a +binary. The last two operands are common for all of them: + + i_bs_init_fail xy j? t? x + i_bs_init_fail_heap s I j? t? x + i_bs_init W t? x + i_bs_init_heap W I t? x + +The instructions are defined like this (formatted with extra +spaces for clarity): + + i_bs_init_fail_heap := bs_init . fail_heap . verify . execute; + i_bs_init_fail := bs_init . fail . verify . execute; + i_bs_init := bs_init . . plain . execute; + i_bs_init_heap := bs_init . heap . execute; + +Note that the first two instruction have three fragments, while the +other two only have two fragments. Here are the fragments: + + bs_init_bits.head() { + Eterm num_bits_term; + Uint num_bits; + Uint alloc; + } + + bs_init_bits.plain(NumBits) { + num_bits = $NumBits; + alloc = 0; + } + + bs_init_bits.heap(NumBits, Alloc) { + num_bits = $NumBits; + alloc = $Alloc; + } + + bs_init_bits.fail(NumBitsTerm) { + num_bits_term = $NumBitsTerm; + alloc = 0; + } + + bs_init_bits.fail_heap(NumBitsTerm, Alloc) { + num_bits_term = $NumBitsTerm; + alloc = $Alloc; + } + + bs_init_bits.verify(Fail) { + // Verify the num_bits_term, fail using $FAIL + // if there is a problem. + . + . + . + } + + bs_init_bits.execute(Live, Dst) { + // Long complicated code to a create a binary. + . + . + . + } + +The full definitions of those instructions can be found in `bs_instrs.tab`. +The generated code can be found in `beam_warm.h`. diff --git a/erts/emulator/nifs/common/prim_buffer_nif.c b/erts/emulator/nifs/common/prim_buffer_nif.c new file mode 100644 index 0000000000..a8ef5fc355 --- /dev/null +++ b/erts/emulator/nifs/common/prim_buffer_nif.c @@ -0,0 +1,512 @@ +/* + * %CopyrightBegin% + * + * Copyright Ericsson 2017. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * %CopyrightEnd% + */ + +#define STATIC_ERLANG_NIF 1 + +#include "erl_nif.h" +#include "config.h" +#include "sys.h" + +#ifdef VALGRIND +# include <valgrind/memcheck.h> +#endif + +#define ACCUMULATOR_SIZE (2 << 10) + +#define FIND_NIF_RESCHEDULE_SIZE (1 << 20) + +/* NIF interface declarations */ +static int load(ErlNifEnv *env, void** priv_data, ERL_NIF_TERM load_info); +static int upgrade(ErlNifEnv *env, void** priv_data, void** old_priv_data, ERL_NIF_TERM load_info); +static void unload(ErlNifEnv *env, void* priv_data); + +static ErlNifResourceType *rtype_buffer; + +static ERL_NIF_TERM am_ok; +static ERL_NIF_TERM am_error; + +static ERL_NIF_TERM am_lock_order_violation; + +static ERL_NIF_TERM am_acquired; +static ERL_NIF_TERM am_busy; + +static ERL_NIF_TERM am_continue; + +static ERL_NIF_TERM am_out_of_memory; +static ERL_NIF_TERM am_not_found; + +typedef struct { +#ifdef DEBUG + erts_atomic32_t concurrent_users; +#endif + + ErlNifBinary accumulator; + size_t accumulated_bytes; + int accumulator_present; + + ErlNifIOQueue *queue; + + erts_atomic32_t external_lock; +} buffer_data_t; + +static ERL_NIF_TERM new_nif(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]); + +static ERL_NIF_TERM peek_head_nif(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]); +static ERL_NIF_TERM skip_nif(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]); +static ERL_NIF_TERM size_nif(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]); + +static ERL_NIF_TERM write_nif(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]); +static ERL_NIF_TERM copying_read_nif(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]); + +static ERL_NIF_TERM find_nif(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]); + +static ERL_NIF_TERM trylock_nif(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]); +static ERL_NIF_TERM unlock_nif(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]); + +static ErlNifFunc nif_funcs[] = { + {"new", 0, new_nif}, + {"size", 1, size_nif}, + {"peek_head", 1, peek_head_nif}, + {"copying_read", 2, copying_read_nif}, + {"write", 2, write_nif}, + {"skip", 2, skip_nif}, + {"find_byte_index", 2, find_nif}, + {"try_lock", 1, trylock_nif}, + {"unlock", 1, unlock_nif}, +}; + +ERL_NIF_INIT(prim_buffer, nif_funcs, load, NULL, upgrade, unload) + +static void gc_buffer(ErlNifEnv *env, void* data); + +static int load(ErlNifEnv *env, void** priv_data, ERL_NIF_TERM load_info) +{ + am_ok = enif_make_atom(env, "ok"); + am_error = enif_make_atom(env, "error"); + + am_lock_order_violation = enif_make_atom(env, "lock_order_violation"); + am_acquired = enif_make_atom(env, "acquired"); + am_busy = enif_make_atom(env, "busy"); + + am_continue = enif_make_atom(env, "continue"); + + am_out_of_memory = enif_make_atom(env, "out_of_memory"); + am_not_found = enif_make_atom(env, "not_found"); + + rtype_buffer = enif_open_resource_type(env, NULL, "gc_buffer", gc_buffer, + ERL_NIF_RT_CREATE, NULL); + + *priv_data = NULL; + + return 0; +} + +static void unload(ErlNifEnv *env, void* priv_data) +{ + +} + +static int upgrade(ErlNifEnv *env, void** priv_data, void** old_priv_data, ERL_NIF_TERM load_info) +{ + if(*old_priv_data != NULL) { + return -1; /* Don't know how to do that */ + } + + if(*priv_data != NULL) { + return -1; /* Don't know how to do that */ + } + + if(load(env, priv_data, load_info)) { + return -1; + } + + return 0; +} + +static void gc_buffer(ErlNifEnv *env, void* data) { + buffer_data_t *buffer = (buffer_data_t*)data; + + if(buffer->accumulator_present) { + enif_release_binary(&buffer->accumulator); + } + + enif_ioq_destroy(buffer->queue); +} + +static int get_buffer_data(ErlNifEnv *env, ERL_NIF_TERM opaque, buffer_data_t **buffer) { + return enif_get_resource(env, opaque, rtype_buffer, (void **)buffer); +} + +/* Copies a number of bytes from the head of the iovec, skipping "vec_skip" + * vector elements followed by "byte_skip" bytes on the target vector. */ +static void copy_from_iovec(SysIOVec *iovec, int vec_len, int vec_skip, + size_t byte_skip, size_t size, char *data) { + + size_t bytes_copied, skip_offset; + int vec_index; + + skip_offset = byte_skip; + vec_index = vec_skip; + bytes_copied = 0; + + while(bytes_copied < size) { + size_t block_size, copy_size; + char *block_start; + + ASSERT(vec_index < vec_len); + + block_start = (char*)iovec[vec_index].iov_base; + block_size = iovec[vec_index].iov_len; + + copy_size = MIN(size - bytes_copied, block_size - skip_offset); + sys_memcpy(&data[bytes_copied], &block_start[skip_offset], copy_size); + + bytes_copied += copy_size; + skip_offset = 0; + + vec_index++; + } +} + +/* Convenience function for copy_from_iovec over queues. */ +static void copy_from_queue(ErlNifIOQueue *queue, int queue_skip, + size_t byte_skip, size_t size, char *data) { + + SysIOVec *queued_data; + int queue_length; + + queued_data = enif_ioq_peek(queue, &queue_length); + ASSERT(queue_skip < queue_length); + + copy_from_iovec(queued_data, queue_length, queue_skip, byte_skip, size, data); +} + +static int enqueue_write_accumulator(buffer_data_t *buffer) { + ASSERT(!buffer->accumulator_present ^ (buffer->accumulated_bytes > 0)); + + if(buffer->accumulator_present && buffer->accumulated_bytes > 0) { + if(!enif_realloc_binary(&buffer->accumulator, buffer->accumulated_bytes)) { + return 0; + } else if(!enif_ioq_enq_binary(buffer->queue, &buffer->accumulator, 0)) { + return 0; + } + + /* The queue owns the accumulator now. */ + buffer->accumulator_present = 0; + buffer->accumulated_bytes = 0; + } + + return 1; +} + +static int combine_small_writes(buffer_data_t *buffer, ErlNifIOVec *iovec) { + ASSERT(!buffer->accumulator_present ^ (buffer->accumulated_bytes > 0)); + + if(buffer->accumulated_bytes + iovec->size >= ACCUMULATOR_SIZE) { + if(iovec->size >= (ACCUMULATOR_SIZE / 2)) { + return 0; + } + + if(!enqueue_write_accumulator(buffer)) { + return 0; + } + } + + if(!buffer->accumulator_present) { + if(!enif_alloc_binary(ACCUMULATOR_SIZE, &buffer->accumulator)) { + return 0; + } + + buffer->accumulator_present = 1; + } + + copy_from_iovec(iovec->iov, iovec->iovcnt, 0, 0, iovec->size, + (char*)&buffer->accumulator.data[buffer->accumulated_bytes]); + buffer->accumulated_bytes += iovec->size; + + return 1; +} + +/* *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** */ + +static ERL_NIF_TERM new_nif(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) { + buffer_data_t *buffer; + ERL_NIF_TERM result; + + buffer = (buffer_data_t*)enif_alloc_resource(rtype_buffer, sizeof(buffer_data_t)); + buffer->queue = enif_ioq_create(ERL_NIF_IOQ_NORMAL); + + if(buffer->queue != NULL) { +#ifdef DEBUG + erts_atomic32_init_nob(&buffer->concurrent_users, 0); +#endif + erts_atomic32_init_nob(&buffer->external_lock, 0); + + buffer->accumulator_present = 0; + buffer->accumulated_bytes = 0; + + result = enif_make_resource(env, buffer); + } else { + result = enif_raise_exception(env, am_out_of_memory); + } + + enif_release_resource(buffer); + + return result; +} + +static ERL_NIF_TERM size_nif(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) { + buffer_data_t *buffer; + + size_t total_size; + + if(argc != 1 || !get_buffer_data(env, argv[0], &buffer)) { + return enif_make_badarg(env); + } + + ASSERT(erts_atomic32_inc_read_acqb(&buffer->concurrent_users) == 1); + + total_size = enif_ioq_size(buffer->queue); + + if(buffer->accumulator_present) { + total_size += buffer->accumulated_bytes; + } else { + ASSERT(buffer->accumulated_bytes == 0); + } + + ASSERT(erts_atomic32_dec_read_relb(&buffer->concurrent_users) == 0); + + return enif_make_uint64(env, total_size); +} + +static ERL_NIF_TERM copying_read_nif(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) { + buffer_data_t *buffer; + + ERL_NIF_TERM result; + unsigned char *data; + Uint64 block_size; + + if(argc != 2 || !get_buffer_data(env, argv[0], &buffer) + || !enif_get_uint64(env, argv[1], &block_size)) { + return enif_make_badarg(env); + } + + ASSERT(erts_atomic32_inc_read_acqb(&buffer->concurrent_users) == 1); + + if(!enqueue_write_accumulator(buffer)) { + return enif_raise_exception(env, am_out_of_memory); + } + + if(enif_ioq_size(buffer->queue) < block_size) { + return enif_make_badarg(env); + } + + data = enif_make_new_binary(env, block_size, &result); + + if(block_size > 0) { + copy_from_queue(buffer->queue, 0, 0, block_size, (char*)data); + enif_ioq_deq(buffer->queue, block_size, NULL); + } + + ASSERT(erts_atomic32_dec_read_relb(&buffer->concurrent_users) == 0); + + return result; +} + +static ERL_NIF_TERM write_nif(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) { + buffer_data_t *buffer; + + ErlNifIOVec vec, *iovec = &vec; + ERL_NIF_TERM tail; + + if(argc != 2 || !get_buffer_data(env, argv[0], &buffer) + || !enif_inspect_iovec(env, 64, argv[1], &tail, &iovec)) { + return enif_make_badarg(env); + } + + ASSERT(erts_atomic32_inc_read_acqb(&buffer->concurrent_users) == 1); + + if(!combine_small_writes(buffer, iovec)) { + if(!enqueue_write_accumulator(buffer) || !enif_ioq_enqv(buffer->queue, iovec, 0)) { + return enif_raise_exception(env, am_out_of_memory); + } + } + + ASSERT(erts_atomic32_dec_read_relb(&buffer->concurrent_users) == 0); + + if(!enif_is_empty_list(env, tail)) { + const ERL_NIF_TERM new_argv[2] = {argv[0], tail}; + + return enif_schedule_nif(env, "write", 0, &write_nif, argc, new_argv); + } + + return am_ok; +} + +static ERL_NIF_TERM peek_head_nif(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) { + buffer_data_t *buffer; + + ERL_NIF_TERM result; + + if(argc != 1 || !get_buffer_data(env, argv[0], &buffer)) { + return enif_make_badarg(env); + } + + ASSERT(erts_atomic32_inc_read_acqb(&buffer->concurrent_users) == 1); + + if(!enqueue_write_accumulator(buffer)) { + return enif_raise_exception(env, am_out_of_memory); + } + + if(!enif_ioq_peek_head(env, buffer->queue, NULL, &result)) { + return enif_make_badarg(env); + } + + ASSERT(erts_atomic32_dec_read_relb(&buffer->concurrent_users) == 0); + + return result; +} + +static ERL_NIF_TERM skip_nif(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) { + buffer_data_t *buffer; + + Uint64 block_size; + + if(argc != 2 || !get_buffer_data(env, argv[0], &buffer) + || !enif_get_uint64(env, argv[1], &block_size)) { + return enif_make_badarg(env); + } + + ASSERT(erts_atomic32_inc_read_acqb(&buffer->concurrent_users) == 1); + + if(!enqueue_write_accumulator(buffer)) { + return enif_raise_exception(env, am_out_of_memory); + } else if(enif_ioq_size(buffer->queue) < block_size) { + return enif_make_badarg(env); + } + + enif_ioq_deq(buffer->queue, block_size, NULL); + + ASSERT(erts_atomic32_dec_read_relb(&buffer->concurrent_users) == 0); + + return am_ok; +} + +static ERL_NIF_TERM find_nif(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) { + buffer_data_t *buffer; + + int queue_length, queue_index; + SysIOVec *queued_data; + size_t queue_size; + + size_t search_offset; + int needle; + + if(argc != 2 || !get_buffer_data(env, argv[0], &buffer) + || !enif_get_int(env, argv[1], &needle)) { + return enif_make_badarg(env); + } + + ASSERT(erts_atomic32_inc_read_acqb(&buffer->concurrent_users) == 1); + + if(!enqueue_write_accumulator(buffer)) { + return enif_raise_exception(env, am_out_of_memory); + } else if(needle < 0 || needle > 255) { + return enif_make_badarg(env); + } + + queued_data = enif_ioq_peek(buffer->queue, &queue_length); + queue_size = enif_ioq_size(buffer->queue); + queue_index = 0; + + search_offset = 0; + + if(queue_size > (FIND_NIF_RESCHEDULE_SIZE / 100)) { + if(enif_thread_type() == ERL_NIF_THR_NORMAL_SCHEDULER) { + int timeslice_percent; + + if(queue_size >= FIND_NIF_RESCHEDULE_SIZE) { + ASSERT(erts_atomic32_dec_read_relb(&buffer->concurrent_users) == 0); + + return enif_schedule_nif(env, "find", + ERL_NIF_DIRTY_JOB_CPU_BOUND, &find_nif, argc, argv); + } + + timeslice_percent = (queue_size * 100) / FIND_NIF_RESCHEDULE_SIZE; + enif_consume_timeslice(env, timeslice_percent); + } + } + + while(queue_index < queue_length) { + char *needle_address; + char *block_start; + size_t block_size; + + block_start = queued_data[queue_index].iov_base; + block_size = queued_data[queue_index].iov_len; + + needle_address = memchr(block_start, needle, block_size); + + if(needle_address != NULL) { + size_t result = search_offset + (needle_address - block_start); + + ASSERT(erts_atomic32_dec_read_relb(&buffer->concurrent_users) == 0); + + return enif_make_tuple2(env, am_ok, enif_make_uint64(env, result)); + } + + search_offset += block_size; + queue_index++; + } + + ASSERT(erts_atomic32_dec_read_relb(&buffer->concurrent_users) == 0); + + return am_not_found; +} + +/* */ + +static ERL_NIF_TERM trylock_nif(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) { + buffer_data_t *buffer; + + if(argc != 1 || !get_buffer_data(env, argv[0], &buffer)) { + return enif_make_badarg(env); + } + + if(erts_atomic32_cmpxchg_acqb(&buffer->external_lock, 1, 0) == 0) { + return am_acquired; + } + + return am_busy; +} + +static ERL_NIF_TERM unlock_nif(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) { + buffer_data_t *buffer; + + if(argc != 1 || !get_buffer_data(env, argv[0], &buffer)) { + return enif_make_badarg(env); + } + + if(erts_atomic32_cmpxchg_relb(&buffer->external_lock, 0, 1) == 0) { + return enif_raise_exception(env, am_lock_order_violation); + } + + return am_ok; +} diff --git a/erts/emulator/nifs/common/prim_file_nif.c b/erts/emulator/nifs/common/prim_file_nif.c new file mode 100644 index 0000000000..6874f41d75 --- /dev/null +++ b/erts/emulator/nifs/common/prim_file_nif.c @@ -0,0 +1,1237 @@ +/* + * %CopyrightBegin% + * + * Copyright Ericsson 2017. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * %CopyrightEnd% + */ + +#define STATIC_ERLANG_NIF 1 + +#include "erl_nif.h" +#include "config.h" +#include "sys.h" + +#ifdef VALGRIND +# include <valgrind/memcheck.h> +#endif + +#include "erl_driver.h" +#include "prim_file_nif.h" + +/* NIF interface declarations */ +static int load(ErlNifEnv *env, void** priv_data, ERL_NIF_TERM load_info); +static int upgrade(ErlNifEnv *env, void** priv_data, void** old_priv_data, ERL_NIF_TERM load_info); +static void unload(ErlNifEnv *env, void* priv_data); + +static ErlNifResourceType *efile_resource_type; + +static ERL_NIF_TERM am_ok; +static ERL_NIF_TERM am_error; +static ERL_NIF_TERM am_continue; + +static ERL_NIF_TERM am_file_info; + +/* File modes */ +static ERL_NIF_TERM am_read; +static ERL_NIF_TERM am_write; +static ERL_NIF_TERM am_exclusive; +static ERL_NIF_TERM am_append; +static ERL_NIF_TERM am_sync; +static ERL_NIF_TERM am_skip_type_check; + +/* enum efile_access_t; read and write are defined above.*/ +static ERL_NIF_TERM am_read_write; +static ERL_NIF_TERM am_none; + +/* enum efile_advise_t */ +static ERL_NIF_TERM am_normal; +static ERL_NIF_TERM am_random; +static ERL_NIF_TERM am_sequential; +static ERL_NIF_TERM am_will_need; +static ERL_NIF_TERM am_dont_need; +static ERL_NIF_TERM am_no_reuse; + +/* enum efile_filetype_t */ +static ERL_NIF_TERM am_device; +static ERL_NIF_TERM am_directory; +static ERL_NIF_TERM am_regular; +static ERL_NIF_TERM am_symlink; +static ERL_NIF_TERM am_other; + +/* enum efile_seek_t, 'eof' marker. */ +static ERL_NIF_TERM am_bof; +static ERL_NIF_TERM am_cur; +static ERL_NIF_TERM am_eof; + +static ERL_NIF_TERM read_info_nif(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]); +static ERL_NIF_TERM set_permissions_nif(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]); +static ERL_NIF_TERM set_owner_nif(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]); +static ERL_NIF_TERM set_time_nif(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]); + +static ERL_NIF_TERM read_link_nif(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]); +static ERL_NIF_TERM list_dir_nif(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]); + +static ERL_NIF_TERM make_hard_link_nif(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]); +static ERL_NIF_TERM make_soft_link_nif(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]); +static ERL_NIF_TERM rename_nif(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]); +static ERL_NIF_TERM make_dir_nif(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]); +static ERL_NIF_TERM del_file_nif(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]); +static ERL_NIF_TERM del_dir_nif(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]); +static ERL_NIF_TERM get_device_cwd_nif(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]); +static ERL_NIF_TERM get_cwd_nif(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]); +static ERL_NIF_TERM set_cwd_nif(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]); + +static ERL_NIF_TERM read_file_nif(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]); + +static ERL_NIF_TERM get_handle_nif(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]); +static ERL_NIF_TERM altname_nif(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]); + +static ERL_NIF_TERM open_nif(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]); + +/* All file handle operations are passed through a wrapper that handles state + * transitions, marking it as busy during the course of the operation, and + * closing on completion if the owner died in the middle of an operation. + * + * This is pretty ugly but required as there's no way to tell when it's safe to + * asynchronously close a file; the event could have fired just before landing + * in a system call which will fail with EBADF at best or alias a newly opened + * fd at worst. + * + * The old driver got away with enqueueing the close operation on the same + * async queue as all of its other operations, but since dirty schedulers use a + * single global queue there's no natural way to schedule an asynchronous close + * "behind" other operations. + * + * The states may transition as follows: + * + * IDLE -> + * BUSY (file_handle_wrapper) | + * CLOSED (owner_death_callback) + * + * BUSY -> + * IDLE (file_handle_wrapper) + * CLOSED (close_nif_impl) + * CLOSE_PENDING (owner_death_callback) + * + * CLOSE_PENDING -> + * CLOSED (file_handle_wrapper) + */ + +typedef ERL_NIF_TERM (*file_op_impl_t)(efile_data_t *d, ErlNifEnv *env, + int argc, const ERL_NIF_TERM argv[]); +static ERL_NIF_TERM file_handle_wrapper(file_op_impl_t operation, ErlNifEnv *env, + int argc, const ERL_NIF_TERM argv[]); + +#define WRAP_FILE_HANDLE_EXPORT(name) \ + static ERL_NIF_TERM name ## _impl (efile_data_t *d, ErlNifEnv *env, \ + int argc, const ERL_NIF_TERM argv[]);\ + static ERL_NIF_TERM name(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) { \ + return file_handle_wrapper( name ## _impl , env, argc, argv); \ + } + +WRAP_FILE_HANDLE_EXPORT(close_nif) +WRAP_FILE_HANDLE_EXPORT(read_nif) +WRAP_FILE_HANDLE_EXPORT(write_nif) +WRAP_FILE_HANDLE_EXPORT(pread_nif) +WRAP_FILE_HANDLE_EXPORT(pwrite_nif) +WRAP_FILE_HANDLE_EXPORT(seek_nif) +WRAP_FILE_HANDLE_EXPORT(sync_nif) +WRAP_FILE_HANDLE_EXPORT(truncate_nif) +WRAP_FILE_HANDLE_EXPORT(allocate_nif) +WRAP_FILE_HANDLE_EXPORT(advise_nif) +WRAP_FILE_HANDLE_EXPORT(get_handle_nif) +WRAP_FILE_HANDLE_EXPORT(ipread_s32bu_p32bu_nif) + +static ErlNifFunc nif_funcs[] = { + /* File handle ops */ + {"open_nif", 2, open_nif, ERL_NIF_DIRTY_JOB_IO_BOUND}, + {"close_nif", 1, close_nif, ERL_NIF_DIRTY_JOB_IO_BOUND}, + {"read_nif", 2, read_nif, ERL_NIF_DIRTY_JOB_IO_BOUND}, + {"write_nif", 2, write_nif, ERL_NIF_DIRTY_JOB_IO_BOUND}, + {"pread_nif", 3, pread_nif, ERL_NIF_DIRTY_JOB_IO_BOUND}, + {"pwrite_nif", 3, pwrite_nif, ERL_NIF_DIRTY_JOB_IO_BOUND}, + {"seek_nif", 3, seek_nif, ERL_NIF_DIRTY_JOB_IO_BOUND}, + {"sync_nif", 2, sync_nif, ERL_NIF_DIRTY_JOB_IO_BOUND}, + {"truncate_nif", 1, truncate_nif, ERL_NIF_DIRTY_JOB_IO_BOUND}, + {"allocate_nif", 3, allocate_nif, ERL_NIF_DIRTY_JOB_IO_BOUND}, + {"advise_nif", 4, advise_nif, ERL_NIF_DIRTY_JOB_IO_BOUND}, + + /* Filesystem ops */ + {"make_hard_link_nif", 2, make_hard_link_nif, ERL_NIF_DIRTY_JOB_IO_BOUND}, + {"make_soft_link_nif", 2, make_soft_link_nif, ERL_NIF_DIRTY_JOB_IO_BOUND}, + {"rename_nif", 2, rename_nif, ERL_NIF_DIRTY_JOB_IO_BOUND}, + {"read_info_nif", 2, read_info_nif, ERL_NIF_DIRTY_JOB_IO_BOUND}, + {"set_permissions_nif", 2, set_permissions_nif, ERL_NIF_DIRTY_JOB_IO_BOUND}, + {"set_owner_nif", 3, set_owner_nif, ERL_NIF_DIRTY_JOB_IO_BOUND}, + {"set_time_nif", 4, set_time_nif, ERL_NIF_DIRTY_JOB_IO_BOUND}, + {"read_link_nif", 1, read_link_nif, ERL_NIF_DIRTY_JOB_IO_BOUND}, + {"list_dir_nif", 1, list_dir_nif, ERL_NIF_DIRTY_JOB_IO_BOUND}, + {"make_dir_nif", 1, make_dir_nif, ERL_NIF_DIRTY_JOB_IO_BOUND}, + {"del_file_nif", 1, del_file_nif, ERL_NIF_DIRTY_JOB_IO_BOUND}, + {"del_dir_nif", 1, del_dir_nif, ERL_NIF_DIRTY_JOB_IO_BOUND}, + {"get_device_cwd_nif", 1, get_device_cwd_nif, ERL_NIF_DIRTY_JOB_IO_BOUND}, + {"set_cwd_nif", 1, set_cwd_nif, ERL_NIF_DIRTY_JOB_IO_BOUND}, + {"get_cwd_nif", 0, get_cwd_nif, ERL_NIF_DIRTY_JOB_IO_BOUND}, + + /* These operations are equivalent to chained calls of other operations, + * but have been moved down to avoid excessive rescheduling. */ + {"ipread_s32bu_p32bu_nif", 3, ipread_s32bu_p32bu_nif, ERL_NIF_DIRTY_JOB_IO_BOUND}, + {"read_file_nif", 1, read_file_nif, ERL_NIF_DIRTY_JOB_IO_BOUND}, + + /* Internal ops. */ + {"get_handle_nif", 1, get_handle_nif}, + {"altname_nif", 1, altname_nif, ERL_NIF_DIRTY_JOB_IO_BOUND}, +}; + +ERL_NIF_INIT(prim_file, nif_funcs, load, NULL, upgrade, unload) + +static void owner_death_callback(ErlNifEnv* env, void* obj, ErlNifPid* pid, ErlNifMonitor* mon); +static void gc_callback(ErlNifEnv *env, void* data); + +static int load(ErlNifEnv *env, void** priv_data, ERL_NIF_TERM load_info) +{ + ErlNifResourceTypeInit callbacks; + + am_ok = enif_make_atom(env, "ok"); + am_error = enif_make_atom(env, "error"); + am_continue = enif_make_atom(env, "continue"); + + am_read = enif_make_atom(env, "read"); + am_write = enif_make_atom(env, "write"); + am_exclusive = enif_make_atom(env, "exclusive"); + am_append = enif_make_atom(env, "append"); + am_sync = enif_make_atom(env, "sync"); + am_skip_type_check = enif_make_atom(env, "skip_type_check"); + + am_read_write = enif_make_atom(env, "read_write"); + am_none = enif_make_atom(env, "none"); + + am_normal = enif_make_atom(env, "normal"); + am_random = enif_make_atom(env, "random"); + am_sequential = enif_make_atom(env, "sequential"); + am_will_need = enif_make_atom(env, "will_need"); + am_dont_need = enif_make_atom(env, "dont_need"); + am_no_reuse = enif_make_atom(env, "no_reuse"); + + am_device = enif_make_atom(env, "device"); + am_directory = enif_make_atom(env, "directory"); + am_regular = enif_make_atom(env, "regular"); + am_symlink = enif_make_atom(env, "symlink"); + am_other = enif_make_atom(env, "other"); + + am_file_info = enif_make_atom(env, "file_info"); + + am_bof = enif_make_atom(env, "bof"); + am_cur = enif_make_atom(env, "cur"); + am_eof = enif_make_atom(env, "eof"); + + callbacks.down = owner_death_callback; + callbacks.dtor = gc_callback; + callbacks.stop = NULL; + + efile_resource_type = enif_open_resource_type_x(env, "efile", &callbacks, + ERL_NIF_RT_CREATE, NULL); + + *priv_data = NULL; + + return 0; +} + +static void unload(ErlNifEnv *env, void* priv_data) +{ + +} + +static int upgrade(ErlNifEnv *env, void** priv_data, void** old_priv_data, ERL_NIF_TERM load_info) +{ + if(*old_priv_data != NULL) { + return -1; /* Don't know how to do that */ + } + if(*priv_data != NULL) { + return -1; /* Don't know how to do that */ + } + if(load(env, priv_data, load_info)) { + return -1; + } + return 0; +} + +static ERL_NIF_TERM posix_error_to_tuple(ErlNifEnv *env, posix_errno_t posix_errno) { + ERL_NIF_TERM error = enif_make_atom(env, erl_errno_id(posix_errno)); + return enif_make_tuple2(env, am_error, error); +} + +static int get_file_data(ErlNifEnv *env, ERL_NIF_TERM opaque, efile_data_t **d) { + return enif_get_resource(env, opaque, efile_resource_type, (void **)d); +} + +static ERL_NIF_TERM file_handle_wrapper(file_op_impl_t operation, ErlNifEnv *env, + int argc, const ERL_NIF_TERM argv[]) { + + efile_data_t *d; + + enum efile_state_t previous_state; + ERL_NIF_TERM result; + + if(argc < 1 || !get_file_data(env, argv[0], &d)) { + return enif_make_badarg(env); + } + + previous_state = erts_atomic32_cmpxchg_acqb(&d->state, + EFILE_STATE_BUSY, EFILE_STATE_IDLE); + + if(previous_state == EFILE_STATE_IDLE) { + result = operation(d, env, argc - 1, &argv[1]); + + previous_state = erts_atomic32_cmpxchg_relb(&d->state, + EFILE_STATE_IDLE, EFILE_STATE_BUSY); + + ASSERT(previous_state != EFILE_STATE_IDLE); + + if(previous_state == EFILE_STATE_CLOSE_PENDING) { + /* This is the only point where a change from CLOSE_PENDING is + * possible, and we're running synchronously, so we can't race with + * anything else here. */ + erts_atomic32_set_acqb(&d->state, EFILE_STATE_CLOSED); + efile_close(d); + } + } else { + /* CLOSE_PENDING should be impossible at this point since it requires + * a transition from BUSY; the only valid state here is CLOSED. */ + ASSERT(previous_state == EFILE_STATE_CLOSED); + + result = posix_error_to_tuple(env, EINVAL); + } + + return result; +} + +static void owner_death_callback(ErlNifEnv* env, void* obj, ErlNifPid* pid, ErlNifMonitor* mon) { + efile_data_t *d = (efile_data_t*)obj; + + (void)env; + (void)pid; + (void)mon; + + for(;;) { + enum efile_state_t previous_state; + + previous_state = erts_atomic32_cmpxchg_acqb(&d->state, + EFILE_STATE_CLOSED, EFILE_STATE_IDLE); + + switch(previous_state) { + case EFILE_STATE_IDLE: + efile_close(d); + return; + case EFILE_STATE_CLOSE_PENDING: + case EFILE_STATE_CLOSED: + /* We're either already closed or managed to mark ourselves for + * closure in the previous iteration. */ + return; + case EFILE_STATE_BUSY: + /* Schedule ourselves to be closed once the current operation + * finishes, retrying the [IDLE -> CLOSED] transition in case we + * narrowly passed the [BUSY -> IDLE] one. */ + erts_atomic32_cmpxchg_nob(&d->state, + EFILE_STATE_CLOSE_PENDING, EFILE_STATE_BUSY); + break; + } + } +} + +static void gc_callback(ErlNifEnv *env, void* data) { + efile_data_t *d = (efile_data_t*)data; + + enum efile_state_t previous_state; + + (void)env; + + previous_state = erts_atomic32_cmpxchg_acqb(&d->state, + EFILE_STATE_CLOSED, EFILE_STATE_IDLE); + + ASSERT(previous_state != EFILE_STATE_CLOSE_PENDING && + previous_state != EFILE_STATE_BUSY); + + if(previous_state == EFILE_STATE_IDLE) { + efile_close(d); + } +} + +static ERL_NIF_TERM efile_filetype_to_atom(enum efile_filetype_t type) { + switch(type) { + case EFILE_FILETYPE_DEVICE: return am_device; + case EFILE_FILETYPE_DIRECTORY: return am_directory; + case EFILE_FILETYPE_REGULAR: return am_regular; + case EFILE_FILETYPE_SYMLINK: return am_symlink; + case EFILE_FILETYPE_OTHER: return am_other; + } + + return am_other; +} + +static ERL_NIF_TERM efile_access_to_atom(enum efile_access_t type) { + if(type & EFILE_ACCESS_READ && !(type & EFILE_ACCESS_WRITE)) { + return am_read; + } else if(type & EFILE_ACCESS_WRITE && !(type & EFILE_ACCESS_READ)) { + return am_write; + } else if(type & EFILE_ACCESS_READ_WRITE) { + return am_read_write; + } + + return am_none; +} + +static enum efile_modes_t efile_translate_modelist(ErlNifEnv *env, ERL_NIF_TERM list) { + enum efile_modes_t modes; + ERL_NIF_TERM head, tail; + + modes = 0; + + while(enif_get_list_cell(env, list, &head, &tail)) { + if(enif_is_identical(head, am_read)) { + modes |= EFILE_MODE_READ; + } else if(enif_is_identical(head, am_write)) { + modes |= EFILE_MODE_WRITE; + } else if(enif_is_identical(head, am_exclusive)) { + modes |= EFILE_MODE_EXCLUSIVE; + } else if(enif_is_identical(head, am_append)) { + modes |= EFILE_MODE_APPEND; + } else if(enif_is_identical(head, am_sync)) { + modes |= EFILE_MODE_SYNC; + } else if(enif_is_identical(head, am_skip_type_check)) { + modes |= EFILE_MODE_SKIP_TYPE_CHECK; + } else { + /* Modes like 'raw', 'ram', 'delayed_writes' etc are handled + * further up the chain. */ + } + + list = tail; + } + + if(modes & (EFILE_MODE_APPEND | EFILE_MODE_EXCLUSIVE)) { + /* 'append' and 'exclusive' are documented as "open for writing." */ + modes |= EFILE_MODE_WRITE; + } else if(!(modes & EFILE_MODE_READ_WRITE)) { + /* Defaulting to read if !(W|R) is undocumented, but specifically + * tested against in file_SUITE. */ + modes |= EFILE_MODE_READ; + } + + return modes; +} + +static ERL_NIF_TERM open_nif(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) { + posix_errno_t posix_errno; + efile_data_t *d; + + ErlNifPid controlling_process; + enum efile_modes_t modes; + ERL_NIF_TERM result; + efile_path_t path; + + if(argc != 2 || !enif_is_list(env, argv[1])) { + return enif_make_badarg(env); + } + + modes = efile_translate_modelist(env, argv[1]); + + if((posix_errno = efile_marshal_path(env, argv[0], &path))) { + return posix_error_to_tuple(env, posix_errno); + } else if((posix_errno = efile_open(&path, modes, efile_resource_type, &d))) { + return posix_error_to_tuple(env, posix_errno); + } + + result = enif_make_resource(env, d); + enif_release_resource(d); + + enif_self(env, &controlling_process); + + if(enif_monitor_process(env, d, &controlling_process, &d->monitor)) { + return posix_error_to_tuple(env, EINVAL); + } + + return enif_make_tuple2(env, am_ok, result); +} + +static ERL_NIF_TERM close_nif_impl(efile_data_t *d, ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) { + enum efile_state_t previous_state; + + if(argc != 0) { + return enif_make_badarg(env); + } + + previous_state = erts_atomic32_cmpxchg_acqb(&d->state, + EFILE_STATE_CLOSED, EFILE_STATE_BUSY); + + ASSERT(previous_state == EFILE_STATE_CLOSE_PENDING || + previous_state == EFILE_STATE_BUSY); + + if(previous_state == EFILE_STATE_BUSY) { + enif_demonitor_process(env, d, &d->monitor); + + if(!efile_close(d)) { + return posix_error_to_tuple(env, d->posix_errno); + } + } + + return am_ok; +} + +static ERL_NIF_TERM read_nif_impl(efile_data_t *d, ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) { + Sint64 bytes_read, block_size; + SysIOVec read_vec[1]; + ErlNifBinary result; + + if(argc != 1 || !enif_is_number(env, argv[0])) { + return enif_make_badarg(env); + } + + if(!enif_get_int64(env, argv[0], &block_size) || block_size < 0) { + return posix_error_to_tuple(env, EINVAL); + } + + if(!enif_alloc_binary(block_size, &result)) { + return posix_error_to_tuple(env, ENOMEM); + } + + read_vec[0].iov_base = result.data; + read_vec[0].iov_len = result.size; + + bytes_read = efile_readv(d, read_vec, 1); + ASSERT(bytes_read <= block_size); + + if(bytes_read < 0) { + return posix_error_to_tuple(env, d->posix_errno); + } else if(bytes_read == 0) { + enif_release_binary(&result); + return am_eof; + } + + if(bytes_read < block_size && !enif_realloc_binary(&result, bytes_read)) { + ERTS_INTERNAL_ERROR("Failed to shrink read result."); + } + + return enif_make_tuple2(env, am_ok, enif_make_binary(env, &result)); +} + +static ERL_NIF_TERM write_nif_impl(efile_data_t *d, ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) { + ErlNifIOVec vec, *input = &vec; + Sint64 bytes_written; + ERL_NIF_TERM tail; + + if(argc != 1 || !enif_inspect_iovec(env, 64, argv[0], &tail, &input)) { + return enif_make_badarg(env); + } + + bytes_written = efile_writev(d, input->iov, input->iovcnt); + + if(bytes_written < 0) { + return posix_error_to_tuple(env, d->posix_errno); + } + + if(!enif_is_empty_list(env, tail)) { + ASSERT(bytes_written > 0); + return enif_make_tuple2(env, am_continue, tail); + } + + return am_ok; +} + +static ERL_NIF_TERM pread_nif_impl(efile_data_t *d, ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) { + Sint64 bytes_read, block_size, offset; + SysIOVec read_vec[1]; + ErlNifBinary result; + + if(argc != 2 || !enif_is_number(env, argv[0]) + || !enif_is_number(env, argv[1])) { + return enif_make_badarg(env); + } + + if(!enif_get_int64(env, argv[0], &offset) || + !enif_get_int64(env, argv[1], &block_size) || + (offset < 0 || block_size < 0)) { + return posix_error_to_tuple(env, EINVAL); + } + + if(!enif_alloc_binary(block_size, &result)) { + return posix_error_to_tuple(env, ENOMEM); + } + + read_vec[0].iov_base = result.data; + read_vec[0].iov_len = result.size; + + bytes_read = efile_preadv(d, offset, read_vec, 1); + + if(bytes_read < 0) { + return posix_error_to_tuple(env, d->posix_errno); + } else if(bytes_read == 0) { + enif_release_binary(&result); + return am_eof; + } + + if(bytes_read < block_size && !enif_realloc_binary(&result, bytes_read)) { + ERTS_INTERNAL_ERROR("Failed to shrink pread result."); + } + + return enif_make_tuple2(env, am_ok, enif_make_binary(env, &result)); +} + +static ERL_NIF_TERM pwrite_nif_impl(efile_data_t *d, ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) { + ErlNifIOVec vec, *input = &vec; + Sint64 bytes_written, offset; + ERL_NIF_TERM tail; + + if(argc != 2 || !enif_is_number(env, argv[0]) + || !enif_inspect_iovec(env, 64, argv[1], &tail, &input)) { + return enif_make_badarg(env); + } + + if(!enif_get_int64(env, argv[0], &offset) || offset < 0) { + return posix_error_to_tuple(env, EINVAL); + } + + bytes_written = efile_pwritev(d, offset, input->iov, input->iovcnt); + + if(bytes_written < 0) { + return posix_error_to_tuple(env, d->posix_errno); + } + + if(!enif_is_empty_list(env, tail)) { + ASSERT(bytes_written > 0); + return enif_make_tuple3(env, am_continue, + enif_make_int64(env, bytes_written), tail); + } + + return am_ok; +} + +static ERL_NIF_TERM seek_nif_impl(efile_data_t *d, ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) { + Sint64 new_position, offset; + enum efile_seek_t seek; + + if(argc != 2 || !enif_get_int64(env, argv[1], &offset)) { + return enif_make_badarg(env); + } + + if(enif_is_identical(argv[0], am_bof)) { + seek = EFILE_SEEK_BOF; + } else if(enif_is_identical(argv[0], am_cur)) { + seek = EFILE_SEEK_CUR; + } else if(enif_is_identical(argv[0], am_eof)) { + seek = EFILE_SEEK_EOF; + } else { + return enif_make_badarg(env); + } + + if(!efile_seek(d, seek, offset, &new_position)) { + return posix_error_to_tuple(env, d->posix_errno); + } + + return enif_make_tuple2(env, am_ok, enif_make_uint64(env, new_position)); +} + +static ERL_NIF_TERM sync_nif_impl(efile_data_t *d, ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) { + int data_only; + + if(argc != 1 || !enif_get_int(env, argv[0], &data_only)) { + return enif_make_badarg(env); + } + + if(!efile_sync(d, data_only)) { + return posix_error_to_tuple(env, d->posix_errno); + } + + return am_ok; +} + +static ERL_NIF_TERM truncate_nif_impl(efile_data_t *d, ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) { + if(argc != 0) { + return enif_make_badarg(env); + } + + if(!efile_truncate(d)) { + return posix_error_to_tuple(env, d->posix_errno); + } + + return am_ok; +} + +static ERL_NIF_TERM allocate_nif_impl(efile_data_t *d, ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) { + Sint64 offset, length; + + if(argc != 2 || !enif_is_number(env, argv[0]) + || !enif_is_number(env, argv[1])) { + return enif_make_badarg(env); + } + + if(!enif_get_int64(env, argv[0], &offset) || + !enif_get_int64(env, argv[1], &length) || + (offset < 0 || length < 0)) { + return posix_error_to_tuple(env, EINVAL); + } + + if(!efile_allocate(d, offset, length)) { + return posix_error_to_tuple(env, d->posix_errno); + } + + return am_ok; +} + +static ERL_NIF_TERM advise_nif_impl(efile_data_t *d, ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) { + enum efile_advise_t advise; + Sint64 offset, length; + + if(argc != 3 || !enif_is_number(env, argv[0]) + || !enif_is_number(env, argv[1])) { + return enif_make_badarg(env); + } + + if(!enif_get_int64(env, argv[0], &offset) || + !enif_get_int64(env, argv[1], &length) || + (offset < 0 || length < 0)) { + return posix_error_to_tuple(env, EINVAL); + } + + if(enif_is_identical(argv[2], am_normal)) { + advise = EFILE_ADVISE_NORMAL; + } else if(enif_is_identical(argv[2], am_random)) { + advise = EFILE_ADVISE_RANDOM; + } else if(enif_is_identical(argv[2], am_sequential)) { + advise = EFILE_ADVISE_SEQUENTIAL; + } else if(enif_is_identical(argv[2], am_will_need)) { + advise = EFILE_ADVISE_WILL_NEED; + } else if(enif_is_identical(argv[2], am_dont_need)) { + advise = EFILE_ADVISE_DONT_NEED; + } else if(enif_is_identical(argv[2], am_no_reuse)) { + advise = EFILE_ADVISE_NO_REUSE; + } else { + /* The tests check for EINVAL instead of badarg. Sigh. */ + return posix_error_to_tuple(env, EINVAL); + } + + if(!efile_advise(d, offset, length, advise)) { + return posix_error_to_tuple(env, d->posix_errno); + } + + return am_ok; +} + +/* This undocumented function reads a pointer and then reads the data block + * described by said pointer. It was reverse-engineered from the old + * implementation so while all tests pass it may not be entirely correct. Our + * current understanding is as follows: + * + * Pointer layout: + * + * <<Size:1/integer-unit:32, Offset:1/integer-unit:32>> + * + * Where Offset is the -absolute- address to the data block. + * + * *) If we fail to read the pointer block in its entirety, we return eof. + * *) If the provided max_payload_size is larger than Size, we return eof. + * *) If we fail to read any data whatsoever at Offset, we return + * {ok, {Size, Offset, eof}} + * *) Otherwise, we return {ok, {Size, Offset, Data}}. Note that the size + * of Data may be smaller than Size if we encounter EOF before we could + * read the entire block. + * + * On errors we'll return {error, posix()} regardless of whether they + * happened before or after reading the pointer block. */ +static ERL_NIF_TERM ipread_s32bu_p32bu_nif_impl(efile_data_t *d, ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) { + Sint64 payload_offset, payload_size; + + SysIOVec read_vec[1]; + Sint64 bytes_read; + + ErlNifBinary payload; + + if(argc != 2 || !enif_is_number(env, argv[0]) + || !enif_is_number(env, argv[1])) { + return enif_make_badarg(env); + } + + { + Sint64 max_payload_size, pointer_offset; + unsigned char pointer_block[8]; + + if(!enif_get_int64(env, argv[0], &pointer_offset) || + !enif_get_int64(env, argv[1], &max_payload_size) || + (pointer_offset < 0 || max_payload_size >= 1u << 31)) { + return posix_error_to_tuple(env, EINVAL); + } + + read_vec[0].iov_base = pointer_block; + read_vec[0].iov_len = sizeof(pointer_block); + + bytes_read = efile_preadv(d, pointer_offset, read_vec, 1); + + if(bytes_read < 0) { + return posix_error_to_tuple(env, d->posix_errno); + } else if(bytes_read < sizeof(pointer_block)) { + return am_eof; + } + + payload_size = (Uint32)get_int32(&pointer_block[0]); + payload_offset = (Uint32)get_int32(&pointer_block[4]); + + if(payload_size > max_payload_size) { + return am_eof; + } + } + + if(!enif_alloc_binary(payload_size, &payload)) { + return posix_error_to_tuple(env, ENOMEM); + } + + read_vec[0].iov_base = payload.data; + read_vec[0].iov_len = payload.size; + + bytes_read = efile_preadv(d, payload_offset, read_vec, 1); + + if(bytes_read < 0) { + return posix_error_to_tuple(env, d->posix_errno); + } else if(bytes_read == 0) { + enif_release_binary(&payload); + + return enif_make_tuple2(env, am_ok, + enif_make_tuple3(env, + enif_make_uint(env, payload_size), + enif_make_uint(env, payload_offset), + am_eof)); + } + + if(bytes_read < payload.size && !enif_realloc_binary(&payload, bytes_read)) { + ERTS_INTERNAL_ERROR("Failed to shrink ipread payload."); + } + + return enif_make_tuple2(env, am_ok, + enif_make_tuple3(env, + enif_make_uint(env, payload_size), + enif_make_uint(env, payload_offset), + enif_make_binary(env, &payload))); +} + +static ERL_NIF_TERM get_handle_nif_impl(efile_data_t *d, ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) { + if(argc != 0) { + return enif_make_badarg(env); + } + + return efile_get_handle(env, d); +} + +static ERL_NIF_TERM read_info_nif(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) { + posix_errno_t posix_errno; + + efile_fileinfo_t info = {0}; + efile_path_t path; + int follow_links; + + if(argc != 2 || !enif_get_int(env, argv[1], &follow_links)) { + return enif_make_badarg(env); + } + + if((posix_errno = efile_marshal_path(env, argv[0], &path))) { + return posix_error_to_tuple(env, posix_errno); + } else if((posix_errno = efile_read_info(&path, follow_links, &info))) { + return posix_error_to_tuple(env, posix_errno); + } + + /* #file_info as declared in file.hrl */ + return enif_make_tuple(env, 14, + am_file_info, + enif_make_uint64(env, info.size), + efile_filetype_to_atom(info.type), + efile_access_to_atom(info.access), + enif_make_int64(env, MAX(EFILE_MIN_FILETIME, info.a_time)), + enif_make_int64(env, MAX(EFILE_MIN_FILETIME, info.m_time)), + enif_make_int64(env, MAX(EFILE_MIN_FILETIME, info.c_time)), + enif_make_uint(env, info.mode), + enif_make_uint(env, info.links), + enif_make_uint(env, info.major_device), + enif_make_uint(env, info.minor_device), + enif_make_uint(env, info.inode), + enif_make_uint(env, info.uid), + enif_make_uint(env, info.gid) + ); +} + +static ERL_NIF_TERM set_permissions_nif(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) { + posix_errno_t posix_errno; + + efile_path_t path; + Uint32 permissions; + + if(argc != 2 || !enif_get_uint(env, argv[1], &permissions)) { + return enif_make_badarg(env); + } + + if((posix_errno = efile_marshal_path(env, argv[0], &path))) { + return posix_error_to_tuple(env, posix_errno); + } else if((posix_errno = efile_set_permissions(&path, permissions))) { + return posix_error_to_tuple(env, posix_errno); + } + + return am_ok; +} + +static ERL_NIF_TERM set_owner_nif(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) { + posix_errno_t posix_errno; + + efile_path_t path; + Uint32 uid, gid; + + if(argc != 3 || !enif_get_uint(env, argv[1], &uid) + || !enif_get_uint(env, argv[2], &gid)) { + return enif_make_badarg(env); + } + + if((posix_errno = efile_marshal_path(env, argv[0], &path))) { + return posix_error_to_tuple(env, posix_errno); + } else if((posix_errno = efile_set_owner(&path, uid, gid))) { + return posix_error_to_tuple(env, posix_errno); + } + + return am_ok; +} + +static ERL_NIF_TERM set_time_nif(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) { + posix_errno_t posix_errno; + + Sint64 accessed, modified, created; + efile_path_t path; + + if(argc != 4 || !enif_get_int64(env, argv[1], &accessed) + || !enif_get_int64(env, argv[2], &modified) + || !enif_get_int64(env, argv[3], &created)) { + return enif_make_badarg(env); + } + + if((posix_errno = efile_marshal_path(env, argv[0], &path))) { + return posix_error_to_tuple(env, posix_errno); + } else if((posix_errno = efile_set_time(&path, accessed, modified, created))) { + return posix_error_to_tuple(env, posix_errno); + } + + return am_ok; +} + +static ERL_NIF_TERM read_link_nif(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) { + posix_errno_t posix_errno; + + efile_path_t path; + ERL_NIF_TERM result; + + if(argc != 1) { + return enif_make_badarg(env); + } + + if((posix_errno = efile_marshal_path(env, argv[0], &path))) { + return posix_error_to_tuple(env, posix_errno); + } else if((posix_errno = efile_read_link(env, &path, &result))) { + return posix_error_to_tuple(env, posix_errno); + } + + return enif_make_tuple2(env, am_ok, result); +} + +static ERL_NIF_TERM list_dir_nif(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) { + posix_errno_t posix_errno; + + efile_path_t path; + ERL_NIF_TERM result; + + if(argc != 1) { + return enif_make_badarg(env); + } + + if((posix_errno = efile_marshal_path(env, argv[0], &path))) { + return posix_error_to_tuple(env, posix_errno); + } else if((posix_errno = efile_list_dir(env, &path, &result))) { + return posix_error_to_tuple(env, posix_errno); + } + + return enif_make_tuple2(env, am_ok, result); +} + +static ERL_NIF_TERM rename_nif(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) { + posix_errno_t posix_errno; + + efile_path_t existing_path, new_path; + + if(argc != 2) { + return enif_make_badarg(env); + } + + if((posix_errno = efile_marshal_path(env, argv[0], &existing_path))) { + return posix_error_to_tuple(env, posix_errno); + } else if((posix_errno = efile_marshal_path(env, argv[1], &new_path))) { + return posix_error_to_tuple(env, posix_errno); + } else if((posix_errno = efile_rename(&existing_path, &new_path))) { + return posix_error_to_tuple(env, posix_errno); + } + + return am_ok; +} + +static ERL_NIF_TERM make_hard_link_nif(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) { + posix_errno_t posix_errno; + + efile_path_t existing_path, new_path; + + if(argc != 2) { + return enif_make_badarg(env); + } + + if((posix_errno = efile_marshal_path(env, argv[0], &existing_path))) { + return posix_error_to_tuple(env, posix_errno); + } else if((posix_errno = efile_marshal_path(env, argv[1], &new_path))) { + return posix_error_to_tuple(env, posix_errno); + } else if((posix_errno = efile_make_hard_link(&existing_path, &new_path))) { + return posix_error_to_tuple(env, posix_errno); + } + + return am_ok; +} + +static ERL_NIF_TERM make_soft_link_nif(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) { + posix_errno_t posix_errno; + + efile_path_t existing_path, new_path; + + if(argc != 2) { + return enif_make_badarg(env); + } + + if((posix_errno = efile_marshal_path(env, argv[0], &existing_path))) { + return posix_error_to_tuple(env, posix_errno); + } else if((posix_errno = efile_marshal_path(env, argv[1], &new_path))) { + return posix_error_to_tuple(env, posix_errno); + } else if((posix_errno = efile_make_soft_link(&existing_path, &new_path))) { + return posix_error_to_tuple(env, posix_errno); + } + + return am_ok; +} + +static ERL_NIF_TERM make_dir_nif(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) { + posix_errno_t posix_errno; + + efile_path_t path; + + if(argc != 1) { + return enif_make_badarg(env); + } + + if((posix_errno = efile_marshal_path(env, argv[0], &path))) { + return posix_error_to_tuple(env, posix_errno); + } else if((posix_errno = efile_make_dir(&path))) { + return posix_error_to_tuple(env, posix_errno); + } + + return am_ok; +} + +static ERL_NIF_TERM del_file_nif(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) { + posix_errno_t posix_errno; + + efile_path_t path; + + if(argc != 1) { + return enif_make_badarg(env); + } + + if((posix_errno = efile_marshal_path(env, argv[0], &path))) { + return posix_error_to_tuple(env, posix_errno); + } else if((posix_errno = efile_del_file(&path))) { + return posix_error_to_tuple(env, posix_errno); + } + + return am_ok; +} + +static ERL_NIF_TERM del_dir_nif(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) { + posix_errno_t posix_errno; + + efile_path_t path; + + if(argc != 1) { + return enif_make_badarg(env); + } + + if((posix_errno = efile_marshal_path(env, argv[0], &path))) { + return posix_error_to_tuple(env, posix_errno); + } else if((posix_errno = efile_del_dir(&path))) { + return posix_error_to_tuple(env, posix_errno); + } + + return am_ok; +} + +static ERL_NIF_TERM get_device_cwd_nif(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) { + posix_errno_t posix_errno; + + ERL_NIF_TERM result; + int device_index; + + if(argc != 1 || !enif_get_int(env, argv[0], &device_index)) { + return enif_make_badarg(env); + } + + if((posix_errno = efile_get_device_cwd(env, device_index, &result))) { + return posix_error_to_tuple(env, posix_errno); + } + + return enif_make_tuple2(env, am_ok, result); +} + +static ERL_NIF_TERM get_cwd_nif(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) { + posix_errno_t posix_errno; + ERL_NIF_TERM result; + + if(argc != 0) { + return enif_make_badarg(env); + } + + if((posix_errno = efile_get_cwd(env, &result))) { + return posix_error_to_tuple(env, posix_errno); + } + + return enif_make_tuple2(env, am_ok, result); +} + +static ERL_NIF_TERM set_cwd_nif(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) { + posix_errno_t posix_errno; + + efile_path_t path; + + if(argc != 1) { + return enif_make_badarg(env); + } + + if((posix_errno = efile_marshal_path(env, argv[0], &path))) { + return posix_error_to_tuple(env, posix_errno); + } else if((posix_errno = efile_set_cwd(&path))) { + return posix_error_to_tuple(env, posix_errno); + } + + return am_ok; +} + +/** @brief Reads an entire file into \c result, stopping after \c size bytes or + * EOF. It will read until EOF if size is 0. */ +static posix_errno_t read_file(efile_data_t *d, size_t size, ErlNifBinary *result) { + size_t initial_buffer_size; + ssize_t bytes_read; + + if(size == 0) { + initial_buffer_size = 16 << 10; + } else { + initial_buffer_size = size; + } + + if(!enif_alloc_binary(initial_buffer_size, result)) { + return ENOMEM; + } + + bytes_read = 0; + + for(;;) { + ssize_t block_bytes_read; + SysIOVec read_vec[1]; + + read_vec[0].iov_base = result->data + bytes_read; + read_vec[0].iov_len = result->size - bytes_read; + + block_bytes_read = efile_readv(d, read_vec, 1); + + if(block_bytes_read < 0) { + enif_release_binary(result); + return d->posix_errno; + } + + bytes_read += block_bytes_read; + + if(block_bytes_read < (result->size - bytes_read)) { + /* EOF */ + break; + } else if(bytes_read == size) { + break; + } + + if(!enif_realloc_binary(result, bytes_read * 2)) { + enif_release_binary(result); + return ENOMEM; + } + } + + /* The file may have shrunk since we queried its size, so we have to do + * this even when the size is known. */ + if(bytes_read < result->size && !enif_realloc_binary(result, bytes_read)) { + ERTS_INTERNAL_ERROR("Failed to shrink read_file result."); + } + + return 0; +} + +static ERL_NIF_TERM read_file_nif(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) { + posix_errno_t posix_errno; + + efile_fileinfo_t info = {0}; + efile_path_t path; + efile_data_t *d; + + ErlNifBinary result; + + if(argc != 1) { + return enif_make_badarg(env); + } + + if((posix_errno = efile_marshal_path(env, argv[0], &path))) { + return posix_error_to_tuple(env, posix_errno); + } else if((posix_errno = efile_read_info(&path, 1, &info))) { + return posix_error_to_tuple(env, posix_errno); + } else if((posix_errno = efile_open(&path, EFILE_MODE_READ, efile_resource_type, &d))) { + return posix_error_to_tuple(env, posix_errno); + } + + posix_errno = read_file(d, info.size, &result); + enif_release_resource(d); + + if(posix_errno) { + return posix_error_to_tuple(env, posix_errno); + } + + return enif_make_tuple2(env, am_ok, enif_make_binary(env, &result)); +} + +static ERL_NIF_TERM altname_nif(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) { + posix_errno_t posix_errno; + + efile_path_t path; + ERL_NIF_TERM result; + + if(argc != 1) { + return enif_make_badarg(env); + } + + if((posix_errno = efile_marshal_path(env, argv[0], &path))) { + return posix_error_to_tuple(env, posix_errno); + } else if((posix_errno = efile_altname(env, &path, &result))) { + return posix_error_to_tuple(env, posix_errno); + } + + return enif_make_tuple2(env, am_ok, result); +} diff --git a/erts/emulator/nifs/common/prim_file_nif.h b/erts/emulator/nifs/common/prim_file_nif.h new file mode 100644 index 0000000000..cc9bc8f5c3 --- /dev/null +++ b/erts/emulator/nifs/common/prim_file_nif.h @@ -0,0 +1,240 @@ +/* + * %CopyrightBegin% + * + * Copyright Ericsson 2017. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * %CopyrightEnd% + */ + +typedef int posix_errno_t; + +enum efile_modes_t { + EFILE_MODE_READ = (1 << 0), + EFILE_MODE_WRITE = (1 << 1), /* Implies truncating file when used alone. */ + EFILE_MODE_APPEND = (1 << 2), + EFILE_MODE_EXCLUSIVE = (1 << 3), + EFILE_MODE_SYNC = (1 << 4), + + EFILE_MODE_SKIP_TYPE_CHECK = (1 << 5), /* Special for device files on Unix. */ + EFILE_MODE_NO_TRUNCATE = (1 << 6), /* Special for reopening on VxWorks. */ + + EFILE_MODE_READ_WRITE = EFILE_MODE_READ | EFILE_MODE_WRITE +}; + +enum efile_access_t { + EFILE_ACCESS_NONE = 0, + EFILE_ACCESS_READ = 1, + EFILE_ACCESS_WRITE = 2, + EFILE_ACCESS_READ_WRITE = EFILE_ACCESS_READ | EFILE_ACCESS_WRITE +}; + +enum efile_seek_t { + EFILE_SEEK_BOF, + EFILE_SEEK_CUR, + EFILE_SEEK_EOF +}; + +enum efile_filetype_t { + EFILE_FILETYPE_DEVICE, + EFILE_FILETYPE_DIRECTORY, + EFILE_FILETYPE_REGULAR, + EFILE_FILETYPE_SYMLINK, + EFILE_FILETYPE_OTHER +}; + +enum efile_advise_t { + EFILE_ADVISE_NORMAL, + EFILE_ADVISE_RANDOM, + EFILE_ADVISE_SEQUENTIAL, + EFILE_ADVISE_WILL_NEED, + EFILE_ADVISE_DONT_NEED, + EFILE_ADVISE_NO_REUSE +}; + +enum efile_state_t { + EFILE_STATE_IDLE = 0, + EFILE_STATE_BUSY = 1, + EFILE_STATE_CLOSE_PENDING = 2, + EFILE_STATE_CLOSED = 3 +}; + +typedef struct { + Sint64 size; /* Size of file */ + Uint32 type; /* Type of file -- one of EFILE_FILETYPE_*. */ + Uint32 access; /* Access to file -- one of EFILE_ACCESS_*. */ + Uint32 mode; /* Access permissions -- bit field. */ + Uint32 links; /* Number of links to file. */ + Uint32 major_device; /* Major device or file system. */ + Uint32 minor_device; /* Minor device (for devices). */ + Uint32 inode; /* Inode number. */ + Uint32 uid; /* User id of owner. */ + Uint32 gid; /* Group id of owner. */ + Sint64 a_time; /* Last time the file was accessed. */ + Sint64 m_time; /* Last time the file was modified. */ + Sint64 c_time; /* Windows: creation time, Unix: last inode + * change. */ +} efile_fileinfo_t; + +/* The smallest value that can be converted freely between universal, local, + * and POSIX time, as required by read_file_info/2. Corresponds to + * {{1902,1,1},{0,0,0}} */ +#define EFILE_MIN_FILETIME -2145916800 + +/* Initializes an efile_data_t; must be used in efile_open on success. */ +#define EFILE_INIT_RESOURCE(__d, __modes) do { \ + erts_atomic32_init_acqb(&(__d)->state, EFILE_STATE_IDLE); \ + (__d)->posix_errno = 0; \ + (__d)->modes = __modes; \ + } while(0) + +typedef struct { + erts_atomic32_t state; + + posix_errno_t posix_errno; + enum efile_modes_t modes; + + ErlNifMonitor monitor; +} efile_data_t; + +typedef ErlNifBinary efile_path_t; + +/* @brief Translates the given "raw name" into the format expected by the APIs + * used by the underlying implementation. The result is transient and does not + * need to be released. + * + * This may change the structure of the path and its results should never be + * passed on to the user. Refer to the OS-specific implementation for details. + * + * @param path The term to translate; it must have been encoded with + * prim_file:internal_native2name for compatibility reasons. */ +posix_errno_t efile_marshal_path(ErlNifEnv *env, ERL_NIF_TERM path, efile_path_t *result); + +/* @brief Returns the underlying handle as an implementation-defined term. + * + * This is an internal function intended to support tests and tricky + * operations like sendfile(2). */ +ERL_NIF_TERM efile_get_handle(ErlNifEnv *env, efile_data_t *d); + +/* @brief Read until EOF or the given iovec has been filled. + * + * @return -1 on failure, or the number of bytes read on success. The return + * value will be 0 if no bytes could be read before EOF or the end of the + * iovec. */ +Sint64 efile_readv(efile_data_t *d, SysIOVec *iov, int iovlen); + +/* @brief Write the entirety of the given iovec. + * + * @return -1 on failure, or the number of bytes written on success. "Partial" + * failures will be reported with -1 and not the number of bytes we managed to + * write to disk before the failure. */ +Sint64 efile_writev(efile_data_t *d, SysIOVec *iov, int iovlen); + +/* @brief As \c efile_readv, but starting from a file offset. */ +Sint64 efile_preadv(efile_data_t *d, Sint64 offset, SysIOVec *iov, int iovlen); + +/* @brief As \c efile_writev, but starting from a file offset. */ +Sint64 efile_pwritev(efile_data_t *d, Sint64 offset, SysIOVec *iov, int iovlen); + +int efile_seek(efile_data_t *d, enum efile_seek_t seek, Sint64 offset, Sint64 *new_position); + +int efile_sync(efile_data_t *d, int data_only); + +int efile_advise(efile_data_t *d, Sint64 offset, Sint64 length, enum efile_advise_t advise); +int efile_allocate(efile_data_t *d, Sint64 offset, Sint64 length); +int efile_truncate(efile_data_t *d); + +posix_errno_t efile_open(const efile_path_t *path, enum efile_modes_t modes, + ErlNifResourceType *nif_type, efile_data_t **d); + +/** @brief Closes a file. The file must have entered the CLOSED state prior to + * calling this to prevent double close. */ +int efile_close(efile_data_t *d); + +/* **** **** **** **** **** **** **** **** **** **** **** **** **** **** **** */ + +posix_errno_t efile_read_info(const efile_path_t *path, int follow_link, efile_fileinfo_t *result); + +/** @brief Sets the file times to the given values. Refer to efile_fileinfo_t + * for a description of each. */ +posix_errno_t efile_set_time(const efile_path_t *path, Sint64 a_time, Sint64 m_time, Sint64 c_time); + +/** @brief On Unix, this sets the file permissions according to the docs for + * file:write_file_info/2. On Windows it uses the "owner write permission" flag + * to toggle whether the file is read-only or not. */ +posix_errno_t efile_set_permissions(const efile_path_t *path, Uint32 permissions); + +/** @brief On Unix, this will set the owner/group to the given values. It will + * do nothing on other platforms. */ +posix_errno_t efile_set_owner(const efile_path_t *path, Uint32 owner, Uint32 group); + +/** @brief Resolves the final path of the given link. */ +posix_errno_t efile_read_link(ErlNifEnv *env, const efile_path_t *path, ERL_NIF_TERM *result); + +/** @brief Lists the contents of the given directory. + * @param result [out] A list of all the directory/file names contained in the + * given directory. */ +posix_errno_t efile_list_dir(ErlNifEnv *env, const efile_path_t *path, ERL_NIF_TERM *result); + +/** @brief Changes the name of an existing file or directory, from old_path + * to new_path. + * + * If old_path and new_path refer to the same file or directory, it does + * nothing and returns success. Otherwise if new_path already exists, it will + * be deleted and replaced by src subject to the following conditions: + * + * If old_path is a directory, new_path may be an empty directory. + * If old_path is a file, new_path may be a file. + * + * Neither of these are guaranteed to be atomic. In any other situation where + * new_path already exists, the rename will fail. + * + * Some possible error codes: + * + * - EACCES: Either paths or one of their parent directories can't be read + * and/or written. + * - EEXIST: new_path is a non-empty directory. + * - EINVAL: old_path is a root directory or new_path is a subdirectory + * of new_path. + * - EISDIR: new_path is a directory, but old_path is not. + * - ENOTDIR: old_path is a directory, but new_path is not. + * - ENOENT: old_path doesn't exist, or either path is "". + * - EXDEV: The paths are on different filesystems. + * + * The implementation of rename may allow cross-filesystem renames, + * but the caller should be prepared to emulate it with copy and + * delete if errno is EXDEV. */ +posix_errno_t efile_rename(const efile_path_t *old_path, const efile_path_t *new_path); + +posix_errno_t efile_make_hard_link(const efile_path_t *existing_path, const efile_path_t *new_path); +posix_errno_t efile_make_soft_link(const efile_path_t *existing_path, const efile_path_t *new_path); +posix_errno_t efile_make_dir(const efile_path_t *path); + +posix_errno_t efile_del_file(const efile_path_t *path); +posix_errno_t efile_del_dir(const efile_path_t *path); + +posix_errno_t efile_get_cwd(ErlNifEnv *env, ERL_NIF_TERM *result); +posix_errno_t efile_set_cwd(const efile_path_t *path); + +/** @brief A Windows-specific function for returning the working directory of a + * given device. + * + * @param device_index The drive index; 1 for A, 2 for B, etc. + * @param result [out] The working directory of the given device + */ +posix_errno_t efile_get_device_cwd(ErlNifEnv *env, int device_index, ERL_NIF_TERM *result); + +/** @brief A Windows-specific function for returning the 8.3-name of a given + * file or directory. */ +posix_errno_t efile_altname(ErlNifEnv *env, const efile_path_t *path, ERL_NIF_TERM *result); diff --git a/erts/emulator/nifs/common/zlib_nif.c b/erts/emulator/nifs/common/zlib_nif.c index fa29b4fb71..b709ed5a6f 100644 --- a/erts/emulator/nifs/common/zlib_nif.c +++ b/erts/emulator/nifs/common/zlib_nif.c @@ -717,7 +717,9 @@ static ERL_NIF_TERM zlib_deflateEnd(ErlNifEnv *env, int argc, const ERL_NIF_TERM static ERL_NIF_TERM zlib_deflateParams(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) { zlib_data_t *d; + int res, level, strategy; + Bytef dummy_buffer; if(argc != 3 || !get_zlib_data(env, argv[0], &d) || !enif_get_int(env, argv[1], &level) @@ -729,12 +731,27 @@ static ERL_NIF_TERM zlib_deflateParams(ErlNifEnv *env, int argc, const ERL_NIF_T return enif_raise_exception(env, am_not_initialized); } - /* deflateParams will flush everything currently in the stream, corrupting - * the heap unless it's empty. We therefore pretend to have a full output - * buffer, forcing a Z_BUF_ERROR if there's anything left to be flushed. */ - d->s.avail_out = 0; + /* This is a bit of a hack; deflateParams flushes with Z_BLOCK which won't + * stop at a byte boundary, so we can't split this operation up, and we + * can't allocate a buffer large enough to fit it in one go since we have + * to support zlib versions that lack deflatePending. + * + * We therefore flush everything prior to this call to ensure that we are + * stopped on a byte boundary and have no pending data. We then hand it a + * dummy buffer to detect when this assumption doesn't hold (Hopefully + * never), and to smooth over an issue with zlib 1.2.11 which always + * returns Z_BUF_ERROR when d->s.avail_out is 0, regardless of whether + * there's any pending data or not. */ + + d->s.next_out = &dummy_buffer; + d->s.avail_out = 1; + res = deflateParams(&d->s, level, strategy); + if(d->s.avail_out == 0) { + return zlib_return(env, Z_STREAM_ERROR); + } + return zlib_return(env, res); } @@ -929,7 +946,7 @@ static ERL_NIF_TERM zlib_inflate(ErlNifEnv *env, int argc, const ERL_NIF_TERM ar return enif_raise_exception(env, am_not_initialized); } - if(d->eos_seen) { + if(d->eos_seen && enif_ioq_size(d->input_queue) > 0) { int res; switch(d->eos_behavior) { @@ -943,11 +960,10 @@ static ERL_NIF_TERM zlib_inflate(ErlNifEnv *env, int argc, const ERL_NIF_TERM ar } d->eos_seen = 0; + break; case EOS_BEHAVIOR_CUT: zlib_reset_input(d); - - return enif_make_tuple2(env, am_finished, enif_make_list(env, 0)); } } diff --git a/erts/emulator/nifs/unix/unix_prim_file.c b/erts/emulator/nifs/unix/unix_prim_file.c new file mode 100644 index 0000000000..4a6c476882 --- /dev/null +++ b/erts/emulator/nifs/unix/unix_prim_file.c @@ -0,0 +1,944 @@ +/* + * %CopyrightBegin% + * + * Copyright Ericsson 2017. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * %CopyrightEnd% + */ + +#include "erl_nif.h" +#include "config.h" +#include "sys.h" + +#ifdef VALGRIND +# include <valgrind/memcheck.h> +#endif + +#include "prim_file_nif.h" + +#if defined(__APPLE__) && defined(__MACH__) && !defined(__DARWIN__) +#define __DARWIN__ 1 +#endif + +#if defined(__DARWIN__) || defined(HAVE_LINUX_FALLOC_H) || defined(HAVE_POSIX_FALLOCATE) +#include <fcntl.h> +#endif + +#ifdef HAVE_LINUX_FALLOC_H +#include <linux/falloc.h> +#endif + +#include <utime.h> + +/* Macros for testing file types. */ +#ifdef NO_UMASK +#define FILE_MODE 0644 +#define DIR_MODE 0755 +#else +#define FILE_MODE 0666 +#define DIR_MODE 0777 +#endif + +/* Old platforms might not have IOV_MAX defined. */ +#if !defined(IOV_MAX) && defined(UIO_MAXIOV) +#define IOV_MAX UIO_MAXIOV +#elif !defined(IOV_MAX) +#define IOV_MAX 16 +#endif + +typedef struct { + efile_data_t common; + int fd; +} efile_unix_t; + +static int has_invalid_null_termination(const ErlNifBinary *path) { + const char *null_pos, *end_pos; + + null_pos = memchr(path->data, '\0', path->size); + end_pos = (const char*)&path->data[path->size] - 1; + + if(null_pos == NULL) { + return 1; + } + + /* prim_file:internal_name2native sometimes feeds us data that is "doubly" + * NUL-terminated, so we'll accept any number of trailing NULs so long as + * they aren't interrupted by anything else. */ + while(null_pos < end_pos && (*null_pos) == '\0') { + null_pos++; + } + + return null_pos != end_pos; +} + +posix_errno_t efile_marshal_path(ErlNifEnv *env, ERL_NIF_TERM path, efile_path_t *result) { + if(!enif_inspect_binary(env, path, result)) { + return EINVAL; + } + + if(has_invalid_null_termination(result)) { + return EINVAL; + } + + return 0; +} + +ERL_NIF_TERM efile_get_handle(ErlNifEnv *env, efile_data_t *d) { + efile_unix_t *u = (efile_unix_t*)d; + + ERL_NIF_TERM result; + unsigned char *bits; + + bits = enif_make_new_binary(env, sizeof(u->fd), &result); + memcpy(bits, &u->fd, sizeof(u->fd)); + + return result; +} + +static int open_file_type_check(const efile_path_t *path, int fd) { + struct stat file_info; + int error; + +#ifndef HAVE_FSTAT + error = stat((const char*)path->data, &file_info); + (void)fd; +#else + error = fstat(fd, &file_info); + (void)path; +#endif + + if(error < 0) { + /* If we failed to stat assume success and let the next call handle the + * error. The old driver checked whether the file was to be used + * immediately in a read within the call, but the new implementation + * never does that. */ + return 1; + } + + /* Allow everything that isn't a directory, and error out on the next call + * if it's unsupported. */ + if(S_ISDIR(file_info.st_mode)) { + return 0; + } + + return 1; +} + +posix_errno_t efile_open(const efile_path_t *path, enum efile_modes_t modes, + ErlNifResourceType *nif_type, efile_data_t **d) { + + int flags, fd; + + flags = 0; + + if(modes & EFILE_MODE_READ && !(modes & EFILE_MODE_WRITE)) { + flags |= O_RDONLY; + } else if(modes & EFILE_MODE_WRITE && !(modes & EFILE_MODE_READ)) { + if(!(modes & EFILE_MODE_NO_TRUNCATE)) { + flags |= O_TRUNC; + } + + flags |= O_WRONLY | O_CREAT; + } else if(modes & EFILE_MODE_READ_WRITE) { + flags |= O_RDWR | O_CREAT; + } else { + return EINVAL; + } + + if(modes & EFILE_MODE_APPEND) { + flags &= ~O_TRUNC; + flags |= O_APPEND; + } + + if(modes & EFILE_MODE_EXCLUSIVE) { + flags |= O_EXCL; + } + + if(modes & EFILE_MODE_SYNC) { +#ifndef O_SYNC + return ENOTSUP; +#else + flags |= O_SYNC; +#endif + } + + do { + fd = open((const char*)path->data, flags, FILE_MODE); + } while(fd == -1 && errno == EINTR); + + if(fd != -1) { + efile_unix_t *u; + + if(!(modes & EFILE_MODE_SKIP_TYPE_CHECK) && !open_file_type_check(path, fd)) { + close(fd); + + /* This is blatantly incorrect, but we're documented as returning + * this for everything that isn't a file. */ + return EISDIR; + } + + u = (efile_unix_t*)enif_alloc_resource(nif_type, sizeof(efile_unix_t)); + u->fd = fd; + + EFILE_INIT_RESOURCE(&u->common, modes); + (*d) = &u->common; + + return 0; + } + + (*d) = NULL; + return errno; +} + +int efile_close(efile_data_t *d) { + efile_unix_t *u = (efile_unix_t*)d; + int fd; + + ASSERT(erts_atomic32_read_nob(&d->state) == EFILE_STATE_CLOSED); + ASSERT(u->fd != -1); + + fd = u->fd; + u->fd = -1; + + /* close(2) either always closes (*BSD, Linux) or leaves the fd in an + * undefined state (POSIX 2008, Solaris), so we must not retry on EINTR. */ + + if(close(fd) < 0) { + u->common.posix_errno = errno; + return 0; + } + + return 1; +} + +static void shift_iov(SysIOVec **iov, int *iovlen, ssize_t shift) { + SysIOVec *head_vec = (*iov); + + ASSERT(shift >= 0); + + while(shift > 0) { + ASSERT(head_vec < &(*iov)[*iovlen]); + + if(shift < head_vec->iov_len) { + head_vec->iov_base = (char*)head_vec->iov_base + shift; + head_vec->iov_len -= shift; + break; + } else { + shift -= head_vec->iov_len; + head_vec++; + } + } + + (*iovlen) -= head_vec - (*iov); + (*iov) = head_vec; +} + +Sint64 efile_readv(efile_data_t *d, SysIOVec *iov, int iovlen) { + efile_unix_t *u = (efile_unix_t*)d; + + Sint64 bytes_read; + ssize_t result; + + bytes_read = 0; + + do { + int use_fallback = 0; + + if(iovlen < 1) { + result = 0; + break; + } + + /* writev(2) implies readv(2) */ +#ifdef HAVE_WRITEV + result = readv(u->fd, iov, MIN(IOV_MAX, iovlen)); + + /* Fall back to using read(2) if readv(2) reports that the combined + * size of iov is greater than SSIZE_T_MAX. */ + use_fallback = (result < 0 && errno == EINVAL); +#else + use_fallback = 1; +#endif + + if(use_fallback) { + result = read(u->fd, iov->iov_base, iov->iov_len); + } + + if(result > 0) { + shift_iov(&iov, &iovlen, result); + bytes_read += result; + } + } while(result > 0 || (result < 0 && errno == EINTR)); + + u->common.posix_errno = errno; + + if(result == 0 && bytes_read > 0) { + return bytes_read; + } + + return result; +} + +Sint64 efile_writev(efile_data_t *d, SysIOVec *iov, int iovlen) { + efile_unix_t *u = (efile_unix_t*)d; + + Sint64 bytes_written; + ssize_t result; + + bytes_written = 0; + + do { + int use_fallback = 0; + + if(iovlen < 1) { + result = 0; + break; + } + +#ifdef HAVE_WRITEV + result = writev(u->fd, iov, MIN(IOV_MAX, iovlen)); + + /* Fall back to using write(2) if writev(2) reports that the combined + * size of iov is greater than SSIZE_T_MAX. */ + use_fallback = (result < 0 && errno == EINVAL); +#else + use_fallback = 1; +#endif + + if(use_fallback) { + result = write(u->fd, iov->iov_base, iov->iov_len); + } + + if(result > 0) { + shift_iov(&iov, &iovlen, result); + bytes_written += result; + } + } while(result > 0 || (result < 0 && errno == EINTR)); + + u->common.posix_errno = errno; + + if(result == 0 && bytes_written > 0) { + return bytes_written; + } + + return result; +} + +Sint64 efile_preadv(efile_data_t *d, Sint64 offset, SysIOVec *iov, int iovlen) { + efile_unix_t *u = (efile_unix_t*)d; + + Uint64 bytes_read; + Sint64 result; + +#if !defined(HAVE_PREADV) && !defined(HAVE_PREAD) + /* This function is documented as leaving the file position undefined, but + * the old driver always reset it so there's probably code in the wild that + * relies on this behavior. */ + off_t original_position = lseek(u->fd, 0, SEEK_CUR); + + if(original_position < 0 || lseek(u->fd, offset, SEEK_SET) < 0) { + u->common.posix_errno = errno; + return -1; + } +#endif + + bytes_read = 0; + + do { + if(iovlen < 1) { + result = 0; + break; + } + +#if defined(HAVE_PREADV) + result = preadv(u->fd, iov, MIN(IOV_MAX, iovlen), offset); +#elif defined(HAVE_PREAD) + result = pread(u->fd, iov->iov_base, iov->iov_len, offset); +#else + result = read(u->fd, iov->iov_base, iov->iov_len); +#endif + + if(result > 0) { + shift_iov(&iov, &iovlen, result); + bytes_read += result; + offset += result; + } + } while(result > 0 || (result < 0 && errno == EINTR)); + + u->common.posix_errno = errno; + +#if !defined(HAVE_PREADV) && !defined(HAVE_PREAD) + if(result >= 0) { + if(lseek(u->fd, original_position, SEEK_SET) < 0) { + u->common.posix_errno = errno; + return -1; + } + } +#endif + + if(result == 0 && bytes_read > 0) { + return bytes_read; + } + + return result; +} + +Sint64 efile_pwritev(efile_data_t *d, Sint64 offset, SysIOVec *iov, int iovlen) { + efile_unix_t *u = (efile_unix_t*)d; + + Sint64 bytes_written; + ssize_t result; + +#if !defined(HAVE_PWRITEV) && !defined(HAVE_PWRITE) + off_t original_position = lseek(u->fd, 0, SEEK_CUR); + + if(original_position < 0 || lseek(u->fd, offset, SEEK_SET) < 0) { + u->common.posix_errno = errno; + return -1; + } +#endif + + bytes_written = 0; + + do { + if(iovlen < 1) { + result = 0; + break; + } + +#if defined(HAVE_PWRITEV) + result = pwritev(u->fd, iov, MIN(IOV_MAX, iovlen), offset); +#elif defined(HAVE_PWRITE) + result = pwrite(u->fd, iov->iov_base, iov->iov_len, offset); +#else + result = write(u->fd, iov->iov_base, iov->iov_len); +#endif + + if(result > 0) { + shift_iov(&iov, &iovlen, result); + bytes_written += result; + offset += result; + } + } while(result > 0 || (result < 0 && errno == EINTR)); + + u->common.posix_errno = errno; + +#if !defined(HAVE_PWRITEV) && !defined(HAVE_PWRITE) + if(result >= 0) { + if(lseek(u->fd, original_position, SEEK_SET) < 0) { + u->common.posix_errno = errno; + return -1; + } + } +#endif + + if(result == 0 && bytes_written > 0) { + return bytes_written; + } + + return result; +} + +int efile_seek(efile_data_t *d, enum efile_seek_t seek, Sint64 offset, Sint64 *new_position) { + efile_unix_t *u = (efile_unix_t*)d; + off_t result; + int whence; + + switch(seek) { + case EFILE_SEEK_BOF: whence = SEEK_SET; break; + case EFILE_SEEK_CUR: whence = SEEK_CUR; break; + case EFILE_SEEK_EOF: whence = SEEK_END; break; + default: ERTS_INTERNAL_ERROR("Invalid seek parameter"); + } + + result = lseek(u->fd, offset, whence); + + /* + * The man page for lseek (on SunOs 5) says: + * + * "if fildes is a remote file descriptor and offset is negative, lseek() + * returns the file pointer even if it is negative." + */ + if(result < 0 && errno == 0) { + errno = EINVAL; + } + + if(result < 0) { + u->common.posix_errno = errno; + return 0; + } + + (*new_position) = result; + + return 1; +} + +int efile_sync(efile_data_t *d, int data_only) { + efile_unix_t *u = (efile_unix_t*)d; + +#if defined(HAVE_FDATASYNC) && !defined(__DARWIN__) + if(data_only) { + if(fdatasync(u->fd) < 0) { + u->common.posix_errno = errno; + return 0; + } + + return 1; + } +#endif + +#if defined(__DARWIN__) && defined(F_FULLFSYNC) + if(fcntl(u->fd, F_FULLFSYNC) < 0) { +#else + if(fsync(u->fd) < 0) { +#endif + u->common.posix_errno = errno; + return 0; + } + + return 1; +} + +int efile_advise(efile_data_t *d, Sint64 offset, Sint64 length, enum efile_advise_t advise) { + efile_unix_t *u = (efile_unix_t*)d; +#ifdef HAVE_POSIX_FADVISE + int p_advise; + + switch(advise) { + case EFILE_ADVISE_NORMAL: p_advise = POSIX_FADV_NORMAL; break; + case EFILE_ADVISE_RANDOM: p_advise = POSIX_FADV_RANDOM; break; + case EFILE_ADVISE_SEQUENTIAL: p_advise = POSIX_FADV_SEQUENTIAL; break; + case EFILE_ADVISE_WILL_NEED: p_advise = POSIX_FADV_WILLNEED; break; + case EFILE_ADVISE_DONT_NEED: p_advise = POSIX_FADV_DONTNEED; break; + case EFILE_ADVISE_NO_REUSE: p_advise = POSIX_FADV_NOREUSE; break; + default: + u->common.posix_errno = EINVAL; + return 0; + } + + if(posix_fadvise(u->fd, offset, length, p_advise) < 0) { + u->common.posix_errno = errno; + return 0; + } + + return 1; +#else + /* We'll pretend to support this syscall as it's only a recommendation even + * on systems that do support it. */ + return 1; +#endif +} + +int efile_allocate(efile_data_t *d, Sint64 offset, Sint64 length) { + efile_unix_t *u = (efile_unix_t*)d; + int ret = -1; + + /* We prefer OS-specific methods, but fall back to posix_fallocate on + * failure. It's unclear whether this has any practical benefit on + * modern systems, but the old driver did it. */ + +#if defined(HAVE_FALLOCATE) + /* Linux-specific */ + do { + ret = fallocate(u->fd, FALLOC_FL_KEEP_SIZE, offset, length); + } while(ret < 0 && errno == EINTR); +#elif defined(F_PREALLOCATE) + /* Mac-specific */ + fstore_t fs = {}; + + fs.fst_flags = F_ALLOCATECONTIG; + fs.fst_posmode = F_VOLPOSMODE; + fs.fst_offset = offset; + fs.fst_length = length; + + ret = fcntl(u->fd, F_PREALLOCATE, &fs); + if(ret < 0) { + fs.fst_flags = F_ALLOCATEALL; + ret = fcntl(u->fd, F_PREALLOCATE, &fs); + } +#elif !defined(HAVE_POSIX_FALLOCATE) + u->common.posix_errno = ENOTSUP; + return 0; +#endif + +#ifdef HAVE_POSIX_FALLOCATE + if(ret < 0) { + do { + ret = posix_fallocate(u->fd, offset, length); + + /* On Linux and Solaris for example, posix_fallocate() returns a + * positive error number on error and it does not set errno. On + * FreeBSD however (9.0 at least), it returns -1 on error and it + * sets errno. */ + if (ret > 0) { + errno = ret; + ret = -1; + } + } while(ret < 0 && errno == EINTR); + } +#endif + + if(ret < 0) { + u->common.posix_errno = errno; + return 0; + } + + return 1; +} + +int efile_truncate(efile_data_t *d) { + efile_unix_t *u = (efile_unix_t*)d; + off_t offset; + + offset = lseek(u->fd, 0, SEEK_CUR); + + if(offset < 0) { + u->common.posix_errno = errno; + return 0; + } + + if(ftruncate(u->fd, offset) < 0) { + u->common.posix_errno = errno; + return 0; + } + + return 1; +} + +posix_errno_t efile_read_info(const efile_path_t *path, int follow_links, efile_fileinfo_t *result) { + struct stat data; + + if(follow_links) { + if(stat((const char*)path->data, &data) < 0) { + return errno; + } + } else { + if(lstat((const char*)path->data, &data) < 0) { + return errno; + } + } + + if(S_ISCHR(data.st_mode) || S_ISBLK(data.st_mode)) { + result->type = EFILE_FILETYPE_DEVICE; + } else if(S_ISDIR(data.st_mode)) { + result->type = EFILE_FILETYPE_DIRECTORY; + } else if(S_ISREG(data.st_mode)) { + result->type = EFILE_FILETYPE_REGULAR; + } else if(S_ISLNK(data.st_mode)) { + result->type = EFILE_FILETYPE_SYMLINK; + } else { + result->type = EFILE_FILETYPE_OTHER; + } + + result->a_time = (Sint64)data.st_atime; + result->m_time = (Sint64)data.st_mtime; + result->c_time = (Sint64)data.st_ctime; + result->size = data.st_size; + + result->major_device = data.st_dev; + result->minor_device = data.st_rdev; + result->links = data.st_nlink; + result->inode = data.st_ino; + result->mode = data.st_mode; + result->uid = data.st_uid; + result->gid = data.st_gid; + +#ifndef NO_ACCESS + result->access = EFILE_ACCESS_NONE; + + if(access((const char*)path->data, R_OK) == 0) { + result->access |= EFILE_ACCESS_READ; + } + if(access((const char*)path->data, W_OK) == 0) { + result->access |= EFILE_ACCESS_WRITE; + } +#else + /* Just look at read/write access for owner. */ + result->access = ((data.st_mode >> 6) & 07) >> 1; +#endif + + return 0; +} + +posix_errno_t efile_set_permissions(const efile_path_t *path, Uint32 permissions) { + const mode_t MUTABLE_MODES = (S_ISUID | S_ISGID | S_IRWXU | S_IRWXG | S_IRWXO); + mode_t new_modes = permissions & MUTABLE_MODES; + + if(chmod((const char*)path->data, new_modes) < 0) { + new_modes &= ~(S_ISUID | S_ISGID); + + if (chmod((const char*)path->data, new_modes) < 0) { + return errno; + } + } + + return 0; +} + +posix_errno_t efile_set_owner(const efile_path_t *path, Uint32 owner, Uint32 group) { + if(chown((const char*)path->data, owner, group) < 0) { + return errno; + } + + return 0; +} + +posix_errno_t efile_set_time(const efile_path_t *path, Sint64 a_time, Sint64 m_time, Sint64 c_time) { + struct utimbuf tval; + + tval.actime = (time_t)a_time; + tval.modtime = (time_t)m_time; + + (void)c_time; + + if(utime((const char*)path->data, &tval) < 0) { + return errno; + } + + return 0; +} + +posix_errno_t efile_read_link(ErlNifEnv *env, const efile_path_t *path, ERL_NIF_TERM *result) { + ErlNifBinary result_bin; + + if(!enif_alloc_binary(256, &result_bin)) { + return ENOMEM; + } + + for(;;) { + ssize_t bytes_copied; + + bytes_copied = readlink((const char*)path->data, (char*)result_bin.data, + result_bin.size); + + if(bytes_copied <= 0) { + posix_errno_t saved_errno = errno; + enif_release_binary(&result_bin); + return saved_errno; + } else if(bytes_copied < result_bin.size) { + if(!enif_realloc_binary(&result_bin, bytes_copied)) { + enif_release_binary(&result_bin); + return ENOMEM; + } + + (*result) = enif_make_binary(env, &result_bin); + + return 0; + } + + /* The result didn't fit into the buffer, so we'll try again with a + * larger one. */ + + if(!enif_realloc_binary(&result_bin, result_bin.size * 2)) { + enif_release_binary(&result_bin); + return ENOMEM; + } + } +} + +static int is_ignored_name(int name_length, const char *name) { + if(name_length == 1 && name[0] == '.') { + return 1; + } else if(name_length == 2 && memcmp(name, "..", 2) == 0) { + return 1; + } + + return 0; +} + +posix_errno_t efile_list_dir(ErlNifEnv *env, const efile_path_t *path, ERL_NIF_TERM *result) { + ERL_NIF_TERM list_head; + + struct dirent *dir_entry; + DIR *dir_stream; + + dir_stream = opendir((const char*)path->data); + if(dir_stream == NULL) { + posix_errno_t saved_errno = errno; + *result = enif_make_list(env, 0); + return saved_errno; + } + + list_head = enif_make_list(env, 0); + dir_entry = readdir(dir_stream); + + while(dir_entry != NULL) { + int name_length = strlen(dir_entry->d_name); + + if(!is_ignored_name(name_length, dir_entry->d_name)) { + unsigned char *name_bytes; + ERL_NIF_TERM name_term; + + name_bytes = enif_make_new_binary(env, name_length, &name_term); + sys_memcpy(name_bytes, dir_entry->d_name, name_length); + + list_head = enif_make_list_cell(env, name_term, list_head); + } + + dir_entry = readdir(dir_stream); + } + + (*result) = list_head; + closedir(dir_stream); + + return 0; +} + +posix_errno_t efile_rename(const efile_path_t *old_path, const efile_path_t *new_path) { + if(rename((const char*)old_path->data, (const char*)new_path->data) < 0) { + if(errno == ENOTEMPTY) { + return EEXIST; + } + + if(strcmp((const char*)old_path->data, "/") == 0) { + /* Alpha reports renaming / as EBUSY and Linux reports it as EACCES + * instead of EINVAL.*/ + return EINVAL; + } + + return errno; + } + + return 0; +} + +posix_errno_t efile_make_hard_link(const efile_path_t *existing_path, const efile_path_t *new_path) { + if(link((const char*)existing_path->data, (const char*)new_path->data) < 0) { + return errno; + } + + return 0; +} + +posix_errno_t efile_make_soft_link(const efile_path_t *existing_path, const efile_path_t *new_path) { + if(symlink((const char*)existing_path->data, (const char*)new_path->data) < 0) { + return errno; + } + + return 0; +} + +posix_errno_t efile_make_dir(const efile_path_t *path) { +#ifdef NO_MKDIR_MODE + if(mkdir((const char*)path->data) < 0) { +#else + if(mkdir((const char*)path->data, DIR_MODE) < 0) { +#endif + return errno; + } + + return 0; +} + +posix_errno_t efile_del_file(const efile_path_t *path) { + if(unlink((const char*)path->data) < 0) { + /* Linux sets the wrong error code. */ + if(errno == EISDIR) { + return EPERM; + } + + return errno; + } + + return 0; +} + +posix_errno_t efile_del_dir(const efile_path_t *path) { + if(rmdir((const char*)path->data) < 0) { + posix_errno_t saved_errno = errno; + + if(saved_errno == ENOTEMPTY) { + saved_errno = EEXIST; + } + + /* The error code might be wrong if we're trying to delete the current + * directory. */ + if(saved_errno == EEXIST) { + struct stat path_stat, cwd_stat; + int has_stat; + + has_stat = (stat((const char*)path->data, &path_stat) == 0); + has_stat &= (stat(".", &cwd_stat) == 0); + + if(has_stat && path_stat.st_ino == cwd_stat.st_ino) { + if(path_stat.st_dev == cwd_stat.st_dev) { + return EINVAL; + } + } + } + + return saved_errno; + } + + return 0; +} + +posix_errno_t efile_set_cwd(const efile_path_t *path) { + if(chdir((const char*)path->data) < 0) { + return errno; + } + + return 0; +} + +posix_errno_t efile_get_device_cwd(ErlNifEnv *env, int device_index, ERL_NIF_TERM *result) { + (void)device_index; + (void)result; + (void)env; + + return ENOTSUP; +} + +posix_errno_t efile_get_cwd(ErlNifEnv *env, ERL_NIF_TERM *result) { + ErlNifBinary result_bin; + size_t bytes_copied; + + if(!enif_alloc_binary(256, &result_bin)) { + return ENOMEM; + } + + while(getcwd((char*)result_bin.data, result_bin.size) == NULL) { + posix_errno_t saved_errno = errno; + + if(saved_errno != ERANGE) { + enif_release_binary(&result_bin); + return saved_errno; + } else { + if(!enif_realloc_binary(&result_bin, result_bin.size * 2)) { + enif_release_binary(&result_bin); + return ENOMEM; + } + } + } + + /* getcwd(2) guarantees null-termination. */ + bytes_copied = strlen((const char*)result_bin.data); + + if(!enif_realloc_binary(&result_bin, bytes_copied)) { + enif_release_binary(&result_bin); + return ENOMEM; + } + + (*result) = enif_make_binary(env, &result_bin); + + return 0; +} + +posix_errno_t efile_altname(ErlNifEnv *env, const efile_path_t *path, ERL_NIF_TERM *result) { + (void)path; + (void)result; + + return ENOTSUP; +} diff --git a/erts/emulator/nifs/win32/win_prim_file.c b/erts/emulator/nifs/win32/win_prim_file.c new file mode 100644 index 0000000000..9b79182f2c --- /dev/null +++ b/erts/emulator/nifs/win32/win_prim_file.c @@ -0,0 +1,1428 @@ +/* + * %CopyrightBegin% + * + * Copyright Ericsson 2017. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * %CopyrightEnd% + */ + +#include "erl_nif.h" +#include "config.h" +#include "sys.h" + +#include "prim_file_nif.h" + +#include <windows.h> +#include <strsafe.h> +#include <wchar.h> + +#define IS_SLASH(a) ((a) == L'\\' || (a) == L'/') + +#define FILE_SHARE_FLAGS (FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE) + +#define LP_PREFIX L"\\\\?\\" +#define LP_PREFIX_SIZE (sizeof(LP_PREFIX) - sizeof(WCHAR)) +#define LP_PREFIX_LENGTH (LP_PREFIX_SIZE / sizeof(WCHAR)) + +#define PATH_LENGTH(path) (path->size / sizeof(WCHAR) - 1) + +#define ASSERT_PATH_FORMAT(path) \ + do { \ + ASSERT(PATH_LENGTH(path) >= 4 && \ + !memcmp(path->data, LP_PREFIX, LP_PREFIX_SIZE)); \ + ASSERT(PATH_LENGTH(path) == wcslen((WCHAR*)path->data)); \ + } while(0) + +#define TICKS_PER_SECOND (10000000ULL) +#define EPOCH_DIFFERENCE (11644473600LL) + +#define FILETIME_TO_EPOCH(epoch, ft) \ + do { \ + ULARGE_INTEGER ull; \ + ull.LowPart = (ft).dwLowDateTime; \ + ull.HighPart = (ft).dwHighDateTime; \ + (epoch) = ((ull.QuadPart / TICKS_PER_SECOND) - EPOCH_DIFFERENCE); \ + } while(0) + +#define EPOCH_TO_FILETIME(ft, epoch) \ + do { \ + ULARGE_INTEGER ull; \ + ull.QuadPart = (((epoch) + EPOCH_DIFFERENCE) * TICKS_PER_SECOND); \ + (ft).dwLowDateTime = ull.LowPart; \ + (ft).dwHighDateTime = ull.HighPart; \ + } while(0) + +typedef struct { + efile_data_t common; + HANDLE handle; +} efile_win_t; + +static int windows_to_posix_errno(DWORD last_error); + +static int has_invalid_null_termination(const ErlNifBinary *path) { + const WCHAR *null_pos, *end_pos; + + null_pos = wmemchr((const WCHAR*)path->data, L'\0', path->size); + end_pos = (const WCHAR*)&path->data[path->size] - 1; + + if(null_pos == NULL) { + return 1; + } + + /* prim_file:internal_name2native sometimes feeds us data that is "doubly" + * NUL-terminated, so we'll accept any number of trailing NULs so long as + * they aren't interrupted by anything else. */ + while(null_pos < end_pos && (*null_pos) == L'\0') { + null_pos++; + } + + return null_pos != end_pos; +} + +static posix_errno_t get_full_path(ErlNifEnv *env, WCHAR *input, efile_path_t *result) { + DWORD maximum_length, actual_length; + int add_long_prefix; + + maximum_length = GetFullPathNameW(input, 0, NULL, NULL); + add_long_prefix = 0; + + if(maximum_length == 0) { + /* POSIX doesn't have the concept of a "path error" in the same way + * Windows does, so we'll return ENOENT since that's what most POSIX + * APIs would return if they were fed such garbage. */ + return ENOENT; + } + + maximum_length += LP_PREFIX_LENGTH; + + if(!enif_alloc_binary(maximum_length * sizeof(WCHAR), result)) { + return ENOMEM; + } + + actual_length = GetFullPathNameW(input, maximum_length, (WCHAR*)result->data, NULL); + + if(actual_length < maximum_length) { + int has_long_path_prefix; + WCHAR *path_start; + + /* Make sure we have a long-path prefix; GetFullPathNameW only adds one + * if the path is relative. */ + has_long_path_prefix = actual_length >= LP_PREFIX_LENGTH && + !sys_memcmp(result->data, LP_PREFIX, LP_PREFIX_SIZE); + + if(!has_long_path_prefix) { + sys_memmove(result->data + LP_PREFIX_SIZE, result->data, + (actual_length + 1) * sizeof(WCHAR)); + sys_memcpy(result->data, LP_PREFIX, LP_PREFIX_SIZE); + actual_length += LP_PREFIX_LENGTH; + } + + path_start = (WCHAR*)result->data; + + /* We're removing trailing slashes since quite a few APIs refuse to + * work with them, and none require them. We only check the last + * character since GetFullPathNameW folds slashes together. */ + if(IS_SLASH(path_start[actual_length - 1])) { + if(path_start[actual_length - 2] != L':') { + path_start[actual_length - 1] = L'\0'; + actual_length--; + } + } + + if(!enif_realloc_binary(result, (actual_length + 1) * sizeof(WCHAR))) { + enif_release_binary(result); + return ENOMEM; + } + + enif_make_binary(env, result); + return 0; + } + + /* We may end up here if the current directory changes to something longer + * between/during GetFullPathName. There's nothing sensible we can do about + * this. */ + + enif_release_binary(result); + + return EINVAL; +} + +posix_errno_t efile_marshal_path(ErlNifEnv *env, ERL_NIF_TERM path, efile_path_t *result) { + ErlNifBinary raw_path; + + if(!enif_inspect_binary(env, path, &raw_path)) { + return EINVAL; + } else if(raw_path.size % sizeof(WCHAR)) { + return EINVAL; + } + + if(has_invalid_null_termination(&raw_path)) { + return EINVAL; + } + + return get_full_path(env, (WCHAR*)raw_path.data, result); +} + +ERL_NIF_TERM efile_get_handle(ErlNifEnv *env, efile_data_t *d) { + efile_win_t *w = (efile_win_t*)d; + + ERL_NIF_TERM result; + unsigned char *bits; + + bits = enif_make_new_binary(env, sizeof(w->handle), &result); + memcpy(bits, &w->handle, sizeof(w->handle)); + + return result; +} + +/** @brief Converts a native path to the preferred form in "erlang space," + * without path-prefixes, forward-slashes, or NUL terminators. */ +static int normalize_path_result(ErlNifBinary *path) { + WCHAR *path_iterator, *path_start, *path_end; + int length; + + path_start = (WCHAR*)path->data; + length = wcslen(path_start); + + ASSERT(length < path->size / sizeof(WCHAR)); + + /* Get rid of the long-path prefix, if present. */ + if(length >= LP_PREFIX_LENGTH) { + if(!sys_memcmp(path_start, LP_PREFIX, LP_PREFIX_SIZE)) { + length -= LP_PREFIX_LENGTH; + + sys_memmove(path_start, &path_start[LP_PREFIX_LENGTH], + length * sizeof(WCHAR)); + } + } + + path_end = &path_start[length]; + path_iterator = path_start; + + /* Convert drive letters to lowercase, if present. */ + if(length >= 2 && path_start[1] == L':') { + WCHAR drive_letter = path_start[0]; + + if(drive_letter >= L'A' && drive_letter <= L'Z') { + path_start[0] = drive_letter - L'A' + L'a'; + } + } + + while(path_iterator < path_end) { + if(*path_iterator == L'\\') { + *path_iterator = L'/'; + } + + path_iterator++; + } + + /* Truncate the result to its actual length; we don't want to include the + * NUL terminator. */ + return enif_realloc_binary(path, length * sizeof(WCHAR)); +} + +/* @brief Checks whether all the given attributes are set on the object at the + * given path. Note that it assumes false on errors. */ +static int has_file_attributes(const efile_path_t *path, DWORD mask) { + DWORD attributes = GetFileAttributesW((WCHAR*)path->data); + + if(attributes == INVALID_FILE_ATTRIBUTES) { + return 0; + } + + return !!((attributes & mask) == mask); +} + +static int is_ignored_name(int name_length, const WCHAR *name) { + if(name_length == 1 && name[0] == L'.') { + return 1; + } else if(name_length == 2 && !sys_memcmp(name, L"..", 2 * sizeof(WCHAR))) { + return 1; + } + + return 0; +} + +static int get_drive_number(const efile_path_t *path) { + const WCHAR *path_start; + int length; + + ASSERT_PATH_FORMAT(path); + + path_start = (WCHAR*)path->data + LP_PREFIX_LENGTH; + length = PATH_LENGTH(path) - LP_PREFIX_LENGTH; + + if(length >= 2 && path_start[1] == L':') { + WCHAR drive_letter = path_start[0]; + + if(drive_letter >= L'A' && drive_letter <= L'Z') { + return drive_letter - L'A' + 1; + } else if(drive_letter >= L'a' && drive_letter <= L'z') { + return drive_letter - L'a' + 1; + } + } + + return -1; +} + +/* @brief Checks whether two *paths* are on the same mount point; they don't + * have to refer to existing or accessible files/directories. */ +static int has_same_mount_point(const efile_path_t *path_a, const efile_path_t *path_b) { + WCHAR *mount_a, *mount_b; + int result = 0; + + mount_a = enif_alloc(path_a->size); + mount_b = enif_alloc(path_b->size); + + if(mount_a != NULL && mount_b != NULL) { + int length_a, length_b; + + length_a = PATH_LENGTH(path_a); + length_b = PATH_LENGTH(path_b); + + if(GetVolumePathNameW((WCHAR*)path_a->data, mount_a, length_a)) { + ASSERT(wcslen(mount_a) <= length_a); + + if(GetVolumePathNameW((WCHAR*)path_b->data, mount_b, length_b)) { + ASSERT(wcslen(mount_b) <= length_b); + + result = !_wcsicmp(mount_a, mount_b); + } + } + } + + if(mount_b != NULL) { + enif_free(mount_b); + } + + if(mount_a != NULL) { + enif_free(mount_a); + } + + return result; +} + +/* Mirrors the PathIsRootW function of the shell API, but doesn't choke on + * paths longer than MAX_PATH. */ +static int is_path_root(const efile_path_t *path) { + const WCHAR *path_start, *path_end; + int length; + + ASSERT_PATH_FORMAT(path); + + path_start = (WCHAR*)path->data + LP_PREFIX_LENGTH; + length = PATH_LENGTH(path) - LP_PREFIX_LENGTH; + + path_end = &path_start[length]; + + if(length == 1) { + /* A single \ refers to the root of the current working directory. */ + return IS_SLASH(path_start[0]); + } else if(length == 3 && iswalpha(path_start[0]) && path_start[1] == L':') { + /* Drive letter. */ + return IS_SLASH(path_start[2]); + } else if(length >= 4) { + /* Check whether we're a UNC root, eg. \\server, \\server\share */ + const WCHAR *path_iterator; + + if(!IS_SLASH(path_start[0]) || !IS_SLASH(path_start[1])) { + return 0; + } + + path_iterator = path_start + 2; + + /* Slide to the slash between the server and share names, if present. */ + while(path_iterator < path_end && !IS_SLASH(*path_iterator)) { + path_iterator++; + } + + /* Slide past the end of the string, stopping at the first slash we + * encounter. */ + do { + path_iterator++; + } while(path_iterator < path_end && !IS_SLASH(*path_iterator)); + + /* If we're past the end of the string and it didnt't end with a slash, + * then we're a root path. */ + return path_iterator >= path_end && !IS_SLASH(path_start[length - 1]); + } + + return 0; +} + +posix_errno_t efile_open(const efile_path_t *path, enum efile_modes_t modes, + ErlNifResourceType *nif_type, efile_data_t **d) { + + DWORD attributes, access_flags, open_mode; + HANDLE handle; + + ASSERT_PATH_FORMAT(path); + + access_flags = 0; + open_mode = 0; + + if(modes & EFILE_MODE_READ && !(modes & EFILE_MODE_WRITE)) { + access_flags = GENERIC_READ; + open_mode = OPEN_EXISTING; + } else if(modes & EFILE_MODE_WRITE && !(modes & EFILE_MODE_READ)) { + access_flags = GENERIC_WRITE; + open_mode = CREATE_ALWAYS; + } else if(modes & EFILE_MODE_READ_WRITE) { + access_flags = GENERIC_READ | GENERIC_WRITE; + open_mode = OPEN_ALWAYS; + } else { + return EINVAL; + } + + if(modes & EFILE_MODE_APPEND) { + access_flags |= FILE_APPEND_DATA; + open_mode = OPEN_ALWAYS; + } + + if(modes & EFILE_MODE_EXCLUSIVE) { + open_mode = CREATE_NEW; + } + + if(modes & EFILE_MODE_SYNC) { + attributes = FILE_FLAG_WRITE_THROUGH; + } else { + attributes = FILE_ATTRIBUTE_NORMAL; + } + + handle = CreateFileW((WCHAR*)path->data, access_flags, + FILE_SHARE_FLAGS, NULL, open_mode, attributes, NULL); + + if(handle != INVALID_HANDLE_VALUE) { + efile_win_t *w; + + w = (efile_win_t*)enif_alloc_resource(nif_type, sizeof(efile_win_t)); + w->handle = handle; + + EFILE_INIT_RESOURCE(&w->common, modes); + (*d) = &w->common; + + return 0; + } else { + DWORD last_error = GetLastError(); + + /* Rewrite all failures on directories to EISDIR to match the old + * driver. */ + if(has_file_attributes(path, FILE_ATTRIBUTE_DIRECTORY)) { + return EISDIR; + } + + return windows_to_posix_errno(last_error); + } +} + +int efile_close(efile_data_t *d) { + efile_win_t *w = (efile_win_t*)d; + HANDLE handle; + + ASSERT(erts_atomic32_read_nob(&d->state) == EFILE_STATE_CLOSED); + ASSERT(w->handle != INVALID_HANDLE_VALUE); + + handle = w->handle; + w->handle = INVALID_HANDLE_VALUE; + + if(!CloseHandle(handle)) { + w->common.posix_errno = windows_to_posix_errno(GetLastError()); + return 0; + } + + return 1; +} + +static void shift_overlapped(OVERLAPPED *overlapped, DWORD shift) { + LARGE_INTEGER offset; + + ASSERT(shift >= 0); + + offset.HighPart = overlapped->OffsetHigh; + offset.LowPart = overlapped->Offset; + + /* ~(Uint64)0 is a magic value ("append to end of file") which needs to be + * preserved. Other positions resulting in overflow would have errored out + * just prior to this point. */ + if(offset.QuadPart != ERTS_UINT64_MAX) { + offset.QuadPart += shift; + } + + /* All unused fields must be zeroed for the next call. */ + sys_memset(overlapped, 0, sizeof(*overlapped)); + overlapped->OffsetHigh = offset.HighPart; + overlapped->Offset = offset.LowPart; +} + +static void shift_iov(SysIOVec **iov, int *iovlen, DWORD shift) { + SysIOVec *head_vec = (*iov); + + ASSERT(shift >= 0); + + while(shift > 0) { + ASSERT(head_vec < &(*iov)[*iovlen]); + + if(shift < head_vec->iov_len) { + head_vec->iov_base = (char*)head_vec->iov_base + shift; + head_vec->iov_len -= shift; + break; + } else { + shift -= head_vec->iov_len; + head_vec++; + } + } + + (*iovlen) -= head_vec - (*iov); + (*iov) = head_vec; +} + +typedef BOOL (WINAPI *io_op_t)(HANDLE, LPVOID, DWORD, LPDWORD, LPOVERLAPPED); + +static Sint64 internal_sync_io(efile_win_t *w, io_op_t operation, + SysIOVec *iov, int iovlen, OVERLAPPED *overlapped) { + + Sint64 bytes_processed = 0; + + for(;;) { + DWORD block_bytes_processed, last_error; + BOOL succeeded; + + if(iovlen < 1) { + return bytes_processed; + } + + succeeded = operation(w->handle, iov->iov_base, iov->iov_len, + &block_bytes_processed, overlapped); + last_error = GetLastError(); + + if(!succeeded && (last_error != ERROR_HANDLE_EOF)) { + w->common.posix_errno = windows_to_posix_errno(last_error); + return -1; + } else if(block_bytes_processed == 0) { + /* EOF */ + return bytes_processed; + } + + if(overlapped != NULL) { + shift_overlapped(overlapped, block_bytes_processed); + } + + shift_iov(&iov, &iovlen, block_bytes_processed); + + bytes_processed += block_bytes_processed; + } +} + +Sint64 efile_readv(efile_data_t *d, SysIOVec *iov, int iovlen) { + efile_win_t *w = (efile_win_t*)d; + + return internal_sync_io(w, ReadFile, iov, iovlen, NULL); +} + +Sint64 efile_writev(efile_data_t *d, SysIOVec *iov, int iovlen) { + efile_win_t *w = (efile_win_t*)d; + + OVERLAPPED __overlapped, *overlapped; + Uint64 bytes_written; + + if(w->common.modes & EFILE_MODE_APPEND) { + overlapped = &__overlapped; + + sys_memset(overlapped, 0, sizeof(*overlapped)); + overlapped->OffsetHigh = 0xFFFFFFFF; + overlapped->Offset = 0xFFFFFFFF; + } else { + overlapped = NULL; + } + + return internal_sync_io(w, WriteFile, iov, iovlen, overlapped); +} + +Sint64 efile_preadv(efile_data_t *d, Sint64 offset, SysIOVec *iov, int iovlen) { + efile_win_t *w = (efile_win_t*)d; + + OVERLAPPED overlapped; + + sys_memset(&overlapped, 0, sizeof(overlapped)); + overlapped.OffsetHigh = (offset >> 32) & 0xFFFFFFFF; + overlapped.Offset = offset & 0xFFFFFFFF; + + return internal_sync_io(w, ReadFile, iov, iovlen, &overlapped); +} + +Sint64 efile_pwritev(efile_data_t *d, Sint64 offset, SysIOVec *iov, int iovlen) { + efile_win_t *w = (efile_win_t*)d; + + OVERLAPPED overlapped; + + sys_memset(&overlapped, 0, sizeof(overlapped)); + overlapped.OffsetHigh = (offset >> 32) & 0xFFFFFFFF; + overlapped.Offset = offset & 0xFFFFFFFF; + + return internal_sync_io(w, WriteFile, iov, iovlen, &overlapped); +} + +int efile_seek(efile_data_t *d, enum efile_seek_t seek, Sint64 offset, Sint64 *new_position) { + efile_win_t *w = (efile_win_t*)d; + + LARGE_INTEGER large_offset, large_new_position; + DWORD whence; + + switch(seek) { + case EFILE_SEEK_BOF: whence = FILE_BEGIN; break; + case EFILE_SEEK_CUR: whence = FILE_CURRENT; break; + case EFILE_SEEK_EOF: whence = FILE_END; break; + default: ERTS_INTERNAL_ERROR("Invalid seek parameter"); + } + + large_offset.QuadPart = offset; + + if(!SetFilePointerEx(w->handle, large_offset, &large_new_position, whence)) { + w->common.posix_errno = windows_to_posix_errno(GetLastError()); + return 0; + } + + (*new_position) = large_new_position.QuadPart; + + return 1; +} + +int efile_sync(efile_data_t *d, int data_only) { + efile_win_t *w = (efile_win_t*)d; + + /* Windows doesn't support data-only syncing. */ + (void)data_only; + + if(!FlushFileBuffers(w->handle)) { + w->common.posix_errno = windows_to_posix_errno(GetLastError()); + return 0; + } + + return 1; +} + +int efile_advise(efile_data_t *d, Sint64 offset, Sint64 length, enum efile_advise_t advise) { + /* Windows doesn't support this, but we'll pretend it does since the call + * is only a recommendation even on systems that do support it. */ + + (void)d; + (void)offset; + (void)length; + (void)advise; + + return 1; +} + +int efile_allocate(efile_data_t *d, Sint64 offset, Sint64 length) { + efile_win_t *w = (efile_win_t*)d; + + (void)d; + (void)offset; + (void)length; + + w->common.posix_errno = ENOTSUP; + + return 0; +} + +int efile_truncate(efile_data_t *d) { + efile_win_t *w = (efile_win_t*)d; + + if(!SetEndOfFile(w->handle)) { + w->common.posix_errno = windows_to_posix_errno(GetLastError()); + return 0; + } + + return 1; +} + +static int is_executable_file(const efile_path_t *path) { + /* We're using the file extension in order to be quirks-compliant with the + * old driver, which never bothered to check the actual permissions. We + * could easily do so now (cf. GetNamedSecurityInfo) but the execute + * permission is only relevant for files that are started with the default + * loader, and batch files run just fine with read permission alone. */ + + int length = PATH_LENGTH(path); + + if(length >= 4) { + const WCHAR *last_four = &((WCHAR*)path->data)[length - 4]; + + if (!_wcsicmp(last_four, L".exe") || + !_wcsicmp(last_four, L".cmd") || + !_wcsicmp(last_four, L".bat") || + !_wcsicmp(last_four, L".com")) { + return 1; + } + } + + return 0; +} + +posix_errno_t efile_read_info(const efile_path_t *path, int follow_links, efile_fileinfo_t *result) { + BY_HANDLE_FILE_INFORMATION native_file_info; + DWORD attributes; + + sys_memset(&native_file_info, 0, sizeof(native_file_info)); + + attributes = GetFileAttributesW((WCHAR*)path->data); + + if(attributes == INVALID_FILE_ATTRIBUTES) { + DWORD last_error = GetLastError(); + + /* Querying a network share root fails with ERROR_BAD_NETPATH, so we'll + * fake it as a directory just like local roots. */ + if(!is_path_root(path) || last_error != ERROR_BAD_NETPATH) { + return windows_to_posix_errno(last_error); + } + + attributes = FILE_ATTRIBUTE_DIRECTORY; + } else if(is_path_root(path)) { + /* Local (or mounted) roots can be queried with GetFileAttributesW but + * lack support for GetFileInformationByHandle, so we'll skip that + * part. */ + } else { + HANDLE handle; + + if(follow_links && (attributes & FILE_ATTRIBUTE_REPARSE_POINT)) { + posix_errno_t posix_errno; + efile_path_t resolved_path; + + posix_errno = internal_read_link(path, &resolved_path); + + if(posix_errno != 0) { + return posix_errno; + } + + return efile_read_info(&resolved_path, 0, result); + } + + handle = CreateFileW((const WCHAR*)path->data, GENERIC_READ, + FILE_SHARE_FLAGS, NULL, OPEN_EXISTING, FILE_FLAG_BACKUP_SEMANTICS, + NULL); + + /* The old driver never cared whether this succeeded. */ + if(handle != INVALID_HANDLE_VALUE) { + GetFileInformationByHandle(handle, &native_file_info); + CloseHandle(handle); + } + + FILETIME_TO_EPOCH(result->m_time, native_file_info.ftLastWriteTime); + FILETIME_TO_EPOCH(result->a_time, native_file_info.ftLastAccessTime); + FILETIME_TO_EPOCH(result->c_time, native_file_info.ftCreationTime); + + if(result->m_time == -EPOCH_DIFFERENCE) { + /* Default to 1970 just like the old driver. */ + result->m_time = 0; + } + + if(result->a_time == -EPOCH_DIFFERENCE) { + result->a_time = result->m_time; + } + + if(result->c_time == -EPOCH_DIFFERENCE) { + result->c_time = result->m_time; + } + } + + if(attributes & FILE_ATTRIBUTE_REPARSE_POINT) { + result->type = EFILE_FILETYPE_SYMLINK; + /* This should be _S_IFLNK, but the old driver always set + * non-directories to _S_IFREG. */ + result->mode |= _S_IFREG; + } else if(attributes & FILE_ATTRIBUTE_DIRECTORY) { + result->type = EFILE_FILETYPE_DIRECTORY; + result->mode |= _S_IFDIR | _S_IEXEC; + } else { + if(is_executable_file(path)) { + result->mode |= _S_IEXEC; + } + + result->type = EFILE_FILETYPE_REGULAR; + result->mode |= _S_IFREG; + } + + if(!(attributes & FILE_ATTRIBUTE_READONLY)) { + result->access = EFILE_ACCESS_READ | EFILE_ACCESS_WRITE; + result->mode |= _S_IREAD | _S_IWRITE; + } else { + result->access = EFILE_ACCESS_READ; + result->mode |= _S_IREAD; + } + + /* Propagate user mode-bits to group/other fields */ + result->mode |= (result->mode & 0700) >> 3; + result->mode |= (result->mode & 0700) >> 6; + + result->size = + ((Uint64)native_file_info.nFileSizeHigh << 32ull) | + (Uint64)native_file_info.nFileSizeLow; + + result->links = MAX(1, native_file_info.nNumberOfLinks); + + result->major_device = get_drive_number(path); + result->minor_device = 0; + result->inode = 0; + result->uid = 0; + result->gid = 0; + + return 0; +} + +posix_errno_t efile_set_permissions(const efile_path_t *path, Uint32 permissions) { + DWORD attributes = GetFileAttributesW((WCHAR*)path->data); + + if(attributes == INVALID_FILE_ATTRIBUTES) { + return windows_to_posix_errno(GetLastError()); + } + + if(permissions & _S_IWRITE) { + attributes &= ~FILE_ATTRIBUTE_READONLY; + } else { + attributes |= FILE_ATTRIBUTE_READONLY; + } + + if(SetFileAttributesW((WCHAR*)path->data, attributes)) { + return 0; + } + + return windows_to_posix_errno(GetLastError()); +} + +posix_errno_t efile_set_owner(const efile_path_t *path, Uint32 owner, Uint32 group) { + (void)path; + (void)owner; + (void)group; + + return 0; +} + +posix_errno_t efile_set_time(const efile_path_t *path, Sint64 a_time, Sint64 m_time, Sint64 c_time) { + FILETIME accessed, modified, created; + DWORD last_error, attributes; + HANDLE handle; + + attributes = GetFileAttributesW((WCHAR*)path->data); + + if(attributes == INVALID_FILE_ATTRIBUTES) { + return windows_to_posix_errno(GetLastError()); + } + + /* If the file is read-only, we have to make it temporarily writable while + * setting new metadata. */ + if(attributes & FILE_ATTRIBUTE_READONLY) { + DWORD without_readonly = attributes & ~FILE_ATTRIBUTE_READONLY; + + if(!SetFileAttributesW((WCHAR*)path->data, without_readonly)) { + return windows_to_posix_errno(GetLastError()); + } + } + + EPOCH_TO_FILETIME(modified, m_time); + EPOCH_TO_FILETIME(accessed, a_time); + EPOCH_TO_FILETIME(created, c_time); + + handle = CreateFileW((WCHAR*)path->data, GENERIC_READ | GENERIC_WRITE, + FILE_SHARE_FLAGS, NULL, OPEN_EXISTING, FILE_FLAG_BACKUP_SEMANTICS, NULL); + last_error = GetLastError(); + + if(handle != INVALID_HANDLE_VALUE) { + if(SetFileTime(handle, &created, &accessed, &modified)) { + last_error = ERROR_SUCCESS; + } else { + last_error = GetLastError(); + } + + CloseHandle(handle); + } + + if(attributes & FILE_ATTRIBUTE_READONLY) { + SetFileAttributesW((WCHAR*)path->data, attributes); + } + + return windows_to_posix_errno(last_error); +} + +static posix_errno_t internal_read_link(const efile_path_t *path, efile_path_t *result) { + DWORD required_length, actual_length; + HANDLE link_handle; + DWORD last_error; + + link_handle = CreateFileW((WCHAR*)path->data, GENERIC_READ, + FILE_SHARE_FLAGS, NULL, OPEN_EXISTING, FILE_FLAG_BACKUP_SEMANTICS, NULL); + last_error = GetLastError(); + + if(link_handle == INVALID_HANDLE_VALUE) { + return windows_to_posix_errno(last_error); + } + + required_length = GetFinalPathNameByHandleW(link_handle, NULL, 0, 0); + last_error = GetLastError(); + + if(required_length <= 0) { + CloseHandle(link_handle); + return windows_to_posix_errno(last_error); + } + + /* Unlike many other path functions (eg. GetFullPathNameW), this one + * includes the NUL terminator in its required length. */ + if(!enif_alloc_binary(required_length * sizeof(WCHAR), result)) { + CloseHandle(link_handle); + return ENOMEM; + } + + actual_length = GetFinalPathNameByHandleW(link_handle, + (WCHAR*)result->data, required_length, 0); + last_error = GetLastError(); + + CloseHandle(link_handle); + + if(actual_length == 0 || actual_length >= required_length) { + enif_release_binary(result); + return windows_to_posix_errno(last_error); + } + + /* GetFinalPathNameByHandle always prepends with "\\?\" and NUL-terminates, + * so we never have to touch-up the resulting path. */ + + ASSERT_PATH_FORMAT(result); + + return 0; +} + +posix_errno_t efile_read_link(ErlNifEnv *env, const efile_path_t *path, ERL_NIF_TERM *result) { + posix_errno_t posix_errno; + ErlNifBinary result_bin; + DWORD attributes; + + ASSERT_PATH_FORMAT(path); + + attributes = GetFileAttributesW((WCHAR*)path->data); + + if(attributes == INVALID_FILE_ATTRIBUTES) { + return windows_to_posix_errno(GetLastError()); + } else if(!(attributes & FILE_ATTRIBUTE_REPARSE_POINT)) { + return EINVAL; + } + + posix_errno = internal_read_link(path, &result_bin); + + if(posix_errno == 0) { + if(!normalize_path_result(&result_bin)) { + enif_release_binary(&result_bin); + return ENOMEM; + } + + (*result) = enif_make_binary(env, &result_bin); + } + + return posix_errno; +} + +posix_errno_t efile_list_dir(ErlNifEnv *env, const efile_path_t *path, ERL_NIF_TERM *result) { + ERL_NIF_TERM list_head; + WIN32_FIND_DATAW data; + HANDLE search_handle; + WCHAR *search_path; + DWORD last_error; + + ASSERT_PATH_FORMAT(path); + + search_path = enif_alloc(path->size + 2 * sizeof(WCHAR)); + + if(search_path == NULL) { + return ENOMEM; + } + + sys_memcpy(search_path, path->data, path->size); + search_path[PATH_LENGTH(path) + 0] = L'\\'; + search_path[PATH_LENGTH(path) + 1] = L'*'; + search_path[PATH_LENGTH(path) + 2] = L'\0'; + + search_handle = FindFirstFileW(search_path, &data); + last_error = GetLastError(); + + enif_free(search_path); + + if(search_handle == INVALID_HANDLE_VALUE) { + return windows_to_posix_errno(last_error); + } + + list_head = enif_make_list(env, 0); + + do { + int name_length = wcslen(data.cFileName); + + if(!is_ignored_name(name_length, data.cFileName)) { + unsigned char *name_bytes; + ERL_NIF_TERM name_term; + size_t name_size; + + name_size = name_length * sizeof(WCHAR); + + name_bytes = enif_make_new_binary(env, name_size, &name_term); + sys_memcpy(name_bytes, data.cFileName, name_size); + + list_head = enif_make_list_cell(env, name_term, list_head); + } + } while(FindNextFileW(search_handle, &data)); + + FindClose(search_handle); + (*result) = list_head; + + return 0; +} + +posix_errno_t efile_rename(const efile_path_t *old_path, const efile_path_t *new_path) { + BOOL old_is_directory, new_is_directory; + DWORD move_flags, last_error; + + ASSERT_PATH_FORMAT(old_path); + ASSERT_PATH_FORMAT(new_path); + + move_flags = MOVEFILE_COPY_ALLOWED | MOVEFILE_WRITE_THROUGH; + + if(MoveFileExW((WCHAR*)old_path->data, (WCHAR*)new_path->data, move_flags)) { + return 0; + } + + last_error = GetLastError(); + + old_is_directory = has_file_attributes(old_path, FILE_ATTRIBUTE_DIRECTORY); + new_is_directory = has_file_attributes(new_path, FILE_ATTRIBUTE_DIRECTORY); + + switch(last_error) { + case ERROR_SHARING_VIOLATION: + case ERROR_ACCESS_DENIED: + if(old_is_directory) { + BOOL moved_into_itself; + + moved_into_itself = (old_path->size <= new_path->size) && + !_wcsnicmp((WCHAR*)old_path->data, (WCHAR*)new_path->data, + PATH_LENGTH(old_path)); + + if(moved_into_itself) { + return EINVAL; + } else if(is_path_root(old_path)) { + return EINVAL; + } + + /* Renaming a directory across volumes needs to be rewritten as + * EXDEV so that the caller can respond by simulating it with + * copy/delete operations. + * + * Files are handled through MOVEFILE_COPY_ALLOWED. */ + if(!has_same_mount_point(old_path, new_path)) { + return EXDEV; + } + } + break; + case ERROR_PATH_NOT_FOUND: + case ERROR_FILE_NOT_FOUND: + return ENOENT; + case ERROR_ALREADY_EXISTS: + case ERROR_FILE_EXISTS: + if(old_is_directory && !new_is_directory) { + return ENOTDIR; + } else if(!old_is_directory && new_is_directory) { + return EISDIR; + } else if(old_is_directory && new_is_directory) { + /* This will fail if the destination isn't empty. */ + if(RemoveDirectoryW((WCHAR*)new_path->data)) { + return efile_rename(old_path, new_path); + } + + return EEXIST; + } else if(!old_is_directory && !new_is_directory) { + /* This is pretty iffy; the public documentation says that the + * operation may EACCES on some systems when either file is open, + * which gives us room to use MOVEFILE_REPLACE_EXISTING and be done + * with it, but the old implementation simulated Unix semantics and + * there's a lot of code that relies on that. + * + * The simulation renames the destination to a scratch name to get + * around the fact that it's impossible to open (and by extension + * rename) a file that's been deleted while open. It has a few + * drawbacks though; + * + * 1) It's not atomic as there's a small window where there's no + * file at all on the destination path. + * 2) It will confuse applications that subscribe to folder + * changes. + * 3) It will fail if we lack general permission to write in the + * same folder. */ + + WCHAR *swap_path = enif_alloc(new_path->size + sizeof(WCHAR) * 64); + + if(swap_path == NULL) { + return ENOMEM; + } else { + static LONGLONG unique_counter = 0; + WCHAR *swap_path_end; + + /* We swap in the same folder as the destination to be + * reasonably sure that it's on the same volume. Note that + * we're avoiding GetTempFileNameW as it will fail on long + * paths. */ + + sys_memcpy(swap_path, (WCHAR*)new_path->data, new_path->size); + swap_path_end = swap_path + PATH_LENGTH(new_path); + + while(!IS_SLASH(*swap_path_end)) { + ASSERT(swap_path_end > swap_path); + swap_path_end--; + } + + StringCchPrintfW(&swap_path_end[1], 64, L"erl-%lx-%llx.tmp", + GetCurrentProcessId(), unique_counter); + InterlockedIncrement64(&unique_counter); + } + + if(MoveFileExW((WCHAR*)new_path->data, swap_path, MOVEFILE_REPLACE_EXISTING)) { + if(MoveFileExW((WCHAR*)old_path->data, (WCHAR*)new_path->data, move_flags)) { + last_error = ERROR_SUCCESS; + DeleteFileW(swap_path); + } else { + last_error = GetLastError(); + MoveFileW(swap_path, (WCHAR*)new_path->data); + } + } else { + last_error = GetLastError(); + DeleteFileW(swap_path); + } + + enif_free(swap_path); + + return windows_to_posix_errno(last_error); + } + + return EEXIST; + } + + return windows_to_posix_errno(last_error); +} + +posix_errno_t efile_make_hard_link(const efile_path_t *existing_path, const efile_path_t *new_path) { + ASSERT_PATH_FORMAT(existing_path); + ASSERT_PATH_FORMAT(new_path); + + if(!CreateHardLinkW((WCHAR*)new_path->data, (WCHAR*)existing_path->data, NULL)) { + return windows_to_posix_errno(GetLastError()); + } + + return 0; +} + +posix_errno_t efile_make_soft_link(const efile_path_t *existing_path, const efile_path_t *new_path) { + DWORD link_flags; + + ASSERT_PATH_FORMAT(existing_path); + ASSERT_PATH_FORMAT(new_path); + + if(has_file_attributes(existing_path, FILE_ATTRIBUTE_DIRECTORY)) { + link_flags = SYMBOLIC_LINK_FLAG_DIRECTORY; + } else { + link_flags = 0; + } + + if(!CreateSymbolicLinkW((WCHAR*)new_path->data, (WCHAR*)existing_path->data, link_flags)) { + return windows_to_posix_errno(GetLastError()); + } + + return 0; +} + +posix_errno_t efile_make_dir(const efile_path_t *path) { + ASSERT_PATH_FORMAT(path); + + if(!CreateDirectoryW((WCHAR*)path->data, NULL)) { + return windows_to_posix_errno(GetLastError()); + } + + return 0; +} + +posix_errno_t efile_del_file(const efile_path_t *path) { + ASSERT_PATH_FORMAT(path); + + if(!DeleteFileW((WCHAR*)path->data)) { + DWORD last_error = GetLastError(); + + switch(last_error) { + case ERROR_INVALID_NAME: + /* Attempted to delete a device or similar. */ + return EACCES; + case ERROR_ACCESS_DENIED: + /* Windows NT reports removing a directory as EACCES instead of + * EPERM. */ + if(has_file_attributes(path, FILE_ATTRIBUTE_DIRECTORY)) { + return EPERM; + } + break; + } + + return windows_to_posix_errno(last_error); + } + + return 0; +} + +posix_errno_t efile_del_dir(const efile_path_t *path) { + ASSERT_PATH_FORMAT(path); + + if(!RemoveDirectoryW((WCHAR*)path->data)) { + DWORD last_error = GetLastError(); + + if(last_error == ERROR_DIRECTORY) { + return ENOTDIR; + } + + return windows_to_posix_errno(last_error); + } + + return 0; +} + +posix_errno_t efile_set_cwd(const efile_path_t *path) { + const WCHAR *path_start; + + ASSERT_PATH_FORMAT(path); + + /* We have to use _wchdir since that's the only function that updates the + * per-drive working directory, but it naively assumes that all paths + * starting with \\ are UNC paths, so we have to skip the \\?\-prefix. */ + path_start = (WCHAR*)path->data + LP_PREFIX_LENGTH; + + if(_wchdir(path_start)) { + return windows_to_posix_errno(GetLastError()); + } + + return 0; +} + +static int is_valid_drive(int device_index) { + WCHAR drive_path[4] = {L'?', L':', L'\\', L'\0'}; + + if(device_index == 0) { + /* Default drive; always valid. */ + return 1; + } else if(device_index > (L'Z' - L'A' + 1)) { + return 0; + } + + drive_path[0] = device_index + L'A' - 1; + + switch(GetDriveTypeW(drive_path)) { + case DRIVE_NO_ROOT_DIR: + case DRIVE_UNKNOWN: + return 0; + } + + return 1; +} + +posix_errno_t efile_get_device_cwd(ErlNifEnv *env, int device_index, ERL_NIF_TERM *result) { + ErlNifBinary result_bin; + + /* _wgetdcwd might crash the entire emulator on debug builds since the CRT + * invalid parameter handler asserts if passed a non-existent drive (Or + * simply one that has been unmounted), so we check it ourselves to avoid + * that. */ + if(!is_valid_drive(device_index)) { + return EACCES; + } + + if(!enif_alloc_binary(MAX_PATH * sizeof(WCHAR), &result_bin)) { + return ENOMEM; + } + + if(_wgetdcwd(device_index, (WCHAR*)result_bin.data, MAX_PATH) == NULL) { + enif_release_binary(&result_bin); + return EACCES; + } + + if(!normalize_path_result(&result_bin)) { + enif_release_binary(&result_bin); + return ENOMEM; + } + + (*result) = enif_make_binary(env, &result_bin); + + return 0; +} + +posix_errno_t efile_get_cwd(ErlNifEnv *env, ERL_NIF_TERM *result) { + return efile_get_device_cwd(env, 0, result); +} + +posix_errno_t efile_altname(ErlNifEnv *env, const efile_path_t *path, ERL_NIF_TERM *result) { + ErlNifBinary result_bin; + + ASSERT_PATH_FORMAT(path); + + if(is_path_root(path)) { + /* Root paths can't be queried so we'll just return them as they are. */ + if(!enif_alloc_binary(path->size, &result_bin)) { + return ENOMEM; + } + + sys_memcpy(result_bin.data, path->data, path->size); + } else { + WIN32_FIND_DATAW data; + HANDLE handle; + + WCHAR *name_buffer; + int name_length; + + /* Reject path wildcards. */ + if(wcspbrk(&((const WCHAR*)path->data)[4], L"?*")) { + return ENOENT; + } + + handle = FindFirstFileW((const WCHAR*)path->data, &data); + + if(handle == INVALID_HANDLE_VALUE) { + return windows_to_posix_errno(GetLastError()); + } + + FindClose(handle); + + name_length = wcslen(data.cAlternateFileName); + + if(name_length > 0) { + name_buffer = data.cAlternateFileName; + } else { + name_length = wcslen(data.cFileName); + name_buffer = data.cFileName; + } + + /* Include NUL-terminator; it will be removed after normalization. */ + name_length += 1; + + if(!enif_alloc_binary(name_length * sizeof(WCHAR), &result_bin)) { + return ENOMEM; + } + + sys_memcpy(result_bin.data, name_buffer, name_length * sizeof(WCHAR)); + } + + if(!normalize_path_result(&result_bin)) { + enif_release_binary(&result_bin); + return ENOMEM; + } + + (*result) = enif_make_binary(env, &result_bin); + + return 0; +} + +static int windows_to_posix_errno(DWORD last_error) { + switch(last_error) { + case ERROR_SUCCESS: + return 0; + case ERROR_INVALID_FUNCTION: + case ERROR_INVALID_DATA: + case ERROR_INVALID_PARAMETER: + case ERROR_INVALID_TARGET_HANDLE: + case ERROR_INVALID_CATEGORY: + case ERROR_NEGATIVE_SEEK: + return EINVAL; + case ERROR_DIR_NOT_EMPTY: + return EEXIST; + case ERROR_BAD_FORMAT: + return ENOEXEC; + case ERROR_PATH_NOT_FOUND: + case ERROR_FILE_NOT_FOUND: + case ERROR_NO_MORE_FILES: + case ERROR_INVALID_NAME: + return ENOENT; + case ERROR_TOO_MANY_OPEN_FILES: + return EMFILE; + case ERROR_ACCESS_DENIED: + case ERROR_INVALID_ACCESS: + case ERROR_CURRENT_DIRECTORY: + case ERROR_SHARING_VIOLATION: + case ERROR_LOCK_VIOLATION: + case ERROR_INVALID_PASSWORD: + case ERROR_DRIVE_LOCKED: + return EACCES; + case ERROR_INVALID_HANDLE: + return EBADF; + case ERROR_NOT_ENOUGH_MEMORY: + case ERROR_OUTOFMEMORY: + case ERROR_OUT_OF_STRUCTURES: + return ENOMEM; + case ERROR_INVALID_DRIVE: + case ERROR_BAD_UNIT: + case ERROR_NOT_READY: + case ERROR_REM_NOT_LIST: + case ERROR_DUP_NAME: + case ERROR_BAD_NETPATH: + case ERROR_NETWORK_BUSY: + case ERROR_DEV_NOT_EXIST: + case ERROR_BAD_NET_NAME: + return ENXIO; + case ERROR_NOT_SAME_DEVICE: + return EXDEV; + case ERROR_WRITE_PROTECT: + return EROFS; + case ERROR_BAD_LENGTH: + case ERROR_BUFFER_OVERFLOW: + return E2BIG; + case ERROR_SEEK: + case ERROR_SECTOR_NOT_FOUND: + return ESPIPE; + case ERROR_NOT_DOS_DISK: + return ENODEV; + case ERROR_GEN_FAILURE: + return ENODEV; + case ERROR_SHARING_BUFFER_EXCEEDED: + case ERROR_NO_MORE_SEARCH_HANDLES: + return EMFILE; + case ERROR_HANDLE_EOF: + case ERROR_BROKEN_PIPE: + return EPIPE; + case ERROR_HANDLE_DISK_FULL: + case ERROR_DISK_FULL: + return ENOSPC; + case ERROR_NOT_SUPPORTED: + return ENOTSUP; + case ERROR_FILE_EXISTS: + case ERROR_ALREADY_EXISTS: + case ERROR_CANNOT_MAKE: + return EEXIST; + case ERROR_ALREADY_ASSIGNED: + return EBUSY; + case ERROR_NO_PROC_SLOTS: + return EAGAIN; + case ERROR_CANT_RESOLVE_FILENAME: + return EMLINK; + case ERROR_PRIVILEGE_NOT_HELD: + return EPERM; + case ERROR_ARENA_TRASHED: + case ERROR_INVALID_BLOCK: + case ERROR_BAD_ENVIRONMENT: + case ERROR_BAD_COMMAND: + case ERROR_CRC: + case ERROR_OUT_OF_PAPER: + case ERROR_READ_FAULT: + case ERROR_WRITE_FAULT: + case ERROR_WRONG_DISK: + case ERROR_NET_WRITE_FAULT: + return EIO; + default: /* not to do with files I expect. */ + return EIO; + } +} diff --git a/erts/emulator/sys/common/erl_osenv.c b/erts/emulator/sys/common/erl_osenv.c new file mode 100644 index 0000000000..9f54d1dff0 --- /dev/null +++ b/erts/emulator/sys/common/erl_osenv.c @@ -0,0 +1,396 @@ +/* + * %CopyrightBegin% + * + * Copyright Ericsson AB 2017. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * %CopyrightEnd% + */ + +#include "erl_osenv.h" + +#include "global.h" +#include "erl_alloc.h" +#include "erl_process.h" + +#define STACKBUF_SIZE (512) + +typedef struct __env_rbtnode_t { + struct __env_rbtnode_t *parent; + struct __env_rbtnode_t *left; + struct __env_rbtnode_t *right; + + int is_red; + + erts_osenv_data_t key; + erts_osenv_data_t value; +} env_rbtnode_t; + +#define ERTS_RBT_PREFIX env +#define ERTS_RBT_T env_rbtnode_t +#define ERTS_RBT_KEY_T erts_osenv_data_t +#define ERTS_RBT_FLAGS_T int +#define ERTS_RBT_INIT_EMPTY_TNODE(T) \ + do { \ + (T)->parent = NULL; \ + (T)->left = NULL; \ + (T)->right = NULL; \ + (T)->is_red = 0; \ + } while(0) +#define ERTS_RBT_IS_RED(T) ((T)->is_red) +#define ERTS_RBT_SET_RED(T) ((T)->is_red = 1) +#define ERTS_RBT_IS_BLACK(T) (!ERTS_RBT_IS_RED(T)) +#define ERTS_RBT_SET_BLACK(T) ((T)->is_red = 0) +#define ERTS_RBT_GET_FLAGS(T) ((T)->is_red) +#define ERTS_RBT_SET_FLAGS(T, F) ((T)->is_red = F) +#define ERTS_RBT_GET_PARENT(T) ((T)->parent) +#define ERTS_RBT_SET_PARENT(T, P) ((T)->parent = P) +#define ERTS_RBT_GET_RIGHT(T) ((T)->right) +#define ERTS_RBT_SET_RIGHT(T, R) ((T)->right = (R)) +#define ERTS_RBT_GET_LEFT(T) ((T)->left) +#define ERTS_RBT_SET_LEFT(T, L) ((T)->left = (L)) +#define ERTS_RBT_GET_KEY(T) ((T)->key) +#define ERTS_RBT_IS_LT(KX, KY) (compare_env_keys(KX, KY) < 0) +#define ERTS_RBT_IS_EQ(KX, KY) (compare_env_keys(KX, KY) == 0) +#define ERTS_RBT_WANT_FOREACH_DESTROY +#define ERTS_RBT_WANT_FOREACH +#define ERTS_RBT_WANT_REPLACE +#define ERTS_RBT_WANT_DELETE +#define ERTS_RBT_WANT_INSERT +#define ERTS_RBT_WANT_LOOKUP + +static int compare_env_keys(const erts_osenv_data_t a, const erts_osenv_data_t b); + +#include "erl_rbtree.h" + +static int compare_env_keys(const erts_osenv_data_t a, const erts_osenv_data_t b) { + int relation = sys_memcmp(a.data, b.data, MIN(a.length, b.length)); + + if(relation != 0) { + return relation; + } + + if(a.length < b.length) { + return -1; + } else if(a.length == b.length) { + return 0; + } else { + return 1; + } +} + +static void *convert_value_to_native(Eterm term, char *stackbuf, + int stackbuf_size, Sint *length) { + int encoding; + void *result; + + if(is_atom(term)) { + return NULL; + } + + encoding = erts_get_native_filename_encoding(); + *length = erts_native_filename_need(term, encoding); + + if(*length < 0) { + return NULL; + } else if(*length >= stackbuf_size) { + result = erts_alloc(ERTS_ALC_T_TMP, *length); + } else { + result = stackbuf; + } + + erts_native_filename_put(term, encoding, (byte*)result); + + return result; +} + +static void *convert_key_to_native(Eterm term, char *stackbuf, + int stackbuf_size, Sint *length) { + byte *name_iterator, *name_end; + void *result; + int encoding; + + result = convert_value_to_native(term, stackbuf, stackbuf_size, length); + + if(result == NULL || length == 0) { + return NULL; + } + + encoding = erts_get_native_filename_encoding(); + + name_iterator = (byte*)result; + name_end = &name_iterator[*length]; + +#ifdef __WIN32__ + /* Windows stores per-drive working directories as variables starting with + * '=', so we skip the first character to tolerate that. */ + name_iterator = erts_raw_env_next_char(name_iterator, encoding); +#endif + + while(name_iterator < name_end) { + if(erts_raw_env_char_is_7bit_ascii_char('=', name_iterator, encoding)) { + if(result != stackbuf) { + erts_free(ERTS_ALC_T_TMP, result); + } + + return NULL; + } + + name_iterator = erts_raw_env_next_char(name_iterator, encoding); + } + + return result; +} + +void erts_osenv_init(erts_osenv_t *env) { + env->variable_count = 0; + env->content_size = 0; + env->tree = NULL; +} + +static void destroy_foreach(env_rbtnode_t *node, void *_state) { + erts_free(ERTS_ALC_T_ENVIRONMENT, node); + (void)_state; +} + +void erts_osenv_clear(erts_osenv_t *env) { + env_rbt_foreach_destroy(&env->tree, &destroy_foreach, NULL); + erts_osenv_init(env); +} + +struct __env_merge { + int overwrite_existing; + erts_osenv_t *env; +}; + +static void merge_foreach(env_rbtnode_t *node, void *_state) { + struct __env_merge *state = (struct __env_merge*)(_state); + env_rbtnode_t *existing_node; + + existing_node = env_rbt_lookup(state->env->tree, node->key); + + if(existing_node == NULL || state->overwrite_existing) { + erts_osenv_put_native(state->env, &node->key, &node->value); + } +} + +void erts_osenv_merge(erts_osenv_t *env, const erts_osenv_t *with, int overwrite) { + struct __env_merge merge_state; + + merge_state.overwrite_existing = overwrite; + merge_state.env = env; + + env_rbt_foreach(with->tree, merge_foreach, &merge_state); +} + +struct __env_foreach_term { + erts_osenv_foreach_term_cb_t user_callback; + struct process *process; + void *user_state; +}; + +static void foreach_term_wrapper(env_rbtnode_t *node, void *_state) { + struct __env_foreach_term *state = (struct __env_foreach_term*)_state; + Eterm key, value; + + key = erts_convert_native_to_filename(state->process, + node->key.length, (byte*)node->key.data); + value = erts_convert_native_to_filename(state->process, + node->value.length, (byte*)node->value.data); + + state->user_callback(state->process, state->user_state, key, value); +} + +void erts_osenv_foreach_term(const erts_osenv_t *env, struct process *process, + void *state, erts_osenv_foreach_term_cb_t callback) { + struct __env_foreach_term wrapper_state; + + wrapper_state.user_callback = callback; + wrapper_state.user_state = state; + wrapper_state.process = process; + + env_rbt_foreach(env->tree, foreach_term_wrapper, &wrapper_state); +} + +int erts_osenv_get_term(const erts_osenv_t *env, Process *process, + Eterm key_term, Eterm *out_term) { + char key_stackbuf[STACKBUF_SIZE]; + erts_osenv_data_t key; + int result; + + key.data = convert_key_to_native(key_term, key_stackbuf, + STACKBUF_SIZE, &key.length); + result = -1; + + if(key.data != NULL) { + env_rbtnode_t *node; + + node = env_rbt_lookup(env->tree, key); + result = 0; + + if(node != NULL) { + (*out_term) = erts_convert_native_to_filename(process, + node->value.length, (byte*)node->value.data); + result = 1; + } + + if(key.data != key_stackbuf) { + erts_free(ERTS_ALC_T_TMP, key.data); + } + } + + return result; +} + +int erts_osenv_put_term(erts_osenv_t *env, Eterm key_term, Eterm value_term) { + char key_stackbuf[STACKBUF_SIZE], value_stackbuf[STACKBUF_SIZE]; + erts_osenv_data_t key, value; + int result; + + key.data = convert_key_to_native(key_term, key_stackbuf, + STACKBUF_SIZE, &key.length); + value.data = convert_value_to_native(value_term, value_stackbuf, + STACKBUF_SIZE, &value.length); + result = -1; + + if(value.data != NULL && key.data != NULL) { + result = erts_osenv_put_native(env, &key, &value); + } + + if(value.data != NULL && value.data != value_stackbuf) { + erts_free(ERTS_ALC_T_TMP, value.data); + } + + if(key.data != NULL && key.data != key_stackbuf) { + erts_free(ERTS_ALC_T_TMP, key.data); + } + + return result; +} + +int erts_osenv_unset_term(erts_osenv_t *env, Eterm key_term) { + char key_stackbuf[STACKBUF_SIZE]; + erts_osenv_data_t key; + int result; + + key.data = convert_key_to_native(key_term, key_stackbuf, + STACKBUF_SIZE, &key.length); + result = -1; + + if(key.data != NULL) { + result = erts_osenv_unset_native(env, &key); + + if(key.data != key_stackbuf) { + erts_free(ERTS_ALC_T_TMP, key.data); + } + } + + return result; +} + +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ + +struct __env_foreach_native { + erts_osenv_foreach_native_cb_t user_callback; + void *user_state; +}; + +static void foreach_native_wrapper(env_rbtnode_t *node, void *_state) { + struct __env_foreach_native *state = (struct __env_foreach_native*)_state; + + state->user_callback(state->user_state, &node->key, &node->value); +} + +void erts_osenv_foreach_native(const erts_osenv_t *env, void *state, + erts_osenv_foreach_native_cb_t callback) { + struct __env_foreach_native wrapper_state; + + wrapper_state.user_callback = callback; + wrapper_state.user_state = state; + + env_rbt_foreach(env->tree, foreach_native_wrapper, &wrapper_state); +} + +int erts_osenv_get_native(const erts_osenv_t *env, + const erts_osenv_data_t *key, + erts_osenv_data_t *value) { + env_rbtnode_t *node = env_rbt_lookup(env->tree, *key); + + if(node != NULL) { + if(value != NULL) { + if(node->value.length > value->length) { + return -1; + } + + sys_memcpy(value->data, node->value.data, node->value.length); + value->length = node->value.length; + } + + return 1; + } + + return 0; +} + +int erts_osenv_put_native(erts_osenv_t *env, const erts_osenv_data_t *key, + const erts_osenv_data_t *value) { + env_rbtnode_t *old_node, *new_node; + + new_node = erts_alloc(ERTS_ALC_T_ENVIRONMENT, sizeof(env_rbtnode_t) + + key->length + value->length); + + new_node->key.data = (char*)(&new_node[1]); + new_node->key.length = key->length; + new_node->value.data = &((char*)new_node->key.data)[key->length]; + new_node->value.length = value->length; + + sys_memcpy(new_node->key.data, key->data, key->length); + sys_memcpy(new_node->value.data, value->data, value->length); + + old_node = env_rbt_lookup(env->tree, *key); + + if(old_node != NULL) { + env->content_size -= old_node->value.length; + env->content_size -= old_node->key.length; + env_rbt_replace(&env->tree, old_node, new_node); + } else { + env_rbt_insert(&env->tree, new_node); + env->variable_count++; + } + + env->content_size += new_node->value.length; + env->content_size += new_node->key.length; + + if(old_node != NULL) { + erts_free(ERTS_ALC_T_ENVIRONMENT, old_node); + } + + return 1; +} + +int erts_osenv_unset_native(erts_osenv_t *env, const erts_osenv_data_t *key) { + env_rbtnode_t *old_node = env_rbt_lookup(env->tree, *key); + + if(old_node != NULL) { + env->content_size -= old_node->value.length; + env->content_size -= old_node->key.length; + env->variable_count -= 1; + + env_rbt_delete(&env->tree, old_node); + erts_free(ERTS_ALC_T_ENVIRONMENT, old_node); + return 1; + } + + return 0; +} diff --git a/erts/emulator/sys/common/erl_osenv.h b/erts/emulator/sys/common/erl_osenv.h new file mode 100644 index 0000000000..4777f2148a --- /dev/null +++ b/erts/emulator/sys/common/erl_osenv.h @@ -0,0 +1,121 @@ +/* + * %CopyrightBegin% + * + * Copyright Ericsson AB 2017. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * %CopyrightEnd% + */ + +/* This is a replacement for getenv(3) and friends, operating on instances so + * we can keep a common implementation for both the global and local (per-port) + * environments. + * + * The instances are not thread-safe on their own but unlike getenv(3) we're + * guaranteed to be the only user, so placing locks around all our accesses + * will suffice. + * + * Use erts_sys_rwlock_global_osenv to access the global environment. */ + +#ifndef __ERL_OSENV_H__ +#define __ERL_OSENV_H__ + +#ifdef HAVE_CONFIG_H +# include "config.h" +#endif + +typedef struct __erts_osenv_data_t erts_osenv_data_t; + +typedef struct __erts_osenv_t { + struct __env_rbtnode_t *tree; + int variable_count; + int content_size; +} erts_osenv_t; + +#include "sys.h" + +struct __erts_osenv_data_t { + Sint length; + void *data; +}; + +void erts_osenv_init(erts_osenv_t *env); +void erts_osenv_clear(erts_osenv_t *env); + +/* @brief Merges \c with into \c env + * + * @param overwrite Whether to overwrite existing entries or keep them as they + * are. */ +void erts_osenv_merge(erts_osenv_t *env, const erts_osenv_t *with, int overwrite); + +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ + +/* @brief Copies env[key] into \c value + * + * @return 1 on success, 0 if the key couldn't be found, and -1 if the input + * was invalid. */ +int erts_osenv_get_term(const erts_osenv_t *env, struct process *process, + Eterm key, Eterm *value); + +/* @brief Copies \c value into \c env[key] + * + * @return 1 on success, -1 if the input was invalid. */ +int erts_osenv_put_term(erts_osenv_t *env, Eterm key, Eterm value); + +/* @brief Removes \c env[key] + * + * @return 1 on success, 0 if the key couldn't be found, and -1 if the input + * was invalid. */ +int erts_osenv_unset_term(erts_osenv_t *env, Eterm key); + +/* @brief Copies env[key] into \c value + * + * @param value [in,out] The buffer to copy the value into, may be NULL if you + * only wish to query presence. + * + * @return 1 on success, 0 if the key couldn't be found, and -1 if if the value + * didn't fit into the buffer. */ +int erts_osenv_get_native(const erts_osenv_t *env, const erts_osenv_data_t *key, + erts_osenv_data_t *value); + +/* @brief Copies \c value into \c env[key] + * + * @return 1 on success, -1 on failure. */ +int erts_osenv_put_native(erts_osenv_t *env, const erts_osenv_data_t *key, + const erts_osenv_data_t *value); + +/* @brief Removes \c key from the env. + * + * @return 1 on success, 0 if the key couldn't be found. */ +int erts_osenv_unset_native(erts_osenv_t *env, const erts_osenv_data_t *key); + +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ + +typedef void (*erts_osenv_foreach_term_cb_t)(struct process *process, + void *state, Eterm key, Eterm value); + +typedef void (*erts_osenv_foreach_native_cb_t)(void *state, + const erts_osenv_data_t *key, + const erts_osenv_data_t *value); + +/* @brief Walks through all environment variables, calling \c callback for each + * one. It's unsafe to modify \c env within the callback. */ +void erts_osenv_foreach_term(const erts_osenv_t *env, struct process *process, + void *state, erts_osenv_foreach_term_cb_t callback); + +/* @copydoc erts_osenv_foreach_term */ +void erts_osenv_foreach_native(const erts_osenv_t *env, void *state, + erts_osenv_foreach_native_cb_t callback); + +#endif diff --git a/erts/emulator/sys/common/erl_sys_common_misc.c b/erts/emulator/sys/common/erl_sys_common_misc.c index 420138ff0a..96bdbacb9e 100644 --- a/erts/emulator/sys/common/erl_sys_common_misc.c +++ b/erts/emulator/sys/common/erl_sys_common_misc.c @@ -150,59 +150,50 @@ int sys_double_to_chars_fast(double f, char *buffer, int buffer_size, int decimals, int compact) { - /* Note that some C compilers don't support "static const" propagation - * so we use a defines */ - #define SYS_DOUBLE_RND_CONST 0.55555555555555555 + #define SYS_DOUBLE_RND_CONST 0.5 #define FRAC_SIZE 52 #define EXP_SIZE 11 - #define EXP_MASK ((1ll << EXP_SIZE) - 1) + #define EXP_MASK (((Uint64)1 << EXP_SIZE) - 1) #define MAX_DECIMALS (sizeof(cs_sys_double_pow10) \ / sizeof(cs_sys_double_pow10[0])) - #define FRAC_MASK ((1ll << FRAC_SIZE) - 1) - #define FRAC_MASK2 ((1ll << (FRAC_SIZE + 1)) - 1) - #define MAX_FLOAT (1ll << (FRAC_SIZE+1)) + #define FRAC_MASK (((Uint64)1 << FRAC_SIZE) - 1) + #define FRAC_MASK2 (((Uint64)1 << (FRAC_SIZE + 1)) - 1) + #define MAX_FLOAT ((Uint64)1 << (FRAC_SIZE+1)) static const double cs_sys_double_pow10[] = { - SYS_DOUBLE_RND_CONST / 1ll, - SYS_DOUBLE_RND_CONST / 10ll, - SYS_DOUBLE_RND_CONST / 100ll, - SYS_DOUBLE_RND_CONST / 1000ll, - SYS_DOUBLE_RND_CONST / 10000ll, - SYS_DOUBLE_RND_CONST / 100000ll, - SYS_DOUBLE_RND_CONST / 1000000ll, - SYS_DOUBLE_RND_CONST / 10000000ll, - SYS_DOUBLE_RND_CONST / 100000000ll, - SYS_DOUBLE_RND_CONST / 1000000000ll, - SYS_DOUBLE_RND_CONST / 10000000000ll, - SYS_DOUBLE_RND_CONST / 100000000000ll, - SYS_DOUBLE_RND_CONST / 1000000000000ll, - SYS_DOUBLE_RND_CONST / 10000000000000ll, - SYS_DOUBLE_RND_CONST / 100000000000000ll, - SYS_DOUBLE_RND_CONST / 1000000000000000ll, - SYS_DOUBLE_RND_CONST / 10000000000000000ll, - SYS_DOUBLE_RND_CONST / 100000000000000000ll, - SYS_DOUBLE_RND_CONST / 1000000000000000000ll + SYS_DOUBLE_RND_CONST / 1e0, + SYS_DOUBLE_RND_CONST / 1e1, + SYS_DOUBLE_RND_CONST / 1e2, + SYS_DOUBLE_RND_CONST / 1e3, + SYS_DOUBLE_RND_CONST / 1e4, + SYS_DOUBLE_RND_CONST / 1e5, + SYS_DOUBLE_RND_CONST / 1e6, + SYS_DOUBLE_RND_CONST / 1e7, + SYS_DOUBLE_RND_CONST / 1e8, + SYS_DOUBLE_RND_CONST / 1e9, + SYS_DOUBLE_RND_CONST / 1e10, + SYS_DOUBLE_RND_CONST / 1e11, + SYS_DOUBLE_RND_CONST / 1e12, + SYS_DOUBLE_RND_CONST / 1e13, + SYS_DOUBLE_RND_CONST / 1e14, + SYS_DOUBLE_RND_CONST / 1e15, + SYS_DOUBLE_RND_CONST / 1e16, + SYS_DOUBLE_RND_CONST / 1e17, + SYS_DOUBLE_RND_CONST / 1e18 }; - long long mantissa, int_part = 0, frac_part = 0; - short exp; + Uint64 mantissa, int_part, frac_part; + int exp; + int fbits; int max; int neg; double fr; - union { long long L; double F; } x; + union { Uint64 L; double F; } x; char *p = buffer; if (decimals < 0) return -1; - /* Round the number to given decimal places. The number of 5's in the - * SYS_DOUBLE_RND_CONST constant is chosen such that adding any more 5's doesn't - * change the double precision of the number, i.e.: - * 1> term_to_binary(0.55555555555555555, [{minor_version, 1}]). - * <<131,70,63,225,199,28,113,199,28,114>> - * 2> term_to_binary(0.5555555555555555555, [{minor_version, 1}]). - * <<131,70,63,225,199,28,113,199,28,114>> - */ if (f >= 0) { neg = 0; fr = decimals < MAX_DECIMALS ? (f + cs_sys_double_pow10[decimals]) : f; @@ -233,7 +224,7 @@ sys_double_to_chars_fast(double f, char *buffer, int buffer_size, int decimals, } exp -= EXP_MASK >> 1; - mantissa |= (1ll << FRAC_SIZE); + mantissa |= ((Uint64)1 << FRAC_SIZE); /* Don't bother with optimizing too large numbers or too large precision */ if (x.F > MAX_FLOAT || decimals >= MAX_DECIMALS) { @@ -248,11 +239,16 @@ sys_double_to_chars_fast(double f, char *buffer, int buffer_size, int decimals, return p - buffer; } else if (exp >= FRAC_SIZE) { int_part = mantissa << (exp - FRAC_SIZE); + frac_part = 0; + fbits = FRAC_SIZE; /* not important as frac_part==0 */ } else if (exp >= 0) { - int_part = mantissa >> (FRAC_SIZE - exp); - frac_part = (mantissa << (exp + 1)) & FRAC_MASK2; + fbits = FRAC_SIZE - exp; + int_part = mantissa >> fbits; + frac_part = mantissa & (((Uint64)1 << fbits) -1); } else /* if (exp < 0) */ { - frac_part = (mantissa & FRAC_MASK2) >> -(exp + 1); + int_part = 0; + frac_part = mantissa; + fbits = FRAC_SIZE - exp; } if (!int_part) { @@ -262,9 +258,8 @@ sys_double_to_chars_fast(double f, char *buffer, int buffer_size, int decimals, } else { int ret, i, n; while (int_part != 0) { - long long j = int_part / 10; - *p++ = (char)(int_part - ((j << 3) + (j << 1)) + '0'); - int_part = j; + *p++ = (char)((int_part % 10) + '0'); + int_part /= 10; } if (neg) *p++ = '-'; @@ -290,11 +285,22 @@ sys_double_to_chars_fast(double f, char *buffer, int buffer_size, int decimals, max = decimals; for (i = 0; i < max; i++) { - /* frac_part *= 10; */ - frac_part = (frac_part << 3) + (frac_part << 1); - - *p++ = (char)((frac_part >> (FRAC_SIZE + 1)) + '0'); - frac_part &= FRAC_MASK2; + if (frac_part > (ERTS_UINT64_MAX/5)) { + frac_part >>= 3; + fbits -= 3; + } + + /* Multiply by 10 (5*2) to extract decimal digit as integer part */ + frac_part *= 5; + fbits--; + + if (fbits >= 64) { + *p++ = '0'; + } + else { + *p++ = (char)((frac_part >> fbits) + '0'); + frac_part &= ((Uint64)1 << fbits) - 1; + } } /* Delete trailing zeroes */ diff --git a/erts/emulator/sys/unix/erl_child_setup.c b/erts/emulator/sys/unix/erl_child_setup.c index 69fc6c2879..57973b10d7 100644 --- a/erts/emulator/sys/unix/erl_child_setup.c +++ b/erts/emulator/sys/unix/erl_child_setup.c @@ -437,6 +437,21 @@ main(int argc, char *argv[]) exit(1); } + /* Ignore SIGTERM. + Some container environments send SIGTERM to all processes + when terminating. We don't want erl_child_setup to terminate + in these cases as that will prevent beam from properly + cleaning up. + */ + sa.sa_handler = SIG_IGN; + sigemptyset(&sa.sa_mask); + sa.sa_flags = 0; + + if (sigaction(SIGTERM, &sa, 0) == -1) { + perror(NULL); + exit(1); + } + forker_hash_init(); SET_CLOEXEC(uds_fd); diff --git a/erts/emulator/sys/unix/erl_unix_sys.h b/erts/emulator/sys/unix/erl_unix_sys.h index b6f5b319ee..e367d565a7 100644 --- a/erts/emulator/sys/unix/erl_unix_sys.h +++ b/erts/emulator/sys/unix/erl_unix_sys.h @@ -133,7 +133,7 @@ #define ERTS_SYS_CONTINOUS_FD_NUMBERS -typedef void *GETENV_STATE; +void erts_sys_env_init(void); /* ** For the erl_timer_sup module. diff --git a/erts/emulator/sys/unix/sys.c b/erts/emulator/sys/unix/sys.c index 6315135151..189ca083d7 100644 --- a/erts/emulator/sys/unix/sys.c +++ b/erts/emulator/sys/unix/sys.c @@ -62,9 +62,6 @@ #include "erl_mseg.h" -extern char **environ; -erts_rwmtx_t environ_rwmtx; - #define MAX_VSIZE 16 /* Max number of entries allowed in an I/O * vector sock_sendv(). */ @@ -77,7 +74,7 @@ erts_rwmtx_t environ_rwmtx; #include "erl_check_io.h" #include "erl_cpu_topology.h" - +#include "erl_osenv.h" extern int driver_interrupt(int, int); extern void do_break(void); @@ -454,10 +451,10 @@ prepare_crash_dump(int secs) close(crashdump_companion_cube_fd); envsz = sizeof(env); - i = erts_sys_getenv__("ERL_CRASH_DUMP_NICE", env, &envsz); + i = erts_sys_explicit_8bit_getenv("ERL_CRASH_DUMP_NICE", env, &envsz); if (i >= 0) { int nice_val; - nice_val = i != 0 ? 0 : atoi(env); + nice_val = i != 1 ? 0 : atoi(env); if (nice_val > 39) { nice_val = 39; } @@ -749,34 +746,6 @@ void os_version(int *pMajor, int *pMinor, int *pBuild) { *pBuild = get_number(&release); /* Pointer to build number. */ } -void init_getenv_state(GETENV_STATE *state) -{ - erts_rwmtx_rlock(&environ_rwmtx); - *state = NULL; -} - -char *getenv_string(GETENV_STATE *state0) -{ - char **state = (char **) *state0; - char *cp; - - ERTS_LC_ASSERT(erts_lc_rwmtx_is_rlocked(&environ_rwmtx)); - - if (state == NULL) - state = environ; - - cp = *state++; - *state0 = (GETENV_STATE) state; - - return cp; -} - -void fini_getenv_state(GETENV_STATE *state) -{ - *state = NULL; - erts_rwmtx_runlock(&environ_rwmtx); -} - void erts_do_break_handling(void) { struct termios temp_mode; @@ -830,90 +799,6 @@ void sys_get_pid(char *buffer, size_t buffer_size){ erts_snprintf(buffer, buffer_size, "%lu",(unsigned long) p); } -int -erts_sys_putenv_raw(char *key, char *value) { - return erts_sys_putenv(key, value); -} -int -erts_sys_putenv(char *key, char *value) -{ - int res; - char *env; - Uint need = strlen(key) + strlen(value) + 2; - -#ifdef HAVE_COPYING_PUTENV - env = erts_alloc(ERTS_ALC_T_TMP, need); -#else - env = erts_alloc(ERTS_ALC_T_PUTENV_STR, need); - erts_atomic_add_nob(&sys_misc_mem_sz, need); -#endif - strcpy(env,key); - strcat(env,"="); - strcat(env,value); - erts_rwmtx_rwlock(&environ_rwmtx); - res = putenv(env); - erts_rwmtx_rwunlock(&environ_rwmtx); -#ifdef HAVE_COPYING_PUTENV - erts_free(ERTS_ALC_T_TMP, env); -#endif - return res; -} - -int -erts_sys_getenv__(char *key, char *value, size_t *size) -{ - int res; - char *orig_value = getenv(key); - if (!orig_value) - res = -1; - else { - size_t len = sys_strlen(orig_value); - if (len >= *size) { - *size = len + 1; - res = 1; - } - else { - *size = len; - sys_memcpy((void *) value, (void *) orig_value, len+1); - res = 0; - } - } - return res; -} - -int -erts_sys_getenv_raw(char *key, char *value, size_t *size) { - return erts_sys_getenv(key, value, size); -} - -/* - * erts_sys_getenv - * returns: - * -1, if environment key is not set with a value - * 0, if environment key is set and value fits into buffer size - * 1, if environment key is set but does not fit into buffer size - * size is set with the needed buffer size value - */ - -int -erts_sys_getenv(char *key, char *value, size_t *size) -{ - int res; - erts_rwmtx_rlock(&environ_rwmtx); - res = erts_sys_getenv__(key, value, size); - erts_rwmtx_runlock(&environ_rwmtx); - return res; -} - -int -erts_sys_unsetenv(char *key) -{ - int res; - erts_rwmtx_rwlock(&environ_rwmtx); - res = unsetenv(key); - erts_rwmtx_rwunlock(&environ_rwmtx); - return res; -} void sys_init_io(void) { } void erts_sys_alloc_init(void) { } @@ -1260,14 +1145,9 @@ erts_sys_main_thread(void) } } - void erl_sys_args(int* argc, char** argv) { - - erts_rwmtx_init(&environ_rwmtx, "environ", NIL, - ERTS_LOCK_FLAGS_PROPERTY_STATIC | ERTS_LOCK_FLAGS_CATEGORY_GENERIC); - ASSERT(argc && argv); max_files = erts_check_io_max_files(); @@ -1275,4 +1155,5 @@ erl_sys_args(int* argc, char** argv) init_smp_sig_notify(); init_smp_sig_suspend(); + erts_sys_env_init(); } diff --git a/erts/emulator/sys/unix/sys_drivers.c b/erts/emulator/sys/unix/sys_drivers.c index 0228e1af54..b7ac89d89a 100644 --- a/erts/emulator/sys/unix/sys_drivers.c +++ b/erts/emulator/sys/unix/sys_drivers.c @@ -55,9 +55,6 @@ #include "erl_threads.h" -extern char **environ; -extern erts_rwmtx_t environ_rwmtx; - extern erts_atomic_t sys_misc_mem_sz; static Eterm forker_port; @@ -180,7 +177,7 @@ erl_sys_late_init(void) opts.read_write = 0; opts.hide_window = 0; opts.wd = NULL; - opts.envir = NULL; + erts_osenv_init(&opts.envir); opts.exit_status = 0; opts.overlapped_io = 0; opts.spawn_type = ERTS_SPAWN_ANY; @@ -443,85 +440,55 @@ static void close_pipes(int ifd[2], int ofd[2]) close(ofd[1]); } -static char **build_unix_environment(char *block) +struct __add_spawn_env_state { + struct iovec *iov; + int *iov_index; + + Sint32 *payload_size; + char *env_block; +}; + +static void add_spawn_env_block_foreach(void *_state, + const erts_osenv_data_t *key, + const erts_osenv_data_t *value) { - int i; - int j; - int len; - char *cp; - char **cpp; - char** old_env; - - ERTS_LC_ASSERT(erts_lc_rwmtx_is_rlocked(&environ_rwmtx)); - - cp = block; - len = 0; - while (*cp != '\0') { - cp += strlen(cp) + 1; - len++; - } - old_env = environ; - while (*old_env++ != NULL) { - len++; - } - - cpp = (char **) erts_alloc_fnf(ERTS_ALC_T_ENVIRONMENT, - sizeof(char *) * (len+1)); - if (cpp == NULL) { - return NULL; - } + struct __add_spawn_env_state *state; + struct iovec *iov; - cp = block; - len = 0; - while (*cp != '\0') { - cpp[len] = cp; - cp += strlen(cp) + 1; - len++; - } - - i = len; - for (old_env = environ; *old_env; old_env++) { - char* old = *old_env; - - for (j = 0; j < len; j++) { - char *s, *t; - - /* check if cpp[j] equals old - before the = sign, - i.e. - "TMPDIR=/tmp/" */ - s = cpp[j]; - t = old; - while (*s == *t && *s != '=') { - s++, t++; - } - if (*s == '=' && *t == '=') { - break; - } - } + state = (struct __add_spawn_env_state*)(_state); + iov = &state->iov[*state->iov_index]; - if (j == len) { /* New version not found */ - cpp[len++] = old; - } - } + iov->iov_base = state->env_block; - for (j = 0; j < i; ) { - size_t last = strlen(cpp[j])-1; - if (cpp[j][last] == '=' && strchr(cpp[j], '=') == cpp[j]+last) { - cpp[j] = cpp[--len]; - if (len < i) { - i--; - } else { - j++; - } - } - else { - j++; - } - } + sys_memcpy(state->env_block, key->data, key->length); + state->env_block += key->length; + *state->env_block++ = '='; + sys_memcpy(state->env_block, value->data, value->length); + state->env_block += value->length; + *state->env_block++ = '\0'; - cpp[len] = NULL; - return cpp; + iov->iov_len = state->env_block - (char*)iov->iov_base; + + (*state->payload_size) += iov->iov_len; + (*state->iov_index)++; +} + +static void *add_spawn_env_block(const erts_osenv_t *env, struct iovec *iov, + int *iov_index, Sint32 *payload_size) { + struct __add_spawn_env_state add_state; + char *env_block; + + env_block = erts_alloc(ERTS_ALC_T_TMP, env->content_size + + env->variable_count * sizeof("=\0")); + + add_state.iov = iov; + add_state.iov_index = iov_index; + add_state.env_block = env_block; + add_state.payload_size = payload_size; + + erts_osenv_foreach_native(env, &add_state, add_spawn_env_block_foreach); + + return env_block; } static ErlDrvData spawn_start(ErlDrvPort port_num, char* name, @@ -531,7 +498,6 @@ static ErlDrvData spawn_start(ErlDrvPort port_num, char* name, #define CMD_LINE_PREFIX_STR_SZ (sizeof(CMD_LINE_PREFIX_STR) - 1) int len; - char **new_environ; ErtsSysDriverData *dd; char *cmd_line; char wd_buff[MAXPATHLEN+1]; @@ -598,19 +564,7 @@ static ErlDrvData spawn_start(ErlDrvPort port_num, char* name, memcpy((void *) (cmd_line + CMD_LINE_PREFIX_STR_SZ), (void *) name, len); cmd_line[CMD_LINE_PREFIX_STR_SZ + len] = '\0'; len = CMD_LINE_PREFIX_STR_SZ + len + 1; - } - - erts_rwmtx_rlock(&environ_rwmtx); - - if (opts->envir == NULL) { - new_environ = environ; - } else if ((new_environ = build_unix_environment(opts->envir)) == NULL) { - erts_rwmtx_runlock(&environ_rwmtx); - close_pipes(ifd, ofd); - erts_free(ERTS_ALC_T_TMP, (void *) cmd_line); - errno = ENOMEM; - return ERL_DRV_ERROR_ERRNO; - } +} if ((cwd = getcwd(wd_buff, MAXPATHLEN+1)) == NULL) { /* on some OSs this call opens a fd in the @@ -619,9 +573,6 @@ static ErlDrvData spawn_start(ErlDrvPort port_num, char* name, int err = errno; close_pipes(ifd, ofd); erts_free(ERTS_ALC_T_TMP, (void *) cmd_line); - if (new_environ != environ) - erts_free(ERTS_ALC_T_ENVIRONMENT, (void *) new_environ); - erts_rwmtx_runlock(&environ_rwmtx); errno = err; return ERL_DRV_ERROR_ERRNO; } @@ -629,6 +580,7 @@ static ErlDrvData spawn_start(ErlDrvPort port_num, char* name, wd = opts->wd; { + void *environment_block; struct iovec *io_vector; int iov_len = 5; char nullbuff[] = "\0"; @@ -641,10 +593,8 @@ static ErlDrvData spawn_start(ErlDrvPort port_num, char* name, if (wd) iov_len++; - /* count number of elements in environment */ - while(new_environ[env_len] != NULL) - env_len++; - iov_len += 1 + env_len; /* num envs including size int */ + /* num envs including size int */ + iov_len += 1 + opts->envir.variable_count; /* count number of element in argument list */ if (opts->spawn_type == ERTS_SPAWN_EXECUTABLE) { @@ -661,10 +611,7 @@ static ErlDrvData spawn_start(ErlDrvPort port_num, char* name, if (!io_vector) { close_pipes(ifd, ofd); - erts_rwmtx_runlock(&environ_rwmtx); erts_free(ERTS_ALC_T_TMP, (void *) cmd_line); - if (new_environ != environ) - erts_free(ERTS_ALC_T_ENVIRONMENT, (void *) new_environ); errno = ENOMEM; return ERL_DRV_ERROR_ERRNO; } @@ -699,16 +646,13 @@ static ErlDrvData spawn_start(ErlDrvPort port_num, char* name, io_vector[i++].iov_len = 1; buffsz += io_vector[i-1].iov_len; + env_len = htonl(opts->envir.variable_count); io_vector[i].iov_base = (void*)&env_len; - env_len = htonl(env_len); io_vector[i++].iov_len = sizeof(env_len); buffsz += io_vector[i-1].iov_len; - for (j = 0; new_environ[j] != NULL; j++) { - io_vector[i].iov_base = new_environ[j]; - io_vector[i++].iov_len = strlen(new_environ[j]) + 1; - buffsz += io_vector[i-1].iov_len; - } + environment_block = add_spawn_env_block(&opts->envir, io_vector, &i, + &buffsz); /* only append arguments if this was a spawn_executable */ if (opts->spawn_type == ERTS_SPAWN_EXECUTABLE) { @@ -744,9 +688,6 @@ static ErlDrvData spawn_start(ErlDrvPort port_num, char* name, int err = errno; close_pipes(ifd, ofd); erts_free(ERTS_ALC_T_TMP, io_vector); - if (new_environ != environ) - erts_free(ERTS_ALC_T_ENVIRONMENT, (void *) new_environ); - erts_rwmtx_runlock(&environ_rwmtx); erts_free(ERTS_ALC_T_TMP, (void *) cmd_line); errno = err; return ERL_DRV_ERROR_ERRNO; @@ -767,16 +708,12 @@ static ErlDrvData spawn_start(ErlDrvPort port_num, char* name, driver_select(port_num, ofd[1], ERL_DRV_WRITE|ERL_DRV_USE, 1); } + erts_free(ERTS_ALC_T_TMP, environment_block); erts_free(ERTS_ALC_T_TMP, io_vector); } erts_free(ERTS_ALC_T_TMP, (void *) cmd_line); - if (new_environ != environ) - erts_free(ERTS_ALC_T_ENVIRONMENT, (void *) new_environ); - - erts_rwmtx_runlock(&environ_rwmtx); - dd = create_driver_data(port_num, ifd[0], ofd[1], opts->packet_bytes, DO_WRITE | DO_READ, opts->exit_status, 0, 0); @@ -1652,15 +1589,13 @@ static ErlDrvData forker_start(ErlDrvPort port_num, char* name, forker_port = erts_drvport2id(port_num); - res = erts_sys_getenv_raw("BINDIR", bindir, &bindirsz); - if (res != 0) { - if (res < 0) - erts_exit(1, - "Environment variable BINDIR is not set\n"); - if (res > 0) - erts_exit(1, - "Value of environment variable BINDIR is too large\n"); + res = erts_sys_explicit_8bit_getenv("BINDIR", bindir, &bindirsz); + if (res == 0) { + erts_exit(1, "Environment variable BINDIR is not set\n"); + } else if(res < 0) { + erts_exit(1, "Value of environment variable BINDIR is too large\n"); } + if (bindir[0] != DIR_SEPARATOR_CHAR) erts_exit(1, "Environment variable BINDIR does not contain an" diff --git a/erts/emulator/sys/unix/sys_env.c b/erts/emulator/sys/unix/sys_env.c new file mode 100644 index 0000000000..4d8301f985 --- /dev/null +++ b/erts/emulator/sys/unix/sys_env.c @@ -0,0 +1,133 @@ + +#ifdef HAVE_CONFIG_H +# include "config.h" +#endif + +#include "sys.h" +#include "erl_osenv.h" +#include "erl_alloc.h" + +#include "erl_thr_progress.h" + +static erts_osenv_t sysenv_global_env; +static erts_rwmtx_t sysenv_rwmtx; + +extern char **environ; + +static void import_initial_env(void); + +void erts_sys_env_init() { + erts_rwmtx_init(&sysenv_rwmtx, "environ", NIL, + ERTS_LOCK_FLAGS_PROPERTY_STATIC | ERTS_LOCK_FLAGS_CATEGORY_GENERIC); + + erts_osenv_init(&sysenv_global_env); + import_initial_env(); +} + +const erts_osenv_t *erts_sys_rlock_global_osenv() { + erts_rwmtx_rlock(&sysenv_rwmtx); + return &sysenv_global_env; +} + +erts_osenv_t *erts_sys_rwlock_global_osenv() { + erts_rwmtx_rwlock(&sysenv_rwmtx); + return &sysenv_global_env; +} + +void erts_sys_rwunlock_global_osenv() { + erts_rwmtx_rwunlock(&sysenv_rwmtx); +} + +void erts_sys_runlock_global_osenv() { + erts_rwmtx_runlock(&sysenv_rwmtx); +} + +int erts_sys_explicit_8bit_putenv(char *key, char *value) { + erts_osenv_data_t env_key, env_value; + int result; + + env_key.length = sys_strlen(key); + env_key.data = key; + + env_value.length = sys_strlen(value); + env_value.data = value; + + { + erts_osenv_t *env = erts_sys_rwlock_global_osenv(); + result = erts_osenv_put_native(env, &env_key, &env_value); + erts_sys_rwunlock_global_osenv(); + } + + return result; +} + +int erts_sys_explicit_8bit_getenv(char *key, char *value, size_t *size) { + erts_osenv_data_t env_key, env_value; + int result; + + env_key.length = sys_strlen(key); + env_key.data = key; + + /* Reserve space for NUL termination. */ + env_value.length = *size - 1; + env_value.data = value; + + { + const erts_osenv_t *env = erts_sys_rlock_global_osenv(); + result = erts_osenv_get_native(env, &env_key, &env_value); + erts_sys_runlock_global_osenv(); + } + + if(result == 1) { + value[env_value.length] = '\0'; + } + + *size = env_value.length; + + return result; +} + +int erts_sys_explicit_host_getenv(char *key, char *value, size_t *size) { + char *orig_value; + size_t length; + + orig_value = getenv(key); + + if(orig_value == NULL) { + return 0; + } + + length = sys_strlen(orig_value); + + if (length >= *size) { + *size = length + 1; + return -1; + } + + sys_memcpy((void*)value, (void*)orig_value, length + 1); + *size = length; + + return 1; +} + +static void import_initial_env(void) { + char **environ_iterator, *environ_variable; + + environ_iterator = environ; + + while ((environ_variable = *(environ_iterator++)) != NULL) { + char *separator_index = strchr(environ_variable, '='); + + if (separator_index != NULL) { + erts_osenv_data_t env_key, env_value; + + env_key.length = separator_index - environ_variable; + env_key.data = environ_variable; + + env_value.length = sys_strlen(separator_index) - 1; + env_value.data = separator_index + 1; + + erts_osenv_put_native(&sysenv_global_env, &env_key, &env_value); + } + } +} diff --git a/erts/emulator/sys/win32/erl_win32_sys_ddll.c b/erts/emulator/sys/win32/erl_win32_sys_ddll.c index 274133a346..fc2179328f 100644 --- a/erts/emulator/sys/win32/erl_win32_sys_ddll.c +++ b/erts/emulator/sys/win32/erl_win32_sys_ddll.c @@ -46,9 +46,17 @@ static TWinDynDriverCallbacks wddc; static TWinDynNifCallbacks nif_callbacks; void erl_sys_ddll_init(void) { + WCHAR cwd_buffer[MAX_PATH]; + tls_index = TlsAlloc(); ERL_INIT_CALLBACK_STRUCTURE(wddc); + /* LOAD_WITH_ALTERED_SEARCH_PATH removes the startup directory from the + * search path, so we add it separately to be backwards compatible. */ + if (GetCurrentDirectoryW(sizeof(cwd_buffer), cwd_buffer)) { + SetDllDirectoryW(cwd_buffer); + } + #define ERL_NIF_API_FUNC_DECL(RET,NAME,ARGS) nif_callbacks.NAME = NAME #include "erl_nif_api_funcs.h" #undef ERL_NIF_API_FUNC_DECL @@ -81,7 +89,10 @@ int erts_sys_ddll_open(const char *full_name, void **handle, ErtsSysDdllError* e ERTS_ALC_T_TMP, &used, EXT_LEN); wcscpy(&wcp[used/2 - 1], FILE_EXT_WCHAR); - if ((hinstance = LoadLibraryW(wcp)) == NULL) { + /* LOAD_WITH_ALTERED_SEARCH_PATH adds the specified DLL's directory to the + * dependency search path. This also removes the directory we started in, + * but we've explicitly added that in in erl_sys_ddll_init. */ + if ((hinstance = LoadLibraryExW(wcp, NULL, LOAD_WITH_ALTERED_SEARCH_PATH)) == NULL) { code = ERL_DE_DYNAMIC_ERROR_OFFSET - GetLastError(); if (err != NULL) { err->str = erts_sys_ddll_error(code); diff --git a/erts/emulator/sys/win32/sys.c b/erts/emulator/sys/win32/sys.c index 0598a12351..a1c630d68a 100644 --- a/erts/emulator/sys/win32/sys.c +++ b/erts/emulator/sys/win32/sys.c @@ -77,6 +77,7 @@ static int create_pipe(LPHANDLE, LPHANDLE, BOOL, BOOL); static int application_type(const wchar_t* originalName, wchar_t fullPath[MAX_PATH], BOOL search_in_path, BOOL handle_quotes, int *error_return); +static void *build_env_block(const erts_osenv_t *env); HANDLE erts_service_event; @@ -1190,7 +1191,6 @@ spawn_start(ErlDrvPort port_num, char* utf8_name, SysDriverOpts* opts) int ok; int neededSelects = 0; SECURITY_ATTRIBUTES sa = {sizeof(SECURITY_ATTRIBUTES), NULL, TRUE}; - char* envir = opts->envir; int errno_return = -1; wchar_t *name; int len; @@ -1265,29 +1265,33 @@ spawn_start(ErlDrvPort port_num, char* utf8_name, SysDriverOpts* opts) name[i] = L'\0'; } DEBUGF(("Spawning \"%S\"\n", name)); - envir = win_build_environment(envir); /* Always a unicode environment */ - ok = create_child_process(name, - hChildStdin, - hChildStdout, - hChildStderr, - &dp->port_pid, - &pid, - opts->hide_window, - (LPVOID) envir, - (wchar_t *) opts->wd, - opts->spawn_type, - (wchar_t **) opts->argv, - &errno_return); - CloseHandle(hChildStdin); - CloseHandle(hChildStdout); - if (close_child_stderr && hChildStderr != INVALID_HANDLE_VALUE && - hChildStderr != 0) { - CloseHandle(hChildStderr); - } - erts_free(ERTS_ALC_T_TMP, name); - - if (envir != NULL) { - erts_free(ERTS_ALC_T_ENVIRONMENT, envir); + + { + void *environment_block = build_env_block(&opts->envir); + + ok = create_child_process(name, + hChildStdin, + hChildStdout, + hChildStderr, + &dp->port_pid, + &pid, + opts->hide_window, + environment_block, + (wchar_t *) opts->wd, + opts->spawn_type, + (wchar_t **) opts->argv, + &errno_return); + + CloseHandle(hChildStdin); + CloseHandle(hChildStdout); + + if (close_child_stderr && hChildStderr != INVALID_HANDLE_VALUE && + hChildStderr != 0) { + CloseHandle(hChildStderr); + } + + erts_free(ERTS_ALC_T_TMP, environment_block); + erts_free(ERTS_ALC_T_TMP, name); } if (!ok) { @@ -1338,6 +1342,41 @@ spawn_start(ErlDrvPort port_num, char* utf8_name, SysDriverOpts* opts) return retval; } +struct __build_env_state { + WCHAR *next_variable; +}; + +static void build_env_foreach(void *_state, const erts_osenv_data_t *key, + const erts_osenv_data_t *value) +{ + struct __build_env_state *state = (struct __build_env_state*)(_state); + + sys_memcpy(state->next_variable, key->data, key->length); + state->next_variable += (int)key->length / sizeof(WCHAR); + *state->next_variable++ = L'='; + + sys_memcpy(state->next_variable, value->data, value->length); + state->next_variable += (int)value->length / sizeof(WCHAR); + *state->next_variable++ = L'\0'; +} + +/* Builds an environment block suitable for CreateProcessW. */ +static void *build_env_block(const erts_osenv_t *env) { + struct __build_env_state build_state; + WCHAR *env_block; + + env_block = erts_alloc(ERTS_ALC_T_TMP, env->content_size + + (env->variable_count * sizeof(L"=\0") + sizeof(L'\0'))); + + build_state.next_variable = env_block; + + erts_osenv_foreach_native(env, &build_state, build_env_foreach); + + (*build_state.next_variable) = L'\0'; + + return env_block; +} + static int create_file_thread(AsyncIo* aio, int mode) { diff --git a/erts/emulator/sys/win32/sys_env.c b/erts/emulator/sys/win32/sys_env.c index 5792816267..c78161b344 100644 --- a/erts/emulator/sys/win32/sys_env.c +++ b/erts/emulator/sys/win32/sys_env.c @@ -1,319 +1,212 @@ -/*
- * %CopyrightBegin%
- *
- * Copyright Ericsson AB 2002-2016. All Rights Reserved.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- * %CopyrightEnd%
- */
-
-#ifdef HAVE_CONFIG_H
-# include "config.h"
-#endif
-
-#include "sys.h"
-#include "erl_sys_driver.h"
-#include "erl_alloc.h"
-
-static WCHAR *merge_environment(WCHAR *current, WCHAR *add);
-static WCHAR *arg_to_env(WCHAR **arg);
-static WCHAR **env_to_arg(WCHAR *env);
-static WCHAR **find_arg(WCHAR **arg, WCHAR *str);
-static int compare(const void *a, const void *b);
-
-static erts_rwmtx_t environ_rwmtx; -
-void
-erts_sys_env_init(void)
-{
- erts_rwmtx_init(&environ_rwmtx, "environ", NIL, - ERTS_LOCK_FLAGS_PROPERTY_STATIC | ERTS_LOCK_FLAGS_CATEGORY_GENERIC);
-}
-
-int
-erts_sys_putenv_raw(char *key, char *value)
-{
- int res;
- erts_rwmtx_rwlock(&environ_rwmtx); - res = (SetEnvironmentVariable((LPCTSTR) key,
- (LPCTSTR) value) ? 0 : 1);
- erts_rwmtx_rwunlock(&environ_rwmtx); - return res;
-}
-
-int
-erts_sys_putenv(char *key, char *value)
-{
- int res;
- WCHAR *wkey = (WCHAR *) key;
- WCHAR *wvalue = (WCHAR *) value;
- erts_rwmtx_rwlock(&environ_rwmtx); - res = (SetEnvironmentVariableW(wkey,
- wvalue) ? 0 : 1);
- erts_rwmtx_rwunlock(&environ_rwmtx); - return res;
-}
-
-int
-erts_sys_getenv(char *key, char *value, size_t *size)
-{
- size_t req_size = 0;
- int res = 0;
- DWORD new_size;
- WCHAR *wkey = (WCHAR *) key;
- WCHAR *wvalue = (WCHAR *) value;
- DWORD wsize = *size / (sizeof(WCHAR) / sizeof(char));
-
- SetLastError(0);
- erts_rwmtx_rlock(&environ_rwmtx); - new_size = GetEnvironmentVariableW(wkey,
- wvalue,
- (DWORD) wsize);
- res = !new_size && GetLastError() == ERROR_ENVVAR_NOT_FOUND ? -1 : 0;
- erts_rwmtx_runlock(&environ_rwmtx); - if (res < 0)
- return res;
- res = new_size > wsize ? 1 : 0;
- *size = new_size * (sizeof(WCHAR) / sizeof(char));
- return res;
-}
-int
-erts_sys_getenv__(char *key, char *value, size_t *size)
-{
- size_t req_size = 0;
- int res = 0;
- DWORD new_size;
-
- SetLastError(0);
- new_size = GetEnvironmentVariable((LPCTSTR) key,
- (LPTSTR) value,
- (DWORD) *size);
- res = !new_size && GetLastError() == ERROR_ENVVAR_NOT_FOUND ? -1 : 0;
- if (res < 0)
- return res;
- res = new_size > *size ? 1 : 0;
- *size = new_size;
- return res;
-}
-
-int
-erts_sys_getenv_raw(char *key, char *value, size_t *size)
-{
- int res;
- erts_rwmtx_rlock(&environ_rwmtx); - res = erts_sys_getenv__(key, value, size);
- erts_rwmtx_runlock(&environ_rwmtx); - return res;
-}
-
-void init_getenv_state(GETENV_STATE *state)
-{
- erts_rwmtx_rlock(&environ_rwmtx); - state->environment_strings = GetEnvironmentStringsW();
- state->next_string = state->environment_strings;
-}
-
-char *getenv_string(GETENV_STATE *state)
-{
- ERTS_LC_ASSERT(erts_lc_rwmtx_is_rlocked(&environ_rwmtx)); - if (state->next_string[0] == L'\0') {
- return NULL;
- } else {
- WCHAR *res = state->next_string;
- state->next_string += wcslen(res) + 1;
- return (char *) res;
- }
-}
-
-void fini_getenv_state(GETENV_STATE *state)
-{
- FreeEnvironmentStringsW(state->environment_strings);
- state->environment_strings = state->next_string = NULL;
- erts_rwmtx_runlock(&environ_rwmtx); -}
-
-int erts_sys_unsetenv(char *key)
-{
- int res = 0;
- WCHAR *wkey = (WCHAR *) key;
-
- SetLastError(0);
- erts_rwmtx_rlock(&environ_rwmtx); - GetEnvironmentVariableW(wkey,
- NULL,
- 0);
- if (GetLastError() != ERROR_ENVVAR_NOT_FOUND) {
- res = (SetEnvironmentVariableW(wkey,
- NULL) ? 0 : 1);
- }
- erts_rwmtx_runlock(&environ_rwmtx); - return res;
-}
-
-char*
-win_build_environment(char* new_env)
-{
- if (new_env == NULL) {
- return NULL;
- } else {
- WCHAR *tmp, *merged, *tmp_new;
-
- tmp_new = (WCHAR *) new_env;
-
- erts_rwmtx_rlock(&environ_rwmtx); - tmp = GetEnvironmentStringsW();
- merged = merge_environment(tmp, tmp_new);
-
- FreeEnvironmentStringsW(tmp);
- erts_rwmtx_runlock(&environ_rwmtx); - return (char *) merged;
- }
-}
-
-static WCHAR *
-merge_environment(WCHAR *old, WCHAR *add)
-{
- WCHAR **a_arg = env_to_arg(add);
- WCHAR **c_arg = env_to_arg(old);
- WCHAR *ret;
- int i, j;
-
- for(i = 0; c_arg[i] != NULL; ++i)
- ;
-
- for(j = 0; a_arg[j] != NULL; ++j)
- ;
-
- c_arg = erts_realloc(ERTS_ALC_T_TMP,
- c_arg, (i+j+1) * sizeof(WCHAR *));
-
- for(j = 0; a_arg[j] != NULL; ++j){
- WCHAR **tmp;
- WCHAR *current = a_arg[j];
- WCHAR *eq_p = wcschr(current,L'=');
- int unset = (eq_p!=NULL && eq_p[1]==L'\0');
-
- if ((tmp = find_arg(c_arg, current)) != NULL) {
- if (!unset) {
- *tmp = current;
- } else {
- *tmp = c_arg[--i];
- c_arg[i] = NULL;
- }
- } else if (!unset) {
- c_arg[i++] = current;
- c_arg[i] = NULL;
- }
- }
- ret = arg_to_env(c_arg);
- erts_free(ERTS_ALC_T_TMP, c_arg);
- erts_free(ERTS_ALC_T_TMP, a_arg);
- return ret;
-}
-
-static WCHAR**
-find_arg(WCHAR **arg, WCHAR *str)
-{
- WCHAR *tmp;
- int len;
-
- if ((tmp = wcschr(str, L'=')) != NULL) {
- tmp++;
- len = tmp - str;
- while (*arg != NULL){
- if (_wcsnicmp(*arg, str, len) == 0){
- return arg;
- }
- ++arg;
- }
- }
- return NULL;
-}
-
-static int
-compare(const void *a, const void *b)
-{
- WCHAR *s1 = *((WCHAR **) a);
- WCHAR *s2 = *((WCHAR **) b);
- WCHAR *e1 = wcschr(s1,L'=');
- WCHAR *e2 = wcschr(s2,L'=');
- int ret;
- int len;
-
- if(!e1)
- e1 = s1 + wcslen(s1);
- if(!e2)
- e2 = s2 + wcslen(s2);
-
- if((e1 - s1) > (e2 - s2))
- len = (e2 - s2);
- else
- len = (e1 - s1);
-
- ret = _wcsnicmp(s1,s2,len);
- if (ret == 0)
- return ((e1 - s1) - (e2 - s2));
- else
- return ret;
-}
-
-static WCHAR**
-env_to_arg(WCHAR *env)
-{
- WCHAR **ret;
- WCHAR *tmp;
- int i;
- int num_strings = 0;
-
- for(tmp = env; *tmp != '\0'; tmp += wcslen(tmp)+1) {
- ++num_strings;
- }
- ret = erts_alloc(ERTS_ALC_T_TMP, sizeof(WCHAR *) * (num_strings + 1));
- i = 0;
- for(tmp = env; *tmp != '\0'; tmp += wcslen(tmp)+1){
- ret[i++] = tmp;
- }
- ret[i] = NULL;
- return ret;
-}
-
-static WCHAR *
-arg_to_env(WCHAR **arg)
-{
- WCHAR *block;
- WCHAR *ptr;
- int i;
- int totlen = 1; /* extra '\0' */
-
- for(i = 0; arg[i] != NULL; ++i) {
- totlen += wcslen(arg[i])+1;
- }
-
- /* sort the environment vector */
- qsort(arg, i, sizeof(WCHAR *), &compare);
-
- if (totlen == 1){
- block = erts_alloc(ERTS_ALC_T_ENVIRONMENT, 2 * sizeof(WCHAR));
- block[0] = block[1] = '\0';
- } else {
- block = erts_alloc(ERTS_ALC_T_ENVIRONMENT, totlen * sizeof(WCHAR));
- ptr = block;
- for(i=0; arg[i] != NULL; ++i){
- wcscpy(ptr, arg[i]);
- ptr += wcslen(ptr)+1;
- }
- *ptr = '\0';
- }
- return block;
-}
+/* + * %CopyrightBegin% + * + * Copyright Ericsson AB 2002-2017. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * %CopyrightEnd% + */ + +#ifdef HAVE_CONFIG_H +# include "config.h" +#endif + +#include "sys.h" +#include "erl_sys_driver.h" +#include "erl_alloc.h" + +static erts_osenv_t sysenv_global_env; +static erts_rwmtx_t sysenv_rwmtx; + +static void import_initial_env(void); + +void erts_sys_env_init() { + erts_rwmtx_init(&sysenv_rwmtx, "environ", NIL, + ERTS_LOCK_FLAGS_PROPERTY_STATIC | ERTS_LOCK_FLAGS_CATEGORY_GENERIC); + + erts_osenv_init(&sysenv_global_env); + import_initial_env(); +} + +const erts_osenv_t *erts_sys_rlock_global_osenv() { + erts_rwmtx_rlock(&sysenv_rwmtx); + return &sysenv_global_env; +} + +erts_osenv_t *erts_sys_rwlock_global_osenv() { + erts_rwmtx_rwlock(&sysenv_rwmtx); + return &sysenv_global_env; +} + +void erts_sys_runlock_global_osenv() { + erts_rwmtx_runlock(&sysenv_rwmtx); +} + +void erts_sys_rwunlock_global_osenv() { + erts_rwmtx_rwunlock(&sysenv_rwmtx); +} + +int erts_sys_explicit_host_getenv(char *key, char *value, size_t *size) { + size_t new_size = GetEnvironmentVariableA(key, value, (DWORD)*size); + + if(new_size == 0 && GetLastError() == ERROR_ENVVAR_NOT_FOUND) { + return 0; + } else if(new_size > *size) { + return -1; + } + + *size = new_size; + return 1; +} + +int erts_sys_explicit_8bit_putenv(char *key, char *value) { + WCHAR *wide_key, *wide_value; + int key_length, value_length; + int result; + + /* Note that we do *NOT* honor the filename encoding flags (+fnu/+fnl) + * here; the previous implementation used SetEnvironmentVariableA and + * things may break if we step away from that. */ + + key_length = MultiByteToWideChar(CP_ACP, 0, key, -1, NULL, 0); + value_length = MultiByteToWideChar(CP_ACP, 0, value, -1, NULL, 0); + + /* Report "not found" if either string isn't convertible. */ + if(key_length == 0 || value_length == 0) { + return 0; + } + + wide_key = erts_alloc(ERTS_ALC_T_TMP, key_length * sizeof(WCHAR)); + wide_value = erts_alloc(ERTS_ALC_T_TMP, value_length * sizeof(WCHAR)); + + MultiByteToWideChar(CP_ACP, 0, key, -1, wide_key, key_length); + MultiByteToWideChar(CP_ACP, 0, value, -1, wide_value, value_length); + + { + erts_osenv_data_t env_key, env_value; + erts_osenv_t *env; + + env = erts_sys_rwlock_global_osenv(); + + /* -1 to exclude the NUL terminator. */ + env_key.length = (key_length - 1) * sizeof(WCHAR); + env_key.data = wide_key; + + env_value.length = (value_length - 1) * sizeof(WCHAR); + env_value.data = wide_value; + + result = erts_osenv_put_native(env, &env_key, &env_value); + erts_sys_rwunlock_global_osenv(); + } + + erts_free(ERTS_ALC_T_TMP, wide_key); + erts_free(ERTS_ALC_T_TMP, wide_value); + + return result; +} + +int erts_sys_explicit_8bit_getenv(char *key, char *value, size_t *size) { + erts_osenv_data_t env_key, env_value; + int key_length, value_length, result; + WCHAR *wide_key, *wide_value; + + key_length = MultiByteToWideChar(CP_ACP, 0, key, -1, NULL, 0); + + /* Report "not found" if the string isn't convertible. */ + if(key_length == 0) { + return 0; + } + + wide_key = erts_alloc(ERTS_ALC_T_TMP, key_length * sizeof(WCHAR)); + MultiByteToWideChar(CP_ACP, 0, key, -1, wide_key, key_length); + + /* We assume that the worst possible size is twice the output buffer width, + * as we could theoretically be on a code page that requires surrogates. */ + value_length = (*size) * 2; + wide_value = erts_alloc(ERTS_ALC_T_TMP, value_length * sizeof(WCHAR)); + + { + const erts_osenv_t *env = erts_sys_rlock_global_osenv(); + + /* -1 to exclude the NUL terminator. */ + env_key.length = (key_length - 1) * sizeof(WCHAR); + env_key.data = wide_key; + + env_value.length = value_length * sizeof(WCHAR); + env_value.data = wide_value; + + result = erts_osenv_get_native(env, &env_key, &env_value); + erts_sys_runlock_global_osenv(); + } + + if(result == 1 && env_value.length > 0) { + /* This function doesn't NUL-terminate if the provided size is >= 0, + * so we pass (*size - 1) to reserve space for it and then do it + * manually. */ + *size = WideCharToMultiByte(CP_ACP, 0, env_value.data, + env_value.length / sizeof(WCHAR), value, *size - 1, NULL, NULL); + + if(*size == 0) { + if(GetLastError() == ERROR_INSUFFICIENT_BUFFER) { + result = -1; + } else { + result = 0; + } + } + } else { + *size = 0; + } + + if(*size > 0) { + value[*size] = '\0'; + } + + erts_free(ERTS_ALC_T_TMP, wide_key); + erts_free(ERTS_ALC_T_TMP, wide_value); + + return result; +} + +static void import_initial_env(void) { + WCHAR *environment_block, *current_variable; + + environment_block = GetEnvironmentStringsW(); + current_variable = environment_block; + + while(wcslen(current_variable) > 0) { + WCHAR *separator_index = wcschr(current_variable, L'='); + + /* We tolerate environment variables starting with '=' as the per-drive + * working directories are stored this way. */ + if(separator_index == current_variable) { + separator_index = wcschr(separator_index + 1, L'='); + } + + if(separator_index != NULL && separator_index != current_variable) { + erts_osenv_data_t env_key, env_value; + + env_key.length = (separator_index - current_variable) * sizeof(WCHAR); + env_key.data = current_variable; + + env_value.length = (wcslen(separator_index) - 1) * sizeof(WCHAR); + env_value.data = separator_index + 1; + + erts_osenv_put_native(&sysenv_global_env, &env_key, &env_value); + } + + current_variable += wcslen(current_variable) + 1; + } + + FreeEnvironmentStringsW(environment_block); +} diff --git a/erts/emulator/test/alloc_SUITE.erl b/erts/emulator/test/alloc_SUITE.erl index 5e5cd6f578..88ff2a7a92 100644 --- a/erts/emulator/test/alloc_SUITE.erl +++ b/erts/emulator/test/alloc_SUITE.erl @@ -31,6 +31,7 @@ mseg_clear_cache/1, erts_mmap/1, cpool/1, + set_dyn_param/1, migration/1]). -include_lib("common_test/include/ct.hrl"). @@ -41,6 +42,7 @@ suite() -> all() -> [basic, coalesce, threads, realloc_copy, bucket_index, + set_dyn_param, bucket_mask, rbtree, mseg_clear_cache, erts_mmap, cpool, migration]. init_per_testcase(Case, Config) when is_list(Config) -> @@ -65,7 +67,11 @@ mseg_clear_cache(Cfg) -> drv_case(Cfg). cpool(Cfg) -> drv_case(Cfg). migration(Cfg) -> - drv_case(Cfg, concurrent, "+MZe true"). + %% Enable test_alloc. + %% Disable driver_alloc to avoid recursive alloc_util calls + %% through enif_mutex_create() in my_creating_mbc(). + drv_case(Cfg, concurrent, "+MZe true +MRe false"), + drv_case(Cfg, concurrent, "+MZe true +MRe false +MZas ageffcbf"). erts_mmap(Config) when is_list(Config) -> case {os:type(), mmsc_flags()} of @@ -110,7 +116,7 @@ erts_mmap_do(Config, SCO, SCRPM, SCRFSD) -> 0 -> O1; _ -> O1 ++ " +MMscrfsd"++integer_to_list(SCRFSD) end, - {ok, Node} = start_node(Config, Opts), + {ok, Node} = start_node(Config, Opts, []), Self = self(), Ref = make_ref(), F = fun() -> @@ -140,6 +146,82 @@ erts_mmap_do(Config, SCO, SCRPM, SCRFSD) -> Result. +%% Test erlang:system_flag(erts_alloc, ...) +set_dyn_param(_Config) -> + {_, _, _, AlcList} = erlang:system_info(allocator), + + {Enabled, Disabled, Others} = + lists:foldl(fun({sys_alloc,_}, {Es, Ds, Os}) -> + {Es, [sys_alloc | Ds], Os}; + + ({AT, Opts}, {Es, Ds, Os}) when is_list(Opts) -> + case lists:keyfind(e, 1, Opts) of + {e, true} -> + {[AT | Es], Ds, Os}; + {e, false} -> + {Es, [AT | Ds], Os}; + false -> + {Es, Ds, [AT | Os]} + end; + + (_, Acc) -> Acc + end, + {[], [], []}, + AlcList), + + Param = sbct, + lists:foreach(fun(AT) -> set_dyn_param_enabled(AT, Param) end, + Enabled), + + lists:foreach(fun(AT) -> + Tpl = {AT, Param, 12345}, + io:format("~p\n", [Tpl]), + notsup = erlang:system_flag(erts_alloc, Tpl) + end, + Disabled), + + lists:foreach(fun(AT) -> + Tpl = {AT, Param, 12345}, + io:format("~p\n", [Tpl]), + {'EXIT',{badarg,_}} = + (catch erlang:system_flag(erts_alloc, Tpl)) + end, + Others), + ok. + +set_dyn_param_enabled(AT, Param) -> + OldVal = get_alc_param(AT, Param), + + Val1 = OldVal div 2, + Tuple = {AT, Param, Val1}, + io:format("~p\n", [Tuple]), + ok = erlang:system_flag(erts_alloc, Tuple), + Val1 = get_alc_param(AT, Param), + + ok = erlang:system_flag(erts_alloc, {AT, Param, OldVal}), + OldVal = get_alc_param(AT, Param), + ok. + +get_alc_param(AT, Param) -> + lists:foldl(fun({instance,_,Istats}, Acc) -> + {options,Opts} = lists:keyfind(options, 1, Istats), + {Param,Val} = lists:keyfind(Param, 1, Opts), + {as,Strategy} = lists:keyfind(as, 1, Opts), + + case {param_for_strat(Param, Strategy), Acc} of + {false, _} -> Acc; + {true, undefined} -> Val; + {true, _} -> + Val = Acc + end + end, + undefined, + erlang:system_info({allocator, AT})). + +param_for_strat(sbct, gf) -> false; +param_for_strat(_, _) -> true. + + %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% %% %% %% Internal functions %% @@ -151,7 +233,9 @@ drv_case(Config) -> drv_case(Config, Mode, NodeOpts) when is_list(Config) -> case os:type() of {Family, _} when Family == unix; Family == win32 -> - {ok, Node} = start_node(Config, NodeOpts), + %%Prog = {prog,"/my/own/otp/bin/cerl -debug"}, + Prog = [], + {ok, Node} = start_node(Config, NodeOpts, Prog), Self = self(), Ref = make_ref(), spawn_link(Node, @@ -217,19 +301,35 @@ wait_for_memory_deallocations() -> end. print_stats(migration) -> - {Btot,Ctot} = lists:foldl(fun({instance,Inr,Istats}, {Bacc,Cacc}) -> - {mbcs,MBCS} = lists:keyfind(mbcs, 1, Istats), - Btup = lists:keyfind(blocks, 1, MBCS), - Ctup = lists:keyfind(carriers, 1, MBCS), - io:format("{instance,~p,~p,~p}\n", [Inr, Btup, Ctup]), - {tuple_add(Bacc,Btup),tuple_add(Cacc,Ctup)}; - (_, Acc) -> Acc - end, - {{blocks,0,0,0},{carriers,0,0,0}}, - erlang:system_info({allocator,test_alloc})), - + IFun = fun({instance,Inr,Istats}, {Bacc,Cacc,Pacc}) -> + {mbcs,MBCS} = lists:keyfind(mbcs, 1, Istats), + Btup = lists:keyfind(blocks, 1, MBCS), + Ctup = lists:keyfind(carriers, 1, MBCS), + + Ptup = case lists:keyfind(mbcs_pool, 1, Istats) of + {mbcs_pool,POOL} -> + {blocks, Bpool} = lists:keyfind(blocks, 1, POOL), + {carriers, Cpool} = lists:keyfind(carriers, 1, POOL), + {pool, Bpool, Cpool}; + false -> + {pool, 0, 0} + end, + io:format("{instance,~p,~p,~p,~p}}\n", + [Inr, Btup, Ctup, Ptup]), + {tuple_add(Bacc,Btup),tuple_add(Cacc,Ctup), + tuple_add(Pacc,Ptup)}; + (_, Acc) -> Acc + end, + + {Btot,Ctot,Ptot} = lists:foldl(IFun, + {{blocks,0,0,0},{carriers,0,0,0},{pool,0,0}}, + erlang:system_info({allocator,test_alloc})), + + {pool, PBtot, PCtot} = Ptot, io:format("Number of blocks : ~p\n", [Btot]), - io:format("Number of carriers: ~p\n", [Ctot]); + io:format("Number of carriers: ~p\n", [Ctot]), + io:format("Number of pooled blocks : ~p\n", [PBtot]), + io:format("Number of pooled carriers: ~p\n", [PCtot]); print_stats(_) -> ok. tuple_add(T1, T2) -> @@ -326,13 +426,13 @@ handle_result(_State, Result0) -> continue end. -start_node(Config, Opts) when is_list(Config), is_list(Opts) -> +start_node(Config, Opts, Prog) when is_list(Config), is_list(Opts) -> case proplists:get_value(debug,Config) of true -> {ok, node()}; - _ -> start_node_1(Config, Opts) + _ -> start_node_1(Config, Opts, Prog) end. -start_node_1(Config, Opts) -> +start_node_1(Config, Opts, Prog) -> Pa = filename:dirname(code:which(?MODULE)), Name = list_to_atom(atom_to_list(?MODULE) ++ "-" @@ -341,7 +441,11 @@ start_node_1(Config, Opts) -> ++ integer_to_list(erlang:system_time(second)) ++ "-" ++ integer_to_list(erlang:unique_integer([positive]))), - test_server:start_node(Name, slave, [{args, Opts++" -pa "++Pa}]). + ErlArg = case Prog of + [] -> []; + _ -> [{erl,[Prog]}] + end, + test_server:start_node(Name, slave, [{args, Opts++" -pa "++Pa} | ErlArg]). stop_node(Node) when Node =:= node() -> ok; stop_node(Node) -> diff --git a/erts/emulator/test/alloc_SUITE_data/allocator_test.h b/erts/emulator/test/alloc_SUITE_data/allocator_test.h index 97ee58cdad..5272f86c98 100644 --- a/erts/emulator/test/alloc_SUITE_data/allocator_test.h +++ b/erts/emulator/test/alloc_SUITE_data/allocator_test.h @@ -156,7 +156,8 @@ typedef void* erts_cond; #define IS_SMP_ENABLED ((int) ALC_TEST0(0xf13)) #define ALLOC_TEST(S) ((void*) ALC_TEST1(0xf14, (S))) #define FREE_TEST(P) ((void) ALC_TEST1(0xf15, (P))) -#define SET_TEST_MBC_USER_HEADER(SZ,CMBC,DMBC) ((int)ALC_TEST3(0xf16, (SZ), (CMBC), (DMBC))) -#define GET_TEST_MBC_SIZE() ((int) ALC_TEST0(0xf17)) +#define REALLOC_TEST(P,S) ((void*) ALC_TEST2(0xf16, (P), (S))) +#define SET_TEST_MBC_USER_HEADER(SZ,CMBC,DMBC) ((int)ALC_TEST3(0xf17, (SZ), (CMBC), (DMBC))) +#define GET_TEST_MBC_SIZE() ((int) ALC_TEST0(0xf18)) #endif diff --git a/erts/emulator/test/alloc_SUITE_data/migration.c b/erts/emulator/test/alloc_SUITE_data/migration.c index b9a4de03b3..1d974225fc 100644 --- a/erts/emulator/test/alloc_SUITE_data/migration.c +++ b/erts/emulator/test/alloc_SUITE_data/migration.c @@ -223,6 +223,42 @@ static int rand_int(MigrationState* state, int low, int high) return low + (x % (high+1-low)); } +enum Operation +{ + ALLOCATE_OP, + FREE_OP, + REALLOC_OP, + CLEANUP_OP +}; + +static enum Operation rand_op(MigrationState* state) +{ + int r = rand_int(state, 1, 100); + switch (state->phase) { + case GROWING: + FATAL_ASSERT(state->nblocks < state->max_nblocks); + if (r > 10 || state->nblocks == 0) + return ALLOCATE_OP; + else if (r > 5) + return FREE_OP; + else + return REALLOC_OP; + + case SHRINKING: + FATAL_ASSERT(state->nblocks > 0); + if (r > 10 || state->nblocks == state->max_nblocks) + return FREE_OP; + else if (r > 5) + return ALLOCATE_OP; + else + return REALLOC_OP; + + case CLEANUP: + return CLEANUP_OP; + default: + FATAL_ASSERT(!"Invalid op phase"); + } +} static void do_cleanup(TestCaseState_t *tcs, MigrationState* state) { @@ -275,53 +311,75 @@ testcase_run(TestCaseState_t *tcs) state->goal_nblocks = rand_int(state, 1, state->max_nblocks); } - switch (state->phase) { - case GROWING: { + switch (rand_op(state)) { + case ALLOCATE_OP: { MyBlock* p; FATAL_ASSERT(!state->blockv[state->nblocks]); - p = ALLOC_TEST(rand_int(state, state->block_size/2, state->block_size)); + p = ALLOC_TEST(rand_int(state, state->block_size/2, state->block_size)); FATAL_ASSERT(p); add_block(p, state); - state->blockv[state->nblocks] = p; - if (++state->nblocks >= state->goal_nblocks) { - /*testcase_printf(tcs, "%d: Grown to %d blocks", tcs->thr_nr, state->nblocks);*/ - state->phase = SHRINKING; - state->goal_nblocks = rand_int(state, 0, state->goal_nblocks-1); - } - else - FATAL_ASSERT(!state->blockv[state->nblocks]); + state->blockv[state->nblocks++] = p; break; } - case SHRINKING: { + case FREE_OP: { int ix = rand_int(state, 0, state->nblocks-1); FATAL_ASSERT(state->blockv[ix]); remove_block(state->blockv[ix]); FREE_TEST(state->blockv[ix]); state->blockv[ix] = state->blockv[--state->nblocks]; state->blockv[state->nblocks] = NULL; - - if (state->nblocks <= state->goal_nblocks) { - /*testcase_printf(tcs, "%d: Shrunk to %d blocks", tcs->thr_nr, state->nblocks);*/ - if (++state->round >= MAX_ROUNDS) { - state->phase = CLEANUP; - } else { - state->phase = GROWING; - state->goal_nblocks = rand_int(state, state->goal_nblocks+1, state->max_nblocks); - } - } break; } + case REALLOC_OP: { + int ix = rand_int(state, 0, state->nblocks-1); + MyBlock* p; + FATAL_ASSERT(state->blockv[ix]); + remove_block(state->blockv[ix]); + p = REALLOC_TEST(state->blockv[ix], rand_int(state, state->block_size/2, state->block_size)); + FATAL_ASSERT(p); + add_block(p, state); + state->blockv[ix] = p; + break; + } + case CLEANUP_OP: + do_cleanup(tcs, state); + break; + default: + FATAL_ASSERT(!"Invalid operation"); + } + + switch (state->phase) { + case GROWING: { + if (state->nblocks >= state->goal_nblocks) { + /*testcase_printf(tcs, "%d: Grown to %d blocks", tcs->thr_nr, state->nblocks);*/ + state->phase = SHRINKING; + state->goal_nblocks = rand_int(state, 0, state->goal_nblocks-1); + } + else + FATAL_ASSERT(!state->blockv[state->nblocks]); + break; + } + case SHRINKING: { + if (state->nblocks <= state->goal_nblocks) { + /*testcase_printf(tcs, "%d: Shrunk to %d blocks", tcs->thr_nr, state->nblocks);*/ + if (++state->round >= MAX_ROUNDS) { + state->phase = CLEANUP; + } else { + state->phase = GROWING; + state->goal_nblocks = rand_int(state, state->goal_nblocks+1, state->max_nblocks); + } + } + break; + } case CLEANUP: - do_cleanup(tcs, state); - break; + case DONE: + break; default: FATAL_ASSERT(!"Invalid phase"); } - if (state->phase == DONE) { - } - else { + if (state->phase != DONE) { testcase_continue(tcs); } } diff --git a/erts/emulator/test/binary_SUITE.erl b/erts/emulator/test/binary_SUITE.erl index 61536bacd7..a3c3daac15 100644 --- a/erts/emulator/test/binary_SUITE.erl +++ b/erts/emulator/test/binary_SUITE.erl @@ -48,6 +48,7 @@ bad_list_to_binary/1, bad_binary_to_list/1, t_split_binary/1, bad_split/1, terms/1, terms_float/1, float_middle_endian/1, + b2t_used_big/1, external_size/1, t_iolist_size/1, t_hash/1, bad_size/1, @@ -57,7 +58,9 @@ otp_5484/1,otp_5933/1, ordering/1,unaligned_order/1,gc_test/1, bit_sized_binary_sizes/1, - otp_6817/1,deep/1,obsolete_funs/1,robustness/1,otp_8117/1, + otp_6817/1,deep/1, + term2bin_tuple_fallbacks/1, + robustness/1,otp_8117/1, otp_8180/1, trapping/1, large/1, error_after_yield/1, cmp_old_impl/1]). @@ -72,12 +75,14 @@ all() -> t_split_binary, bad_split, bad_list_to_binary, bad_binary_to_list, terms, terms_float, float_middle_endian, external_size, t_iolist_size, + b2t_used_big, bad_binary_to_term_2, safe_binary_to_term2, bad_binary_to_term, bad_terms, t_hash, bad_size, bad_term_to_binary, more_bad_terms, otp_5484, otp_5933, ordering, unaligned_order, gc_test, bit_sized_binary_sizes, otp_6817, otp_8117, deep, - obsolete_funs, robustness, otp_8180, trapping, large, + term2bin_tuple_fallbacks, + robustness, otp_8180, trapping, large, error_after_yield, cmp_old_impl]. groups() -> @@ -257,6 +262,7 @@ test_deep_bitstr(List) -> {Bin,bitstring_to_list(Bin)}. bad_list_to_binary(Config) when is_list(Config) -> + test_bad_bin(<<1:1>>), test_bad_bin(atom), test_bad_bin(42), test_bad_bin([1|2]), @@ -425,40 +431,77 @@ bad_term_to_binary(Config) when is_list(Config) -> terms(Config) when is_list(Config) -> TestFun = fun(Term) -> - try - S = io_lib:format("~p", [Term]), - io:put_chars(S) - catch - error:badarg -> - io:put_chars("bit sized binary") - end, + S = io_lib:format("~p", [Term]), + io:put_chars(S), Bin = term_to_binary(Term), case erlang:external_size(Bin) of Sz when is_integer(Sz), size(Bin) =< Sz -> ok end, - Bin1 = term_to_binary(Term, [{minor_version, 1}]), - case erlang:external_size(Bin1, [{minor_version, 1}]) of - Sz1 when is_integer(Sz1), size(Bin1) =< Sz1 -> - ok - end, + Bin1 = term_to_binary(Term, [{minor_version, 1}]), + case erlang:external_size(Bin1, [{minor_version, 1}]) of + Sz1 when is_integer(Sz1), size(Bin1) =< Sz1 -> + ok + end, Term = binary_to_term_stress(Bin), Term = binary_to_term_stress(Bin, [safe]), - Unaligned = make_unaligned_sub_binary(Bin), - Term = binary_to_term_stress(Unaligned), - Term = binary_to_term_stress(Unaligned, []), - Term = binary_to_term_stress(Bin, [safe]), + Bin_sz = byte_size(Bin), + {Term,Bin_sz} = binary_to_term_stress(Bin, [used]), + + BinE = <<Bin/binary, 1, 2, 3>>, + {Term,Bin_sz} = binary_to_term_stress(BinE, [used]), + + BinU = make_unaligned_sub_binary(Bin), + Term = binary_to_term_stress(BinU), + Term = binary_to_term_stress(BinU, []), + Term = binary_to_term_stress(BinU, [safe]), + {Term,Bin_sz} = binary_to_term_stress(BinU, [used]), + + BinUE = make_unaligned_sub_binary(BinE), + {Term,Bin_sz} = binary_to_term_stress(BinUE, [used]), + BinC = erlang:term_to_binary(Term, [compressed]), + BinC_sz = byte_size(BinC), + true = BinC_sz =< size(Bin), Term = binary_to_term_stress(BinC), - true = size(BinC) =< size(Bin), + {Term, BinC_sz} = binary_to_term_stress(BinC, [used]), + Bin = term_to_binary(Term, [{compressed,0}]), terms_compression_levels(Term, size(Bin), 1), - UnalignedC = make_unaligned_sub_binary(BinC), - Term = binary_to_term_stress(UnalignedC) + + BinUC = make_unaligned_sub_binary(BinC), + Term = binary_to_term_stress(BinUC), + {Term,BinC_sz} = binary_to_term_stress(BinUC, [used]), + + BinCE = <<BinC/binary, 1, 2, 3>>, + {Term,BinC_sz} = binary_to_term_stress(BinCE, [used]), + + BinUCE = make_unaligned_sub_binary(BinCE), + Term = binary_to_term_stress(BinUCE), + {Term,BinC_sz} = binary_to_term_stress(BinUCE, [used]) end, test_terms(TestFun), ok. +%% Test binary_to_term(_, [used]) returning a big Used integer. +b2t_used_big(_Config) -> + case erlang:system_info(wordsize) of + 8 -> + {skipped, "This is not a 32-bit machine"}; + 4 -> + %% Use a long utf8 atom for large external format but compact on heap. + BigAtom = binary_to_atom(<< <<16#F0908D88:32>> || _ <- lists:seq(1,255) >>, + utf8), + Atoms = (1 bsl 17) + (1 bsl 9), + BigAtomList = lists:duplicate(Atoms, BigAtom), + BigBin = term_to_binary(BigAtomList), + {BigAtomList, Used} = binary_to_term(BigBin, [used]), + 2 = erts_debug:size(Used), + Used = byte_size(BigBin), + Used = 1 + 1 + 4 + Atoms*(1+2+4*255) + 1, + ok + end. + terms_compression_levels(Term, UncompressedSz, Level) when Level < 10 -> BinC = erlang:term_to_binary(Term, [{compressed,Level}]), Term = binary_to_term_stress(BinC), @@ -1160,7 +1203,7 @@ very_big_num(0, Result) -> Result. make_port() -> - open_port({spawn, efile}, [eof]). + hd(erlang:ports()). make_pid() -> spawn_link(?MODULE, sleeper, []). @@ -1261,40 +1304,28 @@ deep_roundtrip(T) -> B = term_to_binary(T), T = binary_to_term(B). -obsolete_funs(Config) when is_list(Config) -> +term2bin_tuple_fallbacks(Config) when is_list(Config) -> erts_debug:set_internal_state(available_internal_state, true), - X = id({1,2,3}), - Y = id([a,b,c,d]), - Z = id({x,y,z}), - obsolete_fun(fun() -> ok end), - obsolete_fun(fun() -> X end), - obsolete_fun(fun(A) -> {A,X} end), - obsolete_fun(fun() -> {X,Y} end), - obsolete_fun(fun() -> {X,Y,Z} end), - - obsolete_fun(fun ?MODULE:all/1), + term2bin_tf(fun ?MODULE:all/1), + term2bin_tf(<<1:1>>), + term2bin_tf(<<90,80:7>>), erts_debug:set_internal_state(available_internal_state, false), ok. -obsolete_fun(Fun) -> - Tuple = case erlang:fun_info(Fun, type) of - {type,external} -> - {module,M} = erlang:fun_info(Fun, module), - {name,F} = erlang:fun_info(Fun, name), - {M,F}; - {type,local} -> - {module,M} = erlang:fun_info(Fun, module), - {index,I} = erlang:fun_info(Fun, index), - {uniq,U} = erlang:fun_info(Fun, uniq), - {env,E} = erlang:fun_info(Fun, env), - {'fun',M,I,U,list_to_tuple(E)} - end, - Tuple = no_fun_roundtrip(Fun). - -no_fun_roundtrip(Term) -> - binary_to_term_stress(erts_debug:get_internal_state({term_to_binary_no_funs,Term})). +term2bin_tf(Term) -> + Tuple = case Term of + Fun when is_function(Fun) -> + {type, external} = erlang:fun_info(Fun, type), + {module,M} = erlang:fun_info(Fun, module), + {name,F} = erlang:fun_info(Fun, name), + {M,F}; + BS when bit_size(BS) rem 8 =/= 0 -> + Bits = bit_size(BS) rem 8, + {<<BS/bitstring, 0:(8-Bits)>>, Bits} + end, + Tuple = binary_to_term_stress(erts_debug:get_internal_state({term_to_binary_tuple_fallbacks,Term})). %% Test non-standard encodings never generated by term_to_binary/1 %% but recognized by binary_to_term/1. @@ -1442,13 +1473,13 @@ error_after_yield(Type, M, F, AN, AFun, TrapFunc) -> apply(M, F, A), exit({unexpected_success, {M, F, A}}) catch - error:Type -> + error:Type:Stk -> erlang:trace(self(),false,[running,{tracer,Tracer}]), %% We threw the exception from the native %% function we trapped to, but we want %% the BIF that originally was called %% to appear in the stack trace. - [{M, F, A, _} | _] = erlang:get_stacktrace() + [{M, F, A, _} | _] = Stk end end), receive diff --git a/erts/emulator/test/call_trace_SUITE.erl b/erts/emulator/test/call_trace_SUITE.erl index 1251d644ae..d19f7f81ad 100644 --- a/erts/emulator/test/call_trace_SUITE.erl +++ b/erts/emulator/test/call_trace_SUITE.erl @@ -1116,8 +1116,8 @@ get_deep_4_loc(Arg) -> deep_4(Arg), ct:fail(should_not_return_to_here) catch - _:_ -> - [{?MODULE,deep_4,1,Loc0}|_] = erlang:get_stacktrace(), + _:_:Stk -> + [{?MODULE,deep_4,1,Loc0}|_] = Stk, Loc0 end. diff --git a/erts/emulator/test/code_SUITE.erl b/erts/emulator/test/code_SUITE.erl index 77321aa50f..661a2ee6c9 100644 --- a/erts/emulator/test/code_SUITE.erl +++ b/erts/emulator/test/code_SUITE.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 1999-2017. All Rights Reserved. +%% Copyright Ericsson AB 1999-2018. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -25,8 +25,10 @@ multi_proc_purge/1, t_check_old_code/1, external_fun/1,get_chunk/1,module_md5/1, constant_pools/1,constant_refc_binaries/1, + fake_literals/1, false_dependency/1,coverage/1,fun_confusion/1, - t_copy_literals/1, t_copy_literals_frags/1]). + t_copy_literals/1, t_copy_literals_frags/1, + erl_544/1]). -define(line_trace, 1). -include_lib("common_test/include/ct.hrl"). @@ -38,8 +40,10 @@ all() -> call_purged_fun_code_reload, call_purged_fun_code_there, multi_proc_purge, t_check_old_code, external_fun, get_chunk, module_md5, - constant_pools, constant_refc_binaries, false_dependency, - coverage, fun_confusion, t_copy_literals, t_copy_literals_frags]. + constant_pools, constant_refc_binaries, fake_literals, + false_dependency, + coverage, fun_confusion, t_copy_literals, t_copy_literals_frags, + erl_544]. init_per_suite(Config) -> erts_debug:set_internal_state(available_internal_state, true), @@ -554,6 +558,62 @@ wait_for_memory_deallocations() -> wait_for_memory_deallocations() end. +fake_literals(_Config) -> + Mod = fake__literals__module, + try + do_fake_literals(Mod) + after + _ = code:purge(Mod), + _ = code:delete(Mod), + _ = code:purge(Mod), + _ = code:delete(Mod) + end, + ok. + +do_fake_literals(Mod) -> + Tid = ets:new(test, []), + ExtTerms = get_external_terms(), + Term0 = {self(),make_ref(),Tid,fun() -> ok end,ExtTerms}, + Terms = [begin + make_literal_module(Mod, Term0), + Mod:term() + end || _ <- lists:seq(1, 10)], + verify_lit_terms(Terms, Term0), + true = ets:delete(Tid), + ok. + +make_literal_module(Mod, Term) -> + Exp = [{term,0}], + Attr = [], + Fs = [{function,term,0,2, + [{label,1}, + {line,[]}, + {func_info,{atom,Mod},{atom,term},0}, + {label,2}, + {move,{literal,Term},{x,0}}, + return]}], + Asm = {Mod,Exp,Attr,Fs,2}, + {ok,Mod,Beam} = compile:forms(Asm, [from_asm,binary,report]), + code:load_binary(Mod, atom_to_list(Mod), Beam). + +verify_lit_terms([H|T], Term) -> + case H =:= Term of + true -> + verify_lit_terms(T, Term); + false -> + error({bad_term,H}) + end; +verify_lit_terms([], _) -> + ok. + +get_external_terms() -> + {ok,Node} = test_server:start_node(?FUNCTION_NAME, slave, []), + Ref = rpc:call(Node, erlang, make_ref, []), + Ports = rpc:call(Node, erlang, ports, []), + Pid = rpc:call(Node, erlang, self, []), + _ = test_server:stop_node(Node), + {Ref,hd(Ports),Pid}. + %% OTP-7559: c_p->cp could contain garbage and create a false dependency %% to a module in a process. (Thanks to Richard Carlsson.) false_dependency(Config) when is_list(Config) -> @@ -860,6 +920,53 @@ reloader(Mod,Code,Time) -> reloader(Mod,Code,Time) end. +erl_544(Config) when is_list(Config) -> + case file:native_name_encoding() of + utf8 -> + {ok, CWD} = file:get_cwd(), + try + Mod = erl_544, + FileName = atom_to_list(Mod) ++ ".erl", + Priv = proplists:get_value(priv_dir, Config), + Data = proplists:get_value(data_dir, Config), + {ok, FileContent} = file:read_file(filename:join(Data, + FileName)), + Dir = filename:join(Priv, [16#2620,16#2620,16#2620]), + File = filename:join(Dir, FileName), + io:format("~ts~n", [File]), + ok = file:make_dir(Dir), + ok = file:set_cwd(Dir), + ok = file:write_file(File, [FileContent]), + {ok, Mod} = compile:file(File), + Res1 = (catch Mod:err()), + io:format("~p~n", [Res1]), + {'EXIT', {err, [{Mod, err, 0, Info1}|_]}} = Res1, + File = proplists:get_value(file, Info1), + Me = self(), + Go = make_ref(), + Tester = spawn_link(fun () -> + Mod:wait(Me, Go), + Mod:err() + end), + receive Go -> ok end, + Res2 = process_info(Tester, current_stacktrace), + io:format("~p~n", [Res2]), + {current_stacktrace, Stack} = Res2, + [{Mod, wait, 2, Info2}|_] = Stack, + File = proplists:get_value(file, Info2), + StackFun = fun(_, _, _) -> false end, + FormatFun = fun (Term, _) -> io_lib:format("~tp", [Term]) end, + Formated = + lib:format_stacktrace(1, Stack, StackFun, FormatFun), + true = is_list(Formated), + ok + after + ok = file:set_cwd(CWD) + end, + ok; + _Enc -> + {skipped, "Only run when native file name encoding is utf8"} + end. %% Utilities. diff --git a/erts/emulator/test/code_SUITE_data/erl_544.erl b/erts/emulator/test/code_SUITE_data/erl_544.erl new file mode 100644 index 0000000000..c93f3ef5bc --- /dev/null +++ b/erts/emulator/test/code_SUITE_data/erl_544.erl @@ -0,0 +1,35 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2018. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%% +%% %CopyrightEnd% +%% + +-module(erl_544). + +-export([err/0, wait/2]). + +err() -> + erlang:error(err). + +wait(Pid, Msg) -> + erlang:yield(), + Pid ! Msg, + receive + after infinity -> + ok + end, + err(). diff --git a/erts/emulator/test/ddll_SUITE.erl b/erts/emulator/test/ddll_SUITE.erl index 031b05790d..4998fc08be 100644 --- a/erts/emulator/test/ddll_SUITE.erl +++ b/erts/emulator/test/ddll_SUITE.erl @@ -775,7 +775,7 @@ errors(Config) when is_list(Config) -> {error, bad_driver_name} = erl_ddll:load_driver(Path, wrongname_drv), %% We assume that there is a statically linked driver named "ddll": - {error, linked_in_driver} = erl_ddll:unload_driver(efile), + {error, linked_in_driver} = erl_ddll:unload_driver(ram_file_drv), {error, not_loaded} = erl_ddll:unload_driver("__pucko_driver__"), case os:type() of diff --git a/erts/emulator/test/dirty_bif_SUITE.erl b/erts/emulator/test/dirty_bif_SUITE.erl index 981ec4d48d..46eb0cba58 100644 --- a/erts/emulator/test/dirty_bif_SUITE.erl +++ b/erts/emulator/test/dirty_bif_SUITE.erl @@ -108,90 +108,80 @@ dirty_bif_exception(Config) when is_list(Config) -> erts_debug:dirty_cpu(error, Error), ct:fail(expected_exception) catch - error:ErrorType -> - [{erts_debug,dirty_cpu,[error, Error],_}|_] - = erlang:get_stacktrace(), + error:ErrorType:Stk1 -> + [{erts_debug,dirty_cpu,[error, Error],_}|_] = Stk1, ok end, try apply(erts_debug,dirty_cpu,[error, Error]), ct:fail(expected_exception) catch - error:ErrorType -> - [{erts_debug,dirty_cpu,[error, Error],_}|_] - = erlang:get_stacktrace(), + error:ErrorType:Stk2 -> + [{erts_debug,dirty_cpu,[error, Error],_}|_] = Stk2, ok end, try erts_debug:dirty_io(error, Error), ct:fail(expected_exception) catch - error:ErrorType -> - [{erts_debug,dirty_io,[error, Error],_}|_] - = erlang:get_stacktrace(), + error:ErrorType:Stk3 -> + [{erts_debug,dirty_io,[error, Error],_}|_] = Stk3, ok end, try apply(erts_debug,dirty_io,[error, Error]), ct:fail(expected_exception) catch - error:ErrorType -> - [{erts_debug,dirty_io,[error, Error],_}|_] - = erlang:get_stacktrace(), + error:ErrorType:Stk4 -> + [{erts_debug,dirty_io,[error, Error],_}|_] = Stk4, ok end, try erts_debug:dirty(normal, error, Error), ct:fail(expected_exception) catch - error:ErrorType -> - [{erts_debug,dirty,[normal, error, Error],_}|_] - = erlang:get_stacktrace(), + error:ErrorType:Stk5 -> + [{erts_debug,dirty,[normal, error, Error],_}|_] = Stk5, ok end, try apply(erts_debug,dirty,[normal, error, Error]), ct:fail(expected_exception) catch - error:ErrorType -> - [{erts_debug,dirty,[normal, error, Error],_}|_] - = erlang:get_stacktrace(), + error:ErrorType:Stk6 -> + [{erts_debug,dirty,[normal, error, Error],_}|_] = Stk6, ok end, try erts_debug:dirty(dirty_cpu, error, Error), ct:fail(expected_exception) catch - error:ErrorType -> - [{erts_debug,dirty,[dirty_cpu, error, Error],_}|_] - = erlang:get_stacktrace(), + error:ErrorType:Stk7 -> + [{erts_debug,dirty,[dirty_cpu, error, Error],_}|_] = Stk7, ok end, try apply(erts_debug,dirty,[dirty_cpu, error, Error]), ct:fail(expected_exception) catch - error:ErrorType -> - [{erts_debug,dirty,[dirty_cpu, error, Error],_}|_] - = erlang:get_stacktrace(), + error:ErrorType:Stk8 -> + [{erts_debug,dirty,[dirty_cpu, error, Error],_}|_] = Stk8, ok end, try erts_debug:dirty(dirty_io, error, Error), ct:fail(expected_exception) catch - error:ErrorType -> - [{erts_debug,dirty,[dirty_io, error, Error],_}|_] - = erlang:get_stacktrace(), + error:ErrorType:Stk9 -> + [{erts_debug,dirty,[dirty_io, error, Error],_}|_] = Stk9, ok end, try apply(erts_debug,dirty,[dirty_io, error, Error]), ct:fail(expected_exception) catch - error:ErrorType -> - [{erts_debug,dirty,[dirty_io, error, Error],_}|_] - = erlang:get_stacktrace(), + error:ErrorType:Stk10 -> + [{erts_debug,dirty,[dirty_io, error, Error],_}|_] = Stk10, ok end end, @@ -204,25 +194,22 @@ dirty_bif_multischedule_exception(Config) when is_list(Config) -> try erts_debug:dirty_cpu(reschedule,1001) catch - error:badarg -> - [{erts_debug,dirty_cpu,[reschedule, 1001],_}|_] - = erlang:get_stacktrace(), + error:badarg:Stk1 -> + [{erts_debug,dirty_cpu,[reschedule, 1001],_}|_] = Stk1, ok end, try erts_debug:dirty_io(reschedule,1001) catch - error:badarg -> - [{erts_debug,dirty_io,[reschedule, 1001],_}|_] - = erlang:get_stacktrace(), + error:badarg:Stk2 -> + [{erts_debug,dirty_io,[reschedule, 1001],_}|_] = Stk2, ok end, try erts_debug:dirty(normal,reschedule,1001) catch - error:badarg -> - [{erts_debug,dirty,[normal,reschedule,1001],_}|_] - = erlang:get_stacktrace(), + error:badarg:Stk3 -> + [{erts_debug,dirty,[normal,reschedule,1001],_}|_] = Stk3, ok end. @@ -230,6 +217,11 @@ dirty_scheduler_exit(Config) when is_list(Config) -> {ok, Node} = start_node(Config, "+SDio 1"), [ok] = mcall(Node, [fun() -> + %% Perform a dry run to ensure that all required code + %% is loaded. Otherwise the test will fail since code + %% loading is done through dirty IO and it won't make + %% any progress during this test. + _DryRun = test_dirty_scheduler_exit(), Start = erlang:monotonic_time(millisecond), ok = test_dirty_scheduler_exit(), End = erlang:monotonic_time(millisecond), @@ -246,23 +238,22 @@ test_dse(0,Pids) -> timer:sleep(100), kill_dse(Pids,[]); test_dse(N,Pids) -> - Pid = spawn_link(fun () -> erts_debug:dirty_io(wait, 5000) end), + Pid = spawn_link(fun () -> erts_debug:dirty_io(wait, 1000) end), test_dse(N-1,[Pid|Pids]). kill_dse([],Killed) -> - wait_dse(Killed); + wait_dse(Killed, ok); kill_dse([Pid|Pids],AlreadyKilled) -> exit(Pid,kill), kill_dse(Pids,[Pid|AlreadyKilled]). -wait_dse([]) -> - ok; -wait_dse([Pid|Pids]) -> +wait_dse([], Result) -> + Result; +wait_dse([Pid|Pids], Result) -> receive - {'EXIT',Pid,Reason} -> - killed = Reason - end, - wait_dse(Pids). + {'EXIT', Pid, killed} -> wait_dse(Pids, Result); + {'EXIT', Pid, _Other} -> wait_dse(Pids, failed) + end. dirty_call_while_terminated(Config) when is_list(Config) -> Me = self(), diff --git a/erts/emulator/test/dirty_nif_SUITE.erl b/erts/emulator/test/dirty_nif_SUITE.erl index 13806fd5c4..93d0ac392c 100644 --- a/erts/emulator/test/dirty_nif_SUITE.erl +++ b/erts/emulator/test/dirty_nif_SUITE.erl @@ -109,9 +109,8 @@ dirty_nif_exception(Config) when is_list(Config) -> call_dirty_nif_exception(1), ct:fail(expected_badarg) catch - error:badarg -> - [{?MODULE,call_dirty_nif_exception,[1],_}|_] = - erlang:get_stacktrace(), + error:badarg:Stk1 -> + [{?MODULE,call_dirty_nif_exception,[1],_}|_] = Stk1, ok end, try @@ -121,9 +120,8 @@ dirty_nif_exception(Config) when is_list(Config) -> call_dirty_nif_exception(0), ct:fail(expected_badarg) catch - error:badarg -> - [{?MODULE,call_dirty_nif_exception,[0],_}|_] = - erlang:get_stacktrace(), + error:badarg:Stk2 -> + [{?MODULE,call_dirty_nif_exception,[0],_}|_] = Stk2, ok end, %% this checks that a dirty NIF can raise various terms as @@ -138,8 +136,8 @@ nif_raise_exceptions(NifFunc) -> erlang:apply(?MODULE,NifFunc,[Term]), ct:fail({expected,Term}) catch - error:Term -> - [{?MODULE,NifFunc,[Term],_}|_] = erlang:get_stacktrace(), + error:Term:Stk -> + [{?MODULE,NifFunc,[Term],_}|_] = Stk, ok end end, ok, ExcTerms). @@ -151,6 +149,11 @@ dirty_scheduler_exit(Config) when is_list(Config) -> [ok] = mcall(Node, [fun() -> ok = erlang:load_nif(NifLib, []), + %% Perform a dry run to ensure that all required code + %% is loaded. Otherwise the test will fail since code + %% loading is done through dirty IO and it won't make + %% any progress during this test. + _DryRun = test_dirty_scheduler_exit(), Start = erlang:monotonic_time(millisecond), ok = test_dirty_scheduler_exit(), End = erlang:monotonic_time(millisecond), @@ -171,19 +174,18 @@ test_dse(N,Pids) -> test_dse(N-1,[Pid|Pids]). kill_dse([],Killed) -> - wait_dse(Killed); + wait_dse(Killed, ok); kill_dse([Pid|Pids],AlreadyKilled) -> exit(Pid,kill), kill_dse(Pids,[Pid|AlreadyKilled]). -wait_dse([]) -> - ok; -wait_dse([Pid|Pids]) -> +wait_dse([], Result) -> + Result; +wait_dse([Pid|Pids], Result) -> receive - {'EXIT',Pid,Reason} -> - killed = Reason - end, - wait_dse(Pids). + {'EXIT', Pid, killed} -> wait_dse(Pids, Result); + {'EXIT', Pid, _Other} -> wait_dse(Pids, failed) + end. dirty_call_while_terminated(Config) when is_list(Config) -> Me = self(), @@ -287,9 +289,9 @@ access_dirty_heap(Dirty, RGL, N, R) -> %% dirty NIF where the main lock is needed for that access do not get %% blocked. Each test passes its pid to dirty_sleeper, which sends a %% 'ready' message when it's running on a dirty scheduler and just before -%% it starts a 6 second sleep. When it receives the message, it verifies +%% it starts a 2 second sleep. When it receives the message, it verifies %% that access to the dirty process is as it expects. After the dirty -%% process finishes its 6 second sleep but before it returns from the dirty +%% process finishes its 2 second sleep but before it returns from the dirty %% scheduler, it sends a 'done' message. If the tester already received %% that message, the test fails because it means attempting to access the %% dirty process waited for that process to return to a regular scheduler, @@ -353,7 +355,7 @@ dirty_process_trace(Config) when is_list(Config) -> error(missing_trace_return_message) end after - 6500 -> + 2500 -> error(missing_done_message) end, ok @@ -380,7 +382,7 @@ code_purge(Config) when is_list(Config) -> Start = erlang:monotonic_time(), {Pid1, Mon1} = spawn_monitor(fun () -> dirty_code_test:func(fun () -> - %% Sleep for 6 seconds + %% Sleep for 2 seconds %% in dirty nif... dirty_sleeper() end) @@ -388,7 +390,7 @@ code_purge(Config) when is_list(Config) -> {module, dirty_code_test} = erlang:load_module(dirty_code_test, Bin), {Pid2, Mon2} = spawn_monitor(fun () -> dirty_code_test:func(fun () -> - %% Sleep for 6 seconds + %% Sleep for 2 seconds %% in dirty nif... dirty_sleeper() end) @@ -490,7 +492,7 @@ test_dirty_process_access(Start, Test, Finish) -> ok end after - 3000 -> + 1000 -> error(timeout) end, ok = Finish(NifPid). diff --git a/erts/emulator/test/dirty_nif_SUITE_data/dirty_nif_SUITE.c b/erts/emulator/test/dirty_nif_SUITE_data/dirty_nif_SUITE.c index 0321b9898f..2a8b999307 100644 --- a/erts/emulator/test/dirty_nif_SUITE_data/dirty_nif_SUITE.c +++ b/erts/emulator/test/dirty_nif_SUITE_data/dirty_nif_SUITE.c @@ -217,9 +217,9 @@ dirty_sleeper(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) } #ifdef __WIN32__ - Sleep(6000); + Sleep(2000); #else - sleep(6); + sleep(2); #endif if (argc == 1) { diff --git a/erts/emulator/test/distribution_SUITE.erl b/erts/emulator/test/distribution_SUITE.erl index 2d0ae9c83e..e40d346e10 100644 --- a/erts/emulator/test/distribution_SUITE.erl +++ b/erts/emulator/test/distribution_SUITE.erl @@ -35,13 +35,18 @@ -include_lib("common_test/include/ct.hrl"). +%-define(Line, erlang:display({line,?LINE}),). +-define(Line,). + -export([all/0, suite/0, groups/0, ping/1, bulk_send_small/1, + group_leader/1, + optimistic_dflags/1, bulk_send_big/1, bulk_send_bigbig/1, local_send_small/1, local_send_big/1, local_send_legal/1, link_to_busy/1, exit_to_busy/1, lost_exit/1, link_to_dead/1, link_to_dead_new_node/1, - applied_monitor_node/1, ref_port_roundtrip/1, nil_roundtrip/1, + ref_port_roundtrip/1, nil_roundtrip/1, trap_bif_1/1, trap_bif_2/1, trap_bif_3/1, stop_dist/1, dist_auto_connect_never/1, dist_auto_connect_once/1, @@ -61,6 +66,8 @@ %% Internal exports. -export([sender/3, receiver2/2, dummy_waiter/0, dead_process/0, + group_leader_1/1, + optimistic_dflags_echo/0, optimistic_dflags_sender/1, roundtrip/1, bounce/1, do_dist_auto_connect/1, inet_rpc_server/1, dist_parallel_sender/3, dist_parallel_receiver/0, dist_evil_parallel_receiver/0]). @@ -74,8 +81,10 @@ suite() -> all() -> [ping, {group, bulk_send}, {group, local_send}, + group_leader, + optimistic_dflags, link_to_busy, exit_to_busy, lost_exit, link_to_dead, - link_to_dead_new_node, applied_monitor_node, + link_to_dead_new_node, ref_port_roundtrip, nil_roundtrip, stop_dist, {group, trap_bif}, {group, dist_auto_connect}, dist_parallel_send, atom_roundtrip, unicode_atom_roundtrip, @@ -124,6 +133,96 @@ ping(Config) when is_list(Config) -> ok. +%% Test erlang:group_leader(_, ExternalPid), i.e. DOP_GROUP_LEADER +group_leader(Config) when is_list(Config) -> + ?Line Sock = start_relay_node(group_leader_1, []), + ?Line Sock2 = start_relay_node(group_leader_2, []), + try + ?Line Node2 = inet_rpc_nodename(Sock2), + ?Line {ok, ok} = do_inet_rpc(Sock, ?MODULE, group_leader_1, [Node2]) + after + ?Line stop_relay_node(Sock), + ?Line stop_relay_node(Sock2) + end, + ok. + +group_leader_1(Node2) -> + ?Line ExtPid = spawn(Node2, fun F() -> + receive {From, group_leader} -> + From ! {self(), group_leader, group_leader()} + end, + F() + end), + ?Line GL1 = self(), + ?Line group_leader(GL1, ExtPid), + ?Line ExtPid ! {self(), group_leader}, + ?Line {ExtPid, group_leader, GL1} = receive_one(), + + %% Kill connection and repeat test when group_leader/2 triggers auto-connect + ?Line net_kernel:monitor_nodes(true), + ?Line net_kernel:disconnect(Node2), + ?Line {nodedown, Node2} = receive_one(), + ?Line GL2 = spawn(fun() -> dummy end), + ?Line group_leader(GL2, ExtPid), + ?Line {nodeup, Node2} = receive_one(), + ?Line ExtPid ! {self(), group_leader}, + ?Line {ExtPid, group_leader, GL2} = receive_one(), + ok. + +%% Test optimistic distribution flags toward pending connections (DFLAG_DIST_HOPEFULLY) +optimistic_dflags(Config) when is_list(Config) -> + ?Line Sender = start_relay_node(optimistic_dflags_sender, []), + ?Line Echo = start_relay_node(optimistic_dflags_echo, []), + try + ?Line {ok, ok} = do_inet_rpc(Echo, ?MODULE, optimistic_dflags_echo, []), + + ?Line EchoNode = inet_rpc_nodename(Echo), + ?Line {ok, ok} = do_inet_rpc(Sender, ?MODULE, optimistic_dflags_sender, [EchoNode]) + after + ?Line stop_relay_node(Sender), + ?Line stop_relay_node(Echo) + end, + ok. + +optimistic_dflags_echo() -> + P = spawn(fun F() -> + receive {From, Term} -> + From ! {self(), Term} + end, + F() + end), + register(optimistic_dflags_echo, P), + optimistic_dflags_echo ! {self(), hello}, + {P, hello} = receive_one(), + ok. + +optimistic_dflags_sender(EchoNode) -> + ?Line net_kernel:monitor_nodes(true), + + optimistic_dflags_do(EchoNode, <<1:1>>), + optimistic_dflags_do(EchoNode, fun lists:map/2), + ok. + +optimistic_dflags_do(EchoNode, Term) -> + ?Line {optimistic_dflags_echo, EchoNode} ! {self(), Term}, + ?Line {nodeup, EchoNode} = receive_one(), + ?Line {EchoPid, Term} = receive_one(), + %% repeat with pid destination + ?Line net_kernel:disconnect(EchoNode), + ?Line {nodedown, EchoNode} = receive_one(), + ?Line EchoPid ! {self(), Term}, + ?Line {nodeup, EchoNode} = receive_one(), + ?Line {EchoPid, Term} = receive_one(), + + ?Line net_kernel:disconnect(EchoNode), + ?Line {nodedown, EchoNode} = receive_one(), + ok. + + +receive_one() -> + receive M -> M after 1000 -> timeout end. + + bulk_send_small(Config) when is_list(Config) -> bulk_send(64, 32). @@ -639,31 +738,11 @@ link_to_dead_new_node(Config) when is_list(Config) -> end, ok. -%% Test that monitor_node/2 works when applied. -applied_monitor_node(Config) when is_list(Config) -> - NonExisting = list_to_atom("__non_existing__@" ++ hostname()), - - %% Tail-recursive call to apply (since the node is non-existing, - %% there will be a trap). - - true = tail_apply(erlang, monitor_node, [NonExisting, true]), - [{nodedown, NonExisting}] = test_server:messages_get(), - - %% Ordinary call (with trap). - - true = apply(erlang, monitor_node, [NonExisting, true]), - [{nodedown, NonExisting}] = test_server:messages_get(), - - ok. - -tail_apply(M, F, A) -> - apply(M, F, A). - %% Test that sending a port or reference to another node and back again %% doesn't correct them in any way. ref_port_roundtrip(Config) when is_list(Config) -> process_flag(trap_exit, true), - Port = open_port({spawn, efile}, []), + Port = make_port(), Ref = make_ref(), {ok, Node} = start_node(ref_port_roundtrip), net_adm:ping(Node), @@ -684,6 +763,9 @@ ref_port_roundtrip(Config) when is_list(Config) -> end, ok. +make_port() -> + hd(erlang:ports()). + roundtrip(Term) -> exit(Term). @@ -1158,8 +1240,6 @@ contended_atom_cache_entry_test(Config, Type) -> spawn_link( SNode, fun () -> - erts_debug:set_internal_state(available_internal_state, - true), Master = self(), CIX = get_cix(), TestAtoms = case Type of @@ -1244,7 +1324,7 @@ get_cix(CIX) when is_integer(CIX), CIX < 0 -> get_cix(CIX) when is_integer(CIX) -> get_cix(CIX, unwanted_cixs(), - erts_debug:get_internal_state(max_atom_out_cache_index)). + get_internal_state(max_atom_out_cache_index)). get_cix(CIX, Unwanted, MaxCIX) when CIX > MaxCIX -> get_cix(0, Unwanted, MaxCIX); @@ -1256,8 +1336,8 @@ get_cix(CIX, Unwanted, MaxCIX) -> unwanted_cixs() -> lists:map(fun (Node) -> - erts_debug:get_internal_state({atom_out_cache_index, - Node}) + get_internal_state({atom_out_cache_index, + Node}) end, nodes()). @@ -1266,7 +1346,7 @@ get_conflicting_atoms(_CIX, 0) -> []; get_conflicting_atoms(CIX, N) -> Atom = list_to_atom("atom" ++ integer_to_list(erlang:unique_integer([positive]))), - case erts_debug:get_internal_state({atom_out_cache_index, Atom}) of + case get_internal_state({atom_out_cache_index, Atom}) of CIX -> [Atom|get_conflicting_atoms(CIX, N-1)]; _ -> @@ -1277,7 +1357,7 @@ get_conflicting_unicode_atoms(_CIX, 0) -> []; get_conflicting_unicode_atoms(CIX, N) -> Atom = string_to_atom([16#1f608] ++ "atom" ++ integer_to_list(erlang:unique_integer([positive]))), - case erts_debug:get_internal_state({atom_out_cache_index, Atom}) of + case get_internal_state({atom_out_cache_index, Atom}) of CIX -> [Atom|get_conflicting_unicode_atoms(CIX, N-1)]; _ -> @@ -1365,81 +1445,59 @@ bad_dist_structure(Config) when is_list(Config) -> start_monitor(Offender,P), P ! one, send_bad_structure(Offender, P,{?DOP_MONITOR_P_EXIT,'replace',P,normal},2), - pong = rpc:call(Victim, net_adm, ping, [Offender]), + start_monitor(Offender,P), send_bad_structure(Offender, P,{?DOP_MONITOR_P_EXIT,'replace',P,normal,normal},2), - pong = rpc:call(Victim, net_adm, ping, [Offender]), + start_link(Offender,P), send_bad_structure(Offender, P,{?DOP_LINK},0), - pong = rpc:call(Victim, net_adm, ping, [Offender]), + start_link(Offender,P), send_bad_structure(Offender, P,{?DOP_UNLINK,'replace'},2), - pong = rpc:call(Victim, net_adm, ping, [Offender]), + start_link(Offender,P), send_bad_structure(Offender, P,{?DOP_UNLINK,'replace',make_ref()},2), - pong = rpc:call(Victim, net_adm, ping, [Offender]), + start_link(Offender,P), send_bad_structure(Offender, P,{?DOP_UNLINK,make_ref(),P},0), - pong = rpc:call(Victim, net_adm, ping, [Offender]), + start_link(Offender,P), send_bad_structure(Offender, P,{?DOP_UNLINK,normal,normal},0), - pong = rpc:call(Victim, net_adm, ping, [Offender]), + start_monitor(Offender,P), send_bad_structure(Offender, P,{?DOP_MONITOR_P,'replace',P},2), - pong = rpc:call(Victim, net_adm, ping, [Offender]), + start_monitor(Offender,P), send_bad_structure(Offender, P,{?DOP_MONITOR_P,'replace',P,normal},2), - pong = rpc:call(Victim, net_adm, ping, [Offender]), + start_monitor(Offender,P), send_bad_structure(Offender, P,{?DOP_DEMONITOR_P,'replace',P},2), - pong = rpc:call(Victim, net_adm, ping, [Offender]), + start_monitor(Offender,P), send_bad_structure(Offender, P,{?DOP_DEMONITOR_P,'replace',P,normal},2), - pong = rpc:call(Victim, net_adm, ping, [Offender]), + send_bad_structure(Offender, P,{?DOP_EXIT,'replace',P},2), - pong = rpc:call(Victim, net_adm, ping, [Offender]), send_bad_structure(Offender, P,{?DOP_EXIT,make_ref(),normal,normal},0), - pong = rpc:call(Victim, net_adm, ping, [Offender]), send_bad_structure(Offender, P,{?DOP_EXIT_TT,'replace',token,P},2), - pong = rpc:call(Victim, net_adm, ping, [Offender]), send_bad_structure(Offender, P,{?DOP_EXIT_TT,make_ref(),token,normal,normal},0), - pong = rpc:call(Victim, net_adm, ping, [Offender]), send_bad_structure(Offender, P,{?DOP_EXIT2,'replace',P},2), - pong = rpc:call(Victim, net_adm, ping, [Offender]), send_bad_structure(Offender, P,{?DOP_EXIT2,make_ref(),normal,normal},0), - pong = rpc:call(Victim, net_adm, ping, [Offender]), send_bad_structure(Offender, P,{?DOP_EXIT2_TT,'replace',token,P},2), - pong = rpc:call(Victim, net_adm, ping, [Offender]), send_bad_structure(Offender, P,{?DOP_EXIT2_TT,make_ref(),token,normal,normal},0), - pong = rpc:call(Victim, net_adm, ping, [Offender]), send_bad_structure(Offender, P,{?DOP_GROUP_LEADER,'replace'},2), - pong = rpc:call(Victim, net_adm, ping, [Offender]), send_bad_structure(Offender, P,{?DOP_GROUP_LEADER,'replace','atomic'},2), - pong = rpc:call(Victim, net_adm, ping, [Offender]), send_bad_structure(Offender, P,{?DOP_GROUP_LEADER,'replace',P},0), - pong = rpc:call(Victim, net_adm, ping, [Offender]), send_bad_structure(Offender, P,{?DOP_REG_SEND_TT,'replace','',name},2,{message}), - pong = rpc:call(Victim, net_adm, ping, [Offender]), send_bad_structure(Offender, P,{?DOP_REG_SEND_TT,'replace','',name,token},0,{message}), - pong = rpc:call(Victim, net_adm, ping, [Offender]), send_bad_structure(Offender, P,{?DOP_REG_SEND,'replace',''},2,{message}), - pong = rpc:call(Victim, net_adm, ping, [Offender]), send_bad_structure(Offender, P,{?DOP_REG_SEND,'replace','',P},0,{message}), - pong = rpc:call(Victim, net_adm, ping, [Offender]), send_bad_structure(Offender, P,{?DOP_REG_SEND,'replace','',name},0,{message}), - pong = rpc:call(Victim, net_adm, ping, [Offender]), send_bad_structure(Offender, P,{?DOP_REG_SEND,'replace','',name,{token}},2,{message}), - pong = rpc:call(Victim, net_adm, ping, [Offender]), send_bad_structure(Offender, P,{?DOP_SEND_TT,'',P},0,{message}), - pong = rpc:call(Victim, net_adm, ping, [Offender]), send_bad_structure(Offender, P,{?DOP_SEND_TT,'',name,token},0,{message}), - pong = rpc:call(Victim, net_adm, ping, [Offender]), send_bad_structure(Offender, P,{?DOP_SEND,''},0,{message}), - pong = rpc:call(Victim, net_adm, ping, [Offender]), send_bad_structure(Offender, P,{?DOP_SEND,'',name},0,{message}), - pong = rpc:call(Victim, net_adm, ping, [Offender]), send_bad_structure(Offender, P,{?DOP_SEND,'',P,{token}},0,{message}), - pong = rpc:call(Victim, net_adm, ping, [Offender]), P ! two, P ! check_msgs, receive @@ -1685,13 +1743,16 @@ bad_dist_ext_size(Config) when is_list(Config) -> start_node_monitors([Offender,Victim]), Parent = self(), - P = spawn_link(Victim, + P = spawn_opt(Victim, fun () -> Parent ! {self(), started}, receive check_msgs -> ok end, %% DID CRASH HERE bad_dist_ext_check_msgs([one]), Parent ! {self(), messages_checked} - end), + end, + [link, + %% on_heap to force total_heap_size to inspect msg queue + {message_queue_data, on_heap}]), receive {P, started} -> ok end, P ! one, @@ -1714,6 +1775,7 @@ bad_dist_ext_size(Config) when is_list(Config) -> verify_still_up(Offender, Victim), + %% Let process_info(P, total_heap_size) find bad msg and disconnect rpc:call(Victim, erlang, process_info, [P, total_heap_size]), verify_down(Offender, connection_closed, Victim, killed), @@ -1795,10 +1857,11 @@ send_bad_structure(Offender,Victim,Bad,WhereToPutSelf) -> send_bad_structure(Offender,Victim,Bad,WhereToPutSelf,PayLoad) -> Parent = self(), Done = make_ref(), - spawn(Offender, + spawn_link(Offender, fun () -> Node = node(Victim), pong = net_adm:ping(Node), + erlang:monitor_node(Node, true), DCtrl = dctrl(Node), Bad1 = case WhereToPutSelf of 0 -> @@ -1812,7 +1875,16 @@ send_bad_structure(Offender,Victim,Bad,WhereToPutSelf,PayLoad) -> [] -> []; _Other -> [dmsg_ext(PayLoad)] end, + + receive {nodedown, Node} -> exit("premature nodedown") + after 10 -> ok + end, + dctrl_send(DCtrl, DData), + + receive {nodedown, Node} -> ok + after 5000 -> exit("missing nodedown") + end, Parent ! {DData,Done} end), receive @@ -1893,11 +1965,26 @@ send_bad_dhdr(BadNode, ToNode) when is_atom(BadNode), is_atom(ToNode) -> receive Done -> ok end. dctrl(Node) when is_atom(Node) -> - case catch erts_debug:get_internal_state(available_internal_state) of - true -> true; - _ -> erts_debug:set_internal_state(available_internal_state, true) - end, - erts_debug:get_internal_state({dist_ctrl, Node}). + get_internal_state({dist_ctrl, Node}). + +get_internal_state(Op) -> + try erts_debug:get_internal_state(Op) of + R -> R + catch + error:undef -> + erts_debug:set_internal_state(available_internal_state, true), + erts_debug:get_internal_state(Op) + end. + +set_internal_state(Op, Val) -> + try erts_debug:set_internal_state(Op, Val) of + R -> R + catch + error:undef -> + erts_debug:set_internal_state(available_internal_state, true), + erts_debug:set_internal_state(Op, Val) + end. + dmsg_hdr() -> [131, % Version Magic @@ -2040,11 +2127,9 @@ freeze_node(Node, MS) -> Freezer = self(), spawn_link(Node, fun () -> - erts_debug:set_internal_state(available_internal_state, - true), dctrl_dop_send(Freezer, DoingIt), receive after Own -> ok end, - erts_debug:set_internal_state(block, MS+Own) + set_internal_state(block, MS+Own) end), receive DoingIt -> ok end, receive after Own -> ok end. @@ -2248,8 +2333,7 @@ forever(Fun) -> forever(Fun). abort(Why) -> - erts_debug:set_internal_state(available_internal_state, true), - erts_debug:set_internal_state(abort, Why). + set_internal_state(abort, Why). start_busy_dist_port_tracer() -> diff --git a/erts/emulator/test/driver_SUITE.erl b/erts/emulator/test/driver_SUITE.erl index 475e03087a..294c42780d 100644 --- a/erts/emulator/test/driver_SUITE.erl +++ b/erts/emulator/test/driver_SUITE.erl @@ -80,6 +80,7 @@ async_blast/1, thr_msg_blast/1, consume_timeslice/1, + env/1, z_test/1]). -export([bin_prefix/2]). @@ -166,6 +167,7 @@ all() -> %% Keep a_test first and z_test last... async_blast, thr_msg_blast, consume_timeslice, + env, z_test]. groups() -> @@ -1768,7 +1770,10 @@ smp_select0(Config) -> smp_select_loop(_, 0) -> ok; smp_select_loop(Port, N) -> - "ok" = erlang:port_control(Port, ?CHKIO_SMP_SELECT, []), + case erlang:port_control(Port, ?CHKIO_SMP_SELECT, []) of + "yield" -> erlang:yield(); + "ok" -> ok + end, receive stop -> io:format("Worker ~p stopped with ~p laps left\n",[self(), N]), @@ -2360,6 +2365,51 @@ count_proc_sched(Ps, PNs) -> PNs end. +%% +%% Tests whether erl_drv_putenv reflects in os:getenv and vice versa. +%% +env(Config) when is_list(Config) -> + ok = load_driver(proplists:get_value(data_dir, Config), env_drv), + Port = open_port({spawn_driver, env_drv}, []), + true = is_port(Port), + + Keys = ["env_drv_a_key", "env_drv_b_key", "env_drv_c_key"], + Values = ["a_value", "b_value", "c_value"], + + [env_put_test(Port, Key, Value) || Key <- Keys, Value <- Values], + [env_get_test(Port, Key, Value) || Key <- Keys, Value <- Values], + [env_oversize_test(Port, Key) || Key <- Keys], + [env_notfound_test(Port, Key) || Key <- Keys], + + true = port_close(Port), + erl_ddll:unload_driver(env_drv), + ok. + +env_control(Port, Command, Key, Value) -> + KeyBin = list_to_binary(Key), + ValueBin = list_to_binary(Value), + Header = <<(byte_size(KeyBin)), (byte_size(ValueBin))>>, + Payload = <<KeyBin/binary, ValueBin/binary>>, + port_control(Port, Command, <<Header/binary, Payload/binary>>). + +env_put_test(Port, Key, Value) -> + os:unsetenv(Key), + [0] = env_control(Port, 0, Key, Value), + Value = os:getenv(Key). + +env_get_test(Port, Key, ExpectedValue) -> + true = os:putenv(Key, ExpectedValue), + [0] = env_control(Port, 1, Key, ExpectedValue). + +env_oversize_test(Port, Key) -> + os:putenv(Key, [$A || _ <- lists:seq(1, 1024)]), + [127] = env_control(Port, 1, Key, ""). + +env_notfound_test(Port, Key) -> + true = os:unsetenv(Key), + [255] = env_control(Port, 1, Key, ""). + + a_test(Config) when is_list(Config) -> rpc(Config, fun check_io_debug/0). @@ -2614,8 +2664,8 @@ rpc(Config, Fun) -> Result = try Fun() of Res -> Res - catch E:R -> - {'EXIT',E,R,erlang:get_stacktrace()} + catch E:R:Stk -> + {'EXIT',E,R,Stk} end, Self ! {Ref, Result} end), diff --git a/erts/emulator/test/driver_SUITE_data/Makefile.src b/erts/emulator/test/driver_SUITE_data/Makefile.src index 1fedd72200..bcabaa689d 100644 --- a/erts/emulator/test/driver_SUITE_data/Makefile.src +++ b/erts/emulator/test/driver_SUITE_data/Makefile.src @@ -16,7 +16,8 @@ MISC_DRVS = outputv_drv@dll@ \ thr_free_drv@dll@ \ async_blast_drv@dll@ \ thr_msg_blast_drv@dll@ \ - consume_timeslice_drv@dll@ + consume_timeslice_drv@dll@ \ + env_drv@dll@ SYS_INFO_DRVS = sys_info_base_drv@dll@ \ sys_info_prev_drv@dll@ \ diff --git a/erts/emulator/test/driver_SUITE_data/chkio_drv.c b/erts/emulator/test/driver_SUITE_data/chkio_drv.c index d548c4b1dc..ee8f28e8b1 100644 --- a/erts/emulator/test/driver_SUITE_data/chkio_drv.c +++ b/erts/emulator/test/driver_SUITE_data/chkio_drv.c @@ -961,7 +961,6 @@ chkio_drv_control(ErlDrvData drv_data, break; } case CHKIO_SMP_SELECT: { - int rounds = 1; /*rand(); */ ChkioSmpSelect* pip = (ChkioSmpSelect*) cddp->test_data; if (pip == NULL) { erl_drv_mutex_lock(smp_pipes_mtx); @@ -978,7 +977,8 @@ chkio_drv_control(ErlDrvData drv_data, } erl_drv_mutex_unlock(smp_pipes_mtx); } - while (rounds--) { + res_str = NULL; + { int op = rand_r(&pip->rand_state); switch (pip->state) { case Closed: { @@ -988,7 +988,6 @@ chkio_drv_control(ErlDrvData drv_data, fcntl(fds[0], F_SETFL, flags|O_NONBLOCK) < 0) { driver_failure_posix(cddp->port, errno); - rounds = 0; break; } TRACEF(("%T: Created pipe [%d->%d]\n", cddp->id, fds[1], fds[0])); @@ -1075,7 +1074,9 @@ chkio_drv_control(ErlDrvData drv_data, pip->next_write++; } break; - case Waiting: + case Waiting: + res_str = "yield"; + res_len = -1; break; default: fprintf(stderr, "Strange state %d\n", pip->state); @@ -1091,9 +1092,11 @@ chkio_drv_control(ErlDrvData drv_data, else { cddp->test_data = pip; } - } - res_str = "ok"; - res_len = -1; + } + if (!res_str) { + res_str = "ok"; + res_len = -1; + } break; } case CHKIO_DRV_USE: diff --git a/erts/emulator/test/driver_SUITE_data/env_drv.c b/erts/emulator/test/driver_SUITE_data/env_drv.c new file mode 100644 index 0000000000..0e910eeb84 --- /dev/null +++ b/erts/emulator/test/driver_SUITE_data/env_drv.c @@ -0,0 +1,108 @@ +/* + * %CopyrightBegin% + * + * Copyright Ericsson AB 2017. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * %CopyrightEnd% + */ + +/* Tests whether erl_drv_putenv/erl_drv_getenv work correctly and reflect + * changes to os:putenv/getenv. */ + +#include <string.h> +#include <stdio.h> + +#include "erl_driver.h" + +static ErlDrvSSizeT env_drv_ctl(ErlDrvData drv_data, unsigned int cmd, + char* buf, ErlDrvSizeT len, char** rbuf, ErlDrvSizeT rsize); + +static ErlDrvEntry env_drv_entry = { + NULL /* init */, + NULL /* start */, + NULL /* stop */, + NULL /* output */, + NULL /* ready_input */, + NULL /* ready_output */, + "env_drv", + NULL /* finish */, + NULL /* handle */, + env_drv_ctl, + NULL /* timeout */, + NULL /* outputv*/, + NULL /* ready_async */, + NULL /* flush */, + NULL /* call*/, + NULL /* event */, + ERL_DRV_EXTENDED_MARKER, + ERL_DRV_EXTENDED_MAJOR_VERSION, + ERL_DRV_EXTENDED_MINOR_VERSION, + ERL_DRV_FLAG_USE_PORT_LOCKING, + NULL /* handle2 */, + NULL /* handle_monitor */ +}; + +DRIVER_INIT(env_drv) { + return &env_drv_entry; +} + +static int test_putenv(ErlDrvData drv_data, char *buf, ErlDrvSizeT len) { + char key[256], value[256]; + int key_len, value_len; + + key_len = buf[0]; + value_len = buf[1]; + + sprintf(key, "%.*s", key_len, &buf[2]); + sprintf(value, "%.*s", value_len, &buf[2 + key_len]); + + return erl_drv_putenv(key, value); +} + +static int test_getenv(ErlDrvData drv_data, char *buf, ErlDrvSizeT len) { + char expected_value[256], stored_value[256], key[256]; + int expected_value_len, key_len; + size_t stored_value_len; + int res; + + key_len = buf[0]; + sprintf(key, "%.*s", key_len, &buf[2]); + + expected_value_len = buf[1]; + sprintf(expected_value, "%.*s", expected_value_len, &buf[2 + key_len]); + + stored_value_len = sizeof(stored_value); + res = erl_drv_getenv(key, stored_value, &stored_value_len); + + if(res == 0) { + return strcmp(stored_value, expected_value) != 0; + } else if(res == 1) { + return 127; + } + + return 255; +} + +static ErlDrvSSizeT env_drv_ctl(ErlDrvData drv_data, unsigned int cmd, + char* buf, ErlDrvSizeT len, char** rbuf, ErlDrvSizeT rsize) { + + if(cmd == 0) { + (**rbuf) = (char)test_putenv(drv_data, buf, len); + } else { + (**rbuf) = (char)test_getenv(drv_data, buf, len); + } + + return 1; +} diff --git a/erts/emulator/test/efile_SUITE.erl b/erts/emulator/test/efile_SUITE.erl index 08d5597d78..821381bf0d 100644 --- a/erts/emulator/test/efile_SUITE.erl +++ b/erts/emulator/test/efile_SUITE.erl @@ -19,12 +19,9 @@ -module(efile_SUITE). -export([all/0, suite/0]). --export([async_dist/1, - iter_max_files/1, - proc_zero_sized_files/1 - ]). +-export([iter_max_files/1, proc_zero_sized_files/1]). --export([do_iter_max_files/2, do_async_dist/1]). +-export([do_iter_max_files/2]). -include_lib("common_test/include/ct.hrl"). -include_lib("stdlib/include/assert.hrl"). @@ -32,84 +29,7 @@ suite() -> [{ct_hooks,[ts_install_cth]}]. all() -> - [iter_max_files, async_dist, proc_zero_sized_files]. - -do_async_dist(Dir) -> - X = 100, - AT = erlang:system_info(thread_pool_size), - Keys = file_keys(Dir,AT*X,[],[]), - Tab = ets:new(x,[ordered_set]), - [ ets:insert(Tab,{N,0}) || N <- lists:seq(0,AT-1) ], - [ ets:update_counter(Tab,(N rem AT),1) || N <- Keys ], - Res = [ V || {_,V} <- ets:tab2list(Tab) ], - ets:delete(Tab), - {Res, sdev(Res)/X}. - -sdev(List) -> - Len = length(List), - Mean = lists:sum(List)/Len, - math:sqrt(lists:sum([ (X - Mean) * (X - Mean) || X <- List ]) / Len). - -file_keys(_,0,FdList,FnList) -> - [ file:close(FD) || FD <- FdList ], - [ file:delete(FN) || FN <- FnList ], - []; -file_keys(Dir,Num,FdList,FnList) -> - Name = "dummy"++integer_to_list(Num), - FN = filename:join([Dir,Name]), - case file:open(FN,[write,raw]) of - {ok,FD} -> - {file_descriptor,prim_file,{Port,_}} = FD, - <<X:32/integer-big>> = - iolist_to_binary(erlang:port_control(Port,$K,[])), - [X | file_keys(Dir,Num-1,[FD|FdList],[FN|FnList])]; - {error,_} -> - % Try freeing up FD's if there are any - case FdList of - [] -> - exit({cannot_open_file,FN}); - _ -> - [ file:close(FD) || FD <- FdList ], - [ file:delete(F) || F <- FnList ], - file_keys(Dir,Num,[],[]) - end - end. - -%% Check that the distribution of files over async threads is fair -async_dist(Config) when is_list(Config) -> - DataDir = proplists:get_value(data_dir,Config), - Dir = filename:dirname(code:which(?MODULE)), - AsyncSizes = [7,10,100,255,256,64,63,65], - Max = 0.5, - - lists:foreach(fun(Size) -> - {ok,Node} = - test_server:start_node - (test_iter_max_files,slave, - [{args, - "+A "++integer_to_list(Size)++ - " -pa " ++ Dir}]), - {Distr,SD} = rpc:call(Node,?MODULE,do_async_dist, - [DataDir]), - test_server:stop_node(Node), - if - SD > Max -> - io:format("Bad async queue distribution for " - "~p async threads:~n" - " Standard deviation is ~p~n" - " Key distribution:~n ~lp~n", - [Size,SD,Distr]), - exit({bad_async_dist,Size,SD,Distr}); - true -> - io:format("OK async queue distribution for " - "~p async threads:~n" - " Standard deviation is ~p~n" - " Key distribution:~n ~lp~n", - [Size,SD,Distr]), - ok - end - end, AsyncSizes), - ok. + [iter_max_files, proc_zero_sized_files]. %% %% Open as many files as possible. Do this several times and check @@ -117,17 +37,23 @@ async_dist(Config) when is_list(Config) -> %% iter_max_files(Config) when is_list(Config) -> + case os:type() of + {win32, _} -> {skip, "Windows lacks a hard limit on file handles"}; + _ -> iter_max_files_1(Config) + end. + +iter_max_files_1(Config) -> DataDir = proplists:get_value(data_dir,Config), TestFile = filename:join(DataDir, "existing_file"), N = 10, - %% Run on a different node in order to set the max ports + %% Run on a different node in order to make the test more stable. Dir = filename:dirname(code:which(?MODULE)), {ok,Node} = test_server:start_node(test_iter_max_files,slave, - [{args,"+Q 1524 -pa " ++ Dir}]), + [{args,"-pa " ++ Dir}]), L = rpc:call(Node,?MODULE,do_iter_max_files,[N, TestFile]), test_server:stop_node(Node), io:format("Number of files opened in each test:~n~w\n", [L]), - all_equal(L), + verify_max_files(L), Head = hd(L), if Head >= 2 -> ok; true -> ct:fail(too_few_files) @@ -139,12 +65,15 @@ do_iter_max_files(N, Name) when N > 0 -> do_iter_max_files(_, _) -> []. -all_equal([E, E| T]) -> - all_equal([E| T]); -all_equal([_]) -> - ok; -all_equal([]) -> - ok. +%% The attempts shouldn't vary too much; we used to require that they were all +%% exactly equal, but after we reimplemented the file driver as a NIF we +%% noticed that the only reason it was stable on Darwin was because the port +%% limit was hit before ulimit. +verify_max_files(Attempts) -> + N = length(Attempts), + Mean = lists:sum(Attempts) / N, + Variance = lists:sum([(X - Mean) * (X - Mean) || X <- Attempts]) / N, + true = math:sqrt(Variance) =< 1 + (Mean / 1000). max_files(Name) -> Fds = open_files(Name), diff --git a/erts/emulator/test/erl_link_SUITE.erl b/erts/emulator/test/erl_link_SUITE.erl index d8c5b663e3..a66ca7a57d 100644 --- a/erts/emulator/test/erl_link_SUITE.erl +++ b/erts/emulator/test/erl_link_SUITE.erl @@ -200,13 +200,16 @@ monitor_nodes(Config) when is_list(Config) -> monitor_node(A, true), check_monitor_node(self(), A, 1), check_monitor_node(self(), B, 3), + ok = receive {nodedown, C} -> ok after 1000 -> timeout end, + %%OTP-21: monitor_node(_,false) does not trigger auto-connect anymore + %% and therefore no nodedown if it fails. + %%ok = receive {nodedown, C} -> ok after 1000 -> timeout end, + ok = receive {nodedown, C} -> ok after 1000 -> timeout end, + ok = receive {nodedown, D} -> ok after 1000 -> timeout end, + ok = receive {nodedown, D} -> ok after 1000 -> timeout end, check_monitor_node(self(), C, 0), check_monitor_node(self(), D, 0), - receive {nodedown, C} -> ok end, - receive {nodedown, C} -> ok end, - receive {nodedown, C} -> ok end, - receive {nodedown, D} -> ok end, - receive {nodedown, D} -> ok end, + stop_node(A), receive {nodedown, A} -> ok end, check_monitor_node(self(), A, 0), diff --git a/erts/emulator/test/exception_SUITE.erl b/erts/emulator/test/exception_SUITE.erl index be9b63d534..da0292f385 100644 --- a/erts/emulator/test/exception_SUITE.erl +++ b/erts/emulator/test/exception_SUITE.erl @@ -395,12 +395,12 @@ raise(Conf) when is_list(Conf) -> try try foo({'div',{1,0}}) catch - error:badarith -> + error:badarith:A0 -> put(raise, A0 = erlang:get_stacktrace()), erlang:raise(error, badarith, A0) end catch - error:badarith -> + error:badarith:A1 -> A1 = erlang:get_stacktrace(), A1 = get(raise) end, diff --git a/erts/emulator/test/guard_SUITE.erl b/erts/emulator/test/guard_SUITE.erl index 1a93a9f5c2..f2c1595392 100644 --- a/erts/emulator/test/guard_SUITE.erl +++ b/erts/emulator/test/guard_SUITE.erl @@ -500,7 +500,7 @@ all_types() -> {atom, xxxx}, {ref, make_ref()}, {pid, self()}, - {port, open_port({spawn, efile}, [])}, + {port, make_port()}, {function, fun(_) -> "" end}, {function, fun erlang:abs/1}, {binary, list_to_binary([])}, @@ -551,4 +551,7 @@ type_test(bitstring, X) when is_bitstring(X) -> type_test(function, X) when is_function(X) -> function. +make_port() -> + hd(erlang:ports()). + id(I) -> I. diff --git a/erts/emulator/test/iovec_SUITE.erl b/erts/emulator/test/iovec_SUITE.erl index 49dc64b0d2..963b7e2501 100644 --- a/erts/emulator/test/iovec_SUITE.erl +++ b/erts/emulator/test/iovec_SUITE.erl @@ -25,7 +25,8 @@ -export([integer_lists/1, binary_lists/1, empty_lists/1, empty_binary_lists/1, mixed_lists/1, improper_lists/1, illegal_lists/1, cons_bomb/1, sub_binary_lists/1, iolist_to_iovec_idempotence/1, - iolist_to_iovec_correctness/1]). + iolist_to_iovec_correctness/1, unaligned_sub_binaries/1, + direct_binary_arg/1]). -include_lib("common_test/include/ct.hrl"). @@ -36,7 +37,8 @@ suite() -> all() -> [integer_lists, binary_lists, empty_lists, empty_binary_lists, mixed_lists, sub_binary_lists, illegal_lists, improper_lists, cons_bomb, - iolist_to_iovec_idempotence, iolist_to_iovec_correctness]. + iolist_to_iovec_idempotence, iolist_to_iovec_correctness, + unaligned_sub_binaries, direct_binary_arg]. init_per_suite(Config) -> Config. @@ -78,7 +80,7 @@ illegal_lists(Config) when is_list(Config) -> BitStrs = gen_variations(["gurka", <<1:1>>, "gaffel"]), BadInts = gen_variations(["gurka", 890, "gaffel"]), Atoms = gen_variations([gurka, "gaffel"]), - BadTails = [["test" | 0], ["gurka", gaffel]], + BadTails = [["test" | 0], ["gurka" | gaffel], ["gaffel" | <<1:1>>]], Variations = BitStrs ++ BadInts ++ Atoms ++ BadTails, @@ -98,14 +100,7 @@ cons_bomb(Config) when is_list(Config) -> BinBase = gen_variations([<<I:8>> || I <- lists:seq(1, 255)]), MixBase = gen_variations([<<12, 45, 78>>, lists:seq(1, 255)]), - Rounds = - case system_mem_size() of - Mem when Mem >= (16 bsl 30) -> 32; - Mem when Mem >= (3 bsl 30) -> 28; - _ -> 20 - end, - - Variations = gen_variations([IntBase, BinBase, MixBase], Rounds), + Variations = gen_variations([IntBase, BinBase, MixBase], 16), equivalence_test(fun erlang:iolist_to_iovec/1, Variations). iolist_to_iovec_idempotence(Config) when is_list(Config) -> @@ -130,6 +125,21 @@ iolist_to_iovec_correctness(Config) when is_list(Config) -> true = is_iolist_equal(Optimized, Variations), ok. +unaligned_sub_binaries(Config) when is_list(Config) -> + UnalignedBins = [gen_unaligned_binary(I) || I <- lists:seq(32, 4 bsl 10, 512)], + UnalignedVariations = gen_variations(UnalignedBins), + + Optimized = erlang:iolist_to_iovec(UnalignedVariations), + + true = is_iolist_equal(Optimized, UnalignedVariations), + ok. + +direct_binary_arg(Config) when is_list(Config) -> + {'EXIT',{badarg, _}} = (catch erlang:iolist_to_iovec(<<1:1>>)), + [<<1>>] = erlang:iolist_to_iovec(<<1>>), + [] = erlang:iolist_to_iovec(<<>>), + ok. + illegality_test(Fun, Variations) -> [{'EXIT',{badarg, _}} = (catch Fun(Variation)) || Variation <- Variations], ok. @@ -145,11 +155,18 @@ equivalence_test(Fun, [Head | _] = Variations) -> is_iolist_equal(A, B) -> iolist_to_binary(A) =:= iolist_to_binary(B). +gen_unaligned_binary(Size) -> + Bin0 = << <<I>> || I <- lists:seq(1, Size) >>, + <<0:3,Bin:Size/binary,31:5>> = id(<<0:3,Bin0/binary,31:5>>), + Bin. + +id(I) -> I. + %% Generates a bunch of lists whose contents will be equal to Base repeated a %% few times. The lists only differ by their structure, so their reduction to %% a simpler format should yield the same result. gen_variations(Base) -> - gen_variations(Base, 16). + gen_variations(Base, 12). gen_variations(Base, N) -> [gen_flat_list(Base, N), gen_nested_list(Base, N), @@ -169,8 +186,3 @@ gen_nasty_list_1([Head | Base], Result) when is_list(Head) -> gen_nasty_list_1(Base, [[Result], [gen_nasty_list_1(Head, [])]]); gen_nasty_list_1([Head | Base], Result) -> gen_nasty_list_1(Base, [[Result], [Head]]). - -system_mem_size() -> - application:ensure_all_started(os_mon), - {Tot,_Used,_} = memsup:get_memory_data(), - Tot. diff --git a/erts/emulator/test/lcnt_SUITE.erl b/erts/emulator/test/lcnt_SUITE.erl index 504b9b54cf..4e52c2813c 100644 --- a/erts/emulator/test/lcnt_SUITE.erl +++ b/erts/emulator/test/lcnt_SUITE.erl @@ -28,14 +28,16 @@ init_per_testcase/2, end_per_testcase/2]). -export( - [toggle_lock_counting/1, error_on_invalid_category/1, preserve_locks/1]). + [toggle_lock_counting/1, error_on_invalid_category/1, preserve_locks/1, + registered_processes/1, registered_db_tables/1]). suite() -> [{ct_hooks,[ts_install_cth]}, {timetrap, {seconds, 10}}]. all() -> - [toggle_lock_counting, error_on_invalid_category, preserve_locks]. + [toggle_lock_counting, error_on_invalid_category, preserve_locks, + registered_processes, registered_db_tables]. init_per_suite(Config) -> case erlang:system_info(lock_counting) of @@ -154,3 +156,25 @@ preserve_locks(Config) when is_list(Config) -> error_on_invalid_category(Config) when is_list(Config) -> {error, badarg, q_invalid} = erts_debug:lcnt_control(mask, [q_invalid]), ok. + +registered_processes(Config) when is_list(Config) -> + %% There ought to be at least one registered process (init/code_server) + erts_debug:lcnt_control(mask, [process]), + [_, {locks, ProcLocks}] = erts_debug:lcnt_collect(), + true = lists:any( + fun + ({proc_main, RegName, _, _}) when is_atom(RegName) -> true; + (_Lock) -> false + end, ProcLocks), + ok. + +registered_db_tables(Config) when is_list(Config) -> + %% There ought to be at least one registered table (code) + erts_debug:lcnt_control(mask, [db]), + [_, {locks, DbLocks}] = erts_debug:lcnt_collect(), + true = lists:any( + fun + ({db_tab, RegName, _, _}) when is_atom(RegName) -> true; + (_Lock) -> false + end, DbLocks), + ok. diff --git a/erts/emulator/test/map_SUITE.erl b/erts/emulator/test/map_SUITE.erl index 02f3c89318..c9e971af8a 100644 --- a/erts/emulator/test/map_SUITE.erl +++ b/erts/emulator/test/map_SUITE.erl @@ -52,7 +52,7 @@ t_bif_map_values/1, t_bif_map_to_list/1, t_bif_map_from_list/1, - t_bif_erts_internal_maps_to_list/1, + t_bif_map_next/1, %% erlang t_erlang_hash/1, @@ -119,7 +119,7 @@ all() -> [t_build_and_match_literals, t_build_and_match_literals_large, t_bif_map_update, t_bif_map_values, t_bif_map_to_list, t_bif_map_from_list, - t_bif_erts_internal_maps_to_list, + t_bif_map_next, %% erlang t_erlang_hash, t_map_encode_decode, @@ -2364,41 +2364,71 @@ t_bif_map_from_list(Config) when is_list(Config) -> {'EXIT', {badarg,_}} = (catch maps:from_list(id(42))), ok. -t_bif_erts_internal_maps_to_list(Config) when is_list(Config) -> - %% small maps - [] = erts_internal:maps_to_list(#{},-1), - [] = erts_internal:maps_to_list(#{},-2), - [] = erts_internal:maps_to_list(#{},10), - [{a,1},{b,2}] = lists:sort(erts_internal:maps_to_list(#{a=>1,b=>2}, 2)), - [{a,1},{b,2}] = lists:sort(erts_internal:maps_to_list(#{a=>1,b=>2}, -1)), - [{_,_}] = erts_internal:maps_to_list(#{a=>1,b=>2}, 1), - [{a,1},{b,2},{c,3}] = lists:sort(erts_internal:maps_to_list(#{c=>3,a=>1,b=>2},-2)), - [{a,1},{b,2},{c,3}] = lists:sort(erts_internal:maps_to_list(#{c=>3,a=>1,b=>2},3)), - [{a,1},{b,2},{c,3}] = lists:sort(erts_internal:maps_to_list(#{c=>3,a=>1,b=>2},5)), - [{_,_},{_,_}] = erts_internal:maps_to_list(#{c=>3,a=>1,b=>2},2), - [{_,_}] = erts_internal:maps_to_list(#{c=>3,a=>1,b=>2},1), - [] = erts_internal:maps_to_list(#{c=>3,a=>1,b=>2},0), - - %% big maps - M = maps:from_list([{I,ok}||I <- lists:seq(1,500)]), - [] = erts_internal:maps_to_list(M,0), - [{_,_}] = erts_internal:maps_to_list(M,1), - [{_,_},{_,_}] = erts_internal:maps_to_list(M,2), - Ls1 = erts_internal:maps_to_list(M,10), - 10 = length(Ls1), - Ls2 = erts_internal:maps_to_list(M,20), - 20 = length(Ls2), - Ls3 = erts_internal:maps_to_list(M,120), - 120 = length(Ls3), - Ls4 = erts_internal:maps_to_list(M,-1), - 500 = length(Ls4), +t_bif_map_next(Config) when is_list(Config) -> - %% error cases - {'EXIT', {{badmap,[{a,b},b]},_}} = (catch erts_internal:maps_to_list(id([{a,b},b]),id(1))), - {'EXIT', {badarg,_}} = (catch erts_internal:maps_to_list(id(#{}),id(a))), - {'EXIT', {badarg,_}} = (catch erts_internal:maps_to_list(id(#{1=>2}),id(<<>>))), + erts_debug:set_internal_state(available_internal_state, true), + + try + + none = maps:next(maps:iterator(id(#{}))), + + verify_iterator(#{}), + verify_iterator(#{a => 1, b => 2, c => 3}), + + %% Use fatmap in order to test iterating in very deep maps + FM = fatmap(43), + verify_iterator(FM), + + {'EXIT', {{badmap,[{a,b},b]},_}} = (catch maps:iterator(id([{a,b},b]))), + {'EXIT', {badarg,_}} = (catch maps:next(id(a))), + {'EXIT', {badarg,_}} = (catch maps:next(id([a|FM]))), + {'EXIT', {badarg,_}} = (catch maps:next(id([1|#{}]))), + {'EXIT', {badarg,_}} = (catch maps:next(id([-1|#{}]))), + {'EXIT', {badarg,_}} = (catch maps:next(id([-1|FM]))), + {'EXIT', {badarg,_}} = (catch maps:next(id([16#FFFFFFFFFFFFFFFF|FM]))), + {'EXIT', {badarg,_}} = (catch maps:next(id([-16#FFFFFFFFFFFFFFFF|FM]))), + + %% This us a whitebox test that the error code works correctly. + %% It uses a path for a tree of depth 4 and tries to do next on + %% each of those paths. + (fun F(0) -> ok; + F(N) -> + try maps:next([N|FM]) of + none -> + F(N-1); + {_K,_V,_I} -> + F(N-1) + catch error:badarg -> + F(N-1) + end + end)(16#FFFF), + + ok + after + erts_debug:set_internal_state(available_internal_state, false) + end. + +verify_iterator(Map) -> + KVs = t_fold(fun(K, V, A) -> [{K, V} | A] end, [], Map), + + %% Verify that KVs created by iterating Map is of + %% correct size and contains all elements + true = length(KVs) == maps:size(Map), + [maps:get(K, Map) || {K, _} <- KVs], ok. + +t_fold(Fun, Init, Map) -> + t_fold_1(Fun, Init, maps:iterator(Map)). + +t_fold_1(Fun, Acc, Iter) -> + case maps:next(Iter) of + {K, V, NextIter} -> + t_fold_1(Fun, Fun(K,V,Acc), NextIter); + none -> + Acc + end. + t_bif_build_and_check(Config) when is_list(Config) -> ok = check_build_and_remove(750,[fun(K) -> [K,K] end, fun(K) -> [float(K),K] end, diff --git a/erts/emulator/test/nif_SUITE.erl b/erts/emulator/test/nif_SUITE.erl index bec2291867..b6e15ababb 100644 --- a/erts/emulator/test/nif_SUITE.erl +++ b/erts/emulator/test/nif_SUITE.erl @@ -2257,9 +2257,8 @@ nif_schedule(Config) when is_list(Config) -> {B,A} = call_nif_schedule(A, B), ok = try call_nif_schedule(1, 2) catch - error:badarg -> - [{?MODULE,call_nif_schedule,[1,2],_}|_] = - erlang:get_stacktrace(), + error:badarg:Stk -> + [{?MODULE,call_nif_schedule,[1,2],_}|_] = Stk, ok end, ok. @@ -2429,8 +2428,8 @@ nif_raise_exceptions(NifFunc) -> erlang:apply(?MODULE,NifFunc,[Term]), ct:fail({expected,Term}) catch - error:Term -> - [{?MODULE,NifFunc,[Term],_}|_] = erlang:get_stacktrace(), + error:Term:Stk -> + [{?MODULE,NifFunc,[Term],_}|_] = Stk, ok end end, ok, ExcTerms). @@ -3037,18 +3036,37 @@ nif_ioq(Config) -> {enqbraw,a}, {enqbraw,a, 5}, {peek, a}, + {peek_head, a}, {deq, a, 42}, %% Test enqv {enqv, a, 2, 100}, + {peek_head, a}, {deq, a, all}, %% This skips all elements but one in the iolist {enqv, a, 5, iolist_size(nif_ioq_payload(5)) - 1}, + {peek_head, a}, {peek, a}, - %% Test to enqueue a bunch of refc binaries + %% Ensure that enqueued refc binaries are intact after a roundtrip. + %% + %% This test and the ones immediately following it does not go through + %% erlang:iolist_to_iovec/1 {enqv, a, [nif_ioq_payload(refcbin) || _ <- lists:seq(1,20)], 0}, + {peek, a}, + + %% ... heap binaries + {enqv, a, [nif_ioq_payload(heapbin) || _ <- lists:seq(1,20)], 0}, + {peek, a}, + + %% ... plain sub-binaries + {enqv, a, [nif_ioq_payload(subbin) || _ <- lists:seq(1,20)], 0}, + {peek, a}, + + %% ... unaligned binaries + {enqv, a, [nif_ioq_payload(unaligned_bin) || _ <- lists:seq(1,20)], 0}, + {peek, a}, %% Enq stuff to destroy with data in queue {enqv, a, 2, 100}, @@ -3074,9 +3092,13 @@ nif_ioq(Config) -> Q = ioq_nif(create), + false = ioq_nif(peek_head, Q), + {'EXIT', {badarg, _}} = (catch ioq_nif(deq, Q, 1)), {'EXIT', {badarg, _}} = (catch ioq_nif(enqv, Q, 1, 1234)), + false = ioq_nif(peek_head, Q), + {'EXIT', {badarg, _}} = (catch ioq_nif(enqv, Q, [atom_in_list], 0)), {'EXIT', {badarg, _}} = (catch ioq_nif(enqv, Q, [make_ref()], 0)), {'EXIT', {badarg, _}} = (catch ioq_nif(enqv, Q, [256], 0)), @@ -3085,6 +3107,8 @@ nif_ioq(Config) -> {'EXIT', {badarg, _}} = (catch ioq_nif(enqv, Q, [1 bsl 64], 0)), {'EXIT', {badarg, _}} = (catch ioq_nif(enqv, Q, [{tuple}], 0)), + false = ioq_nif(peek_head, Q), + {'EXIT', {badarg, _}} = (catch ioq_nif(inspect, [atom_in_list], use_stack)), {'EXIT', {badarg, _}} = (catch ioq_nif(inspect, [make_ref()], no_stack)), {'EXIT', {badarg, _}} = (catch ioq_nif(inspect, [256], use_stack)), @@ -3145,6 +3169,20 @@ nif_ioq_run([{peek, Name} = H|T], State) -> true = iolist_to_binary(B) == iolist_to_binary(Data), nif_ioq_run(T, State); +nif_ioq_run([{peek_head, Name} = H|T], State) -> + #{ q := IOQ, b := B } = maps:get(Name, State), + RefData = iolist_to_binary(B), + + ct:log("~p", [H]), + + {true, QueueHead} = ioq_nif(peek_head, IOQ), + true = byte_size(QueueHead) > 0, + + {RefHead, _Tail} = split_binary(RefData, byte_size(QueueHead)), + + true = QueueHead =:= RefHead, + + nif_ioq_run(T, State); nif_ioq_run([{deq, Name, all}|T], State) -> #{ q := IOQ, b := B } = maps:get(Name, State), Size = ioq_nif(size, IOQ), @@ -3184,13 +3222,16 @@ nif_ioq_payload(N) when is_integer(N) -> Tail = if N > 3 -> nif_ioq_payload(N-3); true -> [] end, Head = element(1, lists:split(N,[nif_ioq_payload(subbin), nif_ioq_payload(heapbin), - nif_ioq_payload(refcbin) | Tail])), + nif_ioq_payload(refcbin), + nif_ioq_payload(unaligned_bin) | Tail])), erlang:iolist_to_iovec(Head); nif_ioq_payload(subbin) -> Bin = nif_ioq_payload(refcbin), Sz = size(Bin) - 1, <<_:8,SubBin:Sz/binary,_/bits>> = Bin, SubBin; +nif_ioq_payload(unaligned_bin) -> + make_unaligned_binary(<< <<I>> || I <- lists:seq(1, 255) >>); nif_ioq_payload(heapbin) -> <<"a literal heap binary">>; nif_ioq_payload(refcbin) -> @@ -3198,6 +3239,13 @@ nif_ioq_payload(refcbin) -> nif_ioq_payload(Else) -> Else. +make_unaligned_binary(Bin0) -> + Size = byte_size(Bin0), + <<0:3,Bin:Size/binary,31:5>> = id(<<0:3,Bin0/binary,31:5>>), + Bin. + +id(I) -> I. + %% The NIFs: lib_version() -> undefined. call_history() -> ?nif_stub. diff --git a/erts/emulator/test/nif_SUITE_data/nif_SUITE.c b/erts/emulator/test/nif_SUITE_data/nif_SUITE.c index 79560a38aa..fa9ae1015c 100644 --- a/erts/emulator/test/nif_SUITE_data/nif_SUITE.c +++ b/erts/emulator/test/nif_SUITE_data/nif_SUITE.c @@ -3403,6 +3403,15 @@ static ERL_NIF_TERM ioq(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) return enif_make_badarg(env); else return enif_make_atom(env, "true"); + } else if (enif_is_identical(argv[0], enif_make_atom(env, "peek_head"))) { + ERL_NIF_TERM head_term; + + if(enif_ioq_peek_head(env, ioq->q, NULL, &head_term)) { + return enif_make_tuple2(env, + enif_make_atom(env, "true"), head_term); + } + + return enif_make_atom(env, "false"); } else if (enif_is_identical(argv[0], enif_make_atom(env, "peek"))) { int iovlen, num, i, off = 0; SysIOVec *iov = enif_ioq_peek(ioq->q, &iovlen); diff --git a/erts/emulator/test/node_container_SUITE.erl b/erts/emulator/test/node_container_SUITE.erl index be90f929df..7df001fec5 100644 --- a/erts/emulator/test/node_container_SUITE.erl +++ b/erts/emulator/test/node_container_SUITE.erl @@ -966,6 +966,8 @@ check_refc(ThisNodeName,ThisCreation,Table,EntryList) when is_list(EntryList) -> {case Referrer of {system,delayed_delete_timer} -> true; + {system,thread_progress_delete_timer} -> + true; _ -> DDT end, diff --git a/erts/emulator/test/num_bif_SUITE.erl b/erts/emulator/test/num_bif_SUITE.erl index 17555d63c6..592542405f 100644 --- a/erts/emulator/test/num_bif_SUITE.erl +++ b/erts/emulator/test/num_bif_SUITE.erl @@ -118,6 +118,7 @@ t_float(Config) when is_list(Config) -> %% Tests float_to_list/1, float_to_list/2, float_to_binary/1, float_to_binary/2 t_float_to_string(Config) when is_list(Config) -> + rand_seed(), test_fts("0.00000000000000000000e+00", 0.0), test_fts("2.50000000000000000000e+01", 25.0), test_fts("2.50000000000000000000e+00", 2.5), @@ -167,8 +168,8 @@ t_float_to_string(Config) when is_list(Config) -> test_fts("1.12300",1.123, [{decimals, 5}]), test_fts("1.123",1.123, [{decimals, 5}, compact]), test_fts("1.1234",1.1234,[{decimals, 6}, compact]), - test_fts("1.01",1.005, [{decimals, 2}]), - test_fts("-1.01",-1.005,[{decimals, 2}]), + test_fts("1.00",1.005, [{decimals, 2}]), %% 1.005 is really 1.0049999999... + test_fts("-1.00",-1.005,[{decimals, 2}]), test_fts("0.999",0.999, [{decimals, 3}]), test_fts("-0.999",-0.999,[{decimals, 3}]), test_fts("1.0",0.999, [{decimals, 2}, compact]), @@ -184,6 +185,9 @@ t_float_to_string(Config) when is_list(Config) -> test_fts("123000000000000000000.0",1.23e20, [{decimals, 10}, compact]), test_fts("1.2300000000e+20",1.23e20, [{scientific, 10}, compact]), test_fts("1.23000000000000000000e+20",1.23e20, []), + + fts_rand_float_decimals(1000), + ok. test_fts(Expect, Float) -> @@ -197,6 +201,49 @@ test_fts(Expect, Float, Args) -> BinExpect = float_to_binary(Float,Args). +rand_float_reasonable() -> + F = rand_float(), + case abs(F) > 1.0e238 of + true -> rand_float_reasonable(); + false -> F + end. + +fts_rand_float_decimals(0) -> ok; +fts_rand_float_decimals(N) -> + [begin + F0 = rand_float_reasonable(), + L0 = float_to_list(F0, [{decimals, D}]), + L1 = case D of + 0 -> L0 ++ ".0"; + _ -> L0 + end, + F1 = list_to_float(L1), + Diff = abs(F0-F1), + MaxDiff = max_diff_decimals(F0, D), + ok = case Diff =< MaxDiff of + true -> ok; + false -> + io:format("F0 = ~w ~w\n", [F0, <<F0/float>>]), + io:format("L1 = ~s\n", [L1]), + io:format("F1 = ~w ~w\n", [F1, <<F1/float>>]), + io:format("Diff = ~w, MaxDiff = ~w\n", [Diff, MaxDiff]), + error + end + end + || D <- lists:seq(0,15)], + + fts_rand_float_decimals(N-1). + +max_diff_decimals(F, D) -> + IntBits = floor(math:log2(abs(F))) + 1, + FracBits = (52 - IntBits), + Log10_2 = 0.3010299956639812, % math:log10(2) + MaxDec = floor(FracBits * Log10_2), + + Resolution = math:pow(2, IntBits - 53), + + (math:pow(10, -min(D,MaxDec)) / 2) + Resolution. + %% Tests list_to_float/1. t_string_to_float_safe(Config) when is_list(Config) -> @@ -331,18 +378,26 @@ t_trunc_and_friends(_Config) -> -18446744073709551616 = trunc_and_friends(-float(1 bsl 64)), %% Random. + rand_seed(), t_trunc_and_friends_rand(100), ok. +rand_seed() -> + rand:seed(exrop), + io:format("\n*** rand:export_seed() = ~w\n\n", [rand:export_seed()]), + ok. + +rand_float() -> + F0 = rand:uniform() * math:pow(10, 50*rand:normal()), + case rand:uniform() of + U when U < 0.5 -> -F0; + _ -> F0 + end. + t_trunc_and_friends_rand(0) -> ok; t_trunc_and_friends_rand(N) -> - F0 = rand:uniform() * math:pow(10, 50*rand:normal()), - F = case rand:uniform() of - U when U < 0.5 -> -F0; - _ -> F0 - end, - _ = trunc_and_friends(F), + _ = trunc_and_friends(rand_float()), t_trunc_and_friends_rand(N-1). trunc_and_friends(F) -> @@ -491,7 +546,7 @@ t_string_to_integer(Config) when is_list(Config) -> list_to_binary(Value),Base)), {'EXIT', {badarg, _}} = (catch erlang:list_to_integer(Value,Base)) - end,[{" 1",1},{" 1",37},{"2",2},{"C",11}, + end,[{" 1",1},{" 1",37},{"2",2},{"B",11},{"b",11},{":", 16}, {"1111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111z",16}, {"1z111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111",16}, {"111z11111111",16}]), diff --git a/erts/emulator/test/process_SUITE.erl b/erts/emulator/test/process_SUITE.erl index a9f20f9928..a8bcfac84d 100644 --- a/erts/emulator/test/process_SUITE.erl +++ b/erts/emulator/test/process_SUITE.erl @@ -2532,8 +2532,13 @@ system_task_on_suspended(Config) when is_list(Config) -> end. gc_request_when_gc_disabled(Config) when is_list(Config) -> - Master = self(), AIS = erts_debug:set_internal_state(available_internal_state, true), + gc_request_when_gc_disabled_do(ref), + gc_request_when_gc_disabled_do(immed), + erts_debug:set_internal_state(available_internal_state, AIS). + +gc_request_when_gc_disabled_do(ReqIdType) -> + Master = self(), {P, M} = spawn_opt(fun () -> true = erts_debug:set_internal_state(gc_state, false), @@ -2545,7 +2550,10 @@ gc_request_when_gc_disabled(Config) when is_list(Config) -> receive after 100 -> ok end end, [monitor, link]), receive {P, gc_state, false} -> ok end, - ReqId = make_ref(), + ReqId = case ReqIdType of + ref -> make_ref(); + immed -> immed + end, async = garbage_collect(P, [{async, ReqId}]), receive {garbage_collect, ReqId, Result} -> @@ -2554,7 +2562,6 @@ gc_request_when_gc_disabled(Config) when is_list(Config) -> ok end, receive {garbage_collect, ReqId, true} -> ok end, - erts_debug:set_internal_state(available_internal_state, AIS), receive {'DOWN', M, process, P, _Reason} -> ok end, ok. diff --git a/erts/emulator/test/receive_SUITE.erl b/erts/emulator/test/receive_SUITE.erl index a12019ec83..1fe11428b4 100644 --- a/erts/emulator/test/receive_SUITE.erl +++ b/erts/emulator/test/receive_SUITE.erl @@ -25,14 +25,16 @@ -include_lib("common_test/include/ct.hrl"). -export([all/0, suite/0, - call_with_huge_message_queue/1,receive_in_between/1]). + call_with_huge_message_queue/1,receive_in_between/1, + receive_opt_exception/1,receive_opt_recursion/1]). suite() -> [{ct_hooks,[ts_install_cth]}, {timetrap, {minutes, 3}}]. -all() -> - [call_with_huge_message_queue, receive_in_between]. +all() -> + [call_with_huge_message_queue, receive_in_between, + receive_opt_exception, receive_opt_recursion]. call_with_huge_message_queue(Config) when is_list(Config) -> Pid = spawn_link(fun echo_loop/0), @@ -113,6 +115,60 @@ receive_one() -> dummy -> ok end. +receive_opt_exception(_Config) -> + Recurse = fun() -> + %% Overwrite with the same mark, + %% and never consume it. + ThrowFun = fun() -> throw(aborted) end, + aborted = (catch do_receive_opt_exception(ThrowFun)), + ok + end, + do_receive_opt_exception(Recurse), + + %% Eat the second message. + receive + Ref when is_reference(Ref) -> ok + end. + +do_receive_opt_exception(Disturber) -> + %% Create a receive mark. + Ref = make_ref(), + self() ! Ref, + Disturber(), + receive + Ref -> + ok + after 0 -> + error(the_expected_message_was_not_there) + end. + +receive_opt_recursion(_Config) -> + Recurse = fun() -> + %% Overwrite with the same mark, + %% and never consume it. + NoOp = fun() -> ok end, + BlackHole = spawn(NoOp), + expected = do_receive_opt_recursion(BlackHole, NoOp, true), + ok + end, + do_receive_opt_recursion(self(), Recurse, false), + ok. + +do_receive_opt_recursion(Recipient, Disturber, IsInner) -> + Ref = make_ref(), + Recipient ! Ref, + Disturber(), + receive + Ref -> ok + after 0 -> + case IsInner of + true -> + expected; + false -> + error(the_expected_message_was_not_there) + end + end. + %%% %%% Common helpers. %%% diff --git a/erts/emulator/utils/beam_makeops b/erts/emulator/utils/beam_makeops index d7791d23fa..da994fae3e 100755 --- a/erts/emulator/utils/beam_makeops +++ b/erts/emulator/utils/beam_makeops @@ -19,7 +19,7 @@ # %CopyrightEnd% # use strict; -use vars qw($BEAM_FORMAT_NUMBER); +use vars qw($BEAM_FORMAT_NUMBER $GC_REGEXP); use constant COLD => 0; use constant WARM => 1; use constant HOT => 2; @@ -36,6 +36,7 @@ use constant PACK_CMD_LOOSE => '3'; use constant PACK_CMD_WIDE => '4'; $BEAM_FORMAT_NUMBER = undef; +$GC_REGEXP = undef; my $target = \&emulator_output; my $outdir = "."; # Directory for output files. @@ -77,6 +78,10 @@ my %num_specific; my %gen_to_spec; my %specific_op; +# The following hashes are used for error checking. +my %print_name; +my %specific_op_arity; + # Information about each specific operator. Key is the print name (e.g. get_list_xxy). # Value is a hash. my %spec_op_info; @@ -131,7 +136,10 @@ my $loader_types = "nprvlqo"; my $genop_types = $compiler_types . $loader_types; # -# Defines the argument types and their loaded size assuming no packing. +# Define the operand types and their loaded size assuming no packing. +# +# Those are the types that can be used in the definition of a specific +# instruction. # my %arg_size = ('r' => 0, # x(0) - x register zero 'x' => 1, # x(N), N > 0 - x register @@ -154,12 +162,35 @@ my %arg_size = ('r' => 0, # x(0) - x register zero 'A' => 1, # arity value 'P' => 1, # byte offset into tuple or stack 'Q' => 1, # like 'P', but packable - 'h' => 1, # character + 'h' => 1, # character (not used) 'l' => 1, # float reg 'q' => 1, # literal term ); # +# Define the types that may be used in a transformation rule. +# +# %pattern_type defines the types that may be used in a pattern +# on the left side. +# +# %construction_type defines the types that may be used when +# constructing a new instruction on the right side (a subset of +# the pattern types that are possible to construct). +# +my $pattern_types = "acdfjilnopqsuxy"; +my %pattern_type; +@pattern_type{split("", $pattern_types)} = (1) x length($pattern_types); + +my %construction_type; +foreach my $type (keys %pattern_type) { + $construction_type{$type} = 1 + if index($genop_types, $type) >= 0; +} +foreach my $makes_no_sense ('f', 'j', 'o', 'p', 'q') { + delete $construction_type{$makes_no_sense}; +} + +# # Generate bits. # my %type_bit; @@ -194,7 +225,8 @@ sub define_type_bit { define_type_bit('S', $type_bit{'d'}); define_type_bit('j', $type_bit{'f'} | $type_bit{'p'}); - # Aliases (for matching purposes). + # Aliases of 'u'. Those specify how to load the operand and + # what kind of packing can be done. define_type_bit('t', $type_bit{'u'}); define_type_bit('I', $type_bit{'u'}); define_type_bit('W', $type_bit{'u'}); @@ -279,9 +311,15 @@ if ($wordsize == 64) { # Add placeholders for built-in macros. # -$c_code{'IS_PACKED'} = ['$Expr',"built-in macro",('Expr')]; -$c_code{'ARG_POSITION'} = ['$Expr',"built-in macro",('Expr')]; -foreach my $name (keys %c_code) { +my %predef_macros = + (OPERAND_POSITION => ['Expr'], + IF => ['Expr','IfTrue','IfFalse'], + REFRESH_GEN_DEST => [], + ); +foreach my $name (keys %predef_macros) { + my @args = @{$predef_macros{$name}}; + my $body = join(':', map { '$' . $_ } @args); + $c_code{$name} = [$body,"built-in macro",@args], $c_code_used{$name} = 1; } @@ -359,8 +397,10 @@ while (<>) { # if (/^([\w_][\w\d_]+)=(.*)/) { no strict 'refs'; - my($name) = $1; - $$name = $2; + my $name = $1; + my $value = $2; + $value =~ s/;\s*$//; + $$name = $value; next; } @@ -1019,6 +1059,22 @@ sub parse_specific_op { my $key = "$name/$arity"; foreach my $args_ref (@res) { @args = @$args_ref; + my $arity = @args; + my $loc = "$ARGV($.)"; + if (defined $specific_op_arity{$name}) { + my($prev_arity,$loc) = @{$specific_op_arity{$name}}; + if ($arity != $prev_arity) { + error("$name defined with arity $arity, " . + "but previously defined with arity $prev_arity at $loc"); + } + } + $specific_op_arity{$name} = [$arity,$loc]; + my $print_name = print_name($name, @args); + if (defined $print_name{$print_name}) { + error("$name @args: already defined at " . + $print_name{$print_name}); + } + $print_name{$print_name} = $loc; push @{$specific_op{$key}}, [$name,$hotness,@args]; } @@ -1333,7 +1389,9 @@ sub cg_basic { # sub cg_combined_size { - my %params = (@_, pack_options => \@basic_pack_options); + my %params = (@_, + pack_options => \@basic_pack_options, + size_only => 1); $params{pack_options} = \@extended_pack_options if $params{first}; my($size) = code_gen(%params); @@ -1361,6 +1419,7 @@ sub code_gen { my %params = (extra_comments => '', offset => 0, inc => 0, + size_only => 0, @_); my $name = $params{name}; my $extra_comments = $params{extra_comments}; @@ -1393,6 +1452,7 @@ sub code_gen { my $need_block = 0; my $arg_offset = $offset; + my $has_gen_dest = 0; @args = map { s/[?]$//g; $_ } @args; foreach (@args) { my($this_size) = $arg_size{$_}; @@ -1403,6 +1463,7 @@ sub code_gen { "Eterm* dst_ptr = REG_TARGET_PTR(dst);\n"; push(@f, "*dst_ptr"); $this_size = $1; + $has_gen_dest = 1; last SWITCH; }; /^packed:[a-zA-z]:(\d):(.*)/ and do { @@ -1435,6 +1496,7 @@ sub code_gen { $var_decls .= "Eterm dst = " . arg_offset($arg_offset) . ";\n" . "Eterm* dst_ptr = REG_TARGET_PTR(dst);\n"; push(@f, "*dst_ptr"); + $has_gen_dest = 1; last SWITCH; }; defined $arg_size{$_} and do { @@ -1449,10 +1511,10 @@ sub code_gen { } # - # If the implementation is in beam_emu.c, there is nothing - # more to do. + # If the implementation is in beam_emu.c or if + # the caller only wants the size, we are done. # - unless (defined $c_code_ref) { + if (not defined $c_code_ref or $params{size_only}) { return ($size+1, undef, ''); } @@ -1517,9 +1579,36 @@ sub code_gen { "{", "$var_decls$body", "}", ""); + + # Make sure that $REFRESH_GEN_DEST() is used when a + # general destination ('d') may have been clobbered by + # a GC. + my $gc_error = verify_gc_code($code, $has_gen_dest); + if (defined $gc_error) { + warn $gc_error; + error("... from the body of $name at $where"); + } + + # Done. ($size+1, $code, $pack_spec); } +sub verify_gc_code { + my $code = shift; + my $has_gen_dest = shift; + + return unless $has_gen_dest; + + if ($code =~ /$GC_REGEXP/o) { + my $code_after_gc = substr($code, $+[0]); + unless ($code_after_gc =~ /dst_ptr = REG_TARGET_PTR/) { + return "pointer to destination register is invalid after GC -- " . + "use \$REFRESH_GEN_DEST()\n"; + } + } + return undef; +} + sub arg_offset { my $offset = shift; "I[" . ($offset+1) . "]"; @@ -1619,17 +1708,26 @@ sub expand_macro { } # Handle built-in macros. - if ($name eq 'ARG_POSITION') { + if ($name eq 'OPERAND_POSITION') { if ($body =~ /^I\[(\d+)\]$/) { $body = $1; } else { $body = 0; } - } elsif ($name eq 'IS_PACKED') { - $body = ($body =~ /^I\[\d+\]$/) ? 0 : 1; + } elsif ($name eq 'IF') { + my $expr = $new_bindings{Expr}; + my $bool = eval $expr; + if ($@ ne '') { + &error("bad expression '$expr' in \$IF()"); + } + my $part = $bool ? 'IfTrue' : 'IfFalse'; + $body = $new_bindings{$part}; + } elsif ($name eq 'REFRESH_GEN_DEST') { + $body = "dst_ptr = REG_TARGET_PTR(dst)"; } - # Wrap body if needed and return resul.t + + # Wrap body if needed and return result. $body = "do {\n$body\n} while (0)" if needs_do_wrapper($body); ($body,$rest); @@ -2156,12 +2254,19 @@ sub tr_parse_op { if (/^([a-z*]+)(.*)/) { $type = $1; $_ = $2; + error("$type: only a single type is allowed on right side of transformations") + if not $src and length($type) > 1; foreach (split('', $type)) { - error("bad type in $op") - unless defined $type_bit{$_} or $type eq '*'; - $_ eq 'r' and - error("$op: 'r' is not allowed in transformations") - } + next if $src and $type eq '*'; + error("$op: not a type") + unless defined $type_bit{$_}; + error("$op: the type '$_' is not allowed in transformations") + unless defined $pattern_type{$_}; + if (not $src) { + error("$op: type '$_' is not allowed on the right side of transformations") + unless defined $construction_type{$_}; + } + } } # Get an optional condition. (In source.) @@ -2194,10 +2299,18 @@ sub tr_parse_op { } # Get an optional value. (In destination.) - $type_val = $type eq 'x' ? 1023 : 0; + if ($type eq 'x') { + $type_val = 1023; + } elsif ($type eq 'a') { + $type_val = 'am_Empty'; + } else { + $type_val = 0; + } if (/^=(.*)/) { - error("value not allowed in source: $op") + error("$op: value not allowed in source") if $src; + error("$op: the type 'n' must not be given a value") + if $type eq 'n'; $type_val = $1; $_ = ''; } @@ -2207,13 +2320,16 @@ sub tr_parse_op { error("garbage '$_' after operand: $op") unless /^\s*$/; - # Test that destination has no conditions. + # Check the conditions. - unless ($src) { - error("condition not allowed in destination: $op") + if ($src) { + error("$op: the type '$type' is not allowed to be compared with a literal value") + if $cond and not $construction_type{$type}; + } else { + error("$op: condition not allowed in destination") if $cond; - error("variable name and type cannot be combined in destination: $op") - if $var && $type; + error("$op: variable name and type cannot be combined in destination") + if $var and $type; } ($var,$type,$type_val,$cond,$cond_val); |