diff options
Diffstat (limited to 'erts/emulator')
85 files changed, 2778 insertions, 1956 deletions
diff --git a/erts/emulator/beam/atom.c b/erts/emulator/beam/atom.c index bbe1cb3e11..e5b7616a0d 100644 --- a/erts/emulator/beam/atom.c +++ b/erts/emulator/beam/atom.c @@ -452,7 +452,7 @@ init_atom_table(void) /* Ordinary atoms */ for (i = 0; erl_atom_names[i] != 0; i++) { int ix; - a.len = strlen(erl_atom_names[i]); + a.len = sys_strlen(erl_atom_names[i]); a.latin1_chars = a.len; a.name = (byte*)erl_atom_names[i]; a.slot.index = i; diff --git a/erts/emulator/beam/atom.names b/erts/emulator/beam/atom.names index cb4fab51f1..38b5f0c5e3 100644 --- a/erts/emulator/beam/atom.names +++ b/erts/emulator/beam/atom.names @@ -48,7 +48,7 @@ atom Empty='' # # Used in the Beam emulator loop. (Smaller literals usually means tighter code.) # -atom fun infinity timeout normal call return +atom infinity timeout normal call return atom throw error exit atom undefined @@ -69,11 +69,9 @@ atom DOWN='DOWN' atom UP='UP' atom EXIT='EXIT' atom abort -atom aborted atom abs_path atom absoluteURI atom ac -atom accessor atom active atom active_tasks atom active_tasks_all @@ -88,7 +86,6 @@ atom allocated_areas atom allocator atom allocator_sizes atom alloc_util_allocators -atom allow_gc atom allow_passive_connect atom already_loaded atom amd64 @@ -122,9 +119,7 @@ atom bag atom band atom big atom bif_return_trap -atom bif_timer_server atom binary -atom binary_bin_to_list_trap atom binary_copy_trap atom binary_find_trap atom binary_longest_prefix_trap @@ -179,7 +174,6 @@ atom convert_time_unit atom connect atom connected atom connection_closed -atom cons atom const atom context_switches atom control @@ -199,7 +193,6 @@ atom debug_flags atom decimals atom default atom delay_trap -atom depth atom dictionary atom dirty_bif_exception atom dirty_bif_result @@ -246,15 +239,14 @@ atom Eqeq='==' atom erl_tracer atom erlang atom erl_signal_server -atom ERROR='ERROR' atom error_handler atom error_logger atom erts_code_purger atom erts_debug +atom erts_dflags atom erts_internal atom ets atom ETS_TRANSFER='ETS-TRANSFER' -atom event atom exact_reductions atom exception_from atom exception_trace @@ -281,27 +273,20 @@ atom force atom format_cpu_topology atom free atom fullsweep_after -atom fullsweep_if_old_binaries -atom fun -atom function atom functions atom function_clause atom garbage_collecting atom garbage_collection atom garbage_collection_info -atom gc_end atom gc_major_end atom gc_major_start atom gc_max_heap_size atom gc_minor_end atom gc_minor_start -atom gc_start atom Ge='>=' atom generational -atom get_data atom get_seq_token atom get_tcw -atom getenv atom gather_gc_info_result atom gather_io_bytes atom gather_microstate_accounting_result @@ -343,7 +328,6 @@ atom initial_call atom input atom internal atom internal_error -atom internal_status atom instruction_counts atom invalid atom is_constant @@ -389,14 +373,12 @@ atom match_spec_result atom max atom maximum atom max_heap_size -atom max_tables max_processes atom mbuf_size atom md5 atom memory atom memory_internal atom memory_types atom message -atom message_binary atom message_queue_data atom message_queue_len atom messages @@ -443,21 +425,18 @@ atom new_processes atom new_ports atom new_uniq atom newline -atom next atom no atom nomatch atom none atom no_auto_capture atom noconnect atom noconnection -atom nocookie atom node atom node_type atom nodedown atom nodedown_reason atom nodeup atom noeol -atom nofile atom noproc atom normal atom nosuspend @@ -479,7 +458,6 @@ atom notempty_atstart atom notify atom notsup atom nouse_stdio -atom objects atom off_heap atom offset atom ok @@ -549,7 +527,6 @@ atom re_run_trap atom read_concurrency atom ready_input atom ready_output -atom ready_async atom reason atom receive atom recent_size @@ -577,7 +554,8 @@ atom running_procs atom runtime atom safe atom save_calls -atom scheduler +atom sbct +atom scheduler atom scheduler_id atom scheduler_wall_time atom scheduler_wall_time_all @@ -595,7 +573,6 @@ atom sequential_trace_token atom serial atom set atom set_cpu_topology -atom set_data atom set_on_first_link atom set_on_first_spawn atom set_on_link @@ -603,8 +580,6 @@ atom set_on_spawn atom set_seq_token atom set_tcw atom set_tcw_fake -atom separate -atom shared atom sighup atom sigterm atom sigusr1 @@ -620,7 +595,6 @@ atom sigtstp atom sigquit atom silent atom size -atom sl_alloc atom spawn_executable atom spawn_driver atom spawned @@ -628,7 +602,6 @@ atom ssl_tls atom stack_size atom start atom status -atom static atom stderr_to_stdout atom stop atom stream @@ -638,14 +611,11 @@ atom sunrm atom suspend atom suspended atom suspending -atom sys_misc atom system -atom system_error atom system_flag_scheduler_wall_time atom system_limit atom system_version atom system_architecture -atom SYSTEM='SYSTEM' atom table atom term_to_binary_trap atom this @@ -663,7 +633,7 @@ atom total_heap_size atom total_run_queue_lengths atom total_run_queue_lengths_all atom tpkt -atom trace trace_ts traced +atom trace traced atom trace_control_word atom trace_status atom tracer @@ -672,7 +642,6 @@ atom trim atom trim_all atom try_clause atom true -atom tuple atom type atom ucompile atom ucp @@ -689,14 +658,11 @@ atom unblock_normal atom uniq atom unless_suspending atom unloaded -atom unloading atom unloaded_only atom unload_cancelled atom value -atom values atom version atom visible -atom wait atom waiting atom wall_clock atom warning diff --git a/erts/emulator/beam/beam_debug.c b/erts/emulator/beam/beam_debug.c index 8f02d509a9..5eb68b817e 100644 --- a/erts/emulator/beam/beam_debug.c +++ b/erts/emulator/beam/beam_debug.c @@ -228,11 +228,11 @@ erts_debug_instructions_0(BIF_ALIST_0) Eterm res = NIL; for (i = 0; i < num_instructions; i++) { - needed += 2*strlen(opc[i].name); + needed += 2*sys_strlen(opc[i].name); } hp = HAlloc(BIF_P, needed); for (i = num_instructions-1; i >= 0; i--) { - Eterm s = erts_bld_string_n(&hp, 0, opc[i].name, strlen(opc[i].name)); + Eterm s = erts_bld_string_n(&hp, 0, opc[i].name, sys_strlen(opc[i].name)); res = erts_bld_cons(&hp, 0, s, res); } return res; @@ -431,7 +431,7 @@ print_op(fmtfn_t to, void *to_arg, int op, int size, BeamInstr* addr) * the packing program backwards and in reverse. */ - prog = start_prog + strlen(start_prog); + prog = start_prog + sys_strlen(start_prog); while (start_prog < prog) { prog--; switch (*prog) { diff --git a/erts/emulator/beam/beam_emu.c b/erts/emulator/beam/beam_emu.c index ef9abcde08..fbd0e38735 100644 --- a/erts/emulator/beam/beam_emu.c +++ b/erts/emulator/beam/beam_emu.c @@ -749,7 +749,7 @@ void process_main(Eterm * x_reg_array, FloatDef* f_reg_array) dtrace_proc_str(c_p, process_buf); if (ERTS_PROC_IS_EXITING(c_p)) { - strcpy(fun_buf, "<exiting>"); + sys_strcpy(fun_buf, "<exiting>"); } else { ErtsCodeMFA *cmfa = find_function_from_pc(c_p->i); if (cmfa) { @@ -1229,7 +1229,7 @@ void erts_dirty_process_main(ErtsSchedulerData *esdp) dtrace_proc_str(c_p, process_buf); if (ERTS_PROC_IS_EXITING(c_p)) { - strcpy(fun_buf, "<exiting>"); + sys_strcpy(fun_buf, "<exiting>"); } else { ErtsCodeMFA *cmfa = find_function_from_pc(c_p->i); if (cmfa) { diff --git a/erts/emulator/beam/beam_load.c b/erts/emulator/beam/beam_load.c index e242fe9140..0184c567f1 100644 --- a/erts/emulator/beam/beam_load.c +++ b/erts/emulator/beam/beam_load.c @@ -459,7 +459,7 @@ typedef struct LoaderState { #ifdef DEBUG # define GARBAGE 0xCC -# define DEBUG_INIT_GENOP(Dst) memset(Dst, GARBAGE, sizeof(GenOp)) +# define DEBUG_INIT_GENOP(Dst) sys_memset(Dst, GARBAGE, sizeof(GenOp)) #else # define DEBUG_INIT_GENOP(Dst) #endif @@ -1422,7 +1422,7 @@ load_atom_table(LoaderState* stp, ErtsAtomEncoding enc) Atom* ap; ap = atom_tab(atom_val(stp->atom[1])); - memcpy(sbuf, ap->name, ap->len); + sys_memcpy(sbuf, ap->name, ap->len); sbuf[ap->len] = '\0'; LoadError1(stp, "module name in object code is %s", sbuf); } @@ -2089,7 +2089,7 @@ load_code(LoaderState* stp) erts_alloc(ERTS_ALC_T_LOADER_TMP, (arity+last_op->a[arg].val) *sizeof(GenOpArg)); - memcpy(last_op->a, last_op->def_args, + sys_memcpy(last_op->a, last_op->def_args, arity*sizeof(GenOpArg)); arity += last_op->a[arg].val; break; @@ -4915,7 +4915,9 @@ freeze_code(LoaderState* stp) line_items[i] = codev + stp->ci - 1; line_tab->fname_ptr = (Eterm*) &line_items[i + 1]; - memcpy(line_tab->fname_ptr, stp->fname, stp->num_fnames*sizeof(Eterm)); + if (stp->num_fnames) + sys_memcpy(line_tab->fname_ptr, stp->fname, + stp->num_fnames*sizeof(Eterm)); line_tab->loc_size = stp->loc_size; if (stp->loc_size == 2) { @@ -5498,8 +5500,8 @@ transform_engine(LoaderState* st) case TOP_store_rest_args: { GENOP_ARITY(instr, instr->arity+num_rest_args); - memcpy(instr->a, instr->def_args, ap*sizeof(GenOpArg)); - memcpy(instr->a+ap, rest_args, num_rest_args*sizeof(GenOpArg)); + sys_memcpy(instr->a, instr->def_args, ap*sizeof(GenOpArg)); + sys_memcpy(instr->a+ap, rest_args, num_rest_args*sizeof(GenOpArg)); ap += num_rest_args; } break; @@ -6447,7 +6449,7 @@ stub_copy_info(LoaderState* stp, Sint decoded_size; Uint size = stp->chunks[chunk].size; if (size != 0) { - memcpy(info, stp->chunks[chunk].start, size); + sys_memcpy(info, stp->chunks[chunk].start, size); *ptr_word = info; decoded_size = erts_decode_ext_size(info, size); if (decoded_size < 0) { @@ -7017,8 +7019,8 @@ void dbg_set_traced_mfa(const char* m, const char* f, Uint a) { unsigned i = dbg_trace_ix++; ASSERT(i < MFA_MAX); - dbg_trace_m[i] = am_atom_put(m, strlen(m)); - dbg_trace_f[i] = am_atom_put(f, strlen(f)); + dbg_trace_m[i] = am_atom_put(m, sys_strlen(m)); + dbg_trace_f[i] = am_atom_put(f, sys_strlen(f)); dbg_trace_a[i] = a; } diff --git a/erts/emulator/beam/bif.c b/erts/emulator/beam/bif.c index f086c434ea..652b95105f 100644 --- a/erts/emulator/beam/bif.c +++ b/erts/emulator/beam/bif.c @@ -1744,7 +1744,6 @@ BIF_RETTYPE process_flag_2(BIF_ALIST_2) else if (BIF_ARG_1 == am_scheduler) { ErtsRunQueue *old, *new, *curr; Sint sched; - erts_aint32_t state; if (!is_small(BIF_ARG_2)) goto error; @@ -1753,23 +1752,23 @@ BIF_RETTYPE process_flag_2(BIF_ALIST_2) goto error; if (sched == 0) { + old = erts_bind_runq_proc(BIF_P, 0); new = NULL; - state = erts_atomic32_read_band_mb(&BIF_P->state, - ~ERTS_PSFLG_BOUND); } else { + int bound = !0; new = erts_schedid2runq(sched); - erts_atomic_set_nob(&BIF_P->run_queue, (erts_aint_t) new); - state = erts_atomic32_read_bor_mb(&BIF_P->state, - ERTS_PSFLG_BOUND); + old = erts_set_runq_proc(BIF_P, new, &bound); + if (!bound) + old = NULL; } + old_value = old ? make_small(old->ix+1) : make_small(0); + curr = erts_proc_sched_data(BIF_P)->run_queue; - old = (ERTS_PSFLG_BOUND & state) ? curr : NULL; ASSERT(!old || old == curr); - old_value = old ? make_small(old->ix+1) : make_small(0); if (new && new != curr) ERTS_BIF_YIELD_RETURN_X(BIF_P, old_value, am_scheduler); else @@ -3102,7 +3101,7 @@ BIF_RETTYPE list_to_integer_2(BIF_ALIST_2) static int do_float_to_charbuf(Process *p, Eterm efloat, Eterm list, char *fbuf, int sizeof_fbuf) { - const static int arity_two = make_arityval(2); + Eterm arity_two = make_arityval(2); int decimals = SYS_DEFAULT_FLOAT_DECIMALS; int compact = 0; enum fmt_type_ { @@ -3466,7 +3465,7 @@ BIF_RETTYPE binary_to_float_1(BIF_ALIST_1) if (bit_offs) erts_copy_bits(bytes, bit_offs, 1, buf, 0, 1, size*8); else - memcpy(buf, bytes, size); + sys_memcpy(buf, bytes, size); buf[size] = '\0'; @@ -4200,10 +4199,10 @@ BIF_RETTYPE list_to_port_1(BIF_ALIST_1) buf[i] = '\0'; /* null terminal */ cp = &buf[0]; - if (strncmp("#Port<", cp, 6) != 0) + if (sys_strncmp("#Port<", cp, 6) != 0) goto bad; - cp += 6; /* strlen("#Port<") */ + cp += 6; /* sys_strlen("#Port<") */ if (sscanf(cp, "%u.%u>", (unsigned int*)&n, (unsigned int*)&p) < 2) goto bad; @@ -4805,6 +4804,8 @@ BIF_RETTYPE system_flag_2(BIF_ALIST_2) "scheduled for removal in Erlang/OTP 18. For more\n" "information see the erlang:system_flag/2 documentation.\n"); return erts_bind_schedulers(BIF_P, BIF_ARG_2); + } else if (ERTS_IS_ATOM_STR("erts_alloc", BIF_ARG_1)) { + return erts_alloc_set_dyn_param(BIF_P, BIF_ARG_2); } error: BIF_ERROR(BIF_P, BADARG); diff --git a/erts/emulator/beam/bif.tab b/erts/emulator/beam/bif.tab index c0d5e8ce74..be653ee2a0 100644 --- a/erts/emulator/beam/bif.tab +++ b/erts/emulator/beam/bif.tab @@ -551,9 +551,6 @@ bif binary:last/1 bif binary:at/2 bif binary:part/2 binary_binary_part_2 bif binary:part/3 binary_binary_part_3 -bif binary:bin_to_list/1 -bif binary:bin_to_list/2 -bif binary:bin_to_list/3 bif binary:list_to_bin/1 bif binary:copy/1 bif binary:copy/2 @@ -693,6 +690,8 @@ bif erlang:iolist_to_iovec/1 # New in 21.0 # +bif erts_internal:get_dflags/0 bif erts_internal:new_connection/1 bif erts_internal:abort_connection/2 bif erts_internal:map_next/3 +bif ets:whereis/1 diff --git a/erts/emulator/beam/binary.c b/erts/emulator/beam/binary.c index d4db1a4188..95d324d2c1 100644 --- a/erts/emulator/beam/binary.c +++ b/erts/emulator/beam/binary.c @@ -953,7 +953,10 @@ HIPE_WRAPPER_BIF_DISABLE_GC(iolist_to_binary, 1) BIF_RETTYPE iolist_to_binary_1(BIF_ALIST_1) { if (is_binary(BIF_ARG_1)) { - BIF_RET(BIF_ARG_1); + if (binary_bitsize(BIF_ARG_1) == 0) { + BIF_RET(BIF_ARG_1); + } + BIF_ERROR(BIF_P, BADARG); } return erts_list_to_binary_bif(BIF_P, BIF_ARG_1, bif_export[BIF_iolist_to_binary_1]); } diff --git a/erts/emulator/beam/break.c b/erts/emulator/beam/break.c index e7acea0c5f..3967f7f7fc 100644 --- a/erts/emulator/beam/break.c +++ b/erts/emulator/beam/break.c @@ -209,7 +209,6 @@ static void doit_print_monitor(ErtsMonitor *mon, void *vpcontext) void print_process_info(fmtfn_t to, void *to_arg, Process *p) { - time_t approx_started; int garbing = 0; int running = 0; struct saved_calls *scb; @@ -258,8 +257,6 @@ print_process_info(fmtfn_t to, void *to_arg, Process *p) } erts_print(to, to_arg, "Spawned by: %T\n", p->parent); - approx_started = (time_t) p->approx_started; - erts_print(to, to_arg, "Started: %s", ctime(&approx_started)); ERTS_MSGQ_MV_INQ2PRIVQ(p); erts_print(to, to_arg, "Message queue length: %d\n", p->msg.len); @@ -336,6 +333,12 @@ print_process_info(fmtfn_t to, void *to_arg, Process *p) erts_print(to, to_arg, "Heap unused: %bpu\n", (p->hend - p->htop)); erts_print(to, to_arg, "OldHeap unused: %bpu\n", (OLD_HEAP(p) == NULL) ? 0 : (OLD_HEND(p) - OLD_HTOP(p)) ); + erts_print(to, to_arg, "BinVHeap: %b64u\n", p->off_heap.overhead); + erts_print(to, to_arg, "OldBinVHeap: %b64u\n", BIN_OLD_VHEAP(p)); + erts_print(to, to_arg, "BinVHeap unused: %b64u\n", + BIN_VHEAP_SZ(p) - p->off_heap.overhead); + erts_print(to, to_arg, "OldBinVHeap unused: %b64u\n", + BIN_OLD_VHEAP_SZ(p) - BIN_OLD_VHEAP(p)); erts_print(to, to_arg, "Memory: %beu\n", erts_process_memory(p, !0)); if (garbing) { @@ -503,7 +506,7 @@ do_break(void) /* check if we're in console mode and, if so, halt immediately if break is called */ mode = erts_read_env("ERL_CONSOLE_MODE"); - if (mode && strcmp(mode, "window") != 0) + if (mode && sys_strcmp(mode, "window") != 0) erts_exit(0, ""); erts_free_read_env(mode); #endif /* __WIN32__ */ @@ -891,6 +894,21 @@ erl_crash_dump_v(char *file, int line, char* fmt, va_list args) erts_print_scheduler_info(to, to_arg, ERTS_SCHEDULER_IX(i)), erts_cbprintf(to, to_arg, "** crashed **\n")); } + for (i = 0; i < erts_no_dirty_cpu_schedulers; i++) { + ERTS_SYS_TRY_CATCH( + erts_print_scheduler_info(to, to_arg, ERTS_DIRTY_CPU_SCHEDULER_IX(i)), + erts_cbprintf(to, to_arg, "** crashed **\n")); + } + erts_cbprintf(to, to_arg, "=dirty_cpu_run_queue\n"); + erts_print_run_queue_info(to, to_arg, ERTS_DIRTY_CPU_RUNQ); + + for (i = 0; i < erts_no_dirty_io_schedulers; i++) { + ERTS_SYS_TRY_CATCH( + erts_print_scheduler_info(to, to_arg, ERTS_DIRTY_IO_SCHEDULER_IX(i)), + erts_cbprintf(to, to_arg, "** crashed **\n")); + } + erts_cbprintf(to, to_arg, "=dirty_io_run_queue\n"); + erts_print_run_queue_info(to, to_arg, ERTS_DIRTY_IO_RUNQ); #endif diff --git a/erts/emulator/beam/bs_instrs.tab b/erts/emulator/beam/bs_instrs.tab index b11903a47b..94e0000c8b 100644 --- a/erts/emulator/beam/bs_instrs.tab +++ b/erts/emulator/beam/bs_instrs.tab @@ -990,6 +990,9 @@ ctx_to_bin.execute() { Uint hole_size; Uint orig = mb->orig; ErlSubBin* sb = (ErlSubBin *) boxed_val(context); + /* Since we're going to overwrite the match state with the result, an + * ErlBinMatchState must be at least as large as an ErlSubBin. */ + ERTS_CT_ASSERT(sizeof(ErlSubBin) <= sizeof(ErlBinMatchState)); hole_size = 1 + header_arity(sb->thing_word) - ERL_SUB_BIN_SIZE; sb->thing_word = HEADER_SUB_BIN; sb->size = BYTE_OFFSET(size); diff --git a/erts/emulator/beam/copy.c b/erts/emulator/beam/copy.c index 10bf197405..7769a914db 100644 --- a/erts/emulator/beam/copy.c +++ b/erts/emulator/beam/copy.c @@ -611,7 +611,7 @@ Eterm copy_struct_x(Eterm obj, Uint sz, Eterm** hpp, ErlOffHeap* off_heap, Uint Eterm* htop; Eterm* hbot; Eterm* hp; - Eterm* objp; + Eterm* ERTS_RESTRICT objp; Eterm* tp; Eterm res; Eterm elem; @@ -1821,7 +1821,8 @@ all_clean: * * NOTE: Assumes that term is a tuple (ptr is an untagged tuple ptr). */ -Eterm copy_shallow(Eterm* ptr, Uint sz, Eterm** hpp, ErlOffHeap* off_heap) +Eterm copy_shallow(Eterm* ERTS_RESTRICT ptr, Uint sz, Eterm** hpp, + ErlOffHeap* off_heap) { Eterm* tp = ptr; Eterm* hp = *hpp; @@ -1985,7 +1986,7 @@ move_one_frag(Eterm** hpp, ErlHeapFragment* frag, ErlOffHeap* off_heap, int lite if (is_header(val)) { struct erl_off_heap_header* hdr = (struct erl_off_heap_header*)hp; ASSERT(ptr + header_arity(val) < end); - move_boxed(&ptr, val, &hp, &dummy_ref); + ptr = move_boxed(ptr, val, &hp, &dummy_ref); switch (val & _HEADER_SUBTAG_MASK) { case REF_SUBTAG: if (is_ordinary_ref_thing(hdr)) @@ -2002,7 +2003,7 @@ move_one_frag(Eterm** hpp, ErlHeapFragment* frag, ErlOffHeap* off_heap, int lite } else { /* must be a cons cell */ ASSERT(ptr+1 < end); - move_cons(&ptr, val, &hp, &dummy_ref); + move_cons(ptr, val, &hp, &dummy_ref); ptr += 2; } } diff --git a/erts/emulator/beam/dist.c b/erts/emulator/beam/dist.c index 30390cdb5e..cd799e04b8 100644 --- a/erts/emulator/beam/dist.c +++ b/erts/emulator/beam/dist.c @@ -588,13 +588,13 @@ int erts_do_net_exits(DistEntry *dep, Eterm reason) erts_port_task_abort(&dep->dist_cmd); } - if (dep->status & ERTS_DE_SFLG_EXITING) { + if (dep->state == ERTS_DE_STATE_EXITING) { #ifdef DEBUG ASSERT(erts_atomic32_read_nob(&dep->qflgs) & ERTS_DE_QFLG_EXIT); #endif } else { - dep->status |= ERTS_DE_SFLG_EXITING; + dep->state = ERTS_DE_STATE_EXITING; erts_mtx_lock(&dep->qlock); ASSERT(!(erts_atomic32_read_nob(&dep->qflgs) & ERTS_DE_QFLG_EXIT)); erts_atomic32_read_bor_relb(&dep->qflgs, ERTS_DE_QFLG_EXIT); @@ -641,6 +641,14 @@ trap_function(Eterm func, int arity) return erts_export_put(am_erlang, func, arity); } +/* + * Sync with dist_util.erl: + * + * -record(erts_dflags, + * {default, mandatory, addable, rejectable, strict_order}). + */ +static Eterm erts_dflags_record; + void init_dist(void) { init_nodes_monitors(); @@ -657,6 +665,16 @@ void init_dist(void) dist_ctrl_put_data_trap = erts_export_put(am_erts_internal, am_dist_ctrl_put_data, 2); + { + Eterm* hp = erts_alloc(ERTS_ALC_T_LITERAL, (1+6)*sizeof(Eterm)); + erts_dflags_record = TUPLE6(hp, am_erts_dflags, + make_small(DFLAG_DIST_DEFAULT), + make_small(DFLAG_DIST_MANDATORY), + make_small(DFLAG_DIST_ADDABLE), + make_small(DFLAG_DIST_REJECTABLE), + make_small(DFLAG_DIST_STRICT_ORDER)); + erts_set_literal_tag(&erts_dflags_record, hp, (1+6)); + } } #define ErtsDistOutputBuf2Binary(OB) \ @@ -767,7 +785,7 @@ static void clear_dist_entry(DistEntry *dep) erts_atomic64_set_nob(&dep->out, 0); obuf = clear_de_out_queues(dep); - dep->status = 0; + dep->state = ERTS_DE_STATE_IDLE; suspendees = get_suspended_on_de(dep, ERTS_DE_QFLGS_ALL); erts_mtx_unlock(&dep->qlock); @@ -1982,6 +2000,7 @@ erts_dsig_send(ErtsDSigData *dsdp, struct erts_dsig_send_context* ctx) break; } ctx->u.ec.flags = ctx->flags; + ctx->u.ec.hopefull_flags = 0; ctx->u.ec.level = 0; ctx->u.ec.wstack.wstart = NULL; ctx->obuf->msg_start = ctx->obuf->ext_endp; @@ -2005,6 +2024,7 @@ erts_dsig_send(ErtsDSigData *dsdp, struct erts_dsig_send_context* ctx) ctx->data_size = ctx->obuf->ext_endp - ctx->obuf->extp; + ctx->obuf->hopefull_flags = ctx->u.ec.hopefull_flags; /* * Signal encoded; now verify that the connection still exists, * and if so enqueue the signal and schedule it for send. @@ -2012,8 +2032,8 @@ erts_dsig_send(ErtsDSigData *dsdp, struct erts_dsig_send_context* ctx) ctx->obuf->next = NULL; erts_de_rlock(dep); cid = dep->cid; - if (!(dep->status & (ERTS_DE_SFLG_PENDING | ERTS_DE_SFLG_CONNECTED)) - || dep->status & ERTS_DE_SFLG_EXITING + if (dep->state == ERTS_DE_STATE_EXITING + || dep->state == ERTS_DE_STATE_IDLE || dep->connection_id != dsdp->connection_id) { /* Not the same connection as when we started; drop message... */ erts_de_runlock(dep); @@ -2088,7 +2108,7 @@ erts_dsig_send(ErtsDSigData *dsdp, struct erts_dsig_send_context* ctx) } erts_mtx_unlock(&dep->qlock); - if (!(dep->status & ERTS_DE_SFLG_PENDING)) { + if (dep->state != ERTS_DE_STATE_PENDING) { if (is_internal_port(dep->cid)) erts_schedule_dist_command(NULL, dep); } @@ -2270,7 +2290,7 @@ int erts_dist_command(Port *prt, int initial_reds) { Sint reds = initial_reds - ERTS_PORT_REDS_DIST_CMD_START; - Uint32 status; + enum dist_entry_state state; Uint32 flags; Sint qsize, obufsize = 0; ErtsDistOutputQueue oq, foq; @@ -2285,17 +2305,17 @@ erts_dist_command(Port *prt, int initial_reds) erts_de_rlock(dep); flags = dep->flags; - status = dep->status; + state = dep->state; send = dep->send; erts_de_runlock(dep); - if (status & ERTS_DE_SFLG_EXITING) { + if (state == ERTS_DE_STATE_EXITING) { erts_deliver_port_exit(prt, prt->common.id, am_killed, 0, 1); reds -= ERTS_PORT_REDS_DIST_CMD_EXIT; return initial_reds - reds; } - ASSERT(!(status & ERTS_DE_SFLG_PENDING)); + ASSERT(state != ERTS_DE_STATE_PENDING); ASSERT(send); @@ -2831,7 +2851,7 @@ dist_ctrl_get_data_1(BIF_ALIST_1) erts_de_rlock(dep); - if (dep->status & ERTS_DE_SFLG_EXITING) + if (dep->state == ERTS_DE_STATE_EXITING) goto return_none; ASSERT(dep->cid == BIF_P->common.id); @@ -2934,9 +2954,9 @@ erts_dist_port_not_busy(Port *prt) static void kill_connection(DistEntry *dep) { ERTS_LC_ASSERT(erts_lc_is_de_rwlocked(dep)); - ASSERT(dep->status == ERTS_DE_SFLG_CONNECTED); + ASSERT(dep->state == ERTS_DE_STATE_CONNECTED); - dep->status |= ERTS_DE_SFLG_EXITING; + dep->state = ERTS_DE_STATE_EXITING; erts_mtx_lock(&dep->qlock); ASSERT(!(erts_atomic32_read_nob(&dep->qflgs) & ERTS_DE_QFLG_EXIT)); erts_atomic32_read_bor_nob(&dep->qflgs, ERTS_DE_QFLG_EXIT); @@ -2953,7 +2973,7 @@ erts_kill_dist_connection(DistEntry *dep, Uint32 connection_id) { erts_de_rwlock(dep); if (connection_id == dep->connection_id - && dep->status == ERTS_DE_SFLG_CONNECTED) { + && dep->state == ERTS_DE_STATE_CONNECTED) { kill_connection(dep); } @@ -3341,7 +3361,7 @@ BIF_RETTYPE setnode_3(BIF_ALIST_3) goto badarg; } - if (dep->status & ERTS_DE_SFLG_EXITING) { + if (dep->state == ERTS_DE_STATE_EXITING) { /* Suspend on dist entry waiting for the exit to finish */ ErtsProcList *plp = erts_proclist_create(BIF_P); plp->next = NULL; @@ -3351,8 +3371,8 @@ BIF_RETTYPE setnode_3(BIF_ALIST_3) erts_mtx_unlock(&dep->qlock); goto yield; } - if (dep->status != ERTS_DE_SFLG_PENDING) { - if (dep->status == 0) + if (dep->state != ERTS_DE_STATE_PENDING) { + if (dep->state == ERTS_DE_STATE_IDLE) erts_set_dist_entry_pending(dep); else goto badarg; @@ -3390,7 +3410,7 @@ BIF_RETTYPE setnode_3(BIF_ALIST_3) goto done; /* Already set */ } - if (dep->status & ERTS_DE_SFLG_EXITING) { + if (dep->state == ERTS_DE_STATE_EXITING) { /* Suspend on dist entry waiting for the exit to finish */ ErtsProcList *plp = erts_proclist_create(BIF_P); plp->next = NULL; @@ -3400,8 +3420,8 @@ BIF_RETTYPE setnode_3(BIF_ALIST_3) erts_mtx_unlock(&dep->qlock); goto yield; } - if (dep->status != ERTS_DE_SFLG_PENDING) { - if (dep->status == 0) + if (dep->state != ERTS_DE_STATE_PENDING) { + if (dep->state == ERTS_DE_STATE_IDLE) erts_set_dist_entry_pending(dep); else goto badarg; @@ -3437,7 +3457,7 @@ BIF_RETTYPE setnode_3(BIF_ALIST_3) #ifdef DEBUG ASSERT(erts_atomic_read_nob(&dep->qsize) == 0 - || (dep->status & ERTS_DE_SFLG_PENDING)); + || (dep->state == ERTS_DE_STATE_PENDING)); #endif if (flags & DFLAG_DIST_HDR_ATOM_CACHE) @@ -3506,6 +3526,11 @@ BIF_RETTYPE setnode_3(BIF_ALIST_3) goto done; } +BIF_RETTYPE erts_internal_get_dflags_0(BIF_ALIST_0) +{ + return erts_dflags_record; +} + BIF_RETTYPE erts_internal_new_connection_1(BIF_ALIST_1) { DistEntry* dep; @@ -3525,15 +3550,20 @@ BIF_RETTYPE erts_internal_new_connection_1(BIF_ALIST_1) erts_de_rwlock(dep); - if (ERTS_DE_IS_CONNECTED(dep) || dep->status & ERTS_DE_SFLG_PENDING) + switch (dep->state) { + case ERTS_DE_STATE_PENDING: + case ERTS_DE_STATE_CONNECTED: conn_id = dep->connection_id; - else if (dep->status == 0) { + break; + case ERTS_DE_STATE_IDLE: erts_set_dist_entry_pending(dep); conn_id = dep->connection_id; - } - else { - ASSERT(dep->status & ERTS_DE_SFLG_EXITING); + break; + case ERTS_DE_STATE_EXITING: conn_id = (dep->connection_id + 1) & ERTS_DIST_CON_ID_MASK; + break; + default: + erts_exit(ERTS_ABORT_EXIT, "Invalid dep->state (%d)\n", dep->state); } erts_de_rwunlock(dep); hp = HAlloc(BIF_P, 3 + ERTS_MAGIC_REF_THING_SIZE); @@ -3548,10 +3578,10 @@ static Sint abort_connection(DistEntry* dep, Uint32 conn_id) if (dep->connection_id != conn_id) ; - else if (dep->status == ERTS_DE_SFLG_CONNECTED) { + else if (dep->state == ERTS_DE_STATE_CONNECTED) { kill_connection(dep); } - else if (dep->status == ERTS_DE_SFLG_PENDING) { + else if (dep->state == ERTS_DE_STATE_PENDING) { NetExitsContext nec = {dep}; ErtsLink *nlinks; ErtsLink *node_links; @@ -3600,6 +3630,17 @@ static Sint abort_connection(DistEntry* dep, Uint32 conn_id) delete_cache(cache); free_de_out_queues(dep, obuf); + + /* + * We wait to make DistEntry idle and accept new connection attempts + * until all is cleared and deallocated. This to get some back pressure + * against repeated failing connection attempts saturating all CPUs + * with cleanup jobs. + */ + erts_de_rwlock(dep); + ASSERT(dep->state == ERTS_DE_STATE_EXITING); + dep->state = ERTS_DE_STATE_IDLE; + erts_de_rwunlock(dep); return reds; } erts_de_rwunlock(dep); @@ -3631,7 +3672,7 @@ BIF_RETTYPE erts_internal_abort_connection_2(BIF_ALIST_2) int erts_auto_connect(DistEntry* dep, Process *proc, ErtsProcLocks proc_locks) { erts_de_rwlock(dep); - if (dep->status != 0) { + if (dep->state != ERTS_DE_STATE_IDLE) { erts_de_rwunlock(dep); } else { @@ -3907,7 +3948,8 @@ monitor_node(Process* p, Eterm Node, Eterm Bool, Eterm Options) erts_proc_lock(p, ERTS_PROC_LOCK_LINK); erts_de_rlock(dep); - if (!(dep->status & (ERTS_DE_SFLG_PENDING | ERTS_DE_SFLG_CONNECTED))) { + if (dep->state == ERTS_DE_STATE_IDLE) { + ASSERT(!dep->node_links); erts_proc_unlock(p, ERTS_PROC_LOCK_LINK); erts_de_runlock(dep); goto done; diff --git a/erts/emulator/beam/dist.h b/erts/emulator/beam/dist.h index ea4697815f..b1b7ce9c78 100644 --- a/erts/emulator/beam/dist.h +++ b/erts/emulator/beam/dist.h @@ -40,26 +40,52 @@ #define DFLAG_UNICODE_IO 0x1000 #define DFLAG_DIST_HDR_ATOM_CACHE 0x2000 #define DFLAG_SMALL_ATOM_TAGS 0x4000 -#define DFLAG_INTERNAL_TAGS 0x8000 +#define DFLAG_INTERNAL_TAGS 0x8000 /* used by ETS 'compressed' option */ #define DFLAG_UTF8_ATOMS 0x10000 #define DFLAG_MAP_TAG 0x20000 #define DFLAG_BIG_CREATION 0x40000 #define DFLAG_SEND_SENDER 0x80000 -#define DFLAG_NO_MAGIC 0x100000 +#define DFLAG_NO_MAGIC 0x100000 /* internal for pending connection */ -/* Mandatory flags for distribution (sync with dist_util.erl) */ +/* Mandatory flags for distribution */ #define DFLAG_DIST_MANDATORY (DFLAG_EXTENDED_REFERENCES \ | DFLAG_EXTENDED_PIDS_PORTS \ | DFLAG_UTF8_ATOMS \ | DFLAG_NEW_FUN_TAGS) -/* Additional optimistic flags when encoding toward pending connection */ -#define DFLAG_DIST_HOPEFULLY (DFLAG_NO_MAGIC \ - | DFLAG_EXPORT_PTR_TAG \ +/* + * Additional optimistic flags when encoding toward pending connection. + * If remote node (erl_interface) does not supporting these then we may need + * to transcode messages enqueued before connection setup was finished. + */ +#define DFLAG_DIST_HOPEFULLY (DFLAG_EXPORT_PTR_TAG \ | DFLAG_BIT_BINARIES \ | DFLAG_DIST_MONITOR \ | DFLAG_DIST_MONITOR_NAME) +/* Our preferred set of flags. Used for connection setup handshake */ +#define DFLAG_DIST_DEFAULT (DFLAG_DIST_MANDATORY | DFLAG_DIST_HOPEFULLY \ + | DFLAG_FUN_TAGS \ + | DFLAG_NEW_FLOATS \ + | DFLAG_UNICODE_IO \ + | DFLAG_DIST_HDR_ATOM_CACHE \ + | DFLAG_SMALL_ATOM_TAGS \ + | DFLAG_UTF8_ATOMS \ + | DFLAG_MAP_TAG \ + | DFLAG_BIG_CREATION \ + | DFLAG_SEND_SENDER) + +/* Flags addable by local distr implementations */ +#define DFLAG_DIST_ADDABLE DFLAG_DIST_DEFAULT + +/* Flags rejectable by local distr implementation */ +#define DFLAG_DIST_REJECTABLE (DFLAG_DIST_HDR_ATOM_CACHE \ + | DFLAG_HIDDEN_ATOM_CACHE \ + | DFLAG_ATOM_CACHE) + +/* Flags for all features needing strict order delivery */ +#define DFLAG_DIST_STRICT_ORDER DFLAG_DIST_HDR_ATOM_CACHE + /* All flags that should be enabled when term_to_binary/1 is used. */ #define TERM_TO_BINARY_DFLAGS (DFLAG_EXTENDED_REFERENCES \ | DFLAG_NEW_FUN_TAGS \ @@ -111,14 +137,6 @@ typedef struct { int no_suspend; } ErtsDSigData; -#define ERTS_DE_IS_NOT_CONNECTED(DEP) \ - (ERTS_LC_ASSERT(erts_lc_rwmtx_is_rlocked(&(DEP)->rwmtx) \ - || erts_lc_rwmtx_is_rwlocked(&(DEP)->rwmtx)), \ - (is_nil((DEP)->cid) || ((DEP)->status & ERTS_DE_SFLG_EXITING))) - -#define ERTS_DE_IS_CONNECTED(DEP) \ - (!ERTS_DE_IS_NOT_CONNECTED((DEP))) - #define ERTS_DE_BUSY_LIMIT (1024*1024) extern int erts_dist_buf_busy_limit; extern int erts_is_alive; @@ -181,18 +199,18 @@ erts_dsig_prepare(ErtsDSigData *dsdp, retry: erts_de_rlock(dep); - if (ERTS_DE_IS_CONNECTED(dep)) { + if (dep->state == ERTS_DE_STATE_CONNECTED) { res = ERTS_DSIG_PREP_CONNECTED; } - else if (dep->status & ERTS_DE_SFLG_PENDING) { + else if (dep->state == ERTS_DE_STATE_PENDING) { res = ERTS_DSIG_PREP_PENDING; } - else if (dep->status & ERTS_DE_SFLG_EXITING) { + else if (dep->state == ERTS_DE_STATE_EXITING) { res = ERTS_DSIG_PREP_NOT_CONNECTED; goto fail; } else if (connect) { - ASSERT(dep->status == 0); + ASSERT(dep->state == ERTS_DE_STATE_IDLE); erts_de_runlock(dep); if (!erts_auto_connect(dep, proc, proc_locks)) { return ERTS_DSIG_PREP_NOT_ALIVE; @@ -200,7 +218,7 @@ retry: goto retry; } else { - ASSERT(dep->status == 0); + ASSERT(dep->state == ERTS_DE_STATE_IDLE); res = ERTS_DSIG_PREP_NOT_CONNECTED; goto fail; } @@ -328,6 +346,7 @@ typedef struct TTBSizeContext_ { typedef struct TTBEncodeContext_ { Uint flags; + Uint hopefull_flags; int level; byte* ep; Eterm obj; diff --git a/erts/emulator/beam/erl_afit_alloc.c b/erts/emulator/beam/erl_afit_alloc.c index 4ebe37ee1d..23efe3bba4 100644 --- a/erts/emulator/beam/erl_afit_alloc.c +++ b/erts/emulator/beam/erl_afit_alloc.c @@ -181,7 +181,7 @@ static struct { static void ERTS_INLINE atom_init(Eterm *atom, char *name) { - *atom = am_atom_put(name, strlen(name)); + *atom = am_atom_put(name, sys_strlen(name)); } #define AM_INIT(AM) atom_init(&am.AM, #AM) diff --git a/erts/emulator/beam/erl_alloc.c b/erts/emulator/beam/erl_alloc.c index 88285d8be6..f7f5506a72 100644 --- a/erts/emulator/beam/erl_alloc.c +++ b/erts/emulator/beam/erl_alloc.c @@ -160,7 +160,7 @@ enum allctr_type { GOODFIT, BESTFIT, AFIT, - AOFIRSTFIT + FIRSTFIT }; struct au_init { @@ -366,16 +366,9 @@ set_default_exec_alloc_opts(struct au_init *ip) ip->init.util.rmbcmt = 0; ip->init.util.acul = 0; -# ifdef ERTS_HAVE_EXEC_MMAPPER - ip->init.util.mseg_alloc = &erts_alcu_mmapper_mseg_alloc; - ip->init.util.mseg_realloc = &erts_alcu_mmapper_mseg_realloc; - ip->init.util.mseg_dealloc = &erts_alcu_mmapper_mseg_dealloc; - ip->init.util.mseg_mmapper = &erts_exec_mmapper; -# else ip->init.util.mseg_alloc = &erts_alcu_exec_mseg_alloc; ip->init.util.mseg_realloc = &erts_alcu_exec_mseg_realloc; ip->init.util.mseg_dealloc = &erts_alcu_exec_mseg_dealloc; -# endif } #endif /* ERTS_ALC_A_EXEC */ @@ -500,8 +493,9 @@ set_default_test_alloc_opts(struct au_init *ip) SET_DEFAULT_ALLOC_OPTS(ip); ip->enable = 0; /* Disabled by default */ ip->thr_spec = -1 * erts_no_schedulers; - ip->atype = AOFIRSTFIT; - ip->init.aoff.flavor = AOFF_BF; + ip->atype = FIRSTFIT; + ip->init.aoff.crr_order = FF_AOFF; + ip->init.aoff.blk_order = FF_BF; ip->init.util.name_prefix = "test_"; ip->init.util.alloc_no = ERTS_ALC_A_TEST; ip->init.util.mmbcs = 0; /* Main carrier size */ @@ -599,10 +593,10 @@ static ERTS_INLINE int strategy_support_carrier_migration(struct au_init *auip) { /* - * Currently only aoff, aoffcbf and aoffcaobf support carrier + * Currently only aoff* and ageff* support carrier * migration, i.e, type AOFIRSTFIT. */ - return auip->atype == AOFIRSTFIT; + return auip->atype == FIRSTFIT; } static ERTS_INLINE void @@ -617,8 +611,9 @@ adjust_carrier_migration_support(struct au_init *auip) */ if (!strategy_support_carrier_migration(auip)) { /* Default to aoffcbf */ - auip->atype = AOFIRSTFIT; - auip->init.aoff.flavor = AOFF_BF; + auip->atype = FIRSTFIT; + auip->init.aoff.crr_order = FF_AOFF; + auip->init.aoff.blk_order = FF_BF; } } } @@ -1132,7 +1127,7 @@ start_au_allocator(ErtsAlcType_t alctr_n, &init->init.af, &init->init.util); break; - case AOFIRSTFIT: + case FIRSTFIT: as = erts_aoffalc_start((AOFFAllctr_t *) as0, &init->init.aoff, &init->init.util); @@ -1217,31 +1212,41 @@ get_bool_value(char *param_end, char** argv, int* ip) { char *param = argv[*ip]+1; char *value = get_value(param_end, argv, ip); - if (strcmp(value, "true") == 0) + if (sys_strcmp(value, "true") == 0) return 1; - else if (strcmp(value, "false") == 0) + else if (sys_strcmp(value, "false") == 0) return 0; else bad_value(param, param_end, value); return -1; } +static Uint kb_to_bytes(Sint kb, Uint *bytes) +{ + const Uint max = ((~((Uint) 0))/1024) + 1; + + if (kb < 0 || (Uint)kb > max) + return 0; + if ((Uint)kb == max) + *bytes = ~((Uint) 0); + else + *bytes = ((Uint) kb)*1024; + return 1; +} + static Uint get_kb_value(char *param_end, char** argv, int* ip) { Sint tmp; - Uint max = ((~((Uint) 0))/1024) + 1; + Uint bytes = 0; char *rest; char *param = argv[*ip]+1; char *value = get_value(param_end, argv, ip); errno = 0; tmp = (Sint) ErtsStrToSint(value, &rest, 10); - if (errno != 0 || rest == value || tmp < 0 || max < ((Uint) tmp)) + if (errno != 0 || rest == value || !kb_to_bytes(tmp, &bytes)) bad_value(param, param_end, value); - if (max == (Uint) tmp) - return ~((Uint) 0); - else - return ((Uint) tmp)*1024; + return bytes; } static UWord @@ -1328,49 +1333,79 @@ handle_au_arg(struct au_init *auip, switch (sub_param[0]) { case 'a': - if (has_prefix("acul", sub_param)) { - if (!auip->carrier_migration_allowed) { - if (!u_switch) - goto bad_switch; - else { - /* ignore */ - (void) get_acul_value(auip, sub_param + 4, argv, ip); - break; - } - } - auip->init.util.acul = get_acul_value(auip, sub_param + 4, argv, ip); - } + if (sub_param[1] == 'c') { /* Migration parameters "ac*" */ + UWord value; + UWord* wp; + if (!auip->carrier_migration_allowed && !u_switch) + goto bad_switch; + + if (has_prefix("acul", sub_param)) { + value = get_acul_value(auip, sub_param + 4, argv, ip); + wp = &auip->init.util.acul; + } + else if (has_prefix("acnl", sub_param)) { + value = get_amount_value(sub_param + 4, argv, ip); + wp = &auip->init.util.acnl; + } + else if (has_prefix("acfml", sub_param)) { + value = get_amount_value(sub_param + 5, argv, ip); + wp = &auip->init.util.acfml; + } + else + goto bad_switch; + + if (auip->carrier_migration_allowed) + *wp = value; + } else if(has_prefix("asbcst", sub_param)) { auip->init.util.asbcst = get_kb_value(sub_param + 6, argv, ip); } else if(has_prefix("as", sub_param)) { char *alg = get_value(sub_param + 2, argv, ip); - if (strcmp("bf", alg) == 0) { + if (sys_strcmp("bf", alg) == 0) { auip->atype = BESTFIT; auip->init.bf.ao = 0; } - else if (strcmp("aobf", alg) == 0) { + else if (sys_strcmp("aobf", alg) == 0) { auip->atype = BESTFIT; auip->init.bf.ao = 1; } - else if (strcmp("gf", alg) == 0) { + else if (sys_strcmp("gf", alg) == 0) { auip->atype = GOODFIT; } - else if (strcmp("af", alg) == 0) { + else if (sys_strcmp("af", alg) == 0) { auip->atype = AFIT; } - else if (strcmp("aoff", alg) == 0) { - auip->atype = AOFIRSTFIT; - auip->init.aoff.flavor = AOFF_AOFF; + else if (sys_strcmp("aoff", alg) == 0) { + auip->atype = FIRSTFIT; + auip->init.aoff.crr_order = FF_AOFF; + auip->init.aoff.blk_order = FF_AOFF; } - else if (strcmp("aoffcbf", alg) == 0) { - auip->atype = AOFIRSTFIT; - auip->init.aoff.flavor = AOFF_BF; + else if (sys_strcmp("aoffcbf", alg) == 0) { + auip->atype = FIRSTFIT; + auip->init.aoff.crr_order = FF_AOFF; + auip->init.aoff.blk_order = FF_BF; } - else if (strcmp("aoffcaobf", alg) == 0) { - auip->atype = AOFIRSTFIT; - auip->init.aoff.flavor = AOFF_AOBF; + else if (sys_strcmp("aoffcaobf", alg) == 0) { + auip->atype = FIRSTFIT; + auip->init.aoff.crr_order = FF_AOFF; + auip->init.aoff.blk_order = FF_AOBF; } + else if (sys_strcmp("ageffcaoff", alg) == 0) { + auip->atype = FIRSTFIT; + auip->init.aoff.crr_order = FF_AGEFF; + auip->init.aoff.blk_order = FF_AOFF; + } + else if (sys_strcmp("ageffcbf", alg) == 0) { + auip->atype = FIRSTFIT; + auip->init.aoff.crr_order = FF_AGEFF; + auip->init.aoff.blk_order = FF_BF; + } + else if (sys_strcmp("ageffcaobf", alg) == 0) { + auip->atype = FIRSTFIT; + auip->init.aoff.crr_order = FF_AGEFF; + auip->init.aoff.blk_order = FF_AOBF; + } else { bad_value(param, sub_param + 1, alg); } @@ -1526,10 +1561,8 @@ handle_args(int *argc, char **argv, erts_alc_hndl_args_init_t *init) break; case 'X': if (has_prefix("scs", argv[i]+3)) { -#ifdef ERTS_HAVE_EXEC_MMAPPER - init->mseg.exec_mmap.scs = -#endif - get_mb_value(argv[i]+6, argv, &i); + /* Ignore obsolete */ + (void) get_mb_value(argv[i]+6, argv, &i); } else handle_au_arg(&init->exec_alloc, &argv[i][3], argv, &i, 0); @@ -1646,7 +1679,7 @@ handle_args(int *argc, char **argv, erts_alc_hndl_args_init_t *init) } else if (has_prefix("e", param+2)) { arg = get_value(param+3, argv, &i); - if (strcmp("true", arg) != 0) + if (sys_strcmp("true", arg) != 0) bad_value(param, param+3, arg); } else @@ -1658,20 +1691,20 @@ handle_args(int *argc, char **argv, erts_alc_hndl_args_init_t *init) case 'a': { int a; arg = get_value(argv[i]+4, argv, &i); - if (strcmp("min", arg) == 0) { + if (sys_strcmp("min", arg) == 0) { for (a = 0; a < aui_sz; a++) aui[a]->enable = 0; } - else if (strcmp("max", arg) == 0) { + else if (sys_strcmp("max", arg) == 0) { for (a = 0; a < aui_sz; a++) aui[a]->enable = 1; } - else if (strcmp("config", arg) == 0) { + else if (sys_strcmp("config", arg) == 0) { init->erts_alloc_config = 1; } - else if (strcmp("r9c", arg) == 0 - || strcmp("r10b", arg) == 0 - || strcmp("r11b", arg) == 0) { + else if (sys_strcmp("r9c", arg) == 0 + || sys_strcmp("r10b", arg) == 0 + || sys_strcmp("r11b", arg) == 0) { set_default_sl_alloc_opts(&init->sl_alloc); set_default_std_alloc_opts(&init->std_alloc); set_default_ll_alloc_opts(&init->ll_alloc); @@ -1683,7 +1716,7 @@ handle_args(int *argc, char **argv, erts_alc_hndl_args_init_t *init) set_default_driver_alloc_opts(&init->fix_alloc); init->driver_alloc.enable = 0; - if (strcmp("r9c", arg) == 0) { + if (sys_strcmp("r9c", arg) == 0) { init->sl_alloc.enable = 0; init->std_alloc.enable = 0; init->binary_alloc.enable = 0; @@ -1710,18 +1743,18 @@ handle_args(int *argc, char **argv, erts_alc_hndl_args_init_t *init) switch (argv[i][3]) { case 's': arg = get_value(argv[i]+4, argv, &i); - if (strcmp("true", arg) == 0) + if (sys_strcmp("true", arg) == 0) init->instr.stat = 1; - else if (strcmp("false", arg) == 0) + else if (sys_strcmp("false", arg) == 0) init->instr.stat = 0; else bad_value(param, param+3, arg); break; case 'm': arg = get_value(argv[i]+4, argv, &i); - if (strcmp("true", arg) == 0) + if (sys_strcmp("true", arg) == 0) init->instr.map = 1; - else if (strcmp("false", arg) == 0) + else if (sys_strcmp("false", arg) == 0) init->instr.map = 0; else bad_value(param, param+3, arg); @@ -1736,9 +1769,9 @@ handle_args(int *argc, char **argv, erts_alc_hndl_args_init_t *init) case 'l': if (has_prefix("pm", param+2)) { arg = get_value(argv[i]+5, argv, &i); - if (strcmp("all", arg) == 0) + if (sys_strcmp("all", arg) == 0) lock_all_physical_memory = 1; - else if (strcmp("no", arg) == 0) + else if (sys_strcmp("no", arg) == 0) lock_all_physical_memory = 0; else bad_value(param, param+4, arg); @@ -1788,8 +1821,8 @@ handle_args(int *argc, char **argv, erts_alc_hndl_args_init_t *init) /* || init->instr.stat || init->instr.map */) { while (i < *argc) { - if(strcmp(argv[i], "-sname") == 0 - || strcmp(argv[i], "-name") == 0) { + if(sys_strcmp(argv[i], "-sname") == 0 + || sys_strcmp(argv[i], "-name") == 0) { if (i + 1 <*argc) { init->instr.nodename = argv[i+1]; break; @@ -2650,7 +2683,7 @@ erts_allocated_areas(fmtfn_t *print_to_p, void *print_to_arg, void *proc) Eterm atom; if (hpp) atom = am_atom_put(values[i].name, - (int) strlen(values[i].name)); + (int) sys_strlen(values[i].name)); else atom = am_true; @@ -2791,10 +2824,6 @@ erts_allocator_info(fmtfn_t to, void *arg) erts_print(to, arg, "=allocator:erts_mmap.literal_mmap\n"); erts_mmap_info(&erts_literal_mmapper, &to, arg, NULL, NULL, &emis); #endif -#ifdef ERTS_HAVE_EXEC_MMAPPER - erts_print(to, arg, "=allocator:erts_mmap.exec_mmap\n"); - erts_mmap_info(&erts_exec_mmapper, &to, arg, NULL, NULL, &emis); -#endif } #endif @@ -2841,7 +2870,7 @@ erts_allocator_options(void *proc) for (a = ERTS_ALC_A_MIN; a <= ERTS_ALC_A_MAX; a++) { Eterm tmp = NIL; atoms[length] = am_atom_put((char *) ERTS_ALC_A2AD(a), - strlen(ERTS_ALC_A2AD(a))); + sys_strlen(ERTS_ALC_A2AD(a))); if (erts_allctrs_info[a].enabled) { if (erts_allctrs_info[a].alloc_util) { Allctr_t *allctr; @@ -2935,7 +2964,7 @@ erts_allocator_options(void *proc) for (a = ERTS_ALC_A_MIN; a <= ERTS_ALC_A_MAX; a++) { if (erts_allctrs_info[a].enabled) { terms[length++] = am_atom_put((char *) ERTS_ALC_A2AD(a), - strlen(ERTS_ALC_A2AD(a))); + sys_strlen(ERTS_ALC_A2AD(a))); } } @@ -2949,9 +2978,6 @@ erts_allocator_options(void *proc) #if defined(ARCH_64) && defined(ERTS_HAVE_OS_PHYSICAL_MEMORY_RESERVATION) terms[length++] = ERTS_MAKE_AM("literal_mmap"); #endif -#ifdef ERTS_HAVE_EXEC_MMAPPER - terms[length++] = ERTS_MAKE_AM("exec_mmap"); -#endif features = length ? erts_bld_list(hpp, szp, length, terms) : NIL; #if defined(__GLIBC__) @@ -3041,9 +3067,6 @@ reply_alloc_info(void *vair) # if defined(ARCH_64) && defined(ERTS_HAVE_OS_PHYSICAL_MEMORY_RESERVATION) struct erts_mmap_info_struct mmap_info_literal; # endif -# ifdef ERTS_HAVE_EXEC_MMAPPER - struct erts_mmap_info_struct mmap_info_exec; -# endif #endif int i; Eterm (*info_func)(Allctr_t *, @@ -3172,17 +3195,6 @@ reply_alloc_info(void *vair) erts_bld_atom(hpp,szp,"literal_mmap"), ainfo); # endif -# ifdef ERTS_HAVE_EXEC_MMAPPER - ai_list = erts_bld_cons(hpp, szp, - ainfo, ai_list); - ainfo = (air->only_sz ? NIL : - erts_mmap_info(&erts_exec_mmapper, NULL, NULL, - hpp, szp, &mmap_info_exec)); - ainfo = erts_bld_tuple3(hpp, szp, - alloc_atom, - erts_bld_atom(hpp,szp,"exec_mmap"), - ainfo); -# endif #else /* !HAVE_ERTS_MMAP */ ainfo = erts_bld_tuple2(hpp, szp, alloc_atom, am_false); @@ -3392,6 +3404,65 @@ erts_request_alloc_info(struct process *c_p, return 1; } +Eterm erts_alloc_set_dyn_param(Process* c_p, Eterm tuple) +{ + ErtsAllocatorThrSpec_t *tspec; + ErtsAlcType_t ai; + Allctr_t* allctr; + Eterm* tp; + Eterm res; + + if (!is_tuple_arity(tuple, 3)) + goto badarg; + + tp = tuple_val(tuple); + + /* + * Ex: {ets_alloc, sbct, 256000} + */ + if (!is_atom(tp[1]) || !is_atom(tp[2]) || !is_integer(tp[3])) + goto badarg; + + for (ai = ERTS_ALC_A_MIN; ai <= ERTS_ALC_A_MAX; ai++) + if (erts_is_atom_str(erts_alc_a2ad[ai], tp[1], 0)) + break; + + if (ai > ERTS_ALC_A_MAX) + goto badarg; + + if (!erts_allctrs_info[ai].enabled || + !erts_allctrs_info[ai].alloc_util) { + return am_notsup; + } + + if (tp[2] == am_sbct) { + Uint sbct; + int i, ok; + + if (!term_to_Uint(tp[3], &sbct)) + goto badarg; + + tspec = &erts_allctr_thr_spec[ai]; + if (tspec->enabled) { + ok = 0; + for (i = 0; i < tspec->size; i++) { + allctr = tspec->allctr[i]; + ok |= allctr->try_set_dyn_param(allctr, am_sbct, sbct); + } + } + else { + allctr = erts_allctrs_info[ai].extra; + ok = allctr->try_set_dyn_param(allctr, am_sbct, sbct); + } + return ok ? am_ok : am_notsup; + } + return am_notsup; + +badarg: + ERTS_BIF_PREP_ERROR(res, c_p, EXC_BADARG); + return res; +} + /* * The allocator wrapper prelocking stuff below is about the locking order. * It only affects wrappers (erl_mtrace.c and erl_instrument.c) that keep locks @@ -3528,7 +3599,7 @@ UWord erts_alc_test(UWord op, UWord a1, UWord a2, UWord a3) &init.init.af, &init.init.util); break; - case AOFIRSTFIT: + case FIRSTFIT: allctr = erts_aoffalc_start((AOFFAllctr_t *) erts_alloc(ERTS_ALC_T_UNDEF, sizeof(AOFFAllctr_t)), @@ -3622,7 +3693,9 @@ UWord erts_alc_test(UWord op, UWord a1, UWord a2, UWord a3) case 0xf15: erts_free(ERTS_ALC_T_TEST, (void*)a1); return 0; - case 0xf16: { + case 0xf16: return (UWord) erts_realloc(ERTS_ALC_T_TEST, (void*)a1, (Uint)a2); + + case 0xf17: { Uint extra_hdr_sz = UNIT_CEILING((Uint)a1); ErtsAllocatorThrSpec_t* ts = &erts_allctr_thr_spec[ERTS_ALC_A_TEST]; Uint offset = ts->allctr[0]->mbc_header_size; @@ -3649,7 +3722,7 @@ UWord erts_alc_test(UWord op, UWord a1, UWord a2, UWord a3) *(void**)a3 = orig_destroying_mbc; return offset; } - case 0xf17: { + case 0xf18: { ErtsAllocatorThrSpec_t* ts = &erts_allctr_thr_spec[ERTS_ALC_A_TEST]; return ts->allctr[0]->largest_mbc_size; } @@ -3858,7 +3931,7 @@ set_memory_fence(void *ptr, Uint sz, ErtsAlcType_t n) *(ui_ptr++) = sz; *(ui_ptr++) = pattern; - memcpy((void *) (((char *) ui_ptr)+sz), (void *) &pattern, sizeof(UWord)); + sys_memcpy((void *) (((char *) ui_ptr)+sz), (void *) &pattern, sizeof(UWord)); #ifdef HARD_DEBUG *mblkpp = hdbg_alloc((void *) ui_ptr, sz, n); @@ -3898,7 +3971,7 @@ check_memory_fence(void *ptr, Uint *size, ErtsAlcType_t n, int func) (UWord) ptr); } - memcpy((void *) &post_pattern, (void *) (((char *)ptr)+sz), sizeof(UWord)); + sys_memcpy((void *) &post_pattern, (void *) (((char *)ptr)+sz), sizeof(UWord)); if (post_pattern != MK_PATTERN(n) || pre_pattern != post_pattern) { diff --git a/erts/emulator/beam/erl_alloc.h b/erts/emulator/beam/erl_alloc.h index 117f96a4ad..578a3717d9 100644 --- a/erts/emulator/beam/erl_alloc.h +++ b/erts/emulator/beam/erl_alloc.h @@ -126,8 +126,10 @@ typedef struct { void *extra; } ErtsAllocatorFunctions_t; -extern ErtsAllocatorFunctions_t erts_allctrs[ERTS_ALC_A_MAX+1]; -extern ErtsAllocatorInfo_t erts_allctrs_info[ERTS_ALC_A_MAX+1]; +extern ErtsAllocatorFunctions_t + ERTS_WRITE_UNLIKELY(erts_allctrs[ERTS_ALC_A_MAX+1]); +extern ErtsAllocatorInfo_t + ERTS_WRITE_UNLIKELY(erts_allctrs_info[ERTS_ALC_A_MAX+1]); typedef struct { int enabled; @@ -144,7 +146,7 @@ typedef struct ErtsAllocatorWrapper_t_ { void (*unlock)(void); struct ErtsAllocatorWrapper_t_* next; }ErtsAllocatorWrapper_t; -ErtsAllocatorWrapper_t *erts_allctr_wrappers; +extern ErtsAllocatorWrapper_t *erts_allctr_wrappers; extern int erts_allctr_wrapper_prelocked; extern erts_tsd_key_t erts_allctr_prelock_tsd_key; void erts_allctr_wrapper_prelock_init(ErtsAllocatorWrapper_t* wrapper); @@ -169,6 +171,8 @@ __decl_noreturn void erts_realloc_n_enomem(ErtsAlcType_t,void*,Uint) __decl_noreturn void erts_alc_fatal_error(int,int,ErtsAlcType_t,...) __noreturn; +Eterm erts_alloc_set_dyn_param(struct process*, Eterm); + #undef ERTS_HAVE_IS_IN_LITERAL_RANGE #if defined(ARCH_32) || defined(ERTS_HAVE_OS_PHYSICAL_MEMORY_RESERVATION) # define ERTS_HAVE_IS_IN_LITERAL_RANGE @@ -430,7 +434,7 @@ NAME##_free(TYPE *p) \ #ifdef DEBUG #define ERTS_PRE_ALLOC_SIZE(SZ) ((SZ) < 1000 ? (SZ)/10 + 10 : 100) -#define ERTS_PRE_ALLOC_CLOBBER(P, T) memset((void *) (P), 0xfd, sizeof(T)) +#define ERTS_PRE_ALLOC_CLOBBER(P, T) sys_memset((void *) (P), 0xfd, sizeof(T)) #else #define ERTS_PRE_ALLOC_SIZE(SZ) ((SZ) > 1 ? (SZ) : 1) #define ERTS_PRE_ALLOC_CLOBBER(P, T) diff --git a/erts/emulator/beam/erl_alloc_util.c b/erts/emulator/beam/erl_alloc_util.c index 4d4bddb93f..e148be7af6 100644 --- a/erts/emulator/beam/erl_alloc_util.c +++ b/erts/emulator/beam/erl_alloc_util.c @@ -362,8 +362,10 @@ do { \ #define ERTS_CRR_ALCTR_FLG_IN_POOL (((erts_aint_t) 1) << 0) #define ERTS_CRR_ALCTR_FLG_BUSY (((erts_aint_t) 1) << 1) +#define ERTS_CRR_ALCTR_FLG_HOMECOMING (((erts_aint_t) 1) << 2) #define ERTS_CRR_ALCTR_FLG_MASK (ERTS_CRR_ALCTR_FLG_IN_POOL | \ - ERTS_CRR_ALCTR_FLG_BUSY) + ERTS_CRR_ALCTR_FLG_BUSY | \ + ERTS_CRR_ALCTR_FLG_HOMECOMING) #define SBC_HEADER_SIZE \ (UNIT_CEILING(offsetof(Carrier_t, cpool) \ @@ -563,7 +565,7 @@ do { \ DEBUG_CHECK_CARRIER_NO_SZ((AP)); \ } while (0) -#define STAT_MBC_CPOOL_INSERT(AP, CRR) \ +#define STAT_MBC_ABANDON(AP, CRR) \ do { \ UWord csz__ = CARRIER_SZ((CRR)); \ if (IS_MSEG_CARRIER((CRR))) \ @@ -872,7 +874,7 @@ erts_alcu_literal_32_mseg_dealloc(Allctr_t *allctr, void *seg, Uint size, #elif defined(ARCH_64) && defined(ERTS_HAVE_OS_PHYSICAL_MEMORY_RESERVATION) /* For allocators that have their own mmapper (super carrier), - * like literal_alloc and exec_alloc on amd64 + * like literal_alloc. */ void* erts_alcu_mmapper_mseg_alloc(Allctr_t *allctr, Uint *size_p, Uint flags) @@ -914,10 +916,10 @@ erts_alcu_mmapper_mseg_dealloc(Allctr_t *allctr, void *seg, Uint size, } #endif /* ARCH_64 && ERTS_HAVE_OS_PHYSICAL_MEMORY_RESERVATION */ -#if defined(ERTS_ALC_A_EXEC) && !defined(ERTS_HAVE_EXEC_MMAPPER) +#if defined(ERTS_ALC_A_EXEC) /* - * For exec_alloc on non-amd64 that just need memory with PROT_EXEC + * For exec_alloc that need memory with PROT_EXEC */ void* erts_alcu_exec_mseg_alloc(Allctr_t *allctr, Uint *size_p, Uint flags) @@ -956,7 +958,7 @@ erts_alcu_exec_mseg_dealloc(Allctr_t *allctr, void *seg, Uint size, Uint flags) ASSERT(r == 0); (void)r; erts_alcu_mseg_dealloc(allctr, seg, size, flags); } -#endif /* ERTS_ALC_A_EXEC && !ERTS_HAVE_EXEC_MMAPPER */ +#endif /* ERTS_ALC_A_EXEC */ #endif /* HAVE_ERTS_MSEG */ @@ -1153,89 +1155,23 @@ unlink_carrier(CarrierList_t *cl, Carrier_t *crr) ASSERT(crr->next); crr->next->prev = crr->prev; } -} - - #ifdef DEBUG -static int is_in_list(ErtsDoubleLink_t* sentinel, ErtsDoubleLink_t* node) -{ - ErtsDoubleLink_t* p; - - ASSERT(node != sentinel); - for (p = sentinel->next; p != sentinel; p = p->next) { - if (p == node) - return 1; - } - return 0; -} -#endif /* DEBUG */ - -static ERTS_INLINE void -link_edl_after(ErtsDoubleLink_t* after_me, ErtsDoubleLink_t* node) -{ - ErtsDoubleLink_t* before_me = after_me->next; - ASSERT(node != after_me && node != before_me); - node->next = before_me; - node->prev = after_me; - before_me->prev = node; - after_me->next = node; -} - -static ERTS_INLINE void -link_edl_before(ErtsDoubleLink_t* before_me, ErtsDoubleLink_t* node) -{ - ErtsDoubleLink_t* after_me = before_me->prev; - ASSERT(node != before_me && node != after_me); - node->next = before_me; - node->prev = after_me; - before_me->prev = node; - after_me->next = node; -} - -static ERTS_INLINE void -unlink_edl(ErtsDoubleLink_t* node) -{ - node->next->prev = node->prev; - node->prev->next = node->next; -} - -static ERTS_INLINE void -relink_edl_before(ErtsDoubleLink_t* before_me, ErtsDoubleLink_t* node) -{ - if (node != before_me && node != before_me->prev) { - unlink_edl(node); - link_edl_before(before_me, node); - } + crr->next = crr; + crr->prev = crr; +#endif } static ERTS_INLINE int is_abandoned(Carrier_t *crr) { - return crr->cpool.abandoned.next != NULL; -} - -static ERTS_INLINE void -link_abandoned_carrier(ErtsDoubleLink_t* list, Carrier_t *crr) -{ - ASSERT(!is_abandoned(crr)); - - link_edl_after(list, &crr->cpool.abandoned); - - ASSERT(crr->cpool.abandoned.next != &crr->cpool.abandoned); - ASSERT(crr->cpool.abandoned.prev != &crr->cpool.abandoned); + return crr->cpool.state != ERTS_MBC_IS_HOME; } static ERTS_INLINE void unlink_abandoned_carrier(Carrier_t *crr) { - ASSERT(is_in_list(&crr->cpool.orig_allctr->cpool.pooled_list, - &crr->cpool.abandoned) || - is_in_list(&crr->cpool.orig_allctr->cpool.traitor_list, - &crr->cpool.abandoned)); - - unlink_edl(&crr->cpool.abandoned); - - crr->cpool.abandoned.next = NULL; - crr->cpool.abandoned.prev = NULL; + if (crr->cpool.state == ERTS_MBC_WAS_POOLED) { + aoff_remove_pooled_mbc(crr->cpool.orig_allctr, crr); + } } static ERTS_INLINE void @@ -1243,24 +1179,19 @@ clear_busy_pool_carrier(Allctr_t *allctr, Carrier_t *crr) { if (crr) { erts_aint_t max_size; - erts_aint_t new_val; + erts_aint_t iallctr; max_size = (erts_aint_t) allctr->largest_fblk_in_mbc(allctr, crr); erts_atomic_set_nob(&crr->cpool.max_size, max_size); - new_val = (((erts_aint_t) allctr)|ERTS_CRR_ALCTR_FLG_IN_POOL); + iallctr = erts_atomic_read_nob(&crr->allctr); + ERTS_ALC_CPOOL_ASSERT((iallctr & ~ERTS_CRR_ALCTR_FLG_HOMECOMING) + == ((erts_aint_t)allctr | + ERTS_CRR_ALCTR_FLG_IN_POOL | + ERTS_CRR_ALCTR_FLG_BUSY)); -#ifdef ERTS_ALC_CPOOL_DEBUG - { - erts_aint_t old_val = new_val|ERTS_CRR_ALCTR_FLG_BUSY; - - ERTS_ALC_CPOOL_ASSERT(old_val - == erts_atomic_xchg_relb(&crr->allctr, - new_val)); - } -#else - erts_atomic_set_relb(&crr->allctr, new_val); -#endif + iallctr &= ~ERTS_CRR_ALCTR_FLG_BUSY; + erts_atomic_set_relb(&crr->allctr, iallctr); } } @@ -1658,6 +1589,11 @@ dealloc_mbc(Allctr_t *allctr, Carrier_t *crr) } +static void set_new_allctr_abandon_limit(Allctr_t*); +static void abandon_carrier(Allctr_t*, Carrier_t*); +static void poolify_my_carrier(Allctr_t*, Carrier_t*); +static void enqueue_homecoming(Allctr_t*, Carrier_t*); + static ERTS_INLINE Allctr_t* get_pref_allctr(void *extra) { @@ -1724,9 +1660,23 @@ get_used_allctr(Allctr_t *pref_allctr, int pref_lock, void *p, UWord *sizep, erts_aint_t act; ERTS_ALC_CPOOL_ASSERT(!(iallctr & ERTS_CRR_ALCTR_FLG_BUSY)); - act = erts_atomic_cmpxchg_ddrb(&crr->allctr, - iallctr|ERTS_CRR_ALCTR_FLG_BUSY, - iallctr); + if (iallctr & ERTS_CRR_ALCTR_FLG_HOMECOMING) { + /* + * This carrier has just been given back to us by writing + * to crr->allctr with a write barrier (see abandon_carrier). + * + * We need a mathing read barrier to guarantee a correct view + * of the carrier for deallocation work. + */ + act = erts_atomic_cmpxchg_rb(&crr->allctr, + iallctr|ERTS_CRR_ALCTR_FLG_BUSY, + iallctr); + } + else { + act = erts_atomic_cmpxchg_ddrb(&crr->allctr, + iallctr|ERTS_CRR_ALCTR_FLG_BUSY, + iallctr); + } if (act == iallctr) { *busy_pcrr_pp = crr; break; @@ -1742,13 +1692,6 @@ get_used_allctr(Allctr_t *pref_allctr, int pref_lock, void *p, UWord *sizep, erts_mtx_unlock(&pref_allctr->mutex); } } - - ERTS_ALC_CPOOL_ASSERT( - (((iallctr & ~ERTS_CRR_ALCTR_FLG_MASK) == (erts_aint_t) pref_allctr) - ? (((iallctr & ERTS_CRR_ALCTR_FLG_MASK) == ERTS_CRR_ALCTR_FLG_IN_POOL) - || ((iallctr & ERTS_CRR_ALCTR_FLG_MASK) == 0)) - : 1)); - return used_allctr; } } @@ -2000,9 +1943,9 @@ handle_delayed_fix_dealloc(Allctr_t *allctr, void *ptr) /* Carrier migrated; need to redirect block to new owner... */ int cinit = used_allctr->dd.ix - allctr->dd.ix; - ERTS_ALC_CPOOL_ASSERT(!busy_pcrr_p); + ERTS_ALC_CPOOL_ASSERT(!busy_pcrr_p); - DEC_CC(allctr->calls.this_free); + DEC_CC(allctr->calls.this_free); ((ErtsAllctrFixDDBlock_t *) ptr)->fix_type = type; if (ddq_enqueue(&used_allctr->dd.q, ptr, cinit)) @@ -2011,8 +1954,9 @@ handle_delayed_fix_dealloc(Allctr_t *allctr, void *ptr) } } -static void -schedule_dealloc_carrier(Allctr_t *allctr, Carrier_t *crr); +static void schedule_dealloc_carrier(Allctr_t*, Carrier_t*); +static void dealloc_my_carrier(Allctr_t*, Carrier_t*); + static ERTS_INLINE int handle_delayed_dealloc(Allctr_t *allctr, @@ -2074,39 +2018,61 @@ handle_delayed_dealloc(Allctr_t *allctr, res = 1; blk = UMEM2BLK(ptr); - if (IS_FREE_LAST_MBC_BLK(blk)) { + if (blk->bhdr == HOMECOMING_MBC_BLK_HDR) { /* * A multiblock carrier that previously has been migrated away - * from us and now is back to be deallocated. For more info - * see schedule_dealloc_carrier(). - * - * Note that we cannot use FBLK_TO_MBC(blk) since it - * data has been overwritten by the queue. + * from us, was sent back to us either because + * - it became empty and we need to deallocated it, or + * - it was inserted into the pool and we need to update our pooled_tree */ - Carrier_t *crr = FIRST_BLK_TO_MBC(allctr, blk); - - /* Restore word overwritten by the dd-queue as it will be read - * if this carrier is pulled from dc_list by cpool_fetch() - */ - ERTS_ALC_CPOOL_ASSERT(FBLK_TO_MBC(blk) != crr); - ERTS_CT_ASSERT(sizeof(ErtsAllctrDDBlock_t) == sizeof(void*)); -#ifdef MBC_ABLK_OFFSET_BITS - blk->u.carrier = crr; -#else - blk->carrier = crr; -#endif + Carrier_t *crr = ErtsContainerStruct(blk, Carrier_t, + cpool.homecoming_dd.blk); + Block_t* first_blk = MBC_TO_FIRST_BLK(allctr, crr); + erts_aint_t iallctr; ERTS_ALC_CPOOL_ASSERT(ERTS_ALC_IS_CPOOL_ENABLED(allctr)); ERTS_ALC_CPOOL_ASSERT(allctr == crr->cpool.orig_allctr); - ERTS_ALC_CPOOL_ASSERT(((erts_aint_t) allctr) - != (erts_atomic_read_nob(&crr->allctr) - & ~ERTS_CRR_ALCTR_FLG_MASK)); - - erts_atomic_set_nob(&crr->allctr, ((erts_aint_t) allctr)); - schedule_dealloc_carrier(allctr, crr); + iallctr = erts_atomic_read_nob(&crr->allctr); + ASSERT(iallctr & ERTS_CRR_ALCTR_FLG_HOMECOMING); + while (1) { + if ((iallctr & (~ERTS_CRR_ALCTR_FLG_MASK | + ERTS_CRR_ALCTR_FLG_IN_POOL)) + == (erts_aint_t)allctr) { + /* + * Carrier is home (mine and not in pool) + */ + ASSERT(!(iallctr & ERTS_CRR_ALCTR_FLG_BUSY)); + erts_atomic_set_nob(&crr->allctr, (erts_aint_t)allctr); + if (IS_FREE_LAST_MBC_BLK(first_blk)) + dealloc_my_carrier(allctr, crr); + else + ASSERT(crr->cpool.state == ERTS_MBC_IS_HOME); + } + else { + erts_aint_t exp = iallctr; + erts_aint_t want = iallctr & ~ERTS_CRR_ALCTR_FLG_HOMECOMING; + + iallctr = erts_atomic_cmpxchg_nob(&crr->allctr, + want, + exp); + if (iallctr != exp) + continue; /* retry */ + + ASSERT(crr->cpool.state != ERTS_MBC_IS_HOME); + unlink_abandoned_carrier(crr); + if (iallctr & ERTS_CRR_ALCTR_FLG_IN_POOL) + poolify_my_carrier(allctr, crr); + else + crr->cpool.state = ERTS_MBC_WAS_TRAITOR; + } + break; + } } else { + ASSERT(IS_SBC_BLK(blk) || (ABLK_TO_MBC(blk) != + ErtsContainerStruct(blk, Carrier_t, + cpool.homecoming_dd.blk))); INC_CC(allctr->calls.this_free); @@ -2148,20 +2114,26 @@ enqueue_dealloc_other_instance(ErtsAlcType_t type, erts_alloc_notify_delayed_dealloc(allctr->ix); } - -static void -set_new_allctr_abandon_limit(Allctr_t *allctr); -static void -abandon_carrier(Allctr_t *allctr, Carrier_t *crr); - +static ERTS_INLINE void +update_pooled_tree(Allctr_t *allctr, Carrier_t *crr, Uint blk_sz) +{ + if (allctr == crr->cpool.orig_allctr && crr->cpool.state == ERTS_MBC_WAS_POOLED) { + /* + * Update pooled_tree with a potentially new (larger) max_sz + */ + AOFF_RBTree_t* crr_node = &crr->cpool.pooled; + if (blk_sz > crr_node->hdr.bhdr) { + crr_node->hdr.bhdr = blk_sz; + erts_aoff_larger_max_size(crr_node); + } + } +} static ERTS_INLINE void check_abandon_carrier(Allctr_t *allctr, Block_t *fblk, Carrier_t **busy_pcrr_pp) { Carrier_t *crr; - - if (busy_pcrr_pp && *busy_pcrr_pp) - return; + UWord ncrr_in_pool, largest_fblk; if (!ERTS_ALC_IS_CPOOL_ENABLED(allctr)) return; @@ -2170,8 +2142,7 @@ check_abandon_carrier(Allctr_t *allctr, Block_t *fblk, Carrier_t **busy_pcrr_pp) if (--allctr->cpool.check_limit_count <= 0) set_new_allctr_abandon_limit(allctr); - if (!erts_thr_progress_is_managed_thread()) - return; + ASSERT(erts_thr_progress_is_managed_thread()); if (allctr->cpool.disable_abandon) return; @@ -2179,6 +2150,9 @@ check_abandon_carrier(Allctr_t *allctr, Block_t *fblk, Carrier_t **busy_pcrr_pp) if (allctr->mbcs.blocks.curr.size > allctr->cpool.abandon_limit) return; + ncrr_in_pool = erts_atomic_read_nob(&allctr->cpool.stat.no_carriers); + if (ncrr_in_pool >= allctr->cpool.in_pool_limit) + return; crr = FBLK_TO_MBC(fblk); @@ -2189,9 +2163,14 @@ check_abandon_carrier(Allctr_t *allctr, Block_t *fblk, Carrier_t **busy_pcrr_pp) return; if (crr->cpool.thr_prgr != ERTS_THR_PRGR_INVALID - && !erts_thr_progress_has_reached(crr->cpool.thr_prgr)) - return; + && !erts_thr_progress_has_reached(crr->cpool.thr_prgr)) + return; + + largest_fblk = allctr->largest_fblk_in_mbc(allctr, crr); + if (largest_fblk < allctr->cpool.fblk_min_limit) + return; + erts_atomic_set_nob(&crr->cpool.max_size, largest_fblk); abandon_carrier(allctr, crr); } @@ -2237,6 +2216,7 @@ dealloc_block(Allctr_t *allctr, void *ptr, ErtsAlcFixList_t *fix, int dec_cc_on_ else { Carrier_t *busy_pcrr_p; Allctr_t *used_allctr; + used_allctr = get_used_allctr(allctr, ERTS_ALC_TS_PREF_LOCK_NO, ptr, NULL, &busy_pcrr_p); if (used_allctr == allctr) { @@ -2253,10 +2233,10 @@ dealloc_block(Allctr_t *allctr, void *ptr, ErtsAlcFixList_t *fix, int dec_cc_on_ /* Carrier migrated; need to redirect block to new owner... */ int cinit = used_allctr->dd.ix - allctr->dd.ix; - ERTS_ALC_CPOOL_ASSERT(!busy_pcrr_p); + ERTS_ALC_CPOOL_ASSERT(!busy_pcrr_p); - if (dec_cc_on_redirect) - DEC_CC(allctr->calls.this_free); + if (dec_cc_on_redirect) + DEC_CC(allctr->calls.this_free); if (ddq_enqueue(&used_allctr->dd.q, ptr, cinit)) erts_alloc_notify_delayed_dealloc(used_allctr->ix); } @@ -2500,15 +2480,16 @@ mbc_free(Allctr_t *allctr, void *p, Carrier_t **busy_pcrr_pp) ASSERT(blk_sz % sizeof(Unit_t) == 0); ASSERT(IS_MBC_BLK(blk)); - if (is_first_blk - && is_last_blk - && allctr->main_carrier != FIRST_BLK_TO_MBC(allctr, blk)) { - destroy_carrier(allctr, blk, busy_pcrr_pp); + if (is_first_blk && is_last_blk && crr != allctr->main_carrier) { + destroy_carrier(allctr, blk, busy_pcrr_pp); } else { (*allctr->link_free_block)(allctr, blk); HARD_CHECK_BLK_CARRIER(allctr, blk); - check_abandon_carrier(allctr, blk, busy_pcrr_pp); + if (busy_pcrr_pp && *busy_pcrr_pp) + update_pooled_tree(allctr, crr, blk_sz); + else + check_abandon_carrier(allctr, blk, busy_pcrr_pp); } } @@ -2542,8 +2523,19 @@ mbc_realloc(Allctr_t *allctr, void *p, Uint size, Uint32 alcu_flgs, return NULL; #else /* !MBC_REALLOC_ALWAYS_MOVES */ - if (busy_pcrr_pp && *busy_pcrr_pp) - goto realloc_move; /* Don't want to use carrier in pool */ + if (busy_pcrr_pp && *busy_pcrr_pp) { + /* + * Don't want to use carrier in pool + */ + new_p = mbc_alloc(allctr, size); + if (!new_p) + return NULL; + new_blk = UMEM2BLK(new_p); + ASSERT(!(IS_MBC_BLK(new_blk) && ABLK_TO_MBC(new_blk) == *busy_pcrr_pp)); + sys_memcpy(new_p, p, MIN(size, old_blk_sz - ABLK_HDR_SZ)); + mbc_free(allctr, p, busy_pcrr_pp); + return new_p; + } get_blk_sz = blk_sz = UMEMSZ2BLKSZ(allctr, size); @@ -2776,7 +2768,7 @@ mbc_realloc(Allctr_t *allctr, void *p, Uint size, Uint32 alcu_flgs, if (cand_blk_sz < get_blk_sz) { /* We wont fit in cand_blk get a new one */ - realloc_move: + #endif /* !MBC_REALLOC_ALWAYS_MOVES */ new_p = mbc_alloc(allctr, size); @@ -2880,8 +2872,7 @@ mbc_realloc(Allctr_t *allctr, void *p, Uint size, Uint32 alcu_flgs, #define ERTS_ALC_MAX_DEALLOC_CARRIER 10 -#define ERTS_ALC_CPOOL_MAX_FETCH_INSPECT 20 -#define ERTS_ALC_CPOOL_MAX_TRAITOR_INSPECT 10 +#define ERTS_ALC_CPOOL_MAX_FETCH_INSPECT 100 #define ERTS_ALC_CPOOL_CHECK_LIMIT_COUNT 100 #define ERTS_ALC_CPOOL_MAX_FAILED_STAT_READS 3 @@ -3045,19 +3036,18 @@ cpool_insert(Allctr_t *allctr, Carrier_t *crr) ErtsAlcCPoolData_t *cpd1p, *cpd2p; erts_aint_t val; ErtsAlcCPoolData_t *sentinel = &carrier_pool[allctr->alloc_no].sentinel; + Allctr_t *orig_allctr = crr->cpool.orig_allctr; ERTS_ALC_CPOOL_ASSERT(allctr->alloc_no == ERTS_ALC_A_INVALID /* testcase */ || erts_thr_progress_is_managed_thread()); - ERTS_ALC_CPOOL_ASSERT(erts_atomic_read_nob(&crr->allctr) - == (erts_aint_t) allctr); - erts_atomic_add_nob(&allctr->cpool.stat.blocks_size, + erts_atomic_add_nob(&orig_allctr->cpool.stat.blocks_size, (erts_aint_t) crr->cpool.blocks_size); - erts_atomic_add_nob(&allctr->cpool.stat.no_blocks, + erts_atomic_add_nob(&orig_allctr->cpool.stat.no_blocks, (erts_aint_t) crr->cpool.blocks); - erts_atomic_add_nob(&allctr->cpool.stat.carriers_size, + erts_atomic_add_nob(&orig_allctr->cpool.stat.carriers_size, (erts_aint_t) CARRIER_SZ(crr)); - erts_atomic_inc_nob(&allctr->cpool.stat.no_carriers); + erts_atomic_inc_nob(&orig_allctr->cpool.stat.no_carriers); /* * We search in 'next' direction and begin by passing @@ -3118,8 +3108,6 @@ cpool_insert(Allctr_t *allctr, Carrier_t *crr) (erts_aint_t) &crr->cpool, (erts_aint_t) cpd1p); - erts_atomic_set_wb(&crr->allctr, - ((erts_aint_t) allctr)|ERTS_CRR_ALCTR_FLG_IN_POOL); LTTNG3(carrier_pool_put, ERTS_ALC_A2AD(allctr->alloc_no), allctr->ix, CARRIER_SZ(crr)); } @@ -3221,130 +3209,126 @@ cpool_delete(Allctr_t *allctr, Allctr_t *prev_allctr, Carrier_t *crr) static Carrier_t * cpool_fetch(Allctr_t *allctr, UWord size) { - int i, i_stop, has_passed_sentinel; + enum { IGNORANT, HAS_SEEN_SENTINEL, THE_LAST_ONE } loop_state; + int i; Carrier_t *crr; + Carrier_t *reinsert_crr = NULL; ErtsAlcCPoolData_t *cpdp; - ErtsAlcCPoolData_t *cpool_entrance; + ErtsAlcCPoolData_t *cpool_entrance = NULL; ErtsAlcCPoolData_t *sentinel; - ErtsDoubleLink_t* dl; - ErtsDoubleLink_t* first_old_traitor; ERTS_ALC_CPOOL_ASSERT(allctr->alloc_no == ERTS_ALC_A_INVALID /* testcase */ || erts_thr_progress_is_managed_thread()); i = ERTS_ALC_CPOOL_MAX_FETCH_INSPECT; - first_old_traitor = allctr->cpool.traitor_list.next; - cpool_entrance = NULL; LTTNG3(carrier_pool_get, ERTS_ALC_A2AD(allctr->alloc_no), allctr->ix, (unsigned long)size); /* - * Search my own pooled_list, + * Search my own pooled_tree, * i.e my abandoned carriers that were in the pool last time I checked. */ + do { + erts_aint_t exp, act; + + crr = aoff_lookup_pooled_mbc(allctr, size); + if (!crr) + break; + + ASSERT(crr->cpool.state == ERTS_MBC_WAS_POOLED); + ASSERT(crr->cpool.orig_allctr == allctr); + + aoff_remove_pooled_mbc(allctr, crr); + + exp = erts_atomic_read_nob(&crr->allctr); + if (exp & ERTS_CRR_ALCTR_FLG_IN_POOL) { + ASSERT((exp & ~ERTS_CRR_ALCTR_FLG_MASK) == (erts_aint_t)allctr); + if (erts_atomic_read_nob(&crr->cpool.max_size) < size) { + /* + * This carrier has been fetched and inserted back again + * by a foreign allocator. That's why it has a stale search size. + */ + ASSERT(exp & ERTS_CRR_ALCTR_FLG_HOMECOMING); + crr->cpool.pooled.hdr.bhdr = erts_atomic_read_nob(&crr->cpool.max_size); + aoff_add_pooled_mbc(allctr, crr); + INC_CC(allctr->cpool.stat.skip_size); + continue; + } + else if (exp & ERTS_CRR_ALCTR_FLG_BUSY) { + /* + * This must be our own carrier as part of a realloc call. + * Skip it to make things simpler. + * Must wait to re-insert to not be found again by lookup. + */ + ASSERT(!reinsert_crr); + reinsert_crr = crr; + INC_CC(allctr->cpool.stat.skip_busy); + continue; + } - dl = allctr->cpool.pooled_list.next; - while(dl != &allctr->cpool.pooled_list) { - erts_aint_t exp, act; - crr = (Carrier_t *) (((char *) dl) - offsetof(Carrier_t, cpool.abandoned)); - - ASSERT(!is_in_list(&allctr->cpool.traitor_list, dl)); - ASSERT(crr->cpool.orig_allctr == allctr); - dl = dl->next; - exp = erts_atomic_read_rb(&crr->allctr); - if ((exp & ERTS_CRR_ALCTR_FLG_MASK) == ERTS_CRR_ALCTR_FLG_IN_POOL - && erts_atomic_read_nob(&crr->cpool.max_size) >= size) { - /* Try to fetch it... */ - act = erts_atomic_cmpxchg_mb(&crr->allctr, - (erts_aint_t) allctr, - exp); - if (act == exp) { - cpool_delete(allctr, ((Allctr_t *) (act & ~ERTS_CRR_ALCTR_FLG_MASK)), crr); - unlink_abandoned_carrier(crr); - - /* Move sentinel to continue next search from here */ - relink_edl_before(dl, &allctr->cpool.pooled_list); - return crr; - } - exp = act; - } - if (exp & ERTS_CRR_ALCTR_FLG_IN_POOL) { - if (!cpool_entrance) - cpool_entrance = &crr->cpool; - } - else { /* Not in pool, move to traitor_list */ - unlink_abandoned_carrier(crr); - link_abandoned_carrier(&allctr->cpool.traitor_list, crr); - } - if (--i <= 0) { - /* Move sentinel to continue next search from here */ - relink_edl_before(dl, &allctr->cpool.pooled_list); - return NULL; - } - } + /* Try to fetch it... */ + act = erts_atomic_cmpxchg_mb(&crr->allctr, + exp & ~ERTS_CRR_ALCTR_FLG_IN_POOL, + exp); + if (act == exp) { + cpool_delete(allctr, allctr, crr); + crr->cpool.state = ERTS_MBC_IS_HOME; + + if (reinsert_crr) + aoff_add_pooled_mbc(allctr, reinsert_crr); + return crr; + } + exp = act; + INC_CC(allctr->cpool.stat.skip_race); + } + else + INC_CC(allctr->cpool.stat.skip_not_pooled); - /* Now search traitor_list. - * i.e carriers employed by other allocators last time I checked. - * They might have been abandoned since then. - */ + /* Not in pool anymore */ + ASSERT(!(exp & ERTS_CRR_ALCTR_FLG_BUSY)); + crr->cpool.state = ERTS_MBC_WAS_TRAITOR; - i_stop = (i < ERTS_ALC_CPOOL_MAX_TRAITOR_INSPECT ? - 0 : i - ERTS_ALC_CPOOL_MAX_TRAITOR_INSPECT); - dl = first_old_traitor; - while(dl != &allctr->cpool.traitor_list) { - erts_aint_t exp, act; - crr = (Carrier_t *) (((char *) dl) - offsetof(Carrier_t, cpool.abandoned)); - ASSERT(dl != &allctr->cpool.pooled_list); - ASSERT(crr->cpool.orig_allctr == allctr); - dl = dl->next; - exp = erts_atomic_read_rb(&crr->allctr); - if (exp & ERTS_CRR_ALCTR_FLG_IN_POOL) { - if (!(exp & ERTS_CRR_ALCTR_FLG_BUSY) - && erts_atomic_read_nob(&crr->cpool.max_size) >= size) { - /* Try to fetch it... */ - act = erts_atomic_cmpxchg_mb(&crr->allctr, - (erts_aint_t) allctr, - exp); - if (act == exp) { - cpool_delete(allctr, ((Allctr_t *) (act & ~ERTS_CRR_ALCTR_FLG_MASK)), crr); - unlink_abandoned_carrier(crr); + }while (--i > 0); - /* Move sentinel to continue next search from here */ - relink_edl_before(dl, &allctr->cpool.traitor_list); - return crr; - } - exp = act; - } - if (exp & ERTS_CRR_ALCTR_FLG_IN_POOL) { - if (!cpool_entrance) - cpool_entrance = &crr->cpool; + if (reinsert_crr) + aoff_add_pooled_mbc(allctr, reinsert_crr); - /* Move to pooled_list */ - unlink_abandoned_carrier(crr); - link_abandoned_carrier(&allctr->cpool.pooled_list, crr); - } - } - if (--i <= i_stop) { - /* Move sentinel to continue next search from here */ - relink_edl_before(dl, &allctr->cpool.traitor_list); - if (i > 0) - break; - else - return NULL; - } + /* + * Try find a nice cpool_entrance + */ + while (allctr->cpool.pooled_tree) { + erts_aint_t iallctr; + + crr = ErtsContainerStruct(allctr->cpool.pooled_tree, Carrier_t, cpool.pooled); + iallctr = erts_atomic_read_nob(&crr->allctr); + if (iallctr & ERTS_CRR_ALCTR_FLG_IN_POOL) { + cpool_entrance = &crr->cpool; + break; + } + /* Not in pool anymore */ + ASSERT(!(iallctr & ERTS_CRR_ALCTR_FLG_BUSY)); + aoff_remove_pooled_mbc(allctr, crr); + crr->cpool.state = ERTS_MBC_WAS_TRAITOR; + + if (--i <= 0) { + INC_CC(allctr->cpool.stat.fail_pooled); + return NULL; + } } + /* * Finally search the shared pool and try employ foreign carriers */ - sentinel = &carrier_pool[allctr->alloc_no].sentinel; if (cpool_entrance) { - /* We saw a pooled carried above, use it as entrance into the pool + /* + * We saw a pooled carried above, use it as entrance into the pool */ cpdp = cpool_entrance; } else { - /* No pooled carried seen above. Start search at cpool sentinel, + /* + * No pooled carried seen above. Start search at cpool sentinel, * but begin by passing one element before trying to fetch. * This in order to avoid contention with threads inserting elements. */ @@ -3354,8 +3338,8 @@ cpool_fetch(Allctr_t *allctr, UWord size) goto check_dc_list; } - has_passed_sentinel = 0; - while (1) { + loop_state = IGNORANT; + do { erts_aint_t exp; cpdp = cpool_aint2cpd(cpool_read(&cpdp->prev)); if (cpdp == cpool_entrance) { @@ -3364,38 +3348,52 @@ cpool_fetch(Allctr_t *allctr, UWord size) if (cpdp == sentinel) break; } - i = 0; /* Last one to inspect */ + loop_state = THE_LAST_ONE; } else if (cpdp == sentinel) { - if (has_passed_sentinel) { + if (loop_state == HAS_SEEN_SENTINEL) { /* We been here before. cpool_entrance must have been removed */ + INC_CC(allctr->cpool.stat.entrance_removed); break; } cpdp = cpool_aint2cpd(cpool_read(&cpdp->prev)); if (cpdp == sentinel) break; - has_passed_sentinel = 1; + loop_state = HAS_SEEN_SENTINEL; } - crr = (Carrier_t *)(((char *)cpdp) - offsetof(Carrier_t, cpool)); + crr = ErtsContainerStruct(cpdp, Carrier_t, cpool); exp = erts_atomic_read_rb(&crr->allctr); - if (((exp & (ERTS_CRR_ALCTR_FLG_MASK)) == ERTS_CRR_ALCTR_FLG_IN_POOL) - && (erts_atomic_read_nob(&cpdp->max_size) >= size)) { + + if (erts_atomic_read_nob(&cpdp->max_size) < size) { + INC_CC(allctr->cpool.stat.skip_size); + } + else if ((exp & (ERTS_CRR_ALCTR_FLG_IN_POOL | ERTS_CRR_ALCTR_FLG_BUSY)) + == ERTS_CRR_ALCTR_FLG_IN_POOL) { erts_aint_t act; - /* Try to fetch it... */ - act = erts_atomic_cmpxchg_mb(&crr->allctr, - (erts_aint_t) allctr, - exp); + erts_aint_t want = (((erts_aint_t) allctr) + | (exp & ERTS_CRR_ALCTR_FLG_HOMECOMING)); + /* Try to fetch it... */ + act = erts_atomic_cmpxchg_mb(&crr->allctr, want, exp); if (act == exp) { cpool_delete(allctr, ((Allctr_t *) (act & ~ERTS_CRR_ALCTR_FLG_MASK)), crr); if (crr->cpool.orig_allctr == allctr) { unlink_abandoned_carrier(crr); - } + crr->cpool.state = ERTS_MBC_IS_HOME; + } return crr; } } - if (--i <= 0) + + if (exp & ERTS_CRR_ALCTR_FLG_BUSY) + INC_CC(allctr->cpool.stat.skip_busy); + else + INC_CC(allctr->cpool.stat.skip_race); + + if (--i <= 0) { + INC_CC(allctr->cpool.stat.fail_shared); return NULL; - } + } + }while (loop_state != THE_LAST_ONE); check_dc_list: /* Last; check our own pending dealloc carrier list... */ @@ -3404,23 +3402,23 @@ check_dc_list: if (erts_atomic_read_nob(&crr->cpool.max_size) >= size) { Block_t* blk; unlink_carrier(&allctr->cpool.dc_list, crr); -#ifdef ERTS_ALC_CPOOL_DEBUG - ERTS_ALC_CPOOL_ASSERT(erts_atomic_xchg_nob(&crr->allctr, - ((erts_aint_t) allctr)) - == (((erts_aint_t) allctr) & ~ERTS_CRR_ALCTR_FLG_MASK)); -#else - erts_atomic_set_nob(&crr->allctr, ((erts_aint_t) allctr)); -#endif + ERTS_ALC_CPOOL_ASSERT(erts_atomic_read_nob(&crr->allctr) + == ((erts_aint_t) allctr)); blk = MBC_TO_FIRST_BLK(allctr, crr); ASSERT(FBLK_TO_MBC(blk) == crr); allctr->link_free_block(allctr, blk); return crr; } crr = crr->prev; - if (--i <= 0) + if (--i <= 0) { + INC_CC(allctr->cpool.stat.fail_pend_dealloc); return NULL; + } } + if (i != ERTS_ALC_CPOOL_MAX_FETCH_INSPECT) + INC_CC(allctr->cpool.stat.fail); + return NULL; } @@ -3475,9 +3473,6 @@ static void schedule_dealloc_carrier(Allctr_t *allctr, Carrier_t *crr) { Allctr_t *orig_allctr; - Block_t *blk; - int check_pending_dealloc; - erts_aint_t max_size; ASSERT(IS_MB_CARRIER(crr)); @@ -3488,9 +3483,17 @@ schedule_dealloc_carrier(Allctr_t *allctr, Carrier_t *crr) orig_allctr = crr->cpool.orig_allctr; - if (allctr != orig_allctr) { - int cinit = orig_allctr->dd.ix - allctr->dd.ix; - + if (allctr == orig_allctr) { + if (!(erts_atomic_read_nob(&crr->allctr) & ERTS_CRR_ALCTR_FLG_HOMECOMING)) { + dealloc_my_carrier(allctr, crr); + } + /*else + * Carrier was abandoned earlier by other thread and + * is still waiting for us in dd-queue. + * handle_delayed_dealloc() will handle it when crr is dequeued. + */ + } + else { /* * We send the carrier to its origin for deallocation. * This in order: @@ -3499,29 +3502,39 @@ schedule_dealloc_carrier(Allctr_t *allctr, Carrier_t *crr) * - to ensure that we always only reuse empty carriers * originating from our own thread specific mseg_alloc * instance which is beneficial on NUMA systems. - * - * The receiver will recognize that this is a carrier to - * deallocate (and not a block which is the common case) - * since the block is an mbc block that is free and last - * in the carrier. */ - blk = MBC_TO_FIRST_BLK(allctr, crr); - ERTS_ALC_CPOOL_ASSERT(IS_FREE_LAST_MBC_BLK(blk)); - - ERTS_ALC_CPOOL_ASSERT(IS_MBC_FIRST_ABLK(allctr, blk)); - ERTS_ALC_CPOOL_ASSERT(crr == FBLK_TO_MBC(blk)); - ERTS_ALC_CPOOL_ASSERT(crr == FIRST_BLK_TO_MBC(allctr, blk)); - ERTS_ALC_CPOOL_ASSERT(((erts_aint_t) allctr) - == (erts_atomic_read_nob(&crr->allctr) - & ~ERTS_CRR_ALCTR_FLG_MASK)); + erts_aint_t iallctr; +#ifdef ERTS_ALC_CPOOL_DEBUG + Block_t* first_blk = MBC_TO_FIRST_BLK(allctr, crr); + ERTS_ALC_CPOOL_ASSERT(IS_FREE_LAST_MBC_BLK(first_blk)); + + ERTS_ALC_CPOOL_ASSERT(IS_MBC_FIRST_ABLK(allctr, first_blk)); + ERTS_ALC_CPOOL_ASSERT(crr == FBLK_TO_MBC(first_blk)); + ERTS_ALC_CPOOL_ASSERT(crr == FIRST_BLK_TO_MBC(allctr, first_blk)); + ERTS_ALC_CPOOL_ASSERT((erts_atomic_read_nob(&crr->allctr) + & ~ERTS_CRR_ALCTR_FLG_HOMECOMING) + == (erts_aint_t) allctr); +#endif - if (ddq_enqueue(&orig_allctr->dd.q, BLK2UMEM(blk), cinit)) - erts_alloc_notify_delayed_dealloc(orig_allctr->ix); - return; + iallctr = (erts_aint_t)orig_allctr | ERTS_CRR_ALCTR_FLG_HOMECOMING; + if (!(erts_atomic_xchg_nob(&crr->allctr, iallctr) + & ERTS_CRR_ALCTR_FLG_HOMECOMING)) { + enqueue_homecoming(allctr, crr); + } } +} + +static void dealloc_my_carrier(Allctr_t *allctr, Carrier_t *crr) +{ + Block_t *blk; + int check_pending_dealloc; + erts_aint_t max_size; - if (is_abandoned(crr)) - unlink_abandoned_carrier(crr); + ERTS_ALC_CPOOL_ASSERT(allctr == crr->cpool.orig_allctr); + if (is_abandoned(crr)) { + unlink_abandoned_carrier(crr); + crr->cpool.state = ERTS_MBC_IS_HOME; + } if (crr->cpool.thr_prgr == ERTS_THR_PRGR_INVALID || erts_thr_progress_has_reached(crr->cpool.thr_prgr)) { @@ -3553,6 +3566,7 @@ schedule_dealloc_carrier(Allctr_t *allctr, Carrier_t *crr) static ERTS_INLINE void cpool_init_carrier_data(Allctr_t *allctr, Carrier_t *crr) { + crr->cpool.homecoming_dd.blk.bhdr = HOMECOMING_MBC_BLK_HDR; erts_atomic_init_nob(&crr->cpool.next, ERTS_AINT_NULL); erts_atomic_init_nob(&crr->cpool.prev, ERTS_AINT_NULL); crr->cpool.orig_allctr = allctr; @@ -3571,8 +3585,7 @@ cpool_init_carrier_data(Allctr_t *allctr, Carrier_t *crr) limit = (csz/100)*allctr->cpool.util_limit; crr->cpool.abandon_limit = limit; } - crr->cpool.abandoned.next = NULL; - crr->cpool.abandoned.prev = NULL; + crr->cpool.state = ERTS_MBC_IS_HOME; } static void @@ -3598,23 +3611,62 @@ set_new_allctr_abandon_limit(Allctr_t *allctr) static void abandon_carrier(Allctr_t *allctr, Carrier_t *crr) { - erts_aint_t max_size; + erts_aint_t iallctr; - STAT_MBC_CPOOL_INSERT(allctr, crr); + STAT_MBC_ABANDON(allctr, crr); unlink_carrier(&allctr->mbc_list, crr); - if (crr->cpool.orig_allctr == allctr) { - link_abandoned_carrier(&allctr->cpool.pooled_list, crr); + allctr->remove_mbc(allctr, crr); + set_new_allctr_abandon_limit(allctr); + + cpool_insert(allctr, crr); + + + iallctr = erts_atomic_read_nob(&crr->allctr); + if (allctr == crr->cpool.orig_allctr) { + /* preserve HOMECOMING flag */ + ASSERT((iallctr & ~ERTS_CRR_ALCTR_FLG_HOMECOMING) == (erts_aint_t)allctr); + erts_atomic_set_wb(&crr->allctr, iallctr | ERTS_CRR_ALCTR_FLG_IN_POOL); + poolify_my_carrier(allctr, crr); } + else { + ASSERT((iallctr & ~ERTS_CRR_ALCTR_FLG_HOMECOMING) == (erts_aint_t)allctr); + iallctr = ((erts_aint_t)crr->cpool.orig_allctr | + ERTS_CRR_ALCTR_FLG_HOMECOMING | + ERTS_CRR_ALCTR_FLG_IN_POOL); + if (!(erts_atomic_xchg_wb(&crr->allctr, iallctr) + & ERTS_CRR_ALCTR_FLG_HOMECOMING)) { + + enqueue_homecoming(allctr, crr); + } + } +} - allctr->remove_mbc(allctr, crr); +static void +enqueue_homecoming(Allctr_t* allctr, Carrier_t* crr) +{ + Allctr_t* orig_allctr = crr->cpool.orig_allctr; + const int cinit = orig_allctr->dd.ix - allctr->dd.ix; + Block_t* dd_blk = &crr->cpool.homecoming_dd.blk; - max_size = (erts_aint_t) allctr->largest_fblk_in_mbc(allctr, crr); - erts_atomic_set_nob(&crr->cpool.max_size, max_size); + /* + * The receiver will recognize this as a carrier + * (and not a block which is the common case) + * since the block header is HOMECOMING_MBC_BLK_HDR. + */ + ASSERT(dd_blk->bhdr == HOMECOMING_MBC_BLK_HDR); + if (ddq_enqueue(&orig_allctr->dd.q, BLK2UMEM(dd_blk), cinit)) + erts_alloc_notify_delayed_dealloc(orig_allctr->ix); +} - cpool_insert(allctr, crr); +static void +poolify_my_carrier(Allctr_t *allctr, Carrier_t *crr) +{ + ERTS_ALC_CPOOL_ASSERT(allctr == crr->cpool.orig_allctr); - set_new_allctr_abandon_limit(allctr); + crr->cpool.pooled.hdr.bhdr = erts_atomic_read_nob(&crr->cpool.max_size); + aoff_add_pooled_mbc(allctr, crr); + crr->cpool.state = ERTS_MBC_WAS_POOLED; } static void @@ -3771,6 +3823,7 @@ create_carrier(Allctr_t *allctr, Uint umem_sz, UWord flags) crr = cpool_fetch(allctr, blk_sz); if (crr) { STAT_MBC_CPOOL_FETCH(allctr, crr); + INC_CC(allctr->cpool.stat.fetch); link_carrier(&allctr->mbc_list, crr); (*allctr->add_mbc)(allctr, crr); blk = (*allctr->get_free_block)(allctr, blk_sz, NULL, 0); @@ -4128,13 +4181,18 @@ destroy_carrier(Allctr_t *allctr, Block_t *blk, Carrier_t **busy_pcrr_pp) #endif if (busy_pcrr_pp && *busy_pcrr_pp) { + erts_aint_t iallctr = erts_atomic_read_nob(&crr->allctr); ERTS_ALC_CPOOL_ASSERT(*busy_pcrr_pp == crr); - *busy_pcrr_pp = NULL; - ERTS_ALC_CPOOL_ASSERT(erts_atomic_read_nob(&crr->allctr) - == (((erts_aint_t) allctr) - | ERTS_CRR_ALCTR_FLG_IN_POOL - | ERTS_CRR_ALCTR_FLG_BUSY)); - erts_atomic_set_nob(&crr->allctr, ((erts_aint_t) allctr)); + ERTS_ALC_CPOOL_ASSERT((iallctr & ~ERTS_CRR_ALCTR_FLG_HOMECOMING) + == (((erts_aint_t) allctr) + | ERTS_CRR_ALCTR_FLG_IN_POOL + | ERTS_CRR_ALCTR_FLG_BUSY)); + ERTS_ALC_CPOOL_ASSERT(allctr == crr->cpool.orig_allctr); + + *busy_pcrr_pp = NULL; + erts_atomic_set_nob(&crr->allctr, + (iallctr & ~(ERTS_CRR_ALCTR_FLG_IN_POOL | + ERTS_CRR_ALCTR_FLG_BUSY))); cpool_delete(allctr, allctr, crr); } else @@ -4184,7 +4242,6 @@ static struct { Eterm e; Eterm t; Eterm ramv; - Eterm sbct; #if HAVE_ERTS_MSEG Eterm asbcst; Eterm rsbcst; @@ -4201,6 +4258,8 @@ static struct { Eterm smbcs; Eterm mbcgs; Eterm acul; + Eterm acnl; + Eterm acfml; #if HAVE_ERTS_MSEG Eterm mmc; @@ -4212,6 +4271,17 @@ static struct { Eterm mbcs; Eterm mbcs_pool; + Eterm fetch; + Eterm fail_pooled; + Eterm fail_shared; + Eterm fail_pend_dealloc; + Eterm fail; + Eterm skip_size; + Eterm skip_busy; + Eterm skip_not_pooled; + Eterm skip_homecoming; + Eterm skip_race; + Eterm entrance_removed; Eterm sbcs; Eterm sys_alloc_carriers_size; @@ -4245,7 +4315,7 @@ static Eterm fix_type_atoms[ERTS_ALC_NO_FIXED_SIZES]; static ERTS_INLINE void atom_init(Eterm *atom, char *name) { - *atom = am_atom_put(name, strlen(name)); + *atom = am_atom_put(name, sys_strlen(name)); } #define AM_INIT(AM) atom_init(&am.AM, #AM) @@ -4272,7 +4342,6 @@ init_atoms(Allctr_t *allctr) AM_INIT(e); AM_INIT(t); AM_INIT(ramv); - AM_INIT(sbct); #if HAVE_ERTS_MSEG AM_INIT(asbcst); AM_INIT(rsbcst); @@ -4289,6 +4358,8 @@ init_atoms(Allctr_t *allctr) AM_INIT(smbcs); AM_INIT(mbcgs); AM_INIT(acul); + AM_INIT(acnl); + AM_INIT(acfml); #if HAVE_ERTS_MSEG AM_INIT(mmc); @@ -4300,6 +4371,17 @@ init_atoms(Allctr_t *allctr) AM_INIT(mbcs); AM_INIT(mbcs_pool); + AM_INIT(fetch); + AM_INIT(fail_pooled); + AM_INIT(fail_shared); + AM_INIT(fail_pend_dealloc); + AM_INIT(fail); + AM_INIT(skip_size); + AM_INIT(skip_busy); + AM_INIT(skip_not_pooled); + AM_INIT(skip_homecoming); + AM_INIT(skip_race); + AM_INIT(entrance_removed); AM_INIT(sbcs); AM_INIT(sys_alloc_carriers_size); @@ -4334,7 +4416,7 @@ init_atoms(Allctr_t *allctr) for (ix = 0; ix < ERTS_ALC_NO_FIXED_SIZES; ix++) { ErtsAlcType_t n = ERTS_ALC_N_MIN_A_FIXED_SIZE + ix; char *name = (char *) ERTS_ALC_N2TD(n); - size_t len = strlen(name); + size_t len = sys_strlen(name); fix_type_atoms[ix] = am_atom_put(name, len); } } @@ -4583,9 +4665,56 @@ info_cpool(Allctr_t *allctr, if (hpp || szp) { res = NIL; + + if (!sz_only) { + add_3tup(hpp, szp, &res, am.fail_pooled, + bld_unstable_uint(hpp, szp, ERTS_ALC_CC_GIGA_VAL(allctr->cpool.stat.fail_pooled)), + bld_unstable_uint(hpp, szp, ERTS_ALC_CC_VAL(allctr->cpool.stat.fail_pooled))); + + add_3tup(hpp, szp, &res, am.fail_shared, + bld_unstable_uint(hpp, szp, ERTS_ALC_CC_GIGA_VAL(allctr->cpool.stat.fail_shared)), + bld_unstable_uint(hpp, szp, ERTS_ALC_CC_VAL(allctr->cpool.stat.fail_shared))); + + add_3tup(hpp, szp, &res, am.fail_pend_dealloc, + bld_unstable_uint(hpp, szp, ERTS_ALC_CC_GIGA_VAL(allctr->cpool.stat.fail_pend_dealloc)), + bld_unstable_uint(hpp, szp, ERTS_ALC_CC_VAL(allctr->cpool.stat.fail_pend_dealloc))); + + add_3tup(hpp, szp, &res, am.fail, + bld_unstable_uint(hpp, szp, ERTS_ALC_CC_GIGA_VAL(allctr->cpool.stat.fail)), + bld_unstable_uint(hpp, szp, ERTS_ALC_CC_VAL(allctr->cpool.stat.fail))); + + add_3tup(hpp, szp, &res, am.fetch, + bld_unstable_uint(hpp, szp, ERTS_ALC_CC_GIGA_VAL(allctr->cpool.stat.fetch)), + bld_unstable_uint(hpp, szp, ERTS_ALC_CC_VAL(allctr->cpool.stat.fetch))); + + add_3tup(hpp, szp, &res, am.skip_size, + bld_unstable_uint(hpp, szp, ERTS_ALC_CC_GIGA_VAL(allctr->cpool.stat.skip_size)), + bld_unstable_uint(hpp, szp, ERTS_ALC_CC_VAL(allctr->cpool.stat.skip_size))); + + add_3tup(hpp, szp, &res, am.skip_busy, + bld_unstable_uint(hpp, szp, ERTS_ALC_CC_GIGA_VAL(allctr->cpool.stat.skip_busy)), + bld_unstable_uint(hpp, szp, ERTS_ALC_CC_VAL(allctr->cpool.stat.skip_busy))); + + add_3tup(hpp, szp, &res, am.skip_not_pooled, + bld_unstable_uint(hpp, szp, ERTS_ALC_CC_GIGA_VAL(allctr->cpool.stat.skip_not_pooled)), + bld_unstable_uint(hpp, szp, ERTS_ALC_CC_VAL(allctr->cpool.stat.skip_not_pooled))); + + add_3tup(hpp, szp, &res, am.skip_homecoming, + bld_unstable_uint(hpp, szp, ERTS_ALC_CC_GIGA_VAL(allctr->cpool.stat.skip_homecoming)), + bld_unstable_uint(hpp, szp, ERTS_ALC_CC_VAL(allctr->cpool.stat.skip_homecoming))); + + add_3tup(hpp, szp, &res, am.skip_race, + bld_unstable_uint(hpp, szp, ERTS_ALC_CC_GIGA_VAL(allctr->cpool.stat.skip_race)), + bld_unstable_uint(hpp, szp, ERTS_ALC_CC_VAL(allctr->cpool.stat.skip_race))); + + add_3tup(hpp, szp, &res, am.entrance_removed, + bld_unstable_uint(hpp, szp, ERTS_ALC_CC_GIGA_VAL(allctr->cpool.stat.entrance_removed)), + bld_unstable_uint(hpp, szp, ERTS_ALC_CC_VAL(allctr->cpool.stat.entrance_removed))); + add_2tup(hpp, szp, &res, am.carriers_size, bld_unstable_uint(hpp, szp, csz)); + } if (!sz_only) add_2tup(hpp, szp, &res, am.carriers, @@ -4725,20 +4854,20 @@ make_name_atoms(Allctr_t *allctr) char realloc[] = "realloc"; char free[] = "free"; char buf[MAX_ATOM_CHARACTERS]; - size_t prefix_len = strlen(allctr->name_prefix); + size_t prefix_len = sys_strlen(allctr->name_prefix); if (prefix_len > MAX_ATOM_CHARACTERS + sizeof(realloc) - 1) erts_exit(ERTS_ERROR_EXIT,"Too long allocator name: %salloc\n",allctr->name_prefix); - memcpy((void *) buf, (void *) allctr->name_prefix, prefix_len); + sys_memcpy((void *) buf, (void *) allctr->name_prefix, prefix_len); - memcpy((void *) &buf[prefix_len], (void *) alloc, sizeof(alloc) - 1); + sys_memcpy((void *) &buf[prefix_len], (void *) alloc, sizeof(alloc) - 1); allctr->name.alloc = am_atom_put(buf, prefix_len + sizeof(alloc) - 1); - memcpy((void *) &buf[prefix_len], (void *) realloc, sizeof(realloc) - 1); + sys_memcpy((void *) &buf[prefix_len], (void *) realloc, sizeof(realloc) - 1); allctr->name.realloc = am_atom_put(buf, prefix_len + sizeof(realloc) - 1); - memcpy((void *) &buf[prefix_len], (void *) free, sizeof(free) - 1); + sys_memcpy((void *) &buf[prefix_len], (void *) free, sizeof(free) - 1); allctr->name.free = am_atom_put(buf, prefix_len + sizeof(free) - 1); } @@ -4844,7 +4973,7 @@ info_options(Allctr_t *allctr, Uint *szp) { Eterm res = THE_NON_VALUE; - int acul; + UWord acul, acnl, acfml; if (!allctr) { if (print_to_p) @@ -4857,6 +4986,8 @@ info_options(Allctr_t *allctr, } acul = allctr->cpool.util_limit; + acnl = allctr->cpool.in_pool_limit; + acfml = allctr->cpool.fblk_min_limit; if (print_to_p) { char topt[21]; /* Enough for any 64-bit integer */ @@ -4884,7 +5015,7 @@ info_options(Allctr_t *allctr, "option lmbcs: %beu\n" "option smbcs: %beu\n" "option mbcgs: %beu\n" - "option acul: %d\n", + "option acul: %bpu\n", topt, allctr->ramv ? "true" : "false", allctr->sbc_threshold, @@ -4909,9 +5040,15 @@ info_options(Allctr_t *allctr, hpp, szp); if (hpp || szp) { + add_2tup(hpp, szp, &res, + am.acfml, + bld_uint(hpp, szp, acfml)); + add_2tup(hpp, szp, &res, + am.acnl, + bld_uint(hpp, szp, acnl)); add_2tup(hpp, szp, &res, am.acul, - bld_uint(hpp, szp, (UWord) acul)); + bld_uint(hpp, szp, acul)); add_2tup(hpp, szp, &res, am.mbcgs, bld_uint(hpp, szp, allctr->mbc_growth_stages)); @@ -4947,7 +5084,7 @@ info_options(Allctr_t *allctr, bld_uint(hpp, szp, allctr->mseg_opt.abs_shrink_th)); #endif add_2tup(hpp, szp, &res, - am.sbct, + am_sbct, bld_uint(hpp, szp, allctr->sbc_threshold)); add_2tup(hpp, szp, &res, am.ramv, allctr->ramv ? am_true : am_false); add_2tup(hpp, szp, &res, am.t, (allctr->t ? am_true : am_false)); @@ -5481,12 +5618,13 @@ erts_alcu_free_thr_pref(ErtsAlcType_t type, void *extra, void *p) pref_allctr = get_pref_allctr(extra); used_allctr = get_used_allctr(pref_allctr, ERTS_ALC_TS_PREF_LOCK_IF_USED, p, NULL, &busy_pcrr_p); - if (pref_allctr != used_allctr) + if (pref_allctr != used_allctr) { enqueue_dealloc_other_instance(type, - used_allctr, - p, - (used_allctr->dd.ix - - pref_allctr->dd.ix)); + used_allctr, + p, + (used_allctr->dd.ix + - pref_allctr->dd.ix)); + } else { ERTS_ALCU_DBG_CHK_THR_ACCESS(used_allctr); do_erts_alcu_free(type, used_allctr, p, &busy_pcrr_p); @@ -5854,6 +5992,37 @@ erts_alcu_realloc_mv_thr_pref(ErtsAlcType_t type, void *extra, +static Uint adjust_sbct(Allctr_t* allctr, Uint sbct) +{ +#ifndef ARCH_64 + if (sbct > 0) { + Uint max_mbc_block_sz = UNIT_CEILING(sbct - 1 + ABLK_HDR_SZ); + if (max_mbc_block_sz + UNIT_FLOOR(allctr->min_block_size - 1) > MBC_ABLK_SZ_MASK + || max_mbc_block_sz < sbct) { /* wrap around */ + /* + * By limiting sbc_threshold to (hard limit - min_block_size) + * we avoid having to split off free "residue blocks" + * smaller than min_block_size. + */ + max_mbc_block_sz = MBC_ABLK_SZ_MASK - UNIT_FLOOR(allctr->min_block_size - 1); + sbct = max_mbc_block_sz - ABLK_HDR_SZ + 1; + } + } +#endif + return sbct; +} + +int erts_alcu_try_set_dyn_param(Allctr_t* allctr, Eterm param, Uint value) +{ + const Uint MIN_DYN_SBCT = 4000; /* a lame catastrophe prevention */ + + if (param == am_sbct && value >= MIN_DYN_SBCT) { + allctr->sbc_threshold = adjust_sbct(allctr, value); + return 1; + } + return 0; +} + /* ------------------------------------------------------------------------- */ int @@ -5941,10 +6110,7 @@ erts_alcu_start(Allctr_t *allctr, AllctrInit_t *init) allctr->min_block_size = sz; } - allctr->cpool.pooled_list.next = &allctr->cpool.pooled_list; - allctr->cpool.pooled_list.prev = &allctr->cpool.pooled_list; - allctr->cpool.traitor_list.next = &allctr->cpool.traitor_list; - allctr->cpool.traitor_list.prev = &allctr->cpool.traitor_list; + allctr->cpool.pooled_tree = NULL; allctr->cpool.dc_list.first = NULL; allctr->cpool.dc_list.last = NULL; allctr->cpool.abandon_limit = 0; @@ -5954,24 +6120,18 @@ erts_alcu_start(Allctr_t *allctr, AllctrInit_t *init) erts_atomic_init_nob(&allctr->cpool.stat.carriers_size, 0); erts_atomic_init_nob(&allctr->cpool.stat.no_carriers, 0); allctr->cpool.check_limit_count = ERTS_ALC_CPOOL_CHECK_LIMIT_COUNT; - allctr->cpool.util_limit = init->ts ? 0 : init->acul; - - allctr->sbc_threshold = init->sbct; -#ifndef ARCH_64 - if (allctr->sbc_threshold > 0) { - Uint max_mbc_block_sz = UNIT_CEILING(allctr->sbc_threshold - 1 + ABLK_HDR_SZ); - if (max_mbc_block_sz + UNIT_FLOOR(allctr->min_block_size - 1) > MBC_ABLK_SZ_MASK - || max_mbc_block_sz < allctr->sbc_threshold) { /* wrap around */ - /* - * By limiting sbc_threshold to (hard limit - min_block_size) - * we avoid having to split off free "residue blocks" - * smaller than min_block_size. - */ - max_mbc_block_sz = MBC_ABLK_SZ_MASK - UNIT_FLOOR(allctr->min_block_size - 1); - allctr->sbc_threshold = max_mbc_block_sz - ABLK_HDR_SZ + 1; - } + if (!init->ts && init->acul && init->acnl) { + allctr->cpool.util_limit = init->acul; + allctr->cpool.in_pool_limit = init->acnl; + allctr->cpool.fblk_min_limit = init->acfml; } -#endif + else { + allctr->cpool.util_limit = 0; + allctr->cpool.in_pool_limit = 0; + allctr->cpool.fblk_min_limit = 0; + } + + allctr->sbc_threshold = adjust_sbct(allctr, init->sbct); #if HAVE_ERTS_MSEG if (allctr->mseg_opt.abs_shrink_th > ~((UWord) 0) / 100) @@ -6022,6 +6182,9 @@ erts_alcu_start(Allctr_t *allctr, AllctrInit_t *init) allctr->sys_realloc = &erts_alcu_sys_realloc; allctr->sys_dealloc = &erts_alcu_sys_dealloc; } + + allctr->try_set_dyn_param = &erts_alcu_try_set_dyn_param; + #if HAVE_ERTS_MSEG if (init->mseg_alloc) { ASSERT(init->mseg_realloc && init->mseg_dealloc); @@ -6036,6 +6199,7 @@ erts_alcu_start(Allctr_t *allctr, AllctrInit_t *init) allctr->mseg_realloc = &erts_alcu_mseg_realloc; allctr->mseg_dealloc = &erts_alcu_mseg_dealloc; } + /* If a custom carrier alloc function is specified, make sure it's used */ if (init->mseg_alloc && !init->sys_alloc) { allctr->crr_set_flgs = CFLG_FORCE_MSEG; diff --git a/erts/emulator/beam/erl_alloc_util.h b/erts/emulator/beam/erl_alloc_util.h index faeb5ef368..05c8a0db3b 100644 --- a/erts/emulator/beam/erl_alloc_util.h +++ b/erts/emulator/beam/erl_alloc_util.h @@ -61,7 +61,9 @@ typedef struct { UWord lmbcs; UWord smbcs; UWord mbcgs; - int acul; + UWord acul; + UWord acnl; + UWord acfml; void *fix; size_t *fix_type_size; @@ -116,6 +118,8 @@ typedef struct { 1024*1024, /* (bytes) smbcs: smallest mbc size */\ 10, /* (amount) mbcgs: mbc growth stages */\ 0, /* (%) acul: abandon carrier utilization limit */\ + 1000, /* (amount) acnl: abandoned carriers number limit */\ + 0, /* (bytes) acfml: abandoned carrier fblk min limit */\ /* --- Data not options -------------------------------------------- */\ NULL, /* (ptr) fix */\ NULL /* (ptr) fix_type_size */\ @@ -149,6 +153,8 @@ typedef struct { 128*1024, /* (bytes) smbcs: smallest mbc size */\ 10, /* (amount) mbcgs: mbc growth stages */\ 0, /* (%) acul: abandon carrier utilization limit */\ + 1000, /* (amount) acnl: abandoned carriers number limit */\ + 0, /* (bytes) acfml: abandoned carrier fblk min limit */\ /* --- Data not options -------------------------------------------- */\ NULL, /* (ptr) fix */\ NULL /* (ptr) fix_type_size */\ @@ -199,7 +205,7 @@ void* erts_alcu_mmapper_mseg_realloc(Allctr_t*, void *seg, Uint old_size, Uint * void erts_alcu_mmapper_mseg_dealloc(Allctr_t*, void *seg, Uint size, Uint flags); # endif -# if defined(ERTS_ALC_A_EXEC) && !defined(ERTS_HAVE_EXEC_MMAPPER) +# if defined(ERTS_ALC_A_EXEC) void* erts_alcu_exec_mseg_alloc(Allctr_t*, Uint *size_p, Uint flags); void* erts_alcu_exec_mseg_realloc(Allctr_t*, void *seg, Uint old_size, Uint *new_size_p); void erts_alcu_exec_mseg_dealloc(Allctr_t*, void *seg, Uint size, Uint flags); @@ -216,6 +222,8 @@ void erts_alcu_literal_32_sys_dealloc(Allctr_t*, void *ptr, Uint size, int supe void erts_lcnt_update_allocator_locks(int enable); #endif +int erts_alcu_try_set_dyn_param(Allctr_t*, Eterm param, Uint value); + #endif /* !ERL_ALLOC_UTIL__ */ #if defined(GET_ERL_ALLOC_UTIL_IMPL) && !defined(ERL_ALLOC_UTIL_IMPL__) @@ -296,41 +304,7 @@ void erts_lcnt_update_allocator_locks(int enable); typedef union {char c[ERTS_ALLOC_ALIGN_BYTES]; long l; double d;} Unit_t; - -typedef struct ErtsDoubleLink_t_ { - struct ErtsDoubleLink_t_ *next; - struct ErtsDoubleLink_t_ *prev; -}ErtsDoubleLink_t; - -typedef struct { - erts_atomic_t next; - erts_atomic_t prev; - Allctr_t *orig_allctr; /* read-only while carrier is alive */ - ErtsThrPrgrVal thr_prgr; - erts_atomic_t max_size; - UWord abandon_limit; - UWord blocks; - UWord blocks_size; - ErtsDoubleLink_t abandoned; /* node in pooled_list or traitor_list */ -} ErtsAlcCPoolData_t; - - typedef struct Carrier_t_ Carrier_t; -struct Carrier_t_ { - UWord chdr; - Carrier_t *next; - Carrier_t *prev; - erts_atomic_t allctr; - ErtsAlcCPoolData_t cpool; /* Overwritten by block if sbc */ -}; - -#define ERTS_ALC_CARRIER_TO_ALLCTR(C) \ - ((Allctr_t *) (erts_atomic_read_nob(&(C)->allctr) & ~FLG_MASK)) - -typedef struct { - Carrier_t *first; - Carrier_t *last; -} CarrierList_t; typedef struct { UWord bhdr; @@ -344,6 +318,22 @@ typedef struct { #endif } Block_t; +typedef union ErtsAllctrDDBlock_t_ ErtsAllctrDDBlock_t; + +union ErtsAllctrDDBlock_t_ { + erts_atomic_t atmc_next; + ErtsAllctrDDBlock_t *ptr_next; +}; + +typedef struct { + Block_t blk; +#if !MBC_ABLK_OFFSET_BITS + ErtsAllctrDDBlock_t umem_; +#endif +} ErtsFakeDDBlock_t; + + + #define THIS_FREE_BLK_HDR_FLG (((UWord) 1) << 0) #define PREV_FREE_BLK_HDR_FLG (((UWord) 1) << 1) #define LAST_BLK_HDR_FLG (((UWord) 1) << 2) @@ -352,14 +342,13 @@ typedef struct { (THIS_FREE_BLK_HDR_FLG | PREV_FREE_BLK_HDR_FLG | LAST_BLK_HDR_FLG) /* - * FREE_LAST_MBC_BLK_HDR_FLGS is a special flag combo used for - * distinguishing empty mbc's from allocated blocks in - * handle_delayed_dealloc(). + * HOMECOMING_MBC_BLK_HDR is a special block header combo used for + * distinguishing MBC's from allocated blocks in handle_delayed_dealloc(). */ -#define FREE_LAST_MBC_BLK_HDR_FLGS (THIS_FREE_BLK_HDR_FLG | LAST_BLK_HDR_FLG) +#define HOMECOMING_MBC_BLK_HDR (THIS_FREE_BLK_HDR_FLG | LAST_BLK_HDR_FLG) #define IS_FREE_LAST_MBC_BLK(B) \ - (((B)->bhdr & FLG_MASK) == FREE_LAST_MBC_BLK_HDR_FLGS) + (((B)->bhdr & FLG_MASK) == (THIS_FREE_BLK_HDR_FLG | LAST_BLK_HDR_FLG)) #define IS_SBC_BLK(B) (((B)->bhdr & FLG_MASK) == SBC_BLK_HDR_FLG) #define IS_MBC_BLK(B) (!IS_SBC_BLK((B))) @@ -383,6 +372,57 @@ typedef struct { typedef UWord FreeBlkFtr_t; /* Footer of a free block */ +/* This AOFF stuff really belong in erl_ao_firstfit_alloc.h */ +typedef struct AOFF_RBTree_t_ AOFF_RBTree_t; +struct AOFF_RBTree_t_ { + Block_t hdr; + AOFF_RBTree_t *parent; + AOFF_RBTree_t *left; + AOFF_RBTree_t *right; + Uint32 flags; + Uint32 max_sz; /* of all blocks in this sub-tree */ +}; + +void aoff_add_pooled_mbc(Allctr_t*, Carrier_t*); +void aoff_remove_pooled_mbc(Allctr_t*, Carrier_t*); +Carrier_t* aoff_lookup_pooled_mbc(Allctr_t*, Uint size); +void erts_aoff_larger_max_size(AOFF_RBTree_t *node); + +typedef struct { + ErtsFakeDDBlock_t homecoming_dd; + erts_atomic_t next; + erts_atomic_t prev; + Allctr_t *orig_allctr; /* read-only while carrier is alive */ + ErtsThrPrgrVal thr_prgr; + erts_atomic_t max_size; + UWord abandon_limit; + UWord blocks; + UWord blocks_size; + enum { + ERTS_MBC_IS_HOME, + ERTS_MBC_WAS_POOLED, + ERTS_MBC_WAS_TRAITOR + } state; + AOFF_RBTree_t pooled; /* node in pooled_tree */ +} ErtsAlcCPoolData_t; + +struct Carrier_t_ { + UWord chdr; + Carrier_t *next; + Carrier_t *prev; + erts_atomic_t allctr; + ErtsAlcCPoolData_t cpool; /* Overwritten by block if sbc */ +}; + +#define ERTS_ALC_CARRIER_TO_ALLCTR(C) \ + ((Allctr_t *) (erts_atomic_read_nob(&(C)->allctr) & ~FLG_MASK)) + +typedef struct { + Carrier_t *first; + Carrier_t *last; +} CarrierList_t; + + typedef Uint64 CallCounter_t; typedef struct { @@ -419,13 +459,6 @@ typedef struct { #endif -typedef union ErtsAllctrDDBlock_t_ ErtsAllctrDDBlock_t; - -union ErtsAllctrDDBlock_t_ { - erts_atomic_t atmc_next; - ErtsAllctrDDBlock_t *ptr_next; -}; - typedef struct { ErtsAllctrDDBlock_t marker; erts_atomic_t last; @@ -537,25 +570,37 @@ struct Allctr_t_ { UWord crr_set_flgs; UWord crr_clr_flgs; - /* Carriers */ + /* Carriers *employed* by this allocator */ CarrierList_t mbc_list; CarrierList_t sbc_list; struct { - /* pooled_list, traitor list and dc_list contain only - carriers _created_ by this allocator */ - ErtsDoubleLink_t pooled_list; - ErtsDoubleLink_t traitor_list; + /* pooled_tree and dc_list contain only + carriers *created* by this allocator */ + AOFF_RBTree_t* pooled_tree; CarrierList_t dc_list; UWord abandon_limit; int disable_abandon; int check_limit_count; - int util_limit; + UWord util_limit; /* acul */ + UWord in_pool_limit; /* acnl */ + UWord fblk_min_limit; /* acmfl */ struct { erts_atomic_t blocks_size; erts_atomic_t no_blocks; erts_atomic_t carriers_size; erts_atomic_t no_carriers; + CallCounter_t fail_pooled; + CallCounter_t fail_shared; + CallCounter_t fail_pend_dealloc; + CallCounter_t fail; + CallCounter_t fetch; + CallCounter_t skip_size; + CallCounter_t skip_busy; + CallCounter_t skip_not_pooled; + CallCounter_t skip_homecoming; + CallCounter_t skip_race; + CallCounter_t entrance_removed; } stat; } cpool; @@ -589,6 +634,8 @@ struct Allctr_t_ { void* (*sys_realloc)(Allctr_t *allctr, void *ptr, Uint *size_p, Uint old_size, int superalign); void (*sys_dealloc)(Allctr_t *allctr, void *ptr, Uint size, int superalign); + int (*try_set_dyn_param)(Allctr_t*, Eterm param, Uint value); + void (*init_atoms) (void); #ifdef ERTS_ALLOC_UTIL_HARD_DEBUG diff --git a/erts/emulator/beam/erl_ao_firstfit_alloc.c b/erts/emulator/beam/erl_ao_firstfit_alloc.c index 05ba1f9891..0c5545401a 100644 --- a/erts/emulator/beam/erl_ao_firstfit_alloc.c +++ b/erts/emulator/beam/erl_ao_firstfit_alloc.c @@ -20,7 +20,7 @@ /* - * Description: An "address order first fit" allocator + * Description: A family of "first fit" allocator strategies * based on a Red-Black (binary search) Tree. The search, * insert, and delete operations are all O(log n) operations * on a Red-Black Tree. @@ -40,6 +40,10 @@ * sorting order. Blocks within the same carrier are sorted * wrt size instead of address. The 'max_sz' field is maintained * in order to dismiss entire carriers with too small blocks. + * Age Order: + * Carriers are ordered by creation time instead of address. + * Oldest carrier with a large enough free block is chosen. + * No age order supported for blocks. * * Authors: Rickard Green/Sverker Eriksson */ @@ -53,10 +57,12 @@ #include "erl_ao_firstfit_alloc.h" #ifdef DEBUG +# define IS_DEBUG 1 #if 0 #define HARD_DEBUG #endif #else +# define IS_DEBUG 0 #undef HARD_DEBUG #endif @@ -92,18 +98,6 @@ #define RBT_ASSERT(x) #endif - -/* Types... */ -typedef struct AOFF_RBTree_t_ AOFF_RBTree_t; - -struct AOFF_RBTree_t_ { - Block_t hdr; - AOFF_RBTree_t *parent; - AOFF_RBTree_t *left; - AOFF_RBTree_t *right; - Uint32 flags; - Uint32 max_sz; /* of all blocks in this sub-tree */ -}; #define AOFF_BLK_SZ(B) MBC_FBLK_SZ(&(B)->hdr) /* BF block nodes keeps list of all with equal size @@ -121,6 +115,7 @@ typedef struct AOFF_Carrier_t_ AOFF_Carrier_t; struct AOFF_Carrier_t_ { Carrier_t crr; AOFF_RBTree_t rbt_node; /* My node in the carrier tree */ + Sint64 birth_time; AOFF_RBTree_t* root; /* Root of my block tree */ }; #define RBT_NODE_TO_MBC(PTR) ErtsContainerStruct((PTR), AOFF_Carrier_t, rbt_node) @@ -136,12 +131,12 @@ struct AOFF_Carrier_t_ { */ #ifdef HARD_DEBUG -# define HARD_CHECK_IS_MEMBER(ROOT,NODE) rbt_assert_is_member(ROOT,NODE) -# define HARD_CHECK_TREE(CRR,FLV,ROOT,SZ) check_tree(CRR, FLV, ROOT, SZ) -static AOFF_RBTree_t * check_tree(Carrier_t* within_crr, enum AOFF_Flavor flavor, AOFF_RBTree_t* root, Uint); +# define HARD_CHECK_IS_MEMBER(ROOT,NODE) ASSERT(rbt_is_member(ROOT,NODE)) +# define HARD_CHECK_TREE(CRR,ORDER,ROOT,SZ) check_tree(CRR, ORDER, ROOT, SZ) +static AOFF_RBTree_t * check_tree(Carrier_t*, enum AOFFSortOrder, AOFF_RBTree_t*, Uint); #else # define HARD_CHECK_IS_MEMBER(ROOT,NODE) -# define HARD_CHECK_TREE(CRR,FLV,ROOT,SZ) +# define HARD_CHECK_TREE(CRR,ORDER,ROOT,SZ) #endif @@ -179,25 +174,63 @@ static ERTS_INLINE void lower_max_size(AOFF_RBTree_t *node, else ASSERT(new_max == old_max); } -static ERTS_INLINE SWord cmp_blocks(enum AOFF_Flavor flavor, +/* + * Set possibly new larger 'max_sz' of node and propagate change toward root + */ +void erts_aoff_larger_max_size(AOFF_RBTree_t *node) +{ + AOFF_RBTree_t* x = node; + const Uint new_sz = node->hdr.bhdr; + + ASSERT(!x->left || x->left->max_sz <= x->max_sz); + ASSERT(!x->right || x->right->max_sz <= x->max_sz); + + while (new_sz > x->max_sz) { + x->max_sz = new_sz; + x = x->parent; + if (!x) + break; + } +} + +/* Compare nodes for both carrier and block trees */ +static ERTS_INLINE SWord cmp_blocks(enum AOFFSortOrder order, AOFF_RBTree_t* lhs, AOFF_RBTree_t* rhs) { ASSERT(lhs != rhs); - ASSERT(flavor == AOFF_AOFF || FBLK_TO_MBC(&lhs->hdr) == FBLK_TO_MBC(&rhs->hdr)); - if (flavor != AOFF_AOFF) { - SWord diff = (SWord)AOFF_BLK_SZ(lhs) - (SWord)AOFF_BLK_SZ(rhs); - if (diff || flavor == AOFF_BF) return diff; + if (order == FF_AGEFF) { + AOFF_Carrier_t* lc = RBT_NODE_TO_MBC(lhs); + AOFF_Carrier_t* rc = RBT_NODE_TO_MBC(rhs); + Sint64 diff = lc->birth_time - rc->birth_time; + #ifdef ARCH_64 + if (diff) + return diff; + #else + if (diff < 0) + return -1; + else if (diff > 0) + return 1; + #endif + } + else { + ASSERT(order == FF_AOFF || FBLK_TO_MBC(&lhs->hdr) == FBLK_TO_MBC(&rhs->hdr)); + if (order != FF_AOFF) { + SWord diff = (SWord)AOFF_BLK_SZ(lhs) - (SWord)AOFF_BLK_SZ(rhs); + if (diff || order == FF_BF) return diff; + } } return (char*)lhs - (char*)rhs; } -static ERTS_INLINE SWord cmp_cand_blk(enum AOFF_Flavor flavor, +/* Compare candidate block. Only for block tree */ +static ERTS_INLINE SWord cmp_cand_blk(enum AOFFSortOrder order, Block_t* cand_blk, AOFF_RBTree_t* rhs) { - if (flavor != AOFF_AOFF) { + ASSERT(order != FF_AGEFF); + if (order != FF_AOFF) { if (BLK_TO_MBC(cand_blk) == FBLK_TO_MBC(&rhs->hdr)) { SWord diff = (SWord)MBC_BLK_SZ(cand_blk) - (SWord)MBC_FBLK_SZ(&rhs->hdr); - if (diff || flavor == AOFF_BF) return diff; + if (diff || order == FF_BF) return diff; } } return (char*)cand_blk - (char*)rhs; @@ -218,11 +251,8 @@ static UWord aoff_largest_fblk_in_mbc(Allctr_t*, Carrier_t*); /* Generic tree functions used by both carrier and block trees. */ static void rbt_delete(AOFF_RBTree_t** root, AOFF_RBTree_t* del); -static void rbt_insert(enum AOFF_Flavor flavor, AOFF_RBTree_t** root, AOFF_RBTree_t* blk); +static void rbt_insert(enum AOFFSortOrder, AOFF_RBTree_t** root, AOFF_RBTree_t* blk); static AOFF_RBTree_t* rbt_search(AOFF_RBTree_t* root, Uint size); -#ifdef HARD_DEBUG -static int rbt_assert_is_member(AOFF_RBTree_t* root, AOFF_RBTree_t* node); -#endif static Eterm info_options(Allctr_t *, char *, fmtfn_t *, void *, Uint **, Uint *); static void init_atoms(void); @@ -230,10 +260,17 @@ static void init_atoms(void); static int atoms_initialized = 0; +#ifndef ERTS_HAVE_OS_MONOTONIC_TIME_SUPPORT +static erts_atomic64_t birth_time_counter; +#endif + void erts_aoffalc_init(void) { atoms_initialized = 0; +#ifndef ERTS_HAVE_OS_MONOTONIC_TIME_SUPPORT + erts_atomic64_init_nob(&birth_time_counter, 0); +#endif } Allctr_t * @@ -254,11 +291,12 @@ erts_aoffalc_start(AOFFAllctr_t *alc, sys_memcpy((void *) alc, (void *) &zero.allctr, sizeof(AOFFAllctr_t)); - alc->flavor = aoffinit->flavor; + alc->blk_order = aoffinit->blk_order; + alc->crr_order = aoffinit->crr_order; allctr->mbc_header_size = sizeof(AOFF_Carrier_t); allctr->min_mbc_size = MIN_MBC_SZ; allctr->min_mbc_first_free_size = MIN_MBC_FIRST_FREE_SZ; - allctr->min_block_size = (aoffinit->flavor == AOFF_BF ? + allctr->min_block_size = (aoffinit->blk_order == FF_BF ? sizeof(AOFF_RBTreeList_t):sizeof(AOFF_RBTree_t)); allctr->vsn_str = ERTS_ALC_AOFF_ALLOC_VSN_STR; @@ -487,9 +525,9 @@ aoff_unlink_free_block(Allctr_t *allctr, Block_t *blk) AOFF_Carrier_t *crr = (AOFF_Carrier_t*) FBLK_TO_MBC(&del->hdr); ASSERT(crr->rbt_node.hdr.bhdr == crr->root->max_sz); - HARD_CHECK_TREE(&crr->crr, alc->flavor, crr->root, 0); + HARD_CHECK_TREE(&crr->crr, alc->blk_order, crr->root, 0); - if (alc->flavor == AOFF_BF) { + if (alc->blk_order == FF_BF) { ASSERT(del->flags & IS_BF_FLG); if (IS_LIST_ELEM(del)) { /* Remove from list */ @@ -510,14 +548,14 @@ aoff_unlink_free_block(Allctr_t *allctr, Block_t *blk) replace(&crr->root, (AOFF_RBTree_t*)del, LIST_NEXT(del)); - HARD_CHECK_TREE(&crr->crr, alc->flavor, crr->root, 0); + HARD_CHECK_TREE(&crr->crr, alc->blk_order, crr->root, 0); return; } } rbt_delete(&crr->root, (AOFF_RBTree_t*)del); - HARD_CHECK_TREE(&crr->crr, alc->flavor, crr->root, 0); + HARD_CHECK_TREE(&crr->crr, alc->blk_order, crr->root, 0); /* Update the carrier tree with a potentially new (lower) max_sz */ @@ -715,32 +753,33 @@ aoff_link_free_block(Allctr_t *allctr, Block_t *block) ASSERT(allctr == ERTS_ALC_CARRIER_TO_ALLCTR(&blk_crr->crr)); ASSERT(blk_crr->rbt_node.hdr.bhdr == (blk_crr->root ? blk_crr->root->max_sz : 0)); - HARD_CHECK_TREE(&blk_crr->crr, alc->flavor, blk_crr->root, 0); + HARD_CHECK_TREE(&blk_crr->crr, alc->blk_order, blk_crr->root, 0); - rbt_insert(alc->flavor, &blk_crr->root, blk); + rbt_insert(alc->blk_order, &blk_crr->root, blk); - /* Update the carrier tree with a potentially new (larger) max_sz - */ + /* + * Update carrier tree with a potentially new (larger) max_sz + */ crr_node = &blk_crr->rbt_node; if (blk_sz > crr_node->hdr.bhdr) { - ASSERT(blk_sz == blk_crr->root->max_sz); - crr_node->hdr.bhdr = blk_sz; - while (blk_sz > crr_node->max_sz) { - crr_node->max_sz = blk_sz; - crr_node = crr_node->parent; - if (!crr_node) break; - } + ASSERT(blk_sz == blk_crr->root->max_sz); + crr_node->hdr.bhdr = blk_sz; + while (blk_sz > crr_node->max_sz) { + crr_node->max_sz = blk_sz; + crr_node = crr_node->parent; + if (!crr_node) break; + } } - HARD_CHECK_TREE(&blk_crr->crr, alc->flavor, blk_crr->root, 0); + HARD_CHECK_TREE(NULL, alc->crr_order, alc->mbc_root, 0); } static void -rbt_insert(enum AOFF_Flavor flavor, AOFF_RBTree_t** root, AOFF_RBTree_t* blk) +rbt_insert(enum AOFFSortOrder order, AOFF_RBTree_t** root, AOFF_RBTree_t* blk) { Uint blk_sz = AOFF_BLK_SZ(blk); #ifdef DEBUG - blk->flags = (flavor == AOFF_BF) ? IS_BF_FLG : 0; + blk->flags = (order == FF_BF) ? IS_BF_FLG : 0; #else blk->flags = 0; #endif @@ -760,7 +799,7 @@ rbt_insert(enum AOFF_Flavor flavor, AOFF_RBTree_t** root, AOFF_RBTree_t* blk) if (x->max_sz < blk_sz) { x->max_sz = blk_sz; } - diff = cmp_blocks(flavor, blk, x); + diff = cmp_blocks(order, blk, x); if (diff < 0) { if (!x->left) { blk->parent = x; @@ -778,7 +817,7 @@ rbt_insert(enum AOFF_Flavor flavor, AOFF_RBTree_t** root, AOFF_RBTree_t* blk) x = x->right; } else { - ASSERT(flavor == AOFF_BF); + ASSERT(order == FF_BF); ASSERT(blk->flags & IS_BF_FLG); ASSERT(x->flags & IS_BF_FLG); SET_LIST_ELEM(blk); @@ -798,7 +837,7 @@ rbt_insert(enum AOFF_Flavor flavor, AOFF_RBTree_t** root, AOFF_RBTree_t* blk) if (IS_RED(blk->parent)) tree_insert_fixup(root, blk); } - if (flavor == AOFF_BF) { + if (order == FF_BF) { SET_TREE_NODE(blk); LIST_NEXT(blk) = NULL; } @@ -826,6 +865,16 @@ rbt_search(AOFF_RBTree_t* root, Uint size) } } +Carrier_t* aoff_lookup_pooled_mbc(Allctr_t* allctr, Uint size) +{ + AOFF_RBTree_t* node; + + if (!allctr->cpool.pooled_tree) + return NULL; + node = rbt_search(allctr->cpool.pooled_tree, size); + return node ? ErtsContainerStruct(node, Carrier_t, cpool.pooled) : NULL; +} + static Block_t * aoff_get_free_block(Allctr_t *allctr, Uint size, Block_t *cand_blk, Uint cand_size) @@ -850,7 +899,7 @@ aoff_get_free_block(Allctr_t *allctr, Uint size, /* Get block within carrier tree */ #ifdef HARD_DEBUG - dbg_blk = HARD_CHECK_TREE(&crr->crr, alc->flavor, crr->root, size); + dbg_blk = HARD_CHECK_TREE(&crr->crr, alc->blk_order, crr->root, size); #endif blk = rbt_search(crr->root, size); @@ -863,7 +912,7 @@ aoff_get_free_block(Allctr_t *allctr, Uint size, if (!blk) return NULL; - if (cand_blk && cmp_cand_blk(alc->flavor, cand_blk, blk) < 0) { + if (cand_blk && cmp_cand_blk(alc->blk_order, cand_blk, blk) < 0) { return NULL; /* cand_blk was better */ } @@ -872,23 +921,32 @@ aoff_get_free_block(Allctr_t *allctr, Uint size, return (Block_t *) blk; } +static ERTS_INLINE Sint64 get_birth_time(void) +{ +#ifdef ERTS_HAVE_OS_MONOTONIC_TIME_SUPPORT + return (Sint64) erts_os_monotonic_time(); +#else + return (Sint64) erts_atomic64_inc_read_nob(&birth_time_counter); +#endif +} + static void aoff_creating_mbc(Allctr_t *allctr, Carrier_t *carrier) { AOFFAllctr_t *alc = (AOFFAllctr_t *) allctr; AOFF_Carrier_t *crr = (AOFF_Carrier_t*) carrier; AOFF_RBTree_t **root = &alc->mbc_root; - HARD_CHECK_TREE(NULL, 0, *root, 0); + HARD_CHECK_TREE(NULL, alc->crr_order, *root, 0); - /* Link carrier in address order tree - */ crr->rbt_node.hdr.bhdr = 0; - rbt_insert(AOFF_AOFF, root, &crr->rbt_node); + if (alc->crr_order == FF_AGEFF || IS_DEBUG) + crr->birth_time = get_birth_time(); + rbt_insert(alc->crr_order, root, &crr->rbt_node); /* aoff_link_free_block will add free block later */ crr->root = NULL; - HARD_CHECK_TREE(NULL, 0, *root, 0); + HARD_CHECK_TREE(NULL, alc->crr_order, *root, 0); } #define IS_CRR_IN_TREE(CRR,ROOT) \ @@ -911,27 +969,38 @@ static void aoff_add_mbc(Allctr_t *allctr, Carrier_t *carrier) AOFF_RBTree_t **root = &alc->mbc_root; ASSERT(!IS_CRR_IN_TREE(crr, *root)); - HARD_CHECK_TREE(NULL, 0, *root, 0); + HARD_CHECK_TREE(NULL, alc->crr_order, *root, 0); + + rbt_insert(alc->crr_order, root, &crr->rbt_node); + + HARD_CHECK_TREE(NULL, alc->crr_order, *root, 0); +} + +void aoff_add_pooled_mbc(Allctr_t *allctr, Carrier_t *crr) +{ + AOFF_RBTree_t **root = &allctr->cpool.pooled_tree; + + ASSERT(allctr == crr->cpool.orig_allctr); + HARD_CHECK_TREE(NULL, 0, *root, 0); /* Link carrier in address order tree */ - rbt_insert(AOFF_AOFF, root, &crr->rbt_node); + rbt_insert(FF_AOFF, root, &crr->cpool.pooled); HARD_CHECK_TREE(NULL, 0, *root, 0); } static void aoff_remove_mbc(Allctr_t *allctr, Carrier_t *carrier) { - AOFFAllctr_t *alc = (AOFFAllctr_t *) allctr; - AOFF_Carrier_t *crr = (AOFF_Carrier_t*) carrier; - AOFF_RBTree_t **root = &alc->mbc_root; + AOFF_RBTree_t **root = &((AOFFAllctr_t*)allctr)->mbc_root; + AOFF_Carrier_t *crr = (AOFF_Carrier_t*)carrier; ASSERT(allctr == ERTS_ALC_CARRIER_TO_ALLCTR(carrier)); if (!IS_CRR_IN_TREE(crr,*root)) - return; + return; - HARD_CHECK_TREE(NULL, 0, *root, 0); + HARD_CHECK_TREE(NULL, alc->crr_order, *root, 0); rbt_delete(root, &crr->rbt_node); crr->rbt_node.parent = NULL; @@ -939,9 +1008,27 @@ static void aoff_remove_mbc(Allctr_t *allctr, Carrier_t *carrier) crr->rbt_node.right = NULL; crr->rbt_node.max_sz = crr->rbt_node.hdr.bhdr; - HARD_CHECK_TREE(NULL, 0, *root, 0); + HARD_CHECK_TREE(NULL, alc->crr_order, *root, 0); +} + +void aoff_remove_pooled_mbc(Allctr_t *allctr, Carrier_t *crr) +{ + ASSERT(allctr == crr->cpool.orig_allctr); + + HARD_CHECK_TREE(NULL, 0, allctr->cpool.pooled_tree, 0); + + rbt_delete(&allctr->cpool.pooled_tree, &crr->cpool.pooled); +#ifdef DEBUG + crr->cpool.pooled.parent = NULL; + crr->cpool.pooled.left = NULL; + crr->cpool.pooled.right = NULL; + crr->cpool.pooled.max_sz = 0; +#endif + HARD_CHECK_TREE(NULL, 0, allctr->cpool.pooled_tree, 0); + } + static UWord aoff_largest_fblk_in_mbc(Allctr_t* allctr, Carrier_t* carrier) { AOFF_Carrier_t *crr = (AOFF_Carrier_t*) carrier; @@ -955,47 +1042,35 @@ static UWord aoff_largest_fblk_in_mbc(Allctr_t* allctr, Carrier_t* carrier) * info_options() */ +static const char* flavor_str[2][3] = { + {"ageffcaoff", "ageffcaobf", "ageffcbf"}, + { "aoff", "aoffcaobf", "aoffcbf"} +}; +static Eterm flavor_atoms[2][3]; + static struct { Eterm as; - Eterm aoff; - Eterm aoffcaobf; - Eterm aoffcbf; -#ifdef DEBUG - Eterm end_of_atoms; -#endif } am; -static void ERTS_INLINE atom_init(Eterm *atom, char *name) +static void ERTS_INLINE atom_init(Eterm *atom, const char *name) { - *atom = am_atom_put(name, strlen(name)); + *atom = am_atom_put(name, sys_strlen(name)); } #define AM_INIT(AM) atom_init(&am.AM, #AM) static void init_atoms(void) { -#ifdef DEBUG - Eterm *atom; -#endif + int i, j; if (atoms_initialized) return; -#ifdef DEBUG - for (atom = (Eterm *) &am; atom <= &am.end_of_atoms; atom++) { - *atom = THE_NON_VALUE; - } -#endif AM_INIT(as); - AM_INIT(aoff); - AM_INIT(aoffcaobf); - AM_INIT(aoffcbf); -#ifdef DEBUG - for (atom = (Eterm *) &am; atom < &am.end_of_atoms; atom++) { - ASSERT(*atom != THE_NON_VALUE); - } -#endif + for (i = 0; i < 2; i++) + for (j = 0; j < 3; j++) + atom_init(&flavor_atoms[i][j], flavor_str[i][j]); atoms_initialized = 1; } @@ -1021,15 +1096,16 @@ info_options(Allctr_t *allctr, { AOFFAllctr_t* alc = (AOFFAllctr_t*) allctr; Eterm res = THE_NON_VALUE; - const char* flavor_str[3] = {"aoff", "aoffcaobf", "aoffcbf"}; - Eterm flavor_atom[3] = {am.aoff, am.aoffcaobf, am.aoffcbf}; + + ASSERT(alc->crr_order >= 0 && alc->crr_order <= 1); + ASSERT(alc->blk_order >= 1 && alc->blk_order <= 3); if (print_to_p) { erts_print(*print_to_p, print_to_arg, "%sas: %s\n", prefix, - flavor_str[alc->flavor]); + flavor_str[alc->crr_order][alc->blk_order-1]); } if (hpp || szp) { @@ -1039,7 +1115,8 @@ info_options(Allctr_t *allctr, __FILE__, __LINE__);; res = NIL; - add_2tup(hpp, szp, &res, am.as, flavor_atom[alc->flavor]); + add_2tup(hpp, szp, &res, am.as, + flavor_atoms[alc->crr_order][alc->blk_order-1]); } return res; @@ -1057,7 +1134,7 @@ UWord erts_aoffalc_test(UWord op, UWord a1, UWord a2) { switch (op) { - case 0x500: return (UWord) ((AOFFAllctr_t *) a1)->flavor == AOFF_AOBF; + case 0x500: return (UWord) ((AOFFAllctr_t *) a1)->blk_order == FF_AOBF; case 0x501: { AOFF_RBTree_t *node = ((AOFFAllctr_t *) a1)->mbc_root; Uint size = (Uint) a2; @@ -1072,7 +1149,7 @@ erts_aoffalc_test(UWord op, UWord a1, UWord a2) case 0x507: return (UWord) IS_TREE_NODE((AOFF_RBTree_t *) a1); case 0x508: return (UWord) 0; /* IS_BF_ALGO */ case 0x509: return (UWord) ((AOFF_RBTree_t *) a1)->max_sz; - case 0x50a: return (UWord) ((AOFFAllctr_t *) a1)->flavor == AOFF_BF; + case 0x50a: return (UWord) ((AOFFAllctr_t *) a1)->blk_order == FF_BF; case 0x50b: return (UWord) LIST_PREV(a1); default: ASSERT(0); return ~((UWord) 0); } @@ -1085,12 +1162,13 @@ erts_aoffalc_test(UWord op, UWord a1, UWord a2) #ifdef HARD_DEBUG - -static int rbt_assert_is_member(AOFF_RBTree_t* root, AOFF_RBTree_t* node) +static int rbt_is_member(AOFF_RBTree_t* root, AOFF_RBTree_t* node) { while (node != root) { - ASSERT(node->parent); - ASSERT(node->parent->left == node || node->parent->right == node); + if (!node->parent || (node->parent->left != node && + node->parent->right != node)) { + return 0; + } node = node->parent; } return 1; @@ -1132,7 +1210,7 @@ static void print_tree(AOFF_RBTree_t*); */ static AOFF_RBTree_t * -check_tree(Carrier_t* within_crr, enum AOFF_Flavor flavor, AOFF_RBTree_t* root, Uint size) +check_tree(Carrier_t* within_crr, enum AOFFSortOrder order, AOFF_RBTree_t* root, Uint size) { AOFF_RBTree_t *res = NULL; Sint blacks; @@ -1144,7 +1222,8 @@ check_tree(Carrier_t* within_crr, enum AOFF_Flavor flavor, AOFF_RBTree_t* root, #ifdef PRINT_TREE print_tree(root); #endif - ASSERT(within_crr || flavor == AOFF_AOFF); + ASSERT((within_crr && order >= FF_AOFF) || + (!within_crr && order <= FF_AOFF)); if (!root) return res; @@ -1202,7 +1281,7 @@ check_tree(Carrier_t* within_crr, enum AOFF_Flavor flavor, AOFF_RBTree_t* root, ASSERT(((char*)x + AOFF_BLK_SZ(x)) <= ((char*)crr + CARRIER_SZ(crr))); } - if (flavor == AOFF_BF) { + if (order == FF_BF) { AOFF_RBTree_t* y = x; AOFF_RBTree_t* nxt = LIST_NEXT(y); ASSERT(IS_TREE_NODE(x)); @@ -1225,13 +1304,13 @@ check_tree(Carrier_t* within_crr, enum AOFF_Flavor flavor, AOFF_RBTree_t* root, if (x->left) { ASSERT(x->left->parent == x); - ASSERT(cmp_blocks(flavor, x->left, x) < 0); + ASSERT(cmp_blocks(order, x->left, x) < 0); ASSERT(x->left->max_sz <= x->max_sz); } if (x->right) { ASSERT(x->right->parent == x); - ASSERT(cmp_blocks(flavor, x->right, x) > 0); + ASSERT(cmp_blocks(order, x->right, x) > 0); ASSERT(x->right->max_sz <= x->max_sz); } ASSERT(x->max_sz >= AOFF_BLK_SZ(x)); @@ -1240,7 +1319,7 @@ check_tree(Carrier_t* within_crr, enum AOFF_Flavor flavor, AOFF_RBTree_t* root, || x->max_sz == (x->right ? x->right->max_sz : 0)); if (size && AOFF_BLK_SZ(x) >= size) { - if (!res || cmp_blocks(flavor, x, res) < 0) { + if (!res || cmp_blocks(order, x, res) < 0) { res = x; } } diff --git a/erts/emulator/beam/erl_ao_firstfit_alloc.h b/erts/emulator/beam/erl_ao_firstfit_alloc.h index 7349c6ab19..9cf4fc81a8 100644 --- a/erts/emulator/beam/erl_ao_firstfit_alloc.h +++ b/erts/emulator/beam/erl_ao_firstfit_alloc.h @@ -28,14 +28,16 @@ typedef struct AOFFAllctr_t_ AOFFAllctr_t; -enum AOFF_Flavor { - AOFF_AOFF = 0, - AOFF_AOBF = 1, - AOFF_BF = 2 +enum AOFFSortOrder { + FF_AGEFF = 0, + FF_AOFF = 1, + FF_AOBF = 2, + FF_BF = 3 }; typedef struct { - enum AOFF_Flavor flavor; + enum AOFFSortOrder blk_order; + enum AOFFSortOrder crr_order; } AOFFAllctrInit_t; #define ERTS_DEFAULT_AOFF_ALLCTR_INIT {0/*dummy*/} @@ -53,12 +55,12 @@ Allctr_t *erts_aoffalc_start(AOFFAllctr_t *, AOFFAllctrInit_t*, AllctrInit_t *); #define GET_ERL_ALLOC_UTIL_IMPL #include "erl_alloc_util.h" - struct AOFFAllctr_t_ { Allctr_t allctr; /* Has to be first! */ struct AOFF_RBTree_t_* mbc_root; - enum AOFF_Flavor flavor; + enum AOFFSortOrder blk_order; + enum AOFFSortOrder crr_order; }; UWord erts_aoffalc_test(UWord, UWord, UWord); diff --git a/erts/emulator/beam/erl_bestfit_alloc.c b/erts/emulator/beam/erl_bestfit_alloc.c index 6173c408e1..85fc4c3a85 100644 --- a/erts/emulator/beam/erl_bestfit_alloc.c +++ b/erts/emulator/beam/erl_bestfit_alloc.c @@ -875,7 +875,7 @@ static struct { static void ERTS_INLINE atom_init(Eterm *atom, char *name) { - *atom = am_atom_put(name, strlen(name)); + *atom = am_atom_put(name, sys_strlen(name)); } #define AM_INIT(AM) atom_init(&am.AM, #AM) diff --git a/erts/emulator/beam/erl_bif_binary.c b/erts/emulator/beam/erl_bif_binary.c index 41c2ae08d3..469f6a1ea8 100644 --- a/erts/emulator/beam/erl_bif_binary.c +++ b/erts/emulator/beam/erl_bif_binary.c @@ -61,8 +61,6 @@ static Export binary_longest_prefix_trap_export; static BIF_RETTYPE binary_longest_prefix_trap(BIF_ALIST_3); static Export binary_longest_suffix_trap_export; static BIF_RETTYPE binary_longest_suffix_trap(BIF_ALIST_3); -static Export binary_bin_to_list_trap_export; -static BIF_RETTYPE binary_bin_to_list_trap(BIF_ALIST_3); static Export binary_copy_trap_export; static BIF_RETTYPE binary_copy_trap(BIF_ALIST_2); static Uint max_loop_limit; @@ -86,10 +84,6 @@ void erts_init_bif_binary(void) am_erlang, am_binary_longest_suffix_trap, 3, &binary_longest_suffix_trap); - erts_init_trap_export(&binary_bin_to_list_trap_export, - am_erlang, am_binary_bin_to_list_trap, 3, - &binary_bin_to_list_trap); - erts_init_trap_export(&binary_copy_trap_export, am_erlang, am_binary_copy_trap, 2, &binary_copy_trap); @@ -371,7 +365,7 @@ static ACTrie *create_acdata(MyAllocator *my, Uint len, acn->d = 0; acn->final = 0; acn->h = NULL; - memset(acn->g, 0, sizeof(ACNode *) * ALPHABET_SIZE); + sys_memset(acn->g, 0, sizeof(ACNode *) * ALPHABET_SIZE); #ifdef HARDDEBUG act->idc = 0; acn->id = 0; @@ -394,7 +388,7 @@ static BMData *create_bmdata(MyAllocator *my, byte *x, Uint len, init_my_allocator(my, datasize, data); bmd = my_alloc(my, sizeof(BMData)); bmd->x = my_alloc(my,len); - memcpy(bmd->x,x,len); + sys_memcpy(bmd->x,x,len); bmd->len = len; bmd->goodshift = my_alloc(my,sizeof(Uint) * len); *the_bin = mb; @@ -439,7 +433,7 @@ static void ac_add_one_pattern(MyAllocator *my, ACTrie *act, byte *x, Uint len) nn->d = i+1; nn->h = act->root; nn->final = 0; - memset(nn->g, 0, sizeof(ACNode *) * ALPHABET_SIZE); + sys_memset(nn->g, 0, sizeof(ACNode *) * ALPHABET_SIZE); acn->g[x[i]] = nn; ++i; acn = nn; @@ -2270,7 +2264,7 @@ static BIF_RETTYPE do_longest_common(Process *p, Eterm list, int direction) if (cd[i].type == CL_TYPE_HEAP_NOALLOC) { unsigned char *tmp = cd[i].buff; cd[i].buff = erts_alloc(ERTS_ALC_T_BINARY_BUFFER, cd[i].bufflen); - memcpy(cd[i].buff,tmp,cd[i].bufflen); + sys_memcpy(cd[i].buff,tmp,cd[i].bufflen); cd[i].type = CL_TYPE_HEAP; } } @@ -2440,191 +2434,6 @@ BIF_RETTYPE binary_at_2(BIF_ALIST_2) BIF_ERROR(BIF_P,BADARG); } -#define BIN_TO_LIST_OK 0 -#define BIN_TO_LIST_TRAP 1 -/* No badarg, checked before call */ - -#define BIN_TO_LIST_LOOP_FACTOR 10 - -static int do_bin_to_list(Process *p, byte *bytes, Uint bit_offs, - Uint start, Sint *lenp, Eterm *termp) -{ - Uint reds = get_reds(p, BIN_TO_LIST_LOOP_FACTOR); /* reds can never be 0 */ - Uint len = *lenp; - Uint loops; - Eterm *hp; - Eterm term = *termp; - Uint n; - - ASSERT(reds > 0); - - loops = MIN(reds,len); - - BUMP_REDS(p, loops / BIN_TO_LIST_LOOP_FACTOR); - - hp = HAlloc(p,2*loops); - while (loops--) { - --len; - if (bit_offs) { - n = ((((Uint) bytes[start+len]) << bit_offs) | - (((Uint) bytes[start+len+1]) >> (8-bit_offs))) & 0xFF; - } else { - n = bytes[start+len]; - } - - term = CONS(hp,make_small(n),term); - hp +=2; - } - *termp = term; - *lenp = len; - if (len) { - BUMP_ALL_REDS(p); - return BIN_TO_LIST_TRAP; - } - return BIN_TO_LIST_OK; -} - - -static BIF_RETTYPE do_trap_bin_to_list(Process *p, Eterm binary, - Uint start, Sint len, Eterm sofar) -{ - Eterm *hp; - Eterm blob; - - hp = HAlloc(p,3); - hp[0] = make_pos_bignum_header(2); - hp[1] = start; - hp[2] = (Uint) len; - blob = make_big(hp); - BIF_TRAP3(&binary_bin_to_list_trap_export, p, binary, blob, sofar); -} - -static BIF_RETTYPE binary_bin_to_list_trap(BIF_ALIST_3) -{ - Eterm *ptr; - Uint start; - Sint len; - byte *bytes; - Uint bit_offs; - Uint bit_size; - Eterm res = BIF_ARG_3; - - ptr = big_val(BIF_ARG_2); - start = ptr[1]; - len = (Sint) ptr[2]; - - ERTS_GET_BINARY_BYTES(BIF_ARG_1,bytes,bit_offs,bit_size); - if (do_bin_to_list(BIF_P, bytes, bit_offs, start, &len, &res) == - BIN_TO_LIST_OK) { - BIF_RET(res); - } - return do_trap_bin_to_list(BIF_P,BIF_ARG_1,start,len,res); -} - -static BIF_RETTYPE binary_bin_to_list_common(Process *p, - Eterm bin, - Eterm epos, - Eterm elen) -{ - Uint pos; - Sint len; - size_t sz; - byte *bytes; - Uint bit_offs; - Uint bit_size; - Eterm res = NIL; - - if (is_not_binary(bin)) { - goto badarg; - } - if (!term_to_Uint(epos, &pos)) { - goto badarg; - } - if (!term_to_Sint(elen, &len)) { - goto badarg; - } - if (len < 0) { - Uint lentmp = -(Uint)len; - /* overflow */ - if ((Sint)lentmp < 0) { - goto badarg; - } - len = lentmp; - if (len > pos) { - goto badarg; - } - pos -= len; - } - /* overflow */ - if ((pos + len) < pos || (len > 0 && (pos + len) == pos)) { - goto badarg; - } - sz = binary_size(bin); - - if (pos+len > sz) { - goto badarg; - } - ERTS_GET_BINARY_BYTES(bin,bytes,bit_offs,bit_size); - if (bit_size != 0) { - goto badarg; - } - if(do_bin_to_list(p, bytes, bit_offs, pos, &len, &res) == - BIN_TO_LIST_OK) { - BIF_RET(res); - } - return do_trap_bin_to_list(p,bin,pos,len,res); - - badarg: - BIF_ERROR(p,BADARG); -} - -BIF_RETTYPE binary_bin_to_list_3(BIF_ALIST_3) -{ - return binary_bin_to_list_common(BIF_P,BIF_ARG_1,BIF_ARG_2,BIF_ARG_3); -} - -BIF_RETTYPE binary_bin_to_list_2(BIF_ALIST_2) -{ - Eterm *tp; - - if (is_not_tuple(BIF_ARG_2)) { - goto badarg; - } - tp = tuple_val(BIF_ARG_2); - if (arityval(*tp) != 2) { - goto badarg; - } - return binary_bin_to_list_common(BIF_P,BIF_ARG_1,tp[1],tp[2]); - badarg: - BIF_ERROR(BIF_P,BADARG); -} - -BIF_RETTYPE binary_bin_to_list_1(BIF_ALIST_1) -{ - Uint pos = 0; - Sint len; - byte *bytes; - Uint bit_offs; - Uint bit_size; - Eterm res = NIL; - - if (is_not_binary(BIF_ARG_1)) { - goto badarg; - } - len = binary_size(BIF_ARG_1); - ERTS_GET_BINARY_BYTES(BIF_ARG_1,bytes,bit_offs,bit_size); - if (bit_size != 0) { - goto badarg; - } - if(do_bin_to_list(BIF_P, bytes, bit_offs, pos, &len, &res) == - BIN_TO_LIST_OK) { - BIF_RET(res); - } - return do_trap_bin_to_list(BIF_P,BIF_ARG_1,pos,len,res); - badarg: - BIF_ERROR(BIF_P,BADARG); -} - HIPE_WRAPPER_BIF_DISABLE_GC(binary_list_to_bin, 1) BIF_RETTYPE binary_list_to_bin_1(BIF_ALIST_1) @@ -2747,7 +2556,7 @@ static BIF_RETTYPE do_binary_copy(Process *p, Eterm bin, Eterm en) 0); if (!(cbs->temp_alloc)) { /* alignment not needed, need to copy */ byte *tmp = erts_alloc(ERTS_ALC_T_BINARY_BUFFER,size); - memcpy(tmp,cbs->source,size); + sys_memcpy(tmp,cbs->source,size); cbs->source = tmp; cbs->source_type = BC_TYPE_HEAP; } else { @@ -2760,7 +2569,7 @@ static BIF_RETTYPE do_binary_copy(Process *p, Eterm bin, Eterm en) pos = 0; i = 0; while (pos < reds) { - memcpy(t+pos,cbs->source, size); + sys_memcpy(t+pos,cbs->source, size); pos += size; ++i; } @@ -2785,7 +2594,7 @@ static BIF_RETTYPE do_binary_copy(Process *p, Eterm bin, Eterm en) } pos = 0; while (pos < target_size) { - memcpy(t+pos,source, size); + sys_memcpy(t+pos,source, size); pos += size; } erts_free_aligned_binary_bytes(temp_alloc); @@ -2815,7 +2624,7 @@ BIF_RETTYPE binary_copy_trap(BIF_ALIST_2) if ((n-1) * size >= reds) { Uint i = 0; while ((pos - opos) < reds) { - memcpy(t+pos,cbs->source, size); + sys_memcpy(t+pos,cbs->source, size); pos += size; ++i; } @@ -2828,7 +2637,7 @@ BIF_RETTYPE binary_copy_trap(BIF_ALIST_2) Eterm resbin; Uint target_size = cbs->result->orig_size; while (pos < target_size) { - memcpy(t+pos,cbs->source, size); + sys_memcpy(t+pos,cbs->source, size); pos += size; } save = cbs->result; @@ -3220,7 +3029,7 @@ static void dump_bm_data(BMData *bm) static void dump_ac_node(ACNode *node, int indent, int ch) { int i; char *spaces = erts_alloc(ERTS_ALC_T_TMP, 10 * indent + 1); - memset(spaces,' ',10*indent); + sys_memset(spaces,' ',10*indent); spaces[10*indent] = '\0'; erts_printf("%s-> %c\n",spaces,ch); erts_printf("%sId: %u\n",spaces,(unsigned) node->id); diff --git a/erts/emulator/beam/erl_bif_chksum.c b/erts/emulator/beam/erl_bif_chksum.c index 9417803e14..9095bcd380 100644 --- a/erts/emulator/beam/erl_bif_chksum.c +++ b/erts/emulator/beam/erl_bif_chksum.c @@ -516,7 +516,7 @@ md5_2(BIF_ALIST_2) /* No need to check context, this function cannot be called with unaligned or badly sized context as it's always trapped to. */ bytes = binary_bytes(BIF_ARG_1); - memcpy(&context,bytes,sizeof(MD5_CTX)); + sys_memcpy(&context,bytes,sizeof(MD5_CTX)); rest = do_chksum(&md5_wrap,BIF_P,BIF_ARG_2,100,(void *) &context,&res, &err); if (err != 0) { @@ -564,7 +564,7 @@ md5_update_2(BIF_ALIST_2) erts_free_aligned_binary_bytes(temp_alloc); BIF_ERROR(BIF_P, BADARG); } - memcpy(&context,bytes,sizeof(MD5_CTX)); + sys_memcpy(&context,bytes,sizeof(MD5_CTX)); erts_free_aligned_binary_bytes(temp_alloc); rest = do_chksum(&md5_wrap,BIF_P,BIF_ARG_2,100,(void *) &context,&res, &err); @@ -599,7 +599,7 @@ md5_final_1(BIF_ALIST_1) goto error; } bin = erts_new_heap_binary(BIF_P, (byte *)NULL, 16, &result); - memcpy(&ctx_copy, context, sizeof(MD5_CTX)); + sys_memcpy(&ctx_copy, context, sizeof(MD5_CTX)); erts_free_aligned_binary_bytes(temp_alloc); MD5Final(result, &ctx_copy); BIF_RET(bin); diff --git a/erts/emulator/beam/erl_bif_ddll.c b/erts/emulator/beam/erl_bif_ddll.c index f673ef3194..579e9b12f4 100644 --- a/erts/emulator/beam/erl_bif_ddll.c +++ b/erts/emulator/beam/erl_bif_ddll.c @@ -1412,7 +1412,7 @@ static Eterm copy_ref(Eterm ref, Eterm *hp) ErtsORefThing *ptr; ASSERT(is_internal_ordinary_ref(ref)); ptr = ordinary_ref_thing_ptr(ref); - memcpy(hp, ptr, sizeof(ErtsORefThing)); + sys_memcpy(hp, ptr, sizeof(ErtsORefThing)); return (make_internal_ref(hp)); } @@ -1454,9 +1454,9 @@ static void set_driver_reloading(DE_Handle *dh, Process *proc, char *path, char dh->procs = p; dh->status = ERL_DE_RELOAD; dh->reload_full_path = erts_alloc(ERTS_ALC_T_DDLL_HANDLE, sys_strlen(path) + 1); - strcpy(dh->reload_full_path,path); + sys_strcpy(dh->reload_full_path,path); dh->reload_driver_name = erts_alloc(ERTS_ALC_T_DDLL_HANDLE, sys_strlen(name) + 1); - strcpy(dh->reload_driver_name,name); + sys_strcpy(dh->reload_driver_name,name); dh->reload_flags = flags; } @@ -1501,7 +1501,7 @@ static int do_load_driver_entry(DE_Handle *dh, char *path, char *name) goto error; } - if (strcmp(name, dp->driver_name) != 0) { + if (sys_strcmp(name, dp->driver_name) != 0) { res = ERL_DE_LOAD_ERROR_BAD_NAME; goto error; } @@ -1799,7 +1799,7 @@ static char *pick_list_or_atom(Eterm name_term) goto error; } name = erts_alloc(ERTS_ALC_T_DDLL_TMP_BUF, ap->len + 1); - memcpy(name,ap->name,ap->len); + sys_memcpy(name,ap->name,ap->len); name[ap->len] = '\0'; } else { if (erts_iolist_size(name_term, &name_len)) { @@ -1870,7 +1870,7 @@ static erts_driver_t *lookup_driver(char *name) { erts_driver_t *drv; assert_drv_list_locked(); - for (drv = driver_list; drv != NULL && strcmp(drv->name, name); drv = drv->next) + for (drv = driver_list; drv != NULL && sys_strcmp(drv->name, name); drv = drv->next) ; return drv; } diff --git a/erts/emulator/beam/erl_bif_info.c b/erts/emulator/beam/erl_bif_info.c index 903f54e2fb..3f9b584c2e 100644 --- a/erts/emulator/beam/erl_bif_info.c +++ b/erts/emulator/beam/erl_bif_info.c @@ -698,7 +698,6 @@ static Eterm pi_1_keys[] = { am_initial_call, am_status, am_message_queue_len, - am_messages, am_links, am_dictionary, am_trap_exit, @@ -2378,12 +2377,12 @@ BIF_RETTYPE system_info_1(BIF_ALIST_1) BIF_RET(os_version_tuple); } else if (BIF_ARG_1 == am_version) { - int n = strlen(ERLANG_VERSION); + int n = sys_strlen(ERLANG_VERSION); hp = HAlloc(BIF_P, ((sizeof ERLANG_VERSION)-1) * 2); BIF_RET(buf_to_intlist(&hp, ERLANG_VERSION, n, NIL)); } else if (BIF_ARG_1 == am_machine) { - int n = strlen(EMULATOR); + int n = sys_strlen(EMULATOR); hp = HAlloc(BIF_P, n*2); BIF_RET(buf_to_intlist(&hp, EMULATOR, n, NIL)); } @@ -2409,7 +2408,7 @@ BIF_RETTYPE system_info_1(BIF_ALIST_1) res = erts_bld_cons(hpp, hszp, erts_bld_tuple(hpp, hszp, 2, erts_atom_put((byte *)opc[i].name, - strlen(opc[i].name), + sys_strlen(opc[i].name), ERTS_ATOM_ENC_LATIN1, 1), erts_bld_uint(hpp, hszp, @@ -2805,21 +2804,21 @@ BIF_RETTYPE system_info_1(BIF_ALIST_1) Uint sz; Eterm res = NIL, tup, text; Eterm *hp = HAlloc(BIF_P, 3*(2 + 3) + /* three 2-tuples and three cons */ - 2*(strlen(erts_build_flags_CONFIG_H) + - strlen(erts_build_flags_CFLAGS) + - strlen(erts_build_flags_LDFLAGS))); + 2*(sys_strlen(erts_build_flags_CONFIG_H) + + sys_strlen(erts_build_flags_CFLAGS) + + sys_strlen(erts_build_flags_LDFLAGS))); - sz = strlen(erts_build_flags_CONFIG_H); + sz = sys_strlen(erts_build_flags_CONFIG_H); text = buf_to_intlist(&hp, erts_build_flags_CONFIG_H, sz, NIL); tup = TUPLE2(hp, am_config_h, text); hp += 3; res = CONS(hp, tup, res); hp += 2; - sz = strlen(erts_build_flags_CFLAGS); + sz = sys_strlen(erts_build_flags_CFLAGS); text = buf_to_intlist(&hp, erts_build_flags_CFLAGS, sz, NIL); tup = TUPLE2(hp, am_cflags, text); hp += 3; res = CONS(hp, tup, res); hp += 2; - sz = strlen(erts_build_flags_LDFLAGS); + sz = sys_strlen(erts_build_flags_LDFLAGS); text = buf_to_intlist(&hp, erts_build_flags_LDFLAGS, sz, NIL); tup = TUPLE2(hp, am_ldflags, text); hp += 3; res = CONS(hp, tup, res); hp += 2; @@ -3883,7 +3882,7 @@ BIF_RETTYPE erts_debug_get_internal_state_1(BIF_ALIST_1) while (ix >= atom_table_size()) { char tmp[20]; erts_snprintf(tmp, sizeof(tmp), "am%x", atom_table_size()); - erts_atom_put((byte *) tmp, strlen(tmp), ERTS_ATOM_ENC_LATIN1, 1); + erts_atom_put((byte *) tmp, sys_strlen(tmp), ERTS_ATOM_ENC_LATIN1, 1); } return make_atom(ix); } @@ -4449,7 +4448,7 @@ static Eterm lcnt_build_lock_stats_term(Eterm **hpp, Uint *szp, erts_lcnt_lock_s file = stats->file ? stats->file : "undefined"; - af = erts_atom_put((byte *)file, strlen(file), ERTS_ATOM_ENC_LATIN1, 1); + af = erts_atom_put((byte *)file, sys_strlen(file), ERTS_ATOM_ENC_LATIN1, 1); uil = erts_bld_uint( hpp, szp, stats->line); tloc = erts_bld_tuple(hpp, szp, 2, af, uil); @@ -4495,7 +4494,7 @@ static Eterm lcnt_pretty_print_lock_id(erts_lcnt_lock_info_t *info) { } else if(info->flags & ERTS_LOCK_FLAGS_CATEGORY_ALLOCATOR) { if(is_small(id) && !sys_strcmp(info->name, "alcu_allocator")) { const char *name = (const char*)ERTS_ALC_A2AD(signed_val(id)); - id = erts_atom_put((byte*)name, strlen(name), ERTS_ATOM_ENC_LATIN1, 1); + id = erts_atom_put((byte*)name, sys_strlen(name), ERTS_ATOM_ENC_LATIN1, 1); } } @@ -4515,8 +4514,8 @@ static Eterm lcnt_build_lock_term(Eterm **hpp, Uint *szp, lcnt_sample_t *sample, lock_desc = erts_lock_flags_get_type_name(info->flags); - type = erts_atom_put((byte*)lock_desc, strlen(lock_desc), ERTS_ATOM_ENC_LATIN1, 1); - name = erts_atom_put((byte*)info->name, strlen(info->name), ERTS_ATOM_ENC_LATIN1, 1); + type = erts_atom_put((byte*)lock_desc, sys_strlen(lock_desc), ERTS_ATOM_ENC_LATIN1, 1); + name = erts_atom_put((byte*)info->name, sys_strlen(info->name), ERTS_ATOM_ENC_LATIN1, 1); /* Only attempt to resolve ids when actually emitting the term. This ought * to be safe since all immediates are the same size. */ @@ -4552,11 +4551,11 @@ static Eterm lcnt_build_result_term(Eterm **hpp, Uint *szp, erts_lcnt_time_t *du dtns = bld_unstable_uint64(hpp, szp, duration->ns); tdt = erts_bld_tuple(hpp, szp, 2, dts, dtns); - adur = erts_atom_put((byte *)str_duration, strlen(str_duration), ERTS_ATOM_ENC_LATIN1, 1); + adur = erts_atom_put((byte *)str_duration, sys_strlen(str_duration), ERTS_ATOM_ENC_LATIN1, 1); tdur = erts_bld_tuple(hpp, szp, 2, adur, tdt); /* lock tuple */ - aloc = erts_atom_put((byte *)str_locks, strlen(str_locks), ERTS_ATOM_ENC_LATIN1, 1); + aloc = erts_atom_put((byte *)str_locks, sys_strlen(str_locks), ERTS_ATOM_ENC_LATIN1, 1); for(i = 0; i < current_locks->size; i++) { lloc = lcnt_build_lock_term(hpp, szp, ¤t_locks->elements[i], lloc); @@ -4610,7 +4609,7 @@ static Eterm lcnt_build_category_list(Eterm **hpp, Uint *szp, erts_lock_flags_t for(i = 0; lcnt_category_map[i].name != NULL; i++) { if(mask & lcnt_category_map[i].flag) { Eterm category = erts_atom_put((byte*)lcnt_category_map[i].name, - strlen(lcnt_category_map[i].name), + sys_strlen(lcnt_category_map[i].name), ERTS_ATOM_ENC_UTF8, 0); res = erts_bld_cons(hpp, szp, category, res); @@ -4740,14 +4739,14 @@ BIF_RETTYPE erts_debug_lcnt_control_2(BIF_ALIST_2) static void os_info_init(void) { - Eterm type = erts_atom_put((byte *) os_type, strlen(os_type), ERTS_ATOM_ENC_LATIN1, 1); + Eterm type = erts_atom_put((byte *) os_type, sys_strlen(os_type), ERTS_ATOM_ENC_LATIN1, 1); Eterm flav; int major, minor, build; char* buf = erts_alloc(ERTS_ALC_T_TMP, 1024); /* More than enough */ Eterm* hp; os_flavor(buf, 1024); - flav = erts_atom_put((byte *) buf, strlen(buf), ERTS_ATOM_ENC_LATIN1, 1); + flav = erts_atom_put((byte *) buf, sys_strlen(buf), ERTS_ATOM_ENC_LATIN1, 1); erts_free(ERTS_ALC_T_TMP, (void *) buf); hp = erts_alloc(ERTS_ALC_T_LITERAL, (3+4)*sizeof(Eterm)); os_type_tuple = TUPLE2(hp, type, flav); diff --git a/erts/emulator/beam/erl_bif_port.c b/erts/emulator/beam/erl_bif_port.c index 9f0c90ff7b..1feb892b93 100644 --- a/erts/emulator/beam/erl_bif_port.c +++ b/erts/emulator/beam/erl_bif_port.c @@ -71,7 +71,7 @@ BIF_RETTYPE erts_internal_open_port_2(BIF_ALIST_2) BIF_ERROR(BIF_P, EXC_INTERNAL_ERROR); } else if (err_type == -2) { str = erl_errno_id(err_num); - res = erts_atom_put((byte *) str, strlen(str), ERTS_ATOM_ENC_LATIN1, 1); + res = erts_atom_put((byte *) str, sys_strlen(str), ERTS_ATOM_ENC_LATIN1, 1); } else { res = am_einval; } @@ -639,6 +639,27 @@ BIF_RETTYPE port_get_data_1(BIF_ALIST_1) BIF_RET(res); } +Eterm erts_port_data_read(Port* prt) +{ + Eterm res; + erts_aint_t data; + + data = erts_atomic_read_ddrb(&prt->data); + if (data == (erts_aint_t)NULL) + return am_undefined; /* Port terminated by racing thread */ + + if ((data & 0x3) != 0) { + res = (Eterm) (UWord) data; + ASSERT(is_immed(res)); + } + else { + ErtsPortDataHeap *pdhp = (ErtsPortDataHeap *) data; + res = pdhp->data; + } + return res; +} + + /* * Open a port. Most of the work is not done here but rather in * the file io.c. @@ -1124,7 +1145,7 @@ http_bld_string(struct packet_callback_args* pca, Uint **hpp, Uint *szp, ErlHeapBin* bin = (ErlHeapBin*) *hpp; bin->thing_word = header_heap_bin(len); bin->size = len; - memcpy(bin->data, str, len); + sys_memcpy(bin->data, str, len); *hpp += size; } } @@ -1302,9 +1323,9 @@ int ssl_tls_erl(void* arg, unsigned type, unsigned major, unsigned minor, Eterm bin = new_binary(pca->p, NULL, plen+len); byte* bin_ptr = binary_bytes(bin); - memcpy(bin_ptr+plen, buf, len); + sys_memcpy(bin_ptr+plen, buf, len); if (plen) { - memcpy(bin_ptr, prefix, plen); + sys_memcpy(bin_ptr, prefix, plen); } /* {ssl_tls,NIL,ContentType,{Major,Minor},Bin} */ diff --git a/erts/emulator/beam/erl_bif_re.c b/erts/emulator/beam/erl_bif_re.c index bc819505e7..4d769c2d46 100644 --- a/erts/emulator/beam/erl_bif_re.c +++ b/erts/emulator/beam/erl_bif_re.c @@ -514,7 +514,7 @@ re_version_0(BIF_ALIST_0) Eterm ret; size_t version_size = 0; byte *version = (byte *) erts_pcre_version(); - version_size = strlen((const char *) version); + version_size = sys_strlen((const char *) version); ret = new_binary(BIF_P, version, version_size); BIF_RET(ret); } @@ -998,7 +998,7 @@ build_capture(Eterm capture_spec[CAPSPEC_SIZE], const pcre *code) has_dupnames = ((options & PCRE_DUPNAMES) != 0); for(i=0;i<top;++i) { - if (last == NULL || !has_dupnames || strcmp((char *) last+2,(char *) nametable+2)) { + if (last == NULL || !has_dupnames || sys_strcmp((char *) last+2,(char *) nametable+2)) { ASSERT(ri->num_spec >= 0); ++(ri->num_spec); if(ri->num_spec > sallocated) { @@ -1048,7 +1048,7 @@ build_capture(Eterm capture_spec[CAPSPEC_SIZE], const pcre *code) (tmpbsiz = ap->len + 1)); } } - memcpy(tmpb,ap->name,ap->len); + sys_memcpy(tmpb,ap->name,ap->len); tmpb[ap->len] = '\0'; } else { ErlDrvSizeT slen; @@ -1210,7 +1210,7 @@ re_run(Process *p, Eterm arg1, Eterm arg2, Eterm arg3) erts_pcre_fullinfo(result, NULL, PCRE_INFO_CAPTURECOUNT, &capture_count); ovsize = 3*(capture_count+1); restart.code = erts_alloc(ERTS_ALC_T_RE_SUBJECT, code_size); - memcpy(restart.code, result, code_size); + sys_memcpy(restart.code, result, code_size); erts_pcre_free(result); erts_free(ERTS_ALC_T_RE_TMP_BUF, expr); /*unicode = (pflags & PARSE_FLAG_UNICODE) ? 1 : 0;*/ @@ -1252,7 +1252,7 @@ re_run(Process *p, Eterm arg1, Eterm arg2, Eterm arg3) BIF_ERROR(p, BADARG); } restart.code = erts_alloc(ERTS_ALC_T_RE_SUBJECT, code_size); - memcpy(restart.code, code_tmp, code_size); + sys_memcpy(restart.code, code_tmp, code_size); erts_free_aligned_binary_bytes(temp_alloc); } @@ -1364,7 +1364,7 @@ handle_iolist: RestartContext *restartp = ERTS_MAGIC_BIN_DATA(mbp); Eterm magic_ref; Eterm *hp; - memcpy(restartp,&restart,sizeof(RestartContext)); + sys_memcpy(restartp,&restart,sizeof(RestartContext)); BUMP_ALL_REDS(p); hp = HAlloc(p, ERTS_MAGIC_REF_THING_SIZE); magic_ref = erts_mk_magic_ref(&hp, &MSO(p), mbp); @@ -1517,7 +1517,7 @@ re_inspect_2(BIF_ALIST_2) last = NULL; name = nametable; for(i=0;i<top;++i) { - if (last == NULL || !has_dupnames || strcmp((char *) last+2, + if (last == NULL || !has_dupnames || sys_strcmp((char *) last+2, (char *) name+2)) { ++num_names; } @@ -1531,9 +1531,9 @@ re_inspect_2(BIF_ALIST_2) name = nametable; j = 0; for(i=0;i<top;++i) { - if (last == NULL || !has_dupnames || strcmp((char *) last+2, + if (last == NULL || !has_dupnames || sys_strcmp((char *) last+2, (char *) name+2)) { - tmp_vec[j++] = new_binary(BIF_P, (byte *) name+2, strlen((char *) name+2)); + tmp_vec[j++] = new_binary(BIF_P, (byte *) name+2, sys_strlen((char *) name+2)); } last = name; name += entrysize; diff --git a/erts/emulator/beam/erl_bif_unique.h b/erts/emulator/beam/erl_bif_unique.h index 9aa631fde9..6af9d4ac8c 100644 --- a/erts/emulator/beam/erl_bif_unique.h +++ b/erts/emulator/beam/erl_bif_unique.h @@ -307,7 +307,7 @@ erts_iref_storage_clean(ErtsIRefStorage *iref) if (iref->is_magic && erts_refc_dectest(&iref->u.mb->intern.refc, 0) == 0) erts_ref_bin_free(iref->u.mb); #ifdef DEBUG - memset((void *) iref, 0xf, sizeof(ErtsIRefStorage)); + sys_memset((void *) iref, 0xf, sizeof(ErtsIRefStorage)); #endif } @@ -342,7 +342,7 @@ erts_iref_storage_make_ref(ErtsIRefStorage *iref, #ifdef DEBUG if (clean_storage) - memset((void *) iref, 0xf, sizeof(ErtsIRefStorage)); + sys_memset((void *) iref, 0xf, sizeof(ErtsIRefStorage)); #endif return make_internal_ref(hp); diff --git a/erts/emulator/beam/erl_db.c b/erts/emulator/beam/erl_db.c index 3ba0886464..a76d769283 100644 --- a/erts/emulator/beam/erl_db.c +++ b/erts/emulator/beam/erl_db.c @@ -1753,6 +1753,28 @@ BIF_RETTYPE ets_new_2(BIF_ALIST_2) BIF_RET(ret); } +/* +** Retrieves the tid() of a named ets table. +*/ +BIF_RETTYPE ets_whereis_1(BIF_ALIST_1) +{ + DbTable* tb; + Eterm res; + + if (is_not_atom(BIF_ARG_1)) { + BIF_ERROR(BIF_P, BADARG); + } + + if ((tb = db_get_table(BIF_P, BIF_ARG_1, DB_INFO, LCK_READ)) == NULL) { + BIF_RET(am_undefined); + } + + res = make_tid(BIF_P, tb); + db_unlock(tb, LCK_READ); + + BIF_RET(res); +} + /* ** The lookup BIF */ @@ -3126,7 +3148,8 @@ BIF_RETTYPE ets_info_1(BIF_ALIST_1) static Eterm fields[] = {am_protection, am_keypos, am_type, am_named_table, am_node, am_size, am_name, am_heir, am_owner, am_memory, am_compressed, am_write_concurrency, - am_read_concurrency}; + am_read_concurrency, + am_id}; Eterm results[sizeof(fields)/sizeof(Eterm)]; DbTable* tb; Eterm res; @@ -4016,7 +4039,10 @@ static Eterm table_info(Process* p, DbTable* tb, Eterm What) ret = is_table_named(tb) ? am_true : am_false; } else if (What == am_compressed) { ret = tb->common.compress ? am_true : am_false; + } else if (What == am_id) { + ret = make_tid(p, tb); } + /* * For debugging purposes */ @@ -4235,7 +4261,7 @@ erts_ets_colliding_names(Process* p, Eterm name, Uint cnt) while (index >= atom_table_size()) { char tmp[20]; erts_snprintf(tmp, sizeof(tmp), "am%x", atom_table_size()); - erts_atom_put((byte *) tmp, strlen(tmp), ERTS_ATOM_ENC_LATIN1, 1); + erts_atom_put((byte *) tmp, sys_strlen(tmp), ERTS_ATOM_ENC_LATIN1, 1); } list = CONS(hp, make_atom(index), list); hp += 2; diff --git a/erts/emulator/beam/erl_db_util.c b/erts/emulator/beam/erl_db_util.c index e017b9552b..956662aefa 100644 --- a/erts/emulator/beam/erl_db_util.c +++ b/erts/emulator/beam/erl_db_util.c @@ -91,7 +91,7 @@ do { \ (On).siz *= 2; \ (On).data \ = (((On).def == (On).data) \ - ? memcpy(erts_alloc(ERTS_ALC_T_DB_MC_STK, \ + ? sys_memcpy(erts_alloc(ERTS_ALC_T_DB_MC_STK, \ (On).siz*sizeof(*((On).data))), \ (On).def, \ DMC_DEFAULT_SIZE*sizeof(*((On).data))) \ @@ -1518,7 +1518,7 @@ restart: context.current_match < num_progs; ++context.current_match) { /* This loop is long, too long */ - memset(heap.vars, 0, heap.size * sizeof(*heap.vars)); + sys_memset(heap.vars, 0, heap.size * sizeof(*heap.vars)); t = context.matchexpr[context.current_match]; context.stack_used = 0; structure_checked = 0; @@ -2140,7 +2140,7 @@ restart: case matchEqFloat: if (!is_float(*ep)) FAIL(); - if (memcmp(float_val(*ep) + 1, pc, sizeof(double))) + if (sys_memcmp(float_val(*ep) + 1, pc, sizeof(double))) FAIL(); pc += TermWords(2); ++ep; @@ -2742,8 +2742,8 @@ Eterm db_format_dmc_err_info(Process *p, DMCErrInfo *ei) if (vnum >= 0) erts_snprintf(buff,sizeof(buff)+20,tmp->error_string, vnum); else - strcpy(buff,tmp->error_string); - sl = strlen(buff); + sys_strcpy(buff,tmp->error_string); + sl = sys_strlen(buff); shp = HAlloc(p, sl * 2 + 5); sev = (tmp->severity == dmcWarning) ? am_atom_put("warning",7) : @@ -5044,7 +5044,7 @@ static int match_compact(ErlHeapFragment *expr, DMCErrInfo *err_info) ASSERT(j < x); erts_snprintf(buff+1, sizeof(buff) - 1, "%u", (unsigned) j); /* Yes, writing directly into terms, they ARE off heap */ - *p = erts_atom_put((byte *) buff, strlen(buff), + *p = erts_atom_put((byte *) buff, sys_strlen(buff), ERTS_ATOM_ENC_LATIN1, 1); } ++p; @@ -5505,7 +5505,7 @@ void db_match_dis(Binary *bp) ++t; { double num; - memcpy(&num,t,sizeof(double)); + sys_memcpy(&num,t,sizeof(double)); t += TermWords(2); erts_printf("EqFloat\t%f\n", num); } diff --git a/erts/emulator/beam/erl_drv_thread.c b/erts/emulator/beam/erl_drv_thread.c index 71d4534ef9..4cf42fce57 100644 --- a/erts/emulator/beam/erl_drv_thread.c +++ b/erts/emulator/beam/erl_drv_thread.c @@ -439,7 +439,7 @@ erl_drv_tsd_key_create(char *name, ErlDrvTSDKey *key) name_copy = no_name; else { name_copy = erts_alloc_fnf(ERTS_ALC_T_DRV_TSD, - sizeof(char)*(strlen(name) + 1)); + sizeof(char)*(sys_strlen(name) + 1)); if (!name_copy) { *key = -1; return ENOMEM; @@ -750,7 +750,7 @@ erl_drv_steal_main_thread(char *name, + (name ? sys_strlen(name) + 1 : 0))); if (!dtid) return ENOMEM; - memset(dtid,0,sizeof(ErlDrvTid_)); + sys_memset(dtid,0,sizeof(ErlDrvTid_)); dtid->tid = (void * ) -1; dtid->drv_thr = 1; dtid->func = func; @@ -763,8 +763,8 @@ erl_drv_steal_main_thread(char *name, *tid = NULL; /* Ignore options and name... */ - memcpy(buff,&func,sizeof(void* (*)(void*))); - memcpy(buff + sizeof(void* (*)(void*)),&arg,sizeof(void *)); + sys_memcpy(buff,&func,sizeof(void* (*)(void*))); + sys_memcpy(buff + sizeof(void* (*)(void*)),&arg,sizeof(void *)); write(erts_darwin_main_thread_pipe[1],buff,buff_sz); return 0; } diff --git a/erts/emulator/beam/erl_gc.c b/erts/emulator/beam/erl_gc.c index 0da4468f9c..1c64644efc 100644 --- a/erts/emulator/beam/erl_gc.c +++ b/erts/emulator/beam/erl_gc.c @@ -116,6 +116,7 @@ typedef struct { static Uint setup_rootset(Process*, Eterm*, int, Rootset*); static void cleanup_rootset(Rootset *rootset); static Eterm *full_sweep_heaps(Process *p, + ErlHeapFragment *live_hf_end, int hibernate, Eterm *n_heap, Eterm* n_htop, char *oh, Uint oh_size, @@ -142,7 +143,7 @@ static Eterm* sweep_literal_area(Eterm* n_hp, Eterm* n_htop, static Eterm* sweep_literals_to_old_heap(Eterm* heap_ptr, Eterm* heap_end, Eterm* htop, char* src, Uint src_size); static Eterm* collect_live_heap_frags(Process* p, ErlHeapFragment *live_hf_end, - Eterm* heap, Eterm* htop, Eterm* objv, int nobj); + Eterm* htop); static int adjust_after_fullsweep(Process *p, int need, Eterm *objv, int nobj); static void shrink_new_heap(Process *p, Uint new_sz, Eterm *objv, int nobj); static void grow_new_heap(Process *p, Uint new_sz, Eterm* objv, int nobj); @@ -917,6 +918,7 @@ garbage_collect_hibernate(Process* p, int check_long_gc) htop = heap; htop = full_sweep_heaps(p, + ERTS_INVALID_HFRAG_PTR, 1, heap, htop, @@ -1161,7 +1163,7 @@ erts_garbage_collect_literals(Process* p, Eterm* literals, roots++; - while (g_sz--) { + for ( ; g_sz--; g_ptr++) { Eterm gval = *g_ptr; switch (primary_tag(gval)) { @@ -1170,26 +1172,21 @@ erts_garbage_collect_literals(Process* p, Eterm* literals, val = *ptr; if (IS_MOVED_BOXED(val)) { ASSERT(is_boxed(val)); - *g_ptr++ = val; + *g_ptr = val; } else if (ErtsInArea(ptr, area, area_size)) { - move_boxed(&ptr,val,&old_htop,g_ptr++); - } else { - g_ptr++; + move_boxed(ptr,val,&old_htop,g_ptr); } break; case TAG_PRIMARY_LIST: ptr = list_val(gval); val = *ptr; if (IS_MOVED_CONS(val)) { /* Moved */ - *g_ptr++ = ptr[1]; + *g_ptr = ptr[1]; } else if (ErtsInArea(ptr, area, area_size)) { - move_cons(&ptr,val,&old_htop,g_ptr++); - } else { - g_ptr++; - } + move_cons(ptr,val,&old_htop,g_ptr); + } break; default: - g_ptr++; break; } } @@ -1479,25 +1476,29 @@ do_minor(Process *p, ErlHeapFragment *live_hf_end, n_htop = n_heap = (Eterm*) ERTS_HEAP_ALLOC(ERTS_ALC_T_HEAP, sizeof(Eterm)*new_sz); + n = setup_rootset(p, objv, nobj, &rootset); + roots = rootset.roots; + + /* + * All allocations done. Start defile heap with move markers. + * A crash dump due to allocation failure above will see a healthy heap. + */ + if (live_hf_end != ERTS_INVALID_HFRAG_PTR) { /* * Move heap frags that we know are completely live * directly into the new young heap generation. */ - n_htop = collect_live_heap_frags(p, live_hf_end, n_heap, n_htop, - objv, nobj); + n_htop = collect_live_heap_frags(p, live_hf_end, n_htop); } - n = setup_rootset(p, objv, nobj, &rootset); - roots = rootset.roots; - GENSWEEP_NSTACK(p, old_htop, n_htop); while (n--) { Eterm* g_ptr = roots->v; Uint g_sz = roots->sz; roots++; - while (g_sz--) { + for ( ; g_sz--; g_ptr++) { gval = *g_ptr; switch (primary_tag(gval)) { @@ -1507,14 +1508,12 @@ do_minor(Process *p, ErlHeapFragment *live_hf_end, val = *ptr; if (IS_MOVED_BOXED(val)) { ASSERT(is_boxed(val)); - *g_ptr++ = val; + *g_ptr = val; } else if (ErtsInArea(ptr, mature, mature_size)) { - move_boxed(&ptr,val,&old_htop,g_ptr++); + move_boxed(ptr,val,&old_htop,g_ptr); } else if (ErtsInYoungGen(gval, ptr, oh, oh_size)) { - move_boxed(&ptr,val,&n_htop,g_ptr++); - } else { - g_ptr++; - } + move_boxed(ptr,val,&n_htop,g_ptr); + } break; } @@ -1522,19 +1521,15 @@ do_minor(Process *p, ErlHeapFragment *live_hf_end, ptr = list_val(gval); val = *ptr; if (IS_MOVED_CONS(val)) { /* Moved */ - *g_ptr++ = ptr[1]; + *g_ptr = ptr[1]; } else if (ErtsInArea(ptr, mature, mature_size)) { - move_cons(&ptr,val,&old_htop,g_ptr++); + move_cons(ptr,val,&old_htop,g_ptr); } else if (ErtsInYoungGen(gval, ptr, oh, oh_size)) { - move_cons(&ptr,val,&n_htop,g_ptr++); - } else { - g_ptr++; - } + move_cons(ptr,val,&n_htop,g_ptr); + } break; } - default: - g_ptr++; break; } } @@ -1568,9 +1563,9 @@ do_minor(Process *p, ErlHeapFragment *live_hf_end, ASSERT(is_boxed(val)); *n_hp++ = val; } else if (ErtsInArea(ptr, mature, mature_size)) { - move_boxed(&ptr,val,&old_htop,n_hp++); + move_boxed(ptr,val,&old_htop,n_hp++); } else if (ErtsInYoungGen(gval, ptr, oh, oh_size)) { - move_boxed(&ptr,val,&n_htop,n_hp++); + move_boxed(ptr,val,&n_htop,n_hp++); } else { n_hp++; } @@ -1582,9 +1577,9 @@ do_minor(Process *p, ErlHeapFragment *live_hf_end, if (IS_MOVED_CONS(val)) { *n_hp++ = ptr[1]; } else if (ErtsInArea(ptr, mature, mature_size)) { - move_cons(&ptr,val,&old_htop,n_hp++); + move_cons(ptr,val,&old_htop,n_hp++); } else if (ErtsInYoungGen(gval, ptr, oh, oh_size)) { - move_cons(&ptr,val,&n_htop,n_hp++); + move_cons(ptr,val,&n_htop,n_hp++); } else { n_hp++; } @@ -1604,10 +1599,10 @@ do_minor(Process *p, ErlHeapFragment *live_hf_end, *origptr = val; mb->base = binary_bytes(val); } else if (ErtsInArea(ptr, mature, mature_size)) { - move_boxed(&ptr,val,&old_htop,origptr); + move_boxed(ptr,val,&old_htop,origptr); mb->base = binary_bytes(mb->orig); } else if (ErtsInYoungGen(*origptr, ptr, oh, oh_size)) { - move_boxed(&ptr,val,&n_htop,origptr); + move_boxed(ptr,val,&n_htop,origptr); mb->base = binary_bytes(mb->orig); } } @@ -1733,16 +1728,8 @@ major_collection(Process* p, ErlHeapFragment *live_hf_end, n_htop = n_heap = (Eterm *) ERTS_HEAP_ALLOC(ERTS_ALC_T_HEAP, sizeof(Eterm)*new_sz); - if (live_hf_end != ERTS_INVALID_HFRAG_PTR) { - /* - * Move heap frags that we know are completely live - * directly into the heap. - */ - n_htop = collect_live_heap_frags(p, live_hf_end, n_heap, n_htop, - objv, nobj); - } - - n_htop = full_sweep_heaps(p, 0, n_heap, n_htop, oh, oh_size, objv, nobj); + n_htop = full_sweep_heaps(p, live_hf_end, 0, n_heap, n_htop, oh, oh_size, + objv, nobj); /* Move the stack to the end of the heap */ stk_sz = HEAP_END(p) - p->stop; @@ -1789,6 +1776,7 @@ major_collection(Process* p, ErlHeapFragment *live_hf_end, static Eterm * full_sweep_heaps(Process *p, + ErlHeapFragment *live_hf_end, int hibernate, Eterm *n_heap, Eterm* n_htop, char *oh, Uint oh_size, @@ -1805,6 +1793,19 @@ full_sweep_heaps(Process *p, n = setup_rootset(p, objv, nobj, &rootset); + /* + * All allocations done. Start defile heap with move markers. + * A crash dump due to allocation failure above will see a healthy heap. + */ + + if (live_hf_end != ERTS_INVALID_HFRAG_PTR) { + /* + * Move heap frags that we know are completely live + * directly into the heap. + */ + n_htop = collect_live_heap_frags(p, live_hf_end, n_htop); + } + #ifdef HIPE if (hibernate) hipe_empty_nstack(p); @@ -1818,7 +1819,7 @@ full_sweep_heaps(Process *p, Eterm g_sz = roots->sz; roots++; - while (g_sz--) { + for ( ; g_sz--; g_ptr++) { Eterm* ptr; Eterm val; Eterm gval = *g_ptr; @@ -1830,32 +1831,26 @@ full_sweep_heaps(Process *p, val = *ptr; if (IS_MOVED_BOXED(val)) { ASSERT(is_boxed(val)); - *g_ptr++ = val; + *g_ptr = val; } else if (!erts_is_literal(gval, ptr)) { - move_boxed(&ptr,val,&n_htop,g_ptr++); - } else { - g_ptr++; + move_boxed(ptr,val,&n_htop,g_ptr); } - continue; + break; } case TAG_PRIMARY_LIST: { ptr = list_val(gval); val = *ptr; if (IS_MOVED_CONS(val)) { - *g_ptr++ = ptr[1]; + *g_ptr = ptr[1]; } else if (!erts_is_literal(gval, ptr)) { - move_cons(&ptr,val,&n_htop,g_ptr++); - } else { - g_ptr++; + move_cons(ptr,val,&n_htop,g_ptr); } - continue; + break; } - default: { - g_ptr++; - continue; - } + default: + break; } } } @@ -2134,7 +2129,7 @@ sweep(Eterm *n_hp, Eterm *n_htop, ASSERT(is_boxed(val)); *n_hp++ = val; } else if (ERTS_IS_IN_SWEEP_AREA(gval, ptr)) { - move_boxed(&ptr,val,&n_htop,n_hp++); + move_boxed(ptr,val,&n_htop,n_hp++); } else { n_hp++; } @@ -2146,7 +2141,7 @@ sweep(Eterm *n_hp, Eterm *n_htop, if (IS_MOVED_CONS(val)) { *n_hp++ = ptr[1]; } else if (ERTS_IS_IN_SWEEP_AREA(gval, ptr)) { - move_cons(&ptr,val,&n_htop,n_hp++); + move_cons(ptr,val,&n_htop,n_hp++); } else { n_hp++; } @@ -2167,7 +2162,7 @@ sweep(Eterm *n_hp, Eterm *n_htop, *origptr = val; mb->base = binary_bytes(*origptr); } else if (ERTS_IS_IN_SWEEP_AREA(*origptr, ptr)) { - move_boxed(&ptr,val,&n_htop,origptr); + move_boxed(ptr,val,&n_htop,origptr); mb->base = binary_bytes(*origptr); } } @@ -2230,7 +2225,7 @@ sweep_literals_to_old_heap(Eterm* heap_ptr, Eterm* heap_end, Eterm* htop, ASSERT(is_boxed(val)); *heap_ptr++ = val; } else if (ErtsInArea(ptr, src, src_size)) { - move_boxed(&ptr,val,&htop,heap_ptr++); + move_boxed(ptr,val,&htop,heap_ptr++); } else { heap_ptr++; } @@ -2242,7 +2237,7 @@ sweep_literals_to_old_heap(Eterm* heap_ptr, Eterm* heap_end, Eterm* htop, if (IS_MOVED_CONS(val)) { *heap_ptr++ = ptr[1]; } else if (ErtsInArea(ptr, src, src_size)) { - move_cons(&ptr,val,&htop,heap_ptr++); + move_cons(ptr,val,&htop,heap_ptr++); } else { heap_ptr++; } @@ -2263,7 +2258,7 @@ sweep_literals_to_old_heap(Eterm* heap_ptr, Eterm* heap_end, Eterm* htop, *origptr = val; mb->base = binary_bytes(*origptr); } else if (ErtsInArea(ptr, src, src_size)) { - move_boxed(&ptr,val,&htop,origptr); + move_boxed(ptr,val,&htop,origptr); mb->base = binary_bytes(*origptr); } } @@ -2296,11 +2291,11 @@ move_one_area(Eterm* n_htop, char* src, Uint src_size) ASSERT(val != ERTS_HOLE_MARKER); if (is_header(val)) { ASSERT(ptr + header_arity(val) < end); - move_boxed(&ptr, val, &n_htop, &dummy_ref); + ptr = move_boxed(ptr, val, &n_htop, &dummy_ref); } else { /* must be a cons cell */ ASSERT(ptr+1 < end); - move_cons(&ptr, val, &n_htop, &dummy_ref); + move_cons(ptr, val, &n_htop, &dummy_ref); ptr += 2; } } @@ -2313,9 +2308,7 @@ move_one_area(Eterm* n_htop, char* src, Uint src_size) */ static Eterm* -collect_live_heap_frags(Process* p, ErlHeapFragment *live_hf_end, - Eterm* n_hstart, Eterm* n_htop, - Eterm* objv, int nobj) +collect_live_heap_frags(Process* p, ErlHeapFragment *live_hf_end, Eterm* n_htop) { ErlHeapFragment* qb; char* frag_begin; @@ -3281,8 +3274,7 @@ reply_gc_info(void *vgcirp) gcireq_free(vgcirp); } -void erts_sub_binary_to_heap_binary(Eterm **pp, Eterm **hpp, Eterm *orig) { - Eterm *ptr = *pp; +Eterm* erts_sub_binary_to_heap_binary(Eterm *ptr, Eterm **hpp, Eterm *orig) { Eterm *htop = *hpp; Eterm gval; ErlSubBin *sb = (ErlSubBin *)ptr; @@ -3310,7 +3302,7 @@ void erts_sub_binary_to_heap_binary(Eterm **pp, Eterm **hpp, Eterm *orig) { htop += heap_bin_size(sb->size); *hpp = htop; - *pp = ptr; + return ptr; } diff --git a/erts/emulator/beam/erl_gc.h b/erts/emulator/beam/erl_gc.h index 6a529b8443..dec0ab1143 100644 --- a/erts/emulator/beam/erl_gc.h +++ b/erts/emulator/beam/erl_gc.h @@ -33,14 +33,15 @@ #define IS_MOVED_BOXED(x) (!is_header((x))) #define IS_MOVED_CONS(x) (is_non_value((x))) -void erts_sub_binary_to_heap_binary(Eterm **pp, Eterm **hpp, Eterm *orig); +Eterm* erts_sub_binary_to_heap_binary(Eterm *ptr, Eterm **hpp, Eterm *orig); -ERTS_GLB_INLINE void move_cons(Eterm **pp, Eterm car, Eterm **hpp, Eterm *orig); +ERTS_GLB_INLINE void move_cons(Eterm *ERTS_RESTRICT ptr, Eterm car, Eterm **hpp, + Eterm *orig); #if ERTS_GLB_INLINE_INCL_FUNC_DEF -ERTS_GLB_INLINE void move_cons(Eterm **pp, Eterm car, Eterm **hpp, Eterm *orig) +ERTS_GLB_INLINE void move_cons(Eterm *ERTS_RESTRICT ptr, Eterm car, Eterm **hpp, + Eterm *orig) { - Eterm *ptr = *pp; - Eterm *htop = *hpp; + Eterm *ERTS_RESTRICT htop = *hpp; Eterm gval; htop[0] = car; /* copy car */ @@ -53,14 +54,15 @@ ERTS_GLB_INLINE void move_cons(Eterm **pp, Eterm car, Eterm **hpp, Eterm *orig) } #endif -ERTS_GLB_INLINE void move_boxed(Eterm **pp, Eterm hdr, Eterm **hpp, Eterm *orig); +ERTS_GLB_INLINE Eterm* move_boxed(Eterm *ERTS_RESTRICT ptr, Eterm hdr, Eterm **hpp, + Eterm *orig); #if ERTS_GLB_INLINE_INCL_FUNC_DEF -ERTS_GLB_INLINE void move_boxed(Eterm **pp, Eterm hdr, Eterm **hpp, Eterm *orig) +ERTS_GLB_INLINE Eterm* move_boxed(Eterm *ERTS_RESTRICT ptr, Eterm hdr, Eterm **hpp, + Eterm *orig) { Eterm gval; Sint nelts; - Eterm *ptr = *pp; - Eterm *htop = *hpp; + Eterm *ERTS_RESTRICT htop = *hpp; ASSERT(is_header(hdr)); nelts = header_arity(hdr); @@ -71,8 +73,7 @@ ERTS_GLB_INLINE void move_boxed(Eterm **pp, Eterm hdr, Eterm **hpp, Eterm *orig) /* convert sub-binary to heap-binary if applicable */ if (sb->bitsize == 0 && sb->bitoffs == 0 && sb->is_writable == 0 && sb->size <= sizeof(Eterm) * 3) { - erts_sub_binary_to_heap_binary(pp, hpp, orig); - return; + return erts_sub_binary_to_heap_binary(ptr, hpp, orig); } } nelts++; @@ -90,7 +91,7 @@ ERTS_GLB_INLINE void move_boxed(Eterm **pp, Eterm hdr, Eterm **hpp, Eterm *orig) while (nelts--) *htop++ = *ptr++; *hpp = htop; - *pp = ptr; + return ptr; } #endif diff --git a/erts/emulator/beam/erl_goodfit_alloc.c b/erts/emulator/beam/erl_goodfit_alloc.c index 50aa41b4d2..e3ba67f0af 100644 --- a/erts/emulator/beam/erl_goodfit_alloc.c +++ b/erts/emulator/beam/erl_goodfit_alloc.c @@ -170,6 +170,7 @@ static void unlink_free_block (Allctr_t *, Block_t *); static void update_last_aux_mbc (Allctr_t *, Carrier_t *); static Eterm info_options (Allctr_t *, char *, fmtfn_t *, void *, Uint **, Uint *); +static int gfalc_try_set_dyn_param(Allctr_t*, Eterm param, Uint value); static void init_atoms (void); #ifdef ERTS_ALLOC_UTIL_HARD_DEBUG @@ -250,6 +251,8 @@ erts_gfalc_start(GFAllctr_t *gfallctr, if (!erts_alcu_start(allctr, init)) return NULL; + allctr->try_set_dyn_param = gfalc_try_set_dyn_param; + if (allctr->min_block_size != MIN_BLK_SZ) return NULL; @@ -505,7 +508,7 @@ static struct { static void ERTS_INLINE atom_init(Eterm *atom, char *name) { - *atom = am_atom_put(name, strlen(name)); + *atom = am_atom_put(name, sys_strlen(name)); } #define AM_INIT(AM) atom_init(&am.AM, #AM) @@ -584,6 +587,15 @@ info_options(Allctr_t *allctr, return res; } +static int gfalc_try_set_dyn_param(Allctr_t* allctr, Eterm param, Uint value) +{ + if (param == am_sbct) { + /* Cannot change 'sbct' without rearranging buckets */ + return 0; + } + return erts_alcu_try_set_dyn_param(allctr, param, value); +} + /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *\ * NOTE: erts_gfalc_test() is only supposed to be used for testing. * * * diff --git a/erts/emulator/beam/erl_init.c b/erts/emulator/beam/erl_init.c index 4846ccd2d3..8430a5559b 100644 --- a/erts/emulator/beam/erl_init.c +++ b/erts/emulator/beam/erl_init.c @@ -238,7 +238,7 @@ progname(char *fullname) { int i; - i = strlen(fullname); + i = sys_strlen(fullname); while (i >= 0) { if ((fullname[i] != '/') && (fullname[i] != '\\')) i--; @@ -810,7 +810,7 @@ early_init(int *argc, char **argv) /* if (argc && argv) { int i = 1; while (i < *argc) { - if (strcmp(argv[i], "--") == 0) { /* end of emulator options */ + if (sys_strcmp(argv[i], "--") == 0) { /* end of emulator options */ i++; break; } @@ -892,7 +892,7 @@ early_init(int *argc, char **argv) /* else if (argv[i][2] == 'D') { char *arg; char *type = argv[i]+3; - if (strncmp(type, "Pcpu", 4) == 0) { + if (sys_strncmp(type, "Pcpu", 4) == 0) { int ptot, ponln; arg = get_arg(argv[i]+7, argv[i+1], &i); switch (sscanf(arg, "%d:%d", &ptot, &ponln)) { @@ -929,7 +929,7 @@ early_init(int *argc, char **argv) /* VERBOSE(DEBUG_SYSTEM, ("using %d:%d dirty CPU scheduler percentages\n", dirty_cpu_scheds_pctg, dirty_cpu_scheds_onln_pctg)); - } else if (strncmp(type, "cpu", 3) == 0) { + } else if (sys_strncmp(type, "cpu", 3) == 0) { int tot, onln; arg = get_arg(argv[i]+6, argv[i+1], &i); switch (sscanf(arg, "%d:%d", &tot, &onln)) { @@ -980,7 +980,7 @@ early_init(int *argc, char **argv) /* } VERBOSE(DEBUG_SYSTEM, ("using %d:%d dirty CPU scheduler(s)\n", tot, onln)); - } else if (strncmp(type, "io", 2) == 0) { + } else if (sys_strncmp(type, "io", 2) == 0) { arg = get_arg(argv[i]+5, argv[i+1], &i); dirty_io_scheds = atoi(arg); if (dirty_io_scheds < 0 || @@ -1242,7 +1242,7 @@ erl_start(int argc, char **argv) if (argv[i][0] != '-') { erts_usage(); } - if (strcmp(argv[i], "--") == 0) { /* end of emulator options */ + if (sys_strcmp(argv[i], "--") == 0) { /* end of emulator options */ i++; break; } @@ -1270,12 +1270,12 @@ erl_start(int argc, char **argv) ("using display items %d\n",display_items)); break; case 'p': - if (!strncmp(argv[i],"-pc",3)) { + if (!sys_strncmp(argv[i],"-pc",3)) { int printable_chars = ERL_PRINTABLE_CHARACTERS_LATIN1; arg = get_arg(argv[i]+3, argv[i+1], &i); - if (!strcmp(arg,"unicode")) { + if (!sys_strcmp(arg,"unicode")) { printable_chars = ERL_PRINTABLE_CHARACTERS_UNICODE; - } else if (strcmp(arg,"latin1")) { + } else if (sys_strcmp(arg,"latin1")) { erts_fprintf(stderr, "bad range of printable " "characters: %s\n", arg); erts_usage(); @@ -1287,7 +1287,7 @@ erl_start(int argc, char **argv) erts_usage(); } case 'f': - if (!strncmp(argv[i],"-fn",3)) { + if (!sys_strncmp(argv[i],"-fn",3)) { int warning_type = ERL_FILENAME_WARNING_WARNING; arg = get_arg(argv[i]+3, argv[i+1], &i); switch (*arg) { @@ -1470,9 +1470,9 @@ erl_start(int argc, char **argv) } } else if (has_prefix("maxk", sub_param)) { arg = get_arg(sub_param+4, argv[i+1], &i); - if (strcmp(arg,"true") == 0) { + if (sys_strcmp(arg,"true") == 0) { H_MAX_FLAGS |= MAX_HEAP_SIZE_KILL; - } else if (strcmp(arg,"false") == 0) { + } else if (sys_strcmp(arg,"false") == 0) { H_MAX_FLAGS &= ~MAX_HEAP_SIZE_KILL; } else { erts_fprintf(stderr, "bad max heap kill %s\n", arg); @@ -1481,9 +1481,9 @@ erl_start(int argc, char **argv) VERBOSE(DEBUG_SYSTEM, ("using max heap kill %d\n", H_MAX_FLAGS)); } else if (has_prefix("maxel", sub_param)) { arg = get_arg(sub_param+5, argv[i+1], &i); - if (strcmp(arg,"true") == 0) { + if (sys_strcmp(arg,"true") == 0) { H_MAX_FLAGS |= MAX_HEAP_SIZE_LOG; - } else if (strcmp(arg,"false") == 0) { + } else if (sys_strcmp(arg,"false") == 0) { H_MAX_FLAGS &= ~MAX_HEAP_SIZE_LOG; } else { erts_fprintf(stderr, "bad max heap error logger %s\n", arg); @@ -1601,7 +1601,7 @@ erl_start(int argc, char **argv) case 'P': /* set maximum number of processes */ arg = get_arg(argv[i]+2, argv[i+1], &i); - if (strcmp(arg, "legacy") == 0) + if (sys_strcmp(arg, "legacy") == 0) legacy_proc_tab = 1; else { errno = 0; @@ -1617,7 +1617,7 @@ erl_start(int argc, char **argv) case 'Q': /* set maximum number of ports */ arg = get_arg(argv[i]+2, argv[i+1], &i); - if (strcmp(arg, "legacy") == 0) + if (sys_strcmp(arg, "legacy") == 0) legacy_port_tab = 1; else { errno = 0; @@ -1635,11 +1635,11 @@ erl_start(int argc, char **argv) case 'S' : /* Was handled in early_init() just read past it */ if (argv[i][2] == 'D') { char* type = argv[i]+3; - if (strcmp(type, "Pcpu") == 0) + if (sys_strcmp(type, "Pcpu") == 0) (void) get_arg(argv[i]+7, argv[i+1], &i); - if (strcmp(type, "cpu") == 0) + if (sys_strcmp(type, "cpu") == 0) (void) get_arg(argv[i]+6, argv[i+1], &i); - else if (strcmp(type, "io") == 0) + else if (sys_strcmp(type, "io") == 0) (void) get_arg(argv[i]+5, argv[i+1], &i); } else if (argv[i][2] == 'P') (void) get_arg(argv[i]+3, argv[i+1], &i); @@ -1746,6 +1746,7 @@ erl_start(int argc, char **argv) } else if (has_prefix("ecio", sub_param)) { /* ignore argument, eager check io no longer used */ + arg = get_arg(sub_param+4, argv[i+1], &i); } else if (has_prefix("pp", sub_param)) { arg = get_arg(sub_param+2, argv[i+1], &i); diff --git a/erts/emulator/beam/erl_instrument.c b/erts/emulator/beam/erl_instrument.c index 634509f880..2f70e7996e 100644 --- a/erts/emulator/beam/erl_instrument.c +++ b/erts/emulator/beam/erl_instrument.c @@ -103,7 +103,7 @@ static struct { static void ERTS_INLINE atom_init(Eterm *atom, const char *name) { - *atom = am_atom_put((char *) name, strlen(name)); + *atom = am_atom_put((char *) name, sys_strlen(name)); } #define AM_INIT(AM) atom_init(&am.AM, #AM) diff --git a/erts/emulator/beam/erl_io_queue.c b/erts/emulator/beam/erl_io_queue.c index c93b5248d9..d779d1031a 100644 --- a/erts/emulator/beam/erl_io_queue.c +++ b/erts/emulator/beam/erl_io_queue.c @@ -801,12 +801,11 @@ static Eterm iol2v_make_sub_bin(iol2v_state_t *state, Eterm bin_term, ERTS_GET_REAL_BIN(bin_term, orig_pb_term, byte_offset, bit_offset, bit_size); - (void)bit_offset; - (void)bit_size; + ASSERT(bit_size == 0); sb->thing_word = HEADER_SUB_BIN; + sb->bitoffs = bit_offset; sb->bitsize = 0; - sb->bitoffs = 0; sb->orig = orig_pb_term; sb->is_writable = 0; @@ -975,7 +974,7 @@ static int iol2v_append_binary(iol2v_state_t *state, Eterm bin_term) { parent_header = binary_val(parent_binary); binary_size = binary_size(bin_term); - if (bit_offset != 0 || bit_size != 0) { + if (bit_size != 0) { return 0; } else if (binary_size == 0) { state->bytereds_spent += 1; @@ -1017,8 +1016,16 @@ static int iol2v_append_binary(iol2v_state_t *state, Eterm bin_term) { * then just copy it into the accumulator. */ iol2v_expand_acc(state, binary_size); - sys_memcpy(&(state->acc)->orig_bytes[state->acc_size], - binary_data, binary_size); + if (ERTS_LIKELY(bit_offset == 0)) { + sys_memcpy(&(state->acc)->orig_bytes[state->acc_size], + binary_data, binary_size); + } else { + ASSERT(binary_size <= ERTS_UWORD_MAX / 8); + + erts_copy_bits(binary_data, bit_offset, 1, + (byte*)&(state->acc)->orig_bytes[state->acc_size], 0, 1, + binary_size * 8); + } state->acc_size += binary_size; } else { @@ -1029,8 +1036,16 @@ static int iol2v_append_binary(iol2v_state_t *state, Eterm bin_term) { iol2v_expand_acc(state, spill); - sys_memcpy(&(state->acc)->orig_bytes[state->acc_size], - binary_data, spill); + if (ERTS_LIKELY(bit_offset == 0)) { + sys_memcpy(&(state->acc)->orig_bytes[state->acc_size], + binary_data, spill); + } else { + ASSERT(binary_size <= ERTS_UWORD_MAX / 8); + + erts_copy_bits(binary_data, bit_offset, 1, + (byte*)&(state->acc)->orig_bytes[state->acc_size], 0, 1, + spill * 8); + } state->acc_size += spill; @@ -1179,7 +1194,10 @@ BIF_RETTYPE iolist_to_iovec_1(BIF_ALIST_1) { if (is_nil(BIF_ARG_1)) { BIF_RET(NIL); } else if (is_binary(BIF_ARG_1)) { - if (binary_size(BIF_ARG_1) != 0) { + if (binary_bitsize(BIF_ARG_1) != 0) { + ASSERT(!(BIF_P->flags & F_DISABLE_GC)); + BIF_ERROR(BIF_P, BADARG); + } else if (binary_size(BIF_ARG_1) != 0) { Eterm *hp = HAlloc(BIF_P, 2); BIF_RET(CONS(hp, BIF_ARG_1, NIL)); diff --git a/erts/emulator/beam/erl_lock_check.c b/erts/emulator/beam/erl_lock_check.c index 64950fc252..773b138d92 100644 --- a/erts/emulator/beam/erl_lock_check.c +++ b/erts/emulator/beam/erl_lock_check.c @@ -259,7 +259,7 @@ static ERTS_INLINE void lc_free(void *p) { erts_lc_free_block_t *fb = (erts_lc_free_block_t *) p; #ifdef DEBUG - memset((void *) p, 0xdf, sizeof(erts_lc_free_block_t)); + sys_memset((void *) p, 0xdf, sizeof(erts_lc_free_block_t)); #endif lc_lock(); fb->next = free_blocks; @@ -289,12 +289,12 @@ static void *lc_core_alloc(void) } for (i = 1; i < ERTS_LC_FB_CHUNK_SIZE - 1; i++) { #ifdef DEBUG - memset((void *) &fbs[i], 0xdf, sizeof(erts_lc_free_block_t)); + sys_memset((void *) &fbs[i], 0xdf, sizeof(erts_lc_free_block_t)); #endif fbs[i].next = &fbs[i+1]; } #ifdef DEBUG - memset((void *) &fbs[ERTS_LC_FB_CHUNK_SIZE-1], + sys_memset((void *) &fbs[ERTS_LC_FB_CHUNK_SIZE-1], 0xdf, sizeof(erts_lc_free_block_t)); #endif lc_lock(); @@ -695,7 +695,7 @@ erts_lc_get_lock_order_id(char *name) erts_fprintf(stderr, "Missing lock name\n"); else { for (i = 0; i < ERTS_LOCK_ORDER_SIZE; i++) - if (strcmp(erts_lock_order[i].name, name) == 0) + if (sys_strcmp(erts_lock_order[i].name, name) == 0) return i; erts_fprintf(stderr, "Lock name '%s' missing in lock order " @@ -1322,12 +1322,12 @@ erts_lc_init(void) static erts_lc_free_block_t fbs[ERTS_LC_FB_CHUNK_SIZE]; for (i = 0; i < ERTS_LC_FB_CHUNK_SIZE - 1; i++) { #ifdef DEBUG - memset((void *) &fbs[i], 0xdf, sizeof(erts_lc_free_block_t)); + sys_memset((void *) &fbs[i], 0xdf, sizeof(erts_lc_free_block_t)); #endif fbs[i].next = &fbs[i+1]; } #ifdef DEBUG - memset((void *) &fbs[ERTS_LC_FB_CHUNK_SIZE-1], + sys_memset((void *) &fbs[ERTS_LC_FB_CHUNK_SIZE-1], 0xdf, sizeof(erts_lc_free_block_t)); #endif fbs[ERTS_LC_FB_CHUNK_SIZE-1].next = NULL; diff --git a/erts/emulator/beam/erl_message.c b/erts/emulator/beam/erl_message.c index abf194cf94..6f7c71ef98 100644 --- a/erts/emulator/beam/erl_message.c +++ b/erts/emulator/beam/erl_message.c @@ -616,7 +616,7 @@ erts_try_alloc_message_on_heap(Process *pp, } else { in_message_fragment: - if (!((*psp) & ERTS_PSFLG_ON_HEAP_MSGQ)) { + if ((*psp) & ERTS_PSFLG_OFF_HEAP_MSGQ) { mp = erts_alloc_message(sz, hpp); *ohpp = sz == 0 ? NULL : &mp->hfrag.off_heap; } @@ -1079,8 +1079,6 @@ erts_change_message_queue_management(Process *c_p, Eterm new_state) case am_on_heap: c_p->flags |= F_ON_HEAP_MSGQ; c_p->flags &= ~F_OFF_HEAP_MSGQ; - erts_atomic32_read_bor_nob(&c_p->state, - ERTS_PSFLG_ON_HEAP_MSGQ); /* * We are not allowed to clear ERTS_PSFLG_OFF_HEAP_MSGQ * if a off heap change is ongoing. It will be adjusted @@ -1106,8 +1104,6 @@ erts_change_message_queue_management(Process *c_p, Eterm new_state) break; case am_off_heap: c_p->flags &= ~F_ON_HEAP_MSGQ; - erts_atomic32_read_band_nob(&c_p->state, - ~ERTS_PSFLG_ON_HEAP_MSGQ); goto change_to_off_heap; default: res = THE_NON_VALUE; /* badarg */ diff --git a/erts/emulator/beam/erl_msacc.c b/erts/emulator/beam/erl_msacc.c index d659842b7e..d13d6080e1 100644 --- a/erts/emulator/beam/erl_msacc.c +++ b/erts/emulator/beam/erl_msacc.c @@ -81,7 +81,7 @@ void erts_msacc_init(void) { sizeof(Eterm)*ERTS_MSACC_STATE_COUNT); for (i = 0; i < ERTS_MSACC_STATE_COUNT; i++) { erts_msacc_state_atoms[i] = am_atom_put(erts_msacc_states[i], - strlen(erts_msacc_states[i])); + sys_strlen(erts_msacc_states[i])); } } @@ -191,7 +191,7 @@ Eterm erts_msacc_gather_stats(ErtsMsAcc *msacc, ErtsHeapFactory *factory) { map->keys = key; hp[0] = state_map; hp[1] = msacc->id; - hp[2] = am_atom_put(msacc->type,strlen(msacc->type)); + hp[2] = am_atom_put(msacc->type,sys_strlen(msacc->type)); return make_flatmap(map); } diff --git a/erts/emulator/beam/erl_msacc.h b/erts/emulator/beam/erl_msacc.h index 2588dec903..895b1ae319 100644 --- a/erts/emulator/beam/erl_msacc.h +++ b/erts/emulator/beam/erl_msacc.h @@ -159,12 +159,12 @@ struct erl_msacc_t_ { #ifdef ERTS_ENABLE_MSACC -extern erts_tsd_key_t erts_msacc_key; +extern erts_tsd_key_t ERTS_WRITE_UNLIKELY(erts_msacc_key); #ifdef ERTS_MSACC_ALWAYS_ON #define erts_msacc_enabled 1 #else -extern int erts_msacc_enabled; +extern int ERTS_WRITE_UNLIKELY(erts_msacc_enabled); #endif #define ERTS_MSACC_TSD_GET() erts_tsd_get(erts_msacc_key) diff --git a/erts/emulator/beam/erl_mtrace.c b/erts/emulator/beam/erl_mtrace.c index f2a660f085..2807b443a1 100644 --- a/erts/emulator/beam/erl_mtrace.c +++ b/erts/emulator/beam/erl_mtrace.c @@ -314,7 +314,7 @@ disable_trace(int error, char *reason, int eno) erts_fprintf(stderr, "%s: %s\n", mt_dis, reason); else { eno_str = erl_errno_id(eno); - if (strcmp(eno_str, "unknown") == 0) + if (sys_strcmp(eno_str, "unknown") == 0) erts_fprintf(stderr, "%s: %s: %d\n", mt_dis, reason, eno); else erts_fprintf(stderr, "%s: %s: %s\n", mt_dis, reason, eno_str); @@ -401,9 +401,9 @@ write_trace_header(char *nodename, char *pid, char *hostname) PUT_UI32(tracep, ERTS_MT_MAJOR_VSN); PUT_UI32(tracep, ERTS_MT_MINOR_VSN); - n_len = strlen(nodename); - h_len = strlen(hostname); - p_len = strlen(pid); + n_len = sys_strlen(nodename); + h_len = sys_strlen(hostname); + p_len = sys_strlen(pid); hdr_prolog_len = (2*UI32_SZ + 3*UI16_SZ + 3*UI32_SZ @@ -436,15 +436,15 @@ write_trace_header(char *nodename, char *pid, char *hostname) PUT_UI32(tracep, start_time.usec); PUT_UI8(tracep, (byte) n_len); - memcpy((void *) tracep, (void *) nodename, n_len); + sys_memcpy((void *) tracep, (void *) nodename, n_len); tracep += n_len; PUT_UI8(tracep, (byte) h_len); - memcpy((void *) tracep, (void *) hostname, h_len); + sys_memcpy((void *) tracep, (void *) hostname, h_len); tracep += h_len; PUT_UI8(tracep, (byte) p_len); - memcpy((void *) tracep, (void *) pid, p_len); + sys_memcpy((void *) tracep, (void *) pid, p_len); tracep += p_len; ASSERT(startp + hdr_prolog_len == tracep); @@ -472,7 +472,7 @@ write_trace_header(char *nodename, char *pid, char *hostname) str = ERTS_ALC_A2AD(i); ASSERT(str); - str_len = strlen(str); + str_len = sys_strlen(str); if (str_len >= (1 << 8)) { disable_trace(1, "Excessively large allocator string", 0); return 0; @@ -493,7 +493,7 @@ write_trace_header(char *nodename, char *pid, char *hostname) PUT_UI16(tracep, aflags); PUT_UI16(tracep, (Uint16) i); PUT_UI8( tracep, (byte) str_len); - memcpy((void *) tracep, (void *) str, str_len); + sys_memcpy((void *) tracep, (void *) str, str_len); tracep += str_len; if (erts_allctrs_info[i].alloc_util) { PUT_UI8(tracep, 2); @@ -519,7 +519,7 @@ write_trace_header(char *nodename, char *pid, char *hostname) str = ERTS_ALC_N2TD(i); ASSERT(str); - str_len = strlen(str); + str_len = sys_strlen(str); if (str_len >= (1 << 8)) { disable_trace(1, "Excessively large type string", 0); return 0; @@ -543,7 +543,7 @@ write_trace_header(char *nodename, char *pid, char *hostname) PUT_UI16(tracep, nflags); PUT_UI16(tracep, (Uint16) i); PUT_UI8(tracep, (byte) str_len); - memcpy((void *) tracep, (void *) str, str_len); + sys_memcpy((void *) tracep, (void *) str, str_len); tracep += str_len; PUT_UI16(tracep, no); ASSERT(startp + entry_sz == tracep); diff --git a/erts/emulator/beam/erl_nif.c b/erts/emulator/beam/erl_nif.c index 94ebf88b56..c60cc7fecf 100644 --- a/erts/emulator/beam/erl_nif.c +++ b/erts/emulator/beam/erl_nif.c @@ -237,9 +237,11 @@ static void cache_env(ErlNifEnv* env); static void full_flush_env(ErlNifEnv *env); static void flush_env(ErlNifEnv* env); -/* Temporary object header, auto-deallocated when NIF returns - * or when independent environment is cleared. - */ +/* Temporary object header, auto-deallocated when NIF returns or when + * independent environment is cleared. + * + * The payload can be accessed with &tmp_obj_ptr[1] but keep in mind that its + * first element must not require greater alignment than `next`. */ struct enif_tmp_obj_t { struct enif_tmp_obj_t* next; void (*dtor)(struct enif_tmp_obj_t*); @@ -256,6 +258,46 @@ static ERTS_INLINE void free_tmp_objs(ErlNifEnv* env) } } +/* Whether the given environment is bound to a process and will be cleaned up + * when the NIF returns. It's safe to use temp_alloc for objects in + * env->tmp_obj_list when this is true. */ +static ERTS_INLINE int is_proc_bound(ErlNifEnv *env) +{ + return env->mod_nif != NULL; +} + +/* Allocates and attaches an object to the given environment, running its + * destructor when the environment is cleared. To avoid temporary variables the + * address of the allocated object is returned instead of the enif_tmp_obj_t. + * + * The destructor *must* call `erts_free(tmp_obj->allocator, tmp_obj)` to free + * the object. If the destructor needs to refer to the allocated object its + * address will be &tmp_obj[1]. */ +static ERTS_INLINE void *alloc_tmp_obj(ErlNifEnv *env, size_t size, + void (*dtor)(struct enif_tmp_obj_t*)) { + struct enif_tmp_obj_t *tmp_obj; + ErtsAlcType_t allocator; + + allocator = is_proc_bound(env) ? ERTS_ALC_T_TMP : ERTS_ALC_T_NIF; + + tmp_obj = erts_alloc(allocator, sizeof(struct enif_tmp_obj_t) + MAX(1, size)); + + tmp_obj->next = env->tmp_obj_list; + tmp_obj->allocator = allocator; + tmp_obj->dtor = dtor; + + env->tmp_obj_list = tmp_obj; + + return (void*)&tmp_obj[1]; +} + +/* Generic destructor for objects allocated through alloc_tmp_obj that don't + * care about their payload. */ +static void tmp_alloc_dtor(struct enif_tmp_obj_t *tmp_obj) +{ + erts_free(tmp_obj->allocator, tmp_obj); +} + void erts_post_nif(ErlNifEnv* env) { erts_unblock_fpe(env->fpe_was_unmasked); @@ -446,6 +488,7 @@ static void cache_env(ErlNifEnv* env) env->hp_end = env->heap_frag->mem + env->heap_frag->alloc_size; } } + void* enif_priv_data(ErlNifEnv* env) { return env->mod_nif->priv_data; @@ -486,7 +529,7 @@ setup_nif_env(struct enif_msg_environment_t* msg_env, msg_env->env.tmp_obj_list = NULL; msg_env->env.proc = &msg_env->phony_proc; msg_env->env.exception_thrown = 0; - memset(&msg_env->phony_proc, 0, sizeof(Process)); + sys_memset(&msg_env->phony_proc, 0, sizeof(Process)); HEAP_START(&msg_env->phony_proc) = phony_heap; HEAP_TOP(&msg_env->phony_proc) = phony_heap; HEAP_LIMIT(&msg_env->phony_proc) = phony_heap; @@ -1019,11 +1062,6 @@ int enif_is_number(ErlNifEnv* env, ERL_NIF_TERM term) return is_number(term); } -static ERTS_INLINE int is_proc_bound(ErlNifEnv* env) -{ - return env->mod_nif != NULL; -} - static void aligned_binary_dtor(struct enif_tmp_obj_t* obj) { erts_free_aligned_binary_bytes_extra((byte*)obj, obj->allocator); @@ -1058,22 +1096,14 @@ int enif_inspect_binary(ErlNifEnv* env, Eterm bin_term, ErlNifBinary* bin) u.tmp->dtor = &aligned_binary_dtor; env->tmp_obj_list = u.tmp; } - bin->bin_term = bin_term; bin->size = binary_size(bin_term); bin->ref_bin = NULL; ADD_READONLY_CHECK(env, bin->data, bin->size); return 1; } -static void tmp_alloc_dtor(struct enif_tmp_obj_t* obj) -{ - erts_free(obj->allocator, obj); -} - int enif_inspect_iolist_as_binary(ErlNifEnv* env, Eterm term, ErlNifBinary* bin) { - struct enif_tmp_obj_t* tobj; - ErtsAlcType_t allocator; ErlDrvSizeT sz; if (is_binary(term)) { return enif_inspect_binary(env,term,bin); @@ -1081,7 +1111,6 @@ int enif_inspect_iolist_as_binary(ErlNifEnv* env, Eterm term, ErlNifBinary* bin) if (is_nil(term)) { bin->data = (unsigned char*) &bin->data; /* dummy non-NULL */ bin->size = 0; - bin->bin_term = THE_NON_VALUE; bin->ref_bin = NULL; return 1; } @@ -1089,16 +1118,8 @@ int enif_inspect_iolist_as_binary(ErlNifEnv* env, Eterm term, ErlNifBinary* bin) return 0; } - allocator = is_proc_bound(env) ? ERTS_ALC_T_TMP : ERTS_ALC_T_NIF; - tobj = erts_alloc(allocator, sz + sizeof(struct enif_tmp_obj_t)); - tobj->allocator = allocator; - tobj->next = env->tmp_obj_list; - tobj->dtor = &tmp_alloc_dtor; - env->tmp_obj_list = tobj; - - bin->data = (unsigned char*) &tobj[1]; + bin->data = alloc_tmp_obj(env, sz, &tmp_alloc_dtor); bin->size = sz; - bin->bin_term = THE_NON_VALUE; bin->ref_bin = NULL; erts_iolist_to_buf(term, (char*) bin->data, sz); ADD_READONLY_CHECK(env, bin->data, bin->size); @@ -1116,7 +1137,6 @@ int enif_alloc_binary(size_t size, ErlNifBinary* bin) bin->size = size; bin->data = (unsigned char*) refbin->orig_bytes; - bin->bin_term = THE_NON_VALUE; bin->ref_bin = refbin; return 1; } @@ -1150,12 +1170,10 @@ void enif_release_binary(ErlNifBinary* bin) { if (bin->ref_bin != NULL) { Binary* refbin = bin->ref_bin; - ASSERT(bin->bin_term == THE_NON_VALUE); erts_bin_release(refbin); } #ifdef DEBUG bin->data = NULL; - bin->bin_term = THE_NON_VALUE; bin->ref_bin = NULL; #endif } @@ -1312,29 +1330,51 @@ int enif_get_string(ErlNifEnv *env, ERL_NIF_TERM list, char* buf, unsigned len, Eterm enif_make_binary(ErlNifEnv* env, ErlNifBinary* bin) { - if (bin->bin_term != THE_NON_VALUE) { - return bin->bin_term; - } - else if (bin->ref_bin != NULL) { - Binary* bptr = bin->ref_bin; - Eterm bin_term; - - bin_term = erts_build_proc_bin(&MSO(env->proc), - alloc_heap(env, PROC_BIN_SIZE), - bptr); - if (erts_refc_read(&bptr->intern.refc, 1) == 1) { - /* Total ownership transfer */ - bin->ref_bin = NULL; - bin->bin_term = bin_term; - } - return bin_term; - } - else { - flush_env(env); - bin->bin_term = new_binary(env->proc, bin->data, bin->size); - cache_env(env); - return bin->bin_term; + Eterm bin_term; + + if (bin->ref_bin != NULL) { + Binary* binary = bin->ref_bin; + + /* If the binary is smaller than the heap binary limit we'll return a + * heap binary to reduce the number of small refc binaries in the + * system. We can't simply release the refc binary right away however; + * the documentation states that the binary should be considered + * read-only from this point on, which implies that it should still be + * readable. + * + * We could keep it alive until we return by adding it to the temporary + * object list, but that requires an off-heap allocation which is + * potentially quite slow, so we create a dummy ProcBin instead and + * rely on the next minor GC to get rid of it. */ + if (bin->size <= ERL_ONHEAP_BIN_LIMIT) { + ErlHeapBin* hb; + + hb = (ErlHeapBin*)alloc_heap(env, heap_bin_size(bin->size)); + hb->thing_word = header_heap_bin(bin->size); + hb->size = bin->size; + + sys_memcpy(hb->data, bin->data, bin->size); + + erts_build_proc_bin(&MSO(env->proc), + alloc_heap(env, PROC_BIN_SIZE), + binary); + + bin_term = make_binary(hb); + } else { + bin_term = erts_build_proc_bin(&MSO(env->proc), + alloc_heap(env, PROC_BIN_SIZE), + binary); + } + + /* Our (possibly shared) ownership has been transferred to the term. */ + bin->ref_bin = NULL; + } else { + flush_env(env); + bin_term = new_binary(env->proc, bin->data, bin->size); + cache_env(env); } + + return bin_term; } Eterm enif_make_sub_binary(ErlNifEnv* env, ERL_NIF_TERM bin_term, @@ -3284,8 +3324,8 @@ typedef struct { Eterm sublist_start; Eterm sublist_end; - UWord offheap_size; - UWord onheap_size; + UWord referenced_size; + UWord copied_size; UWord iovec_len; } iovec_slice_t; @@ -3295,16 +3335,16 @@ static int examine_iovec_term(Eterm list, UWord max_length, iovec_slice_t *resul result->sublist_start = list; result->sublist_length = 0; - result->offheap_size = 0; - result->onheap_size = 0; + result->referenced_size = 0; + result->copied_size = 0; result->iovec_len = 0; lookahead = result->sublist_start; while (is_list(lookahead)) { - Eterm *binary_header, binary; + UWord byte_size; + Eterm binary; Eterm *cell; - UWord size; cell = list_val(lookahead); binary = CAR(cell); @@ -3313,35 +3353,36 @@ static int examine_iovec_term(Eterm list, UWord max_length, iovec_slice_t *resul return 0; } - size = binary_size(binary); - binary_header = binary_val(binary); + byte_size = binary_size(binary); + + if (byte_size > 0) { + int bit_offset, bit_size; + Eterm parent_binary; + UWord byte_offset; - if (size > 0) { - /* If we're a sub-binary we'll need to check our underlying binary - * to determine whether we're on-heap or not. */ - if (thing_subtag(*binary_header) == SUB_BINARY_SUBTAG) { - ErlSubBin *sb = (ErlSubBin*)binary_header; + int requires_copying; - /* Reject bitstrings */ - if((sb->bitoffs + sb->bitsize) > 0) { - return 0; - } + ERTS_GET_REAL_BIN(binary, parent_binary, byte_offset, + bit_offset, bit_size); - ASSERT(size <= binary_size(sb->orig)); - binary_header = binary_val(sb->orig); + (void)byte_offset; + + if (bit_size != 0) { + return 0; } - if (thing_subtag(*binary_header) == HEAP_BINARY_SUBTAG) { - ASSERT(size <= ERL_ONHEAP_BIN_LIMIT); + /* If we're unaligned or an on-heap binary we'll need to copy + * ourselves over to a temporary buffer. */ + requires_copying = (bit_offset != 0) || + thing_subtag(*binary_val(parent_binary)) == HEAP_BINARY_SUBTAG; - result->iovec_len += 1; - result->onheap_size += size; + if (requires_copying) { + result->copied_size += byte_size; } else { - ASSERT(thing_subtag(*binary_header) == REFC_BINARY_SUBTAG); - - result->iovec_len += 1 + size / MAX_SYSIOVEC_IOVLEN; - result->offheap_size += size; + result->referenced_size += byte_size; } + + result->iovec_len += 1 + byte_size / MAX_SYSIOVEC_IOVLEN; } result->sublist_length += 1; @@ -3361,7 +3402,9 @@ static int examine_iovec_term(Eterm list, UWord max_length, iovec_slice_t *resul return 1; } -static void inspect_raw_binary_data(Eterm binary, ErlNifBinary *result) { +static void marshal_iovec_binary(Eterm binary, ErlNifBinary *copy_buffer, + UWord *copy_offset, ErlNifBinary *result) { + Eterm *parent_header; Eterm parent_binary; @@ -3372,10 +3415,11 @@ static void inspect_raw_binary_data(Eterm binary, ErlNifBinary *result) { ERTS_GET_REAL_BIN(binary, parent_binary, byte_offset, bit_offset, bit_size); + ASSERT(bit_size == 0); + parent_header = binary_val(parent_binary); result->size = binary_size(binary); - result->bin_term = binary; if (thing_subtag(*parent_header) == REFC_BINARY_SUBTAG) { ProcBin *pb = (ProcBin*)parent_header; @@ -3398,24 +3442,48 @@ static void inspect_raw_binary_data(Eterm binary, ErlNifBinary *result) { result->data = &((unsigned char*)&hb->data)[byte_offset]; result->ref_bin = NULL; } + + /* If this isn't an *aligned* refc binary, copy its contents to the buffer + * and reference that instead. */ + + if (result->ref_bin == NULL || bit_offset != 0) { + ASSERT(copy_buffer->ref_bin != NULL && copy_buffer->data != NULL); + ASSERT(result->size <= (copy_buffer->size - *copy_offset)); + + if (bit_offset == 0) { + sys_memcpy(©_buffer->data[*copy_offset], + result->data, result->size); + } else { + erts_copy_bits(result->data, bit_offset, 1, + (byte*)©_buffer->data[*copy_offset], 0, 1, + result->size * 8); + } + + result->data = ©_buffer->data[*copy_offset]; + result->ref_bin = copy_buffer->ref_bin; + + *copy_offset += result->size; + } } static int fill_iovec_with_slice(ErlNifEnv *env, iovec_slice_t *slice, ErlNifIOVec *iovec) { - UWord onheap_offset, iovec_idx; - ErlNifBinary onheap_data; + ErlNifBinary copy_buffer = {0}; + UWord copy_offset, iovec_idx; Eterm sublist_iterator; - /* Set up a common refc binary for all on-heap binaries. */ - if (slice->onheap_size > 0) { - if (!enif_alloc_binary(slice->onheap_size, &onheap_data)) { + /* Set up a common refc binary for all on-heap and unaligned binaries. */ + if (slice->copied_size > 0) { + if (!enif_alloc_binary(slice->copied_size, ©_buffer)) { return 0; } + + ASSERT(copy_buffer.ref_bin != NULL); } sublist_iterator = slice->sublist_start; - onheap_offset = 0; + copy_offset = 0; iovec_idx = 0; while (sublist_iterator != slice->sublist_end) { @@ -3423,20 +3491,7 @@ static int fill_iovec_with_slice(ErlNifEnv *env, Eterm *cell; cell = list_val(sublist_iterator); - inspect_raw_binary_data(CAR(cell), &raw_data); - - /* If this isn't a refc binary, copy its contents to the onheap buffer - * and reference that instead. */ - if (raw_data.size > 0 && raw_data.ref_bin == NULL) { - ASSERT(onheap_offset < onheap_data.size); - ASSERT(slice->onheap_size > 0); - - sys_memcpy(&onheap_data.data[onheap_offset], - raw_data.data, raw_data.size); - - raw_data.data = &onheap_data.data[onheap_offset]; - raw_data.ref_bin = onheap_data.ref_bin; - } + marshal_iovec_binary(CAR(cell), ©_buffer, ©_offset, &raw_data); while (raw_data.size > 0) { UWord chunk_len = MIN(raw_data.size, MAX_SYSIOVEC_IOVLEN); @@ -3467,16 +3522,18 @@ static int fill_iovec_with_slice(ErlNifEnv *env, erts_refc_inc(&refc_binary->intern.refc, 1); } - if (slice->onheap_size > 0) { + if (slice->copied_size > 0) { /* Transfer ownership to the iovec; we've taken references to it in * the above loop. */ - enif_release_binary(&onheap_data); + enif_release_binary(©_buffer); } } else { - if (slice->onheap_size > 0) { - /* Attach the binary to our environment and let the GC take care of - * it after returning. */ - enif_make_binary(env, &onheap_data); + if (slice->copied_size > 0) { + /* Attach the binary to our environment and let the next minor GC + * get rid of it. This is slightly faster than using the tmp object + * list since it avoids off-heap allocations. */ + erts_build_proc_bin(&MSO(env->proc), + alloc_heap(env, PROC_BIN_SIZE), copy_buffer.ref_bin); } } @@ -3502,19 +3559,14 @@ static int create_iovec_from_slice(ErlNifEnv *env, alloc_size = binv_offset; alloc_size += slice->iovec_len * sizeof(Binary*); - /* If we have an environment we'll attach the allocated data to it. The - * GC will take care of releasing it later on. */ + /* When the user passes an environment, we attach the iovec to it so + * the user won't have to bother managing it (similar to + * enif_inspect_binary). It'll disappear once the environment is + * cleaned up. */ if (env != NULL) { - ErlNifBinary gc_bin; - - if (!enif_alloc_binary(alloc_size, &gc_bin)) { - return 0; - } - - alloc_base = (char*)gc_bin.data; - enif_make_binary(env, &gc_bin); + alloc_base = alloc_tmp_obj(env, alloc_size, &tmp_alloc_dtor); } else { - alloc_base = enif_alloc(alloc_size); + alloc_base = erts_alloc(ERTS_ALC_T_NIF, alloc_size); } iovec = (ErlNifIOVec*)alloc_base; @@ -3523,12 +3575,12 @@ static int create_iovec_from_slice(ErlNifEnv *env, iovec->flags = 0; } - iovec->size = slice->offheap_size + slice->onheap_size; + iovec->size = slice->referenced_size + slice->copied_size; iovec->iovcnt = slice->iovec_len; if(!fill_iovec_with_slice(env, slice, iovec)) { if (env == NULL && !(iovec->flags & ERL_NIF_IOVEC_FLAGS_PREALLOC)) { - enif_free(iovec); + erts_free(ERTS_ALC_T_NIF, iovec); } return 0; @@ -4218,34 +4270,31 @@ static unsigned calc_checksum(unsigned char* ptr, unsigned size); struct readonly_check_t { - struct enif_tmp_obj_t hdr; unsigned char* ptr; unsigned size; unsigned checksum; }; static void add_readonly_check(ErlNifEnv* env, unsigned char* ptr, unsigned sz) { - ErtsAlcType_t allocator = is_proc_bound(env) ? ERTS_ALC_T_TMP : ERTS_ALC_T_NIF; - struct readonly_check_t* obj = erts_alloc(allocator, - sizeof(struct readonly_check_t)); - obj->hdr.allocator = allocator; - obj->hdr.next = env->tmp_obj_list; - env->tmp_obj_list = &obj->hdr; - obj->hdr.dtor = &readonly_check_dtor; + struct readonly_check_t* obj; + + obj = alloc_tmp_obj(env, sizeof(struct readonly_check_t), + &readonly_check_dtor); + obj->ptr = ptr; obj->size = sz; - obj->checksum = calc_checksum(ptr, sz); + obj->checksum = calc_checksum(ptr, sz); } -static void readonly_check_dtor(struct enif_tmp_obj_t* o) +static void readonly_check_dtor(struct enif_tmp_obj_t* tmp_obj) { - struct readonly_check_t* obj = (struct readonly_check_t*) o; - unsigned chksum = calc_checksum(obj->ptr, obj->size); - if (chksum != obj->checksum) { + struct readonly_check_t* ro_check = (struct readonly_check_t*)&tmp_obj[1]; + unsigned chksum = calc_checksum(ro_check->ptr, ro_check->size); + if (chksum != ro_check->checksum) { fprintf(stderr, "\r\nReadonly data written by NIF, checksums differ" - " %x != %x\r\nABORTING\r\n", chksum, obj->checksum); + " %x != %x\r\nABORTING\r\n", chksum, ro_check->checksum); abort(); } - erts_free(obj->hdr.allocator, obj); + erts_free(tmp_obj->allocator, tmp_obj); } static unsigned calc_checksum(unsigned char* ptr, unsigned size) { @@ -4320,7 +4369,7 @@ static void get_string_maybe(ErlNifEnv *env, const ERL_NIF_TERM term, str_bin.size > bufsiz) { *ptr = NULL; } else { - memcpy(buf, (char *) str_bin.data, str_bin.size); + sys_memcpy(buf, (char *) str_bin.data, str_bin.size); buf[str_bin.size] = '\0'; *ptr = buf; } @@ -4337,7 +4386,7 @@ ERL_NIF_TERM erl_nif_user_trace_s1(ErlNifEnv* env, int argc, message_bin.size > MESSAGE_BUFSIZ) { return am_badarg; } - memcpy(messagebuf, (char *) message_bin.data, message_bin.size); + sys_memcpy(messagebuf, (char *) message_bin.data, message_bin.size); messagebuf[message_bin.size] = '\0'; DTRACE1(user_trace_s1, messagebuf); return am_true; diff --git a/erts/emulator/beam/erl_nif.h b/erts/emulator/beam/erl_nif.h index 053f7673c4..a99b4db705 100644 --- a/erts/emulator/beam/erl_nif.h +++ b/erts/emulator/beam/erl_nif.h @@ -137,8 +137,9 @@ typedef struct unsigned char* data; /* Internals (avert your eyes) */ - ERL_NIF_TERM bin_term; void* ref_bin; + /* for future additions to be ABI compatible (same struct size) */ + void* __spare__[2]; }ErlNifBinary; #if (defined(__WIN32__) || defined(_WIN32) || defined(_WIN32_)) diff --git a/erts/emulator/beam/erl_node_tables.c b/erts/emulator/beam/erl_node_tables.c index eaf133f5c0..e8901a652f 100644 --- a/erts/emulator/beam/erl_node_tables.c +++ b/erts/emulator/beam/erl_node_tables.c @@ -170,7 +170,7 @@ dist_table_alloc(void *dep_tmpl) dep->cid = NIL; erts_atomic_init_nob(&dep->input_handler, (erts_aint_t) NIL); dep->connection_id = 0; - dep->status = 0; + dep->state = ERTS_DE_STATE_IDLE; dep->flags = 0; dep->version = 0; @@ -223,7 +223,7 @@ dist_table_free(void *vdep) DistEntry *dep = (DistEntry *) vdep; ASSERT(de_refc_read(dep, -1) == -1); - ASSERT(dep->status == 0); + ASSERT(dep->state == ERTS_DE_STATE_IDLE); ASSERT(is_nil(dep->cid)); ASSERT(dep->nlinks == NULL); ASSERT(dep->node_links == NULL); @@ -556,14 +556,14 @@ erts_set_dist_entry_not_connected(DistEntry *dep) ASSERT(dep != erts_this_dist_entry); - if (dep->status & ERTS_DE_SFLG_PENDING) { + if (dep->state == ERTS_DE_STATE_PENDING) { ASSERT(is_nil(dep->cid)); ASSERT(erts_no_of_pending_dist_entries > 0); erts_no_of_pending_dist_entries--; head = &erts_pending_dist_entries; } else { - ASSERT(dep->status != 0); + ASSERT(dep->state != ERTS_DE_STATE_IDLE); ASSERT(is_internal_port(dep->cid) || is_internal_pid(dep->cid)); if (dep->flags & DFLAG_PUBLISHED) { ASSERT(erts_no_of_visible_dist_entries > 0); @@ -588,7 +588,7 @@ erts_set_dist_entry_not_connected(DistEntry *dep) if(dep->next) dep->next->prev = dep->prev; - dep->status &= ~(ERTS_DE_SFLG_PENDING | ERTS_DE_SFLG_CONNECTED); + dep->state = ERTS_DE_STATE_EXITING; dep->flags = 0; dep->prev = NULL; dep->cid = NIL; @@ -610,7 +610,7 @@ erts_set_dist_entry_pending(DistEntry *dep) erts_rwmtx_rwlock(&erts_dist_table_rwmtx); ASSERT(dep != erts_this_dist_entry); - ASSERT(dep->status == 0); + ASSERT(dep->state == ERTS_DE_STATE_IDLE); ASSERT(is_nil(dep->cid)); if(dep->prev) { @@ -627,8 +627,8 @@ erts_set_dist_entry_pending(DistEntry *dep) erts_no_of_not_connected_dist_entries--; - dep->status = ERTS_DE_SFLG_PENDING; - dep->flags = (DFLAG_DIST_MANDATORY | DFLAG_DIST_HOPEFULLY); + dep->state = ERTS_DE_STATE_PENDING; + dep->flags = (DFLAG_DIST_MANDATORY | DFLAG_DIST_HOPEFULLY | DFLAG_NO_MAGIC); dep->connection_id = (dep->connection_id + 1) & ERTS_DIST_CON_ID_MASK; dep->prev = NULL; @@ -652,7 +652,7 @@ erts_set_dist_entry_connected(DistEntry *dep, Eterm cid, Uint flags) ASSERT(dep != erts_this_dist_entry); ASSERT(is_nil(dep->cid)); - ASSERT(dep->status & ERTS_DE_SFLG_PENDING); + ASSERT(dep->state == ERTS_DE_STATE_PENDING); ASSERT(is_internal_port(cid) || is_internal_pid(cid)); if(dep->prev) { @@ -670,8 +670,7 @@ erts_set_dist_entry_connected(DistEntry *dep, Eterm cid, Uint flags) ASSERT(erts_no_of_pending_dist_entries > 0); erts_no_of_pending_dist_entries--; - dep->status &= ~ERTS_DE_SFLG_PENDING; - dep->status |= ERTS_DE_SFLG_CONNECTED; + dep->state = ERTS_DE_STATE_CONNECTED; dep->flags = flags & ~DFLAG_NO_MAGIC; dep->cid = cid; erts_atomic_set_nob(&dep->input_handler, diff --git a/erts/emulator/beam/erl_node_tables.h b/erts/emulator/beam/erl_node_tables.h index 8d29c83e15..58279017c8 100644 --- a/erts/emulator/beam/erl_node_tables.h +++ b/erts/emulator/beam/erl_node_tables.h @@ -57,9 +57,12 @@ #define ERST_INTERNAL_CHANNEL_NO 0 -#define ERTS_DE_SFLG_PENDING (((Uint32) 1) << 0) -#define ERTS_DE_SFLG_CONNECTED (((Uint32) 1) << 1) -#define ERTS_DE_SFLG_EXITING (((Uint32) 1) << 2) +enum dist_entry_state { + ERTS_DE_STATE_IDLE, + ERTS_DE_STATE_PENDING, + ERTS_DE_STATE_CONNECTED, + ERTS_DE_STATE_EXITING +}; #define ERTS_DE_QFLG_BUSY (((erts_aint32_t) 1) << 0) #define ERTS_DE_QFLG_EXIT (((erts_aint32_t) 1) << 1) @@ -86,6 +89,7 @@ struct ErtsDistOutputBuf_ { byte *alloc_endp; #endif ErtsDistOutputBuf *next; + Uint hopefull_flags; byte *extp; byte *ext_endp; byte *msg_start; @@ -121,7 +125,7 @@ typedef struct dist_entry_ { Eterm cid; /* connection handler (pid or port), NIL == free */ Uint32 connection_id; /* Connection id incremented on connect */ - Uint32 status; /* Slot status, like exiting reserved etc */ + enum dist_entry_state state; Uint32 flags; /* Distribution flags, like hidden, atom cache etc. */ unsigned long version; /* Protocol version */ diff --git a/erts/emulator/beam/erl_port.h b/erts/emulator/beam/erl_port.h index 9117eb1f72..0d148ee048 100644 --- a/erts/emulator/beam/erl_port.h +++ b/erts/emulator/beam/erl_port.h @@ -180,6 +180,7 @@ void erts_init_port_data(Port *); void erts_cleanup_port_data(Port *); Uint erts_port_data_size(Port *); ErlOffHeap *erts_port_data_offheap(Port *); +Eterm erts_port_data_read(Port* prt); #define ERTS_PORT_GET_CONNECTED(PRT) \ ((Eterm) erts_atomic_read_nob(&(PRT)->connected)) @@ -195,26 +196,52 @@ struct erl_drv_port_data_lock { Port *prt; }; +ERTS_GLB_INLINE void erts_init_runq_port(Port *prt, ErtsRunQueue *runq); +ERTS_GLB_INLINE void erts_set_runq_port(Port *prt, ErtsRunQueue *runq); +ERTS_GLB_INLINE ErtsRunQueue *erts_get_runq_port(Port *prt); ERTS_GLB_INLINE ErtsRunQueue *erts_port_runq(Port *prt); #if ERTS_GLB_INLINE_INCL_FUNC_DEF +ERTS_GLB_INLINE void +erts_init_runq_port(Port *prt, ErtsRunQueue *runq) +{ + if (!runq) + ERTS_INTERNAL_ERROR("Missing run-queue"); + erts_atomic_init_nob(&prt->run_queue, (erts_aint_t) runq); +} + +ERTS_GLB_INLINE void +erts_set_runq_port(Port *prt, ErtsRunQueue *runq) +{ + if (!runq) + ERTS_INTERNAL_ERROR("Missing run-queue"); + erts_atomic_set_nob(&prt->run_queue, (erts_aint_t) runq); +} + +ERTS_GLB_INLINE ErtsRunQueue * +erts_get_runq_port(Port *prt) +{ + ErtsRunQueue *runq; + runq = (ErtsRunQueue *) erts_atomic_read_nob(&prt->run_queue); + if (!runq) + ERTS_INTERNAL_ERROR("Missing run-queue"); + return runq; +} + + ERTS_GLB_INLINE ErtsRunQueue * erts_port_runq(Port *prt) { ErtsRunQueue *rq1, *rq2; - rq1 = (ErtsRunQueue *) erts_atomic_read_nob(&prt->run_queue); - if (!rq1) - return NULL; + rq1 = erts_get_runq_port(prt); while (1) { erts_runq_lock(rq1); - rq2 = (ErtsRunQueue *) erts_atomic_read_nob(&prt->run_queue); + rq2 = erts_get_runq_port(prt); if (rq1 == rq2) return rq1; erts_runq_unlock(rq1); rq1 = rq2; - if (!rq1) - return NULL; } } diff --git a/erts/emulator/beam/erl_port_task.c b/erts/emulator/beam/erl_port_task.c index a588477320..4a3671df0c 100644 --- a/erts/emulator/beam/erl_port_task.c +++ b/erts/emulator/beam/erl_port_task.c @@ -84,11 +84,10 @@ static void chk_task_queues(Port *pp, ErtsPortTask *execq, int processing_busy_q #define LTTNG_DRIVER(TRACEPOINT, PP) do {} while(0) #endif -#define ERTS_LC_VERIFY_RQ(RQ, PP) \ - do { \ +#define ERTS_LC_VERIFY_RQ(RQ, PP) \ + do { \ ERTS_LC_ASSERT(erts_lc_runq_is_locked(runq)); \ - ERTS_LC_ASSERT((RQ) == ((ErtsRunQueue *) \ - erts_atomic_read_nob(&(PP)->run_queue))); \ + ERTS_LC_ASSERT((RQ) == erts_get_runq_port((PP))); \ } while (0) #define ERTS_PT_STATE_SCHEDULED 0 @@ -1520,19 +1519,15 @@ erts_port_task_schedule(Eterm id, /* Enqueue port on run-queue */ runq = erts_port_runq(pp); - if (!runq) - ERTS_INTERNAL_ERROR("Missing run-queue"); xrunq = erts_check_emigration_need(runq, ERTS_PORT_PRIO_LEVEL); ERTS_LC_ASSERT(runq != xrunq); ERTS_LC_VERIFY_RQ(runq, pp); if (xrunq) { /* Emigrate port ... */ - erts_atomic_set_nob(&pp->run_queue, (erts_aint_t) xrunq); + erts_set_runq_port(pp, xrunq); erts_runq_unlock(runq); runq = erts_port_runq(pp); - if (!runq) - ERTS_INTERNAL_ERROR("Missing run-queue"); } enqueue_port(runq, pp); @@ -1593,8 +1588,6 @@ erts_port_task_free_port(Port *pp) ASSERT(!(erts_atomic32_read_nob(&pp->state) & ERTS_PORT_SFLGS_DEAD)); runq = erts_port_runq(pp); - if (!runq) - ERTS_INTERNAL_ERROR("Missing run-queue"); erts_port_task_sched_lock(&pp->sched); flags = erts_atomic32_read_bor_relb(&pp->sched.flags, ERTS_PTS_FLG_EXIT); @@ -1805,7 +1798,7 @@ erts_port_task_execute(ErtsRunQueue *runq, Port **curr_port_pp) erts_unblock_fpe(fpe_was_unmasked); ERTS_MSACC_POP_STATE_M(); - ASSERT(runq == (ErtsRunQueue *) erts_atomic_read_nob(&pp->run_queue)); + ASSERT(runq == erts_get_runq_port(pp)); active = finalize_exec(pp, &execq, processing_busy_q); @@ -1831,11 +1824,10 @@ erts_port_task_execute(ErtsRunQueue *runq, Port **curr_port_pp) } else { /* Emigrate port... */ - erts_atomic_set_nob(&pp->run_queue, (erts_aint_t) xrunq); + erts_set_runq_port(pp, xrunq); erts_runq_unlock(runq); xrunq = erts_port_runq(pp); - ASSERT(xrunq); enqueue_port(xrunq, pp); erts_runq_unlock(xrunq); erts_notify_inc_runq(xrunq); @@ -2069,7 +2061,7 @@ void erts_enqueue_port(ErtsRunQueue *rq, Port *pp) { ERTS_LC_ASSERT(erts_lc_runq_is_locked(rq)); - ASSERT(rq == (ErtsRunQueue *) erts_atomic_read_nob(&pp->run_queue)); + ASSERT(rq == erts_get_runq_port(pp)); ASSERT(erts_atomic32_read_nob(&pp->sched.flags) & ERTS_PTS_FLG_IN_RUNQ); enqueue_port(rq, pp); } @@ -2080,8 +2072,7 @@ erts_dequeue_port(ErtsRunQueue *rq) Port *pp; ERTS_LC_ASSERT(erts_lc_runq_is_locked(rq)); pp = pop_port(rq); - ASSERT(!pp - || rq == (ErtsRunQueue *) erts_atomic_read_nob(&pp->run_queue)); + ASSERT(!pp || rq == erts_get_runq_port(pp)); ASSERT(!pp || (erts_atomic32_read_nob(&pp->sched.flags) & ERTS_PTS_FLG_IN_RUNQ)); return pp; diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c index a807d60ec7..0d02d10ac9 100644 --- a/erts/emulator/beam/erl_process.c +++ b/erts/emulator/beam/erl_process.c @@ -137,36 +137,6 @@ runq_got_work_to_execute(ErtsRunQueue *rq) return runq_got_work_to_execute_flags(ERTS_RUNQ_FLGS_GET_NOB(rq)); } -#undef RUNQ_READ_RQ -#undef RUNQ_SET_RQ -#define RUNQ_READ_RQ(X) ((ErtsRunQueue *) erts_atomic_read_nob((X))) -#define RUNQ_SET_RQ(X, RQ) erts_atomic_set_nob((X), (erts_aint_t) (RQ)) - -#ifdef DEBUG -# if defined(ARCH_64) -# define ERTS_DBG_SET_INVALID_RUNQP(RQP, N) \ - (RUNQ_SET_RQ((RQP), (0xdeadbeefdead0003LL | ((N) << 4))) -# define ERTS_DBG_VERIFY_VALID_RUNQP(RQP) \ -do { \ - ASSERT((RQP) != NULL); \ - ASSERT(((((Uint) (RQP)) & ((Uint) 0x3))) == ((Uint) 0)); \ - ASSERT((((Uint) (RQP)) & ~((Uint) 0xffff)) != ((Uint) 0xdeadbeefdead0000LL));\ -} while (0) -# else -# define ERTS_DBG_SET_INVALID_RUNQP(RQP, N) \ - (RUNQ_SET_RQ((RQP), (0xdead0003 | ((N) << 4)))) -# define ERTS_DBG_VERIFY_VALID_RUNQP(RQP) \ -do { \ - ASSERT((RQP) != NULL); \ - ASSERT(((((UWord) (RQP)) & ((UWord) 1))) == ((UWord) 0)); \ - ASSERT((((UWord) (RQP)) & ~((UWord) 0xffff)) != ((UWord) 0xdead0000)); \ -} while (0) -# endif -#else -# define ERTS_DBG_SET_INVALID_RUNQP(RQP, N) -# define ERTS_DBG_VERIFY_VALID_RUNQP(RQP) -#endif - const Process erts_invalid_process = {{ERTS_INVALID_PID}}; extern BeamInstr beam_apply[]; @@ -1538,6 +1508,20 @@ erts_proclist_destroy(ErtsProcList *plp) proclist_destroy(plp); } +void +erts_proclist_dump(fmtfn_t to, void *to_arg, ErtsProcList *plp) +{ + ErtsProcList *first = plp; + + while (plp) { + erts_print(to, to_arg, "%T", plp->pid); + plp = plp->next; + if (plp == first) + break; + } + erts_print(to, to_arg, "\n"); +} + void * erts_psd_set_init(Process *p, int ix, void *data) { @@ -3926,21 +3910,16 @@ immigrate(ErtsRunQueue *c_rq, ErtsMigrationPath *mp) Port *prt; prt = erts_dequeue_port(rq); if (prt) - RUNQ_SET_RQ(&prt->run_queue, c_rq); + erts_set_runq_port(prt, c_rq); erts_runq_unlock(rq); if (prt) { - /* port might terminate while we have no lock... */ rq = erts_port_runq(prt); - if (rq) { - if (rq != c_rq) - erts_exit(ERTS_ABORT_EXIT, - "%s:%d:%s(): Internal error", - __FILE__, __LINE__, __func__); - erts_enqueue_port(c_rq, prt); - if (!iflag) - return; /* done */ - erts_runq_unlock(c_rq); - } + if (rq != c_rq) + ERTS_INTERNAL_ERROR("Unexpected run-queue"); + erts_enqueue_port(c_rq, prt); + if (!iflag) + return; /* done */ + erts_runq_unlock(c_rq); } } else { @@ -3954,12 +3933,11 @@ immigrate(ErtsRunQueue *c_rq, ErtsMigrationPath *mp) while (proc) { erts_aint32_t state; state = erts_atomic32_read_acqb(&proc->state); - if (!(ERTS_PSFLG_BOUND & state) - && (prio == (int) ERTS_PSFLGS_GET_PRQ_PRIO(state))) { + if (prio == (int) ERTS_PSFLGS_GET_PRQ_PRIO(state) + && erts_try_change_runq_proc(proc, c_rq)) { ErtsRunQueueInfo *rqi = &rq->procs.prio_info[prio]; unqueue_process(rq, rpq, rqi, prio, prev_proc, proc); erts_runq_unlock(rq); - RUNQ_SET_RQ(&proc->run_queue, c_rq); rq_locked = 0; erts_runq_lock(c_rq); @@ -4140,21 +4118,13 @@ evacuate_run_queue(ErtsRunQueue *rq, while (prt) { ErtsRunQueue *prt_rq; prt = erts_dequeue_port(rq); - RUNQ_SET_RQ(&prt->run_queue, to_rq); + erts_set_runq_port(prt, to_rq); erts_runq_unlock(rq); - /* - * The port might terminate while - * we have no lock on it... - */ prt_rq = erts_port_runq(prt); - if (prt_rq) { - if (prt_rq != to_rq) - erts_exit(ERTS_ABORT_EXIT, - "%s:%d:%s() internal error\n", - __FILE__, __LINE__, __func__); - erts_enqueue_port(to_rq, prt); - erts_runq_unlock(to_rq); - } + if (prt_rq != to_rq) + ERTS_INTERNAL_ERROR("Unexpected run-queue"); + erts_enqueue_port(to_rq, prt); + erts_runq_unlock(to_rq); erts_runq_lock(rq); prt = rq->ports.start; } @@ -4165,8 +4135,6 @@ evacuate_run_queue(ErtsRunQueue *rq, for (prio_q = 0; prio_q < ERTS_NO_PROC_PRIO_QUEUES; prio_q++) { erts_aint32_t state; Process *proc; - int notify = 0; - to_rq = NULL; if (!mp->prio[prio_q].runq) return; @@ -4177,14 +4145,13 @@ evacuate_run_queue(ErtsRunQueue *rq, while (proc) { Process *real_proc; int prio; - erts_aint32_t max_qbit, qbit, real_state; + erts_aint32_t max_qbit, qbit; prio = ERTS_PSFLGS_GET_PRQ_PRIO(state); qbit = ((erts_aint32_t) 1) << prio; if (!(state & ERTS_PSFLG_PROXY)) { real_proc = proc; - real_state = state; } else { real_proc = erts_proc_lookup_raw(proc->common.id); @@ -4192,7 +4159,6 @@ evacuate_run_queue(ErtsRunQueue *rq, free_proxy_proc(proc); goto handle_next_proc; } - real_state = erts_atomic32_read_acqb(&real_proc->state); } max_qbit = (state >> ERTS_PSFLGS_IN_PRQ_MASK_OFFSET); @@ -4227,7 +4193,13 @@ evacuate_run_queue(ErtsRunQueue *rq, goto handle_next_proc; } - if (ERTS_PSFLG_BOUND & real_state) { + prio = (int) ERTS_PSFLGS_GET_PRQ_PRIO(state); + to_rq = mp->prio[prio].runq; + + if (!to_rq) + goto handle_next_proc; + + if (!erts_try_change_runq_proc(proc, to_rq)) { /* Bound processes get stuck here... */ proc->next = NULL; if (sbpp->last) @@ -4237,16 +4209,13 @@ evacuate_run_queue(ErtsRunQueue *rq, sbpp->last = proc; } else { - int prio = (int) ERTS_PSFLGS_GET_PRQ_PRIO(state); erts_runq_unlock(rq); - to_rq = mp->prio[prio].runq; - RUNQ_SET_RQ(&proc->run_queue, to_rq); - erts_runq_lock(to_rq); enqueue_process(to_rq, prio, proc); erts_runq_unlock(to_rq); - notify = 1; + + smp_notify_inc_runq(to_rq); erts_runq_lock(rq); } @@ -4254,8 +4223,7 @@ evacuate_run_queue(ErtsRunQueue *rq, handle_next_proc: proc = dequeue_process(rq, prio_q, &state); } - if (notify) - smp_notify_inc_runq(to_rq); + } } @@ -4309,14 +4277,13 @@ try_steal_task_from_victim(ErtsRunQueue *rq, int *rq_lockedp, ErtsRunQueue *vrq, proc = rpq->first; while (proc) { - erts_aint32_t state = erts_atomic32_read_acqb(&proc->state); - if (!(ERTS_PSFLG_BOUND & state)) { + if (erts_try_change_runq_proc(proc, rq)) { + erts_aint32_t state = erts_atomic32_read_acqb(&proc->state); /* Steal process */ int prio = (int) ERTS_PSFLGS_GET_PRQ_PRIO(state); ErtsRunQueueInfo *rqi = &vrq->procs.prio_info[prio]; unqueue_process(vrq, rpq, rqi, prio, prev_proc, proc); erts_runq_unlock(vrq); - RUNQ_SET_RQ(&proc->run_queue, rq); erts_runq_lock(rq); *rq_lockedp = 1; @@ -4341,26 +4308,14 @@ no_procs: if (vrq->ports.start) { ErtsRunQueue *prt_rq; Port *prt = erts_dequeue_port(vrq); - RUNQ_SET_RQ(&prt->run_queue, rq); + erts_set_runq_port(prt, rq); erts_runq_unlock(vrq); - - /* - * The port might terminate while - * we have no lock on it... - */ - prt_rq = erts_port_runq(prt); - if (!prt_rq) - return 0; - else { - if (prt_rq != rq) - erts_exit(ERTS_ABORT_EXIT, - "%s:%d:%s() internal error\n", - __FILE__, __LINE__, __func__); - *rq_lockedp = 1; - erts_enqueue_port(rq, prt); - return !0; - } + if (prt_rq != rq) + ERTS_INTERNAL_ERROR("Unexpected run-queue"); + *rq_lockedp = 1; + erts_enqueue_port(rq, prt); + return !0; } erts_runq_unlock(vrq); @@ -6116,7 +6071,8 @@ make_proxy_proc(Process *prev_proxy, Process *proc, erts_aint32_t prio) { erts_aint32_t state; Process *proxy; - ErtsRunQueue *rq = RUNQ_READ_RQ(&proc->run_queue); + int bound; + ErtsRunQueue *rq = erts_get_runq_proc(proc, &bound); state = (ERTS_PSFLG_PROXY | ERTS_PSFLG_IN_RUNQ @@ -6129,7 +6085,7 @@ make_proxy_proc(Process *prev_proxy, Process *proc, erts_aint32_t prio) proxy = prev_proxy; ASSERT(erts_atomic32_read_nob(&proxy->state) & ERTS_PSFLG_PROXY); erts_atomic32_set_nob(&proxy->state, state); - RUNQ_SET_RQ(&proc->run_queue, rq); + (void) erts_set_runq_proc(proc, rq, &bound); } else { proxy = erts_alloc(ERTS_ALC_T_PROC, sizeof(Process)); @@ -6142,8 +6098,7 @@ make_proxy_proc(Process *prev_proxy, Process *proc, erts_aint32_t prio) } #endif erts_atomic32_init_nob(&proxy->state, state); - erts_atomic_init_nob(&proxy->run_queue, - erts_atomic_read_nob(&proc->run_queue)); + erts_init_runq_proc(proc, rq, bound); } proxy->common.id = proc->common.id; @@ -6334,18 +6289,21 @@ select_enqueue_run_queue(int enqueue, int enq_prio, Process *p, erts_aint32_t st default: { ErtsRunQueue* runq; + int bound; ASSERT(enqueue == ERTS_ENQUEUE_NORMAL_QUEUE || enqueue == -ERTS_ENQUEUE_NORMAL_QUEUE); - runq = erts_get_runq_proc(p); + runq = erts_get_runq_proc(p, &bound); - if (!(ERTS_PSFLG_BOUND & state)) { + if (!bound) { ErtsRunQueue *new_runq = erts_check_emigration_need(runq, enq_prio); - if (new_runq) { - RUNQ_SET_RQ(&p->run_queue, new_runq); - runq = new_runq; - } + if (new_runq) { + if (erts_try_change_runq_proc(p, new_runq)) + runq = new_runq; + else + runq = erts_get_runq_proc(p, NULL); + } } ASSERT(runq); @@ -10079,6 +10037,10 @@ Process *erts_schedule(ErtsSchedulerData *esdp, Process *p, int calls) p->scheduler_data = esdp; } else { + if (!(state & ERTS_PSFLGS_DIRTY_WORK)) { + /* Dirty work completed... */ + goto sunlock_sched_out_proc; + } if (state & (ERTS_PSFLG_ACTIVE_SYS | ERTS_PSFLG_PENDING_EXIT | ERTS_PSFLG_EXITING)) { @@ -10221,6 +10183,23 @@ Process *erts_schedule(ErtsSchedulerData *esdp, Process *p, int calls) ERTS_PTMR_CLEAR(p); } +#ifdef DEBUG + if (is_normal_sched) { + if (state & ERTS_PSFLGS_DIRTY_WORK) + ERTS_INTERNAL_ERROR("Executing dirty code on normal scheduler"); + } + else { + if (!(state & ERTS_PSFLGS_DIRTY_WORK)) { + if (esdp->type == ERTS_SCHED_DIRTY_CPU) + ERTS_INTERNAL_ERROR("Executing normal code on dirty CPU scheduler"); + else if (esdp->type == ERTS_SCHED_DIRTY_IO) + ERTS_INTERNAL_ERROR("Executing normal code on dirty IO scheduler"); + else + ERTS_INTERNAL_ERROR("Executing normal code on dirty UNKNOWN scheduler"); + } + } +#endif + return p; } } @@ -11462,6 +11441,7 @@ typedef struct { Process *proc; erts_aint32_t state; ErtsRunQueue *run_queue; + int bound; } ErtsEarlyProcInit; static void early_init_process_struct(void *varg, Eterm data) @@ -11472,10 +11452,9 @@ static void early_init_process_struct(void *varg, Eterm data) proc->common.id = make_internal_pid(data); erts_atomic32_init_nob(&proc->dirty_state, 0); proc->dirty_sys_tasks = NULL; + erts_init_runq_proc(proc, arg->run_queue, arg->bound); erts_atomic32_init_relb(&proc->state, arg->state); - RUNQ_SET_RQ(&proc->run_queue, arg->run_queue); - erts_proc_lock_init(proc); /* All locks locked */ } @@ -11484,7 +11463,7 @@ static void early_init_process_struct(void *varg, Eterm data) ** Allocate process and find out where to place next process. */ static Process* -alloc_process(ErtsRunQueue *rq, erts_aint32_t state) +alloc_process(ErtsRunQueue *rq, int bound, erts_aint32_t state) { ErtsEarlyProcInit init_arg; Process *p; @@ -11493,9 +11472,12 @@ alloc_process(ErtsRunQueue *rq, erts_aint32_t state) if (!p) return NULL; + ASSERT(rq); + init_arg.proc = (Process *) p; - init_arg.run_queue = rq; init_arg.state = state; + init_arg.run_queue = rq; + init_arg.bound = bound; ERTS_CT_ASSERT(offsetof(Process,common) == 0); @@ -11511,7 +11493,6 @@ alloc_process(ErtsRunQueue *rq, erts_aint32_t state) ASSERT(internal_pid_serial(p->common.id) <= ERTS_MAX_PID_SERIAL); - p->approx_started = erts_get_approx_time(); p->rcount = 0; p->heap = NULL; @@ -11530,6 +11511,7 @@ erl_create_process(Process* parent, /* Parent of process (default group leader). Eterm args, /* Arguments for function (must be well-formed list). */ ErlSpawnOpts* so) /* Options for spawn. */ { + int bound = 0; Uint flags = 0; ErtsRunQueue *rq = NULL; Process *p; @@ -11566,7 +11548,7 @@ erl_create_process(Process* parent, /* Parent of process (default group leader). ASSERT(0 <= ix && ix < erts_no_run_queues); rq = ERTS_RUNQ_IX(ix); /* Unsupported feature... */ - state |= ERTS_PSFLG_BOUND; + bound = !0; } prio = (erts_aint32_t) so->priority; } @@ -11579,17 +11561,16 @@ erl_create_process(Process* parent, /* Parent of process (default group leader). flags |= F_OFF_HEAP_MSGQ; } else if (so->flags & SPO_ON_HEAP_MSGQ) { - state |= ERTS_PSFLG_ON_HEAP_MSGQ; flags |= F_ON_HEAP_MSGQ; } ASSERT((flags & F_ON_HEAP_MSGQ) || (flags & F_OFF_HEAP_MSGQ)); if (!rq) - rq = erts_get_runq_proc(parent); + rq = erts_get_runq_proc(parent, NULL); - p = alloc_process(rq, state); /* All proc locks are locked by this thread - on success */ + p = alloc_process(rq, bound, state); /* All proc locks are locked by this thread + on success */ if (!p) { erts_send_error_to_logger_str(parent->group_leader, "Too many processes\n"); @@ -11597,11 +11578,6 @@ erl_create_process(Process* parent, /* Parent of process (default group leader). goto error; } - ASSERT((erts_atomic32_read_nob(&p->state) - & ERTS_PSFLG_ON_HEAP_MSGQ) - || (erts_atomic32_read_nob(&p->state) - & ERTS_PSFLG_OFF_HEAP_MSGQ)); - #ifdef SHCOPY_SPAWN arg_size = copy_shared_calculate(args, &info); #else @@ -11947,7 +11923,6 @@ void erts_init_empty_process(Process *p) p->def_arg_reg[5] = 0; p->parent = NIL; - p->approx_started = 0; p->static_flags = 0; p->common.u.alive.started_interval = 0; @@ -11976,7 +11951,7 @@ void erts_init_empty_process(Process *p) p->pending_exit.bp = NULL; erts_proc_lock_init(p); erts_proc_unlock(p, ERTS_PROC_LOCKS_ALL); - RUNQ_SET_RQ(&p->run_queue, ERTS_RUNQ_IX(0)); + erts_init_runq_proc(p, ERTS_RUNQ_IX(0), 0); #if !defined(NO_FPE_SIGNALS) || defined(HIPE) p->fp_exception = 0; @@ -12298,7 +12273,7 @@ save_pending_exiter(Process *p, ErtsProcList *plp) ERTS_LC_ASSERT(ERTS_PROC_LOCK_STATUS & erts_proc_lc_my_proc_locks(p)); - rq = RUNQ_READ_RQ(&p->run_queue); + rq = erts_get_runq_proc(p, NULL); ASSERT(rq && !ERTS_RUNQ_IX_IS_DIRTY(rq->ix)); if (!plp) @@ -13398,16 +13373,33 @@ stack_element_dump(fmtfn_t to, void *to_arg, Eterm* sp, int yreg) return yreg; } +static void print_current_process_info(fmtfn_t, void *to_arg, ErtsSchedulerData*); + /* * Print scheduler information */ void -erts_print_scheduler_info(fmtfn_t to, void *to_arg, ErtsSchedulerData *esdp) { +erts_print_scheduler_info(fmtfn_t to, void *to_arg, ErtsSchedulerData *esdp) +{ int i; erts_aint32_t flg; - Process *p; - erts_print(to, to_arg, "=scheduler:%u\n", esdp->no); + switch (esdp->type) { + case ERTS_SCHED_NORMAL: + erts_print(to, to_arg, "=scheduler:%u\n", esdp->no); + break; + case ERTS_SCHED_DIRTY_CPU: + erts_print(to, to_arg, "=dirty_cpu_scheduler:%u\n", + (esdp->dirty_no + erts_no_schedulers)); + break; + case ERTS_SCHED_DIRTY_IO: + erts_print(to, to_arg, "=dirty_io_scheduler:%u\n", + (esdp->dirty_no + erts_no_schedulers + erts_no_dirty_cpu_schedulers)); + break; + default: + erts_print(to, to_arg, "=unknown_scheduler_type:%u\n", esdp->type); + break; + } flg = erts_atomic32_read_dirty(&esdp->ssi->flags); erts_print(to, to_arg, "Scheduler Sleep Info Flags: "); @@ -13453,10 +13445,24 @@ erts_print_scheduler_info(fmtfn_t to, void *to_arg, ErtsSchedulerData *esdp) { } erts_print(to, to_arg, "\n"); - erts_print(to, to_arg, "Current Port: "); - if (esdp->current_port) - erts_print(to, to_arg, "%T", esdp->current_port->common.id); - erts_print(to, to_arg, "\n"); + if (esdp->type == ERTS_SCHED_NORMAL) { + erts_print(to, to_arg, "Current Port: "); + if (esdp->current_port) + erts_print(to, to_arg, "%T", esdp->current_port->common.id); + erts_print(to, to_arg, "\n"); + + erts_print_run_queue_info(to, to_arg, esdp->run_queue); + } + + /* This *MUST* to be the last information in scheduler block */ + print_current_process_info(to, to_arg, esdp); +} + +void erts_print_run_queue_info(fmtfn_t to, void *to_arg, + ErtsRunQueue *run_queue) +{ + erts_aint32_t flg; + int i; for (i = 0; i < ERTS_NO_PROC_PRIO_LEVELS; i++) { erts_print(to, to_arg, "Run Queue "); @@ -13478,12 +13484,12 @@ erts_print_scheduler_info(fmtfn_t to, void *to_arg, ErtsSchedulerData *esdp) { break; } erts_print(to, to_arg, "Length: %d\n", - erts_atomic32_read_dirty(&esdp->run_queue->procs.prio_info[i].len)); + erts_atomic32_read_dirty(&run_queue->procs.prio_info[i].len)); } erts_print(to, to_arg, "Run Queue Port Length: %d\n", - erts_atomic32_read_dirty(&esdp->run_queue->ports.info.len)); + erts_atomic32_read_dirty(&run_queue->ports.info.len)); - flg = erts_atomic32_read_dirty(&esdp->run_queue->flags); + flg = erts_atomic32_read_dirty(&run_queue->flags); erts_print(to, to_arg, "Run Queue Flags: "); for (i = 0; i < ERTS_RUNQ_FLG_MAX && flg; i++) { erts_aint32_t chk = (1 << i); @@ -13550,9 +13556,15 @@ erts_print_scheduler_info(fmtfn_t to, void *to_arg, ErtsSchedulerData *esdp) { } } erts_print(to, to_arg, "\n"); +} + + +static void print_current_process_info(fmtfn_t to, void *to_arg, + ErtsSchedulerData* esdp) +{ + Process *p = esdp->current_process; + erts_aint32_t flg; - /* This *MUST* to be the last information in scheduler block */ - p = esdp->current_process; erts_print(to, to_arg, "Current Process: "); if (esdp->current_process && !(ERTS_TRACE_FLAGS(p) & F_SENSITIVE)) { flg = erts_atomic32_read_dirty(&p->state); diff --git a/erts/emulator/beam/erl_process.h b/erts/emulator/beam/erl_process.h index 66d7848f89..ebf990c837 100644 --- a/erts/emulator/beam/erl_process.h +++ b/erts/emulator/beam/erl_process.h @@ -105,13 +105,13 @@ struct saved_calls { }; extern Export exp_send, exp_receive, exp_timeout; -extern int erts_sched_compact_load; -extern int erts_sched_balance_util; -extern Uint erts_no_schedulers; -extern Uint erts_no_total_schedulers; -extern Uint erts_no_dirty_cpu_schedulers; -extern Uint erts_no_dirty_io_schedulers; -extern Uint erts_no_run_queues; +extern int ERTS_WRITE_UNLIKELY(erts_sched_compact_load); +extern int ERTS_WRITE_UNLIKELY(erts_sched_balance_util); +extern Uint ERTS_WRITE_UNLIKELY(erts_no_schedulers); +extern Uint ERTS_WRITE_UNLIKELY(erts_no_total_schedulers); +extern Uint ERTS_WRITE_UNLIKELY(erts_no_dirty_cpu_schedulers); +extern Uint ERTS_WRITE_UNLIKELY(erts_no_dirty_io_schedulers); +extern Uint ERTS_WRITE_UNLIKELY(erts_no_run_queues); extern int erts_sched_thread_suggested_stack_size; extern int erts_dcpu_sched_thread_suggested_stack_size; extern int erts_dio_sched_thread_suggested_stack_size; @@ -244,6 +244,9 @@ extern int erts_dio_sched_thread_suggested_stack_size; (erts_aint32_t) (MSK), \ (erts_aint32_t) (FLGS))) +#define ERTS_RUNQ_POINTER_MASK (~((erts_aint_t) 3)) +#define ERTS_RUNQ_BOUND_FLAG ((erts_aint_t) 1) + typedef enum { ERTS_SCHDLR_SSPND_DONE_MSCHED_BLOCKED, ERTS_SCHDLR_SSPND_DONE_NMSCHED_BLOCKED, @@ -522,7 +525,7 @@ typedef union { char align[ERTS_ALC_CACHE_LINE_ALIGN_SIZE(sizeof(ErtsRunQueue))]; } ErtsAlignedRunQueue; -extern ErtsAlignedRunQueue *erts_aligned_run_queues; +extern ErtsAlignedRunQueue * ERTS_WRITE_UNLIKELY(erts_aligned_run_queues); #define ERTS_PROC_REDUCTIONS_EXECUTED(SD, RQ, PRIO, REDS, AREDS)\ do { \ @@ -675,9 +678,9 @@ typedef union { char align[ERTS_ALC_CACHE_LINE_ALIGN_SIZE(sizeof(ErtsSchedulerData))]; } ErtsAlignedSchedulerData; -extern ErtsAlignedSchedulerData *erts_aligned_scheduler_data; -extern ErtsAlignedSchedulerData *erts_aligned_dirty_cpu_scheduler_data; -extern ErtsAlignedSchedulerData *erts_aligned_dirty_io_scheduler_data; +extern ErtsAlignedSchedulerData * ERTS_WRITE_UNLIKELY(erts_aligned_scheduler_data); +extern ErtsAlignedSchedulerData * ERTS_WRITE_UNLIKELY(erts_aligned_dirty_cpu_scheduler_data); +extern ErtsAlignedSchedulerData * ERTS_WRITE_UNLIKELY(erts_aligned_dirty_io_scheduler_data); #if defined(ERTS_ENABLE_LOCK_CHECK) @@ -1019,7 +1022,6 @@ struct process { * Information mainly for post-mortem use (erl crash dump). */ Eterm parent; /* Pid of process that created this process. */ - erts_approx_time_t approx_started; /* Time when started. */ Uint32 static_flags; /* Flags that do *not* change */ @@ -1168,14 +1170,14 @@ void erts_check_for_holes(Process* p); #define ERTS_PSFLG_RUNNING ERTS_PSFLG_BIT(9) #define ERTS_PSFLG_SUSPENDED ERTS_PSFLG_BIT(10) #define ERTS_PSFLG_GC ERTS_PSFLG_BIT(11) -#define ERTS_PSFLG_BOUND ERTS_PSFLG_BIT(12) +/* #define ERTS_PSFLG_ ERTS_PSFLG_BIT(12) */ #define ERTS_PSFLG_TRAP_EXIT ERTS_PSFLG_BIT(13) #define ERTS_PSFLG_ACTIVE_SYS ERTS_PSFLG_BIT(14) #define ERTS_PSFLG_RUNNING_SYS ERTS_PSFLG_BIT(15) #define ERTS_PSFLG_PROXY ERTS_PSFLG_BIT(16) #define ERTS_PSFLG_DELAYED_SYS ERTS_PSFLG_BIT(17) #define ERTS_PSFLG_OFF_HEAP_MSGQ ERTS_PSFLG_BIT(18) -#define ERTS_PSFLG_ON_HEAP_MSGQ ERTS_PSFLG_BIT(19) +/* #define ERTS_PSFLG_ ERTS_PSFLG_BIT(19) */ #define ERTS_PSFLG_DIRTY_CPU_PROC ERTS_PSFLG_BIT(20) #define ERTS_PSFLG_DIRTY_IO_PROC ERTS_PSFLG_BIT(21) #define ERTS_PSFLG_DIRTY_ACTIVE_SYS ERTS_PSFLG_BIT(22) @@ -1270,7 +1272,7 @@ void erts_check_for_holes(Process* p); #define SPO_OFF_HEAP_MSGQ 16 #define SPO_ON_HEAP_MSGQ 32 -extern int erts_default_spo_flags; +extern int ERTS_WRITE_UNLIKELY(erts_default_spo_flags); /* * The following struct contains options for a process to be spawned. @@ -1326,10 +1328,10 @@ extern erts_rwmtx_t erts_cpu_bind_rwmtx; ** erts_system_monitor must be != NIL, to allow testing on just ** the erts_system_monitor_* variables. */ -extern Eterm erts_system_monitor; -extern Uint erts_system_monitor_long_gc; -extern Uint erts_system_monitor_long_schedule; -extern Uint erts_system_monitor_large_heap; +extern Eterm ERTS_WRITE_UNLIKELY(erts_system_monitor); +extern Uint ERTS_WRITE_UNLIKELY(erts_system_monitor_long_gc); +extern Uint ERTS_WRITE_UNLIKELY(erts_system_monitor_long_schedule); +extern Uint ERTS_WRITE_UNLIKELY(erts_system_monitor_large_heap); struct erts_system_monitor_flags_t { unsigned int busy_port : 1; unsigned int busy_dist_port : 1; @@ -1546,6 +1548,7 @@ Uint64 erts_step_proc_interval(void); ErtsProcList *erts_proclist_create(Process *); ErtsProcList *erts_proclist_copy(ErtsProcList *); void erts_proclist_destroy(ErtsProcList *); +void erts_proclist_dump(fmtfn_t to, void *to_arg, ErtsProcList*); ERTS_GLB_INLINE int erts_proclist_same(ErtsProcList *, Process *); ERTS_GLB_INLINE void erts_proclist_store_first(ErtsProcList **, ErtsProcList *); @@ -1792,6 +1795,7 @@ void erts_stack_dump(fmtfn_t to, void *to_arg, Process *); void erts_limited_stack_trace(fmtfn_t to, void *to_arg, Process *); void erts_program_counter_info(fmtfn_t to, void *to_arg, Process *); void erts_print_scheduler_info(fmtfn_t to, void *to_arg, ErtsSchedulerData *esdp); +void erts_print_run_queue_info(fmtfn_t, void *to_arg, ErtsRunQueue*); void erts_dump_extended_process_state(fmtfn_t to, void *to_arg, erts_aint32_t psflg); void erts_dump_process_state(fmtfn_t to, void *to_arg, erts_aint32_t psflg); @@ -2167,7 +2171,12 @@ ERTS_GLB_INLINE int erts_is_scheduler_bound(ErtsSchedulerData *esdp); ERTS_GLB_INLINE Process *erts_get_current_process(void); ERTS_GLB_INLINE Eterm erts_get_current_pid(void); ERTS_GLB_INLINE Uint erts_get_scheduler_id(void); -ERTS_GLB_INLINE ErtsRunQueue *erts_get_runq_proc(Process *p); +ERTS_GLB_INLINE void erts_init_runq_proc(Process *p, ErtsRunQueue *rq, int bnd); +ERTS_GLB_INLINE ErtsRunQueue *erts_set_runq_proc(Process *p, ErtsRunQueue *rq, int *boundp); +ERTS_GLB_INLINE int erts_try_change_runq_proc(Process *p, ErtsRunQueue *rq); +ERTS_GLB_INLINE ErtsRunQueue *erts_bind_runq_proc(Process *p, int bind); +ERTS_GLB_INLINE int erts_proc_runq_is_bound(Process *p); +ERTS_GLB_INLINE ErtsRunQueue *erts_get_runq_proc(Process *p, int *boundp); ERTS_GLB_INLINE ErtsRunQueue *erts_get_runq_current(ErtsSchedulerData *esdp); ERTS_GLB_INLINE void erts_runq_lock(ErtsRunQueue *rq); ERTS_GLB_INLINE int erts_runq_trylock(ErtsRunQueue *rq); @@ -2247,11 +2256,144 @@ Uint erts_get_scheduler_id(void) return esdp ? esdp->no : (Uint) 0; } +/** + * Init run-queue of process. + * + * @param p[in,out] Process + * @param rq[in] Run-queue that process will be assigned to + * @param bnd[in,out] If non-zero binds process to run-queue. + */ + +ERTS_GLB_INLINE void +erts_init_runq_proc(Process *p, ErtsRunQueue *rq, int bnd) +{ + erts_aint_t rqint = (erts_aint_t) rq; + if (bnd) + rqint |= ERTS_RUNQ_BOUND_FLAG; + erts_atomic_init_nob(&p->run_queue, rqint); +} + +/** + * Forcibly set run-queue of process. + * + * @param p[in,out] Process + * @param rq[in] Run-queue that process will be assigned to + * @param bndp[in,out] Pointer to integer. On input non-zero + * value causes the process to be bound to + * the run-queue. On output, indicating + * wether process previously was bound or + * not. + * @return Previous run-queue. + */ + +ERTS_GLB_INLINE ErtsRunQueue * +erts_set_runq_proc(Process *p, ErtsRunQueue *rq, int *bndp) +{ + erts_aint_t rqint = (erts_aint_t) rq; + ASSERT(bndp); + ASSERT(rq); + if (*bndp) + rqint |= ERTS_RUNQ_BOUND_FLAG; + rqint = erts_atomic_xchg_nob(&p->run_queue, rqint); + *bndp = (int) (rqint & ERTS_RUNQ_BOUND_FLAG); + return (ErtsRunQueue *) (rqint & ERTS_RUNQ_POINTER_MASK); +} + +/** + * Try to change run-queue assignment of a process. + * + * @param p[in,out] Process + * @param rq[int] Run-queue that process will be assigned to + * @return Non-zero if the run-queue assignment was + * successfully changed. + */ + +ERTS_GLB_INLINE int +erts_try_change_runq_proc(Process *p, ErtsRunQueue *rq) +{ + erts_aint_t old_rqint, new_rqint; + + new_rqint = (erts_aint_t) rq; + old_rqint = (erts_aint_t) erts_atomic_read_nob(&p->run_queue); + while (1) { + erts_aint_t act_rqint; + + if (old_rqint & ERTS_RUNQ_BOUND_FLAG) + return 0; + + act_rqint = erts_atomic_cmpxchg_nob(&p->run_queue, + new_rqint, + old_rqint); + if (act_rqint == old_rqint) + return !0; + } +} + +/** + * + * Bind or unbind process to/from currently used run-queue. + * + * @param p Process + * @param bind Bind if non-zero; otherwise unbind + * @return Pointer to previously bound run-queue, + * or NULL if previously unbound + */ + +ERTS_GLB_INLINE ErtsRunQueue * +erts_bind_runq_proc(Process *p, int bind) +{ + erts_aint_t rqint; + if (bind) + rqint = erts_atomic_read_bor_nob(&p->run_queue, + ERTS_RUNQ_BOUND_FLAG); + else + rqint = erts_atomic_read_band_nob(&p->run_queue, + ~ERTS_RUNQ_BOUND_FLAG); + if (rqint & ERTS_RUNQ_BOUND_FLAG) + return (ErtsRunQueue *) (rqint & ERTS_RUNQ_POINTER_MASK); + else + return NULL; +} + +/** + * Determine wether a process is bound to a run-queue or not. + * + * @return Returns a non-zero value if bound, + * and zero of not bound. + */ + +ERTS_GLB_INLINE int +erts_proc_runq_is_bound(Process *p) +{ + erts_aint_t rqint = erts_atomic_read_nob(&p->run_queue); + return (int) (rqint & ERTS_RUNQ_BOUND_FLAG); +} + +/** + * Set run-queue of process. + * + * @param p[in,out] Process + * @param bndp[out] Pointer to integer. If non-NULL pointer, + * the integer will be set to a non-zero + * value if the process is bound to the + * run-queue. + * @return Pointer to the normal run-queue that + * the process currently is assigend to. + * A process is always assigned to a + * normal run-queue. + */ + ERTS_GLB_INLINE ErtsRunQueue * -erts_get_runq_proc(Process *p) +erts_get_runq_proc(Process *p, int *bndp) { - ASSERT(ERTS_AINT_NULL != erts_atomic_read_nob(&p->run_queue)); - return (ErtsRunQueue *) erts_atomic_read_nob(&p->run_queue); + erts_aint_t rqint = erts_atomic_read_nob(&p->run_queue); + ErtsRunQueue *rq; + if (bndp) + *bndp = (int) (rqint & ERTS_RUNQ_BOUND_FLAG); + rqint &= ERTS_RUNQ_POINTER_MASK; + rq = (ErtsRunQueue *) rqint; + ASSERT(rq); + return rq; } ERTS_GLB_INLINE ErtsRunQueue * diff --git a/erts/emulator/beam/erl_process_dict.c b/erts/emulator/beam/erl_process_dict.c index 3c80f0e0f6..aee88841ae 100644 --- a/erts/emulator/beam/erl_process_dict.c +++ b/erts/emulator/beam/erl_process_dict.c @@ -79,6 +79,8 @@ /* Array access macro */ #define ARRAY_GET(PDict, Index) (ASSERT((Index) < (PDict)->arraySize), \ (PDict)->data[Index]) +#define ARRAY_GET_PTR(PDict, Index) (ASSERT((Index) < (PDict)->arraySize), \ + &(PDict)->data[Index]) #define ARRAY_PUT(PDict, Index, Val) (ASSERT((Index) < (PDict)->arraySize), \ (PDict)->data[Index] = (Val)) @@ -92,7 +94,7 @@ static void pd_hash_erase_all(Process *p); static Eterm pd_hash_get_with_hval(Process *p, Eterm bucket, Eterm id); static Eterm pd_hash_get_keys(Process *p, Eterm value); static Eterm pd_hash_get_all_keys(Process *p, ProcDict *pd); -static Eterm pd_hash_get_all(Process *p, ProcDict *pd); +static Eterm pd_hash_get_all(Process *p, ProcDict *pd, int keep_dict); static Eterm pd_hash_put(Process *p, Eterm id, Eterm value); static void shrink(Process *p, Eterm* ret); @@ -281,7 +283,7 @@ BIF_RETTYPE get_0(BIF_ALIST_0) { Eterm ret; PD_CHECK(BIF_P->dictionary); - ret = pd_hash_get_all(BIF_P, BIF_P->dictionary); + ret = pd_hash_get_all(BIF_P, BIF_P->dictionary, 1); PD_CHECK(BIF_P->dictionary); BIF_RET(ret); } @@ -329,7 +331,7 @@ BIF_RETTYPE erase_0(BIF_ALIST_0) { Eterm ret; PD_CHECK(BIF_P->dictionary); - ret = pd_hash_get_all(BIF_P, BIF_P->dictionary); + ret = pd_hash_get_all(BIF_P, BIF_P->dictionary, 0); pd_hash_erase_all(BIF_P); PD_CHECK(BIF_P->dictionary); BIF_RET(ret); @@ -541,29 +543,46 @@ static Eterm pd_hash_get_keys(Process *p, Eterm value) static Eterm -pd_hash_get_all(Process *p, ProcDict *pd) +pd_hash_get_all(Process *p, ProcDict *pd, int keep_dict) { Eterm* hp; + Eterm* tp; Eterm res = NIL; Eterm tmp, tmp2; unsigned int i; unsigned int num; + Uint need; if (pd == NULL) { return res; } num = HASH_RANGE(pd); - hp = HAlloc(p, pd->numElements * 2); - + + /* + * If this is not erase/0, then must copy all key-value tuples + * as they may be mutated by put/2. + */ + need = pd->numElements * (keep_dict ? 2+3 : 2); + hp = HAlloc(p, need); + for (i = 0; i < num; ++i) { tmp = ARRAY_GET(pd, i); if (is_boxed(tmp)) { - ASSERT(is_tuple(tmp)); + if (keep_dict) { + tp = tuple_val(tmp); + tmp = TUPLE2(hp, tp[1], tp[2]); + hp += 3; + } res = CONS(hp, tmp, res); hp += 2; } else if (is_list(tmp)) { while (tmp != NIL) { tmp2 = TCAR(tmp); + if (keep_dict) { + tp = tuple_val(tmp2); + tmp2 = TUPLE2(hp, tp[1], tp[2]); + hp += 3; + } res = CONS(hp, tmp2, res); hp += 2; tmp = TCDR(tmp); @@ -577,11 +596,14 @@ static Eterm pd_hash_put(Process *p, Eterm id, Eterm value) { unsigned int hval; Eterm *hp; + Eterm *tp; + Eterm *bucket; Eterm tpl; Eterm old; + Eterm old_val = am_undefined; Eterm tmp; int needed; - int i = 0; + int new_key = 1; #ifdef DEBUG Eterm *hp_limit; #endif @@ -595,7 +617,8 @@ static Eterm pd_hash_put(Process *p, Eterm id, Eterm value) p->dictionary->numElements = 0; } hval = pd_hash_value(p->dictionary, id); - old = ARRAY_GET(p->dictionary, hval); + bucket = ARRAY_GET_PTR(p->dictionary, hval); + old = *bucket; /* * Calculate the number of heap words needed and garbage @@ -603,32 +626,49 @@ static Eterm pd_hash_put(Process *p, Eterm id, Eterm value) */ needed = 3; /* {Key,Value} tuple */ if (is_boxed(old)) { - /* - * We don't want to compare keys twice, so we'll always - * reserve the space for two CONS cells. - */ - needed += 2+2; + ASSERT(is_tuple(old)); + tp = tuple_val(old); + if (EQ(tp[1], id)) { + old_val = tp[2]; + if (is_immed(value)) { + tp[2] = value; /* DESTRUCTIVE HEAP ASSIGNMENT */ + return old_val; + } + new_key = 0; + } + else { + needed += 2+2; + } } else if (is_list(old)) { - i = 0; - for (tmp = old; tmp != NIL && !EQ(tuple_val(TCAR(tmp))[1], id); tmp = TCDR(tmp)) { - ++i; - } - if (is_nil(tmp)) { - i = -1; - needed += 2; - } else { - needed += 2*(i+1); + Eterm* prev_cdr = bucket; + + needed += 2; + for (tmp = old; tmp != NIL; prev_cdr = &TCDR(tmp), tmp = *prev_cdr) { + tp = tuple_val(TCAR(tmp)); + if (EQ(tp[1], id)) { + old_val = tp[2]; + if (is_immed(value)) { + tp[2] = value; /* DESTRUCTIVE HEAP ASSIGNMENT */ + return old_val; + } + new_key = 0; + /* Unlink old {Key,Value} from list */ + *prev_cdr = TCDR(tmp); /* maybe DESTRUCTIVE HEAP ASSIGNMENT */ + break; + } } } if (HeapWordsLeft(p) < needed) { Eterm root[3]; root[0] = id; root[1] = value; - root[2] = old; + root[2] = old_val; erts_garbage_collect(p, needed, root, 3); id = root[0]; value = root[1]; - old = root[2]; + old_val = root[2]; + ASSERT(bucket == ARRAY_GET_PTR(p->dictionary, hval)); + old = *bucket; } #ifdef DEBUG hp_limit = p->htop + needed; @@ -644,66 +684,29 @@ static Eterm pd_hash_put(Process *p, Eterm id, Eterm value) * Update the dictionary. */ if (is_nil(old)) { - ARRAY_PUT(p->dictionary, hval, tpl); - ++(p->dictionary->numElements); + *bucket = tpl; } else if (is_boxed(old)) { ASSERT(is_tuple(old)); - if (EQ(tuple_val(old)[1],id)) { - ARRAY_PUT(p->dictionary, hval, tpl); - return tuple_val(old)[2]; + if (!new_key) { + ASSERT(EQ(tuple_val(old)[1],id)); + *bucket = tpl; + return old_val; } else { hp = HeapOnlyAlloc(p, 4); tmp = CONS(hp, old, NIL); hp += 2; - ++(p->dictionary->numElements); - ARRAY_PUT(p->dictionary, hval, CONS(hp, tpl, tmp)); + *bucket = CONS(hp, tpl, tmp); hp += 2; ASSERT(hp <= hp_limit); } } else if (is_list(old)) { - if (i == -1) { - /* - * New key. Simply prepend the tuple to the beginning of the list. - */ - hp = HeapOnlyAlloc(p, 2); - ARRAY_PUT(p->dictionary, hval, CONS(hp, tpl, old)); - hp += 2; - ASSERT(hp <= hp_limit); - ++(p->dictionary->numElements); - } else { - /* - * i = Number of CDRs to skip to reach the changed element in the list. - * - * Replace old value in list. To avoid pointers from the old generation - * to the new, we must rebuild the list from the beginning up to and - * including the changed element. - */ - Eterm nlist; - int j; - - hp = HeapOnlyAlloc(p, (i+1)*2); - - /* Find the list element to change. */ - for (j = 0, nlist = old; j < i; j++, nlist = TCDR(nlist)) { - ; - } - ASSERT(EQ(tuple_val(TCAR(nlist))[1], id)); - nlist = TCDR(nlist); /* Unchanged part of list. */ - - /* Rebuild list before the updated element. */ - for (tmp = old; i-- > 0; tmp = TCDR(tmp)) { - nlist = CONS(hp, TCAR(tmp), nlist); - hp += 2; - } - ASSERT(EQ(tuple_val(TCAR(tmp))[1], id)); - - /* Put the updated element first in the new list. */ - nlist = CONS(hp, tpl, nlist); - hp += 2; - ASSERT(hp <= hp_limit); - ARRAY_PUT(p->dictionary, hval, nlist); - return tuple_val(TCAR(tmp))[2]; - } + /* + * Simply prepend the tuple to the beginning of the list. + */ + hp = HeapOnlyAlloc(p, 2); + *bucket = CONS(hp, tpl, *bucket); + hp += 2; + ASSERT(hp <= hp_limit); } else { #ifdef DEBUG erts_fprintf(stderr, @@ -714,10 +717,13 @@ static Eterm pd_hash_put(Process *p, Eterm id, Eterm value) erts_exit(ERTS_ERROR_EXIT, "Damaged process dictionary found during put/2."); } + + p->dictionary->numElements += new_key; + if (HASH_RANGE(p->dictionary) <= p->dictionary->numElements) { grow(p); } - return am_undefined; + return old_val; } /* diff --git a/erts/emulator/beam/erl_process_dump.c b/erts/emulator/beam/erl_process_dump.c index f562fc961b..05e7bcdea2 100644 --- a/erts/emulator/beam/erl_process_dump.c +++ b/erts/emulator/beam/erl_process_dump.c @@ -78,8 +78,17 @@ erts_deep_process_dump(fmtfn_t to, void *to_arg) Process *p = erts_pix2proc(i); if (p && p->i != ENULL) { erts_aint32_t state = erts_atomic32_read_acqb(&p->state); - if (!(state & (ERTS_PSFLG_EXITING|ERTS_PSFLG_GC))) - dump_process_info(to, to_arg, p); + if (state & ERTS_PSFLG_EXITING) + continue; + if (state & ERTS_PSFLG_GC) { + ErtsSchedulerData *sdp = erts_get_scheduler_data(); + if (!sdp || p != sdp->current_process) + continue; + + /* We want to dump the garbing process that caused the dump */ + } + + dump_process_info(to, to_arg, p); } } @@ -135,9 +144,12 @@ dump_process_info(fmtfn_t to, void *to_arg, Process *p) ErtsMessage* mp; int yreg = -1; + if (ERTS_TRACE_FLAGS(p) & F_SENSITIVE) + return; + ERTS_MSGQ_MV_INQ2PRIVQ(p); - if ((ERTS_TRACE_FLAGS(p) & F_SENSITIVE) == 0 && p->msg.first) { + if (p->msg.first) { erts_print(to, to_arg, "=proc_messages:%T\n", p->common.id); for (mp = p->msg.first; mp != NULL; mp = mp->next) { Eterm mesg = ERL_MESSAGE_TERM(mp); @@ -152,38 +164,34 @@ dump_process_info(fmtfn_t to, void *to_arg, Process *p) } } - if ((ERTS_TRACE_FLAGS(p) & F_SENSITIVE) == 0) { - if (p->dictionary) { - erts_print(to, to_arg, "=proc_dictionary:%T\n", p->common.id); - erts_deep_dictionary_dump(to, to_arg, - p->dictionary, dump_element_nl); - } + if (p->dictionary) { + erts_print(to, to_arg, "=proc_dictionary:%T\n", p->common.id); + erts_deep_dictionary_dump(to, to_arg, + p->dictionary, dump_element_nl); } - if ((ERTS_TRACE_FLAGS(p) & F_SENSITIVE) == 0) { - erts_print(to, to_arg, "=proc_stack:%T\n", p->common.id); - for (sp = p->stop; sp < STACK_START(p); sp++) { - yreg = stack_element_dump(to, to_arg, sp, yreg); - } + erts_print(to, to_arg, "=proc_stack:%T\n", p->common.id); + for (sp = p->stop; sp < STACK_START(p); sp++) { + yreg = stack_element_dump(to, to_arg, sp, yreg); + } - erts_print(to, to_arg, "=proc_heap:%T\n", p->common.id); - for (sp = p->stop; sp < STACK_START(p); sp++) { - Eterm term = *sp; - - if (!is_catch(term) && !is_CP(term)) { - heap_dump(to, to_arg, term); - } - } - for (mp = p->msg.first; mp != NULL; mp = mp->next) { - Eterm mesg = ERL_MESSAGE_TERM(mp); - if (is_value(mesg)) - heap_dump(to, to_arg, mesg); - mesg = ERL_MESSAGE_TOKEN(mp); - heap_dump(to, to_arg, mesg); - } - if (p->dictionary) { - erts_deep_dictionary_dump(to, to_arg, p->dictionary, heap_dump); - } + erts_print(to, to_arg, "=proc_heap:%T\n", p->common.id); + for (sp = p->stop; sp < STACK_START(p); sp++) { + Eterm term = *sp; + + if (!is_catch(term) && !is_CP(term)) { + heap_dump(to, to_arg, term); + } + } + for (mp = p->msg.first; mp != NULL; mp = mp->next) { + Eterm mesg = ERL_MESSAGE_TERM(mp); + if (is_value(mesg)) + heap_dump(to, to_arg, mesg); + mesg = ERL_MESSAGE_TOKEN(mp); + heap_dump(to, to_arg, mesg); + } + if (p->dictionary) { + erts_deep_dictionary_dump(to, to_arg, p->dictionary, heap_dump); } } @@ -1001,8 +1009,6 @@ erts_dump_extended_process_state(fmtfn_t to, void *to_arg, erts_aint32_t psflg) erts_print(to, to_arg, "SUSPENDED"); break; case ERTS_PSFLG_GC: erts_print(to, to_arg, "GC"); break; - case ERTS_PSFLG_BOUND: - erts_print(to, to_arg, "BOUND"); break; case ERTS_PSFLG_TRAP_EXIT: erts_print(to, to_arg, "TRAP_EXIT"); break; case ERTS_PSFLG_ACTIVE_SYS: @@ -1015,8 +1021,6 @@ erts_dump_extended_process_state(fmtfn_t to, void *to_arg, erts_aint32_t psflg) erts_print(to, to_arg, "DELAYED_SYS"); break; case ERTS_PSFLG_OFF_HEAP_MSGQ: erts_print(to, to_arg, "OFF_HEAP_MSGQ"); break; - case ERTS_PSFLG_ON_HEAP_MSGQ: - erts_print(to, to_arg, "ON_HEAP_MSGQ"); break; case ERTS_PSFLG_DIRTY_CPU_PROC: erts_print(to, to_arg, "DIRTY_CPU_PROC"); break; case ERTS_PSFLG_DIRTY_IO_PROC: diff --git a/erts/emulator/beam/erl_sched_spec_pre_alloc.c b/erts/emulator/beam/erl_sched_spec_pre_alloc.c index ab204303d7..4a6e02281a 100644 --- a/erts/emulator/beam/erl_sched_spec_pre_alloc.c +++ b/erts/emulator/beam/erl_sched_spec_pre_alloc.c @@ -47,6 +47,15 @@ erts_sspa_create(size_t blk_sz, int pa_size, int nthreads, const char* name) int cix; int no_blocks = pa_size; int no_blocks_per_chunk; + size_t aligned_blk_sz; + +#if !defined(ERTS_STRUCTURE_ALIGNED_ALLOC) + /* Force 64-bit alignment... */ + aligned_blk_sz = ((blk_sz - 1) / 8) * 8 + 8; +#else + /* Alignment of structure is enough... */ + aligned_blk_sz = blk_sz; +#endif if (!name) { /* schedulers only variant */ ASSERT(!nthreads); @@ -68,7 +77,7 @@ erts_sspa_create(size_t blk_sz, int pa_size, int nthreads, const char* name) } no_blocks = no_blocks_per_chunk * nthreads; chunk_mem_size = ERTS_ALC_CACHE_LINE_ALIGN_SIZE(sizeof(erts_sspa_chunk_header_t)); - chunk_mem_size += blk_sz * no_blocks_per_chunk; + chunk_mem_size += aligned_blk_sz * no_blocks_per_chunk; chunk_mem_size = ERTS_ALC_CACHE_LINE_ALIGN_SIZE(chunk_mem_size); tot_size = ERTS_ALC_CACHE_LINE_ALIGN_SIZE(sizeof(erts_sspa_data_t)); tot_size += chunk_mem_size * nthreads; @@ -115,7 +124,7 @@ erts_sspa_create(size_t blk_sz, int pa_size, int nthreads, const char* name) blk = (erts_sspa_blk_t *) p; for (i = 0; i < no_blocks_per_chunk; i++) { blk = (erts_sspa_blk_t *) p; - p += blk_sz; + p += aligned_blk_sz; blk->next_ptr = (erts_sspa_blk_t *) p; } diff --git a/erts/emulator/beam/erl_term.h b/erts/emulator/beam/erl_term.h index 7d42d1b396..3dd1c2555c 100644 --- a/erts/emulator/beam/erl_term.h +++ b/erts/emulator/beam/erl_term.h @@ -313,7 +313,8 @@ _ET_DECLARE_CHECKED(Uint,header_arity,Eterm) #define MAX_ARITYVAL ((((Uint)1) << 24) - 1) #define ERTS_MAX_TUPLE_SIZE MAX_ARITYVAL -#define make_arityval(sz) _make_header((sz),_TAG_HEADER_ARITYVAL) +#define make_arityval(sz) (ASSERT((sz) <= MAX_ARITYVAL), \ + _make_header((sz),_TAG_HEADER_ARITYVAL)) #define is_arity_value(x) (((x) & _TAG_HEADER_MASK) == _TAG_HEADER_ARITYVAL) #define is_sane_arity_value(x) ((((x) & _TAG_HEADER_MASK) == _TAG_HEADER_ARITYVAL) && \ (((x) >> _HEADER_ARITY_OFFS) <= MAX_ARITYVAL)) diff --git a/erts/emulator/beam/erl_time.h b/erts/emulator/beam/erl_time.h index 27164d50a0..65211e4e6f 100644 --- a/erts/emulator/beam/erl_time.h +++ b/erts/emulator/beam/erl_time.h @@ -107,9 +107,6 @@ void erts_p_slpq(void); void erts_get_now_cpu(Uint* megasec, Uint* sec, Uint* microsec); #endif -typedef UWord erts_approx_time_t; -erts_approx_time_t erts_get_approx_time(void); - int erts_has_time_correction(void); int erts_check_time_adj_support(int time_correction, ErtsTimeWarpMode time_warp_mode); diff --git a/erts/emulator/beam/erl_time_sup.c b/erts/emulator/beam/erl_time_sup.c index f2e0900fec..8cbdf9fa0f 100644 --- a/erts/emulator/beam/erl_time_sup.c +++ b/erts/emulator/beam/erl_time_sup.c @@ -191,17 +191,6 @@ static struct { ErtsTimeSupData erts_time_sup__ erts_align_attribute(ERTS_CACHE_LINE_SIZE); -/* - * erts_get_approx_time() returns an *approximate* time - * in seconds. NOTE that this time may jump backwards!!! - */ -erts_approx_time_t -erts_get_approx_time(void) -{ - ErtsSystemTime stime = erts_os_system_time(); - return (erts_approx_time_t) ERTS_MONOTONIC_TO_SEC(stime); -} - static ERTS_INLINE void init_time_offset(ErtsMonotonicTime offset) { diff --git a/erts/emulator/beam/erl_trace.c b/erts/emulator/beam/erl_trace.c index 4b996d8fc2..018894f685 100644 --- a/erts/emulator/beam/erl_trace.c +++ b/erts/emulator/beam/erl_trace.c @@ -2533,7 +2533,7 @@ load_tracer_nif(const ErtsTracer tracer) for(i = 0; i < num_of_funcs; i++) { for (j = 0; j < NIF_TRACER_TYPES; j++) { - if (strcmp(tracers[j].name, funcs[i].name) == 0 && tracers[j].arity == funcs[i].arity) { + if (sys_strcmp(tracers[j].name, funcs[i].name) == 0 && tracers[j].arity == funcs[i].arity) { tracers[j].cb = &(funcs[i]); break; } @@ -2783,24 +2783,28 @@ is_tracer_enabled(Process* c_p, ErtsProcLocks c_p_locks, ASSERT(0); } - /* Only remove tracer on self() and ports */ + /* Only remove tracer on (self() or ports) AND we are on a normal scheduler */ if (is_internal_port(t_p->id) || (c_p && c_p->common.id == t_p->id)) { + ErtsSchedulerData *esdp = erts_get_scheduler_data(); ErtsProcLocks c_p_xlocks = 0; - if (is_internal_pid(t_p->id)) { - ERTS_LC_ASSERT(erts_proc_lc_my_proc_locks(c_p) & ERTS_PROC_LOCK_MAIN); - if (c_p_locks != ERTS_PROC_LOCKS_ALL) { - c_p_xlocks = ~c_p_locks & ERTS_PROC_LOCKS_ALL; - if (erts_proc_trylock(c_p, c_p_xlocks) == EBUSY) { - erts_proc_unlock(c_p, c_p_locks & ~ERTS_PROC_LOCK_MAIN); - erts_proc_lock(c_p, ERTS_PROC_LOCKS_ALL_MINOR); + if (esdp && !ERTS_SCHEDULER_IS_DIRTY(esdp)) { + if (is_internal_pid(t_p->id)) { + ERTS_LC_ASSERT(erts_proc_lc_my_proc_locks(c_p) & ERTS_PROC_LOCK_MAIN); + if (c_p_locks != ERTS_PROC_LOCKS_ALL) { + c_p_xlocks = ~c_p_locks & ERTS_PROC_LOCKS_ALL; + if (erts_proc_trylock(c_p, c_p_xlocks) == EBUSY) { + erts_proc_unlock(c_p, c_p_locks & ~ERTS_PROC_LOCK_MAIN); + erts_proc_lock(c_p, ERTS_PROC_LOCKS_ALL_MINOR); + } } } - } - erts_tracer_replace(t_p, erts_tracer_nil); - t_p->trace_flags &= ~TRACEE_FLAGS; - if (c_p_xlocks) - erts_proc_unlock(c_p, c_p_xlocks); + erts_tracer_replace(t_p, erts_tracer_nil); + t_p->trace_flags &= ~TRACEE_FLAGS; + + if (c_p_xlocks) + erts_proc_unlock(c_p, c_p_xlocks); + } } return 0; @@ -3015,7 +3019,7 @@ static void *tracer_alloc_fun(void* tmpl) ErtsTracerNif *obj = erts_alloc(ERTS_ALC_T_TRACER_NIF, sizeof(ErtsTracerNif) + sizeof(ErtsThrPrgrLaterOp)); - memcpy(obj, tmpl, sizeof(*obj)); + sys_memcpy(obj, tmpl, sizeof(*obj)); return obj; } diff --git a/erts/emulator/beam/erl_unicode.c b/erts/emulator/beam/erl_unicode.c index 604f0be127..8673e029e6 100644 --- a/erts/emulator/beam/erl_unicode.c +++ b/erts/emulator/beam/erl_unicode.c @@ -144,7 +144,7 @@ static Eterm make_magic_bin_for_restart(Process *p, RestartContext *rc) cleanup_restart_context_bin); RestartContext *restartp = ERTS_MAGIC_BIN_DATA(mbp); Eterm *hp; - memcpy(restartp,rc,sizeof(RestartContext)); + sys_memcpy(restartp,rc,sizeof(RestartContext)); hp = HAlloc(p, ERTS_MAGIC_REF_THING_SIZE); return erts_mk_magic_ref(&hp, &MSO(p), mbp); } @@ -254,12 +254,12 @@ static Uint copy_utf8_bin(byte *target, byte *source, Uint size, ASSERT(need > 0); ASSERT(from_source > 0); if (size < from_source) { - memcpy(leftover + (*num_leftovers), source, size); + sys_memcpy(leftover + (*num_leftovers), source, size); *num_leftovers += size; return 0; } /* leftover has room for four bytes (see bif) */ - memcpy(leftover + (*num_leftovers),source,from_source); + sys_memcpy(leftover + (*num_leftovers),source,from_source); c = copy_utf8_bin(target, leftover, need, NULL, NULL, &tmp_err_pos, characters); if (tmp_err_pos != 0) { *err_pos = source; @@ -291,7 +291,7 @@ static Uint copy_utf8_bin(byte *target, byte *source, Uint size, size -= 2; copied += 2; } else if (((*source) & ((byte) 0xF0)) == 0xE0) { if (leftover && size < 3) { - memcpy(leftover, source, (int) size); + sys_memcpy(leftover, source, (int) size); *num_leftovers = (int) size; break; } @@ -313,7 +313,7 @@ static Uint copy_utf8_bin(byte *target, byte *source, Uint size, size -= 3; copied += 3; } else if (((*source) & ((byte) 0xF8)) == 0xF0) { if (leftover && size < 4) { - memcpy(leftover, source, (int) size); + sys_memcpy(leftover, source, (int) size); *num_leftovers = (int) size; break; } @@ -2108,7 +2108,7 @@ char *erts_convert_filename_to_encoding(Eterm name, char *statbuf, size_t statbu } else { name_buf = statbuf; } - memcpy(name_buf,bytes,size); + sys_memcpy(name_buf,bytes,size); name_buf[size]=0; } else { name_buf = erts_convert_filename_to_wchar(bytes, size, diff --git a/erts/emulator/beam/erl_utils.h b/erts/emulator/beam/erl_utils.h index 44d8c85867..e4087e0ac8 100644 --- a/erts/emulator/beam/erl_utils.h +++ b/erts/emulator/beam/erl_utils.h @@ -91,7 +91,7 @@ Eterm erts_bld_tuple(Uint **hpp, Uint *szp, Uint arity, ...); #define erts_bld_tuple5(H,S,E1,E2,E3,E4,E5) erts_bld_tuple(H,S,5,E1,E2,E3,E4,E5) Eterm erts_bld_tuplev(Uint **hpp, Uint *szp, Uint arity, Eterm terms[]); Eterm erts_bld_string_n(Uint **hpp, Uint *szp, const char *str, Sint len); -#define erts_bld_string(hpp,szp,str) erts_bld_string_n(hpp,szp,str,strlen(str)) +#define erts_bld_string(hpp,szp,str) erts_bld_string_n(hpp,szp,str,sys_strlen(str)) Eterm erts_bld_list(Uint **hpp, Uint *szp, Sint length, Eterm terms[]); Eterm erts_bld_2tup_list(Uint **hpp, Uint *szp, Sint length, Eterm terms1[], Uint terms2[]); diff --git a/erts/emulator/beam/erl_vm.h b/erts/emulator/beam/erl_vm.h index 76980b5871..f8391fb665 100644 --- a/erts/emulator/beam/erl_vm.h +++ b/erts/emulator/beam/erl_vm.h @@ -69,7 +69,7 @@ # ifdef CHECK_FOR_HOLES # define INIT_HEAP_MEM(p,sz) erts_set_hole_marker(HEAP_TOP(p), (sz)) # else -# define INIT_HEAP_MEM(p,sz) memset(HEAP_TOP(p),0x01,(sz)*sizeof(Eterm*)) +# define INIT_HEAP_MEM(p,sz) sys_memset(HEAP_TOP(p),0x01,(sz)*sizeof(Eterm*)) # endif #else # define INIT_HEAP_MEM(p,sz) ((void)0) diff --git a/erts/emulator/beam/external.c b/erts/emulator/beam/external.c index b12a021e41..fb42969a8f 100644 --- a/erts/emulator/beam/external.c +++ b/erts/emulator/beam/external.c @@ -711,8 +711,8 @@ erts_prepare_dist_ext(ErtsDistExternal *edep, erts_de_rlock(dep); - if (dep->status != ERTS_DE_SFLG_CONNECTED && - dep->status != ERTS_DE_SFLG_PENDING) { + if (dep->state != ERTS_DE_STATE_CONNECTED && + dep->state != ERTS_DE_STATE_PENDING) { erts_de_runlock(dep); return ERTS_PREP_DIST_EXT_CLOSED; } @@ -1953,7 +1953,7 @@ static Eterm erts_term_to_binary_int(Process* p, Eterm Term, int level, Uint fla context_b = erts_create_magic_binary(sizeof(TTBContext), \ ttb_context_destructor); \ context = ERTS_MAGIC_BIN_DATA(context_b); \ - memcpy(context,&c_buff,sizeof(TTBContext)); \ + sys_memcpy(context,&c_buff,sizeof(TTBContext)); \ } \ } while (0) @@ -2873,6 +2873,8 @@ enc_term_int(TTBEncodeContext* ctx, ErtsAtomCacheMap *acmp, Eterm obj, byte* ep, ep[j] = 0; /* Zero unused bits at end of binary */ data_dst = ep; ep += j + 1; + if (ctx) + ctx->hopefull_flags |= DFLAG_BIT_BINARIES; } else { /* * Bit-level binary, but the receiver doesn't support it. @@ -2908,6 +2910,8 @@ enc_term_int(TTBEncodeContext* ctx, ErtsAtomCacheMap *acmp, Eterm obj, byte* ep, ep = enc_atom(acmp, exp->info.mfa.function, ep, dflags); ep = enc_term(acmp, make_small(exp->info.mfa.arity), ep, dflags, off_heap); + if (ctx) + ctx->hopefull_flags |= DFLAG_EXPORT_PTR_TAG; } else { /* Tag, arity */ *ep++ = SMALL_TUPLE_EXT; @@ -4729,11 +4733,13 @@ Sint transcode_dist_obuf(ErtsDistOutputBuf* ob, struct transcode_context* ctx = dep->transcode_ctx; if (!ctx) { /* first call for 'ob' */ - - if (~dflags & (DFLAG_BIT_BINARIES | DFLAG_EXPORT_PTR_TAG)) { + ASSERT(!(ob->hopefull_flags & ~(Uint)(DFLAG_BIT_BINARIES | + DFLAG_EXPORT_PTR_TAG))); + if (~dflags & ob->hopefull_flags) { /* - * Receiver does not support bitstrings and/or export funs. - * We need to transcode control and message terms to use tuple fallbacks. + * Receiver does not support bitstrings and/or export funs + * and output buffer contains such message tags (hopefull_flags). + * Must transcode control and message terms to use tuple fallbacks. */ ctx = erts_alloc(ERTS_ALC_T_DIST_TRANSCODE, sizeof(struct transcode_context)); dep->transcode_ctx = ctx; @@ -4760,7 +4766,7 @@ Sint transcode_dist_obuf(ErtsDistOutputBuf* ob, ctx->state = TRANSCODE_ENC_CTL; } } - else { + else if (!(dflags & DFLAG_DIST_HDR_ATOM_CACHE)) { /* * No need for full transcoding, but primitive receiver (erl_/jinterface) * expects VERSION_MAGIC before both control and message terms. @@ -4777,8 +4783,10 @@ Sint transcode_dist_obuf(ErtsDistOutputBuf* ob, *--(ob->extp) = VERSION_MAGIC; goto done; } + else + goto done; } - else { + else { /* continue after yield */ ASSERT(ctx->dbg_ob == ob); } ctx->b2t.reds = reds * B2T_BYTES_PER_REDUCTION; @@ -4831,6 +4839,7 @@ Sint transcode_dist_obuf(ErtsDistOutputBuf* ob, ob->msg_start = ob->ext_endp; ctx->ttb.wstack.wstart = NULL; ctx->ttb.flags = dflags; + ctx->ttb.hopefull_flags = 0; ctx->ttb.level = 0; ctx->state = TRANSCODE_ENC_MSG; @@ -4843,7 +4852,7 @@ Sint transcode_dist_obuf(ErtsDistOutputBuf* ob, reds /= TERM_TO_BINARY_LOOP_FACTOR; ASSERT(ob->ext_endp <= ob->alloc_endp); - + ASSERT(!ctx->ttb.hopefull_flags); } transcode_free_ctx(dep); diff --git a/erts/emulator/beam/global.h b/erts/emulator/beam/global.h index 86e2c351af..0f23027752 100644 --- a/erts/emulator/beam/global.h +++ b/erts/emulator/beam/global.h @@ -368,7 +368,7 @@ do {\ UWord _wsz = ESTACK_COUNT(s);\ (dst)->start = erts_alloc((s).alloc_type,\ DEF_ESTACK_SIZE * sizeof(Eterm));\ - memcpy((dst)->start, (s).start,_wsz*sizeof(Eterm));\ + sys_memcpy((dst)->start, (s).start,_wsz*sizeof(Eterm));\ (dst)->sp = (dst)->start + _wsz;\ (dst)->end = (dst)->start + DEF_ESTACK_SIZE;\ (dst)->edefault = NULL;\ @@ -536,7 +536,7 @@ do {\ UWord _wsz = WSTACK_COUNT(s);\ (dst)->wstart = erts_alloc(s.alloc_type,\ DEF_WSTACK_SIZE * sizeof(UWord));\ - memcpy((dst)->wstart, s.wstart,_wsz*sizeof(UWord));\ + sys_memcpy((dst)->wstart, s.wstart,_wsz*sizeof(UWord));\ (dst)->wsp = (dst)->wstart + _wsz;\ (dst)->wend = (dst)->wstart + DEF_WSTACK_SIZE;\ (dst)->wdefault = NULL;\ @@ -948,8 +948,8 @@ void erts_update_ranges(BeamInstr* code, Uint size); void erts_remove_from_ranges(BeamInstr* code); UWord erts_ranges_sz(void); void erts_lookup_function_info(FunctionInfo* fi, BeamInstr* pc, int full_info); -ErtsLiteralArea** erts_dump_lit_areas; -Uint erts_dump_num_lit_areas; +extern ErtsLiteralArea** erts_dump_lit_areas; +extern Uint erts_dump_num_lit_areas; /* break.c */ void init_break_handler(void); @@ -1066,7 +1066,7 @@ Eterm copy_struct_x(Eterm, Uint, Eterm**, ErlOffHeap*, Uint*, erts_literal_area_ #define copy_struct_litopt(Obj,Sz,HPP,OH,LitArea) \ copy_struct_x(Obj,Sz,HPP,OH,NULL,LitArea) -Eterm copy_shallow(Eterm*, Uint, Eterm**, ErlOffHeap*); +Eterm copy_shallow(Eterm* ERTS_RESTRICT, Uint, Eterm**, ErlOffHeap*); void erts_move_multi_frags(Eterm** hpp, ErlOffHeap*, ErlHeapFragment* first, Eterm* refs, unsigned nrefs, int literals); diff --git a/erts/emulator/beam/io.c b/erts/emulator/beam/io.c index 2c1b7871c4..e4a5f2b6b6 100644 --- a/erts/emulator/beam/io.c +++ b/erts/emulator/beam/io.c @@ -298,7 +298,6 @@ static Port *create_port(char *name, erts_aint32_t state = ERTS_PORT_SFLG_CONNECTED; erts_aint32_t x_pts_flgs = 0; - ErtsRunQueue *runq; if (!driver_lock) { /* Align size for mutex following port struct */ port_size = size = ERTS_ALC_DATA_ALIGN_SIZE(sizeof(Port)); @@ -347,11 +346,16 @@ static Port *create_port(char *name, p += sizeof(erts_mtx_t); state |= ERTS_PORT_SFLG_PORT_SPECIFIC_LOCK; } - if (erts_get_scheduler_data()) - runq = erts_get_runq_current(NULL); - else - runq = ERTS_RUNQ_IX(0); - erts_atomic_set_nob(&prt->run_queue, (erts_aint_t) runq); + + { + ErtsRunQueue *runq; + ErtsSchedulerData *esdp = erts_get_scheduler_data(); + if (esdp) + runq = erts_get_runq_current(esdp); + else + runq = ERTS_RUNQ_IX(0); + erts_init_runq_port(prt, runq); + } prt->xports = NULL; @@ -584,7 +588,7 @@ erts_open_driver(erts_driver_t* driver, /* Pointer to driver. */ */ for (d = driver_list; d; d = d->next) { - if (strcmp(d->name, name) == 0 && + if (sys_strcmp(d->name, name) == 0 && erts_ddll_driver_ok(d->handle)) { driver = d; break; @@ -634,7 +638,7 @@ erts_open_driver(erts_driver_t* driver, /* Pointer to driver. */ trace_port_open(port, pid, erts_atom_put((byte *) port->name, - strlen(port->name), + sys_strlen(port->name), ERTS_ATOM_ENC_LATIN1, 1)); } @@ -2925,7 +2929,7 @@ erl_drv_init_ack(ErlDrvPort ix, ErlDrvData res) { break; case -2: { char *str = erl_errno_id(errno); - resp = erts_atom_put((byte *) str, strlen(str), + resp = erts_atom_put((byte *) str, sys_strlen(str), ERTS_ATOM_ENC_LATIN1, 1); break; } @@ -5075,6 +5079,93 @@ static void prt_one_lnk(ErtsLink *lnk, void *vprtd) erts_print(prtd->to, prtd->arg, "%T", lnk->pid); } +static void dump_port_state(fmtfn_t to, void *arg, erts_aint32_t state) +{ + erts_aint32_t rest; + int unknown = 0; + char delim = ' '; + + erts_print(to, arg, "State:"); + + rest = state; + while (rest) { + erts_aint32_t chk = (rest ^ (rest-1)) & rest; /* lowest set bit */ + char* s; + + rest &= ~chk; + switch (chk) { + case ERTS_PORT_SFLG_CONNECTED: s = "CONNECTED"; break; + case ERTS_PORT_SFLG_EXITING: s = "EXITING"; break; + case ERTS_PORT_SFLG_DISTRIBUTION: s = "DISTR"; break; + case ERTS_PORT_SFLG_BINARY_IO: s = "BINARY_IO"; break; + case ERTS_PORT_SFLG_SOFT_EOF: s = "SOFT_EOF"; break; + case ERTS_PORT_SFLG_CLOSING: s = "CLOSING"; break; + case ERTS_PORT_SFLG_SEND_CLOSED: s = "SEND_CLOSED"; break; + case ERTS_PORT_SFLG_LINEBUF_IO: s = "LINEBUF_IO"; break; + case ERTS_PORT_SFLG_FREE: s = "FREE"; break; + case ERTS_PORT_SFLG_INITIALIZING: s = "INITIALIZING"; break; + case ERTS_PORT_SFLG_PORT_SPECIFIC_LOCK: s = "PORT_LOCK"; break; + case ERTS_PORT_SFLG_INVALID: s = "INVALID"; break; + case ERTS_PORT_SFLG_HALT: s = "HALT"; break; +#ifdef DEBUG + case ERTS_PORT_SFLG_PORT_DEBUG: s = "DEBUG"; break; +#endif + default: + unknown = 1; + continue; + } + erts_print(to, arg, "%c%s", delim, s); + delim = '|'; + } + if (unknown || !state) + erts_print(to, arg, "%c0x%x\n", delim, state); + else + erts_print(to, arg, "\n"); +} + +static void dump_port_task_flags(fmtfn_t to, void *arg, Port* p) +{ + erts_aint32_t flags = erts_atomic32_read_nob(&p->sched.flags); + erts_aint32_t unknown = 0; + char delim = ' '; + + if (!flags) + return; + + erts_print(to, arg, "Task Flags:"); + + while (flags) { + erts_aint32_t chk = (flags ^ (flags-1)) & flags; /* lowest set bit */ + char* s; + + flags &= ~chk; + switch (chk) { + case ERTS_PTS_FLG_IN_RUNQ: s = "IN_RUNQ"; break; + case ERTS_PTS_FLG_EXEC: s = "EXEC"; break; + case ERTS_PTS_FLG_HAVE_TASKS: s = "HAVE_TASKS"; break; + case ERTS_PTS_FLG_EXIT: s = "EXIT"; break; + case ERTS_PTS_FLG_BUSY_PORT: s = "BUSY_PORT"; break; + case ERTS_PTS_FLG_BUSY_PORT_Q: s = "BUSY_Q"; break; + case ERTS_PTS_FLG_CHK_UNSET_BUSY_PORT_Q: s = "CHK_UNSET_BUSY_Q"; break; + case ERTS_PTS_FLG_HAVE_BUSY_TASKS: s = "BUSY_TASKS"; break; + case ERTS_PTS_FLG_HAVE_NS_TASKS: s = "NS_TASKS"; break; + case ERTS_PTS_FLG_PARALLELISM: s = "PARALLELISM"; break; + case ERTS_PTS_FLG_FORCE_SCHED: s = "FORCE_SCHED"; break; + case ERTS_PTS_FLG_EXITING: s = "EXITING"; break; + case ERTS_PTS_FLG_EXEC_IMM: s = "EXEC_IMM"; break; + default: + unknown |= chk; + continue; + } + erts_print(to, arg, "%c%s", delim, s); + delim = '|'; + } + if (unknown) + erts_print(to, arg, "%cUNKNOWN(0x%x)\n", delim, unknown); + else + erts_print(to, arg, "\n"); +} + void print_port_info(Port *p, fmtfn_t to, void *arg) { @@ -5084,6 +5175,8 @@ print_port_info(Port *p, fmtfn_t to, void *arg) return; erts_print(to, arg, "=port:%T\n", p->common.id); + dump_port_state(to, arg, state); + dump_port_task_flags(to, arg, p); erts_print(to, arg, "Slot: %d\n", internal_port_index(p->common.id)); if (state & ERTS_PORT_SFLG_CONNECTED) { erts_print(to, arg, "Connected: %T", ERTS_PORT_GET_CONNECTED(p)); @@ -5106,6 +5199,10 @@ print_port_info(Port *p, fmtfn_t to, void *arg) erts_doforall_monitors(ERTS_P_MONITORS(p), &prt_one_monitor, &prtd); erts_print(to, arg, "\n"); } + if (p->suspended) { + erts_print(to, arg, "Suspended: "); + erts_proclist_dump(to, arg, p->suspended); + } if (p->common.u.alive.reg != NULL) erts_print(to, arg, "Registered as: %T\n", p->common.u.alive.reg->name); @@ -5123,6 +5220,14 @@ print_port_info(Port *p, fmtfn_t to, void *arg) } else { erts_print(to, arg, "Port controls linked-in driver: %s\n",p->name); } + erts_print(to, arg, "Input: %beu\n", p->bytes_in); + erts_print(to, arg, "Output: %beu\n", p->bytes_out); + erts_print(to, arg, "Queue: %beu\n", erts_ioq_size(&p->ioq)); + { + Eterm port_data = erts_port_data_read(p); + if (port_data != am_undefined) + erts_print(to, arg, "Port Data: %T\n", port_data); + } } void @@ -7219,7 +7324,7 @@ int driver_failure_atom(ErlDrvPort ix, char* string) { return driver_failure_term(ix, erts_atom_put((byte *) string, - strlen(string), + sys_strlen(string), ERTS_ATOM_ENC_LATIN1, 1), 0); diff --git a/erts/emulator/beam/lttng-wrapper.h b/erts/emulator/beam/lttng-wrapper.h index 0bc75c1552..e7f2971bf7 100644 --- a/erts/emulator/beam/lttng-wrapper.h +++ b/erts/emulator/beam/lttng-wrapper.h @@ -61,13 +61,13 @@ #define lttng_proc_to_mfa_str(p, Name) \ do { \ if (ERTS_PROC_IS_EXITING((p))) { \ - strcpy(Name, "<exiting>"); \ + sys_strcpy(Name, "<exiting>"); \ } else { \ BeamInstr *_fptr = find_function_from_pc((p)->i); \ if (_fptr) { \ lttng_mfa_to_str(_fptr[0],_fptr[1],_fptr[2], Name); \ } else { \ - strcpy(Name, "<unknown>"); \ + sys_strcpy(Name, "<unknown>"); \ } \ } \ } while(0) diff --git a/erts/emulator/beam/packet_parser.c b/erts/emulator/beam/packet_parser.c index f14910bc72..de1d481105 100644 --- a/erts/emulator/beam/packet_parser.c +++ b/erts/emulator/beam/packet_parser.c @@ -200,7 +200,7 @@ static int http_init(void) for (i = 0; i < HTTP_HDR_HASH_SIZE; i++) http_hdr_hash[i] = NULL; for (i = 0; http_hdr_strings[i] != NULL; i++) { - ASSERT(strlen(http_hdr_strings[i]) <= HTTP_MAX_NAME_LEN); + ASSERT(sys_strlen(http_hdr_strings[i]) <= HTTP_MAX_NAME_LEN); http_hdr_table[i].index = i; http_hash_insert(http_hdr_strings[i], &http_hdr_table[i], @@ -516,7 +516,7 @@ static http_atom_t* http_hash_lookup(const char* name, int len, while (ap != NULL) { if ((ap->h == h) && (ap->len == len) && - (strncmp(ap->name, name, len) == 0)) + (sys_strncmp(ap->name, name, len) == 0)) return ap; ap = ap->next; } @@ -656,7 +656,7 @@ int packet_parse_http(const char* buf, int len, int* statep, if (*statep == 0) { /* start-line = Request-Line | Status-Line */ - if (n >= 5 && (strncmp(buf, "HTTP/", 5) == 0)) { + if (n >= 5 && (sys_strncmp(buf, "HTTP/", 5) == 0)) { int major = 0; int minor = 0; int status = 0; @@ -750,7 +750,7 @@ int packet_parse_http(const char* buf, int len, int* statep, } if (n < 8) return -1; - if (strncmp(ptr, "HTTP/", 5) != 0) + if (sys_strncmp(ptr, "HTTP/", 5) != 0) return -1; ptr += 5; n -= 5; diff --git a/erts/emulator/beam/sys.h b/erts/emulator/beam/sys.h index 290e0b209a..13ae80e4a5 100644 --- a/erts/emulator/beam/sys.h +++ b/erts/emulator/beam/sys.h @@ -996,16 +996,36 @@ erts_refc_read(erts_refc_t *refcp, erts_aint_t min_val) #endif /* #if ERTS_GLB_INLINE_INCL_FUNC_DEF */ -#define sys_memcpy(s1,s2,n) memcpy(s1,s2,n) -#define sys_memmove(s1,s2,n) memmove(s1,s2,n) -#define sys_memcmp(s1,s2,n) memcmp(s1,s2,n) -#define sys_memset(s,c,n) memset(s,c,n) -#define sys_memzero(s, n) memset(s,'\0',n) -#define sys_strcmp(s1,s2) strcmp(s1,s2) -#define sys_strncmp(s1,s2,n) strncmp(s1,s2,n) -#define sys_strcpy(s1,s2) strcpy(s1,s2) -#define sys_strncpy(s1,s2,n) strncpy(s1,s2,n) -#define sys_strlen(s) strlen(s) +/* Thin wrappers around memcpy and friends, which should always be used in + * place of plain memcpy, memset, etc. + * + * Passing NULL to any of these functions is undefined behavior even though it + * may seemingly work when the length (if any) is zero; a compiler can take + * this as a hint that the passed operand may *never* be NULL and then optimize + * based on that information. + * + * (The weird casts in the assertions silence an "always evaluates to true" + * warning when an operand is the address of an lvalue) */ +#define sys_memcpy(s1,s2,n) \ + (ASSERT((void*)(s1) != NULL && (void*)(s2) != NULL), memcpy(s1,s2,n)) +#define sys_memmove(s1,s2,n) \ + (ASSERT((void*)(s1) != NULL && (void*)(s2) != NULL), memmove(s1,s2,n)) +#define sys_memcmp(s1,s2,n) \ + (ASSERT((void*)(s1) != NULL && (void*)(s2) != NULL), memcmp(s1,s2,n)) +#define sys_memset(s,c,n) \ + (ASSERT((void*)(s) != NULL), memset(s,c,n)) +#define sys_memzero(s, n) \ + (ASSERT((void*)(s) != NULL), memset(s,'\0',n)) +#define sys_strcmp(s1,s2) \ + (ASSERT((void*)(s1) != NULL && (void*)(s2) != NULL), strcmp(s1,s2)) +#define sys_strncmp(s1,s2,n) \ + (ASSERT((void*)(s1) != NULL && (void*)(s2) != NULL), strncmp(s1,s2,n)) +#define sys_strcpy(s1,s2) \ + (ASSERT((void*)(s1) != NULL && (void*)(s2) != NULL), strcpy(s1,s2)) +#define sys_strncpy(s1,s2,n) \ + (ASSERT((void*)(s1) != NULL && (void*)(s2) != NULL), strncpy(s1,s2,n)) +#define sys_strlen(s) \ + (ASSERT((void*)(s) != NULL), strlen(s)) /* define function symbols (needed in sys_drv_api) */ #define sys_fp_alloc sys_alloc diff --git a/erts/emulator/beam/utils.c b/erts/emulator/beam/utils.c index fe9f1c7606..4bf60619ba 100644 --- a/erts/emulator/beam/utils.c +++ b/erts/emulator/beam/utils.c @@ -3152,6 +3152,9 @@ tailrecur_ne: int cmp; byte* a_ptr; byte* b_ptr; + if (eq_only && a_size != b_size) { + RETURN_NEQ(a_size - b_size); + } ERTS_GET_BINARY_BYTES(a, a_ptr, a_bitoffs, a_bitsize); ERTS_GET_BINARY_BYTES(b, b_ptr, b_bitoffs, b_bitsize); if ((a_bitsize | b_bitsize | a_bitoffs | b_bitoffs) == 0) { diff --git a/erts/emulator/hipe/hipe_gc.c b/erts/emulator/hipe/hipe_gc.c index 1a4a4c7952..aaedba1afd 100644 --- a/erts/emulator/hipe/hipe_gc.c +++ b/erts/emulator/hipe/hipe_gc.c @@ -91,7 +91,7 @@ Eterm *fullsweep_nstack(Process *p, Eterm *n_htop) ASSERT(is_boxed(val)); *nsp_i = val; } else if (!erts_is_literal(gval, ptr)) { - move_boxed(&ptr, val, &n_htop, nsp_i); + move_boxed(ptr, val, &n_htop, nsp_i); } } else if (is_list(gval)) { Eterm *ptr = list_val(gval); @@ -100,7 +100,7 @@ Eterm *fullsweep_nstack(Process *p, Eterm *n_htop) *nsp_i = ptr[1]; } else if (!erts_is_literal(gval, ptr)) { ASSERT(erts_dbg_within_proc(ptr, p, NULL)); - move_cons(&ptr, val, &n_htop, nsp_i); + move_cons(ptr, val, &n_htop, nsp_i); } } } @@ -206,10 +206,10 @@ void gensweep_nstack(Process *p, Eterm **ptr_old_htop, Eterm **ptr_n_htop) ASSERT(is_boxed(val)); *nsp_i = val; } else if (ErtsInArea(ptr, mature, mature_size)) { - move_boxed(&ptr, val, &old_htop, nsp_i); + move_boxed(ptr, val, &old_htop, nsp_i); } else if (ErtsInYoungGen(gval, ptr, oh, oh_size)) { ASSERT(erts_dbg_within_proc(ptr, p, NULL)); - move_boxed(&ptr, val, &n_htop, nsp_i); + move_boxed(ptr, val, &n_htop, nsp_i); } } else if (is_list(gval)) { Eterm *ptr = list_val(gval); @@ -217,10 +217,10 @@ void gensweep_nstack(Process *p, Eterm **ptr_old_htop, Eterm **ptr_n_htop) if (IS_MOVED_CONS(val)) { *nsp_i = ptr[1]; } else if (ErtsInArea(ptr, mature, mature_size)) { - move_cons(&ptr, val, &old_htop, nsp_i); + move_cons(ptr, val, &old_htop, nsp_i); } else if (ErtsInYoungGen(gval, ptr, oh, oh_size)) { ASSERT(erts_dbg_within_proc(ptr, p, NULL)); - move_cons(&ptr, val, &n_htop, nsp_i); + move_cons(ptr, val, &n_htop, nsp_i); } } } @@ -278,7 +278,7 @@ Eterm *sweep_literals_nstack(Process *p, Eterm *old_htop, char *area, ASSERT(is_boxed(val)); *nsp_i = val; } else if (ErtsInArea(ptr, area, area_size)) { - move_boxed(&ptr, val, &old_htop, nsp_i); + move_boxed(ptr, val, &old_htop, nsp_i); } } else if (is_list(gval)) { Eterm *ptr = list_val(gval); @@ -286,7 +286,7 @@ Eterm *sweep_literals_nstack(Process *p, Eterm *old_htop, char *area, if (IS_MOVED_CONS(val)) { *nsp_i = ptr[1]; } else if (ErtsInArea(ptr, area, area_size)) { - move_cons(&ptr, val, &old_htop, nsp_i); + move_cons(ptr, val, &old_htop, nsp_i); } } } diff --git a/erts/emulator/internal_doc/CarrierMigration.md b/erts/emulator/internal_doc/CarrierMigration.md index 2a9594db25..3a796d11b7 100644 --- a/erts/emulator/internal_doc/CarrierMigration.md +++ b/erts/emulator/internal_doc/CarrierMigration.md @@ -3,17 +3,17 @@ Carrier Migration The ERTS memory allocators manage memory blocks in two types of raw memory chunks. We call these chunks of raw memory -*carriers*. Singleblock carriers which only contain one large block, -and multiblock carriers which contain multiple blocks. A carrier is +*carriers*. Single-block carriers which only contain one large block, +and multi-block carriers which contain multiple blocks. A carrier is typically created using `mmap()` on unix systems. However, how a carrier is created is of minor importance. An allocator instance -typically manages a mixture of single- and multiblock carriers. +typically manages a mixture of single- and multi-block carriers. Problem ------- When a carrier is empty, i.e. contains only one large free block, it -is deallocated. Since multiblock carriers can contain both allocated +is deallocated. Since multi-block carriers can contain both allocated blocks and free blocks at the same time, an allocator instance might be stuck with a large amount of poorly utilized carriers if the memory load decreases. After a peak in memory usage it is expected that not @@ -23,9 +23,9 @@ can usually be reused if the memory load increases again. However, since each scheduler thread manages its own set of allocator instances, and memory load is not necessarily correlated to CPU load, we might get into a situation where there are lots of poorly utilized -multiblock carriers on some allocator instances while we need to -allocate new multiblock carriers on other allocator instances. In -scenarios like this, the demand for multiblock carriers in the system +multi-block carriers on some allocator instances while we need to +allocate new multi-block carriers on other allocator instances. In +scenarios like this, the demand for multi-block carriers in the system might increase at the same time as the actual memory demand in the system has decreased which is both unwanted and quite unexpected for the end user. @@ -34,7 +34,7 @@ Solution -------- In order to prevent scenarios like this we've implemented support for -migration of multiblock carriers between allocator instances of the +migration of multi-block carriers between allocator instances of the same type. ### Management of Free Blocks ### @@ -44,7 +44,7 @@ and add it to another we need to be able to move references to the free blocks of the carrier between the allocator instances. The allocator instance specific data structure referring to the free blocks it manages often refers to the same carrier from multiple -places. For example, when the address order bestfit strategy is used +places. For example, when the address order best-fit strategy is used this data structure is a binary search tree spanning all carriers that the allocator instance manages. Free blocks in one specific carrier can be referred to from potentially every other carrier that is @@ -135,7 +135,7 @@ carriers between scheduler specific allocator instances of the same allocator type. Each allocator instance keeps track of the current utilization of its -multiblock carriers. When the total utilization falls below the "abandon +multi-block carriers. When the total utilization falls below the "abandon carrier utilization limit" it starts to inspect the utilization of the current carrier when deallocations are made. If also the utilization of the carrier falls below the "abandon carrier utilization limit" it @@ -144,31 +144,45 @@ and inserts the carrier into the pool. Since the carrier has been unlinked from the data structure of available free blocks, no more allocations will be made in the -carrier. The allocator instance putting the carrier into the pool, -however, still has the responsibility of performing deallocations in -it while it remains in the pool. The allocator instance with this -deallocation responsibility is here called the **employer**. - -Each carrier has a flag field containing information about the -employing allocator instance, a flag indicating if the carrier is in -the pool or not, and a flag indicating if it is busy or not. When the -carrier is in the pool, the employing allocator instance needs to mark it -as busy while operating on it. If another thread inspects it in order -to try to fetch it from the pool, it will skip it if it is busy. When -fetching the carrier from the pool, employment will change and further +carrier. + +The allocator instance that created a carrier is called its **owner**. +Ownership never changes. + +The allocator instance that has the responsibility to perform deallocations in a +carrier is called its **employer**. The employer may also perform allocations if +the carrier is not in the pool. Employment may change when a carrier is fetched from +or inserted into the pool. + +Deallocations in a carrier, while it remains in the pool, is always performed +the owner. That is, all pooled carriers are employed by their owners. + +Each carrier has an atomic word containing a pointer to the employing allocator +instance and three bit flags; IN_POOL, BUSY and HOMECOMING. + +When fetching a carrier from the pool, employment may change and further deallocations in the carrier will be redirected to the new employer using the delayed dealloc functionality. -If a carrier in the pool becomes empty, it will be withdrawn from the -pool. All carriers that become empty are also always passed to its -**owning** allocator instance for deallocation using the delayed -dealloc functionality. Since carriers this way always will be -deallocated by the owner that allocated the carrier, the +When a foreign allocator instance abandons a carrier back into the pool, it will +also pass it back to its **owner** using the delayed dealloc queue. When doing +this it will set the HOMECOMING bit flag to mark it as "enqueued". The owner +will later clear the HOMECOMING bit when the carrier is dequeued. This mechanism +prevents a carrier from being enqueued again before it has been dequeued. + +When a carrier becomes empty, it will be deallocated. Carrier deallocation is +always done by the owner that allocated the carrier. By doing this, the underlying functionality of allocating and deallocating carriers can remain simple and doesn't have to bother about multiple threads. In a NUMA system we will also not mix carriers originating from multiple NUMA nodes. +If a carrier in the pool becomes empty, it will be withdrawn from the +pool and be deallocated by the owner which already employs it. + +If a carrier employed by a foreign allocator becomes empty, it will be passed +back to the owner for deallocation using the delayed dealloc functionality. + In short: * The allocator instance that created a carrier **owns** it. @@ -177,34 +191,31 @@ In short: * The allocator instance that uses a carrier **employs** it. * An **employer** can abandon a carrier into the pool. * Pooled carriers are not allocated from. -* Deallocation in a pooled carrier is still performed by its **employer**. -* **Employment** can only change when a carrier is fetched from the pool. +* Pooled carriers are always **employed** by their **owner**. +* **Employment** can only change from **owner** to a foreign allocator + when a carrier is fetched from the pool. + ### Searching the pool ### +When an allocator instance needs more carrier space, it inspects the pool. If no +carrier could be fetched from the pool, it will allocate a new +carrier. Regardless of where the allocator instance gets the carrier from, it +just links in the carrier into its data structure of free blocks. + To harbor real time characteristics, searching the pool is limited. We only inspect a limited number of carriers. If none of those carriers had a free block large enough to satisfy the allocation -request, the search will fail. A carrier in the pool can also be busy +request, the search will fail. A carrier in the pool can also be BUSY if another thread is currently doing block deallocation work on the -carrier. A busy carrier will also be skipped by the search as it can +carrier. A BUSY carrier will also be skipped by the search as it can not satisfy the request. The pool is lock-free and we do not want to block, waiting for the other thread to finish. -#### Before OTP 17.4 #### +### The bad cluster problem ### -When an allocator instance needs more carrier space, it always begins -by inspecting its own carriers that are waiting for thread progress -before they can be deallocated. If no such carrier could be found, it -then inspects the pool. If no carrier could be fetched from the pool, -it will allocate a new carrier. Regardless of where the allocator -instance gets the carrier from it the just links in the carrier into -its data structure of free blocks. - -#### After OTP 17.4 #### - -The old search algorithm had a problem as the search always started at -the same position in the pool, the sentinel. This could lead to +Before OTP-17.4 the search algorithm had a problem as the search always started +at the same position in the pool, the sentinel. This could lead to contention from concurrent searching processes. But even worse, it could lead to a "bad" state when searches fail with a high rate leading to new carriers instead being allocated. These new carriers @@ -236,26 +247,27 @@ The result is that we prefer carriers created by the thread itself, which is good for NUMA performance. And we get more entry points when searching the pool, which will ease contention and clustering. +### Our own pooled tree ### + To do the first search among own carriers, every allocator instance -has two new lists: `pooled_list` and `traitor_list`. These lists are only -accessed by the allocator itself and they only contain the allocator's -own carriers. When an owned carrier is abandoned and put in the -pool, it is also linked into `pooled_list`. When we search our -`pooled_list` and find a carrier that is no longer in the pool, we -move that carrier from `pooled_list` to `traitor_list` as it is now -employed by another allocator. If searching `pooled_list` fails, we -also do a limited search of `traitor_list`. When finding an abandoned -carrier in `traitor_list` it is either employed or moved back to -`pooled_list` if it could not satisfy the allocation request. - -When searching `pooled_list` and `traitor_list` we always start at the -point where the last search ended. This to avoid clustering -problems and increase the probability to find a "good" carrier. As -`pooled_list` and `traitor_list` are only accessed by the owning -allocator instance, they need no thread synchronization at all. +has a `pooled_tree` of carriers. This tree is only accessed by the allocator +itself and can only contain its own carriers. When a carrier is +abandoned and put in the pool, it is also inserted into `pooled_tree`. This is +either done direct, if the carrier was already employed by its owner, or by +first passing it back to the owner via the delayed dealloc queue. + +When we search our `pooled_tree` and find a carrier that is no longer in the +pool, we remove that carrier from `pooled_tree` and mark it as TRAITOR, as it is +now employed by a foreign allocator. We will not find any carriers in +`pooled_tree` that are marked as BUSY by other threads. + +If no carrier in `pooled_tree` had a large enough free block, we search it again +to find any carrier that may act as an entry point into the shared list of all +pooled carriers. This in order to, if possible, avoid starting at the sentinel +and thereby ease the "bad clustering" problem. Furthermore, the search for own carriers that are scheduled -for deallocation is now done as the last search option. The idea is +for deallocation is done as the last search option. The idea is that it is better to reuse a poorly utilized carrier than to resurrect an empty carrier that was just about to be released back to the OS. @@ -271,14 +283,14 @@ load did not. When using the `aoffcaobf` or `aoff` strategies compared to `gf` or `bf`, we loose some performance since we get more modifications in the data structure of free blocks. This performance penalty is however -reduced using the `aoffcbf` strategy. A tradeoff between memory +reduced using the `aoffcbf` strategy. A trade off between memory consumption and performance is however inevitable, and it is up to the user to decide what is most important. Further work ------------ -It would be quite easy to extend this to allow migration of multiblock +It would be quite easy to extend this to allow migration of multi-block carriers between all allocator types. More or less the only obstacle is maintenance of the statistics information. diff --git a/erts/emulator/nifs/common/prim_file_nif.c b/erts/emulator/nifs/common/prim_file_nif.c index 6874f41d75..bbd9becb47 100644 --- a/erts/emulator/nifs/common/prim_file_nif.c +++ b/erts/emulator/nifs/common/prim_file_nif.c @@ -891,10 +891,10 @@ static ERL_NIF_TERM set_owner_nif(ErlNifEnv *env, int argc, const ERL_NIF_TERM a posix_errno_t posix_errno; efile_path_t path; - Uint32 uid, gid; + Sint32 uid, gid; - if(argc != 3 || !enif_get_uint(env, argv[1], &uid) - || !enif_get_uint(env, argv[2], &gid)) { + if(argc != 3 || !enif_get_int(env, argv[1], &uid) + || !enif_get_int(env, argv[2], &gid)) { return enif_make_badarg(env); } diff --git a/erts/emulator/nifs/common/prim_file_nif.h b/erts/emulator/nifs/common/prim_file_nif.h index cc9bc8f5c3..4194cdc7d9 100644 --- a/erts/emulator/nifs/common/prim_file_nif.h +++ b/erts/emulator/nifs/common/prim_file_nif.h @@ -177,7 +177,7 @@ posix_errno_t efile_set_permissions(const efile_path_t *path, Uint32 permissions /** @brief On Unix, this will set the owner/group to the given values. It will * do nothing on other platforms. */ -posix_errno_t efile_set_owner(const efile_path_t *path, Uint32 owner, Uint32 group); +posix_errno_t efile_set_owner(const efile_path_t *path, Sint32 owner, Sint32 group); /** @brief Resolves the final path of the given link. */ posix_errno_t efile_read_link(ErlNifEnv *env, const efile_path_t *path, ERL_NIF_TERM *result); diff --git a/erts/emulator/nifs/unix/unix_prim_file.c b/erts/emulator/nifs/unix/unix_prim_file.c index 57c8ef62e1..1637f9cb71 100644 --- a/erts/emulator/nifs/unix/unix_prim_file.c +++ b/erts/emulator/nifs/unix/unix_prim_file.c @@ -125,24 +125,11 @@ static int open_file_type_check(const efile_path_t *path, int fd) { * immediately in a read within the call, but the new implementation * never does that. */ return 1; - } else { - /* The old driver tolerated opening /dev/null despite the "no devices" - * limitation. It provided no explanation for this but we still need - * to match the behavior. We're checking through stat(2) instead of - * comparing the name to account for links. */ - struct stat null_device_info; - int is_dev_null; - - is_dev_null = (stat("/dev/null", &null_device_info) == 0); - is_dev_null &= (file_info.st_ino == null_device_info.st_ino); - is_dev_null &= (file_info.st_dev == null_device_info.st_dev); - - if(is_dev_null) { - return 1; - } } - if(!S_ISREG(file_info.st_mode)) { + /* Allow everything that isn't a directory, and error out on the next call + * if it's unsupported. */ + if(S_ISDIR(file_info.st_mode)) { return 0; } @@ -700,7 +687,7 @@ posix_errno_t efile_set_permissions(const efile_path_t *path, Uint32 permissions return 0; } -posix_errno_t efile_set_owner(const efile_path_t *path, Uint32 owner, Uint32 group) { +posix_errno_t efile_set_owner(const efile_path_t *path, Sint32 owner, Sint32 group) { if(chown((const char*)path->data, owner, group) < 0) { return errno; } diff --git a/erts/emulator/nifs/win32/win_prim_file.c b/erts/emulator/nifs/win32/win_prim_file.c index 9b79182f2c..8058350b25 100644 --- a/erts/emulator/nifs/win32/win_prim_file.c +++ b/erts/emulator/nifs/win32/win_prim_file.c @@ -801,7 +801,7 @@ posix_errno_t efile_set_permissions(const efile_path_t *path, Uint32 permissions return windows_to_posix_errno(GetLastError()); } -posix_errno_t efile_set_owner(const efile_path_t *path, Uint32 owner, Uint32 group) { +posix_errno_t efile_set_owner(const efile_path_t *path, Sint32 owner, Sint32 group) { (void)path; (void)owner; (void)group; diff --git a/erts/emulator/sys/common/erl_mmap.c b/erts/emulator/sys/common/erl_mmap.c index a9c6e72c5f..5dadd8a5a6 100644 --- a/erts/emulator/sys/common/erl_mmap.c +++ b/erts/emulator/sys/common/erl_mmap.c @@ -296,11 +296,10 @@ typedef struct { }ErtsFreeSegMap; struct ErtsMemMapper_ { - int (*reserve_physical)(char *, UWord, int exec); + int (*reserve_physical)(char *, UWord); void (*unreserve_physical)(char *, UWord); int supercarrier; int no_os_mmap; - int executable; /* is client a native code allocator? */ /* * Super unaligned area is located above super aligned * area. That is, `sa.bot` is beginning of the super @@ -358,10 +357,6 @@ char* erts_literals_start; UWord erts_literals_size; #endif -#ifdef ERTS_HAVE_EXEC_MMAPPER -ErtsMemMapper erts_exec_mmapper; -#endif - #define ERTS_MMAP_SIZE_SC_SA_INC(SZ) \ do { \ @@ -1240,7 +1235,6 @@ Eterm build_free_seg_list(Process* p, ErtsFreeSegMap* map) #if HAVE_MMAP # define ERTS_MMAP_PROT (PROT_READ|PROT_WRITE) -# define ERTS_MMAP_PROT_EXEC (PROT_READ|PROT_WRITE|PROT_EXEC) # if defined(MAP_ANONYMOUS) # define ERTS_MMAP_FLAGS (MAP_ANON|MAP_PRIVATE) # define ERTS_MMAP_FD (-1) @@ -1254,26 +1248,24 @@ Eterm build_free_seg_list(Process* p, ErtsFreeSegMap* map) #endif static ERTS_INLINE void * -os_mmap(void *hint_ptr, UWord size, int try_superalign, int executable) +os_mmap(void *hint_ptr, UWord size, int try_superalign) { #if HAVE_MMAP - const int prot = executable ? ERTS_MMAP_PROT_EXEC : ERTS_MMAP_PROT; void *res; #ifdef MAP_ALIGN if (try_superalign) - res = mmap((void *) ERTS_SUPERALIGNED_SIZE, size, prot, + res = mmap((void *) ERTS_SUPERALIGNED_SIZE, size, ERTS_MMAP_PROT, ERTS_MMAP_FLAGS|MAP_ALIGN, ERTS_MMAP_FD, 0); else #endif - res = mmap((void *) hint_ptr, size, prot, + res = mmap((void *) hint_ptr, size, ERTS_MMAP_PROT, ERTS_MMAP_FLAGS, ERTS_MMAP_FD, 0); if (res == MAP_FAILED) return NULL; return res; #elif HAVE_VIRTUALALLOC - const DWORD prot = executable ? PAGE_EXECUTE_READWRITE : PAGE_READWRITE; return (void *) VirtualAlloc(NULL, (SIZE_T) size, - MEM_COMMIT|MEM_RESERVE, prot); + MEM_COMMIT|MEM_RESERVE, PAGE_READWRITE); #else # error "missing mmap() or similar" #endif @@ -1330,7 +1322,6 @@ os_mremap(void *ptr, UWord old_size, UWord new_size, int try_superalign) #if HAVE_MMAP #define ERTS_MMAP_RESERVE_PROT (ERTS_MMAP_PROT) -#define ERTS_MMAP_RESERVE_PROT_EXEC (ERTS_MMAP_PROT_EXEC) #define ERTS_MMAP_RESERVE_FLAGS (ERTS_MMAP_FLAGS|MAP_FIXED) #define ERTS_MMAP_UNRESERVE_PROT (PROT_NONE) #if defined(__FreeBSD__) @@ -1346,10 +1337,9 @@ os_mremap(void *ptr, UWord old_size, UWord new_size, int try_superalign) #endif /* __FreeBSD__ */ static int -os_reserve_physical(char *ptr, UWord size, int exec) +os_reserve_physical(char *ptr, UWord size) { - const int prot = exec ? ERTS_MMAP_RESERVE_PROT_EXEC : ERTS_MMAP_RESERVE_PROT; - void *res = mmap((void *) ptr, (size_t) size, prot, + void *res = mmap((void *) ptr, (size_t) size, ERTS_MMAP_RESERVE_PROT, ERTS_MMAP_RESERVE_FLAGS, ERTS_MMAP_FD, 0); if (res == (void *) MAP_FAILED) return 0; @@ -1366,37 +1356,11 @@ os_unreserve_physical(char *ptr, UWord size) } static void * -os_mmap_virtual(char *ptr, UWord size, int exec) +os_mmap_virtual(char *ptr, UWord size) { int flags = ERTS_MMAP_VIRTUAL_FLAGS; void* res; -#ifdef ERTS_ALC_A_EXEC - if (exec) { - ASSERT(!ptr); - /* OTP-19.0: Nice hack below cut-and-pasted from hipe_amd64.c */ - -# ifdef MAP_32BIT - /* If we got MAP_32BIT (Linux), then use that to ask for low memory */ - flags |= MAP_32BIT; -# else - /* FreeBSD doesn't have MAP_32BIT, and it doesn't respect - a plain map_hint (returns high mappings even though the - hint refers to a free area), so we have to use both map_hint - and MAP_FIXED to get addresses below the 2GB boundary. - This is even worse than the Linux/ppc64 case. - Similarly, Solaris 10 doesn't have MAP_32BIT, - and it doesn't respect a plain map_hint. */ - ptr = (char*)(512*1024*1024); /* 0.5GB */ - -# if defined(__FreeBSD__) || defined(__sun__) - flags |= MAP_FIXED; -# endif -# endif /* !MAP_32BIT */ - } -#else /* !ERTS_ALC_A_EXEC */ - ASSERT(!exec); -#endif res = mmap((void *) ptr, (size_t) size, ERTS_MMAP_VIRTUAL_PROT, flags, ERTS_MMAP_FD, 0); if (res == (void *) MAP_FAILED) @@ -1411,7 +1375,7 @@ os_mmap_virtual(char *ptr, UWord size, int exec) #endif /* ERTS_HAVE_OS_MMAP */ -static int reserve_noop(char *ptr, UWord size, int exec) +static int reserve_noop(char *ptr, UWord size) { #ifdef ERTS_MMAP_DEBUG_FILL_AREAS Uint32 *uip, *end = (Uint32 *) (ptr + size); @@ -1454,7 +1418,7 @@ alloc_desc_insert_free_seg(ErtsMemMapper* mm, #if ERTS_HAVE_OS_MMAP if (!mm->no_os_mmap) { - ptr = os_mmap(mm->desc.new_area_hint, ERTS_PAGEALIGNED_SIZE, 0, 0); + ptr = os_mmap(mm->desc.new_area_hint, ERTS_PAGEALIGNED_SIZE, 0); if (ptr) { mm->desc.new_area_hint = ptr+ERTS_PAGEALIGNED_SIZE; ERTS_MMAP_SIZE_OS_INC(ERTS_PAGEALIGNED_SIZE); @@ -1473,7 +1437,7 @@ alloc_desc_insert_free_seg(ErtsMemMapper* mm, da_map = &mm->sua.map; desc = lookup_free_seg(da_map, ERTS_PAGEALIGNED_SIZE); if (desc) { - if (mm->reserve_physical(desc->start, ERTS_PAGEALIGNED_SIZE, 0)) + if (mm->reserve_physical(desc->start, ERTS_PAGEALIGNED_SIZE)) ERTS_MMAP_SIZE_SC_SUA_INC(ERTS_PAGEALIGNED_SIZE); else desc = NULL; @@ -1483,7 +1447,7 @@ alloc_desc_insert_free_seg(ErtsMemMapper* mm, da_map = &mm->sa.map; desc = lookup_free_seg(da_map, ERTS_PAGEALIGNED_SIZE); if (desc) { - if (mm->reserve_physical(desc->start, ERTS_PAGEALIGNED_SIZE, 0)) + if (mm->reserve_physical(desc->start, ERTS_PAGEALIGNED_SIZE)) ERTS_MMAP_SIZE_SC_SA_INC(ERTS_PAGEALIGNED_SIZE); else desc = NULL; @@ -1544,7 +1508,7 @@ erts_mmap(ErtsMemMapper* mm, Uint32 flags, UWord *sizep) if (desc) { seg = desc->start; end = seg+asize; - if (!mm->reserve_physical(seg, asize, mm->executable)) + if (!mm->reserve_physical(seg, asize)) goto supercarrier_reserve_failure; if (desc->end == end) { delete_free_seg(&mm->sua.map, desc); @@ -1559,8 +1523,7 @@ erts_mmap(ErtsMemMapper* mm, Uint32 flags, UWord *sizep) } if (asize <= mm->sua.bot - mm->sa.top) { - if (!mm->reserve_physical(mm->sua.bot - asize, asize, - mm->executable)) + if (!mm->reserve_physical(mm->sua.bot - asize, asize)) goto supercarrier_reserve_failure; mm->sua.bot -= asize; seg = mm->sua.bot; @@ -1576,8 +1539,7 @@ erts_mmap(ErtsMemMapper* mm, Uint32 flags, UWord *sizep) char *start = seg = desc->start; seg = (char *) ERTS_SUPERALIGNED_CEILING(seg); end = seg+asize; - if (!mm->reserve_physical(start, (UWord) (end - start), - mm->executable)) + if (!mm->reserve_physical(start, (UWord) (end - start))) goto supercarrier_reserve_failure; ERTS_MMAP_SIZE_SC_SA_INC(asize); if (desc->end == end) { @@ -1609,8 +1571,7 @@ erts_mmap(ErtsMemMapper* mm, Uint32 flags, UWord *sizep) if (asize + (seg - start) <= mm->sua.bot - start) { end = seg + asize; - if (!mm->reserve_physical(start, (UWord) (end - start), - mm->executable)) + if (!mm->reserve_physical(start, (UWord) (end - start))) goto supercarrier_reserve_failure; mm->sa.top = end; ERTS_MMAP_SIZE_SC_SA_INC(asize); @@ -1632,8 +1593,7 @@ erts_mmap(ErtsMemMapper* mm, Uint32 flags, UWord *sizep) seg = (char *) ERTS_SUPERALIGNED_CEILING(org_start); end = seg + asize; - if (!mm->reserve_physical(seg, (UWord) (org_end - seg), - mm->executable)) + if (!mm->reserve_physical(seg, (UWord) (org_end - seg))) goto supercarrier_reserve_failure; ERTS_MMAP_SIZE_SC_SUA_INC(asize); if (org_start != seg) { @@ -1666,13 +1626,13 @@ erts_mmap(ErtsMemMapper* mm, Uint32 flags, UWord *sizep) /* Map using OS primitives */ if (!(ERTS_MMAPFLG_SUPERCARRIER_ONLY & flags) && !mm->no_os_mmap) { if (!(ERTS_MMAPFLG_SUPERALIGNED & flags)) { - seg = os_mmap(NULL, asize, 0, mm->executable); + seg = os_mmap(NULL, asize, 0); if (!seg) goto failure; } else { asize = ERTS_SUPERALIGNED_CEILING(*sizep); - seg = os_mmap(NULL, asize, 1, mm->executable); + seg = os_mmap(NULL, asize, 1); if (!seg) goto failure; @@ -1682,8 +1642,7 @@ erts_mmap(ErtsMemMapper* mm, Uint32 flags, UWord *sizep) os_munmap(seg, asize); - ptr = os_mmap(NULL, asize + ERTS_SUPERALIGNED_SIZE, 1, - mm->executable); + ptr = os_mmap(NULL, asize + ERTS_SUPERALIGNED_SIZE, 1); if (!ptr) goto failure; @@ -2018,8 +1977,7 @@ erts_mremap(ErtsMemMapper* mm, if (next && new_end <= next->end) { if (!mm->reserve_physical(((char *) ptr) + old_size, - asize - old_size, - mm->executable)) + asize - old_size)) goto supercarrier_reserve_failure; if (new_end < next->end) resize_free_seg(&mm->sua.map, next, new_end, next->end); @@ -2037,8 +1995,7 @@ erts_mremap(ErtsMemMapper* mm, if (end == mm->sa.top) { if (new_end <= mm->sua.bot) { if (!mm->reserve_physical(((char *) ptr) + old_size, - asize - old_size, - mm->executable)) + asize - old_size)) goto supercarrier_reserve_failure; mm->sa.top = new_end; new_ptr = ptr; @@ -2050,8 +2007,7 @@ erts_mremap(ErtsMemMapper* mm, adjacent_free_seg(&mm->sa.map, start, end, &prev, &next); if (next && new_end <= next->end) { if (!mm->reserve_physical(((char *) ptr) + old_size, - asize - old_size, - mm->executable)) + asize - old_size)) goto supercarrier_reserve_failure; if (new_end < next->end) resize_free_seg(&mm->sa.map, next, new_end, next->end); @@ -2171,7 +2127,7 @@ static void hard_dbg_mseg_init(void); #endif void -erts_mmap_init(ErtsMemMapper* mm, ErtsMMapInit *init, int executable) +erts_mmap_init(ErtsMemMapper* mm, ErtsMMapInit *init) { static int is_first_call = 1; int virtual_map = 0; @@ -2203,7 +2159,6 @@ erts_mmap_init(ErtsMemMapper* mm, ErtsMMapInit *init, int executable) mm->supercarrier = 0; mm->reserve_physical = reserve_noop; mm->unreserve_physical = unreserve_noop; - mm->executable = executable; #if HAVE_MMAP && !defined(MAP_ANON) mm->mmap_fd = open("/dev/zero", O_RDWR); @@ -2225,7 +2180,7 @@ erts_mmap_init(ErtsMemMapper* mm, ErtsMMapInit *init, int executable) ptr = (char *) ERTS_PAGEALIGNED_CEILING(init->virtual_range.start); end = (char *) ERTS_PAGEALIGNED_FLOOR(init->virtual_range.end); sz = end - ptr; - start = os_mmap_virtual(ptr, sz, executable); + start = os_mmap_virtual(ptr, sz); if (!start || start > ptr || start >= end) erts_exit(1, "erts_mmap: Failed to create virtual range for super carrier\n"); @@ -2250,7 +2205,7 @@ erts_mmap_init(ErtsMemMapper* mm, ErtsMMapInit *init, int executable) sz = ERTS_PAGEALIGNED_CEILING(init->scs); #ifdef ERTS_HAVE_OS_PHYSICAL_MEMORY_RESERVATION if (!init->scrpm) { - start = os_mmap_virtual(NULL, sz, executable); + start = os_mmap_virtual(NULL, sz); mm->reserve_physical = os_reserve_physical; mm->unreserve_physical = os_unreserve_physical; virtual_map = 1; @@ -2262,7 +2217,7 @@ erts_mmap_init(ErtsMemMapper* mm, ErtsMMapInit *init, int executable) * The whole supercarrier will by physically * reserved all the time. */ - start = os_mmap(NULL, sz, 1, executable); + start = os_mmap(NULL, sz, 1); } if (!start) erts_exit(1, @@ -2336,7 +2291,7 @@ erts_mmap_init(ErtsMemMapper* mm, ErtsMMapInit *init, int executable) mm->sua.top -= ERTS_PAGEALIGNED_SIZE; mm->size.supercarrier.used.total += ERTS_PAGEALIGNED_SIZE; #ifdef ERTS_HAVE_OS_PHYSICAL_MEMORY_RESERVATION - if (!virtual_map || os_reserve_physical(mm->sua.top, ERTS_PAGEALIGNED_SIZE, 0)) + if (!virtual_map || os_reserve_physical(mm->sua.top, ERTS_PAGEALIGNED_SIZE)) #endif add_free_desc_area(mm, mm->sua.top, end); mm->desc.reserved += (end - mm->sua.top) / sizeof(ErtsFreeSegDesc); @@ -2349,7 +2304,7 @@ erts_mmap_init(ErtsMemMapper* mm, ErtsMMapInit *init, int executable) * will be used for free segment descritors. */ #ifdef ERTS_HAVE_OS_PHYSICAL_MEMORY_RESERVATION - if (virtual_map && !os_reserve_physical(start, mm->sa.bot - start, 0)) + if (virtual_map && !os_reserve_physical(start, mm->sa.bot - start)) erts_exit(1, "erts_mmap: Failed to reserve physical memory for descriptors\n"); #endif mm->desc.unused_start = start; diff --git a/erts/emulator/sys/common/erl_mmap.h b/erts/emulator/sys/common/erl_mmap.h index 2a07d93c8c..c1f9668b18 100644 --- a/erts/emulator/sys/common/erl_mmap.h +++ b/erts/emulator/sys/common/erl_mmap.h @@ -93,11 +93,6 @@ typedef struct { #define ERTS_MMAP_INIT_LITERAL_INITER \ {{NULL, NULL}, {NULL, NULL}, ERTS_LITERAL_VIRTUAL_AREA_SIZE, 1, (1 << 10), 0} -#define ERTS_HIPE_EXEC_VIRTUAL_AREA_SIZE (UWORD_CONSTANT(512)*1024*1024) - -#define ERTS_MMAP_INIT_HIPE_EXEC_INITER \ - {{NULL, NULL}, {NULL, NULL}, ERTS_HIPE_EXEC_VIRTUAL_AREA_SIZE, 1, (1 << 10), 0} - #define ERTS_SUPERALIGNED_SIZE \ (1 << ERTS_MMAP_SUPERALIGNED_BITS) @@ -140,7 +135,7 @@ void *erts_mmap(ErtsMemMapper*, Uint32 flags, UWord *sizep); void erts_munmap(ErtsMemMapper*, Uint32 flags, void *ptr, UWord size); void *erts_mremap(ErtsMemMapper*, Uint32 flags, void *ptr, UWord old_size, UWord *sizep); int erts_mmap_in_supercarrier(ErtsMemMapper*, void *ptr); -void erts_mmap_init(ErtsMemMapper*, ErtsMMapInit*, int executable); +void erts_mmap_init(ErtsMemMapper*, ErtsMMapInit*); struct erts_mmap_info_struct { UWord sizes[6]; @@ -165,15 +160,6 @@ extern ErtsMemMapper erts_dflt_mmapper; extern ErtsMemMapper erts_literal_mmapper; # endif -# if defined(ERTS_ALC_A_EXEC) && defined(__x86_64__) - /* - * On x86_64, exec_alloc employs its own super carrier 'erts_exec_mmaper' - * to ensure low memory for HiPE AMD64 small code model. - */ -# define ERTS_HAVE_EXEC_MMAPPER -extern ErtsMemMapper erts_exec_mmapper; -# endif - # endif /* ERTS_HAVE_OS_PHYSICAL_MEMORY_RESERVATION */ #endif /* ERTS_WANT_MEM_MAPPERS */ diff --git a/erts/emulator/sys/common/erl_mseg.c b/erts/emulator/sys/common/erl_mseg.c index bf6de9b13a..ced3d61525 100644 --- a/erts/emulator/sys/common/erl_mseg.c +++ b/erts/emulator/sys/common/erl_mseg.c @@ -1406,15 +1406,9 @@ erts_mseg_init(ErtsMsegInit_t *init) erts_mtx_init(&init_atoms_mutex, "mseg_init_atoms", NIL, ERTS_LOCK_FLAGS_PROPERTY_STATIC | ERTS_LOCK_FLAGS_CATEGORY_GENERIC); -#ifdef ERTS_HAVE_EXEC_MMAPPER - /* Initialize erts_exec_mapper *FIRST*, to increase probability - * of getting low memory for HiPE AMD64's small code model. - */ - erts_mmap_init(&erts_exec_mmapper, &init->exec_mmap, 1); -#endif - erts_mmap_init(&erts_dflt_mmapper, &init->dflt_mmap, 0); + erts_mmap_init(&erts_dflt_mmapper, &init->dflt_mmap); #if defined(ARCH_64) && defined(ERTS_HAVE_OS_PHYSICAL_MEMORY_RESERVATION) - erts_mmap_init(&erts_literal_mmapper, &init->literal_mmap, 0); + erts_mmap_init(&erts_literal_mmapper, &init->literal_mmap); #endif if (!IS_2POW(GET_PAGE_SIZE)) diff --git a/erts/emulator/sys/common/erl_mseg.h b/erts/emulator/sys/common/erl_mseg.h index bba0dec499..af275c18be 100644 --- a/erts/emulator/sys/common/erl_mseg.h +++ b/erts/emulator/sys/common/erl_mseg.h @@ -61,7 +61,6 @@ typedef struct { Uint nos; ErtsMMapInit dflt_mmap; ErtsMMapInit literal_mmap; - ErtsMMapInit exec_mmap; } ErtsMsegInit_t; #define ERTS_MSEG_INIT_DEFAULT_INITIALIZER \ @@ -72,7 +71,6 @@ typedef struct { 1000, /* cci: Cache check interval */ \ ERTS_MMAP_INIT_DEFAULT_INITER, \ ERTS_MMAP_INIT_LITERAL_INITER, \ - ERTS_MMAP_INIT_HIPE_EXEC_INITER \ } typedef struct { diff --git a/erts/emulator/sys/unix/erl_child_setup.c b/erts/emulator/sys/unix/erl_child_setup.c index 69fc6c2879..57973b10d7 100644 --- a/erts/emulator/sys/unix/erl_child_setup.c +++ b/erts/emulator/sys/unix/erl_child_setup.c @@ -437,6 +437,21 @@ main(int argc, char *argv[]) exit(1); } + /* Ignore SIGTERM. + Some container environments send SIGTERM to all processes + when terminating. We don't want erl_child_setup to terminate + in these cases as that will prevent beam from properly + cleaning up. + */ + sa.sa_handler = SIG_IGN; + sigemptyset(&sa.sa_mask); + sa.sa_flags = 0; + + if (sigaction(SIGTERM, &sa, 0) == -1) { + perror(NULL); + exit(1); + } + forker_hash_init(); SET_CLOEXEC(uds_fd); diff --git a/erts/emulator/test/alloc_SUITE.erl b/erts/emulator/test/alloc_SUITE.erl index 5e5cd6f578..88ff2a7a92 100644 --- a/erts/emulator/test/alloc_SUITE.erl +++ b/erts/emulator/test/alloc_SUITE.erl @@ -31,6 +31,7 @@ mseg_clear_cache/1, erts_mmap/1, cpool/1, + set_dyn_param/1, migration/1]). -include_lib("common_test/include/ct.hrl"). @@ -41,6 +42,7 @@ suite() -> all() -> [basic, coalesce, threads, realloc_copy, bucket_index, + set_dyn_param, bucket_mask, rbtree, mseg_clear_cache, erts_mmap, cpool, migration]. init_per_testcase(Case, Config) when is_list(Config) -> @@ -65,7 +67,11 @@ mseg_clear_cache(Cfg) -> drv_case(Cfg). cpool(Cfg) -> drv_case(Cfg). migration(Cfg) -> - drv_case(Cfg, concurrent, "+MZe true"). + %% Enable test_alloc. + %% Disable driver_alloc to avoid recursive alloc_util calls + %% through enif_mutex_create() in my_creating_mbc(). + drv_case(Cfg, concurrent, "+MZe true +MRe false"), + drv_case(Cfg, concurrent, "+MZe true +MRe false +MZas ageffcbf"). erts_mmap(Config) when is_list(Config) -> case {os:type(), mmsc_flags()} of @@ -110,7 +116,7 @@ erts_mmap_do(Config, SCO, SCRPM, SCRFSD) -> 0 -> O1; _ -> O1 ++ " +MMscrfsd"++integer_to_list(SCRFSD) end, - {ok, Node} = start_node(Config, Opts), + {ok, Node} = start_node(Config, Opts, []), Self = self(), Ref = make_ref(), F = fun() -> @@ -140,6 +146,82 @@ erts_mmap_do(Config, SCO, SCRPM, SCRFSD) -> Result. +%% Test erlang:system_flag(erts_alloc, ...) +set_dyn_param(_Config) -> + {_, _, _, AlcList} = erlang:system_info(allocator), + + {Enabled, Disabled, Others} = + lists:foldl(fun({sys_alloc,_}, {Es, Ds, Os}) -> + {Es, [sys_alloc | Ds], Os}; + + ({AT, Opts}, {Es, Ds, Os}) when is_list(Opts) -> + case lists:keyfind(e, 1, Opts) of + {e, true} -> + {[AT | Es], Ds, Os}; + {e, false} -> + {Es, [AT | Ds], Os}; + false -> + {Es, Ds, [AT | Os]} + end; + + (_, Acc) -> Acc + end, + {[], [], []}, + AlcList), + + Param = sbct, + lists:foreach(fun(AT) -> set_dyn_param_enabled(AT, Param) end, + Enabled), + + lists:foreach(fun(AT) -> + Tpl = {AT, Param, 12345}, + io:format("~p\n", [Tpl]), + notsup = erlang:system_flag(erts_alloc, Tpl) + end, + Disabled), + + lists:foreach(fun(AT) -> + Tpl = {AT, Param, 12345}, + io:format("~p\n", [Tpl]), + {'EXIT',{badarg,_}} = + (catch erlang:system_flag(erts_alloc, Tpl)) + end, + Others), + ok. + +set_dyn_param_enabled(AT, Param) -> + OldVal = get_alc_param(AT, Param), + + Val1 = OldVal div 2, + Tuple = {AT, Param, Val1}, + io:format("~p\n", [Tuple]), + ok = erlang:system_flag(erts_alloc, Tuple), + Val1 = get_alc_param(AT, Param), + + ok = erlang:system_flag(erts_alloc, {AT, Param, OldVal}), + OldVal = get_alc_param(AT, Param), + ok. + +get_alc_param(AT, Param) -> + lists:foldl(fun({instance,_,Istats}, Acc) -> + {options,Opts} = lists:keyfind(options, 1, Istats), + {Param,Val} = lists:keyfind(Param, 1, Opts), + {as,Strategy} = lists:keyfind(as, 1, Opts), + + case {param_for_strat(Param, Strategy), Acc} of + {false, _} -> Acc; + {true, undefined} -> Val; + {true, _} -> + Val = Acc + end + end, + undefined, + erlang:system_info({allocator, AT})). + +param_for_strat(sbct, gf) -> false; +param_for_strat(_, _) -> true. + + %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% %% %% %% Internal functions %% @@ -151,7 +233,9 @@ drv_case(Config) -> drv_case(Config, Mode, NodeOpts) when is_list(Config) -> case os:type() of {Family, _} when Family == unix; Family == win32 -> - {ok, Node} = start_node(Config, NodeOpts), + %%Prog = {prog,"/my/own/otp/bin/cerl -debug"}, + Prog = [], + {ok, Node} = start_node(Config, NodeOpts, Prog), Self = self(), Ref = make_ref(), spawn_link(Node, @@ -217,19 +301,35 @@ wait_for_memory_deallocations() -> end. print_stats(migration) -> - {Btot,Ctot} = lists:foldl(fun({instance,Inr,Istats}, {Bacc,Cacc}) -> - {mbcs,MBCS} = lists:keyfind(mbcs, 1, Istats), - Btup = lists:keyfind(blocks, 1, MBCS), - Ctup = lists:keyfind(carriers, 1, MBCS), - io:format("{instance,~p,~p,~p}\n", [Inr, Btup, Ctup]), - {tuple_add(Bacc,Btup),tuple_add(Cacc,Ctup)}; - (_, Acc) -> Acc - end, - {{blocks,0,0,0},{carriers,0,0,0}}, - erlang:system_info({allocator,test_alloc})), - + IFun = fun({instance,Inr,Istats}, {Bacc,Cacc,Pacc}) -> + {mbcs,MBCS} = lists:keyfind(mbcs, 1, Istats), + Btup = lists:keyfind(blocks, 1, MBCS), + Ctup = lists:keyfind(carriers, 1, MBCS), + + Ptup = case lists:keyfind(mbcs_pool, 1, Istats) of + {mbcs_pool,POOL} -> + {blocks, Bpool} = lists:keyfind(blocks, 1, POOL), + {carriers, Cpool} = lists:keyfind(carriers, 1, POOL), + {pool, Bpool, Cpool}; + false -> + {pool, 0, 0} + end, + io:format("{instance,~p,~p,~p,~p}}\n", + [Inr, Btup, Ctup, Ptup]), + {tuple_add(Bacc,Btup),tuple_add(Cacc,Ctup), + tuple_add(Pacc,Ptup)}; + (_, Acc) -> Acc + end, + + {Btot,Ctot,Ptot} = lists:foldl(IFun, + {{blocks,0,0,0},{carriers,0,0,0},{pool,0,0}}, + erlang:system_info({allocator,test_alloc})), + + {pool, PBtot, PCtot} = Ptot, io:format("Number of blocks : ~p\n", [Btot]), - io:format("Number of carriers: ~p\n", [Ctot]); + io:format("Number of carriers: ~p\n", [Ctot]), + io:format("Number of pooled blocks : ~p\n", [PBtot]), + io:format("Number of pooled carriers: ~p\n", [PCtot]); print_stats(_) -> ok. tuple_add(T1, T2) -> @@ -326,13 +426,13 @@ handle_result(_State, Result0) -> continue end. -start_node(Config, Opts) when is_list(Config), is_list(Opts) -> +start_node(Config, Opts, Prog) when is_list(Config), is_list(Opts) -> case proplists:get_value(debug,Config) of true -> {ok, node()}; - _ -> start_node_1(Config, Opts) + _ -> start_node_1(Config, Opts, Prog) end. -start_node_1(Config, Opts) -> +start_node_1(Config, Opts, Prog) -> Pa = filename:dirname(code:which(?MODULE)), Name = list_to_atom(atom_to_list(?MODULE) ++ "-" @@ -341,7 +441,11 @@ start_node_1(Config, Opts) -> ++ integer_to_list(erlang:system_time(second)) ++ "-" ++ integer_to_list(erlang:unique_integer([positive]))), - test_server:start_node(Name, slave, [{args, Opts++" -pa "++Pa}]). + ErlArg = case Prog of + [] -> []; + _ -> [{erl,[Prog]}] + end, + test_server:start_node(Name, slave, [{args, Opts++" -pa "++Pa} | ErlArg]). stop_node(Node) when Node =:= node() -> ok; stop_node(Node) -> diff --git a/erts/emulator/test/alloc_SUITE_data/allocator_test.h b/erts/emulator/test/alloc_SUITE_data/allocator_test.h index 97ee58cdad..5272f86c98 100644 --- a/erts/emulator/test/alloc_SUITE_data/allocator_test.h +++ b/erts/emulator/test/alloc_SUITE_data/allocator_test.h @@ -156,7 +156,8 @@ typedef void* erts_cond; #define IS_SMP_ENABLED ((int) ALC_TEST0(0xf13)) #define ALLOC_TEST(S) ((void*) ALC_TEST1(0xf14, (S))) #define FREE_TEST(P) ((void) ALC_TEST1(0xf15, (P))) -#define SET_TEST_MBC_USER_HEADER(SZ,CMBC,DMBC) ((int)ALC_TEST3(0xf16, (SZ), (CMBC), (DMBC))) -#define GET_TEST_MBC_SIZE() ((int) ALC_TEST0(0xf17)) +#define REALLOC_TEST(P,S) ((void*) ALC_TEST2(0xf16, (P), (S))) +#define SET_TEST_MBC_USER_HEADER(SZ,CMBC,DMBC) ((int)ALC_TEST3(0xf17, (SZ), (CMBC), (DMBC))) +#define GET_TEST_MBC_SIZE() ((int) ALC_TEST0(0xf18)) #endif diff --git a/erts/emulator/test/alloc_SUITE_data/migration.c b/erts/emulator/test/alloc_SUITE_data/migration.c index b9a4de03b3..1d974225fc 100644 --- a/erts/emulator/test/alloc_SUITE_data/migration.c +++ b/erts/emulator/test/alloc_SUITE_data/migration.c @@ -223,6 +223,42 @@ static int rand_int(MigrationState* state, int low, int high) return low + (x % (high+1-low)); } +enum Operation +{ + ALLOCATE_OP, + FREE_OP, + REALLOC_OP, + CLEANUP_OP +}; + +static enum Operation rand_op(MigrationState* state) +{ + int r = rand_int(state, 1, 100); + switch (state->phase) { + case GROWING: + FATAL_ASSERT(state->nblocks < state->max_nblocks); + if (r > 10 || state->nblocks == 0) + return ALLOCATE_OP; + else if (r > 5) + return FREE_OP; + else + return REALLOC_OP; + + case SHRINKING: + FATAL_ASSERT(state->nblocks > 0); + if (r > 10 || state->nblocks == state->max_nblocks) + return FREE_OP; + else if (r > 5) + return ALLOCATE_OP; + else + return REALLOC_OP; + + case CLEANUP: + return CLEANUP_OP; + default: + FATAL_ASSERT(!"Invalid op phase"); + } +} static void do_cleanup(TestCaseState_t *tcs, MigrationState* state) { @@ -275,53 +311,75 @@ testcase_run(TestCaseState_t *tcs) state->goal_nblocks = rand_int(state, 1, state->max_nblocks); } - switch (state->phase) { - case GROWING: { + switch (rand_op(state)) { + case ALLOCATE_OP: { MyBlock* p; FATAL_ASSERT(!state->blockv[state->nblocks]); - p = ALLOC_TEST(rand_int(state, state->block_size/2, state->block_size)); + p = ALLOC_TEST(rand_int(state, state->block_size/2, state->block_size)); FATAL_ASSERT(p); add_block(p, state); - state->blockv[state->nblocks] = p; - if (++state->nblocks >= state->goal_nblocks) { - /*testcase_printf(tcs, "%d: Grown to %d blocks", tcs->thr_nr, state->nblocks);*/ - state->phase = SHRINKING; - state->goal_nblocks = rand_int(state, 0, state->goal_nblocks-1); - } - else - FATAL_ASSERT(!state->blockv[state->nblocks]); + state->blockv[state->nblocks++] = p; break; } - case SHRINKING: { + case FREE_OP: { int ix = rand_int(state, 0, state->nblocks-1); FATAL_ASSERT(state->blockv[ix]); remove_block(state->blockv[ix]); FREE_TEST(state->blockv[ix]); state->blockv[ix] = state->blockv[--state->nblocks]; state->blockv[state->nblocks] = NULL; - - if (state->nblocks <= state->goal_nblocks) { - /*testcase_printf(tcs, "%d: Shrunk to %d blocks", tcs->thr_nr, state->nblocks);*/ - if (++state->round >= MAX_ROUNDS) { - state->phase = CLEANUP; - } else { - state->phase = GROWING; - state->goal_nblocks = rand_int(state, state->goal_nblocks+1, state->max_nblocks); - } - } break; } + case REALLOC_OP: { + int ix = rand_int(state, 0, state->nblocks-1); + MyBlock* p; + FATAL_ASSERT(state->blockv[ix]); + remove_block(state->blockv[ix]); + p = REALLOC_TEST(state->blockv[ix], rand_int(state, state->block_size/2, state->block_size)); + FATAL_ASSERT(p); + add_block(p, state); + state->blockv[ix] = p; + break; + } + case CLEANUP_OP: + do_cleanup(tcs, state); + break; + default: + FATAL_ASSERT(!"Invalid operation"); + } + + switch (state->phase) { + case GROWING: { + if (state->nblocks >= state->goal_nblocks) { + /*testcase_printf(tcs, "%d: Grown to %d blocks", tcs->thr_nr, state->nblocks);*/ + state->phase = SHRINKING; + state->goal_nblocks = rand_int(state, 0, state->goal_nblocks-1); + } + else + FATAL_ASSERT(!state->blockv[state->nblocks]); + break; + } + case SHRINKING: { + if (state->nblocks <= state->goal_nblocks) { + /*testcase_printf(tcs, "%d: Shrunk to %d blocks", tcs->thr_nr, state->nblocks);*/ + if (++state->round >= MAX_ROUNDS) { + state->phase = CLEANUP; + } else { + state->phase = GROWING; + state->goal_nblocks = rand_int(state, state->goal_nblocks+1, state->max_nblocks); + } + } + break; + } case CLEANUP: - do_cleanup(tcs, state); - break; + case DONE: + break; default: FATAL_ASSERT(!"Invalid phase"); } - if (state->phase == DONE) { - } - else { + if (state->phase != DONE) { testcase_continue(tcs); } } diff --git a/erts/emulator/test/binary_SUITE.erl b/erts/emulator/test/binary_SUITE.erl index 4bc1838139..a3c3daac15 100644 --- a/erts/emulator/test/binary_SUITE.erl +++ b/erts/emulator/test/binary_SUITE.erl @@ -262,6 +262,7 @@ test_deep_bitstr(List) -> {Bin,bitstring_to_list(Bin)}. bad_list_to_binary(Config) when is_list(Config) -> + test_bad_bin(<<1:1>>), test_bad_bin(atom), test_bad_bin(42), test_bad_bin([1|2]), diff --git a/erts/emulator/test/efile_SUITE.erl b/erts/emulator/test/efile_SUITE.erl index 16d581a567..821381bf0d 100644 --- a/erts/emulator/test/efile_SUITE.erl +++ b/erts/emulator/test/efile_SUITE.erl @@ -46,14 +46,14 @@ iter_max_files_1(Config) -> DataDir = proplists:get_value(data_dir,Config), TestFile = filename:join(DataDir, "existing_file"), N = 10, - %% Run on a different node in order to set the max ports + %% Run on a different node in order to make the test more stable. Dir = filename:dirname(code:which(?MODULE)), {ok,Node} = test_server:start_node(test_iter_max_files,slave, - [{args,"+Q 1524 -pa " ++ Dir}]), + [{args,"-pa " ++ Dir}]), L = rpc:call(Node,?MODULE,do_iter_max_files,[N, TestFile]), test_server:stop_node(Node), io:format("Number of files opened in each test:~n~w\n", [L]), - all_equal(L), + verify_max_files(L), Head = hd(L), if Head >= 2 -> ok; true -> ct:fail(too_few_files) @@ -65,12 +65,15 @@ do_iter_max_files(N, Name) when N > 0 -> do_iter_max_files(_, _) -> []. -all_equal([E, E| T]) -> - all_equal([E| T]); -all_equal([_]) -> - ok; -all_equal([]) -> - ok. +%% The attempts shouldn't vary too much; we used to require that they were all +%% exactly equal, but after we reimplemented the file driver as a NIF we +%% noticed that the only reason it was stable on Darwin was because the port +%% limit was hit before ulimit. +verify_max_files(Attempts) -> + N = length(Attempts), + Mean = lists:sum(Attempts) / N, + Variance = lists:sum([(X - Mean) * (X - Mean) || X <- Attempts]) / N, + true = math:sqrt(Variance) =< 1 + (Mean / 1000). max_files(Name) -> Fds = open_files(Name), diff --git a/erts/emulator/test/iovec_SUITE.erl b/erts/emulator/test/iovec_SUITE.erl index 49dc64b0d2..963b7e2501 100644 --- a/erts/emulator/test/iovec_SUITE.erl +++ b/erts/emulator/test/iovec_SUITE.erl @@ -25,7 +25,8 @@ -export([integer_lists/1, binary_lists/1, empty_lists/1, empty_binary_lists/1, mixed_lists/1, improper_lists/1, illegal_lists/1, cons_bomb/1, sub_binary_lists/1, iolist_to_iovec_idempotence/1, - iolist_to_iovec_correctness/1]). + iolist_to_iovec_correctness/1, unaligned_sub_binaries/1, + direct_binary_arg/1]). -include_lib("common_test/include/ct.hrl"). @@ -36,7 +37,8 @@ suite() -> all() -> [integer_lists, binary_lists, empty_lists, empty_binary_lists, mixed_lists, sub_binary_lists, illegal_lists, improper_lists, cons_bomb, - iolist_to_iovec_idempotence, iolist_to_iovec_correctness]. + iolist_to_iovec_idempotence, iolist_to_iovec_correctness, + unaligned_sub_binaries, direct_binary_arg]. init_per_suite(Config) -> Config. @@ -78,7 +80,7 @@ illegal_lists(Config) when is_list(Config) -> BitStrs = gen_variations(["gurka", <<1:1>>, "gaffel"]), BadInts = gen_variations(["gurka", 890, "gaffel"]), Atoms = gen_variations([gurka, "gaffel"]), - BadTails = [["test" | 0], ["gurka", gaffel]], + BadTails = [["test" | 0], ["gurka" | gaffel], ["gaffel" | <<1:1>>]], Variations = BitStrs ++ BadInts ++ Atoms ++ BadTails, @@ -98,14 +100,7 @@ cons_bomb(Config) when is_list(Config) -> BinBase = gen_variations([<<I:8>> || I <- lists:seq(1, 255)]), MixBase = gen_variations([<<12, 45, 78>>, lists:seq(1, 255)]), - Rounds = - case system_mem_size() of - Mem when Mem >= (16 bsl 30) -> 32; - Mem when Mem >= (3 bsl 30) -> 28; - _ -> 20 - end, - - Variations = gen_variations([IntBase, BinBase, MixBase], Rounds), + Variations = gen_variations([IntBase, BinBase, MixBase], 16), equivalence_test(fun erlang:iolist_to_iovec/1, Variations). iolist_to_iovec_idempotence(Config) when is_list(Config) -> @@ -130,6 +125,21 @@ iolist_to_iovec_correctness(Config) when is_list(Config) -> true = is_iolist_equal(Optimized, Variations), ok. +unaligned_sub_binaries(Config) when is_list(Config) -> + UnalignedBins = [gen_unaligned_binary(I) || I <- lists:seq(32, 4 bsl 10, 512)], + UnalignedVariations = gen_variations(UnalignedBins), + + Optimized = erlang:iolist_to_iovec(UnalignedVariations), + + true = is_iolist_equal(Optimized, UnalignedVariations), + ok. + +direct_binary_arg(Config) when is_list(Config) -> + {'EXIT',{badarg, _}} = (catch erlang:iolist_to_iovec(<<1:1>>)), + [<<1>>] = erlang:iolist_to_iovec(<<1>>), + [] = erlang:iolist_to_iovec(<<>>), + ok. + illegality_test(Fun, Variations) -> [{'EXIT',{badarg, _}} = (catch Fun(Variation)) || Variation <- Variations], ok. @@ -145,11 +155,18 @@ equivalence_test(Fun, [Head | _] = Variations) -> is_iolist_equal(A, B) -> iolist_to_binary(A) =:= iolist_to_binary(B). +gen_unaligned_binary(Size) -> + Bin0 = << <<I>> || I <- lists:seq(1, Size) >>, + <<0:3,Bin:Size/binary,31:5>> = id(<<0:3,Bin0/binary,31:5>>), + Bin. + +id(I) -> I. + %% Generates a bunch of lists whose contents will be equal to Base repeated a %% few times. The lists only differ by their structure, so their reduction to %% a simpler format should yield the same result. gen_variations(Base) -> - gen_variations(Base, 16). + gen_variations(Base, 12). gen_variations(Base, N) -> [gen_flat_list(Base, N), gen_nested_list(Base, N), @@ -169,8 +186,3 @@ gen_nasty_list_1([Head | Base], Result) when is_list(Head) -> gen_nasty_list_1(Base, [[Result], [gen_nasty_list_1(Head, [])]]); gen_nasty_list_1([Head | Base], Result) -> gen_nasty_list_1(Base, [[Result], [Head]]). - -system_mem_size() -> - application:ensure_all_started(os_mon), - {Tot,_Used,_} = memsup:get_memory_data(), - Tot. diff --git a/erts/emulator/test/nif_SUITE.erl b/erts/emulator/test/nif_SUITE.erl index 85c302e310..b6e15ababb 100644 --- a/erts/emulator/test/nif_SUITE.erl +++ b/erts/emulator/test/nif_SUITE.erl @@ -3049,8 +3049,24 @@ nif_ioq(Config) -> {peek_head, a}, {peek, a}, - %% Test to enqueue a bunch of refc binaries + %% Ensure that enqueued refc binaries are intact after a roundtrip. + %% + %% This test and the ones immediately following it does not go through + %% erlang:iolist_to_iovec/1 {enqv, a, [nif_ioq_payload(refcbin) || _ <- lists:seq(1,20)], 0}, + {peek, a}, + + %% ... heap binaries + {enqv, a, [nif_ioq_payload(heapbin) || _ <- lists:seq(1,20)], 0}, + {peek, a}, + + %% ... plain sub-binaries + {enqv, a, [nif_ioq_payload(subbin) || _ <- lists:seq(1,20)], 0}, + {peek, a}, + + %% ... unaligned binaries + {enqv, a, [nif_ioq_payload(unaligned_bin) || _ <- lists:seq(1,20)], 0}, + {peek, a}, %% Enq stuff to destroy with data in queue {enqv, a, 2, 100}, @@ -3206,13 +3222,16 @@ nif_ioq_payload(N) when is_integer(N) -> Tail = if N > 3 -> nif_ioq_payload(N-3); true -> [] end, Head = element(1, lists:split(N,[nif_ioq_payload(subbin), nif_ioq_payload(heapbin), - nif_ioq_payload(refcbin) | Tail])), + nif_ioq_payload(refcbin), + nif_ioq_payload(unaligned_bin) | Tail])), erlang:iolist_to_iovec(Head); nif_ioq_payload(subbin) -> Bin = nif_ioq_payload(refcbin), Sz = size(Bin) - 1, <<_:8,SubBin:Sz/binary,_/bits>> = Bin, SubBin; +nif_ioq_payload(unaligned_bin) -> + make_unaligned_binary(<< <<I>> || I <- lists:seq(1, 255) >>); nif_ioq_payload(heapbin) -> <<"a literal heap binary">>; nif_ioq_payload(refcbin) -> @@ -3220,6 +3239,13 @@ nif_ioq_payload(refcbin) -> nif_ioq_payload(Else) -> Else. +make_unaligned_binary(Bin0) -> + Size = byte_size(Bin0), + <<0:3,Bin:Size/binary,31:5>> = id(<<0:3,Bin0/binary,31:5>>), + Bin. + +id(I) -> I. + %% The NIFs: lib_version() -> undefined. call_history() -> ?nif_stub. |