diff options
Diffstat (limited to 'erts/emulator/beam')
169 files changed, 41607 insertions, 12130 deletions
diff --git a/erts/emulator/beam/atom.c b/erts/emulator/beam/atom.c index 84d2d5e3ed..fe91134ef4 100644 --- a/erts/emulator/beam/atom.c +++ b/erts/emulator/beam/atom.c @@ -3,16 +3,17 @@ * * Copyright Ericsson AB 1996-2013. All Rights Reserved. * - * The contents of this file are subject to the Erlang Public License, - * Version 1.1, (the "License"); you may not use this file except in - * compliance with the License. You should have received a copy of the - * Erlang Public License along with this software. If not, it can be - * retrieved online at http://www.erlang.org/. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Software distributed under the License is distributed on an "AS IS" - * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See - * the License for the specific language governing rights and limitations - * under the License. + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. * * %CopyrightEnd% */ diff --git a/erts/emulator/beam/atom.h b/erts/emulator/beam/atom.h index 5904ae0f7e..ead56c83d8 100644 --- a/erts/emulator/beam/atom.h +++ b/erts/emulator/beam/atom.h @@ -3,16 +3,17 @@ * * Copyright Ericsson AB 1996-2013. All Rights Reserved. * - * The contents of this file are subject to the Erlang Public License, - * Version 1.1, (the "License"); you may not use this file except in - * compliance with the License. You should have received a copy of the - * Erlang Public License along with this software. If not, it can be - * retrieved online at http://www.erlang.org/. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Software distributed under the License is distributed on an "AS IS" - * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See - * the License for the specific language governing rights and limitations - * under the License. + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. * * %CopyrightEnd% */ diff --git a/erts/emulator/beam/atom.names b/erts/emulator/beam/atom.names index ce60bb9bbc..cb6d294a41 100644 --- a/erts/emulator/beam/atom.names +++ b/erts/emulator/beam/atom.names @@ -3,16 +3,17 @@ # # Copyright Ericsson AB 1996-2013. All Rights Reserved. # -# The contents of this file are subject to the Erlang Public License, -# Version 1.1, (the "License"); you may not use this file except in -# compliance with the License. You should have received a copy of the -# Erlang Public License along with this software. If not, it can be -# retrieved online at http://www.erlang.org/. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# Software distributed under the License is distributed on an "AS IS" -# basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See -# the License for the specific language governing rights and limitations -# under the License. +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. # # %CopyrightEnd% # @@ -68,9 +69,12 @@ atom aborted atom abs_path atom absoluteURI atom ac +atom accessor atom active +atom active_tasks atom all atom all_but_first +atom all_names atom alloc_info atom alloc_sizes atom allocated @@ -78,6 +82,7 @@ atom allocated_areas atom allocator atom allocator_sizes atom alloc_util_allocators +atom allow_gc atom allow_passive_connect atom already_loaded atom amd64 @@ -92,21 +97,24 @@ atom args atom arg0 atom arity atom asn1 +atom async atom asynchronous atom atom atom atom_used atom attributes atom await_port_send_result atom await_proc_exit +atom await_result atom await_sched_wall_time_modifications atom awaiting_load atom awaiting_unload atom backtrace backtrace_depth -atom badarg badarith badarity badfile badmatch badsig badfun +atom badarg badarith badarity badfile badfun badkey badmap badmatch badsig atom bag atom band atom big atom bif_return_trap +atom bif_timer_server atom binary atom binary_bin_to_list_trap atom binary_copy_trap @@ -114,6 +122,8 @@ atom binary_longest_prefix_trap atom binary_longest_suffix_trap atom binary_match_trap atom binary_matches_trap +atom binary_to_list_continue +atom binary_to_term_trap atom block atom blocked atom bm @@ -139,9 +149,12 @@ atom caseless atom catchlevel atom cd atom cdr +atom cflags +atom CHANGE='CHANGE' atom characters_to_binary_int atom characters_to_list_int atom clear +atom clock_service atom close atom closed atom code @@ -150,6 +163,8 @@ atom compact atom compat_rel atom compile atom compressed +atom config_h +atom convert_time_unit atom connect atom connected atom connection_closed @@ -174,10 +189,12 @@ atom dexit atom depth atom dgroup_leader atom dictionary +atom dirty_cpu_schedulers_online atom disable_trace atom disabled atom display_items atom dist +atom dist_cmd atom Div='/' atom div atom dlink @@ -190,6 +207,7 @@ atom dotall atom driver atom driver_options atom dsend +atom dsend_continue_trap atom dunlink atom duplicate_bag atom dupnames @@ -228,7 +246,7 @@ atom first atom firstline atom flags atom flush -atom flush_monitor_message +atom flush_monitor_messages atom force atom format_cpu_topology atom free @@ -248,7 +266,10 @@ atom get_data atom get_seq_token atom get_tcw atom getenv +atom gather_gc_info_result +atom gather_io_bytes atom gather_sched_wall_time_result +atom gather_system_check_result atom getting_linked atom getting_unlinked atom global @@ -270,7 +291,6 @@ atom http httph https http_response http_request http_header http_eoh http_error atom id atom if_clause atom ignore -atom imports atom in atom in_exiting atom inactive @@ -299,36 +319,47 @@ atom label atom large_heap atom last_calls atom latin1 +atom ldflags atom Le='=<' atom lf atom line +atom line_delimiter atom line_length atom linked_in_driver atom links atom list +atom list_to_binary_continue atom little atom loaded atom load_cancelled atom load_failure atom local atom long_gc +atom long_schedule atom low atom Lt='<' atom machine atom match +atom match_limit +atom match_limit_recursion atom match_spec atom max atom maximum atom max_tables max_processes atom mbuf_size +atom md5 atom memory +atom memory_internal atom memory_types atom message atom message_binary atom message_queue_len atom messages +atom merge_trap atom meta atom meta_match_spec +atom micro_seconds +atom milli_seconds atom min_heap_size atom min_bin_vheap_size atom minor_version @@ -339,16 +370,22 @@ atom monitored_by atom monitor atom monitor_nodes atom monitors +atom monotonic +atom monotonic_timestamp atom more atom multi_scheduling atom multiline +atom nano_seconds atom name atom named_table +atom namelist +atom native atom native_addresses atom Neq='=/=' atom Neqeq='/=' atom net_kernel atom net_kernel_terminated +atom never_utf atom new atom new_index atom new_uniq @@ -374,6 +411,7 @@ atom nosuspend atom no_float atom no_integer atom no_network +atom no_start_optimize atom not atom not_a_list atom not_loaded @@ -384,6 +422,7 @@ atom notalive atom notbol atom noteol atom notempty +atom notempty_atstart atom notify atom notsup atom nouse_stdio @@ -430,6 +469,8 @@ atom port atom ports atom port_count atom port_limit +atom port_op +atom positive atom print atom priority atom private @@ -441,6 +482,7 @@ atom process_display atom process_limit atom process_dump atom procs +atom proc_sig atom profile atom protected atom protection @@ -466,12 +508,14 @@ atom register atom registered_name atom reload atom rem +atom report_errors atom reset atom restart atom return_from atom return_to atom return_trace atom run_queue +atom run_queue_lengths atom runnable atom runnable_ports atom runnable_procs @@ -487,6 +531,7 @@ atom schedulers_online atom scheme atom scientific atom scope +atom seconds atom sensitive atom sequential_tracer atom sequential_trace_token @@ -516,6 +561,7 @@ atom static atom stderr_to_stdout atom stop atom stream +atom strict_monotonic_timestamp atom sunrm atom suspend atom suspended @@ -528,15 +574,19 @@ atom system_version atom system_architecture atom SYSTEM='SYSTEM' atom table +atom term_to_binary_trap atom this atom thread_pool_size atom threads +atom time_offset atom timeout atom timeout_value atom Times='*' atom timestamp atom total +atom total_active_tasks atom total_heap_size +atom total_run_queue_lengths atom tpkt atom trace trace_ts traced atom trace_control_word @@ -547,6 +597,7 @@ atom true atom tuple atom type atom ucompile +atom ucp atom undef atom ungreedy atom unicode diff --git a/erts/emulator/beam/beam_bif_load.c b/erts/emulator/beam/beam_bif_load.c index 649594a334..0e192b1ebd 100644 --- a/erts/emulator/beam/beam_bif_load.c +++ b/erts/emulator/beam/beam_bif_load.c @@ -3,16 +3,17 @@ * * Copyright Ericsson AB 1999-2013. All Rights Reserved. * - * The contents of this file are subject to the Erlang Public License, - * Version 1.1, (the "License"); you may not use this file except in - * compliance with the License. You should have received a copy of the - * Erlang Public License along with this software. If not, it can be - * retrieved online at http://www.erlang.org/. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Software distributed under the License is distributed on an "AS IS" - * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See - * the License for the specific language governing rights and limitations - * under the License. + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. * * %CopyrightEnd% */ @@ -33,18 +34,16 @@ #include "beam_catches.h" #include "erl_binary.h" #include "erl_nif.h" +#include "erl_bits.h" #include "erl_thr_progress.h" static void set_default_trace_pattern(Eterm module); -static Eterm check_process_code(Process* rp, Module* modp); +static Eterm check_process_code(Process* rp, Module* modp, int allow_gc, int *redsp); static void delete_code(Module* modp); static void decrement_refc(BeamInstr* code); -static int is_native(BeamInstr* code); static int any_heap_ref_ptrs(Eterm* start, Eterm* end, char* mod_start, Uint mod_size); static int any_heap_refs(Eterm* start, Eterm* end, char* mod_start, Uint mod_size); - - BIF_RETTYPE code_is_module_native_1(BIF_ALIST_1) { Module* modp; @@ -59,8 +58,8 @@ BIF_RETTYPE code_is_module_native_1(BIF_ALIST_1) return am_undefined; } erts_rlock_old_code(code_ix); - res = ((modp->curr.code && is_native(modp->curr.code)) || - (modp->old.code != 0 && is_native(modp->old.code))) ? + res = (erts_is_module_native(modp->curr.code) || + erts_is_module_native(modp->old.code)) ? am_true : am_false; erts_runlock_old_code(code_ix); return res; @@ -201,7 +200,7 @@ finish_loading_1(BIF_ALIST_1) * to keep the elements in. */ - n = list_length(BIF_ARG_1); + n = erts_list_length(BIF_ARG_1); if (n == -1) { ERTS_BIF_PREP_ERROR(res, BIF_P, BADARG); goto done; @@ -371,7 +370,7 @@ staging_epilogue(Process* c_p, int commit, Eterm res, int is_blocking, ASSERT(commiter_state.stager == NULL); commiter_state.stager = c_p; erts_schedule_thr_prgr_later_op(smp_code_ix_commiter, NULL, &commiter_state.lop); - erts_smp_proc_inc_refc(c_p); + erts_proc_inc_refc(c_p); erts_suspend(c_p, ERTS_PROC_LOCK_MAIN, NULL); /* * smp_code_ix_commiter() will do the rest "later" @@ -398,7 +397,7 @@ static void smp_code_ix_commiter(void* null) erts_resume(p, ERTS_PROC_LOCK_STATUS); } erts_smp_proc_unlock(p, ERTS_PROC_LOCK_STATUS); - erts_smp_proc_dec_refc(p); + erts_proc_dec_refc(p); } #endif /* ERTS_SMP */ @@ -427,69 +426,82 @@ check_old_code_1(BIF_ALIST_1) } Eterm -check_process_code_2(BIF_ALIST_2) +erts_check_process_code(Process *c_p, Eterm module, int allow_gc, int *redsp) { - Process* rp; Module* modp; + Eterm res; + ErtsCodeIndex code_ix; - if (is_not_atom(BIF_ARG_2)) { - goto error; - } - if (is_internal_pid(BIF_ARG_1)) { - Eterm res; - ErtsCodeIndex code_ix; - - code_ix = erts_active_code_ix(); - modp = erts_get_module(BIF_ARG_2, code_ix); - if (modp == NULL) { /* Doesn't exist. */ - return am_false; - } - erts_rlock_old_code(code_ix); - if (modp->old.code == NULL) { /* No old code. */ - erts_runlock_old_code(code_ix); - return am_false; - } - erts_runlock_old_code(code_ix); - -#ifdef ERTS_SMP - rp = erts_pid2proc_suspend(BIF_P, ERTS_PROC_LOCK_MAIN, - BIF_ARG_1, ERTS_PROC_LOCK_MAIN); -#else - rp = erts_pid2proc(BIF_P, 0, BIF_ARG_1, 0); -#endif - if (!rp) { - BIF_RET(am_false); - } - if (rp == ERTS_PROC_LOCK_BUSY) { - ERTS_BIF_YIELD2(bif_export[BIF_check_process_code_2], BIF_P, - BIF_ARG_1, BIF_ARG_2); - } - erts_rlock_old_code(code_ix); - if (modp->old.code != NULL) { /* must check again */ - res = check_process_code(rp, modp); - } - else { - res = am_false; - } - erts_runlock_old_code(code_ix); -#ifdef ERTS_SMP - if (BIF_P != rp) { - erts_resume(rp, ERTS_PROC_LOCK_MAIN); - erts_smp_proc_unlock(rp, ERTS_PROC_LOCK_MAIN); + (*redsp)++; + + ASSERT(is_atom(module)); + + code_ix = erts_active_code_ix(); + modp = erts_get_module(module, code_ix); + if (!modp) + return am_false; + erts_rlock_old_code(code_ix); + res = modp->old.code ? check_process_code(c_p, modp, allow_gc, redsp) : am_false; + erts_runlock_old_code(code_ix); + + return res; +} + +BIF_RETTYPE erts_internal_check_process_code_2(BIF_ALIST_2) +{ + int reds = 0; + Eterm res; + Eterm olist = BIF_ARG_2; + int allow_gc = 1; + + if (is_not_atom(BIF_ARG_1)) + goto badarg; + + while (is_list(olist)) { + Eterm *lp = list_val(olist); + Eterm opt = CAR(lp); + if (is_tuple(opt)) { + Eterm* tp = tuple_val(opt); + switch (arityval(tp[0])) { + case 2: + switch (tp[1]) { + case am_allow_gc: + switch (tp[2]) { + case am_false: + allow_gc = 0; + break; + case am_true: + allow_gc = 1; + break; + default: + goto badarg; + } + break; + default: + goto badarg; + } + break; + default: + goto badarg; + } } -#endif - BIF_RET(res); - } - else if (is_external_pid(BIF_ARG_1) - && external_pid_dist_entry(BIF_ARG_1) == erts_this_dist_entry) { - BIF_RET(am_false); + else + goto badarg; + olist = CDR(lp); } + if (is_not_nil(olist)) + goto badarg; + + res = erts_check_process_code(BIF_P, BIF_ARG_1, allow_gc, &reds); + + ASSERT(is_value(res)); + + BIF_RET2(res, reds); - error: +badarg: BIF_ERROR(BIF_P, BADARG); } - BIF_RETTYPE delete_module_1(BIF_ALIST_1) { ErtsCodeIndex code_ix; @@ -710,7 +722,7 @@ set_default_trace_pattern(Eterm module) } static Eterm -check_process_code(Process* rp, Module* modp) +check_process_code(Process* rp, Module* modp, int allow_gc, int *redsp) { BeamInstr* start; char* mod_start; @@ -773,6 +785,16 @@ check_process_code(Process* rp, Module* modp) } } + if (rp->flags & F_DISABLE_GC) { + /* + * Cannot proceed. Process has disabled gc in order to + * safely leave inconsistent data on the heap and/or + * off heap lists. Need to wait for gc to be enabled + * again. + */ + return THE_NON_VALUE; + } + /* * See if there are funs that refer to the old version of the module. */ @@ -786,6 +808,8 @@ check_process_code(Process* rp, Module* modp) if (done_gc) { return am_true; } else { + if (!allow_gc) + return am_aborted; /* * Try to get rid of this fun by garbage collecting. * Clear both fvalue and ftrace to make sure they @@ -796,7 +820,7 @@ check_process_code(Process* rp, Module* modp) rp->ftrace = NIL; done_gc = 1; FLAGS(rp) |= F_NEED_FULLSWEEP; - (void) erts_garbage_collect(rp, 0, rp->arg_reg, rp->arity); + *redsp += erts_garbage_collect(rp, 0, rp->arg_reg, rp->arity); goto rescan; } } @@ -850,6 +874,9 @@ check_process_code(Process* rp, Module* modp) Uint lit_size; struct erl_off_heap_header* oh; + if (!allow_gc) + return am_aborted; + /* * Try to get rid of constants by by garbage collecting. * Clear both fvalue and ftrace. @@ -859,11 +886,12 @@ check_process_code(Process* rp, Module* modp) rp->ftrace = NIL; done_gc = 1; FLAGS(rp) |= F_NEED_FULLSWEEP; - (void) erts_garbage_collect(rp, 0, rp->arg_reg, rp->arity); + *redsp += erts_garbage_collect(rp, 0, rp->arg_reg, rp->arity); literals = (Eterm *) modp->old.code[MI_LITERALS_START]; lit_size = (Eterm *) modp->old.code[MI_LITERALS_END] - literals; oh = (struct erl_off_heap_header *) modp->old.code[MI_LITERALS_OFF_HEAP]; + *redsp += lit_size / 10; /* Need, better value... */ erts_garbage_collect_literals(rp, literals, lit_size, oh); } } @@ -911,7 +939,15 @@ any_heap_refs(Eterm* start, Eterm* end, char* mod_start, Uint mod_size) break; case TAG_PRIMARY_HEADER: if (!header_is_transparent(val)) { - Eterm* new_p = p + thing_arityval(val); + Eterm* new_p; + if (header_is_bin_matchstate(val)) { + ErlBinMatchState *ms = (ErlBinMatchState*) p; + ErlBinMatchBuffer *mb = &(ms->mb); + if (in_area(EXPAND_POINTER(mb->orig), mod_start, mod_size)) { + return 1; + } + } + new_p = p + thing_arityval(val); ASSERT(start <= new_p && new_p < end); p = new_p; } @@ -1077,25 +1113,3 @@ beam_make_current_old(Process *c_p, ErtsProcLocks c_p_locks, Eterm module) } return NIL; } - -static int -is_native(BeamInstr* code) -{ - Uint i, num_functions = code[MI_NUM_FUNCTIONS]; - - /* Check NativeAdress of first real function in module - */ - for (i=0; i<num_functions; i++) { - BeamInstr* func_info = (BeamInstr *) code[MI_FUNCTIONS+i]; - Eterm name = (Eterm) func_info[3]; - - if (is_atom(name)) { - return func_info[1] != 0; - } - else ASSERT(is_nil(name)); /* ignore BIF stubs */ - } - /* Not a single non-BIF function? */ - return 0; -} - - diff --git a/erts/emulator/beam/beam_bp.c b/erts/emulator/beam/beam_bp.c index 9b17de34ec..9860968687 100644 --- a/erts/emulator/beam/beam_bp.c +++ b/erts/emulator/beam/beam_bp.c @@ -1,18 +1,19 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2000-2012. All Rights Reserved. + * Copyright Ericsson AB 2000-2013. All Rights Reserved. * - * The contents of this file are subject to the Erlang Public License, - * Version 1.1, (the "License"); you may not use this file except in - * compliance with the License. You should have received a copy of the - * Erlang Public License along with this software. If not, it can be - * retrieved online at http://www.erlang.org/. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Software distributed under the License is distributed on an "AS IS" - * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See - * the License for the specific language governing rights and limitations - * under the License. + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. * * %CopyrightEnd% */ @@ -46,7 +47,8 @@ #if defined(ERTS_ENABLE_LOCK_CHECK) && defined(ERTS_SMP) # define ERTS_SMP_REQ_PROC_MAIN_LOCK(P) \ - if ((P)) erts_proc_lc_require_lock((P), ERTS_PROC_LOCK_MAIN) + if ((P)) erts_proc_lc_require_lock((P), ERTS_PROC_LOCK_MAIN,\ + __FILE__, __LINE__) # define ERTS_SMP_UNREQ_PROC_MAIN_LOCK(P) \ if ((P)) erts_proc_lc_unrequire_lock((P), ERTS_PROC_LOCK_MAIN) #else @@ -73,6 +75,16 @@ extern BeamInstr beam_return_time_trace[1]; /* OpCode(i_return_time_trace) */ erts_smp_atomic32_t erts_active_bp_index; erts_smp_atomic32_t erts_staging_bp_index; +/* + * Inlined helpers + */ + +static ERTS_INLINE ErtsMonotonicTime +get_mtime(Process *c_p) +{ + return erts_get_monotonic_time(ERTS_PROC_GET_SCHDATA(c_p)); +} + /* ************************************************************************* ** Local prototypes */ @@ -95,9 +107,6 @@ static int clear_function_break(BeamInstr *pc, Uint break_flags); static BpDataTime* get_time_break(BeamInstr *pc); static GenericBpData* check_break(BeamInstr *pc, Uint break_flags); -static void bp_time_diff(bp_data_time_item_t *item, - process_breakpoint_time_t *pbt, - Uint ms, Uint s, Uint us); static void bp_meta_unref(BpMetaPid* bmp); static void bp_count_unref(BpCount* bcp); @@ -108,13 +117,8 @@ static void uninstall_breakpoint(BeamInstr* pc); /* bp_hash */ #define BP_TIME_ADD(pi0, pi1) \ do { \ - Uint r; \ (pi0)->count += (pi1)->count; \ - (pi0)->s_time += (pi1)->s_time; \ - (pi0)->us_time += (pi1)->us_time; \ - r = (pi0)->us_time / 1000000; \ - (pi0)->s_time += r; \ - (pi0)->us_time = (pi0)->us_time % 1000000; \ + (pi0)->time += (pi1)->time; \ } while(0) static void bp_hash_init(bp_time_hash_t *hash, Uint n); @@ -641,7 +645,7 @@ erts_generic_breakpoint(Process* c_p, BeamInstr* I, Eterm* reg) erts_smp_atomic_inc_nob(&bp->count->acount); } - if (bp_flags & ERTS_BPF_TIME_TRACE_ACTIVE) { + if (bp_flags & ERTS_BPF_TIME_TRACE_ACTIVE && erts_is_tracer_proc_valid(c_p)) { Eterm w; erts_trace_time_call(c_p, I, bp->time); w = (BeamInstr) *c_p->cp; @@ -729,7 +733,8 @@ erts_bif_trace(int bif_index, Process* p, Eterm* args, BeamInstr* I) } } if (bp_flags & ERTS_BPF_TIME_TRACE_ACTIVE && - IS_TRACED_FL(p, F_TRACE_CALLS)) { + IS_TRACED_FL(p, F_TRACE_CALLS) && + erts_is_tracer_proc_valid(p)) { BeamInstr *pc = (BeamInstr *)ep->code+3; erts_trace_time_call(p, pc, bp->time); } @@ -945,7 +950,7 @@ do_call_trace(Process* c_p, BeamInstr* I, Eterm* reg, void erts_trace_time_call(Process* c_p, BeamInstr* I, BpDataTime* bdt) { - Uint ms,s,us; + ErtsMonotonicTime time; process_breakpoint_time_t *pbt = NULL; bp_data_time_item_t sitem, *item = NULL; bp_time_hash_t *h = NULL; @@ -958,7 +963,7 @@ erts_trace_time_call(Process* c_p, BeamInstr* I, BpDataTime* bdt) * from the process psd */ pbt = ERTS_PROC_GET_CALL_TIME(c_p); - get_sys_now(&ms, &s, &us); + time = get_mtime(c_p); /* get pbt * timestamp = t0 @@ -973,7 +978,7 @@ erts_trace_time_call(Process* c_p, BeamInstr* I, BpDataTime* bdt) } else { ASSERT(pbt->pc); /* add time to previous code */ - bp_time_diff(&sitem, pbt, ms, s, us); + sitem.time = time - pbt->time; sitem.pid = c_p->common.id; sitem.count = 0; @@ -999,8 +1004,7 @@ erts_trace_time_call(Process* c_p, BeamInstr* I, BpDataTime* bdt) /* Add count to this code */ sitem.pid = c_p->common.id; sitem.count = 1; - sitem.s_time = 0; - sitem.us_time = 0; + sitem.time = 0; /* this breakpoint */ ASSERT(bdt); @@ -1017,15 +1021,13 @@ erts_trace_time_call(Process* c_p, BeamInstr* I, BpDataTime* bdt) } pbt->pc = I; - pbt->ms = ms; - pbt->s = s; - pbt->us = us; + pbt->time = time; } void erts_trace_time_return(Process *p, BeamInstr *pc) { - Uint ms,s,us; + ErtsMonotonicTime time; process_breakpoint_time_t *pbt = NULL; bp_data_time_item_t sitem, *item = NULL; bp_time_hash_t *h = NULL; @@ -1038,7 +1040,7 @@ erts_trace_time_return(Process *p, BeamInstr *pc) * from the process psd */ pbt = ERTS_PROC_GET_CALL_TIME(p); - get_sys_now(&ms,&s,&us); + time = get_mtime(p); /* get pbt * lookup bdt from code @@ -1054,7 +1056,7 @@ erts_trace_time_return(Process *p, BeamInstr *pc) */ ASSERT(pbt->pc); - bp_time_diff(&sitem, pbt, ms, s, us); + sitem.time = time - pbt->time; sitem.pid = p->common.id; sitem.count = 0; @@ -1077,9 +1079,7 @@ erts_trace_time_return(Process *p, BeamInstr *pc) } pbt->pc = pc; - pbt->ms = ms; - pbt->s = s; - pbt->us = us; + pbt->time = time; } } @@ -1180,10 +1180,14 @@ int erts_is_time_break(Process *p, BeamInstr *pc, Eterm *retval) { for(ix = 0; ix < hash.n; ix++) { item = &(hash.item[ix]); if (item->pid != NIL) { + ErtsMonotonicTime sec, usec; + usec = ERTS_MONOTONIC_TO_USEC(item->time); + sec = usec / 1000000; + usec = usec - sec*1000000; t = TUPLE4(hp, item->pid, make_small(item->count), - make_small(item->s_time), - make_small(item->us_time)); + make_small((Uint) sec), + make_small((Uint) usec)); hp += 5; *retval = CONS(hp, t, *retval); hp += 2; } @@ -1263,8 +1267,7 @@ static void bp_hash_rehash(bp_time_hash_t *hash, Uint n) { } item[hval].pid = hash->item[ix].pid; item[hval].count = hash->item[ix].count; - item[hval].s_time = hash->item[ix].s_time; - item[hval].us_time = hash->item[ix].us_time; + item[hval].time = hash->item[ix].time; } } @@ -1312,8 +1315,7 @@ static ERTS_INLINE bp_data_time_item_t * bp_hash_put(bp_time_hash_t *hash, bp_da item = &(hash->item[hval]); item->pid = sitem->pid; - item->s_time = sitem->s_time; - item->us_time = sitem->us_time; + item->time = sitem->time; item->count = sitem->count; hash->used++; @@ -1327,41 +1329,7 @@ static void bp_hash_delete(bp_time_hash_t *hash) { hash->item = NULL; } -static void bp_time_diff(bp_data_time_item_t *item, /* out */ - process_breakpoint_time_t *pbt, /* in */ - Uint ms, Uint s, Uint us) { - int ds,dus; -#ifdef DEBUG - int dms; - - - dms = ms - pbt->ms; -#endif - ds = s - pbt->s; - dus = us - pbt->us; - - /* get_sys_now may return zero difftime, - * this is ok. - */ - -#ifdef DEBUG - ASSERT(dms >= 0 || ds >= 0 || dus >= 0); -#endif - - if (dus < 0) { - dus += 1000000; - ds -= 1; - } - if (ds < 0) { - ds += 1000000; - } - - item->s_time = ds; - item->us_time = dus; -} - void erts_schedule_time_break(Process *p, Uint schedule) { - Uint ms, s, us; process_breakpoint_time_t *pbt = NULL; bp_data_time_item_t sitem, *item = NULL; bp_time_hash_t *h = NULL; @@ -1384,8 +1352,7 @@ void erts_schedule_time_break(Process *p, Uint schedule) { pbdt = get_time_break(pbt->pc); if (pbdt) { - get_sys_now(&ms,&s,&us); - bp_time_diff(&sitem, pbt, ms, s, us); + sitem.time = get_mtime(p) - pbt->time; sitem.pid = p->common.id; sitem.count = 0; @@ -1407,10 +1374,7 @@ void erts_schedule_time_break(Process *p, Uint schedule) { * timestamp it and remove the previous * timestamp in the psd. */ - get_sys_now(&ms,&s,&us); - pbt->ms = ms; - pbt->s = s; - pbt->us = us; + pbt->time = get_mtime(p); break; default : ASSERT(0); @@ -1580,6 +1544,7 @@ clear_function_break(BeamInstr *pc, Uint break_flags) } if (common & ERTS_BPF_META_TRACE) { MatchSetUnref(bp->meta_ms); + bp_meta_unref(bp->meta_pid); } if (common & ERTS_BPF_COUNT) { ASSERT((bp->flags & ERTS_BPF_COUNT_ACTIVE) == 0); diff --git a/erts/emulator/beam/beam_bp.h b/erts/emulator/beam/beam_bp.h index b061401863..2b89d6fc71 100644 --- a/erts/emulator/beam/beam_bp.h +++ b/erts/emulator/beam/beam_bp.h @@ -3,16 +3,17 @@ * * Copyright Ericsson AB 2000-2013. All Rights Reserved. * - * The contents of this file are subject to the Erlang Public License, - * Version 1.1, (the "License"); you may not use this file except in - * compliance with the License. You should have received a copy of the - * Erlang Public License along with this software. If not, it can be - * retrieved online at http://www.erlang.org/. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Software distributed under the License is distributed on an "AS IS" - * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See - * the License for the specific language governing rights and limitations - * under the License. + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. * * %CopyrightEnd% */ @@ -28,8 +29,7 @@ typedef struct { Eterm pid; Sint count; - Uint s_time; - Uint us_time; + ErtsMonotonicTime time; } bp_data_time_item_t; typedef struct { @@ -45,9 +45,7 @@ typedef struct bp_data_time { /* Call time */ } BpDataTime; typedef struct { - Uint ms; - Uint s; - Uint us; + ErtsMonotonicTime time; BeamInstr *pc; } process_breakpoint_time_t; /* used within psd */ diff --git a/erts/emulator/beam/beam_catches.c b/erts/emulator/beam/beam_catches.c index d374d0469e..7a1f4901aa 100644 --- a/erts/emulator/beam/beam_catches.c +++ b/erts/emulator/beam/beam_catches.c @@ -3,16 +3,17 @@ * * Copyright Ericsson AB 2000-2013. All Rights Reserved. * - * The contents of this file are subject to the Erlang Public License, - * Version 1.1, (the "License"); you may not use this file except in - * compliance with the License. You should have received a copy of the - * Erlang Public License along with this software. If not, it can be - * retrieved online at http://www.erlang.org/. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Software distributed under the License is distributed on an "AS IS" - * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See - * the License for the specific language governing rights and limitations - * under the License. + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. * * %CopyrightEnd% */ @@ -142,7 +143,7 @@ BeamInstr *beam_catches_car(unsigned i) struct bc_pool* p = &bccix[erts_active_code_ix()]; if (i >= p->tabsize ) { - erl_exit(1, "beam_catches_delmod: index %#x is out of range\r\n", i); + erts_exit(ERTS_ERROR_EXIT, "beam_catches_delmod: index %#x is out of range\r\n", i); } return p->beam_catches[i].cp; } @@ -156,10 +157,10 @@ void beam_catches_delmod(unsigned head, BeamInstr *code, unsigned code_bytes, ASSERT((code_ix == erts_active_code_ix()) != bccix[erts_staging_code_ix()].is_staging); for(i = head; i != (unsigned)-1;) { if (i >= p->tabsize) { - erl_exit(1, "beam_catches_delmod: index %#x is out of range\r\n", i); + erts_exit(ERTS_ERROR_EXIT, "beam_catches_delmod: index %#x is out of range\r\n", i); } if( (char*)p->beam_catches[i].cp - (char*)code >= code_bytes ) { - erl_exit(1, + erts_exit(ERTS_ERROR_EXIT, "beam_catches_delmod: item %#x has cp %p which is not " "in module's range [%p,%p[\r\n", i, p->beam_catches[i].cp, code, ((char*)code + code_bytes)); diff --git a/erts/emulator/beam/beam_catches.h b/erts/emulator/beam/beam_catches.h index 51ef463b2f..59ee64d033 100644 --- a/erts/emulator/beam/beam_catches.h +++ b/erts/emulator/beam/beam_catches.h @@ -3,16 +3,17 @@ * * Copyright Ericsson AB 2000-2013. All Rights Reserved. * - * The contents of this file are subject to the Erlang Public License, - * Version 1.1, (the "License"); you may not use this file except in - * compliance with the License. You should have received a copy of the - * Erlang Public License along with this software. If not, it can be - * retrieved online at http://www.erlang.org/. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Software distributed under the License is distributed on an "AS IS" - * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See - * the License for the specific language governing rights and limitations - * under the License. + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. * * %CopyrightEnd% */ diff --git a/erts/emulator/beam/beam_debug.c b/erts/emulator/beam/beam_debug.c index e36ec2a93e..90985e4f53 100644 --- a/erts/emulator/beam/beam_debug.c +++ b/erts/emulator/beam/beam_debug.c @@ -3,16 +3,17 @@ * * Copyright Ericsson AB 1998-2013. All Rights Reserved. * - * The contents of this file are subject to the Erlang Public License, - * Version 1.1, (the "License"); you may not use this file except in - * compliance with the License. You should have received a copy of the - * Erlang Public License along with this software. If not, it can be - * retrieved online at http://www.erlang.org/. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Software distributed under the License is distributed on an "AS IS" - * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See - * the License for the specific language governing rights and limitations - * under the License. + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. * * %CopyrightEnd% */ @@ -78,7 +79,7 @@ erts_debug_breakpoint_2(BIF_ALIST_2) { Process* p = BIF_P; Eterm MFA = BIF_ARG_1; - Eterm bool = BIF_ARG_2; + Eterm boolean = BIF_ARG_2; Eterm* tp; Eterm mfa[3]; int i; @@ -86,7 +87,7 @@ erts_debug_breakpoint_2(BIF_ALIST_2) Eterm res; BpFunctions f; - if (bool != am_true && bool != am_false) + if (boolean != am_true && boolean != am_false) goto error; if (is_not_tuple(MFA)) { @@ -123,7 +124,7 @@ erts_debug_breakpoint_2(BIF_ALIST_2) erts_smp_thr_progress_block(); erts_bp_match_functions(&f, mfa, specified); - if (bool == am_true) { + if (boolean == am_true) { erts_set_debug_break(&f); erts_install_breakpoints(&f); erts_commit_staged_bp(); @@ -297,8 +298,8 @@ erts_debug_disassemble_1(BIF_ALIST_1) (void) erts_bld_uword(NULL, &hsz, (BeamInstr) code_ptr); hp = HAlloc(p, hsz); addr = erts_bld_uword(&hp, NULL, (BeamInstr) code_ptr); - ASSERT(is_atom(funcinfo[0])); - ASSERT(is_atom(funcinfo[1])); + ASSERT(is_atom(funcinfo[0]) || funcinfo[0] == NIL); + ASSERT(is_atom(funcinfo[1]) || funcinfo[1] == NIL); mfa = TUPLE3(hp, (Eterm) funcinfo[0], (Eterm) funcinfo[1], make_small((Eterm) funcinfo[2])); hp += 4; return TUPLE3(hp, addr, bin, mfa); @@ -579,35 +580,65 @@ print_op(int to, void *to_arg, int op, int size, BeamInstr* addr) unpacked = ap; ap = addr + size; switch (op) { - case op_i_select_val_rfI: - case op_i_select_val_xfI: - case op_i_select_val_yfI: + case op_i_select_val_lins_rfI: + case op_i_select_val_lins_xfI: + case op_i_select_val_lins_yfI: { int n = ap[-1]; + int ix = n; - while (n > 0) { - erts_print(to, to_arg, "%T f(" HEXF ") ", (Eterm) ap[0], ap[1]); - ap += 2; - size += 2; - n--; + while (ix--) { + erts_print(to, to_arg, "%T ", (Eterm) ap[0]); + ap++; + size++; + } + ix = n; + while (ix--) { + erts_print(to, to_arg, "f(" HEXF ") ", (Eterm) ap[0]); + ap++; + size++; } } break; - case op_i_select_tuple_arity_rfI: - case op_i_select_tuple_arity_xfI: - case op_i_select_tuple_arity_yfI: + case op_i_select_val_bins_rfI: + case op_i_select_val_bins_xfI: + case op_i_select_val_bins_yfI: { int n = ap[-1]; while (n > 0) { - Uint arity = arityval(ap[0]); - erts_print(to, to_arg, " {%d} f(" HEXF ")", arity, ap[1]); + erts_print(to, to_arg, "%T f(" HEXF ") ", (Eterm) ap[0], ap[1]); ap += 2; size += 2; n--; } } break; + case op_i_select_tuple_arity_rfI: + case op_i_select_tuple_arity_xfI: + case op_i_select_tuple_arity_yfI: + { + int n = ap[-1]; + int ix = n - 1; /* without sentinel */ + + while (ix--) { + Uint arity = arityval(ap[0]); + erts_print(to, to_arg, "{%d} ", arity, ap[1]); + ap++; + size++; + } + /* print sentinel */ + erts_print(to, to_arg, "{%T} ", ap[0], ap[1]); + ap++; + size++; + ix = n; + while (ix--) { + erts_print(to, to_arg, "f(" HEXF ") ", ap[0]); + ap++; + size++; + } + } + break; case op_i_jump_on_val_rfII: case op_i_jump_on_val_xfII: case op_i_jump_on_val_yfII: @@ -635,6 +666,9 @@ print_op(int to, void *to_arg, int op, int size, BeamInstr* addr) case op_i_put_tuple_rI: case op_i_put_tuple_xI: case op_i_put_tuple_yI: + case op_new_map_dII: + case op_update_map_assoc_jsdII: + case op_update_map_exact_jsdII: { int n = unpacked[-1]; @@ -658,6 +692,32 @@ print_op(int to, void *to_arg, int op, int size, BeamInstr* addr) } } break; + case op_i_get_map_elements_fsI: + { + int n = unpacked[-1]; + + while (n > 0) { + if (n % 3 == 1) { + erts_print(to, to_arg, " %X", ap[0]); + } else if (!is_header(ap[0])) { + erts_print(to, to_arg, " %T", (Eterm) ap[0]); + } else { + switch ((ap[0] >> 2) & 0x03) { + case R_REG_DEF: + erts_print(to, to_arg, " x(0)"); + break; + case X_REG_DEF: + erts_print(to, to_arg, " x(%d)", ap[0] >> 4); + break; + case Y_REG_DEF: + erts_print(to, to_arg, " y(%d)", ap[0] >> 4); + break; + } + } + ap++, size++, n--; + } + } + break; } erts_print(to, to_arg, "\n"); diff --git a/erts/emulator/beam/beam_emu.c b/erts/emulator/beam/beam_emu.c index 944ed6da81..1fe4cc9374 100644 --- a/erts/emulator/beam/beam_emu.c +++ b/erts/emulator/beam/beam_emu.c @@ -1,18 +1,19 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 1996-2013. All Rights Reserved. + * Copyright Ericsson AB 1996-2014. All Rights Reserved. * - * The contents of this file are subject to the Erlang Public License, - * Version 1.1, (the "License"); you may not use this file except in - * compliance with the License. You should have received a copy of the - * Erlang Public License along with this software. If not, it can be - * retrieved online at http://www.erlang.org/. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Software distributed under the License is distributed on an "AS IS" - * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See - * the License for the specific language governing rights and limitations - * under the License. + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. * * %CopyrightEnd% */ @@ -31,6 +32,7 @@ #include "big.h" #include "beam_load.h" #include "erl_binary.h" +#include "erl_map.h" #include "erl_bits.h" #include "dist.h" #include "beam_bp.h" @@ -48,7 +50,7 @@ # define OpCase(OpCode) case op_##OpCode # define CountCase(OpCode) case op_count_##OpCode # define OpCode(OpCode) ((Uint*)op_##OpCode) -# define Goto(Rel) {Go = (int)(Rel); goto emulator_loop;} +# define Goto(Rel) {Go = (int)(UWord)(Rel); goto emulator_loop;} # define LabelAddr(Addr) &&##Addr #else # define OpCase(OpCode) lb_##OpCode @@ -70,7 +72,8 @@ do { \ ERTS_SMP_LC_ASSERT(!erts_thr_progress_is_blocking()); \ } while (0) # define ERTS_SMP_REQ_PROC_MAIN_LOCK(P) \ - if ((P)) erts_proc_lc_require_lock((P), ERTS_PROC_LOCK_MAIN) + if ((P)) erts_proc_lc_require_lock((P), ERTS_PROC_LOCK_MAIN,\ + __FILE__, __LINE__) # define ERTS_SMP_UNREQ_PROC_MAIN_LOCK(P) \ if ((P)) erts_proc_lc_unrequire_lock((P), ERTS_PROC_LOCK_MAIN) # else @@ -133,7 +136,7 @@ do { \ /* We don't check the range if an ordinary switch is used */ #ifdef NO_JUMP_TABLE -#define VALID_INSTR(IP) (0 <= (int)(IP) && ((int)(IP) < (NUMBER_OF_OPCODES*2+10))) +#define VALID_INSTR(IP) ((UWord)(IP) < (NUMBER_OF_OPCODES*2+10)) #else #define VALID_INSTR(IP) \ ((SWord)LabelAddr(emulator_loop) <= (SWord)(IP) && \ @@ -217,6 +220,7 @@ BeamInstr beam_continue_exit[1]; BeamInstr* em_call_error_handler; BeamInstr* em_apply_bif; +BeamInstr* em_call_nif; /* NOTE These should be the only variables containing trace instructions. @@ -238,10 +242,6 @@ BeamInstr beam_return_time_trace[1]; /* OpCode(i_return_time_trace) */ void** beam_ops; #endif -#ifndef ERTS_SMP /* Not supported with smp emulator */ -extern int count_instructions; -#endif - #define SWAPIN \ HTOP = HEAP_TOP(c_p); \ E = c_p->stop @@ -563,7 +563,8 @@ extern int count_instructions; Store(term, Dst); \ } while (0) -#define Move2(src1, dst1, src2, dst2) dst1 = (src1); dst2 = (src2) +#define Move2(S1, D1, S2, D2) D1 = (S1); D2 = (S2) +#define Move3(S1, D1, S2, D2, S3, D3) D1 = (S1); D2 = (S2); D3 = (S3) #define MoveGenDest(src, dstp) \ if ((dstp) == NULL) { r(0) = (src); } else { *(dstp) = src; } @@ -663,6 +664,9 @@ extern int count_instructions; #define EqualImmed(X, Y, Action) if (X != Y) { Action; } #define NotEqualImmed(X, Y, Action) if (X == Y) { Action; } +#define EqualExact(X, Y, Action) if (!EQ(X,Y)) { Action; } +#define IsLessThan(X, Y, Action) if (CMP_GE(X, Y)) { Action; } +#define IsGreaterEqual(X, Y, Action) if (CMP_LT(X, Y)) { Action; } #define IsFloat(Src, Fail) if (is_not_float(Src)) { Fail; } @@ -700,6 +704,26 @@ extern int count_instructions; Fail; \ } +#define IsMap(Src, Fail) if (!is_map(Src)) { Fail; } + +#define GetMapElement(Src, Key, Dst, Fail) \ + do { \ + Eterm _res = get_map_element(Src, Key); \ + if (is_non_value(_res)) { \ + Fail; \ + } \ + Dst = _res; \ + } while (0) + +#define GetMapElementHash(Src, Key, Hx, Dst, Fail) \ + do { \ + Eterm _res = get_map_element_hash(Src, Key, Hx); \ + if (is_non_value(_res)) { \ + Fail; \ + } \ + Dst = _res; \ + } while (0) + #define IsFunction(X, Action) \ do { \ if ( !(is_any_fun(X)) ) { \ @@ -924,6 +948,7 @@ extern int count_instructions; # define NOINLINE #endif + /* * The following functions are called directly by process_main(). * Don't inline them. @@ -942,7 +967,13 @@ static BeamInstr* apply_fun(Process* p, Eterm fun, Eterm args, Eterm* reg) NOINLINE; static Eterm new_fun(Process* p, Eterm* reg, ErlFunEntry* fe, int num_free) NOINLINE; - +static Eterm new_map(Process* p, Eterm* reg, BeamInstr* I) NOINLINE; +static Eterm update_map_assoc(Process* p, Eterm* reg, + Eterm map, BeamInstr* I) NOINLINE; +static Eterm update_map_exact(Process* p, Eterm* reg, + Eterm map, BeamInstr* I) NOINLINE; +static Eterm get_map_element(Eterm map, Eterm key); +static Eterm get_map_element_hash(Eterm map, Eterm key, Uint32 hx); /* * Functions not directly called by process_main(). OK to inline. @@ -1058,16 +1089,32 @@ init_emulator(void) DTRACE2(nif_return, process_name, mfa); \ } -#else /* USE_VM_PROBES */ - -#define DTRACE_LOCAL_CALL(p, m, f, a) do {} while (0) -#define DTRACE_GLOBAL_CALL(p, m, f, a) do {} while (0) -#define DTRACE_RETURN(p, m, f, a) do {} while (0) -#define DTRACE_BIF_ENTRY(p, m, f, a) do {} while (0) -#define DTRACE_BIF_RETURN(p, m, f, a) do {} while (0) -#define DTRACE_NIF_ENTRY(p, m, f, a) do {} while (0) -#define DTRACE_NIF_RETURN(p, m, f, a) do {} while (0) +#define DTRACE_GLOBAL_CALL_FROM_EXPORT(p,e) \ + do { \ + if (DTRACE_ENABLED(global_function_entry)) { \ + BeamInstr* fp = (BeamInstr *) (((Export *) (e))->addressv[erts_active_code_ix()]); \ + DTRACE_GLOBAL_CALL((p), (Eterm)fp[-3], (Eterm)fp[-2], fp[-1]); \ + } \ + } while(0) + +#define DTRACE_RETURN_FROM_PC(p) \ + do { \ + BeamInstr* fp; \ + if (DTRACE_ENABLED(function_return) && (fp = find_function_from_pc((p)->cp))) { \ + DTRACE_RETURN((p), (Eterm)fp[0], (Eterm)fp[1], (Uint)fp[2]); \ + } \ + } while(0) +#else /* USE_VM_PROBES */ +#define DTRACE_LOCAL_CALL(p, m, f, a) do {} while (0) +#define DTRACE_GLOBAL_CALL(p, m, f, a) do {} while (0) +#define DTRACE_GLOBAL_CALL_FROM_EXPORT(p, e) do {} while (0) +#define DTRACE_RETURN(p, m, f, a) do {} while (0) +#define DTRACE_RETURN_FROM_PC(p) do {} while (0) +#define DTRACE_BIF_ENTRY(p, m, f, a) do {} while (0) +#define DTRACE_BIF_RETURN(p, m, f, a) do {} while (0) +#define DTRACE_NIF_ENTRY(p, m, f, a) do {} while (0) +#define DTRACE_NIF_RETURN(p, m, f, a) do {} while (0) #endif /* USE_VM_PROBES */ /* @@ -1140,19 +1187,23 @@ void process_main(void) Eterm (*arith_func)(Process* p, Eterm* reg, Uint live); -#ifndef NO_JUMP_TABLE - static void* opcodes[] = { DEFINE_OPCODES }; #ifdef ERTS_OPCODE_COUNTER_SUPPORT static void* counting_opcodes[] = { DEFINE_COUNTING_OPCODES }; -#endif +#else +#ifndef NO_JUMP_TABLE + static void* opcodes[] = { DEFINE_OPCODES }; #else int Go; #endif +#endif Uint temp_bits; /* Temporary used by BsSkipBits2 & BsGetInteger2 */ Eterm pt_arity; /* Used by do_put_tuple */ + Uint64 start_time = 0; /* Monitor long schedule */ + BeamInstr* start_time_i = NULL; + ERL_BITS_DECLARE_STATEP; /* Has to be last declaration */ @@ -1165,16 +1216,35 @@ void process_main(void) * c_p->arg_reg before calling the scheduler. */ if (!init_done) { + /* This should only be reached during the init phase when only the main + * process is running. I.e. there is no race for init_done. + */ init_done = 1; goto init_emulator; } + c_p = NULL; reds_used = 0; + goto do_schedule1; do_schedule: reds_used = REDS_IN(c_p) - FCALLS; do_schedule1: + + if (start_time != 0) { + Sint64 diff = erts_timestamp_millis() - start_time; + if (diff > 0 && (Uint) diff > erts_system_monitor_long_schedule +#ifdef ERTS_DIRTY_SCHEDULERS + && !ERTS_SCHEDULER_IS_DIRTY(c_p->scheduler_data) +#endif + ) { + BeamInstr *inptr = find_function_from_pc(start_time_i); + BeamInstr *outptr = find_function_from_pc(c_p->i); + monitor_long_schedule_proc(c_p,inptr,outptr,(Uint) diff); + } + } + PROCESS_MAIN_CHK_LOCKS(c_p); ERTS_SMP_UNREQ_PROC_MAIN_LOCK(c_p); #if HALFWORD_HEAP @@ -1183,11 +1253,18 @@ void process_main(void) ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p); c_p = schedule(c_p, reds_used); ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p); + start_time = 0; #ifdef DEBUG pid = c_p->common.id; /* Save for debugging purpouses */ #endif ERTS_SMP_REQ_PROC_MAIN_LOCK(c_p); PROCESS_MAIN_CHK_LOCKS(c_p); + + if (erts_system_monitor_long_schedule != 0) { + start_time = erts_timestamp_millis(); + start_time_i = c_p->i; + } + reg = ERTS_PROC_GET_SCHDATA(c_p)->x_reg_array; freg = ERTS_PROC_GET_SCHDATA(c_p)->f_reg_array; #if !HEAP_ON_C_STACK @@ -1249,7 +1326,7 @@ void process_main(void) (Eterm)fptr[1], (Uint)fptr[2], NULL, fun_buf); } else { - erts_snprintf(fun_buf, sizeof(fun_buf), + erts_snprintf(fun_buf, sizeof(DTRACE_CHARBUF_NAME(fun_buf)), "<unknown/%p>", next); } } @@ -1317,7 +1394,39 @@ void process_main(void) ASSERT(c_p->freason != BADMATCH || is_value(c_p->fvalue)); goto find_func_info; } - + +#define DO_BIG_ARITH(Func,Arg1,Arg2) \ + do { \ + Uint live = Arg(1); \ + SWAPOUT; \ + reg[0] = r(0); \ + reg[live] = (Arg1); \ + reg[live+1] = (Arg2); \ + result = (Func)(c_p, reg, live); \ + r(0) = reg[0]; \ + SWAPIN; \ + ERTS_HOLE_CHECK(c_p); \ + if (is_value(result)) { \ + StoreBifResult(4,result); \ + } \ + goto lb_Cl_error; \ + } while(0) + + OpCase(i_plus_jIxxd): + { + Eterm result; + + if (is_both_small(xb(Arg(2)), xb(Arg(3)))) { + Sint i = signed_val(xb(Arg(2))) + signed_val(xb(Arg(3))); + ASSERT(MY_IS_SSMALL(i) == IS_SSMALL(i)); + if (MY_IS_SSMALL(i)) { + result = make_small(i); + StoreBifResult(4, result); + } + } + DO_BIG_ARITH(ARITH_FUNC(mixed_plus), xb(Arg(2)), xb(Arg(3))); + } + OpCase(i_plus_jId): { Eterm result; @@ -1329,12 +1438,26 @@ void process_main(void) result = make_small(i); STORE_ARITH_RESULT(result); } - } arith_func = ARITH_FUNC(mixed_plus); goto do_big_arith2; } + OpCase(i_minus_jIxxd): + { + Eterm result; + + if (is_both_small(xb(Arg(2)), xb(Arg(3)))) { + Sint i = signed_val(xb(Arg(2))) - signed_val(xb(Arg(3))); + ASSERT(MY_IS_SSMALL(i) == IS_SSMALL(i)); + if (MY_IS_SSMALL(i)) { + result = make_small(i); + StoreBifResult(4, result); + } + } + DO_BIG_ARITH(ARITH_FUNC(mixed_minus), xb(Arg(2)), xb(Arg(3))); + } + OpCase(i_minus_jId): { Eterm result; @@ -1427,6 +1550,52 @@ void process_main(void) Next(2); } + OpCase(move_window3_xxxy): { + BeamInstr *next; + Eterm xt0, xt1, xt2; + Eterm *y = (Eterm *)(((unsigned char *)E) + (Arg(3))); + PreFetch(4, next); + xt0 = xb(Arg(0)); + xt1 = xb(Arg(1)); + xt2 = xb(Arg(2)); + y[0] = xt0; + y[1] = xt1; + y[2] = xt2; + NextPF(4, next); + } + OpCase(move_window4_xxxxy): { + BeamInstr *next; + Eterm xt0, xt1, xt2, xt3; + Eterm *y = (Eterm *)(((unsigned char *)E) + (Arg(4))); + PreFetch(5, next); + xt0 = xb(Arg(0)); + xt1 = xb(Arg(1)); + xt2 = xb(Arg(2)); + xt3 = xb(Arg(3)); + y[0] = xt0; + y[1] = xt1; + y[2] = xt2; + y[3] = xt3; + NextPF(5, next); + } + OpCase(move_window5_xxxxxy): { + BeamInstr *next; + Eterm xt0, xt1, xt2, xt3, xt4; + Eterm *y = (Eterm *)(((unsigned char *)E) + (Arg(5))); + PreFetch(6, next); + xt0 = xb(Arg(0)); + xt1 = xb(Arg(1)); + xt2 = xb(Arg(2)); + xt3 = xb(Arg(3)); + xt4 = xb(Arg(4)); + y[0] = xt0; + y[1] = xt1; + y[2] = xt2; + y[3] = xt3; + y[4] = xt4; + NextPF(6, next); + } + OpCase(i_move_call_only_fcr): { r(0) = Arg(1); } @@ -1474,12 +1643,7 @@ void process_main(void) * is not loaded, it points to code which will invoke the error handler * (see lb_call_error_handler below). */ -#ifdef USE_VM_CALL_PROBES - if (DTRACE_ENABLED(global_function_entry)) { - BeamInstr* fp = (BeamInstr *) (((Export *) Arg(0))->addressv[erts_active_code_ix()]); - DTRACE_GLOBAL_CALL(c_p, (Eterm)fp[-3], (Eterm)fp[-2], fp[-1]); - } -#endif + DTRACE_GLOBAL_CALL_FROM_EXPORT(c_p, Arg(0)); Dispatchx(); OpCase(i_move_call_ext_cre): { @@ -1489,12 +1653,7 @@ void process_main(void) /* FALL THROUGH */ OpCase(i_call_ext_e): SET_CP(c_p, I+2); -#ifdef USE_VM_CALL_PROBES - if (DTRACE_ENABLED(global_function_entry)) { - BeamInstr* fp = (BeamInstr *) (((Export *) Arg(0))->addressv[erts_active_code_ix()]); - DTRACE_GLOBAL_CALL(c_p, (Eterm)fp[-3], (Eterm)fp[-2], fp[-1]); - } -#endif + DTRACE_GLOBAL_CALL_FROM_EXPORT(c_p, Arg(0)); Dispatchx(); OpCase(i_move_call_ext_only_ecr): { @@ -1502,12 +1661,7 @@ void process_main(void) } /* FALL THROUGH */ OpCase(i_call_ext_only_e): -#ifdef USE_VM_CALL_PROBES - if (DTRACE_ENABLED(global_function_entry)) { - BeamInstr* fp = (BeamInstr *) (((Export *) Arg(0))->addressv[erts_active_code_ix()]); - DTRACE_GLOBAL_CALL(c_p, (Eterm)fp[-3], (Eterm)fp[-2], fp[-1]); - } -#endif + DTRACE_GLOBAL_CALL_FROM_EXPORT(c_p, Arg(0)); Dispatchx(); OpCase(init_y): { @@ -1541,18 +1695,9 @@ void process_main(void) Next(1); } - OpCase(return): { -#ifdef USE_VM_CALL_PROBES - BeamInstr* fptr; -#endif SET_I(c_p->cp); - -#ifdef USE_VM_CALL_PROBES - if (DTRACE_ENABLED(function_return) && (fptr = find_function_from_pc(c_p->cp))) { - DTRACE_RETURN(c_p, (Eterm)fptr[0], (Eterm)fptr[1], (Uint)fptr[2]); - } -#endif + DTRACE_RETURN_FROM_PC(c_p); /* * We must clear the CP to make sure that a stale value do not * create a false module dependcy preventing code upgrading. @@ -1975,44 +2120,32 @@ void process_main(void) } GetArg1(1, timeout_value); if (timeout_value != make_small(0)) { -#if !defined(ARCH_64) || HALFWORD_HEAP - Uint time_val; -#endif - if (is_small(timeout_value) && signed_val(timeout_value) > 0 && -#if defined(ARCH_64) && !HALFWORD_HEAP - ((unsigned_val(timeout_value) >> 32) == 0) -#else - 1 -#endif - ) { - /* - * The timer routiner will set c_p->i to the value in - * c_p->def_arg_reg[0]. Note that it is safe to use this - * location because there are no living x registers in - * a receive statement. - * Note that for the halfword emulator, the two first elements - * of the array are used. - */ - BeamInstr** pi = (BeamInstr**) c_p->def_arg_reg; - *pi = I+3; - set_timer(c_p, unsigned_val(timeout_value)); - } else if (timeout_value == am_infinity) { + if (timeout_value == am_infinity) c_p->flags |= F_TIMO; -#if !defined(ARCH_64) || HALFWORD_HEAP - } else if (term_to_Uint(timeout_value, &time_val)) { - BeamInstr** pi = (BeamInstr**) c_p->def_arg_reg; - *pi = I+3; - set_timer(c_p, time_val); -#endif - } else { /* Wrong time */ - OpCase(i_wait_error_locked): { - erts_smp_proc_unlock(c_p, ERTS_PROC_LOCKS_MSG_RECEIVE); - /* Fall through */ + else { + int tres = erts_set_proc_timer_term(c_p, timeout_value); + if (tres == 0) { + /* + * The timer routiner will set c_p->i to the value in + * c_p->def_arg_reg[0]. Note that it is safe to use this + * location because there are no living x registers in + * a receive statement. + * Note that for the halfword emulator, the two first elements + * of the array are used. + */ + BeamInstr** pi = (BeamInstr**) c_p->def_arg_reg; + *pi = I+3; } + else { /* Wrong time */ + OpCase(i_wait_error_locked): { + erts_smp_proc_unlock(c_p, ERTS_PROC_LOCKS_MSG_RECEIVE); + /* Fall through */ + } OpCase(i_wait_error): { - c_p->freason = EXC_TIMEOUT_VALUE; - goto find_func_info; + c_p->freason = EXC_TIMEOUT_VALUE; + goto find_func_info; + } } } @@ -2030,6 +2163,22 @@ void process_main(void) OpCase(wait_f): wait2: { +#ifndef ERTS_SMP + if (ERTS_PROC_IS_EXITING(c_p)) { + /* + * I non smp case: + * + * Currently executing process might be sent an exit + * signal if it is traced by a port that it also is + * linked to, and the port terminates during the + * trace. In this case we do *not* want to clear + * the active flag, which will make the process hang + * in limbo forever. + */ + SWAPOUT; + goto do_schedule; + } +#endif c_p->i = (BeamInstr *) Arg(0); /* L1 */ SWAPOUT; c_p->arity = 0; @@ -2061,7 +2210,7 @@ void process_main(void) if ((c_p->flags & (F_INSLPQUEUE | F_TIMO)) == 0) { BeamInstr** p = (BeamInstr **) c_p->def_arg_reg; *p = I+3; - set_timer(c_p, Arg(1)); + erts_set_proc_timer_uword(c_p, Arg(1)); } goto wait2; } @@ -2089,19 +2238,18 @@ void process_main(void) NextPF(0, next); } - { Eterm select_val2; - OpCase(i_select_tuple_arity2_yfAfAf): + OpCase(i_select_tuple_arity2_yfAAff): select_val2 = yb(Arg(0)); goto do_select_tuple_arity2; - OpCase(i_select_tuple_arity2_xfAfAf): + OpCase(i_select_tuple_arity2_xfAAff): select_val2 = xb(Arg(0)); goto do_select_tuple_arity2; - OpCase(i_select_tuple_arity2_rfAfAf): + OpCase(i_select_tuple_arity2_rfAAff): select_val2 = r(0); I--; @@ -2112,22 +2260,22 @@ void process_main(void) select_val2 = *tuple_val(select_val2); goto do_select_val2; - OpCase(i_select_val2_yfcfcf): + OpCase(i_select_val2_yfccff): select_val2 = yb(Arg(0)); goto do_select_val2; - OpCase(i_select_val2_xfcfcf): + OpCase(i_select_val2_xfccff): select_val2 = xb(Arg(0)); goto do_select_val2; - OpCase(i_select_val2_rfcfcf): + OpCase(i_select_val2_rfccff): select_val2 = r(0); I--; do_select_val2: if (select_val2 == Arg(2)) { - I += 2; - } else if (select_val2 == Arg(4)) { + I += 3; + } else if (select_val2 == Arg(3)) { I += 4; } @@ -2154,20 +2302,50 @@ void process_main(void) do_select_tuple_arity: if (is_tuple(select_val)) { select_val = *tuple_val(select_val); - goto do_binary_search; + goto do_linear_search; + } + SET_I((BeamInstr *) Arg(1)); + Goto(*I); + + OpCase(i_select_val_lins_xfI): + select_val = xb(Arg(0)); + goto do_linear_search; + + OpCase(i_select_val_lins_yfI): + select_val = yb(Arg(0)); + goto do_linear_search; + + OpCase(i_select_val_lins_rfI): + select_val = r(0); + I--; + + do_linear_search: { + BeamInstr *vs = &Arg(3); + int ix = 0; + + for(;;) { + if (vs[ix+0] >= select_val) { ix += 0; break; } + if (vs[ix+1] >= select_val) { ix += 1; break; } + ix += 2; + } + + if (vs[ix] == select_val) { + I += ix + Arg(2) + 2; } + SET_I((BeamInstr *) Arg(1)); Goto(*I); + } - OpCase(i_select_val_xfI): + OpCase(i_select_val_bins_xfI): select_val = xb(Arg(0)); goto do_binary_search; - OpCase(i_select_val_yfI): + OpCase(i_select_val_bins_yfI): select_val = yb(Arg(0)); goto do_binary_search; - OpCase(i_select_val_rfI): + OpCase(i_select_val_bins_rfI): select_val = r(0); I--; @@ -2301,6 +2479,142 @@ void process_main(void) Goto(*I); } + OpCase(new_map_dII): { + Eterm res; + + x(0) = r(0); + SWAPOUT; + res = new_map(c_p, reg, I-1); + SWAPIN; + r(0) = x(0); + StoreResult(res, Arg(0)); + Next(3+Arg(2)); + } + +#define PUT_TERM_REG(term, desc) \ +do { \ + switch ((desc) & _TAG_IMMED1_MASK) { \ + case (R_REG_DEF << _TAG_PRIMARY_SIZE) | TAG_PRIMARY_HEADER: \ + r(0) = (term); \ + break; \ + case (X_REG_DEF << _TAG_PRIMARY_SIZE) | TAG_PRIMARY_HEADER: \ + x((desc) >> _TAG_IMMED1_SIZE) = (term); \ + break; \ + case (Y_REG_DEF << _TAG_PRIMARY_SIZE) | TAG_PRIMARY_HEADER: \ + y((desc) >> _TAG_IMMED1_SIZE) = (term); \ + break; \ + default: \ + ASSERT(0); \ + break; \ + } \ +} while(0) + + OpCase(i_get_map_elements_fsI): { + Eterm map; + BeamInstr *fs; + Uint sz, n; + + GetArg1(1, map); + + /* this instruction assumes Arg1 is a map, + * i.e. that it follows a test is_map if needed. + */ + + n = (Uint)Arg(2) / 3; + fs = &Arg(3); /* pattern fields and target registers */ + + if (is_flatmap(map)) { + flatmap_t *mp; + Eterm *ks; + Eterm *vs; + + mp = (flatmap_t *)flatmap_val(map); + sz = flatmap_get_size(mp); + + if (sz == 0) { + ClauseFail(); + } + + ks = flatmap_get_keys(mp); + vs = flatmap_get_values(mp); + + while(sz) { + if (EQ((Eterm) fs[0], *ks)) { + PUT_TERM_REG(*vs, fs[1]); + n--; + fs += 3; + /* no more values to fetch, we are done */ + if (n == 0) { + I = fs; + Next(-1); + } + } + ks++, sz--, vs++; + } + + ClauseFail(); + } else { + const Eterm *v; + Uint32 hx; + ASSERT(is_hashmap(map)); + while(n--) { + hx = fs[2]; + ASSERT(hx == hashmap_make_hash((Eterm)fs[0])); + if ((v = erts_hashmap_get(hx, (Eterm)fs[0], map)) == NULL) { + ClauseFail(); + } + PUT_TERM_REG(*v, fs[1]); + fs += 3; + } + I = fs; + Next(-1); + } + } +#undef PUT_TERM_REG + + OpCase(update_map_assoc_jsdII): { + Eterm res; + Eterm map; + + GetArg1(1, map); + x(0) = r(0); + SWAPOUT; + res = update_map_assoc(c_p, reg, map, I); + SWAPIN; + if (is_value(res)) { + r(0) = x(0); + StoreResult(res, Arg(2)); + Next(5+Arg(4)); + } else { + /* + * This can only happen if the code was compiled + * with the compiler in OTP 17. + */ + c_p->freason = BADMAP; + c_p->fvalue = map; + goto lb_Cl_error; + } + } + + OpCase(update_map_exact_jsdII): { + Eterm res; + Eterm map; + + GetArg1(1, map); + x(0) = r(0); + SWAPOUT; + res = update_map_exact(c_p, reg, map, I); + SWAPIN; + if (is_value(res)) { + r(0) = x(0); + StoreResult(res, Arg(2)); + Next(5+Arg(4)); + } else { + goto lb_Cl_error; + } + } + + /* * All guards with zero arguments have special instructions: * self/0 @@ -2554,6 +2868,7 @@ void process_main(void) } PreFetch(1, next); ASSERT(!ERTS_PROC_IS_EXITING(c_p)); + ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p); reg[0] = r(0); result = (*bf)(c_p, reg, I); ASSERT(!ERTS_PROC_IS_EXITING(c_p) || is_non_value(result)); @@ -2621,6 +2936,19 @@ void process_main(void) goto do_big_arith2; } + OpCase(i_rem_jIxxd): + { + Eterm result; + + if (xb(Arg(3)) == SMALL_ZERO) { + goto badarith; + } else if (is_both_small(xb(Arg(2)), xb(Arg(3)))) { + result = make_small(signed_val(xb(Arg(2))) % signed_val(xb(Arg(3)))); + StoreBifResult(4, result); + } + DO_BIG_ARITH(ARITH_FUNC(int_rem),xb(Arg(2)),xb(Arg(3))); + } + OpCase(i_rem_jId): { Eterm result; @@ -2636,6 +2964,20 @@ void process_main(void) } } + OpCase(i_band_jIxcd): + { + Eterm result; + + if (is_both_small(xb(Arg(2)), Arg(3))) { + /* + * No need to untag -- TAG & TAG == TAG. + */ + result = xb(Arg(2)) & Arg(3); + StoreBifResult(4, result); + } + DO_BIG_ARITH(ARITH_FUNC(band),xb(Arg(2)),Arg(3)); + } + OpCase(i_band_jId): { Eterm result; @@ -2651,6 +2993,8 @@ void process_main(void) goto do_big_arith2; } +#undef DO_BIG_ARITH + do_big_arith2: { Eterm result; @@ -3282,6 +3626,7 @@ void process_main(void) * I[0]: &&call_nif * I[1]: Function pointer to NIF function * I[2]: Pointer to erl_module_nif + * I[3]: Function pointer to dirty NIF */ BifFunction vbf; @@ -3301,6 +3646,8 @@ void process_main(void) erts_pre_nif(&env, c_p, (struct erl_module_nif*)I[2]); reg[0] = r(0); nif_bif_result = (*fp)(&env, bif_nif_arity, reg); + if (env.exception_thrown) + nif_bif_result = THE_NON_VALUE; erts_post_nif(&env); } ASSERT(!ERTS_PROC_IS_EXITING(c_p) || is_non_value(nif_bif_result)); @@ -3334,7 +3681,7 @@ void process_main(void) vbf = (BifFunction) Arg(0); PROCESS_MAIN_CHK_LOCKS(c_p); bif_nif_arity = I[-1]; - ASSERT(bif_nif_arity <= 3); + ASSERT(bif_nif_arity <= 4); ERTS_SMP_UNREQ_PROC_MAIN_LOCK(c_p); reg[0] = r(0); { @@ -3560,8 +3907,6 @@ void process_main(void) * Allocate the binary struct itself. */ bptr = erts_bin_nrml_alloc(num_bytes); - bptr->flags = 0; - bptr->orig_size = num_bytes; erts_refc_init(&bptr->refc, 1); erts_current_bin = (byte *) bptr->orig_bytes; @@ -3661,8 +4006,6 @@ void process_main(void) * Allocate the binary struct itself. */ bptr = erts_bin_nrml_alloc(tmp_arg1); - bptr->flags = 0; - bptr->orig_size = tmp_arg1; erts_refc_init(&bptr->refc, 1); erts_current_bin = (byte *) bptr->orig_bytes; @@ -3726,7 +4069,7 @@ void process_main(void) tmp_arg1 += Arg1; store_bs_add_result: - if (MY_IS_SSMALL((Sint) tmp_arg1)) { + if (tmp_arg1 <= MAX_SMALL) { tmp_arg1 = make_small(tmp_arg1); } else { /* @@ -4305,7 +4648,19 @@ void process_main(void) flags = Arg(2); BsGetFieldSize(tmp_arg2, (flags >> 3), ClauseFail(), size); if (size >= SMALL_BITS) { - Uint wordsneeded = 1+WSIZE(NBYTES((Uint) size)); + Uint wordsneeded; + /* check bits size before potential gc. + * We do not want a gc and then realize we don't need + * the allocated space (i.e. if the op fails) + * + * remember to reacquire the matchbuffer after gc. + */ + + mb = ms_matchbuffer(tmp_arg1); + if (mb->size - mb->offset < size) { + ClauseFail(); + } + wordsneeded = 1+WSIZE(NBYTES((Uint) size)); TestHeapPreserve(wordsneeded, Arg(1), tmp_arg1); } mb = ms_matchbuffer(tmp_arg1); @@ -4663,7 +5018,7 @@ void process_main(void) #ifdef NO_FPE_SIGNALS OpCase(fclearerror): OpCase(i_fcheckerror): - erl_exit(1, "fclearerror/i_fcheckerror without fpe signals (beam_emu)"); + erts_exit(ERTS_ERROR_EXIT, "fclearerror/i_fcheckerror without fpe signals (beam_emu)"); # define ERTS_NO_FPE_CHECK_INIT ERTS_FP_CHECK_INIT # define ERTS_NO_FPE_ERROR ERTS_FP_ERROR #else @@ -4753,14 +5108,14 @@ void process_main(void) * ... remainder of original BEAM code */ ASSERT(I[-5] == (Uint) OpCode(i_func_info_IaaI)); - c_p->hipe.ncallee = (void(*)(void)) I[-4]; + c_p->hipe.u.ncallee = (void(*)(void)) I[-4]; cmd = HIPE_MODE_SWITCH_CMD_CALL | (I[-1] << 8); ++hipe_trap_count; goto L_hipe_mode_switch; } OpCase(hipe_trap_call_closure): { ASSERT(I[-5] == (Uint) OpCode(i_func_info_IaaI)); - c_p->hipe.ncallee = (void(*)(void)) I[-4]; + c_p->hipe.u.ncallee = (void(*)(void)) I[-4]; cmd = HIPE_MODE_SWITCH_CMD_CALL_CLOSURE | (I[-1] << 8); ++hipe_trap_count; goto L_hipe_mode_switch; @@ -4794,7 +5149,10 @@ void process_main(void) case HIPE_MODE_SWITCH_RES_RETURN: ASSERT(is_value(reg[0])); MoveReturn(reg[0], r(0)); - case HIPE_MODE_SWITCH_RES_CALL: + case HIPE_MODE_SWITCH_RES_CALL_EXPORTED: + c_p->i = c_p->hipe.u.callee_exp->addressv[erts_active_code_ix()]; + /*fall through*/ + case HIPE_MODE_SWITCH_RES_CALL_BEAM: SET_I(c_p->i); r(0) = reg[0]; Dispatch(); @@ -4818,7 +5176,7 @@ void process_main(void) I = handle_error(c_p, I, reg, NULL); goto post_error_handling; default: - erl_exit(1, "hipe_mode_switch: result %u\n", c_p->def_arg_reg[3]); + erts_exit(ERTS_ERROR_EXIT, "hipe_mode_switch: result %u\n", c_p->def_arg_reg[3]); } } OpCase(hipe_call_count): { @@ -4898,7 +5256,7 @@ void process_main(void) OpCase(label_L): OpCase(on_load): OpCase(line_I): - erl_exit(1, "meta op\n"); + erts_exit(ERTS_ERROR_EXIT, "meta op\n"); /* * One-time initialization of Beam emulator. @@ -4911,26 +5269,19 @@ void process_main(void) #ifndef NO_JUMP_TABLE #ifdef ERTS_OPCODE_COUNTER_SUPPORT - - /* Are tables correctly generated by beam_makeops? */ - ASSERT(sizeof(counting_opcodes) == sizeof(opcodes)); - - if (count_instructions) { #ifdef DEBUG - counting_opcodes[op_catch_end_y] = LabelAddr(lb_catch_end_y); + counting_opcodes[op_catch_end_y] = LabelAddr(lb_catch_end_y); #endif - counting_opcodes[op_i_func_info_IaaI] = LabelAddr(lb_i_func_info_IaaI); - beam_ops = counting_opcodes; - } - else -#endif /* #ifndef ERTS_OPCODE_COUNTER_SUPPORT */ - { - beam_ops = opcodes; - } + counting_opcodes[op_i_func_info_IaaI] = LabelAddr(lb_i_func_info_IaaI); + beam_ops = counting_opcodes; +#else /* #ifndef ERTS_OPCODE_COUNTER_SUPPORT */ + beam_ops = opcodes; +#endif /* ERTS_OPCODE_COUNTER_SUPPORT */ #endif /* NO_JUMP_TABLE */ em_call_error_handler = OpCode(call_error_handler); em_apply_bif = OpCode(apply_bif); + em_call_nif = OpCode(call_nif); beam_apply[0] = (BeamInstr) OpCode(i_apply); beam_apply[1] = (BeamInstr) OpCode(normal_exit); @@ -4959,7 +5310,7 @@ void process_main(void) } #ifdef NO_JUMP_TABLE default: - erl_exit(1, "unexpected op code %d\n",Go); + erts_exit(ERTS_ERROR_EXIT, "unexpected op code %d\n",Go); } #endif return; /* Never executed */ @@ -4989,6 +5340,8 @@ translate_gc_bif(void* gcf) return bit_size_1; } else if (gcf == erts_gc_byte_size_1) { return byte_size_1; + } else if (gcf == erts_gc_map_size_1) { + return map_size_1; } else if (gcf == erts_gc_abs_1) { return abs_1; } else if (gcf == erts_gc_float_1) { @@ -5002,7 +5355,7 @@ translate_gc_bif(void* gcf) } else if (gcf == erts_gc_binary_part_3) { return binary_part_3; } else { - erl_exit(1, "bad gc bif"); + erts_exit(ERTS_ERROR_EXIT, "bad gc bif"); } } @@ -5036,7 +5389,9 @@ Eterm error_atom[NUMBER_EXIT_CODES] = { am_notalive, /* 14 */ am_system_limit, /* 15 */ am_try_clause, /* 16 */ - am_notsup /* 17 */ + am_notsup, /* 17 */ + am_badmap, /* 18 */ + am_badkey, /* 19 */ }; /* @@ -5065,7 +5420,7 @@ handle_error(Process* c_p, BeamInstr* pc, Eterm* reg, BifFunction bf) Eterm* hp; Eterm Value = c_p->fvalue; Eterm Args = am_true; - c_p->i = pc; /* In case we call erl_exit(). */ + c_p->i = pc; /* In case we call erts_exit(). */ ASSERT(c_p->freason != TRAP); /* Should have been handled earlier. */ @@ -5129,7 +5484,7 @@ handle_error(Process* c_p, BeamInstr* pc, Eterm* reg, BifFunction bf) c_p->cp = 0; /* To avoid keeping stale references. */ return new_pc; } - if (c_p->catches > 0) erl_exit(1, "Catch not found"); + if (c_p->catches > 0) erts_exit(ERTS_ERROR_EXIT, "Catch not found"); } ERTS_SMP_UNREQ_PROC_MAIN_LOCK(c_p); terminate_proc(c_p, Value); @@ -5239,18 +5594,35 @@ next_catch(Process* c_p, Eterm *reg) { static void terminate_proc(Process* c_p, Eterm Value) { + Eterm *hp; + Eterm Args = NIL; + /* Add a stacktrace if this is an error. */ if (GET_EXC_CLASS(c_p->freason) == EXTAG_ERROR) { Value = add_stacktrace(c_p, Value, c_p->ftrace); } /* EXF_LOG is a primary exception flag */ if (c_p->freason & EXF_LOG) { + int alive = erts_is_alive; erts_dsprintf_buf_t *dsbufp = erts_create_logger_dsbuf(); - erts_dsprintf(dsbufp, "Error in process %T ", c_p->common.id); - if (erts_is_alive) - erts_dsprintf(dsbufp, "on node %T ", erts_this_node->sysname); - erts_dsprintf(dsbufp,"with exit value: %0.*T\n", display_items, Value); - erts_send_error_to_logger(c_p->group_leader, dsbufp); + + /* Build the format message */ + erts_dsprintf(dsbufp, "Error in process ~p "); + if (alive) + erts_dsprintf(dsbufp, "on node ~p "); + erts_dsprintf(dsbufp, "with exit value:~n~p~n"); + + /* Build the args in reverse order */ + hp = HAlloc(c_p, 2); + Args = CONS(hp, Value, Args); + if (alive) { + hp = HAlloc(c_p, 2); + Args = CONS(hp, erts_this_node->sysname, Args); + } + hp = HAlloc(c_p, 2); + Args = CONS(hp, c_p->common.id, Args); + + erts_send_error_term_to_logger(c_p->group_leader, dsbufp, Args); } /* * If we use a shared heap, the process will be garbage-collected. @@ -5292,6 +5664,8 @@ expand_error_value(Process* c_p, Uint freason, Eterm Value) { case (GET_EXC_INDEX(EXC_TRY_CLAUSE)): case (GET_EXC_INDEX(EXC_BADFUN)): case (GET_EXC_INDEX(EXC_BADARITY)): + case (GET_EXC_INDEX(EXC_BADMAP)): + case (GET_EXC_INDEX(EXC_BADKEY)): /* Some common exceptions: value -> {atom, value} */ ASSERT(is_value(Value)); hp = HAlloc(c_p, 3); @@ -5594,7 +5968,7 @@ build_stacktrace(Process* c_p, Eterm exc) { * (e.g. spawn_link(erlang, abs, [1])). */ if (fi.current == NULL) { - erts_set_current_function(&fi, c_p->initial); + erts_set_current_function(&fi, c_p->u.initial); args = am_true; /* Just in case */ } else { args = get_args_from_exc(exc); @@ -5633,7 +6007,6 @@ build_stacktrace(Process* c_p, Eterm exc) { return res; } - static BeamInstr* call_error_handler(Process* p, BeamInstr* fi, Eterm* reg, Eterm func) { @@ -5681,7 +6054,6 @@ call_error_handler(Process* p, BeamInstr* fi, Eterm* reg, Eterm func) return ep->addressv[erts_active_code_ix()]; } - static Export* apply_setup_error_handler(Process* p, Eterm module, Eterm function, Uint arity, Eterm* reg) { @@ -5804,13 +6176,7 @@ apply(Process* p, Eterm module, Eterm function, Eterm args, Eterm* reg) } else if (ERTS_PROC_GET_SAVED_CALLS_BUF(p)) { save_calls(p, ep); } - -#ifdef USE_VM_CALL_PROBES - if (DTRACE_ENABLED(global_function_entry)) { - BeamInstr *fptr = (BeamInstr *) ep->addressv[erts_active_code_ix()]; - DTRACE_GLOBAL_CALL(p, (Eterm)fptr[-3], (Eterm)fptr[-2], (Uint)fptr[-1]); - } -#endif + DTRACE_GLOBAL_CALL_FROM_EXPORT(p, ep); return ep->addressv[erts_active_code_ix()]; } @@ -5859,13 +6225,7 @@ fixed_apply(Process* p, Eterm* reg, Uint arity) } else if (ERTS_PROC_GET_SAVED_CALLS_BUF(p)) { save_calls(p, ep); } - -#ifdef USE_VM_CALL_PROBES - if (DTRACE_ENABLED(global_function_entry)) { - BeamInstr *fptr = (BeamInstr *) ep->addressv[erts_active_code_ix()]; - DTRACE_GLOBAL_CALL(p, (Eterm)fptr[-3], (Eterm)fptr[-2], (Uint)fptr[-1]); - } -#endif + DTRACE_GLOBAL_CALL_FROM_EXPORT(p, ep); return ep->addressv[erts_active_code_ix()]; } @@ -5875,6 +6235,23 @@ erts_hibernate(Process* c_p, Eterm module, Eterm function, Eterm args, Eterm* re int arity; Eterm tmp; +#ifndef ERTS_SMP + if (ERTS_PROC_IS_EXITING(c_p)) { + /* + * I non smp case: + * + * Currently executing process might be sent an exit + * signal if it is traced by a port that it also is + * linked to, and the port terminates during the + * trace. In this case we do *not* want to clear + * the active flag, which will make the process hang + * in limbo forever. Get out of here and terminate + * the process... + */ + return -1; + } +#endif + if (is_not_atom(module) || is_not_atom(function)) { /* * No need to test args here -- done below. @@ -5951,7 +6328,16 @@ erts_hibernate(Process* c_p, Eterm module, Eterm function, Eterm args, Eterm* re ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p); PROCESS_MAIN_CHK_LOCKS(c_p); erts_smp_proc_lock(c_p, ERTS_PROC_LOCK_MSGQ|ERTS_PROC_LOCK_STATUS); -#ifdef ERTS_SMP +#ifndef ERTS_SMP + if (ERTS_PROC_IS_EXITING(c_p)) { + /* + * See comment in the begining of the function... + * + * This second test is needed since gc might be traced. + */ + return -1; + } +#else /* ERTS_SMP */ ERTS_SMP_MSGQ_MV_INQ2PRIVQ(c_p); if (!c_p->msg.len) #endif @@ -6151,6 +6537,7 @@ apply_fun(Process* p, Eterm fun, Eterm args, Eterm* reg) } + static Eterm new_fun(Process* p, Eterm* reg, ErlFunEntry* fe, int num_free) { @@ -6186,7 +6573,529 @@ new_fun(Process* p, Eterm* reg, ErlFunEntry* fe, int num_free) return make_fun(funp); } - +static Eterm get_map_element(Eterm map, Eterm key) +{ + Uint32 hx; + const Eterm *vs; + if (is_flatmap(map)) { + flatmap_t *mp; + Eterm *ks; + Uint i; + Uint n; + + mp = (flatmap_t *)flatmap_val(map); + ks = flatmap_get_keys(mp); + vs = flatmap_get_values(mp); + n = flatmap_get_size(mp); + if (is_immed(key)) { + for (i = 0; i < n; i++) { + if (ks[i] == key) { + return vs[i]; + } + } + } else { + for (i = 0; i < n; i++) { + if (EQ(ks[i], key)) { + return vs[i]; + } + } + } + return THE_NON_VALUE; + } + ASSERT(is_hashmap(map)); + hx = hashmap_make_hash(key); + vs = erts_hashmap_get(hx,key,map); + return vs ? *vs : THE_NON_VALUE; +} + +static Eterm get_map_element_hash(Eterm map, Eterm key, Uint32 hx) +{ + const Eterm *vs; + + if (is_flatmap(map)) { + flatmap_t *mp; + Eterm *ks; + Uint i; + Uint n; + + mp = (flatmap_t *)flatmap_val(map); + ks = flatmap_get_keys(mp); + vs = flatmap_get_values(mp); + n = flatmap_get_size(mp); + if (is_immed(key)) { + for (i = 0; i < n; i++) { + if (ks[i] == key) { + return vs[i]; + } + } + } else { + for (i = 0; i < n; i++) { + if (EQ(ks[i], key)) { + return vs[i]; + } + } + } + return THE_NON_VALUE; + } + + ASSERT(is_hashmap(map)); + ASSERT(hx == hashmap_make_hash(key)); + vs = erts_hashmap_get(hx, key, map); + return vs ? *vs : THE_NON_VALUE; +} + +#define GET_TERM(term, dest) \ +do { \ + Eterm src = (Eterm)(term); \ + switch (src & _TAG_IMMED1_MASK) { \ + case (R_REG_DEF << _TAG_PRIMARY_SIZE) | TAG_PRIMARY_HEADER: \ + dest = x(0); \ + break; \ + case (X_REG_DEF << _TAG_PRIMARY_SIZE) | TAG_PRIMARY_HEADER: \ + dest = x(src >> _TAG_IMMED1_SIZE); \ + break; \ + case (Y_REG_DEF << _TAG_PRIMARY_SIZE) | TAG_PRIMARY_HEADER: \ + dest = y(src >> _TAG_IMMED1_SIZE); \ + break; \ + default: \ + dest = src; \ + break; \ + } \ +} while(0) + + +static Eterm +new_map(Process* p, Eterm* reg, BeamInstr* I) +{ + Uint n = Arg(3); + Uint i; + Uint need = n + 1 /* hdr */ + 1 /*size*/ + 1 /* ptr */ + 1 /* arity */; + Eterm keys; + Eterm *mhp,*thp; + Eterm *E; + BeamInstr *ptr; + flatmap_t *mp; + ErtsHeapFactory factory; + + ptr = &Arg(4); + + if (n > 2*MAP_SMALL_MAP_LIMIT) { + Eterm res; + if (HeapWordsLeft(p) < n) { + erts_garbage_collect(p, n, reg, Arg(2)); + } + + mhp = p->htop; + thp = p->htop; + E = p->stop; + + for (i = 0; i < n/2; i++) { + GET_TERM(*ptr++, *mhp++); + GET_TERM(*ptr++, *mhp++); + } + + p->htop = mhp; + + erts_factory_proc_init(&factory, p); + res = erts_hashmap_from_array(&factory, thp, n/2, 0); + erts_factory_close(&factory); + if (p->mbuf) { + Uint live = Arg(2); + reg[live] = res; + erts_garbage_collect(p, 0, reg, live+1); + res = reg[live]; + E = p->stop; + } + return res; + } + + if (HeapWordsLeft(p) < need) { + erts_garbage_collect(p, need, reg, Arg(2)); + } + + thp = p->htop; + mhp = thp + 1 + n/2; + E = p->stop; + keys = make_tuple(thp); + *thp++ = make_arityval(n/2); + + mp = (flatmap_t *)mhp; mhp += MAP_HEADER_FLATMAP_SZ; + mp->thing_word = MAP_HEADER_FLATMAP; + mp->size = n/2; + mp->keys = keys; + + for (i = 0; i < n/2; i++) { + GET_TERM(*ptr++, *thp++); + GET_TERM(*ptr++, *mhp++); + } + p->htop = mhp; + return make_flatmap(mp); +} + +static Eterm +update_map_assoc(Process* p, Eterm* reg, Eterm map, BeamInstr* I) +{ + Uint n; + Uint num_old; + Uint num_updates; + Uint need; + flatmap_t *old_mp, *mp; + Eterm res; + Eterm* hp; + Eterm* E; + Eterm* old_keys; + Eterm* old_vals; + BeamInstr* new_p; + Eterm new_key; + Eterm* kp; + + new_p = &Arg(5); + num_updates = Arg(4) / 2; + + if (is_not_flatmap(map)) { + Uint32 hx; + Eterm val; + + /* apparently the compiler does not emit is_map instructions, + * bad compiler */ + + if (is_not_hashmap(map)) + return THE_NON_VALUE; + + res = map; + E = p->stop; + while(num_updates--) { + /* assoc can't fail */ + GET_TERM(new_p[0], new_key); + GET_TERM(new_p[1], val); + hx = hashmap_make_hash(new_key); + + res = erts_hashmap_insert(p, hx, new_key, val, res, 0); + if (p->mbuf) { + Uint live = Arg(3); + reg[live] = res; + erts_garbage_collect(p, 0, reg, live+1); + res = reg[live]; + E = p->stop; + } + + new_p += 2; + } + return res; + } + + old_mp = (flatmap_t *) flatmap_val(map); + num_old = flatmap_get_size(old_mp); + + /* + * If the old map is empty, create a new map. + */ + + if (num_old == 0) { + return new_map(p, reg, I+1); + } + + /* + * Allocate heap space for the worst case (i.e. all keys in the + * update list are new). + */ + + need = 2*(num_old+num_updates) + 1 + MAP_HEADER_FLATMAP_SZ; + if (HeapWordsLeft(p) < need) { + Uint live = Arg(3); + reg[live] = map; + erts_garbage_collect(p, need, reg, live+1); + map = reg[live]; + old_mp = (flatmap_t *)flatmap_val(map); + } + + /* + * Build the skeleton for the map, ready to be filled in. + * + * +-----------------------------------+ + * | (Space for aritvyal for keys) | <-----------+ + * +-----------------------------------+ | + * | (Space for key 1) | | <-- kp + * +-----------------------------------+ | + * . | + * . | + * . | + * +-----------------------------------+ | + * | (Space for last key) | | + * +-----------------------------------+ | + * | MAP_HEADER | | + * +-----------------------------------+ | + * | (Space for number of keys/values) | | + * +-----------------------------------+ | + * | Boxed tuple pointer >----------------+ + * +-----------------------------------+ + * | (Space for value 1) | <-- hp + * +-----------------------------------+ + */ + + E = p->stop; + kp = p->htop + 1; /* Point to first key */ + hp = kp + num_old + num_updates; + + res = make_flatmap(hp); + mp = (flatmap_t *)hp; + hp += MAP_HEADER_FLATMAP_SZ; + mp->thing_word = MAP_HEADER_FLATMAP; + mp->keys = make_tuple(kp-1); + + old_vals = flatmap_get_values(old_mp); + old_keys = flatmap_get_keys(old_mp); + + GET_TERM(*new_p, new_key); + n = num_updates; + + /* + * Fill in keys and values, until we run out of either updates + * or old values and keys. + */ + + for (;;) { + Eterm key; + Sint c; + + ASSERT(kp < (Eterm *)mp); + key = *old_keys; + if ((c = CMP_TERM(key, new_key)) < 0) { + /* Copy old key and value */ + *kp++ = key; + *hp++ = *old_vals; + old_keys++, old_vals++, num_old--; + } else { /* Replace or insert new */ + GET_TERM(new_p[1], *hp++); + if (c > 0) { /* If new new key */ + *kp++ = new_key; + } else { /* If replacement */ + *kp++ = key; + old_keys++, old_vals++, num_old--; + } + n--; + if (n == 0) { + break; + } else { + new_p += 2; + GET_TERM(*new_p, new_key); + } + } + if (num_old == 0) { + break; + } + } + + /* + * At this point, we have run out of either old keys and values, + * or the update list. In other words, at least of one n and + * num_old must be zero. + */ + + if (n > 0) { + /* + * All old keys and values have been copied, but there + * are still new keys and values in the update list that + * must be copied. + */ + ASSERT(num_old == 0); + while (n-- > 0) { + GET_TERM(new_p[0], *kp++); + GET_TERM(new_p[1], *hp++); + new_p += 2; + } + } else { + /* + * All updates are now done. We may still have old + * keys and values that we must copy. + */ + ASSERT(n == 0); + while (num_old-- > 0) { + ASSERT(kp < (Eterm *)mp); + *kp++ = *old_keys++; + *hp++ = *old_vals++; + } + } + + /* + * Calculate how many values that are unused at the end of the + * key tuple and fill it out with a bignum header. + */ + if ((n = (Eterm *)mp - kp) > 0) { + *kp = make_pos_bignum_header(n-1); + } + + /* + * Fill in the size of the map in both the key tuple and in the map. + */ + + n = kp - p->htop - 1; /* Actual number of keys/values */ + *p->htop = make_arityval(n); + p->htop = hp; + mp->size = n; + + /* The expensive case, need to build a hashmap */ + if (n > MAP_SMALL_MAP_LIMIT) { + res = erts_hashmap_from_ks_and_vs(p,flatmap_get_keys(mp),flatmap_get_values(mp),n); + if (p->mbuf) { + Uint live = Arg(3); + reg[live] = res; + erts_garbage_collect(p, 0, reg, live+1); + res = reg[live]; + } + } + return res; +} + +/* + * Update values for keys that already exist in the map. + */ + +static Eterm +update_map_exact(Process* p, Eterm* reg, Eterm map, BeamInstr* I) +{ + Uint n; + Uint i; + Uint num_old; + Uint need; + flatmap_t *old_mp, *mp; + Eterm res; + Eterm* hp; + Eterm* E; + Eterm* old_keys; + Eterm* old_vals; + BeamInstr* new_p; + Eterm new_key; + + new_p = &Arg(5); + n = Arg(4) / 2; /* Number of values to be updated */ + ASSERT(n > 0); + + if (is_not_flatmap(map)) { + Uint32 hx; + Eterm val; + + /* apparently the compiler does not emit is_map instructions, + * bad compiler */ + + if (is_not_hashmap(map)) { + p->freason = BADMAP; + p->fvalue = map; + return THE_NON_VALUE; + } + + res = map; + E = p->stop; + while(n--) { + GET_TERM(new_p[0], new_key); + GET_TERM(new_p[1], val); + hx = hashmap_make_hash(new_key); + + res = erts_hashmap_insert(p, hx, new_key, val, res, 1); + if (is_non_value(res)) { + p->fvalue = new_key; + p->freason = BADKEY; + return res; + } + + if (p->mbuf) { + Uint live = Arg(3); + reg[live] = res; + erts_garbage_collect(p, 0, reg, live+1); + res = reg[live]; + E = p->stop; + } + + new_p += 2; + } + return res; + } + + old_mp = (flatmap_t *) flatmap_val(map); + num_old = flatmap_get_size(old_mp); + + /* + * If the old map is empty, fail. + */ + + if (num_old == 0) { + E = p->stop; + p->freason = BADKEY; + GET_TERM(new_p[0], p->fvalue); + return THE_NON_VALUE; + } + + /* + * Allocate the exact heap space needed. + */ + + need = num_old + MAP_HEADER_FLATMAP_SZ; + if (HeapWordsLeft(p) < need) { + Uint live = Arg(3); + reg[live] = map; + erts_garbage_collect(p, need, reg, live+1); + map = reg[live]; + old_mp = (flatmap_t *)flatmap_val(map); + } + + /* + * Update map, keeping the old key tuple. + */ + + hp = p->htop; + E = p->stop; + + old_vals = flatmap_get_values(old_mp); + old_keys = flatmap_get_keys(old_mp); + + res = make_flatmap(hp); + mp = (flatmap_t *)hp; + hp += MAP_HEADER_FLATMAP_SZ; + mp->thing_word = MAP_HEADER_FLATMAP; + mp->size = num_old; + mp->keys = old_mp->keys; + + /* Get array of key/value pairs to be updated */ + GET_TERM(*new_p, new_key); + + /* Update all values */ + for (i = 0; i < num_old; i++) { + if (!EQ(*old_keys, new_key)) { + /* Not same keys */ + *hp++ = *old_vals; + } else { + GET_TERM(new_p[1], *hp); + hp++; + n--; + if (n == 0) { + /* + * All updates done. Copy remaining values + * and return the result. + */ + for (i++, old_vals++; i < num_old; i++) { + *hp++ = *old_vals++; + } + ASSERT(hp == p->htop + need); + p->htop = hp; + return res; + } else { + new_p += 2; + GET_TERM(*new_p, new_key); + } + } + old_vals++, old_keys++; + } + + /* + * Updates left. That means that at least one the keys in the + * update list did not previously exist. + */ + ASSERT(hp == p->htop + need); + p->freason = BADKEY; + p->fvalue = new_key; + return THE_NON_VALUE; +} +#undef GET_TERM int catchlevel(Process *p) { @@ -6234,3 +7143,12 @@ erts_current_reductions(Process *current, Process *p) } } +int +erts_beam_jump_table(void) +{ +#if defined(NO_JUMP_TABLE) + return 0; +#else + return 1; +#endif +} diff --git a/erts/emulator/beam/beam_load.c b/erts/emulator/beam/beam_load.c index bd4e5a52d0..a6dce2d1d2 100644 --- a/erts/emulator/beam/beam_load.c +++ b/erts/emulator/beam/beam_load.c @@ -1,18 +1,19 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 1996-2013. All Rights Reserved. + * Copyright Ericsson AB 1996-2014. All Rights Reserved. * - * The contents of this file are subject to the Erlang Public License, - * Version 1.1, (the "License"); you may not use this file except in - * compliance with the License. You should have received a copy of the - * Erlang Public License along with this software. If not, it can be - * retrieved online at http://www.erlang.org/. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Software distributed under the License is distributed on an "AS IS" - * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See - * the License for the specific language governing rights and limitations - * under the License. + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. * * %CopyrightEnd% */ @@ -36,6 +37,7 @@ #include "beam_catches.h" #include "erl_binary.h" #include "erl_zlib.h" +#include "erl_map.h" #ifdef HIPE #include "hipe_bif0.h" @@ -204,10 +206,7 @@ typedef struct { typedef struct { Eterm term; /* The tagged term (in the heap). */ - Uint heap_size; /* (Exact) size on the heap. */ - SWord offset; /* Offset from temporary location to final. */ - ErlOffHeap off_heap; /* Start of linked list of ProcBins. */ - Eterm* heap; /* Heap for term. */ + ErlHeapFragment* heap_frags; } Literal; /* @@ -245,7 +244,7 @@ typedef struct { /* * This structure contains all information about the module being loaded. */ - +#define MD5_SIZE 16 typedef struct LoaderState { /* * The current logical file within the binary. @@ -292,7 +291,7 @@ typedef struct LoaderState { StringPatch* string_patches; /* Linked list of position into string table to patch. */ BeamInstr catches; /* Linked list of catch_yf instructions. */ unsigned loaded_size; /* Final size of code when loaded. */ - byte mod_md5[16]; /* MD5 for module code. */ + byte mod_md5[MD5_SIZE]; /* MD5 for module code. */ int may_load_nif; /* true if NIFs may later be loaded for this module */ int on_load; /* Index in the code for the on_load function * (or 0 if there is no on_load function) @@ -409,7 +408,7 @@ typedef struct LoaderState { __result = __result << 8 | *Stp->file_p++; \ } \ Dest = __result; \ - } while (0) + } #define GetByte(Stp, Dest) \ if ((Stp)->file_left < 1) { \ @@ -476,6 +475,8 @@ typedef struct LoaderState { static void free_loader_state(Binary* magic); +static ErlHeapFragment* new_literal_fragment(Uint size); +static void free_literal_fragment(ErlHeapFragment*); static void loader_state_dtor(Binary* magic); static Eterm insert_new_code(Process *c_p, ErtsProcLocks c_p_locks, Eterm group_leader, Eterm module, @@ -506,6 +507,9 @@ static GenOp* gen_select_literals(LoaderState* stp, GenOpArg S, static GenOp* const_select_val(LoaderState* stp, GenOpArg S, GenOpArg Fail, GenOpArg Size, GenOpArg* Rest); +static GenOp* gen_get_map_element(LoaderState* stp, GenOpArg Fail, GenOpArg Src, + GenOpArg Size, GenOpArg* Rest); + static int freeze_code(LoaderState* stp); static void final_touch(LoaderState* stp); @@ -521,11 +525,16 @@ static void new_literal_patch(LoaderState* stp, int pos); static void new_string_patch(LoaderState* stp, int pos); static Uint new_literal(LoaderState* stp, Eterm** hpp, Uint heap_size); static int genopargcompare(GenOpArg* a, GenOpArg* b); -static Eterm exported_from_module(Process* p, Eterm mod); -static Eterm functions_in_module(Process* p, Eterm mod); -static Eterm attributes_for_module(Process* p, Eterm mod); -static Eterm compilation_info_for_module(Process* p, Eterm mod); -static Eterm native_addresses(Process* p, Eterm mod); +static Eterm get_module_info(Process* p, ErtsCodeIndex code_ix, + BeamInstr* code, Eterm module, Eterm what); +static Eterm exported_from_module(Process* p, ErtsCodeIndex code_ix, + Eterm mod); +static Eterm functions_in_module(Process* p, BeamInstr* code); +static Eterm attributes_for_module(Process* p, BeamInstr* code); +static Eterm compilation_info_for_module(Process* p, BeamInstr* code); +static Eterm md5_of_module(Process* p, BeamInstr* code); +static Eterm has_native(BeamInstr* code); +static Eterm native_addresses(Process* p, BeamInstr* code); int patch_funentries(Eterm Patchlist); int patch(Eterm Addresses, Uint fe); static int safe_mul(UWord a, UWord b, UWord* resp); @@ -535,7 +544,6 @@ static int must_swap_floats; Uint erts_total_code_size; /**********************************************************************/ - void init_load(void) { FloatDef f; @@ -646,6 +654,7 @@ erts_prepare_loading(Binary* magic, Process *c_p, Eterm group_leader, stp->code[MI_COMPILE_PTR] = 0; stp->code[MI_COMPILE_SIZE] = 0; stp->code[MI_COMPILE_SIZE_ON_HEAP] = 0; + stp->code[MI_MD5_PTR] = 0; /* * Read the atom table. @@ -877,6 +886,28 @@ free_loader_state(Binary* magic) } } +static ErlHeapFragment* new_literal_fragment(Uint size) +{ + ErlHeapFragment* bp; + bp = (ErlHeapFragment*) ERTS_HEAP_ALLOC(ERTS_ALC_T_PREPARED_CODE, + ERTS_HEAP_FRAG_SIZE(size)); + ERTS_INIT_HEAP_FRAG(bp, size); + return bp; +} + +static void free_literal_fragment(ErlHeapFragment* bp) +{ + ASSERT(bp != NULL); + do { + ErlHeapFragment* next_bp = bp->next; + + erts_cleanup_offheap(&bp->off_heap); + ERTS_HEAP_FREE(ERTS_ALC_T_PREPARED_CODE, (void *) bp, + ERTS_HEAP_FRAG_SIZE(bp->size)); + bp = next_bp; + }while (bp != NULL); +} + /* * This destructor function can safely be called multiple times. */ @@ -916,10 +947,9 @@ loader_state_dtor(Binary* magic) if (stp->literals != 0) { int i; for (i = 0; i < stp->num_literals; i++) { - if (stp->literals[i].heap != 0) { - erts_free(ERTS_ALC_T_PREPARED_CODE, - (void *) stp->literals[i].heap); - stp->literals[i].heap = 0; + if (stp->literals[i].heap_frags != 0) { + free_literal_fragment(stp->literals[i].heap_frags); + stp->literals[i].heap_frags = 0; } } erts_free(ERTS_ALC_T_PREPARED_CODE, (void *) stp->literals); @@ -1209,7 +1239,6 @@ verify_chunks(LoaderState* stp) return 0; } - static int load_atom_table(LoaderState* stp) { @@ -1255,7 +1284,6 @@ load_atom_table(LoaderState* stp) return 0; } - static int load_import_table(LoaderState* stp) { @@ -1308,7 +1336,6 @@ load_import_table(LoaderState* stp) return 0; } - static int read_export_table(LoaderState* stp) { @@ -1447,6 +1474,7 @@ read_lambda_table(LoaderState* stp) return 0; } + static int read_literal_table(LoaderState* stp) { @@ -1468,7 +1496,7 @@ read_literal_table(LoaderState* stp) stp->allocated_literals = stp->num_literals; for (i = 0; i < stp->num_literals; i++) { - stp->literals[i].heap = 0; + stp->literals[i].heap_frags = 0; } for (i = 0; i < stp->num_literals; i++) { @@ -1476,28 +1504,38 @@ read_literal_table(LoaderState* stp) Sint heap_size; byte* p; Eterm val; - Eterm* hp; + ErtsHeapFactory factory; GetInt(stp, 4, sz); /* Size of external term format. */ GetString(stp, p, sz); if ((heap_size = erts_decode_ext_size(p, sz)) < 0) { LoadError1(stp, "literal %d: bad external format", i); } - hp = stp->literals[i].heap = erts_alloc(ERTS_ALC_T_PREPARED_CODE, - heap_size*sizeof(Eterm)); - stp->literals[i].off_heap.first = 0; - stp->literals[i].off_heap.overhead = 0; - val = erts_decode_ext(&hp, &stp->literals[i].off_heap, &p); - stp->literals[i].heap_size = hp - stp->literals[i].heap; - if (stp->literals[i].heap_size > heap_size) { - erl_exit(1, "overrun by %d word(s) for literal heap, term %d", - stp->literals[i].heap_size - heap_size, i); - } - if (is_non_value(val)) { - LoadError1(stp, "literal %d: bad external format", i); - } - stp->literals[i].term = val; - stp->total_literal_size += stp->literals[i].heap_size; + + if (heap_size > 0) { + erts_factory_message_init(&factory, NULL, NULL, + new_literal_fragment(heap_size)); + factory.alloc_type = ERTS_ALC_T_PREPARED_CODE; + val = erts_decode_ext(&factory, &p); + + if (is_non_value(val)) { + LoadError1(stp, "literal %d: bad external format", i); + } + erts_factory_close(&factory); + stp->literals[i].heap_frags = factory.heap_frags; + stp->total_literal_size += erts_used_frag_sz(factory.heap_frags); + } + else { + erts_factory_dummy_init(&factory); + val = erts_decode_ext(&factory, &p); + if (is_non_value(val)) { + LoadError1(stp, "literal %d: bad external format", i); + } + ASSERT(is_immed(val)); + stp->literals[i].heap_frags = NULL; + } + stp->literals[i].term = val; + } erts_free(ERTS_ALC_T_TMP, uncompressed); return 1; @@ -1641,7 +1679,6 @@ read_line_table(LoaderState* stp) return 0; } - static int read_code_header(LoaderState* stp) { @@ -1711,7 +1748,6 @@ read_code_header(LoaderState* stp) return 0; } - #define VerifyTag(Stp, Actual, Expected) \ if (Actual != Expected) { \ LoadError2(Stp, "bad tag %d; expected %d", Actual, Expected); \ @@ -1730,7 +1766,6 @@ read_code_header(LoaderState* stp) #define TermWords(t) (((t) / (sizeof(BeamInstr)/sizeof(Eterm))) + !!((t) % (sizeof(BeamInstr)/sizeof(Eterm)))) - static int load_code(LoaderState* stp) { @@ -2367,7 +2402,11 @@ load_code(LoaderState* stp) if (stp->may_load_nif) { const int finfo_ix = ci - FUNC_INFO_SZ; - enum { MIN_FUNC_SZ = 3 }; +#ifdef ERTS_DIRTY_SCHEDULERS + enum { MIN_FUNC_SZ = 4 }; +#else + enum { MIN_FUNC_SZ = 3 }; +#endif if (finfo_ix - last_func_start < MIN_FUNC_SZ && last_func_start) { /* Must make room for call_nif op */ int pad = MIN_FUNC_SZ - (finfo_ix - last_func_start); @@ -2512,7 +2551,6 @@ load_code(LoaderState* stp) return retval; } - #define succ(St, X, Y) ((X).type == (Y).type && (X).val + 1 == (Y).val) #define succ2(St, X, Y) ((X).type == (Y).type && (X).val + 2 == (Y).val) #define succ3(St, X, Y) ((X).type == (Y).type && (X).val + 3 == (Y).val) @@ -2964,19 +3002,19 @@ gen_put_integer(LoaderState* stp, GenOpArg Fail, GenOpArg Size, NEW_GENOP(stp, op); NATIVE_ENDIAN(Flags); - if (Size.type == TAG_i && Size.val < 0) { - error: /* Negative size must fail */ - op->op = genop_badarg_1; - op->arity = 1; - op->a[0] = Fail; - } else if (Size.type == TAG_i) { + if (Size.type == TAG_i) { op->op = genop_i_new_bs_put_integer_imm_4; op->arity = 4; op->a[0] = Fail; op->a[1].type = TAG_u; if (!safe_mul(Size.val, Unit.val, &op->a[1].val)) { - goto error; + error: + op->op = genop_badarg_1; + op->arity = 1; + op->a[0] = Fail; + op->next = NULL; + return op; } op->a[1].val = Size.val * Unit.val; op->a[2].type = Flags.type; @@ -3170,7 +3208,11 @@ gen_increment_from_minus(LoaderState* stp, GenOpArg Reg, GenOpArg Integer, static int negation_is_small(LoaderState* stp, GenOpArg Int) { - return Int.type == TAG_i && IS_SSMALL(-Int.val); + /* Check for the rare case of overflow in BeamInstr (UWord) -> Sint + * Cast to the correct type before using IS_SSMALL (Sint) */ + return Int.type == TAG_i && + !(Int.val & ~((((BeamInstr)1) << ((sizeof(Sint)*8)-1))-1)) && + IS_SSMALL(-((Sint)Int.val)); } @@ -3320,9 +3362,10 @@ gen_select_tuple_arity(LoaderState* stp, GenOpArg S, GenOpArg Fail, { GenOp* op; + GenOpArg *tmp; int arity = Size.val + 3; int size = Size.val / 2; - int i; + int i, j, align = 0; /* * Verify the validity of the list. @@ -3337,9 +3380,37 @@ gen_select_tuple_arity(LoaderState* stp, GenOpArg S, GenOpArg Fail, } /* + * Use a special-cased instruction if there are only two values. + */ + if (size == 2) { + NEW_GENOP(stp, op); + op->next = NULL; + op->op = genop_i_select_tuple_arity2_6; + GENOP_ARITY(op, arity - 1); + op->a[0] = S; + op->a[1] = Fail; + op->a[2].type = TAG_u; + op->a[2].val = Rest[0].val; + op->a[3].type = TAG_u; + op->a[3].val = Rest[2].val; + op->a[4] = Rest[1]; + op->a[5] = Rest[3]; + + return op; + } + + /* * Generate the generic instruction. + * Assumption: + * Few different tuple arities to select on (fewer than 20). + * Use linear scan approach. */ + align = 1; + + arity += 2*align; + size += align; + NEW_GENOP(stp, op); op->next = NULL; op->op = genop_i_select_tuple_arity_3; @@ -3347,39 +3418,36 @@ gen_select_tuple_arity(LoaderState* stp, GenOpArg S, GenOpArg Fail, op->a[0] = S; op->a[1] = Fail; op->a[2].type = TAG_u; - op->a[2].val = Size.val / 2; - for (i = 0; i < Size.val; i += 2) { - op->a[i+3].type = TAG_v; - op->a[i+3].val = make_arityval(Rest[i].val); - op->a[i+4] = Rest[i+1]; - } + op->a[2].val = size; - /* - * Sort the values to make them useful for a binary search. - */ + tmp = (GenOpArg *) erts_alloc(ERTS_ALC_T_LOADER_TMP, sizeof(GenOpArg)*(arity-2*align)); - qsort(op->a+3, size, 2*sizeof(GenOpArg), - (int (*)(const void *, const void *)) genopargcompare); -#ifdef DEBUG - for (i = 3; i < arity-2; i += 2) { - ASSERT(op->a[i].val < op->a[i+2].val); + for (i = 3; i < arity - 2*align; i+=2) { + tmp[i-3].type = TAG_v; + tmp[i-3].val = make_arityval(Rest[i-3].val); + tmp[i-2] = Rest[i-2]; } -#endif /* - * Use a special-cased instruction if there are only two values. + * Sort the values to make them useful for a sentinel search */ - if (size == 2) { - op->op = genop_i_select_tuple_arity2_6; - op->arity--; - op->a[2].type = TAG_u; - op->a[2].val = arityval(op->a[3].val); - op->a[3] = op->a[4]; - op->a[4].type = TAG_u; - op->a[4].val = arityval(op->a[5].val); - op->a[5] = op->a[6]; + + qsort(tmp, size - align, 2*sizeof(GenOpArg), + (int (*)(const void *, const void *)) genopargcompare); + + j = 3; + for (i = 3; i < arity - 2*align; i += 2) { + op->a[j] = tmp[i-3]; + op->a[j + size] = tmp[i-2]; + j++; } + erts_free(ERTS_ALC_T_LOADER_TMP, (void *) tmp); + + op->a[j].type = TAG_u; + op->a[j].val = ~((BeamInstr)0); + op->a[j+size] = Fail; + return op; } @@ -3601,45 +3669,109 @@ gen_select_val(LoaderState* stp, GenOpArg S, GenOpArg Fail, GenOpArg Size, GenOpArg* Rest) { GenOp* op; + GenOpArg *tmp; int arity = Size.val + 3; int size = Size.val / 2; - int i; + int i, j, align = 0; + + if (size == 2) { + + /* + * Use a special-cased instruction if there are only two values. + */ + + NEW_GENOP(stp, op); + op->next = NULL; + op->op = genop_i_select_val2_6; + GENOP_ARITY(op, arity - 1); + op->a[0] = S; + op->a[1] = Fail; + op->a[2] = Rest[0]; + op->a[3] = Rest[2]; + op->a[4] = Rest[1]; + op->a[5] = Rest[3]; + + return op; + + } else if (size > 10) { + + /* binary search instruction */ + + NEW_GENOP(stp, op); + op->next = NULL; + op->op = genop_i_select_val_bins_3; + GENOP_ARITY(op, arity); + op->a[0] = S; + op->a[1] = Fail; + op->a[2].type = TAG_u; + op->a[2].val = size; + for (i = 3; i < arity; i++) { + op->a[i] = Rest[i-3]; + } + + /* + * Sort the values to make them useful for a binary search. + */ + + qsort(op->a+3, size, 2*sizeof(GenOpArg), + (int (*)(const void *, const void *)) genopargcompare); +#ifdef DEBUG + for (i = 3; i < arity-2; i += 2) { + ASSERT(op->a[i].val < op->a[i+2].val); + } +#endif + return op; + } + + /* linear search instruction */ + + align = 1; + + arity += 2*align; + size += align; NEW_GENOP(stp, op); op->next = NULL; - op->op = genop_i_select_val_3; + op->op = genop_i_select_val_lins_3; GENOP_ARITY(op, arity); op->a[0] = S; op->a[1] = Fail; op->a[2].type = TAG_u; op->a[2].val = size; - for (i = 3; i < arity; i++) { - op->a[i] = Rest[i-3]; + + tmp = (GenOpArg *) erts_alloc(ERTS_ALC_T_LOADER_TMP, sizeof(GenOpArg)*(arity-2*align)); + + for (i = 3; i < arity - 2*align; i++) { + tmp[i-3] = Rest[i-3]; } /* - * Sort the values to make them useful for a binary search. + * Sort the values to make them useful for a sentinel search */ - qsort(op->a+3, size, 2*sizeof(GenOpArg), - (int (*)(const void *, const void *)) genopargcompare); -#ifdef DEBUG - for (i = 3; i < arity-2; i += 2) { - ASSERT(op->a[i].val < op->a[i+2].val); + qsort(tmp, size - align, 2*sizeof(GenOpArg), + (int (*)(const void *, const void *)) genopargcompare); + + j = 3; + for (i = 3; i < arity - 2*align; i += 2) { + op->a[j] = tmp[i-3]; + op->a[j+size] = tmp[i-2]; + j++; } -#endif - /* - * Use a special-cased instruction if there are only two values. - */ - if (size == 2) { - op->op = genop_i_select_val2_6; - op->arity--; - op->a[2] = op->a[3]; - op->a[3] = op->a[4]; - op->a[4] = op->a[5]; - op->a[5] = op->a[6]; + erts_free(ERTS_ALC_T_LOADER_TMP, (void *) tmp); + + /* add sentinel */ + + op->a[j].type = TAG_u; + op->a[j].val = ~((BeamInstr)0); + op->a[j+size] = Fail; + +#ifdef DEBUG + for (i = 0; i < size - 1; i++) { + ASSERT(op->a[i+3].val <= op->a[i+4].val); } +#endif return op; } @@ -3791,6 +3923,8 @@ gen_guard_bif1(LoaderState* stp, GenOpArg Fail, GenOpArg Live, GenOpArg Bif, op->a[1].val = (BeamInstr) (void *) erts_gc_bit_size_1; } else if (bf == byte_size_1) { op->a[1].val = (BeamInstr) (void *) erts_gc_byte_size_1; + } else if (bf == map_size_1) { + op->a[1].val = (BeamInstr) (void *) erts_gc_map_size_1; } else if (bf == abs_1) { op->a[1].val = (BeamInstr) (void *) erts_gc_abs_1; } else if (bf == float_1) { @@ -3957,8 +4091,243 @@ tuple_append_put(LoaderState* stp, GenOpArg Arity, GenOpArg Dst, return op; } +/* + * Predicate to test whether the given literal is a map. + */ + +static int +literal_is_map(LoaderState* stp, GenOpArg Lit) +{ + Eterm term; + + ASSERT(Lit.type == TAG_q); + term = stp->literals[Lit.val].term; + return is_map(term); +} + +/* + * Predicate to test whether the given literal is an empty map. + */ + +static int +is_empty_map(LoaderState* stp, GenOpArg Lit) +{ + Eterm term; + + if (Lit.type != TAG_q) { + return 0; + } + term = stp->literals[Lit.val].term; + return is_flatmap(term) && flatmap_get_size(flatmap_val(term)) == 0; +} + +/* + * Pseudo predicate map_key_sort that will sort the Rest operand for + * map instructions as a side effect. + */ + +typedef struct SortGenOpArg { + Eterm term; /* Term to use for comparing */ + GenOpArg arg; /* Original data */ +} SortGenOpArg; + +static int +genopargtermcompare(SortGenOpArg* a, SortGenOpArg* b) +{ + return CMP_TERM(a->term, b->term); +} + +static int +map_key_sort(LoaderState* stp, GenOpArg Size, GenOpArg* Rest) +{ + SortGenOpArg* t; + unsigned size = Size.val; + unsigned i; + + if (size == 2) { + return 1; /* Already sorted. */ + } + + + t = (SortGenOpArg *) erts_alloc(ERTS_ALC_T_TMP, size*sizeof(SortGenOpArg)); + + /* + * Copy original data and sort keys to a temporary array. + */ + for (i = 0; i < size; i += 2) { + t[i].arg = Rest[i]; + switch (Rest[i].type) { + case TAG_a: + t[i].term = Rest[i].val; + ASSERT(is_atom(t[i].term)); + break; + case TAG_i: + t[i].term = make_small(Rest[i].val); + break; + case TAG_n: + t[i].term = NIL; + break; + case TAG_q: + t[i].term = stp->literals[Rest[i].val].term; + break; + default: + /* + * Not a literal key. Not allowed. Only a single + * variable key is allowed in each map instruction. + */ + erts_free(ERTS_ALC_T_TMP, (void *) t); + return 0; + } +#ifdef DEBUG + t[i+1].term = THE_NON_VALUE; +#endif + t[i+1].arg = Rest[i+1]; + } + + /* + * Sort the temporary array. + */ + qsort((void *) t, size / 2, 2 * sizeof(SortGenOpArg), + (int (*)(const void *, const void *)) genopargtermcompare); + + /* + * Copy back the sorted, original data. + */ + for (i = 0; i < size; i++) { + Rest[i] = t[i].arg; + } + + erts_free(ERTS_ALC_T_TMP, (void *) t); + return 1; +} + +static int +hash_genop_arg(LoaderState* stp, GenOpArg Key, Uint32* hx) +{ + switch (Key.type) { + case TAG_a: + *hx = hashmap_make_hash(Key.val); + return 1; + case TAG_i: + *hx = hashmap_make_hash(make_small(Key.val)); + return 1; + case TAG_n: + *hx = hashmap_make_hash(NIL); + return 1; + case TAG_q: + *hx = hashmap_make_hash(stp->literals[Key.val].term); + return 1; + default: + return 0; + } +} + +/* + * Replace a get_map_elements with one key to an instruction with one + * element. + */ + +static GenOp* +gen_get_map_element(LoaderState* stp, GenOpArg Fail, GenOpArg Src, + GenOpArg Size, GenOpArg* Rest) +{ + GenOp* op; + GenOpArg Key; + Uint32 hx = 0; + + ASSERT(Size.type == TAG_u); + + NEW_GENOP(stp, op); + op->next = NULL; + op->a[0] = Fail; + op->a[1] = Src; + op->a[2] = Rest[0]; + + Key = Rest[0]; + if (hash_genop_arg(stp, Key, &hx)) { + op->arity = 5; + op->op = genop_i_get_map_element_hash_5; + op->a[3].type = TAG_u; + op->a[3].val = (BeamInstr) hx; + op->a[4] = Rest[1]; + } else { + op->arity = 4; + op->op = genop_i_get_map_element_4; + op->a[3] = Rest[1]; + } + return op; +} + +static GenOp* +gen_get_map_elements(LoaderState* stp, GenOpArg Fail, GenOpArg Src, + GenOpArg Size, GenOpArg* Rest) +{ + GenOp* op; + Uint32 hx; + Uint i; + GenOpArg* dst; +#ifdef DEBUG + int good_hash; +#endif + + ASSERT(Size.type == TAG_u); + + NEW_GENOP(stp, op); + op->op = genop_i_get_map_elements_3; + GENOP_ARITY(op, 3 + 3*(Size.val/2)); + op->next = NULL; + op->a[0] = Fail; + op->a[1] = Src; + op->a[2].type = TAG_u; + op->a[2].val = 3*(Size.val/2); + + dst = op->a+3; + for (i = 0; i < Size.val / 2; i++) { + dst[0] = Rest[2*i]; + dst[1] = Rest[2*i+1]; +#ifdef DEBUG + good_hash = +#endif + hash_genop_arg(stp, dst[0], &hx); +#ifdef DEBUG + ASSERT(good_hash); +#endif + dst[2].type = TAG_u; + dst[2].val = (BeamInstr) hx; + dst += 3; + } + return op; +} + +static GenOp* +gen_has_map_fields(LoaderState* stp, GenOpArg Fail, GenOpArg Src, + GenOpArg Size, GenOpArg* Rest) +{ + GenOp* op; + Uint i; + Uint n; + + ASSERT(Size.type == TAG_u); + n = Size.val; + + NEW_GENOP(stp, op); + GENOP_ARITY(op, 3 + 2*n); + op->next = NULL; + op->op = genop_get_map_elements_3; + + op->a[0] = Fail; + op->a[1] = Src; + op->a[2].type = TAG_u; + op->a[2].val = 2*n; + + for (i = 0; i < n; i++) { + op->a[3+2*i] = Rest[i]; + op->a[3+2*i+1].type = TAG_x; + op->a[3+2*i+1].val = 0; /* x(0); normally not used */ + } + return op; +} - /* * Freeze the code in memory, move the string table into place, * resolve all labels. @@ -3999,7 +4368,7 @@ freeze_code(LoaderState* stp) } size = (stp->ci * sizeof(BeamInstr)) + (stp->total_literal_size * sizeof(Eterm)) + - strtab_size + attr_size + compile_size + line_size; + strtab_size + attr_size + compile_size + MD5_SIZE + line_size; /* * Move the code to its final location. @@ -4036,8 +4405,9 @@ freeze_code(LoaderState* stp) Uint* low; Uint* high; LiteralPatch* lp; - struct erl_off_heap_header* off_heap = 0; - struct erl_off_heap_header** off_heap_last = &off_heap; + ErlOffHeap code_off_heap; + + ERTS_INIT_OFF_HEAP(&code_off_heap); low = (Uint *) (code+stp->ci); high = low + stp->total_literal_size; @@ -4045,73 +4415,21 @@ freeze_code(LoaderState* stp) code[MI_LITERALS_END] = (BeamInstr) high; ptr = low; for (i = 0; i < stp->num_literals; i++) { - SWord offset; - struct erl_off_heap_header* t_off_heap; - - sys_memcpy(ptr, stp->literals[i].heap, - stp->literals[i].heap_size*sizeof(Eterm)); - offset = ptr - stp->literals[i].heap; - stp->literals[i].offset = offset; - high = ptr + stp->literals[i].heap_size; - while (ptr < high) { - Eterm val = *ptr; - switch (primary_tag(val)) { - case TAG_PRIMARY_LIST: - case TAG_PRIMARY_BOXED: - *ptr++ = offset_ptr(val, offset); - break; - case TAG_PRIMARY_HEADER: - if (header_is_transparent(val)) { - ptr++; - } else { - if (thing_subtag(val) == REFC_BINARY_SUBTAG) { - struct erl_off_heap_header* oh; - - oh = (struct erl_off_heap_header*) ptr; - if (oh->next) { - Eterm** uptr = (Eterm **) (void *) &oh->next; - *uptr += offset; - } - } - ptr += 1 + thing_arityval(val); - } - break; - default: - ptr++; - break; - } - } - ASSERT(ptr == high); - - /* - * Re-link the off_heap list for this term onto the - * off_heap list for the entire module. - */ - t_off_heap = stp->literals[i].off_heap.first; - if (t_off_heap) { - t_off_heap = (struct erl_off_heap_header *) - offset_ptr((UWord) t_off_heap, offset); - while (t_off_heap) { - *off_heap_last = t_off_heap; - off_heap_last = &t_off_heap->next; - t_off_heap = t_off_heap->next; - } - } + if (stp->literals[i].heap_frags) { + move_multi_frags(&ptr, &code_off_heap, stp->literals[i].heap_frags, + &stp->literals[i].term, 1); + } + else ASSERT(is_immed(stp->literals[i].term)); } - code[MI_LITERALS_OFF_HEAP] = (BeamInstr) off_heap; + code[MI_LITERALS_OFF_HEAP] = (BeamInstr) code_off_heap.first; lp = stp->literal_patches; while (lp != 0) { BeamInstr* op_ptr; - Uint literal; Literal* lit; op_ptr = code + lp->pos; lit = &stp->literals[op_ptr[0]]; - literal = lit->term; - if (is_boxed(literal) || is_list(literal)) { - literal = offset_ptr(literal, lit->offset); - } - op_ptr[0] = literal; + op_ptr[0] = lit->term; lp = lp->next; } literal_end += stp->total_literal_size; @@ -4208,11 +4526,20 @@ freeze_code(LoaderState* stp) code[MI_COMPILE_SIZE_ON_HEAP] = decoded_size; } CHKBLK(ERTS_ALC_T_CODE,code); + { + byte* md5_sum = str_table + strtab_size + attr_size + compile_size; + CHKBLK(ERTS_ALC_T_CODE,code); + sys_memcpy(md5_sum, stp->mod_md5, MD5_SIZE); + CHKBLK(ERTS_ALC_T_CODE,code); + code[MI_MD5_PTR] = (BeamInstr) md5_sum; + CHKBLK(ERTS_ALC_T_CODE,code); + } + CHKBLK(ERTS_ALC_T_CODE,code); /* * Make sure that we have not overflowed the allocated code space. */ - ASSERT(str_table + strtab_size + attr_size + compile_size == + ASSERT(str_table + strtab_size + attr_size + compile_size + MD5_SIZE == ((byte *) code) + size); /* @@ -4276,7 +4603,6 @@ freeze_code(LoaderState* stp) return 0; } - static void final_touch(LoaderState* stp) { @@ -4378,7 +4704,6 @@ final_touch(LoaderState* stp) } } - static int transform_engine(LoaderState* st) { @@ -4387,6 +4712,7 @@ transform_engine(LoaderState* st) Uint* restart; /* Where to restart if current match fails. */ GenOpArg def_vars[TE_MAX_VARS]; /* Default buffer for variables. */ GenOpArg* var = def_vars; + int num_vars = 0; int i; /* General index. */ Uint mask; GenOp* instr; @@ -4589,9 +4915,9 @@ transform_engine(LoaderState* st) { int n = *pc++; int formal_arity = gen_opc[instr->op].arity; - int num_vars = n + (instr->arity - formal_arity); int j = formal_arity; + num_vars = n + (instr->arity - formal_arity); var = erts_alloc(ERTS_ALC_T_LOADER_TMP, num_vars * sizeof(GenOpArg)); for (i = 0; i < n; i++) { @@ -4603,7 +4929,6 @@ transform_engine(LoaderState* st) } break; #endif - case TOP_next_arg: ap++; break; @@ -4691,6 +5016,20 @@ transform_engine(LoaderState* st) instr->a[ap].val = var[i].val; ap++; break; +#if defined(TOP_store_rest_args) + case TOP_store_rest_args: + { + int n = *pc++; + int num_extra = num_vars - n; + + ASSERT(n <= num_vars); + GENOP_ARITY(instr, instr->arity+num_extra); + memcpy(instr->a, instr->def_args, ap*sizeof(GenOpArg)); + memcpy(instr->a+ap, var+n, num_extra*sizeof(GenOpArg)); + ap += num_extra; + } + break; +#endif case TOP_try_me_else: restart = pc + 1; restart += *pc++; @@ -4716,7 +5055,6 @@ transform_engine(LoaderState* st) return rval; } - static void short_file(int line, LoaderState* stp, unsigned needed) { @@ -4724,7 +5062,6 @@ short_file(int line, LoaderState* stp, unsigned needed) stp->file_name, needed); } - static void load_printf(int line, LoaderState* context, char *fmt,...) { @@ -4918,7 +5255,8 @@ get_tag_and_value(LoaderState* stp, Uint len_code, arity = count/sizeof(Eterm); *result = new_literal(stp, &hp, arity+1); - (void) bytes_to_big(bigbuf, count, neg, hp); + if (is_nil(bytes_to_big(bigbuf, count, neg, hp))) + goto load_error; if (bigbuf != default_buf) { erts_free(ERTS_ALC_T_LOADER_TMP, (void *) bigbuf); @@ -5022,19 +5360,18 @@ new_literal(LoaderState* stp, Eterm** hpp, Uint heap_size) stp->total_literal_size += heap_size; lit = stp->literals + stp->num_literals; - lit->offset = 0; - lit->heap_size = heap_size; - lit->heap = erts_alloc(ERTS_ALC_T_PREPARED_CODE, heap_size*sizeof(Eterm)); - lit->term = make_boxed(lit->heap); - lit->off_heap.first = 0; - lit->off_heap.overhead = 0; - *hpp = lit->heap; + lit->heap_frags = new_literal_fragment(heap_size); + lit->term = make_boxed(lit->heap_frags->mem); + *hpp = lit->heap_frags->mem; return stp->num_literals++; } Eterm erts_module_info_0(Process* p, Eterm module) { + Module* modp; + ErtsCodeIndex code_ix = erts_active_code_ix(); + BeamInstr* code; Eterm *hp; Eterm list = NIL; Eterm tup; @@ -5043,21 +5380,31 @@ erts_module_info_0(Process* p, Eterm module) return THE_NON_VALUE; } - if (erts_get_module(module, erts_active_code_ix()) == NULL) { + modp = erts_get_module(module, code_ix); + if (modp == NULL) { return THE_NON_VALUE; } + code = modp->curr.code; + if (code == NULL) { + return THE_NON_VALUE; + } + #define BUILD_INFO(What) \ - tup = erts_module_info_1(p, module, What); \ + tup = get_module_info(p, code_ix, code, module, What); \ hp = HAlloc(p, 5); \ tup = TUPLE2(hp, What, tup); \ hp += 3; \ list = CONS(hp, tup, list) + BUILD_INFO(am_md5); +#ifdef HIPE + BUILD_INFO(am_native); +#endif BUILD_INFO(am_compile); BUILD_INFO(am_attributes); - BUILD_INFO(am_imports); BUILD_INFO(am_exports); + BUILD_INFO(am_module); #undef BUILD_INFO return list; } @@ -5065,20 +5412,47 @@ erts_module_info_0(Process* p, Eterm module) Eterm erts_module_info_1(Process* p, Eterm module, Eterm what) { + Module* modp; + ErtsCodeIndex code_ix = erts_active_code_ix(); + BeamInstr* code; + + if (is_not_atom(module)) { + return THE_NON_VALUE; + } + + modp = erts_get_module(module, code_ix); + if (modp == NULL) { + return THE_NON_VALUE; + } + + code = modp->curr.code; + if (code == NULL) { + return THE_NON_VALUE; + } + + return get_module_info(p, code_ix, code, module, what); +} + +static Eterm +get_module_info(Process* p, ErtsCodeIndex code_ix, BeamInstr* code, + Eterm module, Eterm what) +{ if (what == am_module) { return module; - } else if (what == am_imports) { - return NIL; + } else if (what == am_md5) { + return md5_of_module(p, code); } else if (what == am_exports) { - return exported_from_module(p, module); + return exported_from_module(p, code_ix, module); } else if (what == am_functions) { - return functions_in_module(p, module); + return functions_in_module(p, code); } else if (what == am_attributes) { - return attributes_for_module(p, module); + return attributes_for_module(p, code); } else if (what == am_compile) { - return compilation_info_for_module(p, module); + return compilation_info_for_module(p, code); } else if (what == am_native_addresses) { - return native_addresses(p, module); + return native_addresses(p, code); + } else if (what == am_native) { + return has_native(code); } return THE_NON_VALUE; } @@ -5086,16 +5460,12 @@ erts_module_info_1(Process* p, Eterm module, Eterm what) /* * Builds a list of all functions in the given module: * [{Name, Arity},...] - * - * Returns a tagged term, or 0 on error. */ Eterm functions_in_module(Process* p, /* Process whose heap to use. */ - Eterm mod) /* Tagged atom for module. */ + BeamInstr* code) { - Module* modp; - BeamInstr* code; int i; Uint num_functions; Uint need; @@ -5103,15 +5473,6 @@ functions_in_module(Process* p, /* Process whose heap to use. */ Eterm* hp_end; Eterm result = NIL; - if (is_not_atom(mod)) { - return THE_NON_VALUE; - } - - modp = erts_get_module(mod, erts_active_code_ix()); - if (modp == NULL) { - return THE_NON_VALUE; - } - code = modp->curr.code; num_functions = code[MI_NUM_FUNCTIONS]; need = 5*num_functions; hp = HAlloc(p, need); @@ -5139,17 +5500,49 @@ functions_in_module(Process* p, /* Process whose heap to use. */ } /* + * Returns 'true' if mod has any native compiled functions, otherwise 'false' + */ + +static Eterm +has_native(BeamInstr *code) +{ + Eterm result = am_false; +#ifdef HIPE + if (erts_is_module_native(code)) { + result = am_true; + } +#endif + return result; +} + +int +erts_is_module_native(BeamInstr* code) +{ + Uint i, num_functions; + + /* Check NativeAdress of first real function in module */ + if (code != NULL) { + num_functions = code[MI_NUM_FUNCTIONS]; + for (i=0; i<num_functions; i++) { + BeamInstr* func_info = (BeamInstr *) code[MI_FUNCTIONS+i]; + Eterm name = (Eterm) func_info[3]; + if (is_atom(name)) { + return func_info[1] != 0; + } + else ASSERT(is_nil(name)); /* ignore BIF stubs */ + } + } + return 0; +} + +/* * Builds a list of all functions including native addresses. * [{Name,Arity,NativeAddress},...] - * - * Returns a tagged term, or 0 on error. */ static Eterm -native_addresses(Process* p, Eterm mod) +native_addresses(Process* p, BeamInstr* code) { - Module* modp; - BeamInstr* code; int i; Eterm* hp; Uint num_functions; @@ -5157,16 +5550,6 @@ native_addresses(Process* p, Eterm mod) Eterm* hp_end; Eterm result = NIL; - if (is_not_atom(mod)) { - return THE_NON_VALUE; - } - - modp = erts_get_module(mod, erts_active_code_ix()); - if (modp == NULL) { - return THE_NON_VALUE; - } - - code = modp->curr.code; num_functions = code[MI_NUM_FUNCTIONS]; need = (6+BIG_UINT_HEAP_SIZE)*num_functions; hp = HAlloc(p, need); @@ -5190,29 +5573,21 @@ native_addresses(Process* p, Eterm mod) return result; } - /* * Builds a list of all exported functions in the given module: * [{Name, Arity},...] - * - * Returns a tagged term, or 0 on error. */ Eterm exported_from_module(Process* p, /* Process whose heap to use. */ + ErtsCodeIndex code_ix, Eterm mod) /* Tagged atom for module. */ { int i; Eterm* hp = NULL; Eterm* hend = NULL; Eterm result = NIL; - ErtsCodeIndex code_ix; - - if (is_not_atom(mod)) { - return THE_NON_VALUE; - } - code_ix = erts_active_code_ix(); for (i = 0; i < export_list_size(code_ix); i++) { Export* ep = export_list(i,code_ix); @@ -5240,88 +5615,64 @@ exported_from_module(Process* p, /* Process whose heap to use. */ return result; } - /* * Returns a list of all attributes for the module. - * - * Returns a tagged term, or 0 on error. */ Eterm attributes_for_module(Process* p, /* Process whose heap to use. */ - Eterm mod) /* Tagged atom for module. */ - + BeamInstr* code) { - Module* modp; - BeamInstr* code; - Eterm* hp; byte* ext; Eterm result = NIL; - Eterm* end; - - if (is_not_atom(mod) || (is_not_list(result) && is_not_nil(result))) { - return THE_NON_VALUE; - } - modp = erts_get_module(mod, erts_active_code_ix()); - if (modp == NULL) { - return THE_NON_VALUE; - } - code = modp->curr.code; ext = (byte *) code[MI_ATTR_PTR]; if (ext != NULL) { - hp = HAlloc(p, code[MI_ATTR_SIZE_ON_HEAP]); - end = hp + code[MI_ATTR_SIZE_ON_HEAP]; - result = erts_decode_ext(&hp, &MSO(p), &ext); + ErtsHeapFactory factory; + erts_factory_proc_prealloc_init(&factory, p, code[MI_ATTR_SIZE_ON_HEAP]); + result = erts_decode_ext(&factory, &ext); if (is_value(result)) { - ASSERT(hp <= end); + erts_factory_close(&factory); } - HRelease(p,end,hp); } return result; } - /* * Returns a list containing compilation information. - * - * Returns a tagged term, or 0 on error. */ Eterm compilation_info_for_module(Process* p, /* Process whose heap to use. */ - Eterm mod) /* Tagged atom for module. */ + BeamInstr* code) { - Module* modp; - BeamInstr* code; - Eterm* hp; byte* ext; Eterm result = NIL; - Eterm* end; - - if (is_not_atom(mod) || (is_not_list(result) && is_not_nil(result))) { - return THE_NON_VALUE; - } - modp = erts_get_module(mod, erts_active_code_ix()); - if (modp == NULL) { - return THE_NON_VALUE; - } - code = modp->curr.code; ext = (byte *) code[MI_COMPILE_PTR]; if (ext != NULL) { - hp = HAlloc(p, code[MI_COMPILE_SIZE_ON_HEAP]); - end = hp + code[MI_COMPILE_SIZE_ON_HEAP]; - result = erts_decode_ext(&hp, &MSO(p), &ext); + ErtsHeapFactory factory; + erts_factory_proc_prealloc_init(&factory, p, code[MI_COMPILE_SIZE_ON_HEAP]); + result = erts_decode_ext(&factory, &ext); if (is_value(result)) { - ASSERT(hp <= end); + erts_factory_close(&factory); } - HRelease(p,end,hp); } return result; } /* + * Returns the MD5 checksum for a module + */ + +Eterm +md5_of_module(Process* p, /* Process whose heap to use. */ + BeamInstr* code) +{ + return new_binary(p, (byte *) code[MI_MD5_PTR], MD5_SIZE); +} + +/* * Build a single {M,F,A,Loction} item to be part of * a stack trace. */ @@ -5497,7 +5848,7 @@ code_module_md5_1(BIF_ALIST_1) res = am_undefined; goto done; } - res = new_binary(p, stp->mod_md5, sizeof(stp->mod_md5)); + res = new_binary(p, stp->mod_md5, MD5_SIZE); done: erts_free_aligned_binary_bytes(temp_alloc); @@ -5790,6 +6141,7 @@ erts_make_stub_module(Process* p, Eterm Mod, Eterm Beam, Eterm Info) LoaderState* stp; BeamInstr Funcs; BeamInstr Patchlist; + Eterm MD5Bin; Eterm* tp; BeamInstr* code = NULL; BeamInstr* ptrs; @@ -5818,13 +6170,16 @@ erts_make_stub_module(Process* p, Eterm Mod, Eterm Beam, Eterm Info) goto error; } tp = tuple_val(Info); - if (tp[0] != make_arityval(2)) { + if (tp[0] != make_arityval(3)) { goto error; } Funcs = tp[1]; - Patchlist = tp[2]; - - if ((n = list_length(Funcs)) < 0) { + Patchlist = tp[2]; + MD5Bin = tp[3]; + if (is_not_binary(MD5Bin) || (binary_size(MD5Bin) != MD5_SIZE)) { + goto error; + } + if ((n = erts_list_length(Funcs)) < 0) { goto error; } if ((bytes = erts_get_aligned_binary_bytes(Beam, &temp_alloc)) == NULL) { @@ -5873,6 +6228,7 @@ erts_make_stub_module(Process* p, Eterm Mod, Eterm Beam, Eterm Info) code_size = ((WORDS_PER_FUNCTION+1)*n + MI_FUNCTIONS + 2) * sizeof(BeamInstr); code_size += stp->chunks[ATTR_CHUNK].size; code_size += stp->chunks[COMPILE_CHUNK].size; + code_size += MD5_SIZE; code = erts_alloc_fnf(ERTS_ALC_T_CODE, code_size); if (!code) { goto error; @@ -5893,6 +6249,8 @@ erts_make_stub_module(Process* p, Eterm Mod, Eterm Beam, Eterm Info) code[MI_LITERALS_END] = 0; code[MI_LITERALS_OFF_HEAP] = 0; code[MI_ON_LOAD_FUNCTION_PTR] = 0; + code[MI_LINE_TABLE] = 0; + code[MI_MD5_PTR] = 0; ci = MI_FUNCTIONS + n + 1; /* @@ -5978,6 +6336,15 @@ erts_make_stub_module(Process* p, Eterm Mod, Eterm Beam, Eterm Info) if (info == NULL) { goto error; } + { + byte *tmp = NULL; + byte *md5 = NULL; + if ((md5 = erts_get_aligned_binary_bytes(MD5Bin, &tmp)) != NULL) { + sys_memcpy(info, md5, MD5_SIZE); + code[MI_MD5_PTR] = (BeamInstr) info; + } + erts_free_aligned_binary_bytes(tmp); + } /* * Insert the module in the module table. diff --git a/erts/emulator/beam/beam_load.h b/erts/emulator/beam/beam_load.h index 65a8f26d7c..d5af634fad 100644 --- a/erts/emulator/beam/beam_load.h +++ b/erts/emulator/beam/beam_load.h @@ -3,16 +3,17 @@ * * Copyright Ericsson AB 1999-2013. All Rights Reserved. * - * The contents of this file are subject to the Erlang Public License, - * Version 1.1, (the "License"); you may not use this file except in - * compliance with the License. You should have received a copy of the - * Erlang Public License along with this software. If not, it can be - * retrieved online at http://www.erlang.org/. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Software distributed under the License is distributed on an "AS IS" - * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See - * the License for the specific language governing rights and limitations - * under the License. + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. * * %CopyrightEnd% */ @@ -23,10 +24,10 @@ #include "beam_opcodes.h" #include "erl_process.h" +int erts_is_module_native(BeamInstr* code); Eterm beam_make_current_old(Process *c_p, ErtsProcLocks c_p_locks, Eterm module); - typedef struct gen_op_entry { char* name; int arity; @@ -49,6 +50,7 @@ extern void** beam_ops; extern BeamInstr beam_debug_apply[]; extern BeamInstr* em_call_error_handler; extern BeamInstr* em_apply_bif; +extern BeamInstr* em_call_nif; /* * The following variables keep a sorted list of address ranges for @@ -90,7 +92,6 @@ extern Uint erts_total_code_size; #define MI_LITERALS_END 8 #define MI_LITERALS_OFF_HEAP 9 - /* * Pointer to the on_load function (or NULL if none). */ @@ -102,6 +103,11 @@ extern Uint erts_total_code_size; #define MI_LINE_TABLE 11 /* + * Pointer to the module MD5 sum (16 bytes) + */ +#define MI_MD5_PTR 12 + +/* * Start of function pointer table. This table contains pointers to * all functions in the module plus an additional pointer just beyond * the end of the last function. @@ -110,7 +116,7 @@ extern Uint erts_total_code_size; * this table. */ -#define MI_FUNCTIONS 12 +#define MI_FUNCTIONS 13 /* * Layout of the line table. diff --git a/erts/emulator/beam/beam_ranges.c b/erts/emulator/beam/beam_ranges.c index 0f2d5d0c2a..19079ba150 100644 --- a/erts/emulator/beam/beam_ranges.c +++ b/erts/emulator/beam/beam_ranges.c @@ -3,16 +3,17 @@ * * Copyright Ericsson AB 2012. All Rights Reserved. * - * The contents of this file are subject to the Erlang Public License, - * Version 1.1, (the "License"); you may not use this file except in - * compliance with the License. You should have received a copy of the - * Erlang Public License along with this software. If not, it can be - * retrieved online at http://www.erlang.org/. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Software distributed under the License is distributed on an "AS IS" - * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See - * the License for the specific language governing rights and limitations - * under the License. + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. * * %CopyrightEnd% */ @@ -282,7 +283,7 @@ find_range(BeamInstr* pc) while (low < high) { if (pc < mid->start) { high = mid; - } else if (pc > RANGE_END(mid)) { + } else if (pc >= RANGE_END(mid)) { low = mid + 1; } else { erts_smp_atomic_set_nob(&r[active].mid, (erts_aint_t) mid); diff --git a/erts/emulator/beam/benchmark.c b/erts/emulator/beam/benchmark.c index 8613131176..44f5c760c2 100644 --- a/erts/emulator/beam/benchmark.c +++ b/erts/emulator/beam/benchmark.c @@ -3,16 +3,17 @@ * * Copyright Ericsson AB 2002-2012. All Rights Reserved. * - * The contents of this file are subject to the Erlang Public License, - * Version 1.1, (the "License"); you may not use this file except in - * compliance with the License. You should have received a copy of the - * Erlang Public License along with this software. If not, it can be - * retrieved online at http://www.erlang.org/. - * - * Software distributed under the License is distributed on an "AS IS" - * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See - * the License for the specific language governing rights and limitations - * under the License. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. * * %CopyrightEnd% */ @@ -37,37 +38,9 @@ unsigned long long major_gc; #ifdef BM_TIMERS -#if (defined(__i386__) || defined(__x86_64__)) && USE_PERFCTR - -#include "libperfctr.h" -struct vperfctr *system_clock; -double cpu_khz; -BM_NEW_TIMER(start); - -static double get_hrvtime(void) -{ - unsigned long long ticks; - double milli_seconds; - - ticks = vperfctr_read_tsc(system_clock); - milli_seconds = (double)ticks / cpu_khz; - return milli_seconds; -} - -static void stop_hrvtime(void) -{ - if(system_clock) - { - vperfctr_stop(system_clock); - vperfctr_close(system_clock); - system_clock = NULL; - } -} - -#else /* not perfctr, asuming Solaris */ +/* assuming Solaris */ #include <time.h> BM_TIMER_T system_clock; -#endif unsigned long local_pause_times[MAX_PAUSE_TIME]; unsigned long pause_times[MAX_PAUSE_TIME]; @@ -117,40 +90,6 @@ unsigned long long message_sizes[1000]; void init_benchmarking() { #ifdef BM_TIMERS -#if (defined(__i386__) || defined(__x86_64__)) && USE_PERFCTR - /* pass `--with-perfctr=/path/to/perfctr' when configuring */ - struct perfctr_info info; - struct vperfctr_control control; - int i; - - system_clock = vperfctr_open(); - if (system_clock != NULL) - { - if (vperfctr_info(system_clock,&info) >= 0) - { - cpu_khz = (double)info.cpu_khz; - if (info.cpu_features & PERFCTR_FEATURE_RDTSC) - { - memset(&control,0,sizeof control); - control.cpu_control.tsc_on = 1; - } - } - if (vperfctr_control(system_clock,&control) < 0) - { - vperfctr_close(system_clock); - system_clock = NULL; - } - } - - for (i = 0; i < 1000; i++) - { - BM_START_TIMER(system); - BM_STOP_TIMER(system); - } - - timer_time = system_time / 1000; - start_time = 0; -#else int i; for (i = 0; i < 1000; i++) { @@ -158,7 +97,6 @@ void init_benchmarking() BM_STOP_TIMER(system); } timer_time = system_time / 1000; -#endif for (i = 0; i < MAX_PAUSE_TIME; i++) { local_pause_times[i] = 0; diff --git a/erts/emulator/beam/benchmark.h b/erts/emulator/beam/benchmark.h index 766edaac42..0fb0b93f12 100644 --- a/erts/emulator/beam/benchmark.h +++ b/erts/emulator/beam/benchmark.h @@ -3,16 +3,17 @@ * * Copyright Ericsson AB 2002-2012. All Rights Reserved. * - * The contents of this file are subject to the Erlang Public License, - * Version 1.1, (the "License"); you may not use this file except in - * compliance with the License. You should have received a copy of the - * Erlang Public License along with this software. If not, it can be - * retrieved online at http://www.erlang.org/. - * - * Software distributed under the License is distributed on an "AS IS" - * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See - * the License for the specific language governing rights and limitations - * under the License. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. * * %CopyrightEnd% */ @@ -37,10 +38,7 @@ /* BM_TIMERS keeps track of the time spent in diferent parts of the * system. It only measures accual active time, not time spent in idle - * mode. These timers requires hardware support. For Linux, use the - * package perfctr from user.it.uu.se/~mikpe/linux/perfctr. If this - * package is not specified when configuring the system - * (--with-perfctr=PATH), the Solaris hrtime_t will be used. + * mode. Currently, the Solaris hrtime_t will be used. * To add new timers look below. */ #define BM_TIMERS @@ -142,43 +140,12 @@ extern unsigned long long major_gc; * meassure (send time in shared heap for instance). */ -#if (defined(__i386__) || defined(__x86_64__)) && USE_PERFCTR -#include "libperfctr.h" - -#define BM_TIMER_T double - -extern struct vperfctr *system_clock; -extern double cpu_khz; -extern BM_TIMER_T start_time; - -#define BM_START_TIMER(t) start_time = \ - (BM_TIMER_T)vperfctr_read_tsc(system_clock) / \ - cpu_khz; - -#define BM_STOP_TIMER(t) do { \ - BM_TIMER_T tmp = ((BM_TIMER_T)vperfctr_read_tsc(system_clock) / cpu_khz); \ - tmp -= (start_time + timer_time); \ - t##_time += (tmp > 0 ? tmp : 0); \ -} while(0) - -#define BM_TIME_PRINTER(str,time) do { \ - int min,sec,milli,micro; \ - BM_TIMER_T tmp = (time) * 1000; \ - micro = (uint)(tmp - ((int)(tmp / 1000)) * 1000); \ - tmp /= 1000; \ - milli = (uint)(tmp - ((int)(tmp / 1000)) * 1000); \ - tmp /= 1000; \ - sec = (uint)(tmp - ((int)(tmp / 60)) * 60); \ - min = (uint)tmp / 60; \ - erts_fprintf(file,str": %d:%02d.%03d %03d\n",min,sec,milli,micro); \ -} while(0) - -#else /* !USE_PERFCTR (Assuming Solaris) */ +/* (Assuming Solaris) */ -#define BM_TIMER_T hrtime_t -#define BM_START_TIMER(t) system_clock = sys_gethrtime() +#define BM_TIMER_T ErtsMonotonicTime +#define BM_START_TIMER(t) system_clock = ERTS_MONOTONIC_TO_NSEC(erts_os_monotonic_time()) #define BM_STOP_TIMER(t) do { \ - BM_TIMER_T tmp = (sys_gethrtime() - system_clock) - timer_time; \ + BM_TIMER_T tmp = (ERTS_MONOTONIC_TO_NSEC(erts_os_monotonic_time()) - system_clock) - timer_time; \ t##_time += (tmp > 0 ? tmp : 0); \ } while(0) @@ -196,7 +163,6 @@ extern BM_TIMER_T start_time; } while(0) extern BM_TIMER_T system_clock; -#endif /* USE_PERFCTR */ extern BM_TIMER_T timer_time; extern BM_TIMER_T system_time; diff --git a/erts/emulator/beam/bif.c b/erts/emulator/beam/bif.c index 9c438679ea..66c2853534 100644 --- a/erts/emulator/beam/bif.c +++ b/erts/emulator/beam/bif.c @@ -1,18 +1,19 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 1996-2013. All Rights Reserved. + * Copyright Ericsson AB 1996-2014. All Rights Reserved. * - * The contents of this file are subject to the Erlang Public License, - * Version 1.1, (the "License"); you may not use this file except in - * compliance with the License. You should have received a copy of the - * Erlang Public License along with this software. If not, it can be - * retrieved online at http://www.erlang.org/. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Software distributed under the License is distributed on an "AS IS" - * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See - * the License for the specific language governing rights and limitations - * under the License. + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. * * %CopyrightEnd% */ @@ -28,7 +29,9 @@ #include "global.h" #include "erl_process.h" #include "error.h" +#define ERL_WANT_HIPE_BIF_WRAPPER__ #include "bif.h" +#undef ERL_WANT_HIPE_BIF_WRAPPER__ #include "big.h" #include "dist.h" #include "erl_version.h" @@ -40,16 +43,23 @@ #define ERTS_PTAB_WANT_BIF_IMPL__ #include "erl_ptab.h" #include "erl_bits.h" +#include "erl_bif_unique.h" -static Export* flush_monitor_message_trap = NULL; +Export *erts_await_result; +static Export* flush_monitor_messages_trap = NULL; static Export* set_cpu_topology_trap = NULL; static Export* await_proc_exit_trap = NULL; static Export* await_port_send_result_trap = NULL; Export* erts_format_cpu_topology_trap = NULL; +static Export dsend_continue_trap_export; +Export *erts_convert_time_unit_trap = NULL; static Export *await_sched_wall_time_mod_trap; static erts_smp_atomic32_t sched_wall_time; +static erts_smp_mtx_t ports_snapshot_mtx; +erts_smp_atomic_t erts_dead_ports_ptr; /* To store dying ports during snapshot */ + #define DECL_AM(S) Eterm AM_ ## S = am_atom_put(#S, sizeof(#S) - 1) /* @@ -160,7 +170,10 @@ BIF_RETTYPE link_1(BIF_ALIST_1) if (is_internal_port(BIF_ARG_1)) { int send_link_signal = 0; - Port *prt = erts_port_lookup(BIF_ARG_1, ERTS_PORT_SFLGS_INVALID_LOOKUP); + Port *prt = erts_port_lookup(BIF_ARG_1, + (erts_port_synchronous_ops + ? ERTS_PORT_SFLGS_INVALID_DRIVER_LOOKUP + : ERTS_PORT_SFLGS_INVALID_LOOKUP)); if (!prt) { goto res_no_proc; } @@ -388,7 +401,7 @@ remote_demonitor(Process *c_p, DistEntry *dep, Eterm ref, Eterm to) return res; } -static int demonitor(Process *c_p, Eterm ref) +static int demonitor(Process *c_p, Eterm ref, Eterm *multip) { ErtsMonitor *mon = NULL; /* The monitor entry to delete */ Process *rp; /* Local target process */ @@ -412,65 +425,73 @@ static int demonitor(Process *c_p, Eterm ref) goto done; } - if (mon->type != MON_ORIGIN) { - res = ERTS_DEMONITOR_BADARG; - goto done; - } - to = mon->pid; - - if (is_atom(to)) { - /* Monitoring a name at node to */ - ASSERT(is_node_name_atom(to)); - dep = erts_sysname_to_connected_dist_entry(to); - ASSERT(dep != erts_this_dist_entry); - if (dep) - deref_de = 1; - } else { - ASSERT(is_pid(to)); - dep = pid_dist_entry(to); - } - if (dep != erts_this_dist_entry) { - res = remote_demonitor(c_p, dep, ref, to); - /* remote_demonitor() unlocks link lock on c_p */ - unlock_link = 0; - } - else { /* Local monitor */ - if (deref_de) { - deref_de = 0; - erts_deref_dist_entry(dep); + switch (mon->type) { + case MON_TIME_OFFSET: + *multip = am_true; + erts_demonitor_time_offset(ref); + res = ERTS_DEMONITOR_TRUE; + break; + case MON_ORIGIN: + to = mon->pid; + *multip = am_false; + if (is_atom(to)) { + /* Monitoring a name at node to */ + ASSERT(is_node_name_atom(to)); + dep = erts_sysname_to_connected_dist_entry(to); + ASSERT(dep != erts_this_dist_entry); + if (dep) + deref_de = 1; + } else { + ASSERT(is_pid(to)); + dep = pid_dist_entry(to); } - dep = NULL; - rp = erts_pid2proc_opt(c_p, - ERTS_PROC_LOCK_MAIN|ERTS_PROC_LOCK_LINK, - to, - ERTS_PROC_LOCK_LINK, - ERTS_P2P_FLG_ALLOW_OTHER_X); - mon = erts_remove_monitor(&ERTS_P_MONITORS(c_p), ref); + if (dep != erts_this_dist_entry) { + res = remote_demonitor(c_p, dep, ref, to); + /* remote_demonitor() unlocks link lock on c_p */ + unlock_link = 0; + } + else { /* Local monitor */ + if (deref_de) { + deref_de = 0; + erts_deref_dist_entry(dep); + } + dep = NULL; + rp = erts_pid2proc_opt(c_p, + ERTS_PROC_LOCK_MAIN|ERTS_PROC_LOCK_LINK, + to, + ERTS_PROC_LOCK_LINK, + ERTS_P2P_FLG_ALLOW_OTHER_X); + mon = erts_remove_monitor(&ERTS_P_MONITORS(c_p), ref); #ifndef ERTS_SMP - ASSERT(mon); + ASSERT(mon); #else - if (!mon) - res = ERTS_DEMONITOR_FALSE; - else + if (!mon) + res = ERTS_DEMONITOR_FALSE; + else #endif - { - res = ERTS_DEMONITOR_TRUE; - erts_destroy_monitor(mon); - } - if (rp) { - ErtsMonitor *rmon; - rmon = erts_remove_monitor(&ERTS_P_MONITORS(rp), ref); - if (rp != c_p) - erts_smp_proc_unlock(rp, ERTS_PROC_LOCK_LINK); - if (rmon != NULL) - erts_destroy_monitor(rmon); - } - else { - ERTS_SMP_ASSERT_IS_NOT_EXITING(c_p); - } + { + res = ERTS_DEMONITOR_TRUE; + erts_destroy_monitor(mon); + } + if (rp) { + ErtsMonitor *rmon; + rmon = erts_remove_monitor(&ERTS_P_MONITORS(rp), ref); + if (rp != c_p) + erts_smp_proc_unlock(rp, ERTS_PROC_LOCK_LINK); + if (rmon != NULL) + erts_destroy_monitor(rmon); + } + else { + ERTS_SMP_ASSERT_IS_NOT_EXITING(c_p); + } + } + break; + default: + res = ERTS_DEMONITOR_BADARG; + *multip = am_false; + break; } - done: if (unlock_link) @@ -487,7 +508,8 @@ static int demonitor(Process *c_p, Eterm ref) BIF_RETTYPE demonitor_1(BIF_ALIST_1) { - switch (demonitor(BIF_P, BIF_ARG_1)) { + Eterm multi; + switch (demonitor(BIF_P, BIF_ARG_1, &multi)) { case ERTS_DEMONITOR_FALSE: case ERTS_DEMONITOR_TRUE: BIF_RET(am_true); @@ -505,6 +527,7 @@ BIF_RETTYPE demonitor_1(BIF_ALIST_1) BIF_RETTYPE demonitor_2(BIF_ALIST_2) { Eterm res = am_true; + Eterm multi = am_false; int info = 0; int flush = 0; Eterm list = BIF_ARG_2; @@ -527,13 +550,18 @@ BIF_RETTYPE demonitor_2(BIF_ALIST_2) if (is_not_nil(list)) goto badarg; - switch (demonitor(BIF_P, BIF_ARG_1)) { + switch (demonitor(BIF_P, BIF_ARG_1, &multi)) { case ERTS_DEMONITOR_FALSE: if (info) res = am_false; - if (flush) - BIF_TRAP2(flush_monitor_message_trap, BIF_P, BIF_ARG_1, res); + if (flush) { + flush_messages: + BIF_TRAP3(flush_monitor_messages_trap, BIF_P, + BIF_ARG_1, multi, res); + } case ERTS_DEMONITOR_TRUE: + if (multi == am_true && flush) + goto flush_messages; BIF_RET(res); case ERTS_DEMONITOR_YIELD_TRUE: ERTS_BIF_YIELD_RETURN(BIF_P, am_true); @@ -584,22 +612,16 @@ erts_queue_monitor_message(Process *p, ref_copy = copy_struct(ref, ref_size, &hp, ohp); tup = TUPLE5(hp, am_DOWN, ref_copy, type, item_copy, reason_copy); - erts_queue_message(p, p_locksp, bp, tup, NIL -#ifdef USE_VM_PROBES - , NIL -#endif - ); + erts_queue_message(p, p_locksp, bp, tup, NIL); } static BIF_RETTYPE -local_pid_monitor(Process *p, Eterm target) +local_pid_monitor(Process *p, Eterm target, Eterm mon_ref, int boolean) { BIF_RETTYPE ret; - Eterm mon_ref; Process *rp; ErtsProcLocks p_locks = ERTS_PROC_LOCK_MAIN|ERTS_PROC_LOCK_LINK; - mon_ref = erts_make_ref(p); ERTS_BIF_PREP_RET(ret, mon_ref); if (target == p->common.id) { return ret; @@ -612,12 +634,18 @@ local_pid_monitor(Process *p, Eterm target) if (!rp) { erts_smp_proc_unlock(p, ERTS_PROC_LOCK_LINK); p_locks &= ~ERTS_PROC_LOCK_LINK; - erts_queue_monitor_message(p, &p_locks, - mon_ref, am_process, target, am_noproc); + if (boolean) + ret = am_false; + else + erts_queue_monitor_message(p, &p_locks, + mon_ref, am_process, target, am_noproc); } else { ASSERT(rp != p); + if (boolean) + ret = am_true; + erts_add_monitor(&ERTS_P_MONITORS(p), MON_ORIGIN, mon_ref, target, NIL); erts_add_monitor(&ERTS_P_MONITORS(rp), MON_TARGET, mon_ref, p->common.id, NIL); @@ -741,13 +769,28 @@ BIF_RETTYPE monitor_2(BIF_ALIST_2) int deref_de = 0; /* Only process monitors are implemented */ - if (BIF_ARG_1 != am_process) { + switch (BIF_ARG_1) { + case am_time_offset: { + Eterm ref; + if (BIF_ARG_2 != am_clock_service) + goto error; + ref = erts_make_ref(BIF_P); + erts_smp_proc_lock(BIF_P, ERTS_PROC_LOCK_LINK); + erts_add_monitor(&ERTS_P_MONITORS(BIF_P), MON_TIME_OFFSET, + ref, am_clock_service, NIL); + erts_smp_proc_unlock(BIF_P, ERTS_PROC_LOCK_LINK); + erts_monitor_time_offset(BIF_P->common.id, ref); + BIF_RET(ref); + } + case am_process: + break; + default: goto error; } if (is_internal_pid(target)) { local_pid: - ret = local_pid_monitor(BIF_P, target); + ret = local_pid_monitor(BIF_P, target, erts_make_ref(BIF_P), 0); } else if (is_external_pid(target)) { dep = external_pid_dist_entry(target); if (dep == erts_this_dist_entry) @@ -790,7 +833,6 @@ BIF_RETTYPE monitor_2(BIF_ALIST_2) return ret; } - /**********************************************************************/ /* this is a combination of the spawn and link BIFs */ @@ -1363,11 +1405,22 @@ BIF_RETTYPE exit_2(BIF_ALIST_2) */ if (is_internal_port(BIF_ARG_1)) { - Port *prt = erts_port_lookup(BIF_ARG_1, ERTS_PORT_SFLGS_INVALID_LOOKUP); + Eterm ref, *refp; + Uint32 invalid_flags; + Port *prt; + + if (erts_port_synchronous_ops) { + refp = &ref; + invalid_flags = ERTS_PORT_SFLGS_INVALID_DRIVER_LOOKUP; + } + else { + refp = NULL; + invalid_flags = ERTS_PORT_SFLGS_INVALID_LOOKUP; + } + + prt = erts_port_lookup(BIF_ARG_1, invalid_flags); if (prt) { - Eterm ref; - Eterm *refp = erts_port_synchronous_ops ? &ref : NULL; ErtsPortOpResult res; #ifdef DEBUG @@ -1763,6 +1816,8 @@ BIF_RETTYPE whereis_1(BIF_ALIST_1) * erlang:'!'/2 */ +HIPE_WRAPPER_BIF_DISABLE_GC(ebif_bang, 2) + BIF_RETTYPE ebif_bang_2(BIF_ALIST_2) { @@ -1781,34 +1836,36 @@ ebif_bang_2(BIF_ALIST_2) #define SEND_USER_ERROR (-5) #define SEND_INTERNAL_ERROR (-6) #define SEND_AWAIT_RESULT (-7) +#define SEND_YIELD_CONTINUE (-8) + -Sint do_send(Process *p, Eterm to, Eterm msg, int suspend, Eterm *refp); +Sint do_send(Process *p, Eterm to, Eterm msg, Eterm *refp, ErtsSendContext*); static Sint remote_send(Process *p, DistEntry *dep, - Eterm to, Eterm full_to, Eterm msg, int suspend) + Eterm to, Eterm full_to, Eterm msg, + ErtsSendContext* ctx) { Sint res; int code; - ErtsDSigData dsd; ASSERT(is_atom(to) || is_external_pid(to)); - code = erts_dsig_prepare(&dsd, dep, p, ERTS_DSP_NO_LOCK, !suspend); + code = erts_dsig_prepare(&ctx->dsd, dep, p, ERTS_DSP_NO_LOCK, !ctx->suspend); switch (code) { case ERTS_DSIG_PREP_NOT_ALIVE: case ERTS_DSIG_PREP_NOT_CONNECTED: res = SEND_TRAP; break; case ERTS_DSIG_PREP_WOULD_SUSPEND: - ASSERT(!suspend); + ASSERT(!ctx->suspend); res = SEND_YIELD; break; case ERTS_DSIG_PREP_CONNECTED: { if (is_atom(to)) - code = erts_dsig_send_reg_msg(&dsd, to, msg); + code = erts_dsig_send_reg_msg(to, msg, ctx); else - code = erts_dsig_send_msg(&dsd, to, msg); + code = erts_dsig_send_msg(to, msg, ctx); /* * Note that reductions have been bumped on calling * process by erts_dsig_send_reg_msg() or @@ -1816,6 +1873,8 @@ static Sint remote_send(Process *p, DistEntry *dep, */ if (code == ERTS_DSIG_SEND_YIELD) res = SEND_YIELD_RETURN; + else if (code == ERTS_DSIG_SEND_CONTINUE) + res = SEND_YIELD_CONTINUE; else res = 0; break; @@ -1836,7 +1895,8 @@ static Sint remote_send(Process *p, DistEntry *dep, } Sint -do_send(Process *p, Eterm to, Eterm msg, int suspend, Eterm *refp) { +do_send(Process *p, Eterm to, Eterm msg, Eterm *refp, ErtsSendContext* ctx) +{ Eterm portid; Port *pt; Process* rp; @@ -1867,15 +1927,23 @@ do_send(Process *p, Eterm to, Eterm msg, int suspend, Eterm *refp) { erts_send_error_to_logger(p->group_leader, dsbufp); return 0; } - return remote_send(p, dep, to, to, msg, suspend); + return remote_send(p, dep, to, to, msg, ctx); } else if (is_atom(to)) { Eterm id = erts_whereis_name_to_id(p, to); rp = erts_proc_lookup(id); - if (rp) + if (rp) { + if (IS_TRACED(p)) + trace_send(p, to, msg); + if (ERTS_PROC_GET_SAVED_CALLS_BUF(p)) + save_calls(p, &exp_send); goto send_message; + } - pt = erts_port_lookup(id, ERTS_PORT_SFLGS_INVALID_LOOKUP); + pt = erts_port_lookup(id, + (erts_port_synchronous_ops + ? ERTS_PORT_SFLGS_INVALID_DRIVER_LOOKUP + : ERTS_PORT_SFLGS_INVALID_LOOKUP)); if (pt) { portid = id; goto port_common; @@ -1905,13 +1973,16 @@ do_send(Process *p, Eterm to, Eterm msg, int suspend, Eterm *refp) { int ret_val; portid = to; - pt = erts_port_lookup(portid, ERTS_PORT_SFLGS_INVALID_LOOKUP); + pt = erts_port_lookup(portid, + (erts_port_synchronous_ops + ? ERTS_PORT_SFLGS_INVALID_DRIVER_LOOKUP + : ERTS_PORT_SFLGS_INVALID_LOOKUP)); port_common: ret_val = 0; if (pt) { - int ps_flags = suspend ? 0 : ERTS_PORT_SIG_FLG_NOSUSPEND; + int ps_flags = ctx->suspend ? 0 : ERTS_PORT_SIG_FLG_NOSUSPEND; *refp = NIL; switch (erts_port_command(p, ps_flags, pt, msg, refp)) { @@ -1920,12 +1991,12 @@ do_send(Process *p, Eterm to, Eterm msg, int suspend, Eterm *refp) { return SEND_USER_ERROR; case ERTS_PORT_OP_BUSY: /* Nothing has been sent */ - if (suspend) + if (ctx->suspend) erts_suspend(p, ERTS_PROC_LOCK_MAIN, pt); return SEND_YIELD; case ERTS_PORT_OP_BUSY_SCHEDULED: /* Message was sent */ - if (suspend) { + if (ctx->suspend) { erts_suspend(p, ERTS_PROC_LOCK_MAIN, pt); ret_val = SEND_YIELD_RETURN; break; @@ -1994,7 +2065,10 @@ do_send(Process *p, Eterm to, Eterm msg, int suspend, Eterm *refp) { rp = erts_proc_lookup_raw(id); if (rp) goto send_message; - pt = erts_port_lookup(id, ERTS_PORT_SFLGS_INVALID_LOOKUP); + pt = erts_port_lookup(id, + (erts_port_synchronous_ops + ? ERTS_PORT_SFLGS_INVALID_DRIVER_LOOKUP + : ERTS_PORT_SFLGS_INVALID_LOOKUP)); if (pt) { portid = id; goto port_common; @@ -2002,9 +2076,14 @@ do_send(Process *p, Eterm to, Eterm msg, int suspend, Eterm *refp) { return 0; } - ret = remote_send(p, dep, tp[1], to, msg, suspend); - if (dep) - erts_deref_dist_entry(dep); + ret = remote_send(p, dep, tp[1], to, msg, ctx); + if (ret != SEND_YIELD_CONTINUE) { + if (dep) { + erts_deref_dist_entry(dep); + } + } else { + ctx->dep_to_deref = dep; + } return ret; } else { if (IS_TRACED(p)) /* XXX Is this really neccessary ??? */ @@ -2035,9 +2114,11 @@ do_send(Process *p, Eterm to, Eterm msg, int suspend, Eterm *refp) { } } +HIPE_WRAPPER_BIF_DISABLE_GC(send, 3) BIF_RETTYPE send_3(BIF_ALIST_3) { + BIF_RETTYPE retval; Eterm ref; Process *p = BIF_P; Eterm to = BIF_ARG_1; @@ -2045,34 +2126,44 @@ BIF_RETTYPE send_3(BIF_ALIST_3) Eterm opts = BIF_ARG_3; int connect = !0; - int suspend = !0; Eterm l = opts; Sint result; - + DeclareTypedTmpHeap(ErtsSendContext, ctx, BIF_P); + UseTmpHeap(sizeof(ErtsSendContext)/sizeof(Eterm), BIF_P); + + ctx->suspend = !0; + ctx->dep_to_deref = NULL; + ctx->return_term = am_ok; + ctx->dss.reds = (Sint) (ERTS_BIF_REDS_LEFT(p) * TERM_TO_BINARY_LOOP_FACTOR); + ctx->dss.phase = ERTS_DSIG_SEND_PHASE_INIT; + while (is_list(l)) { if (CAR(list_val(l)) == am_noconnect) { connect = 0; } else if (CAR(list_val(l)) == am_nosuspend) { - suspend = 0; + ctx->suspend = 0; } else { - BIF_ERROR(p, BADARG); + ERTS_BIF_PREP_ERROR(retval, p, BADARG); + goto done; } l = CDR(list_val(l)); } if(!is_nil(l)) { - BIF_ERROR(p, BADARG); + ERTS_BIF_PREP_ERROR(retval, p, BADARG); + goto done; } #ifdef DEBUG ref = NIL; #endif - result = do_send(p, to, msg, suspend, &ref); + result = do_send(p, to, msg, &ref, ctx); if (result > 0) { ERTS_VBUMP_REDS(p, result); if (ERTS_IS_PROC_OUT_OF_REDS(p)) goto yield_return; - BIF_RET(am_ok); + ERTS_BIF_PREP_RET(retval, am_ok); + goto done; } switch (result) { @@ -2080,68 +2171,127 @@ BIF_RETTYPE send_3(BIF_ALIST_3) /* May need to yield even though we do not bump reds here... */ if (ERTS_IS_PROC_OUT_OF_REDS(p)) goto yield_return; - BIF_RET(am_ok); + ERTS_BIF_PREP_RET(retval, am_ok); break; case SEND_TRAP: if (connect) { - BIF_TRAP3(dsend3_trap, p, to, msg, opts); + ERTS_BIF_PREP_TRAP3(retval, dsend3_trap, p, to, msg, opts); } else { - BIF_RET(am_noconnect); + ERTS_BIF_PREP_RET(retval, am_noconnect); } break; case SEND_YIELD: - if (suspend) { - ERTS_BIF_YIELD3(bif_export[BIF_send_3], p, to, msg, opts); + if (ctx->suspend) { + ERTS_BIF_PREP_YIELD3(retval, + bif_export[BIF_send_3], p, to, msg, opts); } else { - BIF_RET(am_nosuspend); + ERTS_BIF_PREP_RET(retval, am_nosuspend); } break; case SEND_YIELD_RETURN: - if (!suspend) - BIF_RET(am_nosuspend); + if (!ctx->suspend) { + ERTS_BIF_PREP_RET(retval, am_nosuspend); + break; + } yield_return: - ERTS_BIF_YIELD_RETURN(p, am_ok); + ERTS_BIF_PREP_YIELD_RETURN(retval, p, am_ok); + break; case SEND_AWAIT_RESULT: ASSERT(is_internal_ref(ref)); - BIF_TRAP3(await_port_send_result_trap, p, ref, am_nosuspend, am_ok); + ERTS_BIF_PREP_TRAP3(retval, await_port_send_result_trap, p, ref, am_nosuspend, am_ok); + break; case SEND_BADARG: - BIF_ERROR(p, BADARG); + ERTS_BIF_PREP_ERROR(retval, p, BADARG); break; case SEND_USER_ERROR: - BIF_ERROR(p, EXC_ERROR); + ERTS_BIF_PREP_ERROR(retval, p, EXC_ERROR); break; case SEND_INTERNAL_ERROR: - BIF_ERROR(p, EXC_INTERNAL_ERROR); + ERTS_BIF_PREP_ERROR(retval, p, EXC_INTERNAL_ERROR); + break; + case SEND_YIELD_CONTINUE: + BUMP_ALL_REDS(p); + erts_set_gc_state(p, 0); + ERTS_BIF_PREP_TRAP1(retval, &dsend_continue_trap_export, p, + erts_dsend_export_trap_context(p, ctx)); break; default: - ASSERT(! "Illegal send result"); + erts_exit(ERTS_ABORT_EXIT, "send_3 invalid result %d\n", (int)result); break; } - ASSERT(! "Can not arrive here"); - BIF_ERROR(p, BADARG); + +done: + UnUseTmpHeap(sizeof(ErtsSendContext)/sizeof(Eterm), BIF_P); + return retval; } +HIPE_WRAPPER_BIF_DISABLE_GC(send, 2) + BIF_RETTYPE send_2(BIF_ALIST_2) { return erl_send(BIF_P, BIF_ARG_1, BIF_ARG_2); } +static BIF_RETTYPE dsend_continue_trap_1(BIF_ALIST_1) +{ + Binary* bin = ((ProcBin*) binary_val(BIF_ARG_1))->val; + ErtsSendContext* ctx = (ErtsSendContext*) ERTS_MAGIC_BIN_DATA(bin); + Sint initial_reds = (Sint) (ERTS_BIF_REDS_LEFT(BIF_P) * TERM_TO_BINARY_LOOP_FACTOR); + int result; + + ASSERT(ERTS_MAGIC_BIN_DESTRUCTOR(bin) == erts_dsend_context_dtor); + + ctx->dss.reds = initial_reds; + result = erts_dsig_send(&ctx->dsd, &ctx->dss); + + switch (result) { + case ERTS_DSIG_SEND_OK: + erts_set_gc_state(BIF_P, 1); + BIF_RET(ctx->return_term); + break; + case ERTS_DSIG_SEND_YIELD: /*SEND_YIELD_RETURN*/ + erts_set_gc_state(BIF_P, 1); + if (!ctx->suspend) + BIF_RET(am_nosuspend); + ERTS_BIF_YIELD_RETURN(BIF_P, ctx->return_term); + + case ERTS_DSIG_SEND_CONTINUE: { /*SEND_YIELD_CONTINUE*/ + BUMP_ALL_REDS(BIF_P); + BIF_TRAP1(&dsend_continue_trap_export, BIF_P, BIF_ARG_1); + } + default: + erts_exit(ERTS_ABORT_EXIT, "dsend_continue_trap invalid result %d\n", (int)result); + break; + } + ASSERT(! "Can not arrive here"); + BIF_ERROR(BIF_P, BADARG); +} + Eterm erl_send(Process *p, Eterm to, Eterm msg) { + Eterm retval; Eterm ref; Sint result; + DeclareTypedTmpHeap(ErtsSendContext, ctx, p); + UseTmpHeap(sizeof(ErtsSendContext)/sizeof(Eterm), p); #ifdef DEBUG ref = NIL; #endif + ctx->suspend = !0; + ctx->dep_to_deref = NULL; + ctx->return_term = msg; + ctx->dss.reds = (Sint) (ERTS_BIF_REDS_LEFT(p) * TERM_TO_BINARY_LOOP_FACTOR); + ctx->dss.phase = ERTS_DSIG_SEND_PHASE_INIT; - result = do_send(p, to, msg, !0, &ref); + result = do_send(p, to, msg, &ref, ctx); if (result > 0) { ERTS_VBUMP_REDS(p, result); if (ERTS_IS_PROC_OUT_OF_REDS(p)) goto yield_return; - BIF_RET(msg); + ERTS_BIF_PREP_RET(retval, msg); + goto done; } switch (result) { @@ -2149,35 +2299,46 @@ Eterm erl_send(Process *p, Eterm to, Eterm msg) /* May need to yield even though we do not bump reds here... */ if (ERTS_IS_PROC_OUT_OF_REDS(p)) goto yield_return; - BIF_RET(msg); + ERTS_BIF_PREP_RET(retval, msg); break; case SEND_TRAP: - BIF_TRAP2(dsend2_trap, p, to, msg); + ERTS_BIF_PREP_TRAP2(retval, dsend2_trap, p, to, msg); break; case SEND_YIELD: - ERTS_BIF_YIELD2(bif_export[BIF_send_2], p, to, msg); + ERTS_BIF_PREP_YIELD2(retval, bif_export[BIF_send_2], p, to, msg); break; case SEND_YIELD_RETURN: yield_return: - ERTS_BIF_YIELD_RETURN(p, msg); + ERTS_BIF_PREP_YIELD_RETURN(retval, p, msg); + break; case SEND_AWAIT_RESULT: ASSERT(is_internal_ref(ref)); - BIF_TRAP3(await_port_send_result_trap, p, ref, msg, msg); + ERTS_BIF_PREP_TRAP3(retval, + await_port_send_result_trap, p, ref, msg, msg); + break; case SEND_BADARG: - BIF_ERROR(p, BADARG); + ERTS_BIF_PREP_ERROR(retval, p, BADARG); break; case SEND_USER_ERROR: - BIF_ERROR(p, EXC_ERROR); + ERTS_BIF_PREP_ERROR(retval, p, EXC_ERROR); break; case SEND_INTERNAL_ERROR: - BIF_ERROR(p, EXC_INTERNAL_ERROR); + ERTS_BIF_PREP_ERROR(retval, p, EXC_INTERNAL_ERROR); + break; + case SEND_YIELD_CONTINUE: + BUMP_ALL_REDS(p); + erts_set_gc_state(p, 0); + ERTS_BIF_PREP_TRAP1(retval, &dsend_continue_trap_export, p, + erts_dsend_export_trap_context(p, ctx)); break; default: - ASSERT(! "Illegal send result"); + erts_exit(ERTS_ABORT_EXIT, "invalid send result %d\n", (int)result); break; } - ASSERT(! "Can not arrive here"); - BIF_ERROR(p, BADARG); + +done: + UnUseTmpHeap(sizeof(ErtsSendContext)/sizeof(Eterm), p); + return retval; } /**********************************************************************/ @@ -2543,7 +2704,7 @@ BIF_RETTYPE insert_element_3(BIF_ALIST_3) Eterm* hp; Uint arity; Eterm res; - Sint ix; + Sint ix, c1, c2; if (is_not_tuple(BIF_ARG_2) || is_not_small(BIF_ARG_1)) { BIF_ERROR(BIF_P, BADARG); @@ -2561,14 +2722,12 @@ BIF_RETTYPE insert_element_3(BIF_ALIST_3) res = make_tuple(hp); *hp = make_arityval(arity + 1); - ix--; - arity -= ix; - - while (ix--) { *++hp = *++ptr; } + c1 = ix - 1; + c2 = arity - ix + 1; + while (c1--) { *++hp = *++ptr; } *++hp = BIF_ARG_3; - - while(arity--) { *++hp = *++ptr; } + while (c2--) { *++hp = *++ptr; } BIF_RET(res); } @@ -2579,7 +2738,7 @@ BIF_RETTYPE delete_element_2(BIF_ALIST_3) Eterm* hp; Uint arity; Eterm res; - Sint ix; + Sint ix, c1, c2; if (is_not_tuple(BIF_ARG_2) || is_not_small(BIF_ARG_1)) { BIF_ERROR(BIF_P, BADARG); @@ -2597,14 +2756,12 @@ BIF_RETTYPE delete_element_2(BIF_ALIST_3) res = make_tuple(hp); *hp = make_arityval(arity - 1); - ix--; - arity -= ix; - - while (ix--) { *++hp = *++ptr; } + c1 = ix - 1; + c2 = arity - ix; + while (c1--) { *++hp = *++ptr; } ++ptr; - - while(arity--) { *++hp = *++ptr; } + while (c2--) { *++hp = *++ptr; } BIF_RET(res); } @@ -2656,7 +2813,7 @@ BIF_RETTYPE list_to_atom_1(BIF_ALIST_1) if (i < 0) { erts_free(ERTS_ALC_T_TMP, (void *) buf); - i = list_length(BIF_ARG_1); + i = erts_list_length(BIF_ARG_1); if (i > MAX_ATOM_CHARACTERS) { BIF_ERROR(BIF_P, SYSTEM_LIMIT); } @@ -2744,9 +2901,10 @@ static int do_list_to_integer(Process *p, Eterm orig_list, Eterm *integer, Eterm *rest) { Sint i = 0; + Uint ui = 0; int skip = 0; int neg = 0; - int n = 0; + Sint n = 0; int m; int lg2; Eterm res; @@ -2797,8 +2955,8 @@ static int do_list_to_integer(Process *p, Eterm orig_list, unsigned_val(CAR(list_val(lst))) > '9') { break; } - i = i * 10; - i = i + unsigned_val(CAR(list_val(lst))) - '0'; + ui = ui * 10; + ui = ui + unsigned_val(CAR(list_val(lst))) - '0'; n++; lst = CDR(list_val(lst)); if (is_nil(lst)) { @@ -2822,10 +2980,13 @@ static int do_list_to_integer(Process *p, Eterm orig_list, */ if (n <= SMALL_DIGITS) { /* It must be small */ - if (neg) i = -i; + if (neg) i = -(Sint)ui; + else i = (Sint)ui; res = make_small(i); } else { - lg2 = (n+1)*230/69+1; + /* Convert from log10 to log2 by multiplying with 1/log10(2)=3.3219 + which we round up to (3 + 1/3) */ + lg2 = (n+1)*3 + (n+1)/3 + 1; m = (lg2+D_EXP-1)/D_EXP; /* number of digits */ m = BIG_NEED_SIZE(m); /* number of words + thing */ @@ -2863,9 +3024,6 @@ static int do_list_to_integer(Process *p, Eterm orig_list, res = big_plus_small(res, m, hp); } - if (is_big(res)) /* check if small */ - res = big_plus_small(res, 0, hp); /* includes conversion to small */ - if (neg) { if (is_small(res)) res = make_small(-signed_val(res)); @@ -2875,8 +3033,12 @@ static int do_list_to_integer(Process *p, Eterm orig_list, } } - if (is_big(res)) { - hp += (big_arity(res)+1); + if (is_not_small(res)) { + res = big_plus_small(res, 0, hp); /* includes conversion to small */ + + if (is_not_small(res)) { + hp += (big_arity(res)+1); + } } HRelease(p,hp_end,hp); } @@ -2934,7 +3096,7 @@ BIF_RETTYPE list_to_integer_2(BIF_ALIST_2) char *buf = NULL; int base; - i = list_length(BIF_ARG_1); + i = erts_list_length(BIF_ARG_1); if (i < 0) BIF_ERROR(BIF_P, BADARG); @@ -3273,7 +3435,7 @@ BIF_RETTYPE list_to_float_1(BIF_ALIST_1) Eterm res; char *buf = NULL; - i = list_length(BIF_ARG_1); + i = erts_list_length(BIF_ARG_1); if (i < 0) BIF_ERROR(BIF_P, BADARG); @@ -3388,7 +3550,7 @@ BIF_RETTYPE list_to_tuple_1(BIF_ALIST_1) Eterm* hp; int len; - if ((len = list_length(list)) < 0 || len > ERTS_MAX_TUPLE_SIZE) { + if ((len = erts_list_length(list)) < 0 || len > ERTS_MAX_TUPLE_SIZE) { BIF_ERROR(BIF_P, BADARG); } @@ -3415,91 +3577,6 @@ BIF_RETTYPE self_0(BIF_ALIST_0) /**********************************************************************/ -/* - New representation of refs in R9, see erl_term.h - - In the first data word, only the usual 18 bits are used. Ordinarily, - in "long refs" all words are used (in other words, practically never - wrap around), but for compatibility with older nodes, "short refs" - exist. Short refs come into being by being converted from the old - external format for refs (tag REFERENCE_EXT). Short refs are - converted back to the old external format. - - When converting a long ref to the external format in the case of - preparing for sending to an older node, the ref is truncated by only - using the first word (with 18 significant bits), and using the old tag - REFERENCE_EXT. - - When comparing refs or different size, only the parts up to the length - of the shorter operand are used. This has the desirable effect that a - long ref sent to an old node and back will be treated as equal to - the original, although some of the bits have been lost. - - The hash value for a ref always considers only the first word, since - in the above scenario, the original and the copy should have the same - hash value. -*/ - -static Uint32 reference0; /* Initialized in erts_init_bif */ -static Uint32 reference1; -static Uint32 reference2; -static erts_smp_spinlock_t make_ref_lock; -static erts_smp_mtx_t ports_snapshot_mtx; -erts_smp_atomic_t erts_dead_ports_ptr; /* To store dying ports during snapshot */ - -void -erts_make_ref_in_array(Uint32 ref[ERTS_MAX_REF_NUMBERS]) -{ - erts_smp_spin_lock(&make_ref_lock); - - reference0++; - if (reference0 >= MAX_REFERENCE) { - reference0 = 0; - reference1++; - if (reference1 == 0) { - reference2++; - } - } - - ref[0] = reference0; - ref[1] = reference1; - ref[2] = reference2; - - erts_smp_spin_unlock(&make_ref_lock); -} - -Eterm erts_make_ref_in_buffer(Eterm buffer[REF_THING_SIZE]) -{ - Eterm* hp = buffer; - Uint32 ref[ERTS_MAX_REF_NUMBERS]; - - erts_make_ref_in_array(ref); - write_ref_thing(hp, ref[0], ref[1], ref[2]); - return make_internal_ref(hp); -} - -Eterm erts_make_ref(Process *p) -{ - Eterm* hp; - Uint32 ref[ERTS_MAX_REF_NUMBERS]; - - ERTS_SMP_LC_ASSERT(ERTS_PROC_LOCK_MAIN & erts_proc_lc_my_proc_locks(p)); - - hp = HAlloc(p, REF_THING_SIZE); - - erts_make_ref_in_array(ref); - write_ref_thing(hp, ref[0], ref[1], ref[2]); - - return make_internal_ref(hp); -} - -BIF_RETTYPE make_ref_0(BIF_ALIST_0) -{ - return erts_make_ref(BIF_P); -} - -/**********************************************************************/ - /* return the time of day */ BIF_RETTYPE time_0(BIF_ALIST_0) @@ -3745,45 +3822,6 @@ BIF_RETTYPE now_0(BIF_ALIST_0) /**********************************************************************/ -BIF_RETTYPE garbage_collect_1(BIF_ALIST_1) -{ - int reds; - Process *rp; - - if (is_not_pid(BIF_ARG_1)) { - BIF_ERROR(BIF_P, BADARG); - } - - if (BIF_P->common.id == BIF_ARG_1) - rp = BIF_P; - else { -#ifdef ERTS_SMP - rp = erts_pid2proc_suspend(BIF_P, ERTS_PROC_LOCK_MAIN, - BIF_ARG_1, ERTS_PROC_LOCK_MAIN); - if (rp == ERTS_PROC_LOCK_BUSY) - ERTS_BIF_YIELD1(bif_export[BIF_garbage_collect_1], BIF_P, BIF_ARG_1); -#else - rp = erts_proc_lookup(BIF_ARG_1); -#endif - if (!rp) - BIF_RET(am_false); - } - - /* The GC cost is taken for the process executing this BIF. */ - - FLAGS(rp) |= F_NEED_FULLSWEEP; - reds = erts_garbage_collect(rp, 0, rp->arg_reg, rp->arity); - -#ifdef ERTS_SMP - if (BIF_P != rp) { - erts_resume(rp, ERTS_PROC_LOCK_MAIN); - erts_smp_proc_unlock(rp, ERTS_PROC_LOCK_MAIN); - } -#endif - - BIF_RET2(am_true, reds); -} - BIF_RETTYPE garbage_collect_0(BIF_ALIST_0) { int reds; @@ -3845,7 +3883,7 @@ BIF_RETTYPE erts_debug_display_1(BIF_ALIST_1) erts_dsprintf_buf_t *dsbufp = erts_create_tmp_dsbuf(64); pres = erts_dsprintf(dsbufp, "%.*T\n", INT_MAX, BIF_ARG_1); if (pres < 0) - erl_exit(1, "Failed to convert term to string: %d (%s)\n", + erts_exit(ERTS_ERROR_EXIT, "Failed to convert term to string: %d (%s)\n", -pres, erl_errno_id(-pres)); hp = HAlloc(BIF_P, 2*dsbufp->str_len); /* we need length * 2 heap words */ res = buf_to_intlist(&hp, dsbufp->str, dsbufp->str_len, NIL); @@ -3867,7 +3905,7 @@ BIF_RETTYPE display_string_1(BIF_ALIST_1) } str = (char *) erts_alloc(ERTS_ALC_T_TMP, sizeof(char)*(len + 1)); if (intlist_to_buf(string, str, len) != len) - erl_exit(1, "%s:%d: Internal error\n", __FILE__, __LINE__); + erts_exit(ERTS_ERROR_EXIT, "%s:%d: Internal error\n", __FILE__, __LINE__); str[len] = '\0'; erts_fprintf(stderr, "%s", str); erts_free(ERTS_ALC_T_TMP, (void *) str); @@ -3887,7 +3925,7 @@ BIF_RETTYPE display_nl_0(BIF_ALIST_0) BIF_RETTYPE halt_0(BIF_ALIST_0) { VERBOSE(DEBUG_SYSTEM,("System halted by BIF halt()\n")); - erl_halt(0); + erts_halt(0); ERTS_BIF_YIELD1(bif_export[BIF_halt_1], BIF_P, am_undefined); } @@ -3900,17 +3938,18 @@ static char halt_msg[HALT_MSG_SIZE]; /* ARGSUSED */ BIF_RETTYPE halt_1(BIF_ALIST_1) { - Sint code; + Uint code; - if (is_small(BIF_ARG_1) && (code = signed_val(BIF_ARG_1)) >= 0) { + if (term_to_Uint_mask(BIF_ARG_1, &code)) { + int pos_int_code = (int) (code & INT_MAX); VERBOSE(DEBUG_SYSTEM,("System halted by BIF halt(%T)\n", BIF_ARG_1)); - erl_halt((int)(- code)); + erts_halt(pos_int_code); ERTS_BIF_YIELD1(bif_export[BIF_halt_1], BIF_P, am_undefined); } else if (ERTS_IS_ATOM_STR("abort", BIF_ARG_1)) { VERBOSE(DEBUG_SYSTEM,("System halted by BIF halt(%T)\n", BIF_ARG_1)); erts_smp_proc_unlock(BIF_P, ERTS_PROC_LOCK_MAIN); - erl_exit(ERTS_ABORT_EXIT, ""); + erts_exit(ERTS_ABORT_EXIT, ""); } else if (is_string(BIF_ARG_1) || BIF_ARG_1 == NIL) { int i; @@ -3921,11 +3960,11 @@ BIF_RETTYPE halt_1(BIF_ALIST_1) halt_msg[i] = '\0'; VERBOSE(DEBUG_SYSTEM,("System halted by BIF halt(%T)\n", BIF_ARG_1)); erts_smp_proc_unlock(BIF_P, ERTS_PROC_LOCK_MAIN); - erl_exit(ERTS_DUMP_EXIT, "%s\n", halt_msg); + erts_exit(ERTS_DUMP_EXIT, "%s\n", halt_msg); } else goto error; - return NIL; /* Pedantic (lint does not know about erl_exit) */ + return NIL; /* Pedantic (lint does not know about erts_exit) */ error: BIF_ERROR(BIF_P, BADARG); } @@ -3936,9 +3975,9 @@ BIF_RETTYPE halt_1(BIF_ALIST_1) /* ARGSUSED */ BIF_RETTYPE halt_2(BIF_ALIST_2) { - Sint code; + Uint code; Eterm optlist = BIF_ARG_2; - int flush = 0; + int flush = 1; for (optlist = BIF_ARG_2; is_list(optlist); @@ -3963,23 +4002,24 @@ BIF_RETTYPE halt_2(BIF_ALIST_2) if (is_not_nil(optlist)) goto error; - if (is_small(BIF_ARG_1) && (code = signed_val(BIF_ARG_1)) >= 0) { + if (term_to_Uint_mask(BIF_ARG_1, &code)) { + int pos_int_code = (int) (code & INT_MAX); VERBOSE(DEBUG_SYSTEM, ("System halted by BIF halt(%T, %T)\n", BIF_ARG_1, BIF_ARG_2)); if (flush) { - erl_halt((int)(- code)); + erts_halt(pos_int_code); ERTS_BIF_YIELD1(bif_export[BIF_halt_1], BIF_P, am_undefined); } else { erts_smp_proc_unlock(BIF_P, ERTS_PROC_LOCK_MAIN); - erl_exit((int)(- code), ""); + erts_exit(pos_int_code, ""); } } else if (ERTS_IS_ATOM_STR("abort", BIF_ARG_1)) { VERBOSE(DEBUG_SYSTEM, ("System halted by BIF halt(%T, %T)\n", BIF_ARG_1, BIF_ARG_2)); erts_smp_proc_unlock(BIF_P, ERTS_PROC_LOCK_MAIN); - erl_exit(ERTS_ABORT_EXIT, ""); + erts_exit(ERTS_ABORT_EXIT, ""); } else if (is_string(BIF_ARG_1) || BIF_ARG_1 == NIL) { int i; @@ -3991,11 +4031,11 @@ BIF_RETTYPE halt_2(BIF_ALIST_2) VERBOSE(DEBUG_SYSTEM, ("System halted by BIF halt(%T, %T)\n", BIF_ARG_1, BIF_ARG_2)); erts_smp_proc_unlock(BIF_P, ERTS_PROC_LOCK_MAIN); - erl_exit(ERTS_DUMP_EXIT, "%s\n", halt_msg); + erts_exit(ERTS_DUMP_EXIT, "%s\n", halt_msg); } else goto error; - return NIL; /* Pedantic (lint does not know about erl_exit) */ + return NIL; /* Pedantic (lint does not know about erts_exit) */ error: BIF_ERROR(BIF_P, BADARG); } @@ -4004,16 +4044,19 @@ BIF_RETTYPE halt_2(BIF_ALIST_2) BIF_RETTYPE function_exported_3(BIF_ALIST_3) { + int arity; if (is_not_atom(BIF_ARG_1) || is_not_atom(BIF_ARG_2) || is_not_small(BIF_ARG_3)) { BIF_ERROR(BIF_P, BADARG); } - if (erts_find_function(BIF_ARG_1, BIF_ARG_2, signed_val(BIF_ARG_3), - erts_active_code_ix()) == NULL) { - BIF_RET(am_false); + arity = signed_val(BIF_ARG_3); + if (erts_find_function(BIF_ARG_1, BIF_ARG_2, arity, + erts_active_code_ix()) != NULL || + erts_is_builtin(BIF_ARG_1, BIF_ARG_2, arity)) { + BIF_RET(am_true); } - BIF_RET(am_true); + BIF_RET(am_false); } /**********************************************************************/ @@ -4048,7 +4091,7 @@ term2list_dsprintf(Process *p, Eterm term) erts_dsprintf_buf_t *dsbufp = erts_create_tmp_dsbuf(64); pres = erts_dsprintf(dsbufp, "%T", term); if (pres < 0) - erl_exit(1, "Failed to convert term to list: %d (%s)\n", + erts_exit(ERTS_ERROR_EXIT, "Failed to convert term to list: %d (%s)\n", -pres, erl_errno_id(-pres)); hp = HAlloc(p, 2*dsbufp->str_len); /* we need length * 2 heap words */ res = buf_to_intlist(&hp, dsbufp->str, dsbufp->str_len, NIL); @@ -4190,6 +4233,7 @@ BIF_RETTYPE list_to_pid_1(BIF_ALIST_1) goto bad; enp = erts_find_or_insert_node(dep->sysname, dep->creation); + ASSERT(enp != erts_this_node); etp = (ExternalThing *) HAlloc(BIF_P, EXTERNAL_THING_HEAD_SIZE + 1); etp->header = make_external_pid_header(1); @@ -4353,7 +4397,11 @@ BIF_RETTYPE system_flag_2(BIF_ALIST_2) switch (erts_set_schedulers_online(BIF_P, ERTS_PROC_LOCK_MAIN, signed_val(BIF_ARG_2), - &old_no)) { + &old_no +#ifdef ERTS_DIRTY_SCHEDULERS + , 0 +#endif + )) { case ERTS_SCHDLR_SSPND_DONE: BIF_RET(make_small(old_no)); case ERTS_SCHDLR_SSPND_YIELD_RESTART: @@ -4485,6 +4533,55 @@ BIF_RETTYPE system_flag_2(BIF_ALIST_2) ref, old ? am_true : am_false); } +#if defined(ERTS_SMP) && defined(ERTS_DIRTY_SCHEDULERS) + } else if (BIF_ARG_1 == am_dirty_cpu_schedulers_online) { + Sint old_no; + if (!is_small(BIF_ARG_2)) + goto error; + switch (erts_set_schedulers_online(BIF_P, + ERTS_PROC_LOCK_MAIN, + signed_val(BIF_ARG_2), + &old_no, + 1)) { + case ERTS_SCHDLR_SSPND_DONE: + BIF_RET(make_small(old_no)); + case ERTS_SCHDLR_SSPND_YIELD_RESTART: + ERTS_VBUMP_ALL_REDS(BIF_P); + BIF_TRAP2(bif_export[BIF_system_flag_2], + BIF_P, BIF_ARG_1, BIF_ARG_2); + case ERTS_SCHDLR_SSPND_YIELD_DONE: + ERTS_BIF_YIELD_RETURN_X(BIF_P, make_small(old_no), + am_dirty_cpu_schedulers_online); + case ERTS_SCHDLR_SSPND_EINVAL: + goto error; + default: + ASSERT(0); + BIF_ERROR(BIF_P, EXC_INTERNAL_ERROR); + break; + } +#endif + } else if (BIF_ARG_1 == am_time_offset + && ERTS_IS_ATOM_STR("finalize", BIF_ARG_2)) { + ErtsTimeOffsetState res; + erts_smp_proc_unlock(BIF_P, ERTS_PROC_LOCK_MAIN); + res = erts_finalize_time_offset(); + erts_smp_proc_lock(BIF_P, ERTS_PROC_LOCK_MAIN); + switch (res) { + case ERTS_TIME_OFFSET_PRELIMINARY: { + DECL_AM(preliminary); + BIF_RET(AM_preliminary); + } + case ERTS_TIME_OFFSET_FINAL: { + DECL_AM(final); + BIF_RET(AM_final); + } + case ERTS_TIME_OFFSET_VOLATILE: { + DECL_AM(volatile); + BIF_RET(AM_volatile); + } + default: + ERTS_INTERNAL_ERROR("Unknown state"); + } } else if (ERTS_IS_ATOM_STR("scheduling_statistics", BIF_ARG_1)) { int what; if (ERTS_IS_ATOM_STR("disable", BIF_ARG_2)) @@ -4508,7 +4605,7 @@ BIF_RETTYPE system_flag_2(BIF_ALIST_2) BIF_P->group_leader, "A call to erlang:system_flag(cpu_topology, _) was made.\n" "The cpu_topology argument is deprecated and scheduled\n" - "for removal in erts-5.10/OTP-R16. For more information\n" + "for removal in Erlang/OTP 18. For more information\n" "see the erlang:system_flag/2 documentation.\n"); BIF_TRAP1(set_cpu_topology_trap, BIF_P, BIF_ARG_2); } else if (ERTS_IS_ATOM_STR("scheduler_bind_type", BIF_ARG_1)) { @@ -4516,7 +4613,7 @@ BIF_RETTYPE system_flag_2(BIF_ALIST_2) BIF_P->group_leader, "A call to erlang:system_flag(scheduler_bind_type, _) was\n" "made. The scheduler_bind_type argument is deprecated and\n" - "scheduled for removal in erts-5.10/OTP-R16. For more\n" + "scheduled for removal in Erlang/OTP 18. For more\n" "information see the erlang:system_flag/2 documentation.\n"); return erts_bind_schedulers(BIF_P, BIF_ARG_2); } @@ -4635,6 +4732,17 @@ BIF_RETTYPE bump_reductions_1(BIF_ALIST_1) BIF_RET2(am_true, reds); } +BIF_RETTYPE erts_internal_cmp_term_2(BIF_ALIST_2) { + Sint res = CMP_TERM(BIF_ARG_1,BIF_ARG_2); + + /* ensure -1, 0, 1 result */ + if (res < 0) { + BIF_RET(make_small(-1)); + } else if (res > 0) { + BIF_RET(make_small(1)); + } + BIF_RET(make_small(0)); +} /* * Processes doing yield on return in a bif ends up in bif_return_trap(). */ @@ -4761,11 +4869,6 @@ void erts_init_trap_export(Export* ep, Eterm m, Eterm f, Uint a, void erts_init_bif(void) { - reference0 = 0; - reference1 = 0; - reference2 = 0; - - erts_smp_spinlock_init(&make_ref_lock, "make_ref"); erts_smp_mtx_init(&ports_snapshot_mtx, "ports_snapshot"); erts_smp_atomic_init_nob(&erts_dead_ports_ptr, (erts_aint_t) NULL); @@ -4782,9 +4885,21 @@ void erts_init_bif(void) #endif , &bif_return_trap); - flush_monitor_message_trap = erts_export_put(am_erlang, - am_flush_monitor_message, - 2); + erts_await_result = erts_export_put(am_erts_internal, + am_await_result, + 1); + + erts_init_trap_export(&dsend_continue_trap_export, + am_erts_internal, am_dsend_continue_trap, 1, + dsend_continue_trap_1); + + flush_monitor_messages_trap = erts_export_put(am_erts_internal, + am_flush_monitor_messages, + 3); + + erts_convert_time_unit_trap = erts_export_put(am_erlang, + am_convert_time_unit, + 3); set_cpu_topology_trap = erts_export_put(am_erlang, am_set_cpu_topology, diff --git a/erts/emulator/beam/bif.h b/erts/emulator/beam/bif.h index 51b77a95ed..c6ed60376a 100644 --- a/erts/emulator/beam/bif.h +++ b/erts/emulator/beam/bif.h @@ -3,16 +3,17 @@ * * Copyright Ericsson AB 1996-2013. All Rights Reserved. * - * The contents of this file are subject to the Erlang Public License, - * Version 1.1, (the "License"); you may not use this file except in - * compliance with the License. You should have received a copy of the - * Erlang Public License along with this software. If not, it can be - * retrieved online at http://www.erlang.org/. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Software distributed under the License is distributed on an "AS IS" - * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See - * the License for the specific language governing rights and limitations - * under the License. + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. * * %CopyrightEnd% */ @@ -20,7 +21,9 @@ #ifndef __BIF_H__ #define __BIF_H__ +extern Export *erts_await_result; extern Export* erts_format_cpu_topology_trap; +extern Export *erts_convert_time_unit_trap; #define BIF_RETTYPE Eterm @@ -30,10 +33,12 @@ extern Export* erts_format_cpu_topology_trap; #define BIF_ALIST_1 Process* A__p, Eterm* BIF__ARGS #define BIF_ALIST_2 Process* A__p, Eterm* BIF__ARGS #define BIF_ALIST_3 Process* A__p, Eterm* BIF__ARGS +#define BIF_ALIST_4 Process* A__p, Eterm* BIF__ARGS #define BIF_ARG_1 (BIF__ARGS[0]) #define BIF_ARG_2 (BIF__ARGS[1]) #define BIF_ARG_3 (BIF__ARGS[2]) +#define BIF_ARG_4 (BIF__ARGS[3]) #define ERTS_IS_PROC_OUT_OF_REDS(p) \ ((p)->fcalls > 0 \ @@ -124,12 +129,85 @@ do { \ return THE_NON_VALUE; \ } while(0) +#define ERTS_BIF_ERROR_TRAPPED0(Proc, Reason, Bif) \ +do { \ + (Proc)->freason = (Reason); \ + (Proc)->current = (Bif)->code; \ + return THE_NON_VALUE; \ +} while (0) + +#define ERTS_BIF_ERROR_TRAPPED1(Proc, Reason, Bif, A0) \ +do { \ + Eterm* reg = ERTS_PROC_GET_SCHDATA((Proc))->x_reg_array; \ + (Proc)->freason = (Reason); \ + (Proc)->current = (Bif)->code; \ + reg[0] = (Eterm) (A0); \ + return THE_NON_VALUE; \ +} while (0) + +#define ERTS_BIF_ERROR_TRAPPED2(Proc, Reason, Bif, A0, A1) \ +do { \ + Eterm* reg = ERTS_PROC_GET_SCHDATA((Proc))->x_reg_array; \ + (Proc)->freason = (Reason); \ + (Proc)->current = (Bif)->code; \ + reg[0] = (Eterm) (A0); \ + reg[1] = (Eterm) (A1); \ + return THE_NON_VALUE; \ +} while (0) + +#define ERTS_BIF_ERROR_TRAPPED3(Proc, Reason, Bif, A0, A1, A2) \ +do { \ + Eterm* reg = ERTS_PROC_GET_SCHDATA((Proc))->x_reg_array; \ + (Proc)->freason = (Reason); \ + (Proc)->current = (Bif)->code; \ + reg[0] = (Eterm) (A0); \ + reg[1] = (Eterm) (A1); \ + reg[2] = (Eterm) (A2); \ + return THE_NON_VALUE; \ +} while (0) + #define ERTS_BIF_PREP_ERROR(Ret, Proc, Reason) \ do { \ (Proc)->freason = (Reason); \ (Ret) = THE_NON_VALUE; \ } while (0) +#define ERTS_BIF_PREP_ERROR_TRAPPED0(Ret, Proc, Reason, Bif) \ +do { \ + (Proc)->freason = (Reason); \ + (Proc)->current = (Bif)->code; \ + (Ret) = THE_NON_VALUE; \ +} while (0) + +#define ERTS_BIF_PREP_ERROR_TRAPPED1(Ret, Proc, Reason, Bif, A0) \ +do { \ + Eterm* reg = ERTS_PROC_GET_SCHDATA((Proc))->x_reg_array; \ + (Proc)->freason = (Reason); \ + (Proc)->current = (Bif)->code; \ + reg[0] = (Eterm) (A0); \ + (Ret) = THE_NON_VALUE; \ +} while (0) + +#define ERTS_BIF_PREP_ERROR_TRAPPED2(Ret, Proc, Reason, Bif, A0, A1) \ +do { \ + Eterm* reg = ERTS_PROC_GET_SCHDATA((Proc))->x_reg_array; \ + (Proc)->freason = (Reason); \ + (Proc)->current = (Bif)->code; \ + reg[0] = (Eterm) (A0); \ + reg[1] = (Eterm) (A1); \ + (Ret) = THE_NON_VALUE; \ +} while (0) + +#define ERTS_BIF_PREP_ERROR_TRAPPED3(Ret, Proc, Reason, Bif, A0, A1, A2) \ +do { \ + Eterm* reg = ERTS_PROC_GET_SCHDATA((Proc))->x_reg_array; \ + (Proc)->freason = (Reason); \ + (Proc)->current = (Bif)->code; \ + reg[0] = (Eterm) (A0); \ + reg[1] = (Eterm) (A1); \ + reg[2] = (Eterm) (A2); \ + (Ret) = THE_NON_VALUE; \ +} while (0) #define ERTS_BIF_PREP_TRAP0(Ret, Trap, Proc) \ do { \ @@ -392,6 +470,54 @@ erts_bif_prep_await_proc_exit_apply_trap(Process *c_p, Eterm args[], int nargs); +#ifdef ERL_WANT_HIPE_BIF_WRAPPER__ + +#ifndef HIPE + +#define HIPE_WRAPPER_BIF_DISABLE_GC(BIF_NAME, ARITY) + +#else + +#include "erl_fun.h" +#include "hipe_mode_switch.h" + +/* + * Hipe wrappers used by native code for BIFs that disable GC while trapping. + * Also add usage of the wrapper in ../hipe/hipe_bif_list.m4 + * + * Problem: + * When native code calls a BIF that traps, hipe_mode_switch will push a + * "trap frame" on the Erlang stack in order to find its way back from beam_emu + * back to native caller when finally done. If GC is disabled and stack/heap + * is full there is no place to push the "trap frame". + * + * Solution: + * We reserve space on stack for the "trap frame" here before the BIF is called. + * If the BIF does not trap, the space is reclaimed here before returning. + * If the BIF traps, hipe_push_beam_trap_frame() will detect that a "trap frame" + * already is reserved and use it. + */ + + +#define HIPE_WRAPPER_BIF_DISABLE_GC(BIF_NAME, ARITY) \ +BIF_RETTYPE hipe_wrapper_ ## BIF_NAME ## _ ## ARITY (Process* c_p, \ + Eterm* args); \ +BIF_RETTYPE hipe_wrapper_ ## BIF_NAME ## _ ## ARITY (Process* c_p, \ + Eterm* args) \ +{ \ + BIF_RETTYPE res; \ + hipe_reserve_beam_trap_frame(c_p, args, ARITY); \ + res = BIF_NAME ## _ ## ARITY (c_p, args); \ + if (is_value(res) || c_p->freason != TRAP) { \ + hipe_unreserve_beam_trap_frame(c_p); \ + } \ + return res; \ +} + +#endif + +#endif /* ERL_WANT_HIPE_BIF_WRAPPER__ */ + #include "erl_bif_table.h" #endif diff --git a/erts/emulator/beam/bif.tab b/erts/emulator/beam/bif.tab index 8bc994c8c3..3177d5dae7 100644 --- a/erts/emulator/beam/bif.tab +++ b/erts/emulator/beam/bif.tab @@ -3,16 +3,17 @@ # # Copyright Ericsson AB 1996-2013. All Rights Reserved. # -# The contents of this file are subject to the Erlang Public License, -# Version 1.1, (the "License"); you may not use this file except in -# compliance with the License. You should have received a copy of the -# Erlang Public License along with this software. If not, it can be -# retrieved online at http://www.erlang.org/. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# Software distributed under the License is distributed on an "AS IS" -# basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See -# the License for the specific language governing rights and limitations -# under the License. +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. # # %CopyrightEnd% # @@ -45,8 +46,7 @@ bif erlang:apply/3 bif erlang:atom_to_list/1 bif erlang:binary_to_list/1 bif erlang:binary_to_list/3 -bif erlang:binary_to_term/1 -bif erlang:check_process_code/2 +bif erlang:binary_to_term/1 bif erlang:crc32/1 bif erlang:crc32/2 bif erlang:crc32_combine/3 @@ -67,7 +67,6 @@ bif erlang:float_to_list/1 bif erlang:float_to_list/2 bif erlang:fun_info/2 bif erlang:garbage_collect/0 -bif erlang:garbage_collect/1 bif erlang:get/0 bif erlang:get/1 bif erlang:get_keys/1 @@ -94,6 +93,8 @@ bif erlang:loaded/0 bif erlang:localtime/0 bif erlang:localtime_to_universaltime/2 bif erlang:make_ref/0 +bif erlang:unique_integer/0 +bif erlang:unique_integer/1 bif erlang:md5/1 bif erlang:md5_init/0 bif erlang:md5_update/2 @@ -106,6 +107,13 @@ ubif erlang:node/1 ubif erlang:node/0 bif erlang:nodes/1 bif erlang:now/0 +bif erlang:monotonic_time/0 +bif erlang:monotonic_time/1 +bif erlang:system_time/0 +bif erlang:system_time/1 +bif erlang:time_offset/0 +bif erlang:time_offset/1 +bif erlang:timestamp/0 bif erlang:open_port/2 @@ -154,8 +162,23 @@ bif erts_internal:port_command/3 bif erts_internal:port_control/3 bif erts_internal:port_close/1 bif erts_internal:port_connect/2 -bif erts_internal:port_set_data/2 -bif erts_internal:port_get_data/1 + +bif erts_internal:request_system_task/3 +bif erts_internal:check_process_code/2 + +bif erts_internal:map_to_tuple_keys/1 +bif erts_internal:map_type/1 +bif erts_internal:map_hashmap_children/1 + +bif erts_internal:time_unit/0 + +bif erts_internal:is_system_process/1 + +bif erts_internal:system_check/1 + +# inet_db support +bif erlang:port_set_data/2 +bif erlang:port_get_data/1 # Tracing & debugging. bif erlang:trace_pattern/2 @@ -189,15 +212,20 @@ bif math:erf/1 bif math:erfc/1 bif math:exp/1 bif math:log/1 +bif math:log2/1 bif math:log10/1 bif math:sqrt/1 bif math:atan2/2 bif math:pow/2 bif erlang:start_timer/3 +bif erlang:start_timer/4 bif erlang:send_after/3 +bif erlang:send_after/4 bif erlang:cancel_timer/1 +bif erlang:cancel_timer/2 bif erlang:read_timer/1 +bif erlang:read_timer/2 bif erlang:make_tuple/2 bif erlang:append_element/2 @@ -342,6 +370,8 @@ bif os:getenv/0 bif os:getenv/1 bif os:getpid/0 bif os:timestamp/0 +bif os:system_time/0 +bif os:system_time/1 # # Bifs in the erl_ddll module (the module actually does not exist) @@ -475,7 +505,7 @@ bif erlang:call_on_load_function/1 bif erlang:finish_after_on_load/2 # -# New Bifs in R13B4 +# New Bifs in R13B04 # bif erlang:binary_to_term/2 @@ -570,6 +600,45 @@ bif erlang:float_to_binary/2 bif erlang:binary_to_float/1 bif io:printable_range/0 +bif os:unsetenv/1 + +# +# New in 17.0 +# + +bif re:inspect/2 + +ubif erlang:is_map/1 +ubif erlang:map_size/1 +bif maps:to_list/1 +bif maps:find/2 +bif maps:get/2 +bif maps:from_list/1 +bif maps:is_key/2 +bif maps:keys/1 +bif maps:merge/2 +bif maps:new/0 +bif maps:put/3 +bif maps:remove/2 +bif maps:update/3 +bif maps:values/1 + +bif erts_internal:cmp_term/2 + +bif ets:take/2 + +# +# New in 17.1 +# + +bif erlang:fun_info_mfa/1 + +# New in 18.0 +# + +bif erlang:get_keys/0 +bif ets:update_counter/4 +bif erts_debug:map_info/1 # # Obsolete diff --git a/erts/emulator/beam/big.c b/erts/emulator/beam/big.c index acfcc845e4..87d3be2b0f 100644 --- a/erts/emulator/beam/big.c +++ b/erts/emulator/beam/big.c @@ -1,18 +1,19 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 1996-2012. All Rights Reserved. + * Copyright Ericsson AB 1996-2014. All Rights Reserved. * - * The contents of this file are subject to the Erlang Public License, - * Version 1.1, (the "License"); you may not use this file except in - * compliance with the License. You should have received a copy of the - * Erlang Public License along with this software. If not, it can be - * retrieved online at http://www.erlang.org/. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Software distributed under the License is distributed on an "AS IS" - * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See - * the License for the specific language governing rights and limitations - * under the License. + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. * * %CopyrightEnd% */ @@ -150,14 +151,14 @@ #define D2GTE(a1,a0,b1,b0) (!D2LT(a1,a0,b1,b0)) #define D2LTE(a1,a0,b1,b0) (!D2GT(a1,a0,b1,b0)) -// Add (A+B), A=(a1B+a0) B=(b1B+b0) +/* Add (A+B), A=(a1B+a0) B=(b1B+b0) */ #define D2ADD(a1,a0,b1,b0,c1,c0) do { \ ErtsDigit __ci = 0; \ DSUM(a0,b0,__ci,c0); \ DSUMc(a1,b1,__ci,c1); \ } while(0) -// Subtract (A-B), A=(a1B+a0), B=(b1B+b0) (A>=B) +/* Subtract (A-B), A=(a1B+a0), B=(b1B+b0) (A>=B) */ #define D2SUB(a1,a0,b1,b0,c1,c0) do { \ ErtsDigit __bi; \ DSUB(a0,b0,__bi,c0); \ @@ -274,7 +275,9 @@ _b = _b << _s; \ _vn1 = _b >> H_EXP; \ _vn0 = _b & LO_MASK; \ - _un32 = (_a1 << _s) | ((_a0>>(D_EXP-_s)) & (-_s >> (D_EXP-1))); \ + /* If needed to avoid undefined behaviour */ \ + if (_s) _un32 = (_a1 << _s) | ((_a0>>(D_EXP-_s)) & (-_s >> (D_EXP-1))); \ + else _un32 = _a1; \ _un10 = _a0 << _s; \ _un1 = _un10 >> H_EXP; \ _un0 = _un10 & LO_MASK; \ @@ -1325,9 +1328,9 @@ static dsize_t I_lshift(ErtsDigit* x, dsize_t xl, Sint y, return 1; } else { - SWord ay = (y < 0) ? -y : y; - int bw = ay / D_EXP; - int sw = ay % D_EXP; + Uint ay = (y < 0) ? -y : y; + Uint bw = ay / D_EXP; + Uint sw = ay % D_EXP; dsize_t rl; ErtsDigit a1=0; ErtsDigit a0=0; @@ -1337,7 +1340,7 @@ static dsize_t I_lshift(ErtsDigit* x, dsize_t xl, Sint y, while(bw--) *r++ = 0; - if (sw) { // NOTE! x >> 32 is not = 0! + if (sw) { /* NOTE! x >> 32 is not = 0! */ while(xl--) { a0 = (*x << sw) | a1; a1 = (*x >> (D_EXP - sw)); @@ -1368,7 +1371,7 @@ static dsize_t I_lshift(ErtsDigit* x, dsize_t xl, Sint y, } if (sign) { - int zl = bw; + Uint zl = bw; ErtsDigit* z = x; while(zl--) { @@ -1384,7 +1387,7 @@ static dsize_t I_lshift(ErtsDigit* x, dsize_t xl, Sint y, x += (xl-1); r += (rl-1); xl -= bw; - if (sw) { // NOTE! x >> 32 is not = 0! + if (sw) { /* NOTE! x >> 32 is not = 0! */ while(xl--) { a1 = (*x >> sw) | a0; a0 = (*x << (D_EXP-sw)); @@ -1506,13 +1509,15 @@ Eterm uword_to_big(UWord x, Eterm *y) */ Eterm small_to_big(Sint x, Eterm *y) { + Uint xu; if (x >= 0) { + xu = x; *y = make_pos_bignum_header(1); } else { - x = -x; + xu = -(Uint)x; *y = make_neg_bignum_header(1); } - BIG_DIGIT(y, 0) = x; + BIG_DIGIT(y, 0) = xu; return make_big(y); } @@ -1540,21 +1545,24 @@ Eterm erts_uint64_to_big(Uint64 x, Eterm **hpp) Eterm erts_sint64_to_big(Sint64 x, Eterm **hpp) { Eterm *hp = *hpp; + Uint64 ux; int neg; - if (x >= 0) + if (x >= 0) { neg = 0; + ux = x; + } else { neg = 1; - x = -x; + ux = -(Uint64)x; } #if defined(ARCH_32) || HALFWORD_HEAP - if (x >= (((Uint64) 1) << 32)) { + if (ux >= (((Uint64) 1) << 32)) { if (neg) *hp = make_neg_bignum_header(2); else *hp = make_pos_bignum_header(2); - BIG_DIGIT(hp, 0) = (Uint) (x & ((Uint) 0xffffffff)); - BIG_DIGIT(hp, 1) = (Uint) ((x >> 32) & ((Uint) 0xffffffff)); + BIG_DIGIT(hp, 0) = (Uint) (ux & ((Uint) 0xffffffff)); + BIG_DIGIT(hp, 1) = (Uint) ((ux >> 32) & ((Uint) 0xffffffff)); *hpp += 3; } else @@ -1564,12 +1572,52 @@ Eterm erts_sint64_to_big(Sint64 x, Eterm **hpp) *hp = make_neg_bignum_header(1); else *hp = make_pos_bignum_header(1); - BIG_DIGIT(hp, 0) = (Uint) x; + BIG_DIGIT(hp, 0) = (Uint) ux; *hpp += 2; } return make_big(hp); } +Eterm +erts_uint64_array_to_big(Uint **hpp, int neg, int len, Uint64 *array) +{ + Uint *headerp; + int i, pot_digits, digits; + + headerp = *hpp; + + pot_digits = digits = 0; + for (i = 0; i < len; i++) { +#if defined(ARCH_32) || HALFWORD_HEAP + Uint low_val = array[i] & ((Uint) 0xffffffff); + Uint high_val = (array[i] >> 32) & ((Uint) 0xffffffff); + BIG_DIGIT(headerp, pot_digits) = low_val; + pot_digits++; + if (low_val) + digits = pot_digits; + BIG_DIGIT(headerp, pot_digits) = high_val; + pot_digits++; + if (high_val) + digits = pot_digits; +#else + Uint val = array[i]; + BIG_DIGIT(headerp, pot_digits) = val; + pot_digits++; + if (val) + digits = pot_digits; +#endif + } + + if (neg) + *headerp = make_neg_bignum_header(digits); + else + *headerp = make_pos_bignum_header(digits); + + *hpp = headerp + 1 + digits; + + return make_big(headerp); +} + /* ** Convert a bignum to a double float */ @@ -1603,9 +1651,11 @@ big_to_double(Wterm x, double* resp) /* * Logic has been copied from erl_bif_guard.c and slightly * modified to use a static instead of dynamic heap + * + * HALFWORD: Return relative term with 'heap' as base. */ Eterm -double_to_big(double x, Eterm *heap) +double_to_big(double x, Eterm *heap, Uint hsz) { int is_negative; int ds; @@ -1633,9 +1683,10 @@ double_to_big(double x, Eterm *heap) sz = BIG_NEED_SIZE(ds); /* number of words including arity */ hp = heap; - res = make_big(hp); + res = make_big_rel(hp, heap); xp = (ErtsDigit*) (hp + 1); + ASSERT(ds < hsz); for (i = ds - 1; i >= 0; i--) { ErtsDigit d; @@ -1890,6 +1941,8 @@ Eterm bytes_to_big(byte *xp, dsize_t xsz, int xsgn, Eterm *r) *rwp = d; rwp++; } + if (rsz > BIG_ARITY_MAX) + return NIL; if (xsgn) { *r = make_neg_bignum_header(rsz); } @@ -1975,6 +2028,32 @@ term_to_Uint(Eterm term, Uint *up) } } +/* same as term_to_Uint() + but also accept larger bignums by masking + */ +int +term_to_Uint_mask(Eterm term, Uint *up) +{ + if (is_small(term)) { + Sint i = signed_val(term); + if (i < 0) { + *up = BADARG; + return 0; + } + *up = (Uint) i; + return 1; + } else if (is_big(term) && !big_sign(term)) { + ErtsDigit* xr = big_v(term); + + ERTS_CT_ASSERT(sizeof(ErtsDigit) == sizeof(Uint)); + *up = (Uint)*xr; /* just pick first word */ + return 1; + } else { + *up = BADARG; + return 0; + } +} + int term_to_UWord(Eterm term, UWord *up) { @@ -2468,7 +2547,7 @@ int term_equals_2pow32(Eterm x) if (!is_big(x)) return 0; bp = big_val(x); -#if D_EXP == 16 // 16 bit platfrom not really supported!!! +#if D_EXP == 16 /* 16 bit platfrom not really supported!!! */ return (BIG_SIZE(bp) == 3) && !BIG_DIGIT(bp,0) && !BIG_DIGIT(bp,1) && BIG_DIGIT(bp,2) == 1; #elif D_EXP == 32 @@ -2528,17 +2607,18 @@ const byte d_base_exp_lookup[] = { 31, 19, 15, 13, 11, 11, 10, 9, 9, 8, 8, 8, 8, * end * How much can the characters which fit in 31 bit represent */ -const Uint d_base_base_lookup[] = { 2147483648, 1162261467, 1073741824, - 1220703125, 362797056, 1977326743, 1073741824, 387420489, 1000000000, - 214358881, 429981696, 815730721, 1475789056, 170859375, 268435456, - 410338673, 612220032, 893871739, 1280000000, 1801088541, 113379904, - 148035889, 191102976, 244140625, 308915776, 387420489, 481890304, - 594823321, 729000000, 887503681, 1073741824, 1291467969, 1544804416, - 1838265625, 60466176, 69343957, 79235168, 90224199, 102400000, - 115856201, 130691232, 147008443, 164916224, 184528125, 205962976, - 229345007, 254803968, 282475249, 312500000, 345025251, 380204032, - 418195493, 459165024, 503284375, 550731776, 601692057, 656356768, - 714924299, 777600000, 844596301, 916132832, 992436543, 1073741824 }; +const Uint d_base_base_lookup[] = { 2147483648u, 1162261467u, 1073741824u, + 1220703125u, 362797056u, 1977326743u, 1073741824u, 387420489u, + 1000000000u, 214358881u, 429981696u, 815730721u, 1475789056u, + 170859375u, 268435456u, 410338673u, 612220032u, 893871739u, 1280000000u, + 1801088541u, 113379904u, 148035889u, 191102976u, 244140625u, 308915776u, + 387420489u, 481890304u, 594823321u, 729000000u, 887503681u, 1073741824u, + 1291467969u, 1544804416u, 1838265625u, 60466176u, 69343957u, 79235168u, + 90224199u, 102400000u, 115856201u, 130691232u, 147008443u, 164916224u, + 184528125u, 205962976u, 229345007u, 254803968u, 282475249u, 312500000u, + 345025251u, 380204032u, 418195493u, 459165024u, 503284375u, 550731776u, + 601692057u, 656356768u, 714924299u, 777600000u, 844596301u, 916132832u, + 992436543u, 1073741824u }; Eterm erts_chars_to_integer(Process *BIF_P, char *bytes, Uint size, const int base) { @@ -2564,6 +2644,9 @@ Eterm erts_chars_to_integer(Process *BIF_P, char *bytes, size--; } + if (size == 0) + goto bytebuf_to_integer_1_error; + if (size < SMALL_DIGITS && base <= 10) { /* * * Take shortcut if we know that all chars are '0' < b < '9' and @@ -2663,9 +2746,6 @@ Eterm erts_chars_to_integer(Process *BIF_P, char *bytes, res = big_plus_small(res, m, hp); } - if (is_big(res)) /* check if small */ - res = big_plus_small(res, 0, hp); /* includes conversion to small */ - if (neg) { if (is_small(res)) res = make_small(-signed_val(res)); @@ -2675,8 +2755,12 @@ Eterm erts_chars_to_integer(Process *BIF_P, char *bytes, } } - if (is_big(res)) { - hp += (big_arity(res) + 1); + if (is_not_small(res)) { + res = big_plus_small(res, 0, hp); /* includes conversion to small */ + + if (is_not_small(res)) { + hp += (big_arity(res) + 1); + } } HRelease(BIF_P, hp_end, hp); goto bytebuf_to_integer_1_done; diff --git a/erts/emulator/beam/big.h b/erts/emulator/beam/big.h index 1a7b14170f..3c1f1e6b23 100644 --- a/erts/emulator/beam/big.h +++ b/erts/emulator/beam/big.h @@ -1,18 +1,19 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 1996-2013. All Rights Reserved. + * Copyright Ericsson AB 1996-2014. All Rights Reserved. * - * The contents of this file are subject to the Erlang Public License, - * Version 1.1, (the "License"); you may not use this file except in - * compliance with the License. You should have received a copy of the - * Erlang Public License along with this software. If not, it can be - * retrieved online at http://www.erlang.org/. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Software distributed under the License is distributed on an "AS IS" - * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See - * the License for the specific language governing rights and limitations - * under the License. + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. * * %CopyrightEnd% */ @@ -85,6 +86,7 @@ typedef Uint dsize_t; /* Vector size type */ /* The heap size needed for a bignum */ #define BIG_NEED_SIZE(x) ((x) + 1) +#define BIG_NEED_FOR_BITS(bits) BIG_NEED_SIZE(((bits)-1)/D_EXP + 1) #define BIG_UINT_HEAP_SIZE (1 + 1) /* always, since sizeof(Uint) <= sizeof(Eterm) */ @@ -101,9 +103,12 @@ typedef Uint dsize_t; /* Vector size type */ #define ERTS_SINT64_HEAP_SIZE(X) \ (IS_SSMALL((X)) \ ? 0 \ - : ERTS_UINT64_BIG_HEAP_SIZE__((X) >= 0 ? (X) : -(X))) + : ERTS_UINT64_BIG_HEAP_SIZE__((X) >= 0 ? (X) : -(Uint64)(X))) #define ERTS_UINT64_HEAP_SIZE(X) \ (IS_USMALL(0, (X)) ? 0 : ERTS_UINT64_BIG_HEAP_SIZE__((X))) +#define ERTS_MAX_SINT64_HEAP_SIZE (1 + 2) +#define ERTS_MAX_UINT64_HEAP_SIZE (1 + 2) +#define ERTS_UINT64_ARRAY_TO_BIG_MAX_HEAP_SZ(LEN) (2*(LEN)+1) #else @@ -111,6 +116,9 @@ typedef Uint dsize_t; /* Vector size type */ (IS_SSMALL((X)) ? 0 : (1 + 1)) #define ERTS_UINT64_HEAP_SIZE(X) \ (IS_USMALL(0, (X)) ? 0 : (1 + 1)) +#define ERTS_MAX_SINT64_HEAP_SIZE (1 + 1) +#define ERTS_MAX_UINT64_HEAP_SIZE (1 + 1) +#define ERTS_UINT64_ARRAY_TO_BIG_MAX_HEAP_SZ(LEN) ((LEN)+1) #endif @@ -141,7 +149,7 @@ Eterm big_lshift(Eterm, Sint, Eterm*); int big_comp (Wterm, Wterm); int big_ucomp (Eterm, Eterm); int big_to_double(Wterm x, double* resp); -Eterm double_to_big(double, Eterm*); +Eterm double_to_big(double, Eterm*, Uint hsz); Eterm small_to_big(Sint, Eterm*); Eterm uint_to_big(Uint, Eterm*); Eterm uword_to_big(UWord, Eterm*); @@ -153,9 +161,11 @@ Eterm bytes_to_big(byte*, dsize_t, int, Eterm*); byte* big_to_bytes(Eterm, byte*); int term_to_Uint(Eterm, Uint*); +int term_to_Uint_mask(Eterm, Uint*); int term_to_UWord(Eterm, UWord*); int term_to_Sint(Eterm, Sint*); #if HAVE_INT64 +Eterm erts_uint64_array_to_big(Uint **, int, int, Uint64 *); int term_to_Uint64(Eterm, Uint64*); int term_to_Sint64(Eterm, Sint64*); #endif diff --git a/erts/emulator/beam/binary.c b/erts/emulator/beam/binary.c index 33abac2f3d..247c8f1122 100644 --- a/erts/emulator/beam/binary.c +++ b/erts/emulator/beam/binary.c @@ -3,16 +3,17 @@ * * Copyright Ericsson AB 1996-2013. All Rights Reserved. * - * The contents of this file are subject to the Erlang Public License, - * Version 1.1, (the "License"); you may not use this file except in - * compliance with the License. You should have received a copy of the - * Erlang Public License along with this software. If not, it can be - * retrieved online at http://www.erlang.org/. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Software distributed under the License is distributed on an "AS IS" - * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See - * the License for the specific language governing rights and limitations - * under the License. + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. * * %CopyrightEnd% */ @@ -26,17 +27,18 @@ #include "global.h" #include "erl_process.h" #include "error.h" +#define ERL_WANT_HIPE_BIF_WRAPPER__ #include "bif.h" +#undef ERL_WANT_HIPE_BIF_WRAPPER__ #include "big.h" #include "erl_binary.h" #include "erl_bits.h" -#ifdef DEBUG -static int list_to_bitstr_buf(Eterm obj, char* buf, Uint len); -#else -static int list_to_bitstr_buf(Eterm obj, char* buf); -#endif -static int bitstr_list_len(Eterm obj, Uint* num_bytes); +static Export binary_to_list_continue_export; +static Export list_to_binary_continue_export; + +static BIF_RETTYPE binary_to_list_continue(BIF_ALIST_1); +static BIF_RETTYPE list_to_binary_continue(BIF_ALIST_1); void erts_init_binary(void) @@ -45,10 +47,19 @@ erts_init_binary(void) if ((((UWord) &((Binary *) 0)->orig_bytes[0]) % ((UWord) 8)) != 0) { /* I assume that any compiler should be able to optimize this away. If not, this test is not very expensive... */ - erl_exit(ERTS_ABORT_EXIT, + erts_exit(ERTS_ABORT_EXIT, "Internal error: Address of orig_bytes[0] of a Binary" " is *not* 8-byte aligned\n"); } + + erts_init_trap_export(&binary_to_list_continue_export, + am_erts_internal, am_binary_to_list_continue, 1, + &binary_to_list_continue); + + erts_init_trap_export(&list_to_binary_continue_export, + am_erts_internal, am_list_to_binary_continue, 1, + &list_to_binary_continue); + } /* @@ -75,8 +86,6 @@ new_binary(Process *p, byte *buf, Uint len) * Allocate the binary struct itself. */ bptr = erts_bin_nrml_alloc(len); - bptr->flags = 0; - bptr->orig_size = len; erts_refc_init(&bptr->refc, 1); if (buf != NULL) { sys_memcpy(bptr->orig_bytes, buf, len); @@ -114,8 +123,6 @@ Eterm erts_new_mso_binary(Process *p, byte *buf, int len) * Allocate the binary struct itself. */ bptr = erts_bin_nrml_alloc(len); - bptr->flags = 0; - bptr->orig_size = len; erts_refc_init(&bptr->refc, 1); if (buf != NULL) { sys_memcpy(bptr->orig_bytes, buf, len); @@ -169,7 +176,6 @@ erts_realloc_binary(Eterm bin, size_t size) } else { /* REFC */ ProcBin* pb = (ProcBin *) bval; Binary* newbin = erts_bin_realloc(pb->val, size); - newbin->orig_size = size; pb->val = newbin; pb->size = size; pb->bytes = (byte*) newbin->orig_bytes; @@ -333,6 +339,132 @@ BIF_RETTYPE integer_to_binary_1(BIF_ALIST_1) BIF_RET(res); } +#define ERTS_B2L_BYTES_PER_REDUCTION 256 + +typedef struct { + Eterm res; + Eterm *hp; +#ifdef DEBUG + Eterm *hp_end; +#endif + byte *bytes; + Uint size; + Uint bitoffs; +} ErtsB2LState; + +static void b2l_state_destructor(Binary *mbp) +{ + ASSERT(ERTS_MAGIC_BIN_DESTRUCTOR(mbp) == b2l_state_destructor); +} + +static BIF_RETTYPE +binary_to_list_chunk(Process *c_p, + Eterm mb_eterm, + ErtsB2LState* sp, + int reds_left, + int gc_disabled) +{ + BIF_RETTYPE ret; + int bump_reds; + Uint size; + byte *bytes; + + size = (reds_left + 1)*ERTS_B2L_BYTES_PER_REDUCTION; + if (size > sp->size) + size = sp->size; + bytes = sp->bytes + (sp->size - size); + + bump_reds = (size - 1)/ERTS_B2L_BYTES_PER_REDUCTION + 1; + BUMP_REDS(c_p, bump_reds); + + ASSERT(is_list(sp->res) || is_nil(sp->res)); + + sp->res = erts_bin_bytes_to_list(sp->res, + sp->hp, + bytes, + size, + sp->bitoffs); + sp->size -= size; + sp->hp += 2*size; + + if (sp->size > 0) { + + if (!gc_disabled) + erts_set_gc_state(c_p, 0); + + ASSERT(c_p->flags & F_DISABLE_GC); + ASSERT(is_value(mb_eterm)); + ERTS_BIF_PREP_TRAP1(ret, + &binary_to_list_continue_export, + c_p, + mb_eterm); + } + else { + + ASSERT(sp->hp == sp->hp_end); + ASSERT(sp->size == 0); + + if (!gc_disabled || !erts_set_gc_state(c_p, 1)) + ERTS_BIF_PREP_RET(ret, sp->res); + else + ERTS_BIF_PREP_YIELD_RETURN(ret, c_p, sp->res); + ASSERT(!(c_p->flags & F_DISABLE_GC)); + } + + return ret; +} + +static ERTS_INLINE BIF_RETTYPE +binary_to_list(Process *c_p, Eterm *hp, Eterm tail, byte *bytes, Uint size, Uint bitoffs) +{ + int reds_left = ERTS_BIF_REDS_LEFT(c_p); + if (size < reds_left*ERTS_B2L_BYTES_PER_REDUCTION) { + Eterm res; + BIF_RETTYPE ret; + int bump_reds = (size - 1)/ERTS_B2L_BYTES_PER_REDUCTION + 1; + BUMP_REDS(c_p, bump_reds); + res = erts_bin_bytes_to_list(tail, hp, bytes, size, bitoffs); + ERTS_BIF_PREP_RET(ret, res); + return ret; + } + else { + Binary *mbp = erts_create_magic_binary(sizeof(ErtsB2LState), + b2l_state_destructor); + ErtsB2LState *sp = ERTS_MAGIC_BIN_DATA(mbp); + Eterm mb; + + sp->res = tail; + sp->hp = hp; +#ifdef DEBUG + sp->hp_end = sp->hp + 2*size; +#endif + sp->bytes = bytes; + sp->size = size; + sp->bitoffs = bitoffs; + + hp = HAlloc(c_p, PROC_BIN_SIZE); + mb = erts_mk_magic_binary_term(&hp, &MSO(c_p), mbp); + return binary_to_list_chunk(c_p, mb, sp, reds_left, 0); + } +} + +static BIF_RETTYPE binary_to_list_continue(BIF_ALIST_1) +{ + Binary *mbp = ((ProcBin *) binary_val(BIF_ARG_1))->val; + + ASSERT(ERTS_MAGIC_BIN_DESTRUCTOR(mbp) == b2l_state_destructor); + + ASSERT(BIF_P->flags & F_DISABLE_GC); + + return binary_to_list_chunk(BIF_P, + BIF_ARG_1, + (ErtsB2LState*) ERTS_MAGIC_BIN_DATA(mbp), + ERTS_BIF_REDS_LEFT(BIF_P), + 1); +} + +HIPE_WRAPPER_BIF_DISABLE_GC(binary_to_list, 1) + BIF_RETTYPE binary_to_list_1(BIF_ALIST_1) { Eterm real_bin; @@ -354,14 +486,15 @@ BIF_RETTYPE binary_to_list_1(BIF_ALIST_1) } else { Eterm* hp = HAlloc(BIF_P, 2 * size); byte* bytes = binary_bytes(real_bin)+offset; - - BIF_RET(erts_bin_bytes_to_list(NIL, hp, bytes, size, bitoffs)); + return binary_to_list(BIF_P, hp, NIL, bytes, size, bitoffs); } error: BIF_ERROR(BIF_P, BADARG); } +HIPE_WRAPPER_BIF_DISABLE_GC(binary_to_list, 3) + BIF_RETTYPE binary_to_list_3(BIF_ALIST_3) { byte* bytes; @@ -387,12 +520,13 @@ BIF_RETTYPE binary_to_list_3(BIF_ALIST_3) } i = stop-start+1; hp = HAlloc(BIF_P, 2*i); - BIF_RET(erts_bin_bytes_to_list(NIL, hp, bytes+start-1, i, bitoffs)); - + return binary_to_list(BIF_P, hp, NIL, bytes+start-1, i, bitoffs); error: BIF_ERROR(BIF_P, BADARG); } +HIPE_WRAPPER_BIF_DISABLE_GC(bitstring_to_list, 1) + BIF_RETTYPE bitstring_to_list_1(BIF_ALIST_1) { Eterm real_bin; @@ -431,113 +565,441 @@ BIF_RETTYPE bitstring_to_list_1(BIF_ALIST_1) previous = CONS(hp, make_binary(last), previous); hp += 2; } - BIF_RET(erts_bin_bytes_to_list(previous, hp, bytes, size, bitoffs)); + + return binary_to_list(BIF_P, hp, previous, bytes, size, bitoffs); } /* Turn a possibly deep list of ints (and binaries) into */ /* One large binary object */ -/* - * This bif also exists in the binary module, under the name - * binary:list_to_bin/1, why it's divided into interface and - * implementation. Also the backend for iolist_to_binary_1. - */ +typedef enum { + ERTS_L2B_OK, + ERTS_L2B_YIELD, + ERTS_L2B_TYPE_ERROR, + ERTS_L2B_OVERFLOW_ERROR +} ErtsL2BResult; -BIF_RETTYPE erts_list_to_binary_bif(Process *p, Eterm arg) -{ +#define ERTS_L2B_STATE_INITER(C_P, ARG, BIF, SZFunc, TBufFunc) \ + {ERTS_IOLIST2BUF_STATE_INITER((C_P), (ARG)), \ + (ARG), THE_NON_VALUE, (BIF), (SZFunc), (TBufFunc)} + +#define ERTS_L2B_STATE_MOVE(TO, FROM) \ + sys_memcpy((void *) (TO), (void *) (FROM), sizeof(ErtsL2BState)) + +typedef struct ErtsL2BState_ ErtsL2BState; + +struct ErtsL2BState_ { + ErtsIOList2BufState buf; + Eterm arg; Eterm bin; - ErlDrvSizeT size; - byte* bytes; -#ifdef DEBUG - ErlDrvSizeT offset; -#endif + Export *bif; + int (*iolist_to_buf_size)(ErtsIOListState *); + ErlDrvSizeT (*iolist_to_buf)(ErtsIOList2BufState *); +}; + +static ERTS_INLINE ErtsL2BResult +list_to_binary_engine(ErtsL2BState *sp) +{ + ErlDrvSizeT res; + Process *c_p = sp->buf.iolist.c_p; + + /* + * have_size == 0 while sp->iolist_to_buf_size() + * has not finished the calculation. + */ + + if (!sp->buf.iolist.have_size) { + switch (sp->iolist_to_buf_size(&sp->buf.iolist)) { + case ERTS_IOLIST_YIELD: + return ERTS_L2B_YIELD; + case ERTS_IOLIST_OVERFLOW: + return ERTS_L2B_OVERFLOW_ERROR; + case ERTS_IOLIST_TYPE: + return ERTS_L2B_TYPE_ERROR; + case ERTS_IOLIST_OK: + break; + default: + ASSERT(0); + break; + } + + ASSERT(sp->buf.iolist.have_size); + + /* + * Size calculated... Setup state for + * sp->iolist_to_buf_*() + */ + + sp->bin = new_binary(c_p, + (byte *) NULL, + sp->buf.iolist.size); + + if (sp->buf.iolist.size == 0) + return ERTS_L2B_OK; - if (is_nil(arg)) { - BIF_RET(new_binary(p,(byte*)"",0)); + sp->buf.buf = (char *) binary_bytes(sp->bin); + sp->buf.len = sp->buf.iolist.size; + sp->buf.iolist.obj = sp->arg; + + if (sp->buf.iolist.reds_left <= 0) { + BUMP_ALL_REDS(c_p); + return ERTS_L2B_YIELD; + } } - if (is_not_list(arg)) { - goto error; + + ASSERT(sp->buf.iolist.size != 0); + ASSERT(is_value(sp->bin)); + ASSERT(sp->buf.buf); + + res = sp->iolist_to_buf(&sp->buf); + + if (!ERTS_IOLIST_TO_BUF_FAILED(res)) { + ASSERT(res == 0); + return ERTS_L2B_OK; } - switch (erts_iolist_size(arg, &size)) { - case ERTS_IOLIST_OVERFLOW: BIF_ERROR(p, SYSTEM_LIMIT); - case ERTS_IOLIST_TYPE: goto error; - default: ; + + switch (res) { + case ERTS_IOLIST_TO_BUF_YIELD: + return ERTS_L2B_YIELD; + case ERTS_IOLIST_TO_BUF_OVERFLOW: + return ERTS_L2B_OVERFLOW_ERROR; + case ERTS_IOLIST_TO_BUF_TYPE_ERROR: + return ERTS_L2B_TYPE_ERROR; + default: + ERTS_INTERNAL_ERROR("Invalid return value from iolist_to_buf_yielding()"); + return ERTS_L2B_TYPE_ERROR; } - bin = new_binary(p, (byte *)NULL, size); - bytes = binary_bytes(bin); -#ifdef DEBUG - offset = -#endif - erts_iolist_to_buf(arg, (char*) bytes, size); +} + +static void +l2b_state_destructor(Binary *mbp) +{ + ErtsL2BState *sp = ERTS_MAGIC_BIN_DATA(mbp); + ASSERT(ERTS_MAGIC_BIN_DESTRUCTOR(mbp) == l2b_state_destructor); + DESTROY_SAVED_ESTACK(&sp->buf.iolist.estack); +} + +static ERTS_INLINE Eterm +l2b_final_touch(Process *c_p, ErtsL2BState *sp) +{ + Eterm *hp; + ErlSubBin* sbin; + if (sp->buf.offset == 0) + return sp->bin; + + hp = HAlloc(c_p, ERL_SUB_BIN_SIZE); + ASSERT(sp->buf.offset > 0); + sbin = (ErlSubBin *) hp; + sbin->thing_word = HEADER_SUB_BIN; + sbin->size = sp->buf.iolist.size-1; + sbin->offs = 0; + sbin->orig = sp->bin; + sbin->bitoffs = 0; + sbin->bitsize = sp->buf.offset; + sbin->is_writable = 0; + return make_binary(sbin); +} - ASSERT(offset == 0); - BIF_RET(bin); +static BIF_RETTYPE +list_to_binary_chunk(Eterm mb_eterm, + ErtsL2BState* sp, + int reds_left, + int gc_disabled) +{ + Eterm err = BADARG; + BIF_RETTYPE ret; + Process *c_p = sp->buf.iolist.c_p; + + sp->buf.iolist.reds_left = reds_left; - error: - BIF_ERROR(p, BADARG); + switch (list_to_binary_engine(sp)) { + + case ERTS_L2B_OK: { + Eterm result = l2b_final_touch(c_p, sp); + if (!gc_disabled || !erts_set_gc_state(c_p, 1)) + ERTS_BIF_PREP_RET(ret, result); + else + ERTS_BIF_PREP_YIELD_RETURN(ret, c_p, result); + ASSERT(!(c_p->flags & F_DISABLE_GC)); + break; + } + case ERTS_L2B_YIELD: + if (!gc_disabled) { + /* first yield... */ + Eterm *hp; + Binary *mbp = erts_create_magic_binary(sizeof(ErtsL2BState), + l2b_state_destructor); + ErtsL2BState *new_sp = ERTS_MAGIC_BIN_DATA(mbp); + + ERTS_L2B_STATE_MOVE(new_sp, sp); + sp = new_sp; + + hp = HAlloc(c_p, PROC_BIN_SIZE); + mb_eterm = erts_mk_magic_binary_term(&hp, &MSO(c_p), mbp); + + ASSERT(is_value(mb_eterm)); + + erts_set_gc_state(c_p, 0); + } + + ASSERT(c_p->flags & F_DISABLE_GC); + + ERTS_BIF_PREP_TRAP1(ret, + &list_to_binary_continue_export, + c_p, + mb_eterm); + break; + + case ERTS_L2B_OVERFLOW_ERROR: + err = SYSTEM_LIMIT; + /* fall through */ + + case ERTS_L2B_TYPE_ERROR: + if (!gc_disabled) + ERTS_BIF_PREP_ERROR(ret, c_p, err); + else { + if (erts_set_gc_state(c_p, 1)) + ERTS_VBUMP_ALL_REDS(c_p); + + ERTS_BIF_PREP_ERROR_TRAPPED1(ret, + c_p, + err, + sp->bif, + sp->arg); + } + + ASSERT(!(c_p->flags & F_DISABLE_GC)); + break; + + default: + ERTS_INTERNAL_ERROR("Invalid return value from list_to_binary_engine()"); + ERTS_BIF_PREP_ERROR(ret,c_p, EXC_INTERNAL_ERROR); + break; + } + return ret; } +static BIF_RETTYPE list_to_binary_continue(BIF_ALIST_1) +{ + Binary *mbp = ((ProcBin *) binary_val(BIF_ARG_1))->val; + ASSERT(ERTS_MAGIC_BIN_DESTRUCTOR(mbp) == l2b_state_destructor); + + ASSERT(BIF_P->flags & F_DISABLE_GC); + + return list_to_binary_chunk(BIF_ARG_1, + ERTS_MAGIC_BIN_DATA(mbp), + ERTS_BIF_REDS_LEFT(BIF_P), + 1); +} + +BIF_RETTYPE erts_list_to_binary_bif(Process *c_p, Eterm arg, Export *bif) +{ + BIF_RETTYPE ret; + + if (is_nil(arg)) + ERTS_BIF_PREP_RET(ret, new_binary(c_p, (byte *) "", 0)); + else if (is_not_list(arg)) + ERTS_BIF_PREP_ERROR(ret, c_p, BADARG); + else { + /* check for [binary()] case */ + Eterm h = CAR(list_val(arg)); + Eterm t = CDR(list_val(arg)); + if (is_binary(h) + && is_nil(t) + && !(HEADER_SUB_BIN == *(binary_val(h)) + && (((ErlSubBin *)binary_val(h))->bitoffs != 0 + || ((ErlSubBin *)binary_val(h))->bitsize != 0))) { + ERTS_BIF_PREP_RET(ret, h); + } + else { + ErtsL2BState state = ERTS_L2B_STATE_INITER(c_p, + arg, + bif, + erts_iolist_size_yielding, + erts_iolist_to_buf_yielding); + int orig_reds_left = ERTS_BIF_REDS_LEFT(c_p); + + /* + * First try to do it all at once without having to use + * yielding iolist_to_buf(). + */ + state.buf.iolist.reds_left = orig_reds_left; + switch (erts_iolist_size_yielding(&state.buf.iolist)) { + case ERTS_IOLIST_OK: { + ErlDrvSizeT size = state.buf.iolist.size; + Eterm bin; + char *buf; + + if (size == 0) { + ERTS_BIF_PREP_RET(ret, new_binary(c_p, (byte *) NULL, 0)); + break; /* done */ + } + + bin = new_binary(c_p, (byte *) NULL, size); + buf = (char *) binary_bytes(bin); + + if (size < ERTS_IOLIST_TO_BUF_BYTES_PER_RED*CONTEXT_REDS) { + /* An (over) estimation of reductions needed */ + int reds_left = state.buf.iolist.reds_left; + int to_buf_reds = orig_reds_left - reds_left; + to_buf_reds += size/ERTS_IOLIST_TO_BUF_BYTES_PER_RED; + if (to_buf_reds <= reds_left) { + ErlDrvSizeT res; + + res = erts_iolist_to_buf(arg, buf, size); + if (res == 0) { + BUMP_REDS(c_p, to_buf_reds); + ERTS_BIF_PREP_RET(ret, bin); + break; /* done */ + } + if (!ERTS_IOLIST_TO_BUF_FAILED(res)) + ERTS_INTERNAL_ERROR("iolist_size/iolist_to_buf missmatch"); + if (res == ERTS_IOLIST_TO_BUF_OVERFLOW) + goto overflow; + goto type_error; + } + } + /* + * Since size has been computed list_to_binary_chunk() expects + * state prepared for iolist_to_buf. + */ + state.bin = bin; + state.buf.buf = buf; + state.buf.len = size; + state.buf.iolist.obj = arg; + /* Fall through... */ + } + case ERTS_IOLIST_YIELD: + ret = list_to_binary_chunk(THE_NON_VALUE, + &state, + state.buf.iolist.reds_left, + 0); + break; + case ERTS_IOLIST_OVERFLOW: + overflow: + ERTS_BIF_PREP_ERROR(ret, c_p, SYSTEM_LIMIT); + break; + case ERTS_IOLIST_TYPE: + type_error: + default: + ERTS_BIF_PREP_ERROR(ret, c_p, BADARG); + break; + } + } + } + return ret; +} + +HIPE_WRAPPER_BIF_DISABLE_GC(list_to_binary, 1) + BIF_RETTYPE list_to_binary_1(BIF_ALIST_1) { - return erts_list_to_binary_bif(BIF_P, BIF_ARG_1); + return erts_list_to_binary_bif(BIF_P, BIF_ARG_1, bif_export[BIF_list_to_binary_1]); } -/* Turn a possibly deep list of ints (and binaries) into */ -/* One large binary object */ +HIPE_WRAPPER_BIF_DISABLE_GC(iolist_to_binary, 1) BIF_RETTYPE iolist_to_binary_1(BIF_ALIST_1) { if (is_binary(BIF_ARG_1)) { BIF_RET(BIF_ARG_1); } - return erts_list_to_binary_bif(BIF_P, BIF_ARG_1); + return erts_list_to_binary_bif(BIF_P, BIF_ARG_1, bif_export[BIF_iolist_to_binary_1]); } +static int bitstr_list_len(ErtsIOListState *); +static ErlDrvSizeT list_to_bitstr_buf_yielding(ErtsIOList2BufState *); +static ErlDrvSizeT list_to_bitstr_buf_not_yielding(ErtsIOList2BufState *); + +HIPE_WRAPPER_BIF_DISABLE_GC(list_to_bitstring, 1) + BIF_RETTYPE list_to_bitstring_1(BIF_ALIST_1) { - Eterm bin; - Uint sz; - int offset; - byte* bytes; - ErlSubBin* sb1; - Eterm* hp; - - if (is_nil(BIF_ARG_1)) { - BIF_RET(new_binary(BIF_P,(byte*)"",0)); - } - if (is_not_list(BIF_ARG_1)) { - error: - BIF_ERROR(BIF_P, BADARG); - } - switch (bitstr_list_len(BIF_ARG_1, &sz)) { - case ERTS_IOLIST_TYPE: - goto error; - case ERTS_IOLIST_OVERFLOW: - BIF_ERROR(BIF_P, SYSTEM_LIMIT); - } - bin = new_binary(BIF_P, (byte *)NULL, sz); - bytes = binary_bytes(bin); -#ifdef DEBUG - offset = list_to_bitstr_buf(BIF_ARG_1, (char*) bytes, sz); -#else - offset = list_to_bitstr_buf(BIF_ARG_1, (char*) bytes); -#endif - ASSERT(offset >= 0); - if (offset > 0) { - hp = HAlloc(BIF_P, ERL_SUB_BIN_SIZE); - sb1 = (ErlSubBin *) hp; - sb1->thing_word = HEADER_SUB_BIN; - sb1->size = sz-1; - sb1->offs = 0; - sb1->orig = bin; - sb1->bitoffs = 0; - sb1->bitsize = offset; - sb1->is_writable = 0; - bin = make_binary(sb1); + BIF_RETTYPE ret; + + if (is_nil(BIF_ARG_1)) + ERTS_BIF_PREP_RET(ret, new_binary(BIF_P, (byte *) "", 0)); + else if (is_not_list(BIF_ARG_1)) + ERTS_BIF_PREP_ERROR(ret, BIF_P, BADARG); + else { + /* check for [bitstring()] case */ + Eterm h = CAR(list_val(BIF_ARG_1)); + Eterm t = CDR(list_val(BIF_ARG_1)); + if (is_binary(h) && is_nil(t)) { + ERTS_BIF_PREP_RET(ret, h); + } + else { + ErtsL2BState state = ERTS_L2B_STATE_INITER(BIF_P, + BIF_ARG_1, + bif_export[BIF_list_to_bitstring_1], + bitstr_list_len, + list_to_bitstr_buf_yielding); + int orig_reds_left = ERTS_BIF_REDS_LEFT(BIF_P); + + /* + * First try to do it all at once without having to use + * yielding list_to_bitstr_buf(). + */ + state.buf.iolist.reds_left = orig_reds_left; + switch (bitstr_list_len(&state.buf.iolist)) { + case ERTS_IOLIST_OK: { + ErlDrvSizeT size = state.buf.iolist.size; + + state.bin = new_binary(BIF_P, (byte *) NULL, size); + state.buf.buf = (char *) binary_bytes(state.bin); + state.buf.len = size; + state.buf.iolist.obj = BIF_ARG_1; + + if (size < ERTS_IOLIST_TO_BUF_BYTES_PER_RED*CONTEXT_REDS) { + /* An (over) estimation of reductions needed */ + int reds_left = state.buf.iolist.reds_left; + int to_buf_reds = orig_reds_left - reds_left; + to_buf_reds += size/ERTS_IOLIST_TO_BUF_BYTES_PER_RED; + if (to_buf_reds <= reds_left) { + ErlDrvSizeT res; + + res = list_to_bitstr_buf_not_yielding(&state.buf); + if (res == 0) { + Eterm res_bin = l2b_final_touch(BIF_P, &state); + BUMP_REDS(BIF_P, to_buf_reds); + ERTS_BIF_PREP_RET(ret, res_bin); + break; /* done */ + } + if (!ERTS_IOLIST_TO_BUF_FAILED(res)) + ERTS_INTERNAL_ERROR("iolist_size/iolist_to_buf missmatch"); + if (res == ERTS_IOLIST_TO_BUF_OVERFLOW) + goto overflow; + goto type_error; + } + } + /* + * Since size has been computed list_to_binary_chunk() expects + * the state prepared for list_to_bitstr_buf. + */ + + /* Fall through... */ + } + case ERTS_IOLIST_YIELD: + ret = list_to_binary_chunk(THE_NON_VALUE, + &state, + state.buf.iolist.reds_left, + 0); + break; + case ERTS_IOLIST_OVERFLOW: + overflow: + ERTS_BIF_PREP_ERROR(ret, BIF_P, SYSTEM_LIMIT); + break; + case ERTS_IOLIST_TYPE: + type_error: + default: + ERTS_BIF_PREP_ERROR(ret, BIF_P, BADARG); + break; + } + } } - - BIF_RET(bin); + + return ret; } BIF_RETTYPE split_binary_2(BIF_ALIST_2) @@ -594,123 +1056,353 @@ BIF_RETTYPE split_binary_2(BIF_ALIST_2) * Local functions. */ +static int +list_to_bitstr_buf_bcopy(ErtsIOList2BufState *state, Eterm obj, int *yield_countp); + /* * The input list is assumed to be type-correct and the buffer is * assumed to be of sufficient size. Those assumptions are verified in * the DEBUG-built emulator. */ -static int +static ErlDrvSizeT +list_to_bitstr_buf(int yield_support, ErtsIOList2BufState *state) +{ + +#undef LIST_TO_BITSTR_BUF_BCOPY_DBG +#undef LIST_TO_BITSTR_BUF_BCOPY #ifdef DEBUG -list_to_bitstr_buf(Eterm obj, char* buf, Uint len) +#define LIST_TO_BITSTR_BUF_BCOPY_DBG \ + len -= size + (offset>7); #else -list_to_bitstr_buf(Eterm obj, char* buf) +#define LIST_TO_BITSTR_BUF_BCOPY_DBG #endif -{ - Eterm* objp; - int offset = 0; +#define LIST_TO_BITSTR_BUF_BCOPY(CONSP) \ + do { \ + byte* bptr; \ + Uint bitsize; \ + Uint bitoffs; \ + Uint num_bits; \ + size_t size = binary_size(obj); \ + if (yield_support) { \ + size_t max_size = ERTS_IOLIST_TO_BUF_BYTES_PER_YIELD_COUNT; \ + if (yield_count > 0) \ + max_size *= yield_count+1; \ + if (size > max_size) { \ + state->objp = CONSP; \ + goto L_bcopy_yield; \ + } \ + if (size >= ERTS_IOLIST_TO_BUF_BYTES_PER_YIELD_COUNT) { \ + int cost = (int) size; \ + cost /= ERTS_IOLIST_TO_BUF_BYTES_PER_YIELD_COUNT; \ + yield_count -= cost; \ + } \ + } \ + ASSERT(size <= len); \ + ERTS_GET_BINARY_BYTES(obj, bptr, bitoffs, bitsize); \ + num_bits = 8*size+bitsize; \ + copy_binary_to_buffer(buf, offset, bptr, bitoffs, num_bits); \ + offset += bitsize; \ + buf += size + (offset>7); \ + LIST_TO_BITSTR_BUF_BCOPY_DBG; \ + offset = offset & 7; \ + } while(0) + +#ifdef DEBUG + ErlDrvSizeT len; +#endif + Eterm obj; + char *buf; + Eterm *objp = NULL; + int offset; + int init_yield_count = 0, yield_count; DECLARE_ESTACK(s); - goto L_again; - - while (!ESTACK_ISEMPTY(s)) { - obj = ESTACK_POP(s); - L_again: - if (is_list(obj)) { - L_iter_list: - objp = list_val(obj); - obj = CAR(objp); - if (is_byte(obj)) { - ASSERT(len > 0); - if (offset == 0) { - *buf++ = unsigned_val(obj); - } else { - *buf = (char)((unsigned_val(obj) >> offset) | - ((*buf >> (8-offset)) << (8-offset))); - buf++; - *buf = (unsigned_val(obj) << (8-offset)); - } + + obj = state->iolist.obj; + buf = state->buf; + offset = state->offset; #ifdef DEBUG - len--; + len = state->len; #endif - } else if (is_binary(obj)) { - byte* bptr; - size_t size = binary_size(obj); - Uint bitsize; - Uint bitoffs; - Uint num_bits; - - ASSERT(size <= len); - ERTS_GET_BINARY_BYTES(obj, bptr, bitoffs, bitsize); - num_bits = 8*size+bitsize; - copy_binary_to_buffer(buf, offset, bptr, bitoffs, num_bits); - offset += bitsize; - buf += size + (offset>7); + + if (!yield_support) { + yield_count = init_yield_count = 0; /* Shut up faulty warning... >:-( */ + goto L_again; + } + else { + + if (state->iolist.reds_left <= 0) + return ERTS_IOLIST_TO_BUF_YIELD; + + ESTACK_CHANGE_ALLOCATOR(s, ERTS_ALC_T_SAVED_ESTACK); + init_yield_count = (ERTS_IOLIST_TO_BUF_YIELD_COUNT_PER_RED + * state->iolist.reds_left); + yield_count = init_yield_count; + + if (!state->iolist.estack.start) + goto L_again; + else { + int chk_stack; + /* Restart; restore state... */ + ESTACK_RESTORE(s, &state->iolist.estack); + + if (!state->bcopy.bptr) + chk_stack = 0; + else { + chk_stack = 1; + if (list_to_bitstr_buf_bcopy(state, THE_NON_VALUE, &yield_count)) { + /* Yield again... */ + BUMP_ALL_REDS(state->iolist.c_p); + state->iolist.reds_left = 0; + ESTACK_SAVE(s, &state->iolist.estack); + return ERTS_IOLIST_TO_BUF_YIELD; + } + buf = state->buf; + offset = state->offset; #ifdef DEBUG - len -= size + (offset>7); + len = state->len; #endif - offset = offset & 7; - } else if (is_list(obj)) { - ESTACK_PUSH(s, CDR(objp)); - goto L_iter_list; /* on head */ - } else { - ASSERT(is_nil(obj)); } - obj = CDR(objp); - if (is_list(obj)) { - goto L_iter_list; /* on tail */ - } else if (is_binary(obj)) { - byte* bptr; - size_t size = binary_size(obj); - Uint bitsize; - Uint bitoffs; - Uint num_bits; - - ASSERT(size <= len); - ERTS_GET_BINARY_BYTES(obj, bptr, bitoffs, bitsize); - num_bits = 8*size+bitsize; - copy_binary_to_buffer(buf, offset, bptr, bitoffs, num_bits); - offset += bitsize; - buf += size+(offset>7); + objp = state->objp; + state->objp = NULL; + + if (objp) + goto L_tail; + if (!chk_stack) + goto L_again; + /* check stack */ + } + } + + while (!ESTACK_ISEMPTY(s)) { + obj = ESTACK_POP(s); + L_again: + if (is_list(obj)) { + while (1) { /* Tail loop */ + while (1) { /* Head loop */ + if (yield_support && --yield_count <= 0) + goto L_yield; + objp = list_val(obj); + obj = CAR(objp); + if (is_byte(obj)) { + ASSERT(len > 0); + if (offset == 0) { + *buf++ = unsigned_val(obj); + } else { + *buf = (char)((unsigned_val(obj) >> offset) | + ((*buf >> (8-offset)) << (8-offset))); + buf++; + *buf = (unsigned_val(obj) << (8-offset)); + } #ifdef DEBUG - len -= size+(offset>7); + len--; #endif - offset = offset & 7; - } else { - ASSERT(is_nil(obj)); + } else if (is_binary(obj)) { + LIST_TO_BITSTR_BUF_BCOPY(objp); + } else if (is_list(obj)) { + ESTACK_PUSH(s, CDR(objp)); + continue; /* Head loop */ + } else { + ASSERT(is_nil(obj)); + } + break; + } + + L_tail: + + obj = CDR(objp); + if (is_list(obj)) { + continue; /* Tail loop */ + } else if (is_binary(obj)) { + LIST_TO_BITSTR_BUF_BCOPY(NULL); + } else { + ASSERT(is_nil(obj)); + } + break; } } else if (is_binary(obj)) { - byte* bptr; - size_t size = binary_size(obj); - Uint bitsize; - Uint bitoffs; - Uint num_bits; - - ASSERT(size <= len); - ERTS_GET_BINARY_BYTES(obj, bptr, bitoffs, bitsize); - num_bits = 8*size+bitsize; - copy_binary_to_buffer(buf, offset, bptr, bitoffs, num_bits); - offset += bitsize; - buf += size + (offset>7); -#ifdef DEBUG - len -= size + (offset>7); -#endif - offset = offset & 7; + LIST_TO_BITSTR_BUF_BCOPY(NULL); } else { + if (yield_support && --yield_count <= 0) + goto L_yield; ASSERT(is_nil(obj)); } } DESTROY_ESTACK(s); - return offset; + + if (yield_support) { + int reds; + CLEAR_SAVED_ESTACK(&state->iolist.estack); + reds = ((init_yield_count - yield_count - 1) + / ERTS_IOLIST_TO_BUF_YIELD_COUNT_PER_RED) + 1; + BUMP_REDS(state->iolist.c_p, reds); + state->iolist.reds_left -= reds; + if (state->iolist.reds_left < 0) + state->iolist.reds_left = 0; + } + state->buf = buf; + state->offset = offset; + return 0; + +L_bcopy_yield: + + state->buf = buf; + state->offset = offset; +#ifdef DEBUG + state->len = len; +#endif + + if (list_to_bitstr_buf_bcopy(state, obj, &yield_count) == 0) + ERTS_INTERNAL_ERROR("Missing yield"); + + BUMP_ALL_REDS(state->iolist.c_p); + state->iolist.reds_left = 0; + ESTACK_SAVE(s, &state->iolist.estack); + return ERTS_IOLIST_TO_BUF_YIELD; + +L_yield: + + BUMP_ALL_REDS(state->iolist.c_p); + state->iolist.reds_left = 0; + state->iolist.obj = obj; + state->buf = buf; + state->offset = offset; + ESTACK_SAVE(s, &state->iolist.estack); +#ifdef DEBUG + state->len = len; +#endif + return ERTS_IOLIST_TO_BUF_YIELD; + + +#undef LIST_TO_BITSTR_BUF_BCOPY_DBG +#undef LIST_TO_BITSTR_BUF_BCOPY + +} + +static ErlDrvSizeT +list_to_bitstr_buf_yielding(ErtsIOList2BufState *state) +{ + return list_to_bitstr_buf(1, state); +} + +static ErlDrvSizeT +list_to_bitstr_buf_not_yielding(ErtsIOList2BufState *state) +{ + return list_to_bitstr_buf(0, state); } static int -bitstr_list_len(Eterm obj, Uint* num_bytes) +list_to_bitstr_buf_bcopy(ErtsIOList2BufState *state, Eterm obj, int *yield_countp) +{ + int res; + char *buf = state->buf; + char *next_buf; + int offset = state->offset; + int next_offset; +#ifdef DEBUG + ErlDrvSizeT len = state->len; + ErlDrvSizeT next_len; +#endif + byte* bptr; + size_t size; + size_t max_size; + Uint bitoffs; + Uint num_bits; + Uint bitsize; + int yield_count = *yield_countp; + + if (state->bcopy.bptr) { + bptr = state->bcopy.bptr; + size = state->bcopy.size; + bitoffs = state->bcopy.bitoffs; + bitsize = state->bcopy.bitsize; + state->bcopy.bptr = NULL; + } + else { + + ASSERT(is_binary(obj)); + + size = binary_size(obj); + + ASSERT(size <= len); + + ERTS_GET_BINARY_BYTES(obj, bptr, bitoffs, bitsize); + } + + max_size = (size_t) ERTS_IOLIST_TO_BUF_BYTES_PER_YIELD_COUNT; + if (yield_count > 0) + max_size *= (size_t) (yield_count+1); + + if (size <= max_size) { + if (size >= ERTS_IOLIST_TO_BUF_BYTES_PER_YIELD_COUNT) { + int cost = (int) size; + cost /= ERTS_IOLIST_TO_BUF_BYTES_PER_YIELD_COUNT; + yield_count -= cost; + } + next_offset = offset + bitsize; + next_buf = buf + size+(next_offset>7); +#ifdef DEBUG + next_len = len - size+(next_offset>7); +#endif + next_offset &= 7; + num_bits = 8*size+bitsize; + res = 0; + } + else { + ASSERT(0 < max_size && max_size < size); + yield_count = 0; + state->bcopy.bptr = bptr + max_size; + state->bcopy.bitoffs = bitoffs; + state->bcopy.bitsize = bitsize; + state->bcopy.size = size - max_size; + next_buf = buf + max_size; +#ifdef DEBUG + next_len = len - max_size; +#endif + next_offset = offset; + num_bits = 8*max_size; + size = max_size; + res = 1; + } + + copy_binary_to_buffer(buf, offset, bptr, bitoffs, num_bits); + + state->offset = next_offset; + state->buf = next_buf; +#ifdef DEBUG + state->len = next_len; +#endif + *yield_countp = yield_count; + + return res; +} + +static int +bitstr_list_len(ErtsIOListState *state) { Eterm* objp; - Uint len = 0; - Uint offs = 0; + Eterm obj; + Uint len, offs; + int res, init_yield_count, yield_count; DECLARE_ESTACK(s); + + if (state->reds_left <= 0) + return ERTS_IOLIST_YIELD; + + len = (Uint) state->size; + offs = state->offs; + obj = state->obj; + + ESTACK_CHANGE_ALLOCATOR(s, ERTS_ALC_T_SAVED_ESTACK); + init_yield_count = ERTS_IOLIST_SIZE_YIELDS_COUNT_PER_RED; + init_yield_count *= state->reds_left; + yield_count = init_yield_count; + if (state->estack.start) { + /* Restart; restore estack... */ + ESTACK_RESTORE(s, &state->estack); + } + goto L_again; #define SAFE_ADD(Var, Val) \ @@ -737,46 +1429,55 @@ bitstr_list_len(Eterm obj, Uint* num_bytes) obj = ESTACK_POP(s); L_again: if (is_list(obj)) { - L_iter_list: - objp = list_val(obj); - /* Head */ - obj = CAR(objp); - if (is_byte(obj)) { - len++; - if (len == 0) { - goto L_overflow_error; + while (1) { /* Tail loop */ + while (1) { /* Head loop */ + if (--yield_count <= 0) + goto L_yield; + objp = list_val(obj); + /* Head */ + obj = CAR(objp); + if (is_byte(obj)) { + len++; + if (len == 0) { + goto L_overflow_error; + } + } else if (is_binary(obj)) { + SAFE_ADD(len, binary_size(obj)); + SAFE_ADD_BITSIZE(offs, obj); + } else if (is_list(obj)) { + ESTACK_PUSH(s, CDR(objp)); + continue; /* Head loop */ + } else if (is_not_nil(obj)) { + goto L_type_error; + } + break; } - } else if (is_binary(obj)) { - SAFE_ADD(len, binary_size(obj)); - SAFE_ADD_BITSIZE(offs, obj); - } else if (is_list(obj)) { - ESTACK_PUSH(s, CDR(objp)); - goto L_iter_list; /* on head */ - } else if (is_not_nil(obj)) { - goto L_type_error; + /* Tail */ + obj = CDR(objp); + if (is_list(obj)) + continue; /* Tail loop */ + else if (is_binary(obj)) { + SAFE_ADD(len, binary_size(obj)); + SAFE_ADD_BITSIZE(offs, obj); + } else if (is_not_nil(obj)) { + goto L_type_error; + } + break; } - /* Tail */ - obj = CDR(objp); - if (is_list(obj)) - goto L_iter_list; /* on tail */ - else if (is_binary(obj)) { + } else { + if (--yield_count <= 0) + goto L_yield; + if (is_binary(obj)) { SAFE_ADD(len, binary_size(obj)); SAFE_ADD_BITSIZE(offs, obj); } else if (is_not_nil(obj)) { goto L_type_error; } - } else if (is_binary(obj)) { - SAFE_ADD(len, binary_size(obj)); - SAFE_ADD_BITSIZE(offs, obj); - } else if (is_not_nil(obj)) { - goto L_type_error; } } #undef SAFE_ADD #undef SAFE_ADD_BITSIZE - DESTROY_ESTACK(s); - /* * Make sure that the number of bits in the bitstring will fit * in an Uint to ensure that the binary can be matched using @@ -789,15 +1490,42 @@ bitstr_list_len(Eterm obj, Uint* num_bytes) if (len << 3 < len) { goto L_overflow_error; } - *num_bytes = len; - return ERTS_IOLIST_OK; + state->size = len; - L_type_error: - DESTROY_ESTACK(s); - return ERTS_IOLIST_TYPE; + res = ERTS_IOLIST_OK; + + L_return: { + int yc = init_yield_count - yield_count; + int reds; + + DESTROY_ESTACK(s); + CLEAR_SAVED_ESTACK(&state->estack); + + reds = (yc - 1)/ERTS_IOLIST_SIZE_YIELDS_COUNT_PER_RED + 1; + BUMP_REDS(state->c_p, reds); + state->reds_left -= reds; + state->size = (ErlDrvSizeT) len; + state->have_size = 1; + return res; + } L_overflow_error: - DESTROY_ESTACK(s); - return ERTS_IOLIST_OVERFLOW; + res = ERTS_IOLIST_OVERFLOW; + len = 0; + goto L_return; + + L_type_error: + res = ERTS_IOLIST_TYPE; + len = 0; + goto L_return; + + L_yield: + BUMP_ALL_REDS(state->c_p); + state->reds_left = 0; + state->size = len; + state->offs = offs; + state->obj = obj; + ESTACK_SAVE(s, &state->estack); + return ERTS_IOLIST_YIELD; } diff --git a/erts/emulator/beam/break.c b/erts/emulator/beam/break.c index 9aa1e5f30d..0ddf7f4e6d 100644 --- a/erts/emulator/beam/break.c +++ b/erts/emulator/beam/break.c @@ -1,18 +1,19 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 1996-2012. All Rights Reserved. + * Copyright Ericsson AB 1996-2013. All Rights Reserved. * - * The contents of this file are subject to the Erlang Public License, - * Version 1.1, (the "License"); you may not use this file except in - * compliance with the License. You should have received a copy of the - * Erlang Public License along with this software. If not, it can be - * retrieved online at http://www.erlang.org/. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Software distributed under the License is distributed on an "AS IS" - * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See - * the License for the specific language governing rights and limitations - * under the License. + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. * * %CopyrightEnd% */ @@ -36,7 +37,7 @@ #include "atom.h" #include "beam_load.h" #include "erl_instrument.h" -#include "erl_bif_timer.h" +#include "erl_hl_timer.h" #include "erl_thr_progress.h" /* Forward declarations -- should really appear somewhere else */ @@ -76,7 +77,10 @@ process_info(int to, void *to_arg) for (i = 0; i < max; i++) { Process *p = erts_pix2proc(i); if (p && p->i != ENULL) { - if (!ERTS_PROC_IS_EXITING(p)) + /* Do not include processes with no heap, + * they are most likely just created and has invalid data + */ + if (!ERTS_PROC_IS_EXITING(p) && p->heap != NULL) print_process_info(to, to_arg, p); } } @@ -100,19 +104,21 @@ process_killer(void) erts_printf("(k)ill (n)ext (r)eturn:\n"); while(1) { if ((j = sys_get_key(0)) <= 0) - erl_exit(0, ""); + erts_exit(0, ""); switch(j) { case 'k': { ErtsProcLocks rp_locks = ERTS_PROC_LOCKS_XSIG_SEND; erts_aint32_t state; - erts_smp_proc_inc_refc(rp); + erts_proc_inc_refc(rp); erts_smp_proc_lock(rp, rp_locks); state = erts_smp_atomic32_read_acqb(&rp->state); if (state & (ERTS_PSFLG_FREE - | ERTS_PSFLG_EXITING - | ERTS_PSFLG_ACTIVE - | ERTS_PSFLG_IN_RUNQ - | ERTS_PSFLG_RUNNING)) { + | ERTS_PSFLG_EXITING + | ERTS_PSFLG_ACTIVE + | ERTS_PSFLG_ACTIVE_SYS + | ERTS_PSFLG_IN_RUNQ + | ERTS_PSFLG_RUNNING + | ERTS_PSFLG_RUNNING_SYS)) { erts_printf("Can only kill WAITING processes this way\n"); } else { @@ -126,7 +132,7 @@ process_killer(void) 0); } erts_smp_proc_unlock(rp, rp_locks); - erts_smp_proc_dec_refc(rp); + erts_proc_dec_refc(rp); } case 'n': br = 1; break; case 'r': return; @@ -176,7 +182,6 @@ static void doit_print_monitor(ErtsMonitor *mon, void *vpcontext) ASSERT(is_node_name_atom(mon->pid)); erts_print(to, to_arg, "%s{to,{%T,%T},%T}", prefix, mon->name, mon->pid, mon->ref); - erts_print(to, to_arg,"}"); } else if (is_atom(mon->name)){ /* local by name */ erts_print(to, to_arg, "%s{to,{%T,%T},%T}", prefix, mon->name, erts_this_dist_entry->sysname, mon->ref); @@ -205,25 +210,12 @@ print_process_info(int to, void *to_arg, Process *p) erts_print(to, to_arg, "State: "); state = erts_smp_atomic32_read_acqb(&p->state); - if (state & ERTS_PSFLG_FREE) - erts_print(to, to_arg, "Non Existing\n"); /* Should never happen */ - else if (state & ERTS_PSFLG_EXITING) - erts_print(to, to_arg, "Exiting\n"); - else if (state & ERTS_PSFLG_GC) { - garbing = 1; - running = 1; - erts_print(to, to_arg, "Garbing\n"); - } - else if (state & ERTS_PSFLG_SUSPENDED) - erts_print(to, to_arg, "Suspended\n"); - else if (state & ERTS_PSFLG_RUNNING) { - running = 1; - erts_print(to, to_arg, "Running\n"); - } - else if (state & ERTS_PSFLG_ACTIVE) - erts_print(to, to_arg, "Scheduled\n"); - else - erts_print(to, to_arg, "Waiting\n"); + erts_dump_process_state(to, to_arg, state); + if (state & ERTS_PSFLG_GC) { + garbing = 1; + running = 1; + } else if (state & ERTS_PSFLG_RUNNING) + running = 1; /* * If the process is registered as a global process, display the @@ -236,9 +228,9 @@ print_process_info(int to, void *to_arg, Process *p) * Display the initial function name */ erts_print(to, to_arg, "Spawned as: %T:%T/%bpu\n", - p->initial[INITIAL_MOD], - p->initial[INITIAL_FUN], - p->initial[INITIAL_ARI]); + p->u.initial[INITIAL_MOD], + p->u.initial[INITIAL_FUN], + p->u.initial[INITIAL_ARI]); if (p->current != NULL) { if (running) { @@ -331,6 +323,7 @@ print_process_info(int to, void *to_arg, Process *p) erts_print(to, to_arg, "Heap unused: %bpu\n", (p->hend - p->htop)); erts_print(to, to_arg, "OldHeap unused: %bpu\n", (OLD_HEAP(p) == NULL) ? 0 : (OLD_HEND(p) - OLD_HTOP(p)) ); + erts_print(to, to_arg, "Memory: %beu\n", erts_process_memory(p)); if (garbing) { print_garb_info(to, to_arg, p); @@ -345,6 +338,10 @@ print_process_info(int to, void *to_arg, Process *p) #endif erts_stack_dump(to, to_arg, p); } + + /* Display all states */ + erts_print(to, to_arg, "Internal State: "); + erts_dump_extended_process_state(to, to_arg, state); } static void @@ -496,7 +493,7 @@ do_break(void) halt immediately if break is called */ mode = erts_read_env("ERL_CONSOLE_MODE"); if (mode && strcmp(mode, "window") != 0) - erl_exit(0, ""); + erts_exit(0, ""); erts_free_read_env(mode); #endif /* __WIN32__ */ @@ -506,7 +503,7 @@ do_break(void) while (1) { if ((i = sys_get_key(0)) <= 0) - erl_exit(0, ""); + erts_exit(0, ""); switch (i) { case 'q': case 'a': @@ -516,9 +513,9 @@ do_break(void) * The usual reason for a read error is Ctrl-C. Treat this as * 'a' to avoid infinite loop. */ - erl_exit(0, ""); + erts_exit(0, ""); case 'A': /* Halt generating crash dump */ - erl_exit(1, "Crash dump requested by user"); + erts_exit(ERTS_ERROR_EXIT, "Crash dump requested by user"); case 'c': return; case 'p': @@ -539,7 +536,9 @@ do_break(void) erts_printf("Erlang (%s) emulator version " ERLANG_VERSION "\n", EMULATOR); +#if ERTS_SAVED_COMPILE_TIME erts_printf("Compiled on " ERLANG_COMPILE_DATE "\n"); +#endif return; case 'd': distribution_info(ERTS_PRINT_STDOUT, NULL); @@ -674,27 +673,39 @@ erl_crash_dump_v(char *file, int line, char* fmt, va_list args) char* dumpname; int secs; int env_erl_crash_dump_seconds_set = 1; + int i; if (ERTS_SOMEONE_IS_CRASH_DUMPING) return; #ifdef ERTS_SMP + /* Order all managed threads to block, this has to be done + first to guarantee that this is the only thread to generate + crash dump. */ + erts_thr_progress_fatal_error_block(&tpd_buf); + +#ifdef ERTS_THR_HAVE_SIG_FUNCS /* - * Wait for all managed threads to block. If all threads haven't blocked - * after a minute, we go anyway and hope for the best... - * - * We do not release system again. We expect an exit() or abort() after - * dump has been written. + * We suspend all scheduler threads so that we can dump some + * data about the currently running processes and scheduler data. + * We have to be very very careful when doing this as the schedulers + * could be anywhere. */ - erts_thr_progress_fatal_error_block(60000, &tpd_buf); - /* Either worked or not... */ + for (i = 0; i < erts_no_schedulers; i++) { + erts_tid_t tid = ERTS_SCHEDULER_IX(i)->tid; + if (!erts_equal_tids(tid,erts_thr_self())) + sys_thr_suspend(tid); + } + +#endif /* Allow us to pass certain places without locking... */ erts_smp_atomic32_set_mb(&erts_writing_erl_crash_dump, 1); erts_smp_tsd_set(erts_is_crash_dumping_key, (void *) 1); -#else + +#else /* !ERTS_SMP */ erts_writing_erl_crash_dump = 1; -#endif +#endif /* ERTS_SMP */ envsz = sizeof(env); /* ERL_CRASH_DUMP_SECONDS not set @@ -747,13 +758,14 @@ erl_crash_dump_v(char *file, int line, char* fmt, va_list args) dumpname = "erl_crash.dump"; else dumpname = &dumpnamebuf[0]; + + erts_fprintf(stderr,"\nCrash dump is being written to: %s...", dumpname); fd = open(dumpname,O_WRONLY | O_CREAT | O_TRUNC,0640); - if (fd < 0) + if (fd < 0) return; /* Can't create the crash dump, skip it */ - time(&now); - erts_fdprintf(fd, "=erl_crash_dump:0.1\n%s", ctime(&now)); + erts_fdprintf(fd, "=erl_crash_dump:0.3\n%s", ctime(&now)); if (file != NULL) erts_fdprintf(fd, "The error occurred in file %s, line %d\n", file, line); @@ -764,10 +776,77 @@ erl_crash_dump_v(char *file, int line, char* fmt, va_list args) } erts_fdprintf(fd, "System version: "); erts_print_system_version(fd, NULL, NULL); +#if ERTS_SAVED_COMPILE_TIME erts_fdprintf(fd, "%s\n", "Compiled: " ERLANG_COMPILE_DATE); +#endif + erts_fdprintf(fd, "Taints: "); erts_print_nif_taints(fd, NULL); erts_fdprintf(fd, "Atoms: %d\n", atom_table_size()); + +#ifdef USE_THREADS + /* We want to note which thread it was that called erts_exit */ + if (erts_get_scheduler_data()) { + erts_fdprintf(fd, "Calling Thread: scheduler:%d\n", + erts_get_scheduler_data()->no); + } else { + if (!erts_thr_getname(erts_thr_self(), dumpnamebuf, MAXPATHLEN)) + erts_fdprintf(fd, "Calling Thread: %s\n", dumpnamebuf); + else + erts_fdprintf(fd, "Calling Thread: %p\n", erts_thr_self()); + } +#else + erts_fdprintf(fd, "Calling Thread: scheduler:1\n"); +#endif + +#if defined(ERTS_HAVE_TRY_CATCH) + + /* + * erts_print_scheduler_info is not guaranteed to be safe to call + * here for all schedulers as we may have suspended a scheduler + * in the middle of updating the STACK_TOP and STACK_START + * variables and thus when scanning the stack we could get + * segmentation faults. We protect against this very unlikely + * scenario by using the ERTS_SYS_TRY_CATCH. + */ + for (i = 0; i < erts_no_schedulers; i++) { + ERTS_SYS_TRY_CATCH( + erts_print_scheduler_info(fd, NULL, ERTS_SCHEDULER_IX(i)), + erts_fdprintf(fd, "** crashed **\n")); + } +#endif + +#ifdef ERTS_SMP + +#if defined(ERTS_THR_HAVE_SIG_FUNCS) + + /* We resume all schedulers so that we are in a known safe state + when we write the rest of the crash dump */ + for (i = 0; i < erts_no_schedulers; i++) { + erts_tid_t tid = ERTS_SCHEDULER_IX(i)->tid; + if (!erts_equal_tids(tid,erts_thr_self())) + sys_thr_resume(tid); + } +#endif + + /* + * Wait for all managed threads to block. If all threads haven't blocked + * after a minute, we go anyway and hope for the best... + * + * We do not release system again. We expect an exit() or abort() after + * dump has been written. + */ + erts_thr_progress_fatal_error_wait(60000); + /* Either worked or not... */ +#endif + +#ifndef ERTS_HAVE_TRY_CATCH + /* This is safe to call here, as all schedulers are blocked */ + for (i = 0; i < erts_no_schedulers; i++) { + erts_print_scheduler_info(fd, NULL, ERTS_SCHEDULER_IX(i)); + } +#endif + info(fd, NULL); /* General system info */ if (erts_ptab_initialized(&erts_proc)) process_info(fd, NULL); /* Info about each process and port */ @@ -797,7 +876,7 @@ erl_crash_dump_v(char *file, int line, char* fmt, va_list args) erts_fdprintf(fd, "=end\n"); close(fd); - erts_fprintf(stderr,"\nCrash dump was written to: %s\n", dumpname); + erts_fprintf(stderr,"done\n"); } void diff --git a/erts/emulator/beam/code_ix.c b/erts/emulator/beam/code_ix.c index c66d5a2f05..209d008d18 100644 --- a/erts/emulator/beam/code_ix.c +++ b/erts/emulator/beam/code_ix.c @@ -3,16 +3,17 @@ * * Copyright Ericsson AB 2012. All Rights Reserved. * - * The contents of this file are subject to the Erlang Public License, - * Version 1.1, (the "License"); you may not use this file except in - * compliance with the License. You should have received a copy of the - * Erlang Public License along with this software. If not, it can be - * retrieved online at http://www.erlang.org/. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Software distributed under the License is distributed on an "AS IS" - * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See - * the License for the specific language governing rights and limitations - * under the License. + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. * * %CopyrightEnd% */ @@ -58,7 +59,8 @@ void erts_code_ix_init(void) erts_smp_atomic32_init_nob(&the_staging_code_index, 0); erts_smp_mtx_init(&code_write_permission_mtx, "code_write_permission"); #ifdef ERTS_ENABLE_LOCK_CHECK - erts_tsd_key_create(&has_code_write_permission); + erts_tsd_key_create(&has_code_write_permission, + "erts_has_code_write_permission"); #endif CIX_TRACE("init"); } @@ -129,7 +131,7 @@ int erts_try_seize_code_write_permission(Process* c_p) ASSERT(code_writing_process != c_p); qitem = erts_alloc(ERTS_ALC_T_CODE_IX_LOCK_Q, sizeof(*qitem)); qitem->p = c_p; - erts_smp_proc_inc_refc(c_p); + erts_proc_inc_refc(c_p); qitem->next = code_write_queue; code_write_queue = qitem; erts_suspend(c_p, ERTS_PROC_LOCK_MAIN, NULL); @@ -150,7 +152,7 @@ void erts_release_code_write_permission(void) } erts_smp_proc_unlock(qitem->p, ERTS_PROC_LOCK_STATUS); code_write_queue = qitem->next; - erts_smp_proc_dec_refc(qitem->p); + erts_proc_dec_refc(qitem->p); erts_free(ERTS_ALC_T_CODE_IX_LOCK_Q, qitem); } code_writing_process = NULL; diff --git a/erts/emulator/beam/code_ix.h b/erts/emulator/beam/code_ix.h index 16ad900228..5f00b409ef 100644 --- a/erts/emulator/beam/code_ix.h +++ b/erts/emulator/beam/code_ix.h @@ -3,16 +3,17 @@ * * Copyright Ericsson AB 2012-2013. All Rights Reserved. * - * The contents of this file are subject to the Erlang Public License, - * Version 1.1, (the "License"); you may not use this file except in - * compliance with the License. You should have received a copy of the - * Erlang Public License along with this software. If not, it can be - * retrieved online at http://www.erlang.org/. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Software distributed under the License is distributed on an "AS IS" - * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See - * the License for the specific language governing rights and limitations - * under the License. + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. * * %CopyrightEnd% */ diff --git a/erts/emulator/beam/copy.c b/erts/emulator/beam/copy.c index 23c0fca6aa..64be43edb4 100644 --- a/erts/emulator/beam/copy.c +++ b/erts/emulator/beam/copy.c @@ -3,16 +3,17 @@ * * Copyright Ericsson AB 1996-2012. All Rights Reserved. * - * The contents of this file are subject to the Erlang Public License, - * Version 1.1, (the "License"); you may not use this file except in - * compliance with the License. You should have received a copy of the - * Erlang Public License along with this software. If not, it can be - * retrieved online at http://www.erlang.org/. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Software distributed under the License is distributed on an "AS IS" - * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See - * the License for the specific language governing rights and limitations - * under the License. + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. * * %CopyrightEnd% */ @@ -21,17 +22,20 @@ # include "config.h" #endif +#define ERL_WANT_GC_INTERNALS__ + #include "sys.h" #include "erl_vm.h" #include "global.h" #include "erl_process.h" #include "erl_gc.h" #include "big.h" +#include "erl_map.h" #include "erl_binary.h" #include "erl_bits.h" #include "dtrace-wrapper.h" -static void move_one_frag(Eterm** hpp, Eterm* src, Uint src_sz, ErlOffHeap*); +static void move_one_frag(Eterm** hpp, ErlHeapFragment*, ErlOffHeap*); /* * Copy object "obj" to process p. @@ -47,14 +51,15 @@ copy_object(Eterm obj, Process* to) if (DTRACE_ENABLED(copy_object)) { DTRACE_CHARBUF(proc_name, 64); - erts_snprintf(proc_name, sizeof(proc_name), "%T", to->common.id); + erts_snprintf(proc_name, sizeof(DTRACE_CHARBUF_NAME(proc_name)), + "%T", to->common.id); DTRACE2(copy_object, proc_name, size); } #endif res = copy_struct(obj, size, &hp, &to->off_heap); #ifdef DEBUG if (eq(obj, res) == 0) { - erl_exit(ERTS_ABORT_EXIT, "copy not equal to source\n"); + erts_exit(ERTS_ABORT_EXIT, "copy not equal to source\n"); } #endif return res; @@ -123,6 +128,52 @@ Uint size_object(Eterm obj) obj = *bptr; break; } + case MAP_SUBTAG: + switch (MAP_HEADER_TYPE(hdr)) { + case MAP_HEADER_TAG_FLATMAP_HEAD : + { + Uint n; + flatmap_t *mp; + mp = (flatmap_t*)flatmap_val_rel(obj,base); + ptr = (Eterm *)mp; + n = flatmap_get_size(mp) + 1; + sum += n + 2; + ptr += 2; /* hdr + size words */ + while (n--) { + obj = *ptr++; + if (!IS_CONST(obj)) { + ESTACK_PUSH(s, obj); + } + } + goto pop_next; + } + case MAP_HEADER_TAG_HAMT_HEAD_BITMAP : + case MAP_HEADER_TAG_HAMT_HEAD_ARRAY : + case MAP_HEADER_TAG_HAMT_NODE_BITMAP : + { + Eterm *head; + Uint sz; + head = hashmap_val_rel(obj, base); + sz = hashmap_bitcount(MAP_HEADER_VAL(hdr)); + sum += 1 + sz + header_arity(hdr); + head += 1 + header_arity(hdr); + + if (sz == 0) { + goto pop_next; + } + while(sz-- > 1) { + obj = head[sz]; + if (!IS_CONST(obj)) { + ESTACK_PUSH(s, obj); + } + } + obj = head[0]; + } + break; + default: + erts_exit(ERTS_ABORT_EXIT, "size_object: bad hashmap type %d\n", MAP_HEADER_TYPE(hdr)); + } + break; case SUB_BINARY_SUBTAG: { Eterm real_bin; @@ -150,8 +201,8 @@ Uint size_object(Eterm obj) goto pop_next; } break; - case BIN_MATCHSTATE_SUBTAG: - erl_exit(ERTS_ABORT_EXIT, + case BIN_MATCHSTATE_SUBTAG: + erts_exit(ERTS_ABORT_EXIT, "size_object: matchstate term not allowed"); default: sum += thing_arityval(hdr) + 1; @@ -168,7 +219,7 @@ Uint size_object(Eterm obj) obj = ESTACK_POP(s); break; default: - erl_exit(ERTS_ABORT_EXIT, "size_object: bad tag for %#x\n", obj); + erts_exit(ERTS_ABORT_EXIT, "size_object: bad tag for %#x\n", obj); } } } @@ -221,7 +272,7 @@ Eterm copy_struct(Eterm obj, Uint sz, Eterm** hpp, ErlOffHeap* off_heap) goto L_copy_list; case TAG_PRIMARY_BOXED: argp = &res; goto L_copy_boxed; default: - erl_exit(ERTS_ABORT_EXIT, + erts_exit(ERTS_ABORT_EXIT, "%s, line %d: Internal error in copy_struct: 0x%08x\n", __FILE__, __LINE__,obj); } @@ -280,7 +331,7 @@ Eterm copy_struct(Eterm obj, Uint sz, Eterm** hpp, ErlOffHeap* off_heap) case TAG_PRIMARY_IMMED1: *tailp = obj; goto L_copy; case TAG_PRIMARY_BOXED: argp = tailp; goto L_copy_boxed; default: - erl_exit(ERTS_ABORT_EXIT, + erts_exit(ERTS_ABORT_EXIT, "%s, line %d: Internal error in copy_struct: 0x%08x\n", __FILE__, __LINE__,obj); } @@ -428,7 +479,7 @@ Eterm copy_struct(Eterm obj, Uint sz, Eterm** hpp, ErlOffHeap* off_heap) { ExternalThing *etp = (ExternalThing *) htop; - i = thing_arityval(hdr) + 1; + i = thing_arityval(hdr) + 1; tp = htop; while (i--) { @@ -442,8 +493,30 @@ Eterm copy_struct(Eterm obj, Uint sz, Eterm** hpp, ErlOffHeap* off_heap) *argp = make_external_rel(tp, dst_base); } break; + case MAP_SUBTAG: + tp = htop; + switch (MAP_HEADER_TYPE(hdr)) { + case MAP_HEADER_TAG_FLATMAP_HEAD : + i = flatmap_get_size(objp) + 3; + *argp = make_flatmap_rel(htop, dst_base); + while (i--) { + *htop++ = *objp++; + } + break; + case MAP_HEADER_TAG_HAMT_HEAD_BITMAP : + case MAP_HEADER_TAG_HAMT_HEAD_ARRAY : + *htop++ = *objp++; + case MAP_HEADER_TAG_HAMT_NODE_BITMAP : + i = 1 + hashmap_bitcount(MAP_HEADER_VAL(hdr)); + while (i--) { *htop++ = *objp++; } + *argp = make_hashmap_rel(tp, dst_base); + break; + default: + erts_exit(ERTS_ABORT_EXIT, "copy_struct: bad hashmap type %d\n", MAP_HEADER_TYPE(hdr)); + } + break; case BIN_MATCHSTATE_SUBTAG: - erl_exit(ERTS_ABORT_EXIT, + erts_exit(ERTS_ABORT_EXIT, "copy_struct: matchstate term not allowed"); default: i = thing_arityval(hdr)+1; @@ -467,13 +540,13 @@ Eterm copy_struct(Eterm obj, Uint sz, Eterm** hpp, ErlOffHeap* off_heap) #ifdef DEBUG if (htop != hbot) - erl_exit(ERTS_ABORT_EXIT, + erts_exit(ERTS_ABORT_EXIT, "Internal error in copy_struct() when copying %T:" " htop=%p != hbot=%p (sz=%beu)\n", org_obj, htop, hbot, org_sz); #else if (htop > hbot) { - erl_exit(ERTS_ABORT_EXIT, + erts_exit(ERTS_ABORT_EXIT, "Internal error in copy_struct(): htop, hbot overrun\n"); } #endif @@ -536,7 +609,6 @@ Eterm copy_shallow(Eterm* ptr, Uint sz, Eterm** hpp, ErlOffHeap* off_heap) erts_refc_inc(&funp->fe->refc, 2); } goto off_heap_common; - case EXTERNAL_PID_SUBTAG: case EXTERNAL_PORT_SUBTAG: case EXTERNAL_REF_SUBTAG: @@ -572,7 +644,6 @@ Eterm copy_shallow(Eterm* ptr, Uint sz, Eterm** hpp, ErlOffHeap* off_heap) } } *hpp = hp; - return res; } @@ -591,8 +662,7 @@ void move_multi_frags(Eterm** hpp, ErlOffHeap* off_heap, ErlHeapFragment* first, unsigned i; for (bp=first; bp!=NULL; bp=bp->next) { - move_one_frag(hpp, bp->mem, bp->used_size, off_heap); - OH_OVERHEAD(off_heap, bp->off_heap.overhead); + move_one_frag(hpp, bp, off_heap); } hp_end = *hpp; for (hp=hp_start; hp<hp_end; ++hp) { @@ -628,10 +698,10 @@ void move_multi_frags(Eterm** hpp, ErlOffHeap* off_heap, ErlHeapFragment* first, } static void -move_one_frag(Eterm** hpp, Eterm* src, Uint src_sz, ErlOffHeap* off_heap) +move_one_frag(Eterm** hpp, ErlHeapFragment* frag, ErlOffHeap* off_heap) { - Eterm* ptr = src; - Eterm* end = ptr + src_sz; + Eterm* ptr = frag->mem; + Eterm* end = ptr + frag->used_size; Eterm dummy_ref; Eterm* hp = *hpp; @@ -662,5 +732,7 @@ move_one_frag(Eterm** hpp, Eterm* src, Uint src_sz, ErlOffHeap* off_heap) } } *hpp = hp; + OH_OVERHEAD(off_heap, frag->off_heap.overhead); + frag->off_heap.first = NULL; } diff --git a/erts/emulator/beam/dist.c b/erts/emulator/beam/dist.c index 0781665f05..787241b960 100644 --- a/erts/emulator/beam/dist.c +++ b/erts/emulator/beam/dist.c @@ -1,18 +1,19 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 1996-2013. All Rights Reserved. + * Copyright Ericsson AB 1996-2014. All Rights Reserved. * - * The contents of this file are subject to the Erlang Public License, - * Version 1.1, (the "License"); you may not use this file except in - * compliance with the License. You should have received a copy of the - * Erlang Public License along with this software. If not, it can be - * retrieved online at http://www.erlang.org/. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Software distributed under the License is distributed on an "AS IS" - * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See - * the License for the specific language governing rights and limitations - * under the License. + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. * * %CopyrightEnd% */ @@ -44,6 +45,8 @@ #include "erl_thr_progress.h" #include "dtrace-wrapper.h" +#define DIST_CTL_DEFAULT_SIZE 64 + /* Turn this on to get printouts of all distribution messages * which go on the line */ @@ -65,9 +68,13 @@ static void bw(byte *buf, ErlDrvSizeT sz) static void dist_msg_dbg(ErtsDistExternal *edep, char *what, byte *buf, int sz) { + ErtsHeapFactory factory; + DeclareTmpHeapNoproc(ctl_default,DIST_CTL_DEFAULT_SIZE); + Eterm* ctl = ctl_default; byte *extp = edep->extp; Eterm msg; - Sint size = erts_decode_dist_ext_size(edep); + Sint ctl_len; + Sint size = ctl_len = erts_decode_dist_ext_size(edep); if (size < 0) { erts_fprintf(stderr, "DIST MSG DEBUG: erts_decode_dist_ext_size(%s) failed:\n", @@ -75,10 +82,9 @@ dist_msg_dbg(ErtsDistExternal *edep, char *what, byte *buf, int sz) bw(buf, sz); } else { - Eterm *hp; ErlHeapFragment *mbuf = new_message_buffer(size); - hp = mbuf->mem; - msg = erts_decode_dist_ext(&hp, &mbuf->off_heap, edep); + erts_factory_static_init(&factory, ctl, ctl_len, &mbuf->off_heap); + msg = erts_decode_dist_ext(&factory, edep); if (is_value(msg)) erts_fprintf(stderr, " %s: %T\n", what, msg); else { @@ -119,7 +125,7 @@ Export* dmonitor_p_trap = NULL; /* forward declarations */ static void clear_dist_entry(DistEntry*); -static int dsig_send(ErtsDSigData *, Eterm, Eterm, int); +static int dsig_send_ctl(ErtsDSigData* dsdp, Eterm ctl, int force_busy); static void send_nodes_mon_msgs(Process *, Eterm, Eterm, Eterm, Eterm); static void init_nodes_monitors(void); @@ -353,7 +359,7 @@ static void doit_link_net_exits_sub(ErtsLink *sublnk, void *vlnecp) static void doit_link_net_exits(ErtsLink *lnk, void *vnecp) { LinkNetExitsContext lnec = {(NetExitsContext *) vnecp, lnk}; - ASSERT(lnk->type == LINK_PID) + ASSERT(lnk->type == LINK_PID); erts_sweep_links(ERTS_LINK_ROOT(lnk), &doit_link_net_exits_sub, (void *) &lnec); #ifdef DEBUG ERTS_LINK_ROOT(lnk) = NULL; @@ -369,7 +375,7 @@ static void doit_node_link_net_exits(ErtsLink *lnk, void *vnecp) Process *rp; ErtsLink *rlnk; Uint i,n; - ASSERT(lnk->type == LINK_NODE) + ASSERT(lnk->type == LINK_NODE); if (is_internal_pid(lnk->pid)) { ErtsProcLocks rp_locks = ERTS_PROC_LOCK_LINK; rp = erts_pid2proc(NULL, 0, lnk->pid, rp_locks); @@ -388,11 +394,7 @@ static void doit_node_link_net_exits(ErtsLink *lnk, void *vnecp) Eterm tup; Eterm *hp = erts_alloc_message_heap(3,&bp,&ohp,rp,&rp_locks); tup = TUPLE2(hp, am_nodedown, name); - erts_queue_message(rp, &rp_locks, bp, tup, NIL -#ifdef USE_VM_PROBES - , NIL -#endif - ); + erts_queue_message(rp, &rp_locks, bp, tup, NIL); } erts_smp_proc_unlock(rp, rp_locks); } @@ -455,7 +457,7 @@ int erts_do_net_exits(DistEntry *dep, Eterm reason) Eterm nd_reason = (reason == am_no_network ? am_no_network : am_net_kernel_terminated); - erts_smp_rwmtx_rwlock(&erts_dist_table_rwmtx); + erts_smp_rwmtx_rlock(&erts_dist_table_rwmtx); for (tdep = erts_hidden_dist_entries; tdep; tdep = tdep->next) no_dist_port++; @@ -464,7 +466,7 @@ int erts_do_net_exits(DistEntry *dep, Eterm reason) /* KILL all port controllers */ if (no_dist_port == 0) - erts_smp_rwmtx_rwunlock(&erts_dist_table_rwmtx); + erts_smp_rwmtx_runlock(&erts_dist_table_rwmtx); else { Eterm def_buf[128]; int i = 0; @@ -483,7 +485,7 @@ int erts_do_net_exits(DistEntry *dep, Eterm reason) ASSERT(is_internal_port(tdep->cid)); dist_port[i++] = tdep->cid; } - erts_smp_rwmtx_rwunlock(&erts_dist_table_rwmtx); + erts_smp_rwmtx_runlock(&erts_dist_table_rwmtx); for (i = 0; i < no_dist_port; i++) { Port *prt = erts_port_lookup(dist_port[i], @@ -622,9 +624,7 @@ alloc_dist_obuf(Uint size) ErtsDistOutputBuf *obuf; Uint obuf_size = sizeof(ErtsDistOutputBuf)+sizeof(byte)*(size-1); Binary *bin = erts_bin_drv_alloc(obuf_size); - bin->flags = BIN_FLAG_DRV; erts_refc_init(&bin->refc, 1); - bin->orig_size = (SWord) obuf_size; obuf = (ErtsDistOutputBuf *) &bin->orig_bytes[0]; #ifdef DEBUG obuf->dbg_pattern = ERTS_DIST_OUTPUT_BUF_DBG_PATTERN; @@ -709,6 +709,55 @@ static void clear_dist_entry(DistEntry *dep) } } +void erts_dsend_context_dtor(Binary* ctx_bin) +{ + ErtsSendContext* ctx = ERTS_MAGIC_BIN_DATA(ctx_bin); + switch (ctx->dss.phase) { + case ERTS_DSIG_SEND_PHASE_MSG_SIZE: + DESTROY_SAVED_WSTACK(&ctx->dss.u.sc.wstack); + break; + case ERTS_DSIG_SEND_PHASE_MSG_ENCODE: + DESTROY_SAVED_WSTACK(&ctx->dss.u.ec.wstack); + break; + default:; + } + if (ctx->dss.phase >= ERTS_DSIG_SEND_PHASE_ALLOC && ctx->dss.obuf) { + free_dist_obuf(ctx->dss.obuf); + } + if (ctx->dep_to_deref) + erts_deref_dist_entry(ctx->dep_to_deref); +} + +Eterm erts_dsend_export_trap_context(Process* p, ErtsSendContext* ctx) +{ + struct exported_ctx { + ErtsSendContext ctx; + ErtsAtomCacheMap acm; + }; + Binary* ctx_bin = erts_create_magic_binary(sizeof(struct exported_ctx), + erts_dsend_context_dtor); + struct exported_ctx* dst = ERTS_MAGIC_BIN_DATA(ctx_bin); + Uint ctl_size = !HALFWORD_HEAP ? 0 : (arityval(ctx->ctl_heap[0]) + 1); + Eterm* hp = HAlloc(p, ctl_size + PROC_BIN_SIZE); + + sys_memcpy(&dst->ctx, ctx, sizeof(ErtsSendContext)); + ASSERT(ctx->dss.ctl == make_tuple(ctx->ctl_heap)); +#if !HALFWORD_HEAP + dst->ctx.dss.ctl = make_tuple(dst->ctx.ctl_heap); +#else + /* Must put control tuple in low mem */ + sys_memcpy(hp, ctx->ctl_heap, ctl_size*sizeof(Eterm)); + dst->ctx.dss.ctl = make_tuple(hp); + hp += ctl_size; +#endif + if (ctx->dss.acmp) { + sys_memcpy(&dst->acm, ctx->dss.acmp, sizeof(ErtsAtomCacheMap)); + dst->ctx.dss.acmp = &dst->acm; + } + return erts_mk_magic_binary_term(&hp, &MSO(p), ctx_bin); +} + + /* * The erts_dsig_send_*() functions implemented below, sends asynchronous * distributed signals to other Erlang nodes. Before sending a distributed @@ -731,7 +780,7 @@ erts_dsig_send_link(ErtsDSigData *dsdp, Eterm local, Eterm remote) int res; UseTmpHeapNoproc(4); - res = dsig_send(dsdp, ctl, THE_NON_VALUE, 0); + res = dsig_send_ctl(dsdp, ctl, 0); UnUseTmpHeapNoproc(4); return res; } @@ -744,7 +793,7 @@ erts_dsig_send_unlink(ErtsDSigData *dsdp, Eterm local, Eterm remote) int res; UseTmpHeapNoproc(4); - res = dsig_send(dsdp, ctl, THE_NON_VALUE, 0); + res = dsig_send_ctl(dsdp, ctl, 0); UnUseTmpHeapNoproc(4); return res; } @@ -772,7 +821,7 @@ erts_dsig_send_m_exit(ErtsDSigData *dsdp, Eterm watcher, Eterm watched, erts_smp_de_links_unlock(dsdp->dep); #endif - res = dsig_send(dsdp, ctl, THE_NON_VALUE, 1); + res = dsig_send_ctl(dsdp, ctl, 1); UnUseTmpHeapNoproc(6); return res; } @@ -793,7 +842,7 @@ erts_dsig_send_monitor(ErtsDSigData *dsdp, Eterm watcher, Eterm watched, make_small(DOP_MONITOR_P), watcher, watched, ref); - res = dsig_send(dsdp, ctl, THE_NON_VALUE, 0); + res = dsig_send_ctl(dsdp, ctl, 0); UnUseTmpHeapNoproc(5); return res; } @@ -815,18 +864,17 @@ erts_dsig_send_demonitor(ErtsDSigData *dsdp, Eterm watcher, make_small(DOP_DEMONITOR_P), watcher, watched, ref); - res = dsig_send(dsdp, ctl, THE_NON_VALUE, force); + res = dsig_send_ctl(dsdp, ctl, force); UnUseTmpHeapNoproc(5); return res; } int -erts_dsig_send_msg(ErtsDSigData *dsdp, Eterm remote, Eterm message) +erts_dsig_send_msg(Eterm remote, Eterm message, ErtsSendContext* ctx) { Eterm ctl; - DeclareTmpHeapNoproc(ctl_heap,5); Eterm token = NIL; - Process *sender = dsdp->proc; + Process *sender = ctx->dsd.proc; int res; #ifdef USE_VM_PROBES Sint tok_label = 0; @@ -838,8 +886,7 @@ erts_dsig_send_msg(ErtsDSigData *dsdp, Eterm remote, Eterm message) DTRACE_CHARBUF(receiver_name, 64); #endif - UseTmpHeapNoproc(5); - if (SEQ_TRACE_TOKEN(sender) != NIL + if (SEQ_TRACE_TOKEN(sender) != NIL #ifdef USE_VM_PROBES && SEQ_TRACE_TOKEN(sender) != am_have_dt_utag #endif @@ -851,9 +898,12 @@ erts_dsig_send_msg(ErtsDSigData *dsdp, Eterm remote, Eterm message) #ifdef USE_VM_PROBES *node_name = *sender_name = *receiver_name = '\0'; if (DTRACE_ENABLED(message_send) || DTRACE_ENABLED(message_send_remote)) { - erts_snprintf(node_name, sizeof(node_name), "%T", dsdp->dep->sysname); - erts_snprintf(sender_name, sizeof(sender_name), "%T", sender->common.id); - erts_snprintf(receiver_name, sizeof(receiver_name), "%T", remote); + erts_snprintf(node_name, sizeof(DTRACE_CHARBUF_NAME(node_name)), + "%T", ctx->dsd.dep->sysname); + erts_snprintf(sender_name, sizeof(DTRACE_CHARBUF_NAME(sender_name)), + "%T", sender->common.id); + erts_snprintf(receiver_name, sizeof(DTRACE_CHARBUF_NAME(receiver_name)), + "%T", remote); msize = size_object(message); if (token != NIL && token != am_have_dt_utag) { tok_label = signed_val(SEQ_TRACE_T_LABEL(token)); @@ -864,26 +914,28 @@ erts_dsig_send_msg(ErtsDSigData *dsdp, Eterm remote, Eterm message) #endif if (token != NIL) - ctl = TUPLE4(&ctl_heap[0], + ctl = TUPLE4(&ctx->ctl_heap[0], make_small(DOP_SEND_TT), am_Cookie, remote, token); else - ctl = TUPLE3(&ctl_heap[0], make_small(DOP_SEND), am_Cookie, remote); + ctl = TUPLE3(&ctx->ctl_heap[0], make_small(DOP_SEND), am_Cookie, remote); DTRACE6(message_send, sender_name, receiver_name, msize, tok_label, tok_lastcnt, tok_serial); DTRACE7(message_send_remote, sender_name, node_name, receiver_name, msize, tok_label, tok_lastcnt, tok_serial); - res = dsig_send(dsdp, ctl, message, 0); - UnUseTmpHeapNoproc(5); + ctx->dss.ctl = ctl; + ctx->dss.msg = message; + ctx->dss.force_busy = 0; + res = erts_dsig_send(&ctx->dsd, &ctx->dss); return res; } int -erts_dsig_send_reg_msg(ErtsDSigData *dsdp, Eterm remote_name, Eterm message) +erts_dsig_send_reg_msg(Eterm remote_name, Eterm message, + ErtsSendContext* ctx) { Eterm ctl; - DeclareTmpHeapNoproc(ctl_heap,6); Eterm token = NIL; - Process *sender = dsdp->proc; + Process *sender = ctx->dsd.proc; int res; #ifdef USE_VM_PROBES Sint tok_label = 0; @@ -895,7 +947,6 @@ erts_dsig_send_reg_msg(ErtsDSigData *dsdp, Eterm remote_name, Eterm message) DTRACE_CHARBUF(receiver_name, 128); #endif - UseTmpHeapNoproc(6); if (SEQ_TRACE_TOKEN(sender) != NIL #ifdef USE_VM_PROBES && SEQ_TRACE_TOKEN(sender) != am_have_dt_utag @@ -908,9 +959,11 @@ erts_dsig_send_reg_msg(ErtsDSigData *dsdp, Eterm remote_name, Eterm message) #ifdef USE_VM_PROBES *node_name = *sender_name = *receiver_name = '\0'; if (DTRACE_ENABLED(message_send) || DTRACE_ENABLED(message_send_remote)) { - erts_snprintf(node_name, sizeof(node_name), "%T", dsdp->dep->sysname); - erts_snprintf(sender_name, sizeof(sender_name), "%T", sender->common.id); - erts_snprintf(receiver_name, sizeof(receiver_name), + erts_snprintf(node_name, sizeof(DTRACE_CHARBUF_NAME(node_name)), + "%T", ctx->dsd.dep->sysname); + erts_snprintf(sender_name, sizeof(DTRACE_CHARBUF_NAME(sender_name)), + "%T", sender->common.id); + erts_snprintf(receiver_name, sizeof(DTRACE_CHARBUF_NAME(receiver_name)), "{%T,%s}", remote_name, node_name); msize = size_object(message); if (token != NIL && token != am_have_dt_utag) { @@ -922,17 +975,19 @@ erts_dsig_send_reg_msg(ErtsDSigData *dsdp, Eterm remote_name, Eterm message) #endif if (token != NIL) - ctl = TUPLE5(&ctl_heap[0], make_small(DOP_REG_SEND_TT), + ctl = TUPLE5(&ctx->ctl_heap[0], make_small(DOP_REG_SEND_TT), sender->common.id, am_Cookie, remote_name, token); else - ctl = TUPLE4(&ctl_heap[0], make_small(DOP_REG_SEND), + ctl = TUPLE4(&ctx->ctl_heap[0], make_small(DOP_REG_SEND), sender->common.id, am_Cookie, remote_name); DTRACE6(message_send, sender_name, receiver_name, msize, tok_label, tok_lastcnt, tok_serial); DTRACE7(message_send_remote, sender_name, node_name, receiver_name, msize, tok_label, tok_lastcnt, tok_serial); - res = dsig_send(dsdp, ctl, message, 0); - UnUseTmpHeapNoproc(6); + ctx->dss.ctl = ctl; + ctx->dss.msg = message; + ctx->dss.force_busy = 0; + res = erts_dsig_send(&ctx->dsd, &ctx->dss); return res; } @@ -971,11 +1026,14 @@ erts_dsig_send_exit_tt(ErtsDSigData *dsdp, Eterm local, Eterm remote, #ifdef USE_VM_PROBES *node_name = *sender_name = *remote_name = '\0'; if (DTRACE_ENABLED(process_exit_signal_remote)) { - erts_snprintf(node_name, sizeof(node_name), "%T", dsdp->dep->sysname); - erts_snprintf(sender_name, sizeof(sender_name), "%T", sender->common.id); - erts_snprintf(remote_name, sizeof(remote_name), + erts_snprintf(node_name, sizeof(DTRACE_CHARBUF_NAME(node_name)), + "%T", dsdp->dep->sysname); + erts_snprintf(sender_name, sizeof(DTRACE_CHARBUF_NAME(sender_name)), + "%T", sender->common.id); + erts_snprintf(remote_name, sizeof(DTRACE_CHARBUF_NAME(remote_name)), "{%T,%s}", remote, node_name); - erts_snprintf(reason_str, sizeof(reason), "%T", reason); + erts_snprintf(reason_str, sizeof(DTRACE_CHARBUF_NAME(reason_str)), + "%T", reason); if (token != NIL && token != am_have_dt_utag) { tok_label = signed_val(SEQ_TRACE_T_LABEL(token)); tok_lastcnt = signed_val(SEQ_TRACE_T_LASTCNT(token)); @@ -986,7 +1044,7 @@ erts_dsig_send_exit_tt(ErtsDSigData *dsdp, Eterm local, Eterm remote, DTRACE7(process_exit_signal_remote, sender_name, node_name, remote_name, reason_str, tok_label, tok_lastcnt, tok_serial); /* forced, i.e ignore busy */ - res = dsig_send(dsdp, ctl, THE_NON_VALUE, 1); + res = dsig_send_ctl(dsdp, ctl, 1); UnUseTmpHeapNoproc(6); return res; } @@ -1002,7 +1060,7 @@ erts_dsig_send_exit(ErtsDSigData *dsdp, Eterm local, Eterm remote, Eterm reason) ctl = TUPLE4(&ctl_heap[0], make_small(DOP_EXIT), local, remote, reason); /* forced, i.e ignore busy */ - res = dsig_send(dsdp, ctl, THE_NON_VALUE, 1); + res = dsig_send_ctl(dsdp, ctl, 1); UnUseTmpHeapNoproc(5); return res; } @@ -1018,7 +1076,7 @@ erts_dsig_send_exit2(ErtsDSigData *dsdp, Eterm local, Eterm remote, Eterm reason ctl = TUPLE4(&ctl_heap[0], make_small(DOP_EXIT2), local, remote, reason); - res = dsig_send(dsdp, ctl, THE_NON_VALUE, 0); + res = dsig_send_ctl(dsdp, ctl, 0); UnUseTmpHeapNoproc(5); return res; } @@ -1035,7 +1093,7 @@ erts_dsig_send_group_leader(ErtsDSigData *dsdp, Eterm leader, Eterm remote) ctl = TUPLE3(&ctl_heap[0], make_small(DOP_GROUP_LEADER), leader, remote); - res = dsig_send(dsdp, ctl, THE_NON_VALUE, 0); + res = dsig_send_ctl(dsdp, ctl, 0); UnUseTmpHeapNoproc(4); return res; } @@ -1083,7 +1141,6 @@ int erts_net_message(Port *prt, byte *buf, ErlDrvSizeT len) { -#define DIST_CTL_DEFAULT_SIZE 64 ErtsDistExternal ede; byte *t; Sint ctl_len; @@ -1096,7 +1153,7 @@ int erts_net_message(Port *prt, Process* rp; DeclareTmpHeapNoproc(ctl_default,DIST_CTL_DEFAULT_SIZE); Eterm* ctl = ctl_default; - ErlOffHeap off_heap; + ErtsHeapFactory factory; Eterm* hp; Sint type; Eterm token; @@ -1110,9 +1167,6 @@ int erts_net_message(Port *prt, #endif UseTmpHeapNoproc(DIST_CTL_DEFAULT_SIZE); - /* Thanks to Luke Gorrie */ - off_heap.first = NULL; - off_heap.overhead = 0; ERTS_SMP_CHK_NO_PROC_LOCKS; @@ -1173,14 +1227,15 @@ int erts_net_message(Port *prt, } hp = ctl; - arg = erts_decode_dist_ext(&hp, &off_heap, &ede); + erts_factory_tmp_init(&factory, ctl, ctl_len, ERTS_ALC_T_DCTRL_BUF); + arg = erts_decode_dist_ext(&factory, &ede); if (is_non_value(arg)) { #ifdef ERTS_DIST_MSG_DBG - erts_fprintf(stderr, "DIST MSG DEBUG: erts_dist_ext_size(CTL) failed:\n"); + erts_fprintf(stderr, "DIST MSG DEBUG: erts_decode_dist_ext(CTL) failed:\n"); bw(buf, orig_len); #endif PURIFY_MSG("data error"); - goto data_error; + goto decode_error; } ctl_len = t - buf; @@ -1509,12 +1564,12 @@ int erts_net_message(Port *prt, break; } rp = erts_pid2proc(NULL, 0, mon->pid, rp_locks); + + erts_destroy_monitor(mon); if (rp == NULL) { break; } - erts_destroy_monitor(mon); - mon = erts_remove_monitor(&ERTS_P_MONITORS(rp), ref); if (mon == NULL) { @@ -1660,7 +1715,7 @@ int erts_net_message(Port *prt, goto invalid_message; } - erts_cleanup_offheap(&off_heap); + erts_factory_close(&factory); if (ctl != ctl_default) { erts_free(ERTS_ALC_T_DCTRL_BUF, (void *) ctl); } @@ -1673,203 +1728,248 @@ int erts_net_message(Port *prt, erts_dsprintf(dsbufp, "Invalid distribution message: %.200T", arg); erts_send_error_to_logger_nogl(dsbufp); } - data_error: +decode_error: PURIFY_MSG("data error"); - erts_cleanup_offheap(&off_heap); + erts_factory_close(&factory); if (ctl != ctl_default) { erts_free(ERTS_ALC_T_DCTRL_BUF, (void *) ctl); } +data_error: UnUseTmpHeapNoproc(DIST_CTL_DEFAULT_SIZE); erts_deliver_port_exit(prt, dep->cid, am_killed, 0); ERTS_SMP_CHK_NO_PROC_LOCKS; return -1; } -static int -dsig_send(ErtsDSigData *dsdp, Eterm ctl, Eterm msg, int force_busy) +static int dsig_send_ctl(ErtsDSigData* dsdp, Eterm ctl, int force_busy) +{ + struct erts_dsig_send_context ctx; + int ret; + ctx.ctl = ctl; + ctx.msg = THE_NON_VALUE; + ctx.force_busy = force_busy; + ctx.phase = ERTS_DSIG_SEND_PHASE_INIT; +#ifdef DEBUG + ctx.reds = 1; /* provoke assert below (no reduction count without msg) */ +#endif + ret = erts_dsig_send(dsdp, &ctx); + ASSERT(ret != ERTS_DSIG_SEND_CONTINUE); + return ret; +} + +int +erts_dsig_send(ErtsDSigData *dsdp, struct erts_dsig_send_context* ctx) { + int retval; + Sint initial_reds = ctx->reds; Eterm cid; - int suspended = 0; - int resume = 0; - Uint32 pass_through_size; - Uint data_size, dhdr_ext_size; - ErtsAtomCacheMap *acmp; - ErtsDistOutputBuf *obuf; - DistEntry *dep = dsdp->dep; - Uint32 flags = dep->flags; - Process *c_p = dsdp->proc; - if (!c_p || dsdp->no_suspend) - force_busy = 1; + while (1) { + switch (ctx->phase) { + case ERTS_DSIG_SEND_PHASE_INIT: + ctx->flags = dsdp->dep->flags; + ctx->c_p = dsdp->proc; - ERTS_SMP_LC_ASSERT(!c_p - || (ERTS_PROC_LOCK_MAIN - == erts_proc_lc_my_proc_locks(c_p))); + if (!ctx->c_p || dsdp->no_suspend) + ctx->force_busy = 1; - if (!erts_is_alive) - return ERTS_DSIG_SEND_OK; + ERTS_SMP_LC_ASSERT(!ctx->c_p + || (ERTS_PROC_LOCK_MAIN + == erts_proc_lc_my_proc_locks(ctx->c_p))); - if (flags & DFLAG_DIST_HDR_ATOM_CACHE) { - acmp = erts_get_atom_cache_map(c_p); - pass_through_size = 0; - } - else { - acmp = NULL; - pass_through_size = 1; - } + if (!erts_is_alive) + return ERTS_DSIG_SEND_OK; -#ifdef ERTS_DIST_MSG_DBG - erts_fprintf(stderr, ">>%s CTL: %T\n", pass_through_size ? "P" : " ", ctl); - if (is_value(msg)) - erts_fprintf(stderr, " MSG: %T\n", msg); -#endif + if (ctx->flags & DFLAG_DIST_HDR_ATOM_CACHE) { + ctx->acmp = erts_get_atom_cache_map(ctx->c_p); + ctx->pass_through_size = 0; + } + else { + ctx->acmp = NULL; + ctx->pass_through_size = 1; + } - data_size = pass_through_size; - erts_reset_atom_cache_map(acmp); - data_size += erts_encode_dist_ext_size(ctl, flags, acmp); - if (is_value(msg)) - data_size += erts_encode_dist_ext_size(msg, flags, acmp); - erts_finalize_atom_cache_map(acmp, flags); + #ifdef ERTS_DIST_MSG_DBG + erts_fprintf(stderr, ">>%s CTL: %T\n", ctx->pass_through_size ? "P" : " ", ctx->ctl); + if (is_value(ctx->msg)) + erts_fprintf(stderr, " MSG: %T\n", ctx->msg); + #endif + + ctx->data_size = ctx->pass_through_size; + erts_reset_atom_cache_map(ctx->acmp); + erts_encode_dist_ext_size(ctx->ctl, ctx->flags, ctx->acmp, &ctx->data_size); + + if (is_value(ctx->msg)) { + ctx->u.sc.wstack.wstart = NULL; + ctx->u.sc.flags = ctx->flags; + ctx->u.sc.level = 0; + ctx->phase = ERTS_DSIG_SEND_PHASE_MSG_SIZE; + } else { + ctx->phase = ERTS_DSIG_SEND_PHASE_ALLOC; + } + break; - dhdr_ext_size = erts_encode_ext_dist_header_size(acmp); - data_size += dhdr_ext_size; + case ERTS_DSIG_SEND_PHASE_MSG_SIZE: + if (erts_encode_dist_ext_size_int(ctx->msg, ctx, &ctx->data_size)) { + retval = ERTS_DSIG_SEND_CONTINUE; + goto done; + } - obuf = alloc_dist_obuf(data_size); - obuf->ext_endp = &obuf->data[0] + pass_through_size + dhdr_ext_size; + ctx->phase = ERTS_DSIG_SEND_PHASE_ALLOC; + case ERTS_DSIG_SEND_PHASE_ALLOC: + erts_finalize_atom_cache_map(ctx->acmp, ctx->flags); + + ctx->dhdr_ext_size = erts_encode_ext_dist_header_size(ctx->acmp); + ctx->data_size += ctx->dhdr_ext_size; + + ctx->obuf = alloc_dist_obuf(ctx->data_size); + ctx->obuf->ext_endp = &ctx->obuf->data[0] + ctx->pass_through_size + ctx->dhdr_ext_size; + + /* Encode internal version of dist header */ + ctx->obuf->extp = erts_encode_ext_dist_header_setup(ctx->obuf->ext_endp, ctx->acmp); + /* Encode control message */ + erts_encode_dist_ext(ctx->ctl, &ctx->obuf->ext_endp, ctx->flags, ctx->acmp, NULL, NULL); + if (is_value(ctx->msg)) { + ctx->u.ec.flags = ctx->flags; + ctx->u.ec.level = 0; + ctx->u.ec.wstack.wstart = NULL; + ctx->phase = ERTS_DSIG_SEND_PHASE_MSG_ENCODE; + } else { + ctx->phase = ERTS_DSIG_SEND_PHASE_FIN; + } + break; - /* Encode internal version of dist header */ - obuf->extp = erts_encode_ext_dist_header_setup(obuf->ext_endp, acmp); - /* Encode control message */ - erts_encode_dist_ext(ctl, &obuf->ext_endp, flags, acmp); - if (is_value(msg)) { - /* Encode message */ - erts_encode_dist_ext(msg, &obuf->ext_endp, flags, acmp); - } + case ERTS_DSIG_SEND_PHASE_MSG_ENCODE: + if (erts_encode_dist_ext(ctx->msg, &ctx->obuf->ext_endp, ctx->flags, ctx->acmp, &ctx->u.ec, &ctx->reds)) { + retval = ERTS_DSIG_SEND_CONTINUE; + goto done; + } - ASSERT(obuf->extp < obuf->ext_endp); - ASSERT(&obuf->data[0] <= obuf->extp - pass_through_size); - ASSERT(obuf->ext_endp <= &obuf->data[0] + data_size); + ctx->phase = ERTS_DSIG_SEND_PHASE_FIN; + case ERTS_DSIG_SEND_PHASE_FIN: { + DistEntry *dep = dsdp->dep; + int suspended = 0; + int resume = 0; - data_size = obuf->ext_endp - obuf->extp; + ASSERT(ctx->obuf->extp < ctx->obuf->ext_endp); + ASSERT(&ctx->obuf->data[0] <= ctx->obuf->extp - ctx->pass_through_size); + ASSERT(ctx->obuf->ext_endp <= &ctx->obuf->data[0] + ctx->data_size); - /* - * Signal encoded; now verify that the connection still exists, - * and if so enqueue the signal and schedule it for send. - */ - obuf->next = NULL; - erts_smp_de_rlock(dep); - cid = dep->cid; - if (cid != dsdp->cid - || dep->connection_id != dsdp->connection_id - || dep->status & ERTS_DE_SFLG_EXITING) { - /* Not the same connection as when we started; drop message... */ - erts_smp_de_runlock(dep); - free_dist_obuf(obuf); - } - else { - ErtsProcList *plp = NULL; - erts_smp_mtx_lock(&dep->qlock); - dep->qsize += size_obuf(obuf); - if (dep->qsize >= erts_dist_buf_busy_limit) - dep->qflgs |= ERTS_DE_QFLG_BUSY; - if (!force_busy && (dep->qflgs & ERTS_DE_QFLG_BUSY)) { - erts_smp_mtx_unlock(&dep->qlock); + ctx->data_size = ctx->obuf->ext_endp - ctx->obuf->extp; - plp = erts_proclist_create(c_p); - erts_suspend(c_p, ERTS_PROC_LOCK_MAIN, NULL); - suspended = 1; - erts_smp_mtx_lock(&dep->qlock); - } + /* + * Signal encoded; now verify that the connection still exists, + * and if so enqueue the signal and schedule it for send. + */ + ctx->obuf->next = NULL; + erts_smp_de_rlock(dep); + cid = dep->cid; + if (cid != dsdp->cid + || dep->connection_id != dsdp->connection_id + || dep->status & ERTS_DE_SFLG_EXITING) { + /* Not the same connection as when we started; drop message... */ + erts_smp_de_runlock(dep); + free_dist_obuf(ctx->obuf); + } + else { + ErtsProcList *plp = NULL; + erts_smp_mtx_lock(&dep->qlock); + dep->qsize += size_obuf(ctx->obuf); + if (dep->qsize >= erts_dist_buf_busy_limit) + dep->qflgs |= ERTS_DE_QFLG_BUSY; + if (!ctx->force_busy && (dep->qflgs & ERTS_DE_QFLG_BUSY)) { + erts_smp_mtx_unlock(&dep->qlock); + + plp = erts_proclist_create(ctx->c_p); + erts_suspend(ctx->c_p, ERTS_PROC_LOCK_MAIN, NULL); + suspended = 1; + erts_smp_mtx_lock(&dep->qlock); + } - /* Enqueue obuf on dist entry */ - if (dep->out_queue.last) - dep->out_queue.last->next = obuf; - else - dep->out_queue.first = obuf; - dep->out_queue.last = obuf; + /* Enqueue obuf on dist entry */ + if (dep->out_queue.last) + dep->out_queue.last->next = ctx->obuf; + else + dep->out_queue.first = ctx->obuf; + dep->out_queue.last = ctx->obuf; + + if (!ctx->force_busy) { + if (!(dep->qflgs & ERTS_DE_QFLG_BUSY)) { + if (suspended) + resume = 1; /* was busy when we started, but isn't now */ + #ifdef USE_VM_PROBES + if (resume && DTRACE_ENABLED(dist_port_not_busy)) { + DTRACE_CHARBUF(port_str, 64); + DTRACE_CHARBUF(remote_str, 64); + + erts_snprintf(port_str, sizeof(DTRACE_CHARBUF_NAME(port_str)), + "%T", cid); + erts_snprintf(remote_str, sizeof(DTRACE_CHARBUF_NAME(remote_str)), + "%T", dep->sysname); + DTRACE3(dist_port_not_busy, erts_this_node_sysname, + port_str, remote_str); + } + #endif + } + else { + /* Enqueue suspended process on dist entry */ + ASSERT(plp); + erts_proclist_store_last(&dep->suspended, plp); + } + } - if (!force_busy) { - if (!(dep->qflgs & ERTS_DE_QFLG_BUSY)) { - if (suspended) - resume = 1; /* was busy when we started, but isn't now */ -#ifdef USE_VM_PROBES - if (resume && DTRACE_ENABLED(dist_port_not_busy)) { - DTRACE_CHARBUF(port_str, 64); - DTRACE_CHARBUF(remote_str, 64); - - erts_snprintf(port_str, sizeof(port_str), "%T", cid); - erts_snprintf(remote_str, sizeof(remote_str), - "%T", dep->sysname); - DTRACE3(dist_port_not_busy, erts_this_node_sysname, - port_str, remote_str); - } -#endif + erts_smp_mtx_unlock(&dep->qlock); + erts_schedule_dist_command(NULL, dep); + erts_smp_de_runlock(dep); + + if (resume) { + erts_resume(ctx->c_p, ERTS_PROC_LOCK_MAIN); + erts_proclist_destroy(plp); + /* + * Note that the calling process still have to yield as if it + * suspended. If not, the calling process could later be + * erroneously scheduled when it shouldn't be. + */ + } } - else { - /* Enqueue suspended process on dist entry */ - ASSERT(plp); - erts_proclist_store_last(&dep->suspended, plp); + ctx->obuf = NULL; + + if (suspended) { + #ifdef USE_VM_PROBES + if (!resume && DTRACE_ENABLED(dist_port_busy)) { + DTRACE_CHARBUF(port_str, 64); + DTRACE_CHARBUF(remote_str, 64); + DTRACE_CHARBUF(pid_str, 16); + + erts_snprintf(port_str, sizeof(DTRACE_CHARBUF_NAME(port_str)), "%T", cid); + erts_snprintf(remote_str, sizeof(DTRACE_CHARBUF_NAME(remote_str)), + "%T", dep->sysname); + erts_snprintf(pid_str, sizeof(DTRACE_CHARBUF_NAME(pid_str)), + "%T", ctx->c_p->common.id); + DTRACE4(dist_port_busy, erts_this_node_sysname, + port_str, remote_str, pid_str); + } + #endif + if (!resume && erts_system_monitor_flags.busy_dist_port) + monitor_generic(ctx->c_p, am_busy_dist_port, cid); + retval = ERTS_DSIG_SEND_YIELD; + } else { + retval = ERTS_DSIG_SEND_OK; } + goto done; } - - erts_smp_mtx_unlock(&dep->qlock); - erts_schedule_dist_command(NULL, dep); - erts_smp_de_runlock(dep); - - if (resume) { - erts_resume(c_p, ERTS_PROC_LOCK_MAIN); - erts_proclist_destroy(plp); - /* - * Note that the calling process still have to yield as if it - * suspended. If not, the calling process could later be - * erroneously scheduled when it shouldn't be. - */ + default: + erts_exit(ERTS_ABORT_EXIT, "dsig_send invalid phase (%d)\n", (int)ctx->phase); } } - if (c_p) { - int reds; - /* - * Bump reductions on calling process. - * - * This is the reduction cost: Always a base cost of 8 reductions - * plus 16 reductions per kilobyte generated external data. - */ - - data_size >>= (10-4); -#if defined(ARCH_64) && !HALFWORD_HEAP - data_size &= 0x003fffffffffffff; -#elif defined(ARCH_32) || HALFWORD_HEAP - data_size &= 0x003fffff; -#else -# error "Ohh come on ... !?!" -#endif - reds = 8 + ((int) data_size > 1000000 ? 1000000 : (int) data_size); - BUMP_REDS(c_p, reds); - } - - if (suspended) { -#ifdef USE_VM_PROBES - if (!resume && DTRACE_ENABLED(dist_port_busy)) { - DTRACE_CHARBUF(port_str, 64); - DTRACE_CHARBUF(remote_str, 64); - DTRACE_CHARBUF(pid_str, 16); - - erts_snprintf(port_str, sizeof(port_str), "%T", cid); - erts_snprintf(remote_str, sizeof(remote_str), "%T", dep->sysname); - erts_snprintf(pid_str, sizeof(pid_str), "%T", c_p->common.id); - DTRACE4(dist_port_busy, erts_this_node_sysname, - port_str, remote_str, pid_str); - } -#endif - if (!resume && erts_system_monitor_flags.busy_dist_port) - monitor_generic(c_p, am_busy_dist_port, cid); - return ERTS_DSIG_SEND_YIELD; +done: + if (ctx->msg && ctx->c_p) { + BUMP_REDS(ctx->c_p, (initial_reds - ctx->reds) / TERM_TO_BINARY_LOOP_FACTOR); } - return ERTS_DSIG_SEND_OK; + return retval; } - static Uint dist_port_command(Port *prt, ErtsDistOutputBuf *obuf) { @@ -1880,7 +1980,7 @@ dist_port_command(Port *prt, ErtsDistOutputBuf *obuf) ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(prt)); if (size > (Uint) INT_MAX) - erl_exit(ERTS_ABORT_EXIT, + erts_exit(ERTS_ABORT_EXIT, "Absurdly large distribution output data buffer " "(%beu bytes) passed.\n", size); @@ -1890,8 +1990,9 @@ dist_port_command(Port *prt, ErtsDistOutputBuf *obuf) DTRACE_CHARBUF(port_str, 64); DTRACE_CHARBUF(remote_str, 64); - erts_snprintf(port_str, sizeof(port_str), "%T", prt->common.id); - erts_snprintf(remote_str, sizeof(remote_str), + erts_snprintf(port_str, sizeof(DTRACE_CHARBUF_NAME(port_str)), + "%T", prt->common.id); + erts_snprintf(remote_str, sizeof(DTRACE_CHARBUF_NAME(remote_str)), "%T", prt->dist_entry->sysname); DTRACE4(dist_output, erts_this_node_sysname, port_str, remote_str, size); @@ -1919,7 +2020,7 @@ dist_port_commandv(Port *prt, ErtsDistOutputBuf *obuf) ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(prt)); if (size > (Uint) INT_MAX) - erl_exit(ERTS_ABORT_EXIT, + erts_exit(ERTS_ABORT_EXIT, "Absurdly large distribution output data buffer " "(%beu bytes) passed.\n", size); @@ -1944,8 +2045,9 @@ dist_port_commandv(Port *prt, ErtsDistOutputBuf *obuf) DTRACE_CHARBUF(port_str, 64); DTRACE_CHARBUF(remote_str, 64); - erts_snprintf(port_str, sizeof(port_str), "%T", prt->common.id); - erts_snprintf(remote_str, sizeof(remote_str), + erts_snprintf(port_str, sizeof(DTRACE_CHARBUF_NAME(port_str)), + "%T", prt->common.id); + erts_snprintf(remote_str, sizeof(DTRACE_CHARBUF_NAME(remote_str)), "%T", prt->dist_entry->sysname); DTRACE4(dist_outputv, erts_this_node_sysname, port_str, remote_str, size); @@ -1988,6 +2090,7 @@ erts_dist_command(Port *prt, int reds_limit) DistEntry *dep = prt->dist_entry; Uint (*send)(Port *prt, ErtsDistOutputBuf *obuf); erts_aint32_t sched_flags; + ErtsSchedulerData *esdp = erts_get_scheduler_data(); ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(prt)); @@ -2042,12 +2145,12 @@ erts_dist_command(Port *prt, int reds_limit) ErtsDistOutputBuf *fob; size = (*send)(prt, foq.first); + esdp->io.out += (Uint64) size; #ifdef ERTS_RAW_DIST_MSG_DBG erts_fprintf(stderr, ">> "); bw(foq.first->extp, size); #endif reds += ERTS_PORT_REDS_DIST_CMD_DATA(size); - erts_smp_atomic_add_nob(&erts_bytes_out, size); fob = foq.first; obufsize += size_obuf(fob); foq.first = foq.first->next; @@ -2127,12 +2230,12 @@ erts_dist_command(Port *prt, int reds_limit) ASSERT(&oq.first->data[0] <= oq.first->extp && oq.first->extp < oq.first->ext_endp); size = (*send)(prt, oq.first); + esdp->io.out += (Uint64) size; #ifdef ERTS_RAW_DIST_MSG_DBG erts_fprintf(stderr, ">> "); bw(oq.first->extp, size); #endif reds += ERTS_PORT_REDS_DIST_CMD_DATA(size); - erts_smp_atomic_add_nob(&erts_bytes_out, size); fob = oq.first; obufsize += size_obuf(fob); oq.first = oq.first->next; @@ -2280,8 +2383,9 @@ erts_dist_port_not_busy(Port *prt) DTRACE_CHARBUF(port_str, 64); DTRACE_CHARBUF(remote_str, 64); - erts_snprintf(port_str, sizeof(port_str), "%T", prt->common.id); - erts_snprintf(remote_str, sizeof(remote_str), + erts_snprintf(port_str, sizeof(DTRACE_CHARBUF_NAME(port_str)), + "%T", prt->common.id); + erts_snprintf(remote_str, sizeof(DTRACE_CHARBUF_NAME(remote_str)), "%T", prt->dist_entry->sysname); DTRACE3(dist_port_not_busy, erts_this_node_sysname, port_str, remote_str); @@ -2423,7 +2527,7 @@ info_dist_entry(int to, void *arg, DistEntry *dep, int visible, int connected) erts_print(to, arg, "Name: %T", dep->sysname); #ifdef DEBUG - erts_print(to, arg, " (refc=%d)", erts_refc_read(&dep->refc, 1)); + erts_print(to, arg, " (refc=%d)", erts_refc_read(&dep->refc, 0)); #endif erts_print(to, arg, "\n"); if (!connected && is_nil(dep->cid)) { @@ -2469,7 +2573,9 @@ int distribution_info(int to, void *arg) /* Called by break handler */ } for (dep = erts_not_connected_dist_entries; dep; dep = dep->next) { - info_dist_entry(to, arg, dep, 0, 0); + if (dep != erts_this_dist_entry) { + info_dist_entry(to, arg, dep, 0, 0); + } } return(0); @@ -2547,22 +2653,17 @@ BIF_RETTYPE setnode_2(BIF_ALIST_2) if (!net_kernel) goto error; - /* By setting dist_entry==erts_this_dist_entry and DISTRIBUTION on - net_kernel do_net_exist will be called when net_kernel - is terminated !! */ - (void) ERTS_PROC_SET_DIST_ENTRY(net_kernel, - ERTS_PROC_LOCK_MAIN, - erts_this_dist_entry); - erts_refc_inc(&erts_this_dist_entry->refc, 2); + /* By setting F_DISTRIBUTION on net_kernel, + * do_net_exist will be called when net_kernel is terminated !! */ net_kernel->flags |= F_DISTRIBUTION; if (net_kernel != BIF_P) erts_smp_proc_unlock(net_kernel, ERTS_PROC_LOCK_MAIN); #ifdef DEBUG - erts_smp_rwmtx_rwlock(&erts_dist_table_rwmtx); + erts_smp_rwmtx_rlock(&erts_dist_table_rwmtx); ASSERT(!erts_visible_dist_entries && !erts_hidden_dist_entries); - erts_smp_rwmtx_rwunlock(&erts_dist_table_rwmtx); + erts_smp_rwmtx_runlock(&erts_dist_table_rwmtx); #endif erts_smp_proc_unlock(BIF_P, ERTS_PROC_LOCK_MAIN); @@ -2912,13 +3013,13 @@ BIF_RETTYPE nodes_1(BIF_ALIST_1) length = 0; - erts_smp_rwmtx_rwlock(&erts_dist_table_rwmtx); + erts_smp_rwmtx_rlock(&erts_dist_table_rwmtx); - ASSERT(erts_no_of_not_connected_dist_entries >= 0); + ASSERT(erts_no_of_not_connected_dist_entries > 0); ASSERT(erts_no_of_hidden_dist_entries >= 0); ASSERT(erts_no_of_visible_dist_entries >= 0); if(not_connected) - length += erts_no_of_not_connected_dist_entries; + length += (erts_no_of_not_connected_dist_entries - 1); if(hidden) length += erts_no_of_hidden_dist_entries; if(visible) @@ -2929,7 +3030,7 @@ BIF_RETTYPE nodes_1(BIF_ALIST_1) result = NIL; if (length == 0) { - erts_smp_rwmtx_rwunlock(&erts_dist_table_rwmtx); + erts_smp_rwmtx_runlock(&erts_dist_table_rwmtx); goto done; } @@ -2940,8 +3041,10 @@ BIF_RETTYPE nodes_1(BIF_ALIST_1) #endif if(not_connected) for(dep = erts_not_connected_dist_entries; dep; dep = dep->next) { - result = CONS(hp, dep->sysname, result); - hp += 2; + if (dep != erts_this_dist_entry) { + result = CONS(hp, dep->sysname, result); + hp += 2; + } } if(hidden) for(dep = erts_hidden_dist_entries; dep; dep = dep->next) { @@ -2958,7 +3061,7 @@ BIF_RETTYPE nodes_1(BIF_ALIST_1) hp += 2; } ASSERT(endp == hp); - erts_smp_rwmtx_rwunlock(&erts_dist_table_rwmtx); + erts_smp_rwmtx_runlock(&erts_dist_table_rwmtx); done: UnUseTmpHeap(2,BIF_P); @@ -3222,11 +3325,7 @@ send_nodes_mon_msg(Process *rp, } ASSERT(hend == hp); - erts_queue_message(rp, rp_locksp, bp, msg, NIL -#ifdef USE_VM_PROBES - , NIL -#endif - ); + erts_queue_message(rp, rp_locksp, bp, msg, NIL); } static void @@ -3246,10 +3345,10 @@ send_nodes_mon_msgs(Process *c_p, Eterm what, Eterm node, Eterm type, Eterm reas DTRACE_CHARBUF(type_str, 12); DTRACE_CHARBUF(reason_str, 64); - erts_snprintf(what_str, sizeof(what_str), "%T", what); - erts_snprintf(node_str, sizeof(node_str), "%T", node); - erts_snprintf(type_str, sizeof(type_str), "%T", type); - erts_snprintf(reason_str, sizeof(reason_str), "%T", reason); + erts_snprintf(what_str, sizeof(DTRACE_CHARBUF_NAME(what_str)), "%T", what); + erts_snprintf(node_str, sizeof(DTRACE_CHARBUF_NAME(node_str)), "%T", node); + erts_snprintf(type_str, sizeof(DTRACE_CHARBUF_NAME(type_str)), "%T", type); + erts_snprintf(reason_str, sizeof(DTRACE_CHARBUF_NAME(reason_str)), "%T", reason); DTRACE5(dist_monitor, erts_this_node_sysname, what_str, node_str, type_str, reason_str); } @@ -3283,7 +3382,7 @@ send_nodes_mon_msgs(Process *c_p, Eterm what, Eterm node, Eterm type, Eterm reas continue; break; default: - erl_exit(ERTS_ABORT_EXIT, "Bad node type found\n"); + erts_exit(ERTS_ABORT_EXIT, "Bad node type found\n"); } } @@ -3592,7 +3691,7 @@ erts_processes_monitoring_nodes(Process *c_p) case ERTS_NODES_MON_OPT_TYPES: type = am_all; break; case ERTS_NODES_MON_OPT_TYPE_VISIBLE: type = am_visible; break; case ERTS_NODES_MON_OPT_TYPE_HIDDEN: type = am_hidden; break; - default: erl_exit(ERTS_ABORT_EXIT, "Bad node type found\n"); + default: erts_exit(ERTS_ABORT_EXIT, "Bad node type found\n"); } olist = erts_bld_cons(hpp, szp, erts_bld_tuple(hpp, szp, 2, diff --git a/erts/emulator/beam/dist.h b/erts/emulator/beam/dist.h index ff8f5e106f..fb777d9ac1 100644 --- a/erts/emulator/beam/dist.h +++ b/erts/emulator/beam/dist.h @@ -1,18 +1,19 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 1996-2013. All Rights Reserved. + * Copyright Ericsson AB 1996-2014. All Rights Reserved. * - * The contents of this file are subject to the Erlang Public License, - * Version 1.1, (the "License"); you may not use this file except in - * compliance with the License. You should have received a copy of the - * Erlang Public License along with this software. If not, it can be - * retrieved online at http://www.erlang.org/. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Software distributed under the License is distributed on an "AS IS" - * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See - * the License for the specific language governing rights and limitations - * under the License. + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. * * %CopyrightEnd% */ @@ -22,6 +23,7 @@ #include "erl_process.h" #include "erl_node_tables.h" +#include "zlib.h" #define DFLAG_PUBLISHED 0x01 #define DFLAG_ATOM_CACHE 0x02 @@ -40,13 +42,16 @@ #define DFLAG_SMALL_ATOM_TAGS 0x4000 #define DFLAG_INTERNAL_TAGS 0x8000 #define DFLAG_UTF8_ATOMS 0x10000 +#define DFLAG_MAP_TAG 0x20000 /* All flags that should be enabled when term_to_binary/1 is used. */ #define TERM_TO_BINARY_DFLAGS (DFLAG_EXTENDED_REFERENCES \ | DFLAG_NEW_FUN_TAGS \ + | DFLAG_NEW_FLOATS \ | DFLAG_EXTENDED_PIDS_PORTS \ | DFLAG_EXPORT_PTR_TAG \ - | DFLAG_BIT_BINARIES) + | DFLAG_BIT_BINARIES \ + | DFLAG_MAP_TAG) /* opcodes used in distribution messages */ #define DOP_LINK 1 @@ -261,17 +266,105 @@ erts_destroy_dist_link(ErtsDistLinkData *dldp) #endif + + +/* Define for testing */ +/* #define EXTREME_TTB_TRAPPING 1 */ + +#ifndef EXTREME_TTB_TRAPPING +#define TERM_TO_BINARY_LOOP_FACTOR 32 +#else +#define TERM_TO_BINARY_LOOP_FACTOR 1 +#endif + +typedef enum { TTBSize, TTBEncode, TTBCompress } TTBState; +typedef struct TTBSizeContext_ { + Uint flags; + int level; + Uint result; + Eterm obj; + ErtsWStack wstack; +} TTBSizeContext; + +typedef struct TTBEncodeContext_ { + Uint flags; + int level; + byte* ep; + Eterm obj; + ErtsWStack wstack; + Binary *result_bin; +} TTBEncodeContext; + +typedef struct { + Uint real_size; + Uint dest_len; + byte *dbytes; + Binary *result_bin; + Binary *destination_bin; + z_stream stream; +} TTBCompressContext; + +typedef struct { + int alive; + TTBState state; + union { + TTBSizeContext sc; + TTBEncodeContext ec; + TTBCompressContext cc; + } s; +} TTBContext; + +enum erts_dsig_send_phase { + ERTS_DSIG_SEND_PHASE_INIT, + ERTS_DSIG_SEND_PHASE_MSG_SIZE, + ERTS_DSIG_SEND_PHASE_ALLOC, + ERTS_DSIG_SEND_PHASE_MSG_ENCODE, + ERTS_DSIG_SEND_PHASE_FIN +}; + +struct erts_dsig_send_context { + enum erts_dsig_send_phase phase; + Sint reds; + + Eterm ctl; + Eterm msg; + int force_busy; + Uint32 pass_through_size; + Uint data_size, dhdr_ext_size; + ErtsAtomCacheMap *acmp; + ErtsDistOutputBuf *obuf; + Uint32 flags; + Process *c_p; + union { + TTBSizeContext sc; + TTBEncodeContext ec; + }u; +}; + +typedef struct { + int suspend; + + Eterm ctl_heap[6]; + ErtsDSigData dsd; + DistEntry* dep_to_deref; + struct erts_dsig_send_context dss; + + Eterm return_term; +}ErtsSendContext; + + /* * erts_dsig_send_* return values. */ #define ERTS_DSIG_SEND_OK 0 #define ERTS_DSIG_SEND_YIELD 1 +#define ERTS_DSIG_SEND_CONTINUE 2 extern int erts_dsig_send_link(ErtsDSigData *, Eterm, Eterm); -extern int erts_dsig_send_msg(ErtsDSigData *, Eterm, Eterm); +extern int erts_dsig_send_msg(Eterm, Eterm, ErtsSendContext*); extern int erts_dsig_send_exit_tt(ErtsDSigData *, Eterm, Eterm, Eterm, Eterm); extern int erts_dsig_send_unlink(ErtsDSigData *, Eterm, Eterm); -extern int erts_dsig_send_reg_msg(ErtsDSigData *, Eterm, Eterm); +extern int erts_dsig_send_reg_msg(Eterm, Eterm, ErtsSendContext*); extern int erts_dsig_send_group_leader(ErtsDSigData *, Eterm, Eterm); extern int erts_dsig_send_exit(ErtsDSigData *, Eterm, Eterm, Eterm); extern int erts_dsig_send_exit2(ErtsDSigData *, Eterm, Eterm, Eterm); @@ -279,6 +372,10 @@ extern int erts_dsig_send_demonitor(ErtsDSigData *, Eterm, Eterm, Eterm, int); extern int erts_dsig_send_monitor(ErtsDSigData *, Eterm, Eterm, Eterm); extern int erts_dsig_send_m_exit(ErtsDSigData *, Eterm, Eterm, Eterm, Eterm); +extern int erts_dsig_send(ErtsDSigData *dsdp, struct erts_dsig_send_context* ctx); +extern void erts_dsend_context_dtor(Binary*); +extern Eterm erts_dsend_export_trap_context(Process* p, ErtsSendContext* ctx); + extern int erts_dist_command(Port *prt, int reds); extern void erts_dist_port_not_busy(Port *prt); extern void erts_kill_dist_connection(DistEntry *dep, Uint32); diff --git a/erts/emulator/beam/dtrace-wrapper.h b/erts/emulator/beam/dtrace-wrapper.h index 6ec0c91e21..d343dc5ab0 100644 --- a/erts/emulator/beam/dtrace-wrapper.h +++ b/erts/emulator/beam/dtrace-wrapper.h @@ -4,16 +4,17 @@ * Copyright Dustin Sallings, Michal Ptaszek, Scott Lystig Fritchie 2011-2012. * All Rights Reserved. * - * The contents of this file are subject to the Erlang Public License, - * Version 1.1, (the "License"); you may not use this file except in - * compliance with the License. You should have received a copy of the - * Erlang Public License along with this software. If not, it can be - * retrieved online at http://www.erlang.org/. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Software distributed under the License is distributed on an "AS IS" - * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See - * the License for the specific language governing rights and limitations - * under the License. + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. * * %CopyrightEnd% */ diff --git a/erts/emulator/beam/elib_memmove.c b/erts/emulator/beam/elib_memmove.c index d2fe8649ed..d4ca30158e 100644 --- a/erts/emulator/beam/elib_memmove.c +++ b/erts/emulator/beam/elib_memmove.c @@ -3,16 +3,17 @@ * * Copyright Ericsson AB 1997-2009. All Rights Reserved. * - * The contents of this file are subject to the Erlang Public License, - * Version 1.1, (the "License"); you may not use this file except in - * compliance with the License. You should have received a copy of the - * Erlang Public License along with this software. If not, it can be - * retrieved online at http://www.erlang.org/. - * - * Software distributed under the License is distributed on an "AS IS" - * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See - * the License for the specific language governing rights and limitations - * under the License. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. * * %CopyrightEnd% */ diff --git a/erts/emulator/beam/erl_afit_alloc.c b/erts/emulator/beam/erl_afit_alloc.c index 5850f80843..4b0541c10e 100644 --- a/erts/emulator/beam/erl_afit_alloc.c +++ b/erts/emulator/beam/erl_afit_alloc.c @@ -3,16 +3,17 @@ * * Copyright Ericsson AB 2003-2013. All Rights Reserved. * - * The contents of this file are subject to the Erlang Public License, - * Version 1.1, (the "License"); you may not use this file except in - * compliance with the License. You should have received a copy of the - * Erlang Public License along with this software. If not, it can be - * retrieved online at http://www.erlang.org/. - * - * Software distributed under the License is distributed on an "AS IS" - * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See - * the License for the specific language governing rights and limitations - * under the License. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. * * %CopyrightEnd% */ @@ -48,10 +49,9 @@ struct AFFreeBlock_t_ { #define MIN_MBC_FIRST_FREE_SZ (4*1024) /* Prototypes of callback functions */ -static Block_t * get_free_block (Allctr_t *, Uint, - Block_t *, Uint, Uint32); -static void link_free_block (Allctr_t *, Block_t *, Uint32); -static void unlink_free_block (Allctr_t *, Block_t *, Uint32); +static Block_t * get_free_block (Allctr_t *, Uint, Block_t *, Uint); +static void link_free_block (Allctr_t *, Block_t *); +static void unlink_free_block (Allctr_t *, Block_t *); static Eterm info_options (Allctr_t *, char *, int *, @@ -84,8 +84,7 @@ erts_afalc_start(AFAllctr_t *afallctr, sys_memcpy((void *) afallctr, (void *) &zero.allctr, sizeof(AFAllctr_t)); - init->sbmbct = 0; /* Small mbc not supported by afit */ - + allctr->mbc_header_size = sizeof(Carrier_t); allctr->min_mbc_size = MIN_MBC_SZ; allctr->min_mbc_first_free_size = MIN_MBC_FIRST_FREE_SZ; allctr->min_block_size = sizeof(AFFreeBlock_t); @@ -100,6 +99,9 @@ erts_afalc_start(AFAllctr_t *afallctr, allctr->get_next_mbc_size = NULL; allctr->creating_mbc = NULL; allctr->destroying_mbc = NULL; + allctr->add_mbc = NULL; + allctr->remove_mbc = NULL; + allctr->largest_fblk_in_mbc = NULL; allctr->init_atoms = init_atoms; #ifdef ERTS_ALLOC_UTIL_HARD_DEBUG @@ -116,8 +118,7 @@ erts_afalc_start(AFAllctr_t *afallctr, } static Block_t * -get_free_block(Allctr_t *allctr, Uint size, Block_t *cand_blk, Uint cand_size, - Uint32 flags) +get_free_block(Allctr_t *allctr, Uint size, Block_t *cand_blk, Uint cand_size) { AFAllctr_t *afallctr = (AFAllctr_t *) allctr; @@ -135,7 +136,7 @@ get_free_block(Allctr_t *allctr, Uint size, Block_t *cand_blk, Uint cand_size, } static void -link_free_block(Allctr_t *allctr, Block_t *block, Uint32 flags) +link_free_block(Allctr_t *allctr, Block_t *block) { AFFreeBlock_t *blk = (AFFreeBlock_t *) block; AFAllctr_t *afallctr = (AFAllctr_t *) allctr; @@ -156,7 +157,7 @@ link_free_block(Allctr_t *allctr, Block_t *block, Uint32 flags) } static void -unlink_free_block(Allctr_t *allctr, Block_t *block, Uint32 flags) +unlink_free_block(Allctr_t *allctr, Block_t *block) { AFFreeBlock_t *blk = (AFFreeBlock_t *) block; AFAllctr_t *afallctr = (AFAllctr_t *) allctr; @@ -240,7 +241,7 @@ info_options(Allctr_t *allctr, if (hpp || szp) { if (!atoms_initialized) - erl_exit(1, "%s:%d: Internal error: Atoms not initialized", + erts_exit(ERTS_ERROR_EXIT, "%s:%d: Internal error: Atoms not initialized", __FILE__, __LINE__);; res = NIL; @@ -259,10 +260,10 @@ info_options(Allctr_t *allctr, * to erts_afalc_test() * \* */ -unsigned long -erts_afalc_test(unsigned long op, unsigned long a1, unsigned long a2) +UWord +erts_afalc_test(UWord op, UWord a1, UWord a2) { switch (op) { - default: ASSERT(0); return ~((unsigned long) 0); + default: ASSERT(0); return ~((UWord) 0); } } diff --git a/erts/emulator/beam/erl_afit_alloc.h b/erts/emulator/beam/erl_afit_alloc.h index cf7b99c463..ef050ff50e 100644 --- a/erts/emulator/beam/erl_afit_alloc.h +++ b/erts/emulator/beam/erl_afit_alloc.h @@ -3,16 +3,17 @@ * * Copyright Ericsson AB 2003-2013. All Rights Reserved. * - * The contents of this file are subject to the Erlang Public License, - * Version 1.1, (the "License"); you may not use this file except in - * compliance with the License. You should have received a copy of the - * Erlang Public License along with this software. If not, it can be - * retrieved online at http://www.erlang.org/. - * - * Software distributed under the License is distributed on an "AS IS" - * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See - * the License for the specific language governing rights and limitations - * under the License. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. * * %CopyrightEnd% */ @@ -56,7 +57,7 @@ struct AFAllctr_t_ { AFFreeBlock_t * free_list; }; -unsigned long erts_afalc_test(unsigned long, unsigned long, unsigned long); +UWord erts_afalc_test(UWord, UWord, UWord); #endif /* #if defined(GET_ERL_AF_ALLOC_IMPL) && !defined(ERL_AF_ALLOC_IMPL__) */ diff --git a/erts/emulator/beam/erl_alloc.c b/erts/emulator/beam/erl_alloc.c index d748f86d75..5f8cd2bbf7 100644 --- a/erts/emulator/beam/erl_alloc.c +++ b/erts/emulator/beam/erl_alloc.c @@ -3,16 +3,17 @@ * * Copyright Ericsson AB 2002-2013. All Rights Reserved. * - * The contents of this file are subject to the Erlang Public License, - * Version 1.1, (the "License"); you may not use this file except in - * compliance with the License. You should have received a copy of the - * Erlang Public License along with this software. If not, it can be - * retrieved online at http://www.erlang.org/. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Software distributed under the License is distributed on an "AS IS" - * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See - * the License for the specific language governing rights and limitations - * under the License. + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. * * %CopyrightEnd% */ @@ -39,7 +40,7 @@ #include "erl_instrument.h" #include "erl_mseg.h" #include "erl_monitors.h" -#include "erl_bif_timer.h" +#include "erl_hl_timer.h" #include "erl_cpu_topology.h" #include "erl_thr_queue.h" #if defined(ERTS_ALC_T_DRV_SEL_D_STATE) || defined(ERTS_ALC_T_DRV_EV_D_STATE) @@ -71,6 +72,23 @@ #define AU_ALLOC_DEFAULT_ENABLE(X) (X) #endif +#define ERTS_ALC_DEFAULT_ENABLED_ACUL 60 +#define ERTS_ALC_DEFAULT_ENABLED_ACUL_EHEAP_ALLOC 45 +#define ERTS_ALC_DEFAULT_ENABLED_ACUL_LL_ALLOC 85 + +#define ERTS_ALC_DEFAULT_ACUL ERTS_ALC_DEFAULT_ENABLED_ACUL +#define ERTS_ALC_DEFAULT_ACUL_EHEAP_ALLOC ERTS_ALC_DEFAULT_ENABLED_ACUL_EHEAP_ALLOC +#define ERTS_ALC_DEFAULT_ACUL_LL_ALLOC ERTS_ALC_DEFAULT_ENABLED_ACUL_LL_ALLOC + +#ifndef ERTS_SMP +# undef ERTS_ALC_DEFAULT_ACUL +# define ERTS_ALC_DEFAULT_ACUL 0 +# undef ERTS_ALC_DEFAULT_ACUL_EHEAP_ALLOC +# define ERTS_ALC_DEFAULT_ACUL_EHEAP_ALLOC 0 +# undef ERTS_ALC_DEFAULT_ACUL_LL_ALLOC +# define ERTS_ALC_DEFAULT_ACUL_LL_ALLOC 0 +#endif + #ifdef DEBUG static Uint install_debug_functions(void); #if 0 @@ -83,6 +101,8 @@ static Uint install_debug_functions(void); #endif #endif +static int lock_all_physical_memory = 0; + ErtsAllocatorFunctions_t erts_allctrs[ERTS_ALC_A_MAX+1]; ErtsAllocatorInfo_t erts_allctrs_info[ERTS_ALC_A_MAX+1]; ErtsAllocatorThrSpec_t erts_allctr_thr_spec[ERTS_ALC_A_MAX+1]; @@ -101,11 +121,9 @@ typedef union { char align_aoffa[ERTS_ALC_CACHE_LINE_ALIGN_SIZE(sizeof(AOFFAllctr_t))]; } ErtsAllocatorState_t; -static ErtsAllocatorState_t sbmbc_alloc_state; static ErtsAllocatorState_t std_alloc_state; static ErtsAllocatorState_t ll_alloc_state; #if HALFWORD_HEAP -static ErtsAllocatorState_t sbmbc_low_alloc_state; static ErtsAllocatorState_t std_low_alloc_state; static ErtsAllocatorState_t ll_low_alloc_state; #endif @@ -116,10 +134,12 @@ static ErtsAllocatorState_t binary_alloc_state; static ErtsAllocatorState_t ets_alloc_state; static ErtsAllocatorState_t driver_alloc_state; static ErtsAllocatorState_t fix_alloc_state; +static ErtsAllocatorState_t test_alloc_state; typedef struct { erts_smp_atomic32_t refc; int only_sz; + int internal; Uint req_sched; Process *proc; Eterm ref; @@ -162,6 +182,7 @@ enum allctr_type { struct au_init { int enable; int thr_spec; + int carrier_migration_allowed; enum allctr_type atype; struct { AllctrInit_t util; @@ -200,7 +221,6 @@ typedef struct { char *mtrace; char *nodename; } instr; - struct au_init sbmbc_alloc; struct au_init sl_alloc; struct au_init std_alloc; struct au_init ll_alloc; @@ -211,13 +231,13 @@ typedef struct { struct au_init driver_alloc; struct au_init fix_alloc; #if HALFWORD_HEAP - struct au_init sbmbc_low_alloc; struct au_init std_low_alloc; struct au_init ll_low_alloc; #endif + struct au_init test_alloc; } erts_alc_hndl_args_init_t; -#define ERTS_AU_INIT__ {0, 0, GOODFIT, DEFAULT_ALLCTR_INIT, {1,1,1,1}} +#define ERTS_AU_INIT__ {0, 0, 1, GOODFIT, DEFAULT_ALLCTR_INIT, {1,1,1,1}} #define SET_DEFAULT_ALLOC_OPTS(IP) \ do { \ @@ -226,34 +246,6 @@ do { \ } while (0) static void -set_default_sbmbc_alloc_opts(struct au_init *ip) -{ - SET_DEFAULT_ALLOC_OPTS(ip); - ip->enable = 0; - ip->thr_spec = 0; - ip->atype = BESTFIT; - ip->init.bf.ao = 1; - ip->init.util.ramv = 0; - ip->init.util.mmsbc = 0; - ip->init.util.mmmbc = 500; - ip->init.util.sbct = ~((UWord) 0); - ip->init.util.name_prefix = "sbmbc_"; - ip->init.util.alloc_no = ERTS_ALC_A_SBMBC; -#ifndef SMALL_MEMORY - ip->init.util.mmbcs = 2*1024*1024; /* Main carrier size */ -#else - ip->init.util.mmbcs = 1*1024*1024; /* Main carrier size */ -#endif - ip->init.util.ts = ERTS_ALC_MTA_SBMBC; - ip->init.util.asbcst = 0; - ip->init.util.rsbcst = 0; - ip->init.util.rsbcmt = 0; - ip->init.util.rmbcmt = 0; - ip->init.util.sbmbct = 0; - ip->init.util.sbmbcs = 0; -} - -static void set_default_sl_alloc_opts(struct au_init *ip) { SET_DEFAULT_ALLOC_OPTS(ip); @@ -261,7 +253,6 @@ set_default_sl_alloc_opts(struct au_init *ip) ip->thr_spec = 1; ip->atype = GOODFIT; ip->init.util.name_prefix = "sl_"; - ip->init.util.mmmbc = 5; ip->init.util.alloc_no = ERTS_ALC_A_SHORT_LIVED; #ifndef SMALL_MEMORY ip->init.util.mmbcs = 128*1024; /* Main carrier size */ @@ -274,7 +265,7 @@ set_default_sl_alloc_opts(struct au_init *ip) ip->init.util.force = 1; ip->init.util.low_mem = 1; #endif - + ip->init.util.acul = ERTS_ALC_DEFAULT_ACUL; } static void @@ -285,7 +276,6 @@ set_default_std_alloc_opts(struct au_init *ip) ip->thr_spec = 1; ip->atype = BESTFIT; ip->init.util.name_prefix = "std_"; - ip->init.util.mmmbc = 5; ip->init.util.alloc_no = ERTS_ALC_A_STANDARD; #ifndef SMALL_MEMORY ip->init.util.mmbcs = 128*1024; /* Main carrier size */ @@ -293,6 +283,7 @@ set_default_std_alloc_opts(struct au_init *ip) ip->init.util.mmbcs = 32*1024; /* Main carrier size */ #endif ip->init.util.ts = ERTS_ALC_MTA_STANDARD; + ip->init.util.acul = ERTS_ALC_DEFAULT_ACUL; } static void @@ -305,7 +296,6 @@ set_default_ll_alloc_opts(struct au_init *ip) ip->init.bf.ao = 1; ip->init.util.ramv = 0; ip->init.util.mmsbc = 0; - ip->init.util.mmmbc = 0; ip->init.util.sbct = ~((UWord) 0); ip->init.util.name_prefix = "ll_"; ip->init.util.alloc_no = ERTS_ALC_A_LONG_LIVED; @@ -319,8 +309,7 @@ set_default_ll_alloc_opts(struct au_init *ip) ip->init.util.rsbcst = 0; ip->init.util.rsbcmt = 0; ip->init.util.rmbcmt = 0; - ip->init.util.sbmbct = 0; - ip->init.util.sbmbcs = 0; + ip->init.util.acul = ERTS_ALC_DEFAULT_ACUL_LL_ALLOC; } static void @@ -329,6 +318,7 @@ set_default_temp_alloc_opts(struct au_init *ip) SET_DEFAULT_ALLOC_OPTS(ip); ip->enable = AU_ALLOC_DEFAULT_ENABLE(1); ip->thr_spec = 1; + ip->carrier_migration_allowed = 0; ip->atype = AFIT; ip->init.util.name_prefix = "temp_"; ip->init.util.alloc_no = ERTS_ALC_A_TEMPORARY; @@ -353,7 +343,6 @@ set_default_eheap_alloc_opts(struct au_init *ip) ip->enable = AU_ALLOC_DEFAULT_ENABLE(1); ip->thr_spec = 1; ip->atype = GOODFIT; - ip->init.util.mmmbc = 100; ip->init.util.name_prefix = "eheap_"; ip->init.util.alloc_no = ERTS_ALC_A_EHEAP; #ifndef SMALL_MEMORY @@ -367,6 +356,7 @@ set_default_eheap_alloc_opts(struct au_init *ip) ip->init.util.force = 1; ip->init.util.low_mem = 1; #endif + ip->init.util.acul = ERTS_ALC_DEFAULT_ACUL_EHEAP_ALLOC; } static void @@ -376,7 +366,6 @@ set_default_binary_alloc_opts(struct au_init *ip) ip->enable = AU_ALLOC_DEFAULT_ENABLE(1); ip->thr_spec = 1; ip->atype = BESTFIT; - ip->init.util.mmmbc = 50; ip->init.util.name_prefix = "binary_"; ip->init.util.alloc_no = ERTS_ALC_A_BINARY; #ifndef SMALL_MEMORY @@ -385,6 +374,7 @@ set_default_binary_alloc_opts(struct au_init *ip) ip->init.util.mmbcs = 32*1024; /* Main carrier size */ #endif ip->init.util.ts = ERTS_ALC_MTA_BINARY; + ip->init.util.acul = ERTS_ALC_DEFAULT_ACUL; } static void @@ -394,7 +384,6 @@ set_default_ets_alloc_opts(struct au_init *ip) ip->enable = AU_ALLOC_DEFAULT_ENABLE(1); ip->thr_spec = 1; ip->atype = BESTFIT; - ip->init.util.mmmbc = 100; ip->init.util.name_prefix = "ets_"; ip->init.util.alloc_no = ERTS_ALC_A_ETS; #ifndef SMALL_MEMORY @@ -403,6 +392,7 @@ set_default_ets_alloc_opts(struct au_init *ip) ip->init.util.mmbcs = 32*1024; /* Main carrier size */ #endif ip->init.util.ts = ERTS_ALC_MTA_ETS; + ip->init.util.acul = ERTS_ALC_DEFAULT_ACUL; } static void @@ -420,6 +410,7 @@ set_default_driver_alloc_opts(struct au_init *ip) ip->init.util.mmbcs = 32*1024; /* Main carrier size */ #endif ip->init.util.ts = ERTS_ALC_MTA_DRIVER; + ip->init.util.acul = ERTS_ALC_DEFAULT_ACUL; } static void @@ -440,8 +431,36 @@ set_default_fix_alloc_opts(struct au_init *ip, ip->init.util.mmbcs = 128*1024; /* Main carrier size */ #endif ip->init.util.ts = ERTS_ALC_MTA_FIXED_SIZE; + ip->init.util.acul = ERTS_ALC_DEFAULT_ACUL; +} + +static void +set_default_test_alloc_opts(struct au_init *ip) +{ + SET_DEFAULT_ALLOC_OPTS(ip); + ip->enable = 0; /* Disabled by default */ + ip->thr_spec = -1 * erts_no_schedulers; + ip->atype = AOFIRSTFIT; + ip->init.aoff.flavor = AOFF_BF; + ip->init.util.name_prefix = "test_"; + ip->init.util.alloc_no = ERTS_ALC_A_TEST; + ip->init.util.mmbcs = 0; /* Main carrier size */ + ip->init.util.ts = ERTS_ALC_MTA_TEST; + ip->init.util.acul = ERTS_ALC_DEFAULT_ACUL; + + /* Use a constant minimal MBC size */ +#if ERTS_SA_MB_CARRIERS + ip->init.util.smbcs = ERTS_SACRR_UNIT_SZ; + ip->init.util.lmbcs = ERTS_SACRR_UNIT_SZ; + ip->init.util.sbct = ERTS_SACRR_UNIT_SZ; +#else + ip->init.util.smbcs = 1 << 12; + ip->init.util.lmbcs = 1 << 12; + ip->init.util.sbct = 1 << 12; +#endif } + #ifdef ERTS_SMP static void @@ -462,13 +481,6 @@ adjust_tpref(struct au_init *ip, int no_sched) /* ... shrink smallest multi-block carrier size */ if (ip->default_.smbcs) ip->init.util.smbcs /= ERTS_MIN(4, no_sched); - /* ... and more than three allocators shrink - max mseg multi-block carriers */ - if (ip->default_.mmmbc && no_sched > 2) { - ip->init.util.mmmbc /= ERTS_MIN(4, no_sched - 1); - if (ip->init.util.mmmbc < 3) - ip->init.util.mmmbc = 3; - } } } @@ -495,6 +507,70 @@ refuse_af_strategy(struct au_init *init) static void hdbg_init(void); #endif +static void adjust_fix_alloc_sizes(UWord extra_block_size) +{ + + if (extra_block_size && erts_allctrs_info[ERTS_ALC_A_FIXED_SIZE].enabled) { + int j; + +#ifdef ERTS_SMP + if (erts_allctrs_info[ERTS_ALC_A_FIXED_SIZE].thr_spec) { + int i; + ErtsAllocatorThrSpec_t* tspec; + + tspec = &erts_allctr_thr_spec[ERTS_ALC_A_FIXED_SIZE]; + ASSERT(tspec->enabled); + + for (i=0; i < tspec->size; i++) { + Allctr_t* allctr = tspec->allctr[i]; + for (j=0; j < ERTS_ALC_NO_FIXED_SIZES; ++j) { + allctr->fix[j].type_size += extra_block_size; + } + } + } + else +#endif + { + Allctr_t* allctr = erts_allctrs_info[ERTS_ALC_A_FIXED_SIZE].extra; + for (j=0; j < ERTS_ALC_NO_FIXED_SIZES; ++j) { + allctr->fix[j].type_size += extra_block_size; + } + } + } +} + +static ERTS_INLINE int +strategy_support_carrier_migration(struct au_init *auip) +{ + /* + * Currently only aoff, aoffcbf and aoffcaobf support carrier + * migration, i.e, type AOFIRSTFIT. + */ + return auip->atype == AOFIRSTFIT; +} + +static ERTS_INLINE void +adjust_carrier_migration_support(struct au_init *auip) +{ +#ifdef ERTS_SMP + if (auip->init.util.acul) { + auip->thr_spec = -1; /* Need thread preferred */ + + /* + * If strategy cannot handle carrier migration, + * default to a strategy that can... + */ + if (!strategy_support_carrier_migration(auip)) { + /* Default to aoffcbf */ + auip->atype = AOFIRSTFIT; + auip->init.aoff.flavor = AOFF_BF; + } + } +#else + auip->init.util.acul = 0; +#endif +} + void erts_alloc_init(int *argc, char **argv, ErtsAllocInitOpts *eaiop) { @@ -515,9 +591,9 @@ erts_alloc_init(int *argc, char **argv, ErtsAllocInitOpts *eaiop) = sizeof(Process); #if !HALFWORD_HEAP fix_type_sizes[ERTS_ALC_FIX_TYPE_IX(ERTS_ALC_T_MONITOR_SH)] - = ERTS_MONITOR_SH_SIZE; + = ERTS_MONITOR_SH_SIZE * sizeof(Uint); fix_type_sizes[ERTS_ALC_FIX_TYPE_IX(ERTS_ALC_T_NLINK_SH)] - = ERTS_LINK_SH_SIZE; + = ERTS_LINK_SH_SIZE * sizeof(Uint); #endif fix_type_sizes[ERTS_ALC_FIX_TYPE_IX(ERTS_ALC_T_DRV_EV_D_STATE)] = sizeof(ErtsDrvEventDataState); @@ -529,19 +605,33 @@ erts_alloc_init(int *argc, char **argv, ErtsAllocInitOpts *eaiop) fix_type_sizes[ERTS_ALC_FIX_TYPE_IX(ERTS_ALC_T_THR_Q_EL_SL)] = sizeof(ErtsThrQElement_t); #endif + fix_type_sizes[ERTS_ALC_FIX_TYPE_IX(ERTS_ALC_T_LL_PTIMER)] + = erts_timer_type_size(ERTS_ALC_T_LL_PTIMER); + fix_type_sizes[ERTS_ALC_FIX_TYPE_IX(ERTS_ALC_T_HL_PTIMER)] + = erts_timer_type_size(ERTS_ALC_T_HL_PTIMER); + fix_type_sizes[ERTS_ALC_FIX_TYPE_IX(ERTS_ALC_T_BIF_TIMER)] + = erts_timer_type_size(ERTS_ALC_T_BIF_TIMER); +#ifdef ERTS_BTM_ACCESSOR_SUPPORT + fix_type_sizes[ERTS_ALC_FIX_TYPE_IX(ERTS_ALC_T_ABIF_TIMER)] + = erts_timer_type_size(ERTS_ALC_T_ABIF_TIMER); +#endif + #ifdef HARD_DEBUG hdbg_init(); #endif - erts_have_sbmbc_alloc = 0; + lock_all_physical_memory = 0; + ncpu = eaiop->ncpu; if (ncpu < 1) ncpu = 1; + erts_tsd_key_create(&erts_allctr_prelock_tsd_key, + "erts_allctr_prelock_tsd_key"); + erts_sys_alloc_init(); erts_init_utils_mem(); - set_default_sbmbc_alloc_opts(&init.sbmbc_alloc); set_default_sl_alloc_opts(&init.sl_alloc); set_default_std_alloc_opts(&init.std_alloc); set_default_ll_alloc_opts(&init.ll_alloc); @@ -552,12 +642,26 @@ erts_alloc_init(int *argc, char **argv, ErtsAllocInitOpts *eaiop) set_default_driver_alloc_opts(&init.driver_alloc); set_default_fix_alloc_opts(&init.fix_alloc, fix_type_sizes); + set_default_test_alloc_opts(&init.test_alloc); if (argc && argv) handle_args(argc, argv, &init); + if (lock_all_physical_memory) { +#ifdef HAVE_MLOCKALL + errno = 0; + if (mlockall(MCL_CURRENT|MCL_FUTURE) != 0) { + int err = errno; + char *errstr = err ? strerror(err) : "unknown"; + erts_exit(1, "Failed to lock physical memory: %s (%d)\n", + errstr, err); + } +#else + erts_exit(1, "Failed to lock physical memory: Not supported\n"); +#endif + } + #ifndef ERTS_SMP - init.sbmbc_alloc.thr_spec = 0; init.sl_alloc.thr_spec = 0; init.std_alloc.thr_spec = 0; init.ll_alloc.thr_spec = 0; @@ -568,9 +672,21 @@ erts_alloc_init(int *argc, char **argv, ErtsAllocInitOpts *eaiop) init.fix_alloc.thr_spec = 0; #endif + /* Make adjustments for carrier migration support */ + init.temp_alloc.init.util.acul = 0; + adjust_carrier_migration_support(&init.sl_alloc); + adjust_carrier_migration_support(&init.std_alloc); + adjust_carrier_migration_support(&init.ll_alloc); + adjust_carrier_migration_support(&init.eheap_alloc); + adjust_carrier_migration_support(&init.binary_alloc); + adjust_carrier_migration_support(&init.ets_alloc); + adjust_carrier_migration_support(&init.driver_alloc); + adjust_carrier_migration_support(&init.fix_alloc); + if (init.erts_alloc_config) { /* Adjust flags that erts_alloc_config won't like */ - init.sbmbc_alloc.thr_spec = 0; + + /* No thread specific instances */ init.temp_alloc.thr_spec = 0; init.sl_alloc.thr_spec = 0; init.std_alloc.thr_spec = 0; @@ -579,7 +695,18 @@ erts_alloc_init(int *argc, char **argv, ErtsAllocInitOpts *eaiop) init.binary_alloc.thr_spec = 0; init.ets_alloc.thr_spec = 0; init.driver_alloc.thr_spec = 0; - init.fix_alloc.thr_spec = 0; + init.fix_alloc.thr_spec = 0; + + /* No carrier migration */ + init.temp_alloc.init.util.acul = 0; + init.sl_alloc.init.util.acul = 0; + init.std_alloc.init.util.acul = 0; + init.ll_alloc.init.util.acul = 0; + init.eheap_alloc.init.util.acul = 0; + init.binary_alloc.init.util.acul = 0; + init.ets_alloc.init.util.acul = 0; + init.driver_alloc.init.util.acul = 0; + init.fix_alloc.init.util.acul = 0; } #ifdef ERTS_SMP @@ -588,7 +715,6 @@ erts_alloc_init(int *argc, char **argv, ErtsAllocInitOpts *eaiop) init.temp_alloc.thr_spec = erts_no_schedulers; /* Others must use thread preferred interface */ - adjust_tpref(&init.sbmbc_alloc, erts_no_schedulers); adjust_tpref(&init.sl_alloc, erts_no_schedulers); adjust_tpref(&init.std_alloc, erts_no_schedulers); adjust_tpref(&init.ll_alloc, erts_no_schedulers); @@ -607,7 +733,6 @@ erts_alloc_init(int *argc, char **argv, ErtsAllocInitOpts *eaiop) * The following allocators cannot be run with afit strategy. * Make sure they don't... */ - refuse_af_strategy(&init.sbmbc_alloc); refuse_af_strategy(&init.sl_alloc); refuse_af_strategy(&init.std_alloc); refuse_af_strategy(&init.ll_alloc); @@ -627,6 +752,7 @@ erts_alloc_init(int *argc, char **argv, ErtsAllocInitOpts *eaiop) init.mseg.nos = erts_no_schedulers; erts_mseg_init(&init.mseg); #endif + erts_alcu_init(&init.alloc_util); erts_afalc_init(); erts_bfalc_init(); @@ -651,11 +777,6 @@ erts_alloc_init(int *argc, char **argv, ErtsAllocInitOpts *eaiop) #if HALFWORD_HEAP /* Init low memory variants by cloning */ - init.sbmbc_low_alloc = init.sbmbc_alloc; - init.sbmbc_low_alloc.init.util.name_prefix = "sbmbc_low_"; - init.sbmbc_low_alloc.init.util.alloc_no = ERTS_ALC_A_SBMBC_LOW; - init.sbmbc_low_alloc.init.util.low_mem = 1; - init.std_low_alloc = init.std_alloc; init.std_low_alloc.init.util.name_prefix = "std_low_"; init.std_low_alloc.init.util.alloc_no = ERTS_ALC_A_STANDARD_LOW; @@ -668,13 +789,11 @@ erts_alloc_init(int *argc, char **argv, ErtsAllocInitOpts *eaiop) init.ll_low_alloc.init.util.force = 1; init.ll_low_alloc.init.util.low_mem = 1; - set_au_allocator(ERTS_ALC_A_SBMBC_LOW, &init.sbmbc_low_alloc, ncpu); set_au_allocator(ERTS_ALC_A_STANDARD_LOW, &init.std_low_alloc, ncpu); set_au_allocator(ERTS_ALC_A_LONG_LIVED_LOW, &init.ll_low_alloc, ncpu); #endif /* HALFWORD */ set_au_allocator(ERTS_ALC_A_TEMPORARY, &init.temp_alloc, ncpu); - set_au_allocator(ERTS_ALC_A_SBMBC, &init.sbmbc_alloc, ncpu); set_au_allocator(ERTS_ALC_A_SHORT_LIVED, &init.sl_alloc, ncpu); set_au_allocator(ERTS_ALC_A_STANDARD, &init.std_alloc, ncpu); set_au_allocator(ERTS_ALC_A_LONG_LIVED, &init.ll_alloc, ncpu); @@ -683,16 +802,17 @@ erts_alloc_init(int *argc, char **argv, ErtsAllocInitOpts *eaiop) set_au_allocator(ERTS_ALC_A_ETS, &init.ets_alloc, ncpu); set_au_allocator(ERTS_ALC_A_DRIVER, &init.driver_alloc, ncpu); set_au_allocator(ERTS_ALC_A_FIXED_SIZE, &init.fix_alloc, ncpu); + set_au_allocator(ERTS_ALC_A_TEST, &init.test_alloc, ncpu); for (i = ERTS_ALC_A_MIN; i <= ERTS_ALC_A_MAX; i++) { if (!erts_allctrs[i].alloc) - erl_exit(ERTS_ABORT_EXIT, + erts_exit(ERTS_ABORT_EXIT, "Missing alloc function for %s\n", ERTS_ALC_A2AD(i)); if (!erts_allctrs[i].realloc) - erl_exit(ERTS_ABORT_EXIT, + erts_exit(ERTS_ABORT_EXIT, "Missing realloc function for %s\n", ERTS_ALC_A2AD(i)); if (!erts_allctrs[i].free) - erl_exit(ERTS_ABORT_EXIT, + erts_exit(ERTS_ABORT_EXIT, "Missing free function for %s\n", ERTS_ALC_A2AD(i)); } @@ -701,20 +821,6 @@ erts_alloc_init(int *argc, char **argv, ErtsAllocInitOpts *eaiop) erts_mtrace_init(init.instr.mtrace, init.instr.nodename); - /* sbmbc_alloc() needs to be started first */ - start_au_allocator(ERTS_ALC_A_SBMBC, - &init.sbmbc_alloc, - &sbmbc_alloc_state); -#if HALFWORD_HEAP - start_au_allocator(ERTS_ALC_A_SBMBC_LOW, - &init.sbmbc_low_alloc, - &sbmbc_low_alloc_state); - erts_have_sbmbc_alloc = (init.sbmbc_alloc.enable - && init.sbmbc_low_alloc.enable); -#else - erts_have_sbmbc_alloc = init.sbmbc_alloc.enable; -#endif - start_au_allocator(ERTS_ALC_A_TEMPORARY, &init.temp_alloc, &temp_alloc_state); @@ -758,6 +864,10 @@ erts_alloc_init(int *argc, char **argv, ErtsAllocInitOpts *eaiop) &init.fix_alloc, &fix_alloc_state); + start_au_allocator(ERTS_ALC_A_TEST, + &init.test_alloc, + &test_alloc_state); + erts_mtrace_install_wrapper_functions(); extra_block_size += erts_instr_init(init.instr.stat, init.instr.map); @@ -768,7 +878,7 @@ erts_alloc_init(int *argc, char **argv, ErtsAllocInitOpts *eaiop) #ifdef DEBUG extra_block_size += install_debug_functions(); #endif - + adjust_fix_alloc_sizes(extra_block_size); } void @@ -780,7 +890,7 @@ erts_alloc_late_init(void) static void * erts_realloc_fixed_size(ErtsAlcType_t type, void *extra, void *p, Uint size) { - erl_exit(ERTS_ABORT_EXIT, + erts_exit(ERTS_ABORT_EXIT, "Attempt to reallocate a block of the fixed size type %s\n", ERTS_ALC_T2TD(type)); } @@ -902,7 +1012,7 @@ start_au_allocator(ErtsAlcType_t alctr_n, * tspec->size) + ERTS_CACHE_LINE_SIZE - 1)); if (!states) - erl_exit(ERTS_ABORT_EXIT, + erts_exit(ERTS_ABORT_EXIT, "Failed to allocate allocator states for %salloc\n", init->init.util.name_prefix); tspec->allctr = (Allctr_t **) states; @@ -930,7 +1040,7 @@ start_au_allocator(ErtsAlcType_t alctr_n, (tot_fix_list_size + ERTS_CACHE_LINE_SIZE - 1)); if (!fix_lists) - erl_exit(ERTS_ABORT_EXIT, + erts_exit(ERTS_ABORT_EXIT, "Failed to allocate fix lists for %salloc\n", init->init.util.name_prefix); @@ -1004,7 +1114,7 @@ start_au_allocator(ErtsAlcType_t alctr_n, } if (!as) - erl_exit(ERTS_ABORT_EXIT, + erts_exit(ERTS_ABORT_EXIT, "Failed to start %salloc\n", init->init.util.name_prefix); ASSERT(as == (void *) as0); @@ -1104,6 +1214,26 @@ get_kb_value(char *param_end, char** argv, int* ip) return ((Uint) tmp)*1024; } +static UWord +get_mb_value(char *param_end, char** argv, int* ip) +{ + SWord tmp; + UWord max = ((~((UWord) 0))/(1024*1024)) + 1; + char *rest; + char *param = argv[*ip]+1; + char *value = get_value(param_end, argv, ip); + errno = 0; + tmp = (SWord) ErtsStrToSint(value, &rest, 10); + if (errno != 0 || rest == value || tmp < 0 || max < ((UWord) tmp)) + bad_value(param, param_end, value); + if (max == (UWord) tmp) + return ~((UWord) 0); + else + return ((UWord) tmp)*1024*1024; +} + + +#if 0 static Uint get_byte_value(char *param_end, char** argv, int* ip) { @@ -1117,6 +1247,7 @@ get_byte_value(char *param_end, char** argv, int* ip) bad_value(param, param_end, value); return (Uint) tmp; } +#endif static Uint get_amount_value(char *param_end, char** argv, int* ip) @@ -1132,17 +1263,57 @@ get_amount_value(char *param_end, char** argv, int* ip) return (Uint) tmp; } +static Uint +get_acul_value(struct au_init *auip, char *param_end, char** argv, int* ip) +{ + Sint tmp; + char *rest; + char *param = argv[*ip]+1; + char *value = get_value(param_end, argv, ip); + if (sys_strcmp(value, "de") == 0) { + switch (auip->init.util.alloc_no) { + case ERTS_ALC_A_LONG_LIVED: +#if HALFWORD_HEAP + case ERTS_ALC_A_LONG_LIVED_LOW: +#endif + return ERTS_ALC_DEFAULT_ENABLED_ACUL_LL_ALLOC; + case ERTS_ALC_A_EHEAP: + return ERTS_ALC_DEFAULT_ENABLED_ACUL_EHEAP_ALLOC; + default: + return ERTS_ALC_DEFAULT_ENABLED_ACUL; + } + } + errno = 0; + tmp = (Sint) ErtsStrToSint(value, &rest, 10); + if (errno != 0 || rest == value || tmp < 0 || 100 < tmp) + bad_value(param, param_end, value); + return (Uint) tmp; +} + static void handle_au_arg(struct au_init *auip, char* sub_param, char** argv, - int* ip) + int* ip, + int u_switch) { char *param = argv[*ip]+1; switch (sub_param[0]) { case 'a': - if(has_prefix("asbcst", sub_param)) { + if (has_prefix("acul", sub_param)) { + if (!auip->carrier_migration_allowed) { + if (!u_switch) + goto bad_switch; + else { + /* ignore */ + (void) get_acul_value(auip, sub_param + 4, argv, ip); + break; + } + } + auip->init.util.acul = get_acul_value(auip, sub_param + 4, argv, ip); + } + else if(has_prefix("asbcst", sub_param)) { auip->init.util.asbcst = get_kb_value(sub_param + 6, argv, ip); } else if(has_prefix("as", sub_param)) { @@ -1163,21 +1334,27 @@ handle_au_arg(struct au_init *auip, } else if (strcmp("aoff", alg) == 0) { auip->atype = AOFIRSTFIT; + auip->init.aoff.flavor = AOFF_AOFF; + } + else if (strcmp("aoffcbf", alg) == 0) { + auip->atype = AOFIRSTFIT; + auip->init.aoff.flavor = AOFF_BF; + } + else if (strcmp("aoffcaobf", alg) == 0) { + auip->atype = AOFIRSTFIT; + auip->init.aoff.flavor = AOFF_AOBF; } else { bad_value(param, sub_param + 1, alg); } + if (!strategy_support_carrier_migration(auip)) + auip->init.util.acul = 0; } else goto bad_switch; break; case 'e': auip->enable = get_bool_value(sub_param+1, argv, ip); -#if !HAVE_ERTS_SBMBC - if (auip->init.util.alloc_no == ERTS_ALC_A_SBMBC) { - auip->enable = 0; - } -#endif break; case 'l': if (has_prefix("lmbcs", sub_param)) { @@ -1237,18 +1414,6 @@ handle_au_arg(struct au_init *auip, if(has_prefix("sbct", sub_param)) { auip->init.util.sbct = get_kb_value(sub_param + 4, argv, ip); } - else if (has_prefix("sbmbcs", sub_param)) { -#if HAVE_ERTS_SBMBC - auip->init.util.sbmbcs = -#endif - get_byte_value(sub_param + 6, argv, ip); - } - else if (has_prefix("sbmbct", sub_param)) { -#if HAVE_ERTS_SBMBC - auip->init.util.sbmbct = -#endif - get_byte_value(sub_param + 6, argv, ip); - } else if (has_prefix("smbcs", sub_param)) { auip->default_.smbcs = 0; auip->init.util.smbcs = get_kb_value(sub_param + 5, argv, ip); @@ -1264,6 +1429,7 @@ handle_au_arg(struct au_init *auip, } else if (res == 0) { auip->thr_spec = 0; + auip->init.util.acul = 0; break; } goto bad_switch; @@ -1278,7 +1444,6 @@ static void handle_args(int *argc, char **argv, erts_alc_hndl_args_init_t *init) { struct au_init *aui[] = { - &init->sbmbc_alloc, &init->binary_alloc, &init->std_alloc, &init->ets_alloc, @@ -1288,6 +1453,7 @@ handle_args(int *argc, char **argv, erts_alc_hndl_args_init_t *init) &init->fix_alloc, &init->sl_alloc, &init->temp_alloc + /* test_alloc not affected by +Mea??? or +Mu??? */ }; int aui_sz = (int) sizeof(aui)/sizeof(aui[0]); char *arg; @@ -1305,25 +1471,22 @@ handle_args(int *argc, char **argv, erts_alc_hndl_args_init_t *init) case 'M': switch (argv[i][2]) { case 'B': - handle_au_arg(&init->binary_alloc, &argv[i][3], argv, &i); - break; - case 'C': - handle_au_arg(&init->sbmbc_alloc, &argv[i][3], argv, &i); + handle_au_arg(&init->binary_alloc, &argv[i][3], argv, &i, 0); break; case 'D': - handle_au_arg(&init->std_alloc, &argv[i][3], argv, &i); + handle_au_arg(&init->std_alloc, &argv[i][3], argv, &i, 0); break; case 'E': - handle_au_arg(&init->ets_alloc, &argv[i][3], argv, &i); + handle_au_arg(&init->ets_alloc, &argv[i][3], argv, &i, 0); break; case 'F': - handle_au_arg(&init->fix_alloc, &argv[i][3], argv, &i); + handle_au_arg(&init->fix_alloc, &argv[i][3], argv, &i, 0); break; case 'H': - handle_au_arg(&init->eheap_alloc, &argv[i][3], argv, &i); + handle_au_arg(&init->eheap_alloc, &argv[i][3], argv, &i, 0); break; case 'L': - handle_au_arg(&init->ll_alloc, &argv[i][3], argv, &i); + handle_au_arg(&init->ll_alloc, &argv[i][3], argv, &i, 0); break; case 'M': if (has_prefix("amcbf", argv[i]+3)) { @@ -1344,18 +1507,45 @@ handle_args(int *argc, char **argv, erts_alc_hndl_args_init_t *init) #endif get_amount_value(argv[i]+6, argv, &i); } + else if (has_prefix("scs", argv[i]+3)) { +#if HAVE_ERTS_MSEG + init->mseg.mmap.scs = +#endif + get_mb_value(argv[i]+6, argv, &i); + } + else if (has_prefix("sco", argv[i]+3)) { +#if HAVE_ERTS_MSEG + init->mseg.mmap.sco = +#endif + get_bool_value(argv[i]+6, argv, &i); + } + else if (has_prefix("scrpm", argv[i]+3)) { +#if HAVE_ERTS_MSEG + init->mseg.mmap.scrpm = +#endif + get_bool_value(argv[i]+8, argv, &i); + } + else if (has_prefix("scrfsd", argv[i]+3)) { +#if HAVE_ERTS_MSEG + init->mseg.mmap.scrfsd = +#endif + get_amount_value(argv[i]+9, argv, &i); + } else { bad_param(param, param+2); } break; case 'R': - handle_au_arg(&init->driver_alloc, &argv[i][3], argv, &i); + handle_au_arg(&init->driver_alloc, &argv[i][3], argv, &i, 0); break; case 'S': - handle_au_arg(&init->sl_alloc, &argv[i][3], argv, &i); + handle_au_arg(&init->sl_alloc, &argv[i][3], argv, &i, 0); break; case 'T': - handle_au_arg(&init->temp_alloc, &argv[i][3], argv, &i); + handle_au_arg(&init->temp_alloc, &argv[i][3], argv, &i, 0); + break; + case 'Z': + handle_au_arg(&init->test_alloc, &argv[i][3], argv, &i, 0); break; case 'Y': { /* sys_alloc */ if (has_prefix("tt", param+2)) { @@ -1414,9 +1604,6 @@ handle_args(int *argc, char **argv, erts_alc_hndl_args_init_t *init) else if (strcmp("max", arg) == 0) { for (a = 0; a < aui_sz; a++) aui[a]->enable = 1; -#if !HAVE_ERTS_SBMBC - init->sbmbc_alloc.enable = 0; -#endif } else if (strcmp("config", arg) == 0) { init->erts_alloc_config = 1; @@ -1444,8 +1631,8 @@ handle_args(int *argc, char **argv, erts_alc_hndl_args_init_t *init) for (a = 0; a < aui_sz; a++) { aui[a]->thr_spec = 0; + aui[a]->init.util.acul = 0; aui[a]->init.util.ramv = 0; - aui[a]->init.util.mmmbc = 10; aui[a]->init.util.lmbcs = 5*1024*1024; } } @@ -1485,6 +1672,19 @@ handle_args(int *argc, char **argv, erts_alc_hndl_args_init_t *init) bad_param(param, param+2); } break; + case 'l': + if (has_prefix("pm", param+2)) { + arg = get_value(argv[i]+5, argv, &i); + if (strcmp("all", arg) == 0) + lock_all_physical_memory = 1; + else if (strcmp("no", arg) == 0) + lock_all_physical_memory = 0; + else + bad_value(param, param+4, arg); + break; + } + bad_param(param, param+2); + break; case 'u': if (has_prefix("ycs", argv[i]+3)) { init->alloc_util.ycs @@ -1494,6 +1694,10 @@ handle_args(int *argc, char **argv, erts_alc_hndl_args_init_t *init) init->alloc_util.mmc = get_amount_value(argv[i]+6, argv, &i); } + else if (has_prefix("sac", argv[i]+3)) { + init->alloc_util.sac + = get_bool_value(argv[i]+6, argv, &i); + } else { int a; int start = i; @@ -1508,7 +1712,7 @@ handle_args(int *argc, char **argv, erts_alc_hndl_args_init_t *init) argv[start + 1] = val; i = start; } - handle_au_arg(aui[a], &argv[i][3], argv, &i); + handle_au_arg(aui[a], &argv[i][3], argv, &i, 1); } } break; @@ -1574,6 +1778,9 @@ erts_alloc_register_scheduler(void *vesdp) int ix = (int) esdp->no; int aix; +#ifdef ERTS_DIRTY_SCHEDULERS + ASSERT(!ERTS_SCHEDULER_IS_DIRTY(esdp)); +#endif for (aix = ERTS_ALC_A_MIN; aix <= ERTS_ALC_A_MAX; aix++) { ErtsAllocatorThrSpec_t *tspec = &erts_allctr_thr_spec[aix]; esdp->alloc_data.deallctr[aix] = NULL; @@ -1702,7 +1909,7 @@ erts_alc_fatal_error(int error, int func, ErtsAlcType_t n, ...) case ERTS_ALC_O_FREE: op_str = "free"; break; default: op_str = "UNKNOWN"; break; } - erl_exit(ERTS_ABORT_EXIT, + erts_exit(ERTS_ABORT_EXIT, "%s: %s operation not supported (memory type: \"%s\")\n", allctr_str, op_str, t_str); break; @@ -1716,18 +1923,18 @@ erts_alc_fatal_error(int error, int func, ErtsAlcType_t n, ...) va_start(argp, n); size = va_arg(argp, Uint); va_end(argp); - erl_exit(1, + erts_exit(ERTS_DUMP_EXIT, "%s: Cannot %s %lu bytes of memory (of type \"%s\").\n", allctr_str, op, size, t_str); break; } case ERTS_ALC_E_NOALLCTR: - erl_exit(ERTS_ABORT_EXIT, + erts_exit(ERTS_ABORT_EXIT, "erts_alloc: Unknown allocator type: %d\n", ERTS_ALC_T2A(ERTS_ALC_N2T(n))); break; default: - erl_exit(ERTS_ABORT_EXIT, "erts_alloc: Unknown error: %d\n", error); + erts_exit(ERTS_ABORT_EXIT, "erts_alloc: Unknown error: %d\n", error); break; } } @@ -2054,15 +2261,12 @@ erts_memory(int *print_to_p, void *print_to_arg, void *proc, Eterm earg) return am_badarg; } - /* All alloc_util allocators except sbmbc_alloc *have* to be enabled */ + /* All alloc_util allocators *have* to be enabled, except test_alloc */ for (ai = ERTS_ALC_A_MIN; ai <= ERTS_ALC_A_MAX; ai++) { switch (ai) { case ERTS_ALC_A_SYSTEM: - case ERTS_ALC_A_SBMBC: -#if HALFWORD_HEAP - case ERTS_ALC_A_SBMBC_LOW: -#endif + case ERTS_ALC_A_TEST: break; default: if (!erts_allctrs_info[ai].enabled @@ -2102,13 +2306,9 @@ erts_memory(int *print_to_p, void *print_to_arg, void *proc, Eterm earg) * Often not thread safe and usually never * contain any allocated memory. */ - case ERTS_ALC_A_SBMBC: - /* Included in other allocators */ -#if HALFWORD_HEAP - case ERTS_ALC_A_SBMBC_LOW: - /* Included in other allocators */ -#endif continue; + case ERTS_ALC_A_TEST: + continue; case ERTS_ALC_A_EHEAP: save = &size.processes; break; @@ -2142,7 +2342,6 @@ erts_memory(int *print_to_p, void *print_to_arg, void *proc, Eterm earg) if (want_tot_or_sys || want.processes || want.processes_used) { - int max_processes = erts_ptab_max(&erts_proc); UWord tmp; if (ERTS_MEM_NEED_ALL_ALCU) @@ -2152,7 +2351,7 @@ erts_memory(int *print_to_p, void *print_to_arg, void *proc, Eterm earg) fi, ERTS_ALC_NO_FIXED_SIZES); tmp = alcu_size(ERTS_ALC_A_EHEAP, NULL, 0); } - tmp += max_processes*sizeof(erts_smp_atomic_t); + tmp += erts_ptab_mem_size(&erts_proc); tmp += erts_bif_timer_memory_size(); tmp += erts_tot_link_lh_size(); @@ -2177,6 +2376,24 @@ erts_memory(int *print_to_p, void *print_to_arg, void *proc, Eterm earg) &size.processes_used, fi, ERTS_ALC_T_MSG_REF); + add_fix_values(&size.processes, + &size.processes_used, + fi, + ERTS_ALC_T_LL_PTIMER); + add_fix_values(&size.processes, + &size.processes_used, + fi, + ERTS_ALC_T_HL_PTIMER); + add_fix_values(&size.processes, + &size.processes_used, + fi, + ERTS_ALC_T_BIF_TIMER); +#ifdef ERTS_BTM_ACCESSOR_SUPPORT + add_fix_values(&size.processes, + &size.processes_used, + fi, + ERTS_ALC_T_ABIF_TIMER); +#endif } if (want.atom || want.atom_used) { @@ -2278,13 +2495,11 @@ struct aa_values { Eterm erts_allocated_areas(int *print_to_p, void *print_to_arg, void *proc) { -#define MAX_AA_VALUES (23) +#define MAX_AA_VALUES (24) struct aa_values values[MAX_AA_VALUES]; Eterm res = THE_NON_VALUE; int i, length; Uint reserved_atom_space, atom_space; - int max_processes = erts_ptab_max(&erts_proc); - int max_ports = erts_ptab_max(&erts_port); if (proc) { ERTS_SMP_LC_ASSERT(ERTS_PROC_LOCK_MAIN @@ -2315,8 +2530,8 @@ erts_allocated_areas(int *print_to_p, void *print_to_arg, void *proc) values[i].arity = 2; values[i].name = "static"; - values[i].ui[0] = - max_ports*sizeof(erts_smp_atomic_t) /* Port table */ + values[i].ui[0] = + sizeof(ErtsPTab)*2 /* proc & port tables */ + erts_timer_wheel_memory_size(); /* Timer wheel */ i++; @@ -2395,7 +2610,12 @@ erts_allocated_areas(int *print_to_p, void *print_to_arg, void *proc) values[i].arity = 2; values[i].name = "process_table"; - values[i].ui[0] = max_processes*sizeof(Process*); + values[i].ui[0] = erts_ptab_mem_size(&erts_proc); + i++; + + values[i].arity = 2; + values[i].name = "port_table"; + values[i].ui[0] = erts_ptab_mem_size(&erts_port); i++; values[i].arity = 2; @@ -2497,14 +2717,17 @@ erts_alloc_util_allocators(void *proc) /* * Currently all allocators except sys_alloc are * alloc_util allocators. + * Also hide test_alloc which is disabled by default + * and only intended for our own testing. */ - sz = ((ERTS_ALC_A_MAX + 1 - ERTS_ALC_A_MIN) - 1)*2; + sz = ((ERTS_ALC_A_MAX + 1 - ERTS_ALC_A_MIN) - 2)*2; ASSERT(sz > 0); hp = HAlloc((Process *) proc, sz); res = NIL; for (i = ERTS_ALC_A_MAX; i >= ERTS_ALC_A_MIN; i--) { switch (i) { case ERTS_ALC_A_SYSTEM: + case ERTS_ALC_A_TEST: break; default: { char *alc_str = (char *) ERTS_ALC_A2AD(i); @@ -2549,7 +2772,7 @@ erts_allocator_info(int to, void *arg) as = erts_allctr_thr_spec[a].allctr[ai]; } /* Binary alloc has its own thread safety... */ - erts_alcu_info(as, 0, &to, arg, NULL, NULL); + erts_alcu_info(as, 0, 0, &to, arg, NULL, NULL); } else { switch (a) { @@ -2575,6 +2798,7 @@ erts_allocator_info(int to, void *arg) #if HAVE_ERTS_MSEG { + struct erts_mmap_info_struct emis; #ifdef ERTS_SMP int max = (int) erts_no_schedulers; #else @@ -2585,6 +2809,8 @@ erts_allocator_info(int to, void *arg) erts_print(to, arg, "=allocator:mseg_alloc[%d]\n", i); erts_mseg_info(i, &to, arg, 0, NULL, NULL); } + erts_print(to, arg, "=allocator:mseg_alloc.erts_mmap\n"); + erts_mmap_info(&to, arg, NULL, NULL, &emis); } #endif @@ -2609,8 +2835,8 @@ erts_allocator_options(void *proc) #endif Uint sz, *szp, *hp, **hpp; Eterm res, features, settings; - Eterm atoms[ERTS_ALC_A_MAX-ERTS_ALC_A_MIN+5]; - Uint terms[ERTS_ALC_A_MAX-ERTS_ALC_A_MIN+5]; + Eterm atoms[ERTS_ALC_A_MAX-ERTS_ALC_A_MIN+7]; + Uint terms[ERTS_ALC_A_MAX-ERTS_ALC_A_MIN+7]; int a, length; SysAllocStat sas; Uint *endp = NULL; @@ -2708,6 +2934,11 @@ erts_allocator_options(void *proc) terms[length++] = erts_bld_2tup_list(hpp, szp, 3, o, v); } + atoms[length] = am_atom_put("lock_physical_memory", 20); + terms[length++] = (lock_all_physical_memory + ? am_atom_put("all", 3) + : am_atom_put("no", 2)); + settings = erts_bld_2tup_list(hpp, szp, length, atoms, terms); length = 0; @@ -2723,6 +2954,9 @@ erts_allocator_options(void *proc) if (use_mseg) terms[length++] = am_atom_put("mseg_alloc", 10); #endif +#if ERTS_HAVE_ERTS_SYS_ALIGNED_ALLOC + terms[length++] = am_atom_put("sys_aligned_alloc", 17); +#endif features = length ? erts_bld_list(hpp, szp, length, terms) : NIL; @@ -2808,9 +3042,11 @@ reply_alloc_info(void *vair) Uint sz, *szp; ErlOffHeap *ohp = NULL; ErlHeapFragment *bp = NULL; + struct erts_mmap_info_struct emis; int i; Eterm (*info_func)(Allctr_t *, int, + int, int *, void *, Uint **, @@ -2919,15 +3155,23 @@ reply_alloc_info(void *vair) ? NIL : erts_mseg_info(0, NULL, NULL, hpp != NULL, hpp, szp)); - ainfo = erts_bld_tuple(hpp, szp, 3, - alloc_atom, - make_small(0), - ainfo); + ainfo = erts_bld_tuple3(hpp, szp, + alloc_atom, + make_small(0), + ainfo); + + ai_list = erts_bld_cons(hpp, szp, + ainfo, ai_list); + ainfo = (air->only_sz ? NIL : erts_mmap_info(NULL, NULL, hpp, szp, &emis)); + ainfo = erts_bld_tuple3(hpp, szp, + alloc_atom, + erts_bld_atom(hpp,szp,"erts_mmap"), + ainfo); #else - ainfo = erts_bld_tuple(hpp, szp, 2, alloc_atom, - am_false); + ainfo = erts_bld_tuple2(hpp, szp, alloc_atom, + am_false); #endif - break; + break; default: alloc_atom = erts_bld_atom(hpp, szp, (char *) ERTS_ALC_A2AD(ai)); @@ -2939,13 +3183,13 @@ reply_alloc_info(void *vair) allctr = erts_allctr_thr_spec[ai].allctr[0]; else allctr = erts_allctrs_info[ai].extra; - ainfo = info_func(allctr, hpp != NULL, NULL, - NULL, hpp, szp); + ainfo = info_func(allctr, air->internal, hpp != NULL, + NULL, NULL, hpp, szp); ainfo = erts_bld_tuple(hpp, szp, 3, alloc_atom, make_small(0), ainfo); } else { - erl_exit(ERTS_ABORT_EXIT, "%s:%d: internal error\n", + erts_exit(ERTS_ABORT_EXIT, "%s:%d: internal error\n", __FILE__, __LINE__); } } @@ -2975,7 +3219,7 @@ reply_alloc_info(void *vair) alloc_atom = erts_bld_atom(hpp, szp, (char *) ERTS_ALC_A2AD(ai)); allctr = erts_allctr_thr_spec[ai].allctr[sched_id]; - ainfo = info_func(allctr, hpp != NULL, NULL, + ainfo = info_func(allctr, air->internal, hpp != NULL, NULL, NULL, hpp, szp); ai_list = erts_bld_cons(hpp, szp, erts_bld_tuple( @@ -3011,17 +3255,13 @@ reply_alloc_info(void *vair) HRelease(rp, hp_end, hp); } - erts_queue_message(rp, &rp_locks, bp, msg, NIL -#ifdef USE_VM_PROBES - , NIL -#endif - ); + erts_queue_message(rp, &rp_locks, bp, msg, NIL); if (air->req_sched == sched_id) rp_locks &= ~ERTS_PROC_LOCK_MAIN; erts_smp_proc_unlock(rp, rp_locks); - erts_smp_proc_dec_refc(rp); + erts_proc_dec_refc(rp); if (erts_smp_atomic32_dec_read_nob(&air->refc) == 0) aireq_free(air); @@ -3031,7 +3271,8 @@ int erts_request_alloc_info(struct process *c_p, Eterm ref, Eterm allocs, - int only_sz) + int only_sz, + int internal) { ErtsAllocInfoReq *air = aireq_alloc(); Eterm req_ai[ERTS_ALC_A_MAX+1+2] = {0}; @@ -3043,6 +3284,8 @@ erts_request_alloc_info(struct process *c_p, air->only_sz = only_sz; + air->internal = internal; + air->proc = c_p; if (is_not_internal_ref(ref)) @@ -3092,7 +3335,7 @@ erts_request_alloc_info(struct process *c_p, erts_smp_atomic32_init_nob(&air->refc, (erts_aint32_t) erts_no_schedulers); - erts_smp_proc_add_refc(c_p, (Sint32) erts_no_schedulers); + erts_proc_add_refc(c_p, (Sint) erts_no_schedulers); #ifdef ERTS_SMP if (erts_no_schedulers > 1) @@ -3107,6 +3350,55 @@ erts_request_alloc_info(struct process *c_p, return 1; } +/* + * The allocator wrapper prelocking stuff below is about the locking order. + * It only affects wrappers (erl_mtrace.c and erl_instrument.c) that keep locks + * during alloc/realloc/free. + * + * Some query functions in erl_alloc_util.c lock the allocator mutex and then + * use erts_printf that in turn may call the sys allocator through the wrappers. + * To avoid breaking locking order these query functions first "pre-locks" all + * allocator wrappers. + */ +ErtsAllocatorWrapper_t *erts_allctr_wrappers; +int erts_allctr_wrapper_prelocked = 0; +erts_tsd_key_t erts_allctr_prelock_tsd_key; + +void erts_allctr_wrapper_prelock_init(ErtsAllocatorWrapper_t* wrapper) +{ + ASSERT(wrapper->lock && wrapper->unlock); + wrapper->next = erts_allctr_wrappers; + erts_allctr_wrappers = wrapper; +} + +void erts_allctr_wrapper_pre_lock(void) +{ + if (erts_allctr_wrappers) { + ErtsAllocatorWrapper_t* wrapper = erts_allctr_wrappers; + for ( ; wrapper; wrapper = wrapper->next) { + wrapper->lock(); + } + ASSERT(!erts_allctr_wrapper_prelocked); + erts_allctr_wrapper_prelocked = 1; + erts_tsd_set(erts_allctr_prelock_tsd_key, (void*)1); + } +} + +void erts_allctr_wrapper_pre_unlock(void) +{ + if (erts_allctr_wrappers) { + ErtsAllocatorWrapper_t* wrapper = erts_allctr_wrappers; + + erts_allctr_wrapper_prelocked = 0; + erts_tsd_set(erts_allctr_prelock_tsd_key, (void*)0); + for ( ; wrapper; wrapper = wrapper->next) { + wrapper->unlock(); + } + } +} + + + /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *\ * Deprecated functions * * * @@ -3134,12 +3426,9 @@ void *safe_realloc(void *ptr, Uint sz) * Keep alloc_SUITE_data/allocator_test.h updated if changes are made * * to erts_alc_test() * \* */ -#define ERTS_ALC_TEST_ABORT erl_exit(ERTS_ABORT_EXIT, "%s:%d: Internal error\n") +#define ERTS_ALC_TEST_ABORT erts_exit(ERTS_ABORT_EXIT, "%s:%d: Internal error\n") -UWord erts_alc_test(UWord op, - UWord a1, - UWord a2, - UWord a3) +UWord erts_alc_test(UWord op, UWord a1, UWord a2, UWord a3) { switch (op >> 8) { case 0x0: return erts_alcu_test(op, a1, a2); @@ -3191,14 +3480,13 @@ UWord erts_alc_test(UWord op, init.atype = GOODFIT; init.init.util.name_prefix = (char *) a1; init.init.util.ts = a2 ? 1 : 0; - init.init.util.sbmbct = 0; if ((char **) a3) { char **argv = (char **) a3; int i = 0; while (argv[i]) { if (argv[i][0] == '-' && argv[i][1] == 't') - handle_au_arg(&init, &argv[i][2], argv, &i); + handle_au_arg(&init, &argv[i][2], argv, &i, 0); else return (UWord) NULL; i++; @@ -3318,6 +3606,46 @@ UWord erts_alc_test(UWord op, ERTS_ALC_TEST_ABORT; break; #endif /* #ifdef USE_THREADS */ +#ifdef ERTS_SMP + case 0xf13: return (UWord) 1; +#else + case 0xf13: return (UWord) 0; +#endif + case 0xf14: return (UWord) erts_alloc(ERTS_ALC_T_TEST, (Uint)a1); + + case 0xf15: erts_free(ERTS_ALC_T_TEST, (void*)a1); return 0; + + case 0xf16: { + Uint extra_hdr_sz = UNIT_CEILING((Uint)a1); + ErtsAllocatorThrSpec_t* ts = &erts_allctr_thr_spec[ERTS_ALC_A_TEST]; + Uint offset = ts->allctr[0]->mbc_header_size; + void* orig_creating_mbc = ts->allctr[0]->creating_mbc; + void* orig_destroying_mbc = ts->allctr[0]->destroying_mbc; + void* new_creating_mbc = *(void**)a2; /* inout arg */ + void* new_destroying_mbc = *(void**)a3; /* inout arg */ + int i; + + for (i=0; i < ts->size; i++) { + Allctr_t* ap = ts->allctr[i]; + if (ap->mbc_header_size != offset + || ap->creating_mbc != orig_creating_mbc + || ap->destroying_mbc != orig_destroying_mbc + || ap->mbc_list.first != NULL) + return -1; + } + for (i=0; i < ts->size; i++) { + ts->allctr[i]->mbc_header_size += extra_hdr_sz; + ts->allctr[i]->creating_mbc = new_creating_mbc; + ts->allctr[i]->destroying_mbc = new_destroying_mbc; + } + *(void**)a2 = orig_creating_mbc; + *(void**)a3 = orig_destroying_mbc; + return offset; + } + case 0xf17: { + ErtsAllocatorThrSpec_t* ts = &erts_allctr_thr_spec[ERTS_ALC_A_TEST]; + return ts->allctr[0]->largest_mbc_size; + } default: break; } @@ -3560,7 +3888,7 @@ check_memory_fence(void *ptr, Uint *size, ErtsAlcType_t n, int func) found_type = GET_TYPE_OF_PATTERN(pre_pattern); if (pre_pattern != MK_PATTERN(n)) { if ((FIXED_FENCE_PATTERN_MASK & pre_pattern) != FIXED_FENCE_PATTERN) - erl_exit(ERTS_ABORT_EXIT, + erts_exit(ERTS_ABORT_EXIT, "ERROR: Fence at beginning of memory block (p=0x%u) " "clobbered.\n", (UWord) ptr); @@ -3577,12 +3905,12 @@ check_memory_fence(void *ptr, Uint *size, ErtsAlcType_t n, int func) char *op_str; if ((FIXED_FENCE_PATTERN_MASK & post_pattern) != FIXED_FENCE_PATTERN) - erl_exit(ERTS_ABORT_EXIT, + erts_exit(ERTS_ABORT_EXIT, "ERROR: Fence at end of memory block (p=0x%u, sz=%u) " "clobbered.\n", (UWord) ptr, (UWord) sz); if (found_type != GET_TYPE_OF_PATTERN(post_pattern)) - erl_exit(ERTS_ABORT_EXIT, + erts_exit(ERTS_ABORT_EXIT, "ERROR: Fence around memory block (p=0x%u, sz=%u) " "clobbered.\n", (UWord) ptr, (UWord) sz); @@ -3605,7 +3933,7 @@ check_memory_fence(void *ptr, Uint *size, ErtsAlcType_t n, int func) default: op_str = "???"; break; } - erl_exit(ERTS_ABORT_EXIT, + erts_exit(ERTS_ABORT_EXIT, "ERROR: Memory block (p=0x%u, sz=%u) allocated as type \"%s\"," " but %s as type \"%s\".\n", (UWord) ptr, (UWord) sz, ftype, op_str, otype); @@ -3717,7 +4045,7 @@ static Uint install_debug_functions(void) { int i; - ASSERT(sizeof(erts_allctrs) == sizeof(real_allctrs)); + ERTS_CT_ASSERT(sizeof(erts_allctrs) == sizeof(real_allctrs)); sys_memcpy((void *)real_allctrs,(void *)erts_allctrs,sizeof(erts_allctrs)); diff --git a/erts/emulator/beam/erl_alloc.h b/erts/emulator/beam/erl_alloc.h index ba5ec9c367..f540bae20d 100644 --- a/erts/emulator/beam/erl_alloc.h +++ b/erts/emulator/beam/erl_alloc.h @@ -1,18 +1,19 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2002-2012. All Rights Reserved. + * Copyright Ericsson AB 2002-2013. All Rights Reserved. * - * The contents of this file are subject to the Erlang Public License, - * Version 1.1, (the "License"); you may not use this file except in - * compliance with the License. You should have received a copy of the - * Erlang Public License along with this software. If not, it can be - * retrieved online at http://www.erlang.org/. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Software distributed under the License is distributed on an "AS IS" - * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See - * the License for the specific language governing rights and limitations - * under the License. + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. * * %CopyrightEnd% */ @@ -54,6 +55,16 @@ void erts_sys_alloc_init(void); void *erts_sys_alloc(ErtsAlcType_t, void *, Uint); void *erts_sys_realloc(ErtsAlcType_t, void *, void *, Uint); void erts_sys_free(ErtsAlcType_t, void *, void *); +#if ERTS_HAVE_ERTS_SYS_ALIGNED_ALLOC +/* + * Note 'alignment' must remain the same in calls to + * 'erts_sys_aligned_realloc()' and 'erts_sys_aligned_free()' + * as in the initial call to 'erts_sys_aligned_alloc()'. + */ +void *erts_sys_aligned_alloc(UWord alignment, UWord size); +void *erts_sys_aligned_realloc(UWord alignment, void *ptr, UWord size, UWord old_size); +void erts_sys_aligned_free(UWord alignment, void *ptr); +#endif Eterm erts_memory(int *, void *, void *, Eterm); Eterm erts_allocated_areas(int *, void *, void *); @@ -65,7 +76,7 @@ Eterm erts_allocator_options(void *proc); struct process; int erts_request_alloc_info(struct process *c_p, Eterm ref, Eterm allocs, - int only_sz); + int only_sz, int internal); #define ERTS_ALLOC_INIT_DEF_OPTS_INITER {0} typedef struct { @@ -100,14 +111,6 @@ UWord erts_alc_test(UWord, #define ERTS_ALC_MIN_LONG_LIVED_TIME (10*60*1000) -#if HALFWORD_HEAP -#define ERTS_IS_SBMBC_ALLOCATOR_NO__(NO) \ - ((NO) == ERTS_ALC_A_SBMBC || (NO) == ERTS_ALC_A_SBMBC_LOW) -#else -#define ERTS_IS_SBMBC_ALLOCATOR_NO__(NO) \ - ((NO) == ERTS_ALC_A_SBMBC) -#endif - typedef struct { int alloc_util; int enabled; @@ -135,6 +138,18 @@ typedef struct { extern ErtsAllocatorThrSpec_t erts_allctr_thr_spec[ERTS_ALC_A_MAX+1]; +typedef struct ErtsAllocatorWrapper_t_ { + void (*lock)(void); + void (*unlock)(void); + struct ErtsAllocatorWrapper_t_* next; +}ErtsAllocatorWrapper_t; +ErtsAllocatorWrapper_t *erts_allctr_wrappers; +extern int erts_allctr_wrapper_prelocked; +extern erts_tsd_key_t erts_allctr_prelock_tsd_key; +void erts_allctr_wrapper_prelock_init(ErtsAllocatorWrapper_t* wrapper); +void erts_allctr_wrapper_pre_lock(void); +void erts_allctr_wrapper_pre_unlock(void); + void erts_alloc_register_scheduler(void *vesdp); #ifdef ERTS_SMP void erts_alloc_scheduler_handle_delayed_dealloc(void *vesdp, @@ -188,14 +203,15 @@ void *erts_realloc(ErtsAlcType_t type, void *ptr, Uint size); void erts_free(ErtsAlcType_t type, void *ptr); void *erts_alloc_fnf(ErtsAlcType_t type, Uint size); void *erts_realloc_fnf(ErtsAlcType_t type, void *ptr, Uint size); +int erts_is_allctr_wrapper_prelocked(void); #endif /* #if !ERTS_ALC_DO_INLINE */ void *erts_alloc_permanent_cache_aligned(ErtsAlcType_t type, Uint size); #ifndef ERTS_CACHE_LINE_SIZE -/* Assume a cache line size of 64 bytes */ -# define ERTS_CACHE_LINE_SIZE ((UWord) 64) +/* Assumed cache line size */ +# define ERTS_CACHE_LINE_SIZE ((UWord) ASSUMED_CACHE_LINE_SIZE) # define ERTS_CACHE_LINE_MASK (ERTS_CACHE_LINE_SIZE - 1) #endif @@ -258,6 +274,13 @@ void *erts_realloc_fnf(ErtsAlcType_t type, void *ptr, Uint size) size); } +ERTS_ALC_INLINE +int erts_is_allctr_wrapper_prelocked(void) +{ + return erts_allctr_wrapper_prelocked /* locked */ + && !!erts_tsd_get(erts_allctr_prelock_tsd_key); /* by me */ +} + #endif /* #if ERTS_ALC_DO_INLINE || defined(ERTS_ALC_INTERNAL__) */ #define ERTS_ALC_GET_THR_IX() ((int) erts_get_scheduler_id()) @@ -470,7 +493,7 @@ static TYPE * \ NAME##_alloc(void) \ { \ ErtsSchedulerData *esdp = erts_get_scheduler_data(); \ - if (!esdp) \ + if (!esdp || ERTS_SCHEDULER_IS_DIRTY(esdp)) \ return NULL; \ return (TYPE *) erts_sspa_alloc(sspa_data_##NAME##__, \ (int) esdp->no - 1); \ diff --git a/erts/emulator/beam/erl_alloc.types b/erts/emulator/beam/erl_alloc.types index 5e3615ccc2..7738531142 100644 --- a/erts/emulator/beam/erl_alloc.types +++ b/erts/emulator/beam/erl_alloc.types @@ -1,18 +1,19 @@ # # %CopyrightBegin% # -# Copyright Ericsson AB 2003-2013. All Rights Reserved. +# Copyright Ericsson AB 2003-2014. All Rights Reserved. # -# The contents of this file are subject to the Erlang Public License, -# Version 1.1, (the "License"); you may not use this file except in -# compliance with the License. You should have received a copy of the -# Erlang Public License along with this software. If not, it can be -# retrieved online at http://www.erlang.org/. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# Software distributed under the License is distributed on an "AS IS" -# basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See -# the License for the specific language governing rights and limitations -# under the License. +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. # # %CopyrightEnd% # @@ -76,11 +77,6 @@ allocator SYSTEM true sys_alloc -allocator SBMBC true sbmbc_alloc -+if halfword -allocator SBMBC_LOW true sbmbc_low_alloc -+endif - +if smp allocator TEMPORARY true temp_alloc @@ -116,7 +112,7 @@ allocator STANDARD_LOW false std_low_alloc allocator BINARY true binary_alloc allocator DRIVER true driver_alloc - +allocator TEST true test_alloc # --- Class declarations ----------------------------------------------------- # @@ -146,7 +142,6 @@ class SYSTEM system_data # # <TYPE> <ALLOCATOR> <CLASS> <DESCRIPTION> -type SBMBC SBMBC SYSTEM small_block_mbc type PROC FIXED_SIZE PROCESSES proc type PORT DRIVER SYSTEM port type ATOM LONG_LIVED ATOM atom_entry @@ -156,6 +151,7 @@ type LINK_LH STANDARD PROCESSES link_lh type SUSPEND_MON STANDARD PROCESSES suspend_monitor type PEND_SUSPEND SHORT_LIVED PROCESSES pending_suspend type PROC_LIST SHORT_LIVED PROCESSES proc_list +type SAVED_ESTACK SHORT_LIVED PROCESSES saved_estack type FUN_ENTRY LONG_LIVED CODE fun_entry type ATOM_TXT LONG_LIVED ATOM atom_text type BEAM_REGISTER EHEAP PROCESSES beam_register @@ -168,9 +164,13 @@ type MSG_ROOTS TEMPORARY PROCESSES msg_roots type ROOTSET TEMPORARY PROCESSES root_set type LOADER_TMP TEMPORARY CODE loader_tmp type PREPARED_CODE SHORT_LIVED CODE prepared_code -type BIF_TIMER_TABLE LONG_LIVED SYSTEM bif_timer_table -type SL_BIF_TIMER SHORT_LIVED PROCESSES bif_timer_sl -type LL_BIF_TIMER STANDARD PROCESSES bif_timer_ll +type TIMER_SERVICE LONG_LIVED SYSTEM timer_service +type LL_PTIMER FIXED_SIZE PROCESSES ll_ptimer +type HL_PTIMER FIXED_SIZE PROCESSES hl_ptimer +type BIF_TIMER FIXED_SIZE PROCESSES bif_timer +# type ABIF_TIMER FIXED_SIZE PROCESSES accessor_bif_timer +type TIMER_REQUEST SHORT_LIVED PROCESSES timer_request +type BTM_YIELD_STATE SHORT_LIVED PROCESSES btm_yield_state type REG_TABLE STANDARD SYSTEM reg_tab type FUN_TABLE STANDARD CODE fun_tab type DIST_TABLE STANDARD SYSTEM dist_tab @@ -272,6 +272,10 @@ type CODE_IX_LOCK_Q SHORT_LIVED SYSTEM code_ix_lock_q type PROC_INTERVAL LONG_LIVED SYSTEM process_interval type BUSY_CALLER_TAB SHORT_LIVED SYSTEM busy_caller_table type BUSY_CALLER SHORT_LIVED SYSTEM busy_caller +type PROC_SYS_TSK SHORT_LIVED PROCESSES proc_sys_task +type PROC_SYS_TSK_QS SHORT_LIVED PROCESSES proc_sys_task_queues +type NEW_TIME_OFFSET SHORT_LIVED SYSTEM new_time_offset +type IOB_REQ SHORT_LIVED SYSTEM io_bytes_request +if threads_no_smp # Need thread safe allocs, but std_alloc and fix_alloc are not; @@ -326,8 +330,6 @@ type ACTIVE_PROCS STANDARD PROCESSES active_procs +endif +if smp -type SL_PTIMER SHORT_LIVED SYSTEM ptimer_sl -type LL_PTIMER STANDARD SYSTEM ptimer_ll type SYS_MSG_Q SHORT_LIVED PROCESSES system_messages_queue type FP_EXCEPTION LONG_LIVED SYSTEM fp_exception type LL_MPATHS LONG_LIVED SYSTEM ll_migration_paths @@ -350,7 +352,6 @@ type SSB SHORT_LIVED PROCESSES ssb +if halfword -type SBMBC_LOW SBMBC_LOW SYSTEM small_block_mbc_low type DDLL_PROCESS STANDARD_LOW SYSTEM ddll_processes type MONITOR_LH STANDARD_LOW PROCESSES monitor_lh type NLINK_LH STANDARD_LOW PROCESSES nlink_lh @@ -360,11 +361,15 @@ type DB_MS_PSDO_PROC LONG_LIVED_LOW ETS db_match_pseudo_proc type SCHDLR_DATA LONG_LIVED_LOW SYSTEM scheduler_data type LL_TEMP_TERM LONG_LIVED_LOW SYSTEM ll_temp_term +type NIF_TRAP_EXPORT STANDARD_LOW CODE nif_trap_export_entry type EXPORT LONG_LIVED_LOW CODE export_entry type MONITOR_SH STANDARD_LOW PROCESSES monitor_sh type NLINK_SH STANDARD_LOW PROCESSES nlink_sh type AINFO_REQ STANDARD_LOW SYSTEM alloc_info_request type SCHED_WTIME_REQ STANDARD_LOW SYSTEM sched_wall_time_request +type GC_INFO_REQ STANDARD_LOW SYSTEM gc_info_request +type PORT_DATA_HEAP STANDARD_LOW SYSTEM port_data_heap +type SYS_CHECK_REQ STANDARD_LOW SYSTEM system_check_request +else # "fullword" @@ -377,11 +382,15 @@ type DB_MS_PSDO_PROC LONG_LIVED ETS db_match_pseudo_proc type SCHDLR_DATA LONG_LIVED SYSTEM scheduler_data type LL_TEMP_TERM LONG_LIVED SYSTEM ll_temp_term +type NIF_TRAP_EXPORT STANDARD CODE nif_trap_export_entry type EXPORT LONG_LIVED CODE export_entry type MONITOR_SH FIXED_SIZE PROCESSES monitor_sh type NLINK_SH FIXED_SIZE PROCESSES nlink_sh type AINFO_REQ SHORT_LIVED SYSTEM alloc_info_request type SCHED_WTIME_REQ SHORT_LIVED SYSTEM sched_wall_time_request +type GC_INFO_REQ SHORT_LIVED SYSTEM gc_info_request +type PORT_DATA_HEAP STANDARD SYSTEM port_data_heap +type SYS_CHECK_REQ SHORT_LIVED SYSTEM system_check_request +endif @@ -396,11 +405,13 @@ type DRV_EV_STATE LONG_LIVED SYSTEM driver_event_state type DRV_EV_D_STATE FIXED_SIZE SYSTEM driver_event_data_state type DRV_SEL_D_STATE FIXED_SIZE SYSTEM driver_select_data_state type FD_LIST SHORT_LIVED SYSTEM fd_list +type ACTIVE_FD_ARR SHORT_LIVED SYSTEM active_fd_array type POLLSET LONG_LIVED SYSTEM pollset type POLLSET_UPDREQ SHORT_LIVED SYSTEM pollset_update_req type POLL_FDS LONG_LIVED SYSTEM poll_fds type POLL_RES_EVS LONG_LIVED SYSTEM poll_result_events type FD_STATUS LONG_LIVED SYSTEM fd_status +type SELECT_FDS LONG_LIVED SYSTEM select_fds +if unix @@ -411,9 +422,31 @@ type CS_PROG_PATH LONG_LIVED SYSTEM cs_prog_path type ENVIRONMENT TEMPORARY SYSTEM environment type PUTENV_STR SYSTEM SYSTEM putenv_string type PRT_REP_EXIT STANDARD SYSTEM port_report_exit +type SYS_BLOCKING STANDARD SYSTEM sys_blocking + ++if smp +type SYS_WRITE_BUF TEMPORARY SYSTEM sys_write_buf ++else +type SYS_WRITE_BUF BINARY SYSTEM sys_write_buf ++endif + ++endif + ++if ose + +type SYS_READ_BUF TEMPORARY SYSTEM sys_read_buf +type FD_TAB LONG_LIVED SYSTEM fd_tab +type FD_ENTRY_BUF STANDARD SYSTEM fd_entry_buf +type FD_SIG_LIST SHORT_LIVED SYSTEM fd_sig_list +type DRV_EV STANDARD SYSTEM driver_event +type CS_PROG_PATH LONG_LIVED SYSTEM cs_prog_path +type ENVIRONMENT TEMPORARY SYSTEM environment +type PUTENV_STR SYSTEM SYSTEM putenv_string +type PRT_REP_EXIT STANDARD SYSTEM port_report_exit +endif + +if win32 type DRV_DATA_BUF SYSTEM SYSTEM drv_data_buf @@ -425,4 +458,7 @@ type CON_VPRINTF_BUF TEMPORARY SYSTEM con_vprintf_buf +endif +# This type should only be used for test +type TEST TEST SYSTEM testing + # ---------------------------------------------------------------------------- diff --git a/erts/emulator/beam/erl_alloc_util.c b/erts/emulator/beam/erl_alloc_util.c index ac7f420708..d9eaa7b32c 100644 --- a/erts/emulator/beam/erl_alloc_util.c +++ b/erts/emulator/beam/erl_alloc_util.c @@ -3,16 +3,17 @@ * * Copyright Ericsson AB 2002-2013. All Rights Reserved. * - * The contents of this file are subject to the Erlang Public License, - * Version 1.1, (the "License"); you may not use this file except in - * compliance with the License. You should have received a copy of the - * Erlang Public License along with this software. If not, it can be - * retrieved online at http://www.erlang.org/. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Software distributed under the License is distributed on an "AS IS" - * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See - * the License for the specific language governing rights and limitations - * under the License. + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. * * %CopyrightEnd% */ @@ -71,31 +72,22 @@ #define ALLOC_ZERO_EQ_NULL 0 +#ifndef ERTS_MSEG_FLG_2POW +# define ERTS_MSEG_FLG_2POW 0 +#endif +#ifndef ERTS_MSEG_FLG_NONE +# define ERTS_MSEG_FLG_NONE 0 +#endif + static int atoms_initialized = 0; static int initialized = 0; -int erts_have_sbmbc_alloc; - -#if HAVE_ERTS_MSEG - -#define MSEG_UNIT_SHIFT MSEG_ALIGN_BITS -#define MSEG_UNIT_SZ (1 << MSEG_UNIT_SHIFT) -#define MSEG_UNIT_MASK ((~(UWord)0) << MSEG_UNIT_SHIFT) - -#define MSEG_UNIT_FLOOR(X) ((X) & MSEG_UNIT_MASK) -#define MSEG_UNIT_CEILING(X) MSEG_UNIT_FLOOR((X) + ~MSEG_UNIT_MASK) - -#endif - #define INV_SYS_ALLOC_CARRIER_MASK ((UWord) (sys_alloc_carrier_size - 1)) #define SYS_ALLOC_CARRIER_MASK (~INV_SYS_ALLOC_CARRIER_MASK) #define SYS_ALLOC_CARRIER_FLOOR(X) ((X) & SYS_ALLOC_CARRIER_MASK) #define SYS_ALLOC_CARRIER_CEILING(X) \ SYS_ALLOC_CARRIER_FLOOR((X) + INV_SYS_ALLOC_CARRIER_MASK) -#undef ASSERT -#define ASSERT ASSERT_EXPR - #if 0 /* Can be useful for debugging */ #define MBC_REALLOC_ALWAYS_MOVES @@ -107,16 +99,16 @@ static Uint sys_alloc_carrier_size; #if HAVE_ERTS_MSEG static Uint max_mseg_carriers; #endif +static int allow_sys_alloc_carriers; #define ONE_GIGA (1000000000) -#define INC_CC(CC) ((CC).no == ONE_GIGA - 1 \ - ? ((CC).giga_no++, (CC).no = 0) \ - : (CC).no++) +#define ERTS_ALC_CC_GIGA_VAL(CC) ((CC) / ONE_GIGA) +#define ERTS_ALC_CC_VAL(CC) ((CC) % ONE_GIGA) -#define DEC_CC(CC) ((CC).no == 0 \ - ? ((CC).giga_no--, (CC).no = ONE_GIGA - 1) \ - : (CC).no--) +#define INC_CC(CC) ((CC)++) + +#define DEC_CC(CC) ((CC)--) /* Multi block carrier (MBC) memory layout in R16: @@ -173,14 +165,6 @@ MBC after deallocating first block: #define SET_BLK_SZ_FTR(B, SZ) \ (((FreeBlkFtr_t *) (((char *) (B)) + (SZ)))[-1] = (SZ)) -#define THIS_FREE_BLK_HDR_FLG (((UWord) 1) << 0) -#define PREV_FREE_BLK_HDR_FLG (((UWord) 1) << 1) -#define LAST_BLK_HDR_FLG (((UWord) 1) << 2) - -/* Special flag combo for (allocated) SBC blocks -*/ -#define SBC_BLK_HDR_FLG (THIS_FREE_BLK_HDR_FLG | PREV_FREE_BLK_HDR_FLG | LAST_BLK_HDR_FLG) - #define SET_MBC_ABLK_SZ(B, SZ) \ (ASSERT(((SZ) & FLG_MASK) == 0), \ (B)->bhdr = (((B)->bhdr) & ~MBC_ABLK_SZ_MASK) | (SZ)) @@ -208,9 +192,9 @@ MBC after deallocating first block: #if MBC_ABLK_OFFSET_BITS -# define MBC_SZ_MAX_LIMIT ((((UWord)1 << MBC_ABLK_OFFSET_BITS) - 1) << MSEG_ALIGN_BITS) +# define MBC_SZ_MAX_LIMIT ((((UWord)1 << MBC_ABLK_OFFSET_BITS) - 1) << ERTS_SUPER_ALIGN_BITS) -# define BLK_CARRIER_OFFSET(B, C) (((char*)(B) - (char*)(C)) >> MSEG_UNIT_SHIFT) +# define BLK_CARRIER_OFFSET(B, C) (((char*)(B) - (char*)(C)) >> ERTS_SACRR_UNIT_SHIFT) # define SET_MBC_ABLK_HDR(B, Sz, F, C) \ (ASSERT(((Sz) & ~MBC_ABLK_SZ_MASK) == 0), \ @@ -223,19 +207,8 @@ MBC after deallocating first block: (B)->bhdr = ((Sz) | (F)), \ (B)->u.carrier = (C)) -# define ABLK_TO_MBC(B) \ - (ASSERT(IS_MBC_BLK(B) && IS_ALLOCED_BLK(B)), \ - (Carrier_t*)((MSEG_UNIT_FLOOR((UWord)(B)) - \ - (((B)->bhdr >> MBC_ABLK_OFFSET_SHIFT) << MSEG_UNIT_SHIFT)))) - -# define FBLK_TO_MBC(B) \ - (ASSERT(IS_MBC_BLK(B) && IS_FREE_BLK(B)), \ - (B)->u.carrier) - -# define BLK_TO_MBC(B) (IS_FREE_BLK(B) ? FBLK_TO_MBC(B) : ABLK_TO_MBC(B)) - # define IS_MBC_FIRST_ABLK(AP,B) \ - ((((UWord)(B) & ~MSEG_UNIT_MASK) == MBC_HEADER_SIZE(AP)) \ + ((((UWord)(B) & ~ERTS_SACRR_UNIT_MASK) == MBC_HEADER_SIZE(AP)) \ && ((B)->bhdr & MBC_ABLK_OFFSET_MASK) == 0) # define IS_MBC_FIRST_FBLK(AP,B) \ @@ -272,10 +245,6 @@ MBC after deallocating first block: (B)->bhdr = ((Sz) | (F)), \ (B)->carrier = (C)) -# define BLK_TO_MBC(B) ((B)->carrier) -# define ABLK_TO_MBC(B) BLK_TO_MBC(B) -# define FBLK_TO_MBC(B) BLK_TO_MBC(B) - # define IS_MBC_FIRST_BLK(AP,B) \ ((char*)(B) == (char*)((B)->carrier) + MBC_HEADER_SIZE(AP)) # define IS_MBC_FIRST_ABLK(AP,B) IS_MBC_FIRST_BLK(AP,B) @@ -300,8 +269,6 @@ MBC after deallocating first block: ((B)->bhdr & PREV_FREE_BLK_HDR_FLG) #define IS_PREV_BLK_ALLOCED(B) \ (!IS_PREV_BLK_FREE((B))) -#define IS_FREE_BLK(B) \ - (ASSERT(!IS_SBC_BLK(B)), (B)->bhdr & THIS_FREE_BLK_HDR_FLG) #define IS_ALLOCED_BLK(B) \ (!IS_FREE_BLK((B))) #define IS_LAST_BLK(B) \ @@ -318,13 +285,6 @@ MBC after deallocating first block: #define GET_BLK_HDR_FLGS(B) \ ((B)->bhdr & FLG_MASK) -#define IS_SBC_BLK(B) \ - (((B)->bhdr & FLG_MASK) == SBC_BLK_HDR_FLG) -#define IS_MBC_BLK(B) \ - (!IS_SBC_BLK((B))) - -#define MBC_BLK_SZ(B) (IS_FREE_BLK(B) ? MBC_FBLK_SZ(B) : MBC_ABLK_SZ(B)) - #define NXT_BLK(B) \ (ASSERT(IS_MBC_BLK(B)), \ (Block_t *) (((char *) (B)) + MBC_BLK_SZ((B)))) @@ -338,9 +298,97 @@ MBC after deallocating first block: /* Carriers ... */ -#define SBC_HEADER_SIZE (UNIT_CEILING(sizeof(Carrier_t) + ABLK_HDR_SZ) \ - - ABLK_HDR_SZ) -#define MBC_HEADER_SIZE(AP) SBC_HEADER_SIZE +/* #define ERTS_ALC_CPOOL_DEBUG */ + +#if defined(DEBUG) && !defined(ERTS_ALC_CPOOL_DEBUG) +# define ERTS_ALC_CPOOL_DEBUG +#endif + +#ifndef ERTS_SMP +# undef ERTS_ALC_CPOOL_DEBUG +#endif + +#ifdef ERTS_ALC_CPOOL_DEBUG +# define ERTS_ALC_CPOOL_ASSERT(A) \ + ((void) ((A) \ + ? 1 \ + : (erts_alcu_assert_failed(#A, \ + (char *) __FILE__, \ + __LINE__, \ + (char *) __func__), \ + 0))) +#else +# define ERTS_ALC_CPOOL_ASSERT(A) ((void) 1) +#endif + +#ifdef ERTS_SMP +#define ERTS_ALC_IS_CPOOL_ENABLED(A) ((A)->cpool.util_limit) +#else +#define ERTS_ALC_IS_CPOOL_ENABLED(A) (0) +#endif + +#ifdef ERTS_SMP + +#define ERTS_ALC_CPOOL_MAX_DISABLE_ABANDON 1000 +#define ERTS_ALC_CPOOL_ALLOC_OP_INC 8 +#define ERTS_ALC_CPOOL_FREE_OP_DEC 10 + +#define ERTS_ALC_CPOOL_ALLOC_OP(A) \ +do { \ + if ((A)->cpool.disable_abandon < ERTS_ALC_CPOOL_MAX_DISABLE_ABANDON) { \ + (A)->cpool.disable_abandon += ERTS_ALC_CPOOL_ALLOC_OP_INC; \ + if ((A)->cpool.disable_abandon > ERTS_ALC_CPOOL_MAX_DISABLE_ABANDON) \ + (A)->cpool.disable_abandon = ERTS_ALC_CPOOL_MAX_DISABLE_ABANDON; \ + } \ +} while (0) + + +#if ERTS_ALC_CPOOL_ALLOC_OP_INC >= ERTS_ALC_CPOOL_FREE_OP_DEC +# error "Implementation assume ERTS_ALC_CPOOL_ALLOC_OP_INC < ERTS_ALC_CPOOL_FREE_OP_DEC" +#endif + +#define ERTS_ALC_CPOOL_REALLOC_OP(A) \ +do { \ + if ((A)->cpool.disable_abandon) { \ + (A)->cpool.disable_abandon -= (ERTS_ALC_CPOOL_FREE_OP_DEC \ + - ERTS_ALC_CPOOL_ALLOC_OP_INC); \ + if ((A)->cpool.disable_abandon < 0) \ + (A)->cpool.disable_abandon = 0; \ + } \ +} while (0) + +#define ERTS_ALC_CPOOL_FREE_OP(A) \ +do { \ + if ((A)->cpool.disable_abandon) { \ + (A)->cpool.disable_abandon -= ERTS_ALC_CPOOL_FREE_OP_DEC; \ + if ((A)->cpool.disable_abandon < 0) \ + (A)->cpool.disable_abandon = 0; \ + } \ +} while (0) + +#else +#define ERTS_ALC_CPOOL_ALLOC_OP(A) +#define ERTS_ALC_CPOOL_REALLOC_OP(A) +#define ERTS_ALC_CPOOL_FREE_OP(A) +#endif + +#define ERTS_CRR_ALCTR_FLG_IN_POOL (((erts_aint_t) 1) << 0) +#define ERTS_CRR_ALCTR_FLG_BUSY (((erts_aint_t) 1) << 1) +#define ERTS_CRR_ALCTR_FLG_MASK (ERTS_CRR_ALCTR_FLG_IN_POOL | \ + ERTS_CRR_ALCTR_FLG_BUSY) + +#ifdef ERTS_SMP +#define SBC_HEADER_SIZE \ + (UNIT_CEILING(offsetof(Carrier_t, cpool) \ + + ABLK_HDR_SZ) \ + - ABLK_HDR_SZ) +#else +#define SBC_HEADER_SIZE \ + (UNIT_CEILING(sizeof(Carrier_t) \ + + ABLK_HDR_SZ) \ + - ABLK_HDR_SZ) +#endif +#define MBC_HEADER_SIZE(AP) ((AP)->mbc_header_size) #define MSEG_CARRIER_HDR_FLAG (((UWord) 1) << 0) @@ -352,7 +400,8 @@ MBC after deallocating first block: #define SCH_SBC SBC_CARRIER_HDR_FLAG #define SET_CARRIER_HDR(C, Sz, F, AP) \ - (ASSERT(((Sz) & FLG_MASK) == 0), (C)->chdr = ((Sz) | (F)), (C)->allctr = (AP)) + (ASSERT(((Sz) & FLG_MASK) == 0), (C)->chdr = ((Sz) | (F)), \ + erts_smp_atomic_init_nob(&(C)->allctr, (erts_aint_t) (AP))) #define BLK_TO_SBC(B) \ ((Carrier_t *) (((char *) (B)) - SBC_HEADER_SIZE)) @@ -385,6 +434,7 @@ MBC after deallocating first block: #define CFLG_FORCE_SYS_ALLOC (1 << 3) #define CFLG_FORCE_SIZE (1 << 4) #define CFLG_MAIN_CARRIER (1 << 5) +#define CFLG_NO_CPOOL (1 << 6) #ifdef ERTS_ALLOC_UTIL_HARD_DEBUG static void check_blk_carrier(Allctr_t *, Block_t *); @@ -412,11 +462,7 @@ static void check_blk_carrier(Allctr_t *, Block_t *); ASSERT(((AP)->mbcs.curr.norm.sys_alloc.no \ && (AP)->mbcs.curr.norm.sys_alloc.size) \ || (!(AP)->mbcs.curr.norm.sys_alloc.no \ - && !(AP)->mbcs.curr.norm.sys_alloc.size)); \ - ASSERT(((AP)->sbmbcs.curr.small_block.no \ - && (AP)->sbmbcs.curr.small_block.size) \ - || (!(AP)->sbmbcs.curr.small_block.no \ - && !(AP)->sbmbcs.curr.small_block.size)) + && !(AP)->mbcs.curr.norm.sys_alloc.size)); #else #define DEBUG_CHECK_CARRIER_NO_SZ(AP) @@ -487,17 +533,6 @@ do { \ + (AP)->mbcs.curr.norm.sys_alloc.size) -#define STAT_SBMBC_ALLOC(AP, CSZ) \ -do { \ - (AP)->sbmbcs.curr.small_block.no++; \ - (AP)->sbmbcs.curr.small_block.size += (CSZ); \ - if ((AP)->sbmbcs.max.no < (AP)->sbmbcs.curr.small_block.no) \ - (AP)->sbmbcs.max.no = (AP)->sbmbcs.curr.small_block.no; \ - if ((AP)->sbmbcs.max.size < (AP)->sbmbcs.curr.small_block.size) \ - (AP)->sbmbcs.max.size = (AP)->sbmbcs.curr.small_block.size; \ - DEBUG_CHECK_CARRIER_NO_SZ((AP)); \ -} while (0) - #define STAT_MSEG_MBC_ALLOC(AP, CSZ) \ do { \ (AP)->mbcs.curr.norm.mseg.no++; \ @@ -514,13 +549,19 @@ do { \ DEBUG_CHECK_CARRIER_NO_SZ((AP)); \ } while (0) -#define STAT_SBMBC_FREE(AP, CSZ) \ +#define STAT_MBC_CPOOL_FETCH(AP, CRR) \ do { \ - ASSERT((AP)->sbmbcs.curr.small_block.no > 0); \ - (AP)->sbmbcs.curr.small_block.no--; \ - ASSERT((AP)->sbmbcs.curr.small_block.size >= (CSZ)); \ - (AP)->sbmbcs.curr.small_block.size -= (CSZ); \ - DEBUG_CHECK_CARRIER_NO_SZ((AP)); \ + UWord csz__ = CARRIER_SZ((CRR)); \ + if (IS_MSEG_CARRIER((CRR))) \ + STAT_MSEG_MBC_ALLOC((AP), csz__); \ + else \ + STAT_SYS_ALLOC_MBC_ALLOC((AP), csz__); \ + (AP)->mbcs.blocks.curr.no += (CRR)->cpool.blocks; \ + if ((AP)->mbcs.blocks.max.no < (AP)->mbcs.blocks.curr.no) \ + (AP)->mbcs.blocks.max.no = (AP)->mbcs.blocks.curr.no; \ + (AP)->mbcs.blocks.curr.size += (CRR)->cpool.blocks_size; \ + if ((AP)->mbcs.blocks.max.size < (AP)->mbcs.blocks.curr.size) \ + (AP)->mbcs.blocks.max.size = (AP)->mbcs.blocks.curr.size; \ } while (0) #define STAT_MSEG_MBC_FREE(AP, CSZ) \ @@ -541,28 +582,88 @@ do { \ DEBUG_CHECK_CARRIER_NO_SZ((AP)); \ } while (0) -#define STAT_MBC_BLK_ALLOC(AP, BSZ, FLGS) \ +#define STAT_MBC_CPOOL_INSERT(AP, CRR) \ +do { \ + UWord csz__ = CARRIER_SZ((CRR)); \ + if (IS_MSEG_CARRIER((CRR))) \ + STAT_MSEG_MBC_FREE((AP), csz__); \ + else \ + STAT_SYS_ALLOC_MBC_FREE((AP), csz__); \ + ERTS_ALC_CPOOL_ASSERT((AP)->mbcs.blocks.curr.no \ + >= (CRR)->cpool.blocks); \ + (AP)->mbcs.blocks.curr.no -= (CRR)->cpool.blocks; \ + ERTS_ALC_CPOOL_ASSERT((AP)->mbcs.blocks.curr.size \ + >= (CRR)->cpool.blocks_size); \ + (AP)->mbcs.blocks.curr.size -= (CRR)->cpool.blocks_size; \ +} while (0) + +#ifdef ERTS_SMP +#define STAT_MBC_BLK_ALLOC_CRR(CRR, BSZ) \ do { \ - CarriersStats_t *cstats__ = (((FLGS) & ERTS_ALCU_FLG_SBMBC) \ - ? &(AP)->sbmbcs \ - : &(AP)->mbcs); \ + (CRR)->cpool.blocks++; \ + (CRR)->cpool.blocks_size += (BSZ); \ +} while (0) +#else +#define STAT_MBC_BLK_ALLOC_CRR(CRR, BSZ) ((void) (CRR)) /* Get rid of warning */ +#endif + +#define STAT_MBC_BLK_ALLOC(AP, CRR, BSZ, FLGS) \ +do { \ + CarriersStats_t *cstats__ = &(AP)->mbcs; \ cstats__->blocks.curr.no++; \ if (cstats__->blocks.max.no < cstats__->blocks.curr.no) \ cstats__->blocks.max.no = cstats__->blocks.curr.no; \ cstats__->blocks.curr.size += (BSZ); \ if (cstats__->blocks.max.size < cstats__->blocks.curr.size) \ cstats__->blocks.max.size = cstats__->blocks.curr.size; \ + STAT_MBC_BLK_ALLOC_CRR((CRR), (BSZ)); \ } while (0) -#define STAT_MBC_BLK_FREE(AP, BSZ, FLGS) \ +static ERTS_INLINE int +stat_cpool_mbc_blk_free(Allctr_t *allctr, + Carrier_t *crr, + Carrier_t **busy_pcrr_pp, + UWord blksz) +{ +#ifdef ERTS_SMP + + ERTS_ALC_CPOOL_ASSERT(crr->cpool.blocks > 0); + crr->cpool.blocks--; + ERTS_ALC_CPOOL_ASSERT(crr->cpool.blocks_size >= blksz); + crr->cpool.blocks_size -= blksz; + + if (!busy_pcrr_pp || !*busy_pcrr_pp) + return 0; + + ERTS_ALC_CPOOL_ASSERT(crr == *busy_pcrr_pp); + +#ifdef ERTS_ALC_CPOOL_DEBUG + ERTS_ALC_CPOOL_ASSERT( + erts_atomic_dec_read_nob(&allctr->cpool.stat.no_blocks) >= 0); + ERTS_ALC_CPOOL_ASSERT( + erts_atomic_add_read_nob(&allctr->cpool.stat.blocks_size, + -((erts_aint_t) blksz)) >= 0); +#else + erts_atomic_dec_nob(&allctr->cpool.stat.no_blocks); + erts_atomic_add_nob(&allctr->cpool.stat.blocks_size, + -((erts_aint_t) blksz)); +#endif + + return 1; +#else + return 0; +#endif +} + +#define STAT_MBC_BLK_FREE(AP, CRR, BPCRRPP, BSZ, FLGS) \ do { \ - CarriersStats_t *cstats__ = (((FLGS) & ERTS_ALCU_FLG_SBMBC) \ - ? &(AP)->sbmbcs \ - : &(AP)->mbcs); \ - ASSERT(cstats__->blocks.curr.no > 0); \ - cstats__->blocks.curr.no--; \ - ASSERT(cstats__->blocks.curr.size >= (BSZ)); \ - cstats__->blocks.curr.size -= (BSZ); \ + if (!stat_cpool_mbc_blk_free((AP), (CRR), (BPCRRPP), (BSZ))) { \ + CarriersStats_t *cstats__ = &(AP)->mbcs; \ + ASSERT(cstats__->blocks.curr.no > 0); \ + cstats__->blocks.curr.no--; \ + ASSERT(cstats__->blocks.curr.size >= (BSZ)); \ + cstats__->blocks.curr.size -= (BSZ); \ + } \ } while (0) /* Debug stuff... */ @@ -588,17 +689,21 @@ do { \ #ifdef DEBUG #ifdef USE_THREADS +# ifdef ERTS_SMP +# define IS_ACTUALLY_BLOCKING (erts_thr_progress_is_blocking()) +# else +# define IS_ACTUALLY_BLOCKING 0 +# endif #define ERTS_ALCU_DBG_CHK_THR_ACCESS(A) \ do { \ - if (!(A)->thread_safe) { \ - if (!(A)->debug.saved_tid) { \ + if (!(A)->thread_safe && !IS_ACTUALLY_BLOCKING) { \ + if (!(A)->debug.saved_tid) { \ (A)->debug.tid = erts_thr_self(); \ (A)->debug.saved_tid = 1; \ } \ else { \ ERTS_SMP_LC_ASSERT( \ - ethr_equal_tids((A)->debug.tid, erts_thr_self()) \ - || erts_thr_progress_is_blocking()); \ + ethr_equal_tids((A)->debug.tid, erts_thr_self())); \ } \ } \ } while (0) @@ -609,13 +714,42 @@ do { \ #define ERTS_ALCU_DBG_CHK_THR_ACCESS(A) #endif - static void make_name_atoms(Allctr_t *allctr); static Block_t *create_carrier(Allctr_t *, Uint, UWord); -static void destroy_carrier(Allctr_t *, Block_t *); -static void mbc_free(Allctr_t *allctr, void *p); +static void destroy_carrier(Allctr_t *, Block_t *, Carrier_t **); +static void mbc_free(Allctr_t *allctr, void *p, Carrier_t **busy_pcrr_pp); +static void dealloc_block(Allctr_t *, void *, ErtsAlcFixList_t *, int); +/* internal data... */ + +#if 0 + +static ERTS_INLINE void * +internal_alloc(UWord size) +{ + void *res = erts_sys_alloc(0, NULL, size); + if (!res) + erts_alloc_enomem(ERTS_ALC_T_UNDEF, size); + return res; +} + +static ERTS_INLINE void * +internal_realloc(void *ptr, UWord size) +{ + void *res = erts_sys_realloc(0, NULL, ptr, size); + if (!res) + erts_alloc_enomem(ERTS_ALC_T_UNDEF, size); + return res; +} + +static ERTS_INLINE void +internal_free(void *ptr) +{ + erts_sys_free(0, NULL, ptr); +} + +#endif /* mseg ... */ @@ -625,8 +759,9 @@ static ERTS_INLINE void * alcu_mseg_alloc(Allctr_t *allctr, Uint *size_p, Uint flags) { void *res; - - res = erts_mseg_alloc_opt(allctr->alloc_no, size_p, flags, &allctr->mseg_opt); + UWord size = (UWord) *size_p; + res = erts_mseg_alloc_opt(allctr->alloc_no, &size, flags, &allctr->mseg_opt); + *size_p = (Uint) size; INC_CC(allctr->calls.mseg_alloc); return res; } @@ -635,9 +770,10 @@ static ERTS_INLINE void * alcu_mseg_realloc(Allctr_t *allctr, void *seg, Uint old_size, Uint *new_size_p) { void *res; - - res = erts_mseg_realloc_opt(allctr->alloc_no, seg, old_size, new_size_p, + UWord new_size = (UWord) *new_size_p; + res = erts_mseg_realloc_opt(allctr->alloc_no, seg, (UWord) old_size, &new_size, ERTS_MSEG_FLG_NONE, &allctr->mseg_opt); + *new_size_p = (Uint) new_size; INC_CC(allctr->calls.mseg_realloc); return res; } @@ -645,18 +781,22 @@ alcu_mseg_realloc(Allctr_t *allctr, void *seg, Uint old_size, Uint *new_size_p) static ERTS_INLINE void alcu_mseg_dealloc(Allctr_t *allctr, void *seg, Uint size, Uint flags) { - erts_mseg_dealloc_opt(allctr->alloc_no, seg, size, flags, &allctr->mseg_opt); + erts_mseg_dealloc_opt(allctr->alloc_no, seg, (UWord) size, flags, &allctr->mseg_opt); INC_CC(allctr->calls.mseg_dealloc); } #endif static ERTS_INLINE void * -alcu_sys_alloc(Allctr_t *allctr, Uint size) +alcu_sys_alloc(Allctr_t *allctr, Uint size, int superalign) { void *res; - - res = erts_sys_alloc(0, NULL, size); +#if ERTS_SA_MB_CARRIERS && ERTS_HAVE_ERTS_SYS_ALIGNED_ALLOC + if (superalign) + res = erts_sys_aligned_alloc(ERTS_SACRR_UNIT_SZ, size); + else +#endif + res = erts_sys_alloc(0, NULL, size); INC_CC(allctr->calls.sys_alloc); if (erts_mtrace_enabled) erts_mtrace_crr_alloc(res, allctr->alloc_no, ERTS_ALC_A_SYSTEM, size); @@ -664,11 +804,16 @@ alcu_sys_alloc(Allctr_t *allctr, Uint size) } static ERTS_INLINE void * -alcu_sys_realloc(Allctr_t *allctr, void *ptr, Uint size) +alcu_sys_realloc(Allctr_t *allctr, void *ptr, Uint size, Uint old_size, int superalign) { void *res; - res = erts_sys_realloc(0, NULL, ptr, size); +#if ERTS_SA_MB_CARRIERS && ERTS_HAVE_ERTS_SYS_ALIGNED_ALLOC + if (superalign) + res = erts_sys_aligned_realloc(ERTS_SACRR_UNIT_SZ, ptr, size, old_size); + else +#endif + res = erts_sys_realloc(0, NULL, ptr, size); INC_CC(allctr->calls.sys_realloc); if (erts_mtrace_enabled) erts_mtrace_crr_realloc(res, @@ -680,9 +825,14 @@ alcu_sys_realloc(Allctr_t *allctr, void *ptr, Uint size) } static ERTS_INLINE void -alcu_sys_free(Allctr_t *allctr, void *ptr) +alcu_sys_free(Allctr_t *allctr, void *ptr, int superalign) { - erts_sys_free(0, NULL, ptr); +#if ERTS_SA_MB_CARRIERS && ERTS_HAVE_ERTS_SYS_ALIGNED_ALLOC + if (superalign) + erts_sys_aligned_free(ERTS_SACRR_UNIT_SZ, ptr); + else +#endif + erts_sys_free(0, NULL, ptr); INC_CC(allctr->calls.sys_free); if (erts_mtrace_enabled) erts_mtrace_crr_free(allctr->alloc_no, ERTS_ALC_A_SYSTEM, ptr); @@ -777,10 +927,117 @@ unlink_carrier(CarrierList_t *cl, Carrier_t *crr) } } -static Block_t *create_sbmbc(Allctr_t *allctr, Uint umem_sz); -static void destroy_sbmbc(Allctr_t *allctr, Block_t *blk); -static Block_t *create_carrier(Allctr_t *, Uint, UWord); -static void destroy_carrier(Allctr_t *, Block_t *); +#ifdef ERTS_SMP + +#ifdef DEBUG +static int is_in_list(ErtsDoubleLink_t* sentinel, ErtsDoubleLink_t* node) +{ + ErtsDoubleLink_t* p; + + ASSERT(node != sentinel); + for (p = sentinel->next; p != sentinel; p = p->next) { + if (p == node) + return 1; + } + return 0; +} +#endif /* DEBUG */ + +static ERTS_INLINE void +link_edl_after(ErtsDoubleLink_t* after_me, ErtsDoubleLink_t* node) +{ + ErtsDoubleLink_t* before_me = after_me->next; + ASSERT(node != after_me && node != before_me); + node->next = before_me; + node->prev = after_me; + before_me->prev = node; + after_me->next = node; +} + +static ERTS_INLINE void +link_edl_before(ErtsDoubleLink_t* before_me, ErtsDoubleLink_t* node) +{ + ErtsDoubleLink_t* after_me = before_me->prev; + ASSERT(node != before_me && node != after_me); + node->next = before_me; + node->prev = after_me; + before_me->prev = node; + after_me->next = node; +} + +static ERTS_INLINE void +unlink_edl(ErtsDoubleLink_t* node) +{ + node->next->prev = node->prev; + node->prev->next = node->next; +} + +static ERTS_INLINE void +relink_edl_before(ErtsDoubleLink_t* before_me, ErtsDoubleLink_t* node) +{ + if (node != before_me && node != before_me->prev) { + unlink_edl(node); + link_edl_before(before_me, node); + } +} + +static ERTS_INLINE int is_abandoned(Carrier_t *crr) +{ + return crr->cpool.abandoned.next != NULL; +} + +static ERTS_INLINE void +link_abandoned_carrier(ErtsDoubleLink_t* list, Carrier_t *crr) +{ + ASSERT(!is_abandoned(crr)); + + link_edl_after(list, &crr->cpool.abandoned); + + ASSERT(crr->cpool.abandoned.next != &crr->cpool.abandoned); + ASSERT(crr->cpool.abandoned.prev != &crr->cpool.abandoned); +} + +static ERTS_INLINE void +unlink_abandoned_carrier(Carrier_t *crr) +{ + ASSERT(is_in_list(&crr->cpool.orig_allctr->cpool.pooled_list, + &crr->cpool.abandoned) || + is_in_list(&crr->cpool.orig_allctr->cpool.traitor_list, + &crr->cpool.abandoned)); + + unlink_edl(&crr->cpool.abandoned); + + crr->cpool.abandoned.next = NULL; + crr->cpool.abandoned.prev = NULL; +} + +static ERTS_INLINE void +clear_busy_pool_carrier(Allctr_t *allctr, Carrier_t *crr) +{ + if (crr) { + erts_aint_t max_size; + erts_aint_t new_val; + + max_size = (erts_aint_t) allctr->largest_fblk_in_mbc(allctr, crr); + erts_atomic_set_nob(&crr->cpool.max_size, max_size); + + new_val = (((erts_aint_t) allctr)|ERTS_CRR_ALCTR_FLG_IN_POOL); + +#ifdef ERTS_ALC_CPOOL_DEBUG + { + erts_aint_t old_val = new_val|ERTS_CRR_ALCTR_FLG_BUSY; + + ERTS_ALC_CPOOL_ASSERT(old_val + == erts_smp_atomic_xchg_relb(&crr->allctr, + new_val)); + } +#else + erts_smp_atomic_set_relb(&crr->allctr, new_val); +#endif + } +} + +#endif /* ERTS_SMP */ #if 0 #define ERTS_DBG_CHK_FIX_LIST(A, FIX, IX, B) \ @@ -802,13 +1059,159 @@ chk_fix_list(Allctr_t *allctr, ErtsAlcFixList_t *fix, int ix, int before) #define ERTS_DBG_CHK_FIX_LIST(A, FIX, IX, B) #endif -erts_aint32_t -erts_alcu_fix_alloc_shrink(Allctr_t *allctr, erts_aint32_t flgs) +static void *mbc_alloc(Allctr_t *allctr, Uint size); + +#ifdef ERTS_SMP +typedef struct { + ErtsAllctrDDBlock_t ddblock__; /* must be first */ + ErtsAlcType_t fix_type; +} ErtsAllctrFixDDBlock_t; +#endif + +#define ERTS_ALC_FIX_NO_UNUSE (((ErtsAlcType_t) 1) << ERTS_ALC_N_BITS) + +static ERTS_INLINE void +dealloc_fix_block(Allctr_t *allctr, + ErtsAlcType_t type, + void *ptr, + ErtsAlcFixList_t *fix, + int dec_cc_on_redirect) +{ +#ifdef ERTS_SMP + /* May be redirected... */ + ASSERT((type & ERTS_ALC_FIX_NO_UNUSE) == 0); + ((ErtsAllctrFixDDBlock_t *) ptr)->fix_type = type | ERTS_ALC_FIX_NO_UNUSE; +#endif + dealloc_block(allctr, ptr, fix, dec_cc_on_redirect); +} + +static ERTS_INLINE void +sched_fix_shrink(Allctr_t *allctr, int on) +{ + if (on && !allctr->fix_shrink_scheduled) { + allctr->fix_shrink_scheduled = 1; + erts_set_aux_work_timeout(allctr->ix, + (ERTS_SSI_AUX_WORK_FIX_ALLOC_LOWER_LIM + | ERTS_SSI_AUX_WORK_FIX_ALLOC_DEALLOC), + 1); + } + else if (!on && allctr->fix_shrink_scheduled) { + allctr->fix_shrink_scheduled = 0; + erts_set_aux_work_timeout(allctr->ix, + (ERTS_SSI_AUX_WORK_FIX_ALLOC_LOWER_LIM + | ERTS_SSI_AUX_WORK_FIX_ALLOC_DEALLOC), + 0); + } +} + +static ERTS_INLINE void +fix_cpool_check_shrink(Allctr_t *allctr, + ErtsAlcType_t type, + ErtsAlcFixList_t *fix, + Carrier_t **busy_pcrr_pp) +{ + if (fix->u.cpool.shrink_list > 0) { + if (fix->list_size == 0) + fix->u.cpool.shrink_list = 0; + else { + void *p; +#ifdef ERTS_SMP + if (busy_pcrr_pp) { + clear_busy_pool_carrier(allctr, *busy_pcrr_pp); + *busy_pcrr_pp = NULL; + } +#endif + fix->u.cpool.shrink_list--; + p = fix->list; + fix->list = *((void **) p); + fix->list_size--; + if (fix->u.cpool.min_list_size > fix->list_size) + fix->u.cpool.min_list_size = fix->list_size; + + dealloc_fix_block(allctr, type, p, fix, 0); + } + } +} + +static ERTS_INLINE void * +fix_cpool_alloc(Allctr_t *allctr, ErtsAlcType_t type, Uint size) +{ + void *res; + ErtsAlcFixList_t *fix; + + ASSERT(ERTS_ALC_N_MIN_A_FIXED_SIZE <= type + && type <= ERTS_ALC_N_MAX_A_FIXED_SIZE); + + fix = &allctr->fix[type - ERTS_ALC_N_MIN_A_FIXED_SIZE]; + + res = fix->list; + if (res) { + fix->list = *((void **) res); + fix->list_size--; + if (fix->u.cpool.min_list_size > fix->list_size) + fix->u.cpool.min_list_size = fix->list_size; + fix->u.cpool.used++; + fix_cpool_check_shrink(allctr, type, fix, NULL); + return res; + } + if (size < 2*sizeof(UWord)) + size += sizeof(UWord); + if (size >= allctr->sbc_threshold) { + Block_t *blk; + blk = create_carrier(allctr, size, CFLG_SBC); + res = blk ? BLK2UMEM(blk) : NULL; + } + else + res = mbc_alloc(allctr, size); + if (res) { + fix->u.cpool.used++; + fix->u.cpool.allocated++; + } + return res; +} + +static ERTS_INLINE void +fix_cpool_free(Allctr_t *allctr, + ErtsAlcType_t type, + void *p, + Carrier_t **busy_pcrr_pp, + int unuse) +{ + ErtsAlcFixList_t *fix; + + ASSERT(ERTS_ALC_N_MIN_A_FIXED_SIZE <= type + && type <= ERTS_ALC_N_MAX_A_FIXED_SIZE); + + fix = &allctr->fix[type - ERTS_ALC_N_MIN_A_FIXED_SIZE]; + + if (unuse) + fix->u.cpool.used--; + + if ((!busy_pcrr_pp || !*busy_pcrr_pp) + && !fix->u.cpool.shrink_list + && fix->list_size < ERTS_ALCU_FIX_MAX_LIST_SZ) { + *((void **) p) = fix->list; + fix->list = p; + fix->list_size++; + sched_fix_shrink(allctr, 1); + } + else { + Block_t *blk = UMEM2BLK(p); + if (IS_SBC_BLK(blk)) + destroy_carrier(allctr, blk, NULL); + else + mbc_free(allctr, p, busy_pcrr_pp); + fix->u.cpool.allocated--; + fix_cpool_check_shrink(allctr, type, fix, busy_pcrr_pp); + } +} + +static ERTS_INLINE erts_aint32_t +fix_cpool_alloc_shrink(Allctr_t *allctr, erts_aint32_t flgs) { int all_empty = 1; erts_aint32_t res = 0; int ix, o; - ErtsAlcFixList_t *fix = allctr->fix; int flush = flgs == 0; #ifdef USE_THREADS @@ -817,56 +1220,203 @@ erts_alcu_fix_alloc_shrink(Allctr_t *allctr, erts_aint32_t flgs) #endif for (ix = 0; ix < ERTS_ALC_NO_FIXED_SIZES; ix++) { + ErtsAlcFixList_t *fix = &allctr->fix[ix]; + ErtsAlcType_t type; ERTS_DBG_CHK_FIX_LIST(allctr, fix, ix, 1); - if (flgs & ERTS_SSI_AUX_WORK_FIX_ALLOC_LOWER_LIM) { - fix[ix].limit = fix[ix].max_used; - if (fix[ix].limit < fix[ix].used) - fix[ix].limit = fix[ix].used; - fix[ix].max_used = fix[ix].used; - ASSERT(fix[ix].limit >= 0); - - } - if (flush) { - fix[ix].limit = 0; - fix[ix].max_used = fix[ix].used; - ASSERT(fix[ix].limit >= 0); + if (flush) + fix->u.cpool.shrink_list = fix->list_size; + else if (flgs & ERTS_SSI_AUX_WORK_FIX_ALLOC_LOWER_LIM) { + fix->u.cpool.shrink_list = fix->u.cpool.min_list_size; + fix->u.cpool.min_list_size = fix->list_size; } + type = (ErtsAlcType_t) (ix + ERTS_ALC_N_MIN_A_FIXED_SIZE); for (o = 0; o < ERTS_ALC_FIX_MAX_SHRINK_OPS || flush; o++) { - Block_t *blk; void *ptr; - if (!flush && fix[ix].limit >= fix[ix].allocated) + if (fix->u.cpool.shrink_list == 0) break; - if (fix[ix].list_size == 0) + if (fix->list_size == 0) { + fix->u.cpool.shrink_list = 0; break; - ptr = fix[ix].list; - fix[ix].list = *((void **) ptr); - fix[ix].list_size--; + } + ptr = fix->list; + fix->list = *((void **) ptr); + fix->list_size--; + fix->u.cpool.shrink_list--; + dealloc_fix_block(allctr, type, ptr, fix, 0); + } + if (fix->u.cpool.min_list_size > fix->list_size) + fix->u.cpool.min_list_size = fix->list_size; + if (fix->list_size != 0) { + if (fix->u.cpool.shrink_list > 0) + res |= ERTS_SSI_AUX_WORK_FIX_ALLOC_DEALLOC; + all_empty = 0; + } + } + + if (all_empty) + sched_fix_shrink(allctr, 0); + +#ifdef USE_THREADS + if (allctr->thread_safe) + erts_mtx_unlock(&allctr->mutex); +#endif + + return res; +} + +static ERTS_INLINE void * +fix_nocpool_alloc(Allctr_t *allctr, ErtsAlcType_t type, Uint size) +{ + ErtsAlcFixList_t *fix; + void *res; - blk = UMEM2BLK(ptr); + ASSERT(ERTS_ALC_N_MIN_A_FIXED_SIZE <= type + && type <= ERTS_ALC_N_MAX_A_FIXED_SIZE); + fix = &allctr->fix[type - ERTS_ALC_N_MIN_A_FIXED_SIZE]; + + ERTS_DBG_CHK_FIX_LIST(allctr, fix, ix, 1); + fix->u.nocpool.used++; + res = fix->list; + if (res) { + fix->list_size--; + fix->list = *((void **) res); + if (fix->list && fix->u.nocpool.allocated > fix->u.nocpool.limit) { + Block_t *blk; + void *p = fix->list; + fix->list = *((void **) p); + fix->list_size--; + blk = UMEM2BLK(p); if (IS_SBC_BLK(blk)) - destroy_carrier(allctr, blk); + destroy_carrier(allctr, blk, NULL); else - mbc_free(allctr, ptr); + mbc_free(allctr, p, NULL); + fix->u.nocpool.allocated--; + } + ERTS_DBG_CHK_FIX_LIST(allctr, fix, ix, 0); + return res; + } + if (size < 2*sizeof(UWord)) + size += sizeof(UWord); + if (fix->u.nocpool.limit < fix->u.nocpool.used) + fix->u.nocpool.limit = fix->u.nocpool.used; + if (fix->u.nocpool.max_used < fix->u.nocpool.used) + fix->u.nocpool.max_used = fix->u.nocpool.used; + fix->u.nocpool.allocated++; + + if (size >= allctr->sbc_threshold) { + Block_t *blk; + blk = create_carrier(allctr, size, CFLG_SBC); + res = blk ? BLK2UMEM(blk) : NULL; + } + else + res = mbc_alloc(allctr, size); + + if (!res) { + fix->u.nocpool.allocated--; + fix->u.nocpool.used--; + } + return res; +} + +static ERTS_INLINE void +fix_nocpool_free(Allctr_t *allctr, + ErtsAlcType_t type, + void *p) +{ + Block_t *blk; + ErtsAlcFixList_t *fix; + + ASSERT(ERTS_ALC_N_MIN_A_FIXED_SIZE <= type + && type <= ERTS_ALC_N_MAX_A_FIXED_SIZE); + + fix = &allctr->fix[type - ERTS_ALC_N_MIN_A_FIXED_SIZE]; + + ERTS_DBG_CHK_FIX_LIST(allctr, fix, ix, 1); + fix->u.nocpool.used--; + if (fix->u.nocpool.allocated < fix->u.nocpool.limit + && fix->list_size < ERTS_ALCU_FIX_MAX_LIST_SZ) { + *((void **) p) = fix->list; + fix->list = p; + fix->list_size++; + sched_fix_shrink(allctr, 1); + ERTS_DBG_CHK_FIX_LIST(allctr, fix, ix, 0); + return; + } + fix->u.nocpool.allocated--; + if (fix->list && fix->u.nocpool.allocated > fix->u.nocpool.limit) { + blk = UMEM2BLK(p); + if (IS_SBC_BLK(blk)) + destroy_carrier(allctr, blk, NULL); + else + mbc_free(allctr, p, NULL); + p = fix->list; + fix->list = *((void **) p); + fix->list_size--; + fix->u.nocpool.allocated--; + } + + blk = UMEM2BLK(p); + if (IS_SBC_BLK(blk)) + destroy_carrier(allctr, blk, NULL); + else + mbc_free(allctr, p, NULL); + ERTS_DBG_CHK_FIX_LIST(allctr, fix, ix, 0); +} + +static ERTS_INLINE erts_aint32_t +fix_nocpool_alloc_shrink(Allctr_t *allctr, erts_aint32_t flgs) +{ + int all_empty = 1; + erts_aint32_t res = 0; + int ix, o; + int flush = flgs == 0; + +#ifdef USE_THREADS + if (allctr->thread_safe) + erts_mtx_lock(&allctr->mutex); +#endif + + for (ix = 0; ix < ERTS_ALC_NO_FIXED_SIZES; ix++) { + ErtsAlcFixList_t *fix = &allctr->fix[ix]; + ERTS_DBG_CHK_FIX_LIST(allctr, fix, ix, 1); + if (flgs & ERTS_SSI_AUX_WORK_FIX_ALLOC_LOWER_LIM) { + fix->u.nocpool.limit = fix->u.nocpool.max_used; + if (fix->u.nocpool.limit < fix->u.nocpool.used) + fix->u.nocpool.limit = fix->u.nocpool.used; + fix->u.nocpool.max_used = fix->u.nocpool.used; + ASSERT(fix->u.nocpool.limit >= 0); - fix[ix].allocated--; } - if (fix[ix].list_size != 0) { - if (fix[ix].limit < fix[ix].allocated) + if (flush) { + fix->u.nocpool.limit = 0; + fix->u.nocpool.max_used = fix->u.nocpool.used; + ASSERT(fix->u.nocpool.limit >= 0); + } + for (o = 0; o < ERTS_ALC_FIX_MAX_SHRINK_OPS || flush; o++) { + void *ptr; + + if (!flush && fix->u.nocpool.limit >= fix->u.nocpool.allocated) + break; + if (fix->list_size == 0) + break; + ptr = fix->list; + fix->list = *((void **) ptr); + fix->list_size--; + dealloc_block(allctr, ptr, NULL, 0); + fix->u.nocpool.allocated--; + } + if (fix->list_size != 0) { + if (fix->u.nocpool.limit < fix->u.nocpool.allocated) res |= ERTS_SSI_AUX_WORK_FIX_ALLOC_DEALLOC; all_empty = 0; } ERTS_DBG_CHK_FIX_LIST(allctr, fix, ix, 0); } - if (all_empty && allctr->fix_shrink_scheduled) { - allctr->fix_shrink_scheduled = 0; - erts_set_aux_work_timeout(allctr->ix, - (ERTS_SSI_AUX_WORK_FIX_ALLOC_LOWER_LIM - | ERTS_SSI_AUX_WORK_FIX_ALLOC_DEALLOC), - 0); - } + if (all_empty) + sched_fix_shrink(allctr, 0); #ifdef USE_THREADS if (allctr->thread_safe) @@ -876,11 +1426,28 @@ erts_alcu_fix_alloc_shrink(Allctr_t *allctr, erts_aint32_t flgs) return res; } -#ifdef ERTS_SMP +erts_aint32_t +erts_alcu_fix_alloc_shrink(Allctr_t *allctr, erts_aint32_t flgs) +{ + if (ERTS_ALC_IS_CPOOL_ENABLED(allctr)) + return fix_cpool_alloc_shrink(allctr, flgs); + else + return fix_nocpool_alloc_shrink(allctr, flgs); +} -#define ERTS_ALCU_DD_FIX_TYPE_OFFS \ - ((sizeof(ErtsAllctrDDBlock_t)-1)/sizeof(UWord) + 1) +static void dealloc_carrier(Allctr_t *allctr, Carrier_t *crr, int superaligned); + +static ERTS_INLINE void +dealloc_mbc(Allctr_t *allctr, Carrier_t *crr) +{ + ASSERT(IS_MB_CARRIER(crr)); + if (allctr->destroying_mbc) + allctr->destroying_mbc(allctr, crr); + dealloc_carrier(allctr, crr, 1); +} + +#ifdef ERTS_SMP static ERTS_INLINE Allctr_t* get_pref_allctr(void *extra) @@ -890,12 +1457,15 @@ get_pref_allctr(void *extra) pref_ix = ERTS_ALC_GET_THR_IX(); - ASSERT(sizeof(UWord) == sizeof(Allctr_t *)); + ERTS_CT_ASSERT(sizeof(UWord) == sizeof(Allctr_t *)); ASSERT(0 <= pref_ix && pref_ix < tspec->size); return tspec->allctr[pref_ix]; } +#define ERTS_ALC_TS_PREF_LOCK_IF_USED (1) +#define ERTS_ALC_TS_PREF_LOCK_NO (0) + /* SMP note: * get_used_allctr() must be safe WITHOUT locking the allocator while * concurrent threads may be updating adjacent blocks. @@ -904,22 +1474,85 @@ get_pref_allctr(void *extra) * the "PREV_FREE" flag bit. */ static ERTS_INLINE Allctr_t* -get_used_allctr(void *extra, void *p, UWord *sizep) +get_used_allctr(Allctr_t *pref_allctr, int pref_lock, void *p, UWord *sizep, + Carrier_t **busy_pcrr_pp) { Block_t* blk = UMEM2BLK(p); - Carrier_t* crr; + Carrier_t *crr; + erts_aint_t iallctr; + Allctr_t *used_allctr; + + *busy_pcrr_pp = NULL; if (IS_SBC_BLK(blk)) { crr = BLK_TO_SBC(blk); if (sizep) *sizep = SBC_BLK_SZ(blk) - ABLK_HDR_SZ; + iallctr = erts_smp_atomic_read_dirty(&crr->allctr); } else { crr = ABLK_TO_MBC(blk); + if (sizep) *sizep = MBC_ABLK_SZ(blk) - ABLK_HDR_SZ; + if (!ERTS_ALC_IS_CPOOL_ENABLED(pref_allctr)) + iallctr = erts_smp_atomic_read_dirty(&crr->allctr); + else { + int locked_pref_allctr = 0; + iallctr = erts_smp_atomic_read_ddrb(&crr->allctr); + + if (ERTS_ALC_TS_PREF_LOCK_IF_USED == pref_lock + && pref_allctr->thread_safe) { + used_allctr = (Allctr_t *) (iallctr & ~ERTS_CRR_ALCTR_FLG_MASK); + if (pref_allctr == used_allctr) { + erts_mtx_lock(&pref_allctr->mutex); + locked_pref_allctr = 1; + } + } + + while ((iallctr & ((~ERTS_CRR_ALCTR_FLG_MASK)|ERTS_CRR_ALCTR_FLG_IN_POOL)) + == (((erts_aint_t) pref_allctr)|ERTS_CRR_ALCTR_FLG_IN_POOL)) { + erts_aint_t act; + + ERTS_ALC_CPOOL_ASSERT(!(iallctr & ERTS_CRR_ALCTR_FLG_BUSY)); + act = erts_smp_atomic_cmpxchg_ddrb(&crr->allctr, + iallctr|ERTS_CRR_ALCTR_FLG_BUSY, + iallctr); + if (act == iallctr) { + *busy_pcrr_pp = crr; + break; + } + iallctr = act; + } + + used_allctr = (Allctr_t *) (iallctr & ~ERTS_CRR_ALCTR_FLG_MASK); + + if (ERTS_ALC_TS_PREF_LOCK_IF_USED == pref_lock) { + if (locked_pref_allctr && used_allctr != pref_allctr) { + /* Was taken out of pool; now owned by someone else */ + erts_mtx_unlock(&pref_allctr->mutex); + } + } + + ERTS_ALC_CPOOL_ASSERT( + (((iallctr & ~ERTS_CRR_ALCTR_FLG_MASK) == (erts_aint_t) pref_allctr) + ? (((iallctr & ERTS_CRR_ALCTR_FLG_MASK) == ERTS_CRR_ALCTR_FLG_IN_POOL) + || ((iallctr & ERTS_CRR_ALCTR_FLG_MASK) == 0)) + : 1)); + + return used_allctr; + } } - return crr->allctr; + + used_allctr = (Allctr_t *) (iallctr & ~ERTS_CRR_ALCTR_FLG_MASK); + + if (ERTS_ALC_TS_PREF_LOCK_IF_USED == pref_lock + && used_allctr == pref_allctr + && pref_allctr->thread_safe) { + erts_mtx_lock(&pref_allctr->mutex); + } + + return used_allctr; } static void @@ -1014,7 +1647,7 @@ check_insert_marker(ErtsAllctrDDQueue_t *ddq, erts_aint_t ilast) } static ERTS_INLINE int -ddq_enqueue(ErtsAlcType_t type, ErtsAllctrDDQueue_t *ddq, void *ptr, int cinit) +ddq_enqueue(ErtsAllctrDDQueue_t *ddq, void *ptr, int cinit) { int last_elem; int um_refc_ix = 0; @@ -1115,6 +1748,63 @@ store_earliest_thr_prgr(ErtsThrPrgrVal *prev_val, ErtsAllctrDDQueue_t *ddq) } } +static void +check_pending_dealloc_carrier(Allctr_t *allctr, + int *need_thr_progress, + ErtsThrPrgrVal *thr_prgr_p, + int *need_more_work); + +static void +handle_delayed_fix_dealloc(Allctr_t *allctr, void *ptr) +{ + ErtsAlcType_t type; + + type = ((ErtsAllctrFixDDBlock_t *) ptr)->fix_type; + + ASSERT(ERTS_ALC_N_MIN_A_FIXED_SIZE + <= (type & ~ERTS_ALC_FIX_NO_UNUSE)); + ASSERT((type & ~ERTS_ALC_FIX_NO_UNUSE) + <= ERTS_ALC_N_MAX_A_FIXED_SIZE); + + if (!ERTS_ALC_IS_CPOOL_ENABLED(allctr)) + fix_nocpool_free(allctr, (type & ~ERTS_ALC_FIX_NO_UNUSE), ptr); + else { + Block_t *blk = UMEM2BLK(ptr); + Carrier_t *busy_pcrr_p; + Allctr_t *used_allctr; + + if (IS_SBC_BLK(blk)) { + busy_pcrr_p = NULL; + goto doit; + } + + used_allctr = get_used_allctr(allctr, ERTS_ALC_TS_PREF_LOCK_NO, ptr, + NULL, &busy_pcrr_p); + if (used_allctr == allctr) { + doit: + fix_cpool_free(allctr, (type & ~ERTS_ALC_FIX_NO_UNUSE), + ptr, &busy_pcrr_p, + !(type & ERTS_ALC_FIX_NO_UNUSE)); + clear_busy_pool_carrier(allctr, busy_pcrr_p); + } + else { + /* Carrier migrated; need to redirect block to new owner... */ + int cinit = used_allctr->dd.ix - allctr->dd.ix; + + ERTS_ALC_CPOOL_ASSERT(!busy_pcrr_p); + + DEC_CC(allctr->calls.this_free); + + ((ErtsAllctrFixDDBlock_t *) ptr)->fix_type = type; + if (ddq_enqueue(&used_allctr->dd.q, ptr, cinit)) + erts_alloc_notify_delayed_dealloc(used_allctr->ix); + } + } +} + +static void +schedule_dealloc_carrier(Allctr_t *allctr, Carrier_t *crr); + static ERTS_INLINE int handle_delayed_dealloc(Allctr_t *allctr, int allctr_locked, @@ -1146,7 +1836,6 @@ handle_delayed_dealloc(Allctr_t *allctr, while (1) { Block_t *blk; void *ptr; - int ix; if (use_limit && ++ops > ops_limit) { if (ddq->head.first != ddq->head.unref_end) { @@ -1175,52 +1864,48 @@ handle_delayed_dealloc(Allctr_t *allctr, res = 1; - INC_CC(allctr->calls.this_free); + blk = UMEM2BLK(ptr); + if (IS_FREE_LAST_MBC_BLK(blk)) { + /* + * A multiblock carrier that previously has been migrated away + * from us and now is back to be deallocated. For more info + * see schedule_dealloc_carrier(). + * + * Note that we cannot use FBLK_TO_MBC(blk) since it + * data has been overwritten by the queue. + */ + Carrier_t *crr = FIRST_BLK_TO_MBC(allctr, blk); + + /* Restore word overwritten by the dd-queue as it will be read + * if this carrier is pulled from dc_list by cpool_fetch() + */ + ERTS_ALC_CPOOL_ASSERT(FBLK_TO_MBC(blk) != crr); + ERTS_CT_ASSERT(sizeof(ErtsAllctrDDBlock_t) == sizeof(void*)); +#ifdef MBC_ABLK_OFFSET_BITS + blk->u.carrier = crr; +#else + blk->carrier = crr; +#endif - if (fix) { - ErtsAlcType_t type; - - type = (ErtsAlcType_t) ((UWord *) ptr)[ERTS_ALCU_DD_FIX_TYPE_OFFS]; - ix = type - ERTS_ALC_N_MIN_A_FIXED_SIZE; - ERTS_DBG_CHK_FIX_LIST(allctr, fix, ix, 1); - fix[ix].used--; - if (fix[ix].allocated < fix[ix].limit - && fix[ix].list_size < ERTS_ALCU_FIX_MAX_LIST_SZ) { - *((void **) ptr) = fix[ix].list; - fix[ix].list = ptr; - fix[ix].list_size++; - if (!allctr->fix_shrink_scheduled) { - allctr->fix_shrink_scheduled = 1; - erts_set_aux_work_timeout( - allctr->ix, - (ERTS_SSI_AUX_WORK_FIX_ALLOC_LOWER_LIM - | ERTS_SSI_AUX_WORK_FIX_ALLOC_DEALLOC), - 1); - } - ERTS_DBG_CHK_FIX_LIST(allctr, fix, ix, 0); - continue; - } - fix[ix].allocated--; - if (fix[ix].list && fix[ix].allocated > fix[ix].limit) { - blk = UMEM2BLK(ptr); - if (IS_SBC_BLK(blk)) - destroy_carrier(allctr, blk); - else - mbc_free(allctr, ptr); - ptr = fix[ix].list; - fix[ix].list = *((void **) ptr); - fix[ix].list_size--; - fix[ix].allocated--; - } + ERTS_ALC_CPOOL_ASSERT(ERTS_ALC_IS_CPOOL_ENABLED(allctr)); + ERTS_ALC_CPOOL_ASSERT(allctr == crr->cpool.orig_allctr); + ERTS_ALC_CPOOL_ASSERT(((erts_aint_t) allctr) + != (erts_smp_atomic_read_nob(&crr->allctr) + & ~ERTS_CRR_ALCTR_FLG_MASK)); + + erts_smp_atomic_set_nob(&crr->allctr, ((erts_aint_t) allctr)); + + schedule_dealloc_carrier(allctr, crr); } + else { - blk = UMEM2BLK(ptr); + INC_CC(allctr->calls.this_free); - if (IS_SBC_BLK(blk)) - destroy_carrier(allctr, blk); - else - mbc_free(allctr, ptr); - ERTS_DBG_CHK_FIX_LIST(allctr, fix, ix, 0); + if (fix) + handle_delayed_fix_dealloc(allctr, ptr); + else + dealloc_block(allctr, ptr, NULL, 1); + } } if (need_thr_progress && !(need_thr_prgr | need_mr_wrk)) { @@ -1230,6 +1915,12 @@ handle_delayed_dealloc(Allctr_t *allctr, store_earliest_thr_prgr(thr_prgr_p, ddq); } + if (ERTS_ALC_IS_CPOOL_ENABLED(allctr)) + check_pending_dealloc_carrier(allctr, + need_thr_progress, + thr_prgr_p, + need_more_work); + if (allctr->thread_safe && !allctr_locked) erts_mtx_unlock(&allctr->mutex); return res; @@ -1242,15 +1933,61 @@ enqueue_dealloc_other_instance(ErtsAlcType_t type, int cinit) { if (allctr->fix) - ((UWord *) ptr)[ERTS_ALCU_DD_FIX_TYPE_OFFS] = (UWord) type; + ((ErtsAllctrFixDDBlock_t*) ptr)->fix_type = type; - if (ddq_enqueue(type, &allctr->dd.q, ptr, cinit)) + if (ddq_enqueue(&allctr->dd.q, ptr, cinit)) erts_alloc_notify_delayed_dealloc(allctr->ix); } #endif #ifdef ERTS_SMP +static void +set_new_allctr_abandon_limit(Allctr_t *allctr); +static void +abandon_carrier(Allctr_t *allctr, Carrier_t *crr); + + +static ERTS_INLINE void +check_abandon_carrier(Allctr_t *allctr, Block_t *fblk, Carrier_t **busy_pcrr_pp) +{ + Carrier_t *crr; + + if (busy_pcrr_pp && *busy_pcrr_pp) + return; + + if (!ERTS_ALC_IS_CPOOL_ENABLED(allctr)) + return; + + allctr->cpool.check_limit_count--; + if (--allctr->cpool.check_limit_count <= 0) + set_new_allctr_abandon_limit(allctr); + + if (!erts_thr_progress_is_managed_thread()) + return; + + if (allctr->cpool.disable_abandon) + return; + + if (allctr->mbcs.blocks.curr.size > allctr->cpool.abandon_limit) + return; + + + crr = FBLK_TO_MBC(fblk); + + if (allctr->main_carrier == crr) + return; + + if (crr->cpool.blocks_size > crr->cpool.abandon_limit) + return; + + if (crr->cpool.thr_prgr != ERTS_THR_PRGR_INVALID + && !erts_thr_progress_has_reached(crr->cpool.thr_prgr)) + return; + + abandon_carrier(allctr, crr); +} + void erts_alcu_check_delayed_dealloc(Allctr_t *allctr, int limit, @@ -1268,78 +2005,103 @@ erts_alcu_check_delayed_dealloc(Allctr_t *allctr, } #endif -#define ERTS_ALCU_HANDLE_DD_IN_OP(Allctr, Locked) \ - handle_delayed_dealloc((Allctr), (Locked), 1, \ +#define ERTS_ALCU_HANDLE_DD_IN_OP(Allctr, Locked) \ + handle_delayed_dealloc((Allctr), (Locked), 1, \ ERTS_ALCU_DD_OPS_LIM_LOW, NULL, NULL, NULL) +static void +dealloc_block(Allctr_t *allctr, void *ptr, ErtsAlcFixList_t *fix, int dec_cc_on_redirect) +{ + Block_t *blk = UMEM2BLK(ptr); + + ERTS_SMP_LC_ASSERT(!allctr->thread_safe + || erts_lc_mtx_is_locked(&allctr->mutex)); + + if (IS_SBC_BLK(blk)) { + destroy_carrier(allctr, blk, NULL); +#ifdef ERTS_SMP + if (fix && ERTS_ALC_IS_CPOOL_ENABLED(allctr)) { + ErtsAlcType_t type = ((ErtsAllctrFixDDBlock_t *) ptr)->fix_type; + if (!(type & ERTS_ALC_FIX_NO_UNUSE)) + fix->u.cpool.used--; + fix->u.cpool.allocated--; + } +#endif + } +#ifndef ERTS_SMP + else + mbc_free(allctr, ptr, NULL); +#else + else if (!ERTS_ALC_IS_CPOOL_ENABLED(allctr)) + mbc_free(allctr, ptr, NULL); + else { + Carrier_t *busy_pcrr_p; + Allctr_t *used_allctr; + used_allctr = get_used_allctr(allctr, ERTS_ALC_TS_PREF_LOCK_NO, ptr, + NULL, &busy_pcrr_p); + if (used_allctr == allctr) { + if (fix) { + ErtsAlcType_t type = ((ErtsAllctrFixDDBlock_t *) ptr)->fix_type; + if (!(type & ERTS_ALC_FIX_NO_UNUSE)) + fix->u.cpool.used--; + fix->u.cpool.allocated--; + } + mbc_free(allctr, ptr, &busy_pcrr_p); + clear_busy_pool_carrier(allctr, busy_pcrr_p); + } + else { + /* Carrier migrated; need to redirect block to new owner... */ + int cinit = used_allctr->dd.ix - allctr->dd.ix; + + ERTS_ALC_CPOOL_ASSERT(!busy_pcrr_p); + + if (dec_cc_on_redirect) + DEC_CC(allctr->calls.this_free); + if (ddq_enqueue(&used_allctr->dd.q, ptr, cinit)) + erts_alloc_notify_delayed_dealloc(used_allctr->ix); + } + } +#endif +} + /* Multi block carrier alloc/realloc/free ... */ /* NOTE! mbc_alloc() may in case of memory shortage place the requested * block in a sbc. */ static ERTS_INLINE void * -mbc_alloc_block(Allctr_t *allctr, Uint size, Uint *blk_szp, Uint32 *alcu_flgsp) +mbc_alloc_block(Allctr_t *allctr, Uint size, Uint *blk_szp) { Block_t *blk; Uint get_blk_sz; - Uint sbmbct; ASSERT(size); ASSERT(size < allctr->sbc_threshold); *blk_szp = get_blk_sz = UMEMSZ2BLKSZ(allctr, size); - sbmbct = allctr->sbmbc_threshold; - if (sbmbct) { - if (get_blk_sz < sbmbct) { - *alcu_flgsp |= ERTS_ALCU_FLG_SBMBC; - if (get_blk_sz + allctr->min_block_size > sbmbct) { - /* Since we use block size to determine if blocks are - located in sbmbc or not... */ - get_blk_sz += allctr->min_block_size; - } - } - } - -#ifdef ERTS_SMP - if (allctr->dd.use) - ERTS_ALCU_HANDLE_DD_IN_OP(allctr, 1); -#endif - - blk = (*allctr->get_free_block)(allctr, get_blk_sz, NULL, 0, *alcu_flgsp); - -#ifdef ERTS_SMP - if (!blk && allctr->dd.use) { - if (ERTS_ALCU_HANDLE_DD_IN_OP(allctr, 1)) - blk = (*allctr->get_free_block)(allctr, get_blk_sz, NULL, 0, - *alcu_flgsp); - } -#endif + blk = (*allctr->get_free_block)(allctr, get_blk_sz, NULL, 0); if (!blk) { - if ((*alcu_flgsp) & ERTS_ALCU_FLG_SBMBC) - blk = create_sbmbc(allctr, get_blk_sz); - else { - blk = create_carrier(allctr, get_blk_sz, CFLG_MBC); -#if !HALFWORD_HEAP && !HAVE_SUPER_ALIGNED_MB_CARRIERS - if (!blk) { - /* Emergency! We couldn't create the carrier as we wanted. - Try to place it in a sys_alloced sbc. */ - blk = create_carrier(allctr, - size, - (CFLG_SBC - | CFLG_FORCE_SIZE - | CFLG_FORCE_SYS_ALLOC)); - } -#endif + blk = create_carrier(allctr, get_blk_sz, CFLG_MBC); +#if !HALFWORD_HEAP && !ERTS_SUPER_ALIGNED_MSEG_ONLY + if (!blk) { + /* Emergency! We couldn't create the carrier as we wanted. + Try to place it in a sys_alloced sbc. */ + blk = create_carrier(allctr, + size, + (CFLG_SBC + | CFLG_FORCE_SIZE + | CFLG_FORCE_SYS_ALLOC)); } +#endif } #ifdef ERTS_ALLOC_UTIL_HARD_DEBUG if (IS_MBC_BLK(blk)) { - (*allctr->link_free_block)(allctr, blk, *alcu_flgsp); + (*allctr->link_free_block)(allctr, blk); HARD_CHECK_BLK_CARRIER(allctr, blk); - (*allctr->unlink_free_block)(allctr, blk, *alcu_flgsp); + (*allctr->unlink_free_block)(allctr, blk); } #endif @@ -1353,8 +2115,7 @@ mbc_alloc_finalize(Allctr_t *allctr, UWord flags, Carrier_t *crr, Uint want_blk_sz, - int valid_blk_info, - Uint32 alcu_flgs) + int valid_blk_info) { Uint blk_sz; Uint nxt_blk_sz; @@ -1386,7 +2147,7 @@ mbc_alloc_finalize(Allctr_t *allctr, SET_PREV_BLK_FREE(allctr, nxt_nxt_blk); } } - (*allctr->link_free_block)(allctr, nxt_blk, alcu_flgs); + (*allctr->link_free_block)(allctr, nxt_blk); ASSERT(IS_NOT_LAST_BLK(blk)); ASSERT(IS_FREE_BLK(nxt_blk)); @@ -1427,7 +2188,8 @@ mbc_alloc_finalize(Allctr_t *allctr, ASSERT(ABLK_TO_MBC(blk) == crr); } - STAT_MBC_BLK_ALLOC(allctr, blk_sz, alcu_flgs); + ERTS_ALC_CPOOL_ALLOC_OP(allctr); + STAT_MBC_BLK_ALLOC(allctr, crr, blk_sz, alcu_flgs); ASSERT(IS_ALLOCED_BLK(blk)); ASSERT(blk_sz == MBC_BLK_SZ(blk)); @@ -1447,8 +2209,7 @@ mbc_alloc(Allctr_t *allctr, Uint size) { Block_t *blk; Uint blk_sz; - Uint32 alcu_flgs = 0; - blk = mbc_alloc_block(allctr, size, &blk_sz, &alcu_flgs); + blk = mbc_alloc_block(allctr, size, &blk_sz); if (!blk) return NULL; if (IS_MBC_BLK(blk)) @@ -1458,35 +2219,34 @@ mbc_alloc(Allctr_t *allctr, Uint size) GET_BLK_HDR_FLGS(blk), FBLK_TO_MBC(blk), blk_sz, - 1, - alcu_flgs); + 1); return BLK2UMEM(blk); } static void -mbc_free(Allctr_t *allctr, void *p) +mbc_free(Allctr_t *allctr, void *p, Carrier_t **busy_pcrr_pp) { Uint is_first_blk; Uint is_last_blk; - Uint32 alcu_flgs = 0; Uint blk_sz; Block_t *blk; Block_t *nxt_blk; - + Carrier_t *crr; ASSERT(p); blk = UMEM2BLK(p); blk_sz = MBC_ABLK_SZ(blk); - if (blk_sz < allctr->sbmbc_threshold) - alcu_flgs |= ERTS_ALCU_FLG_SBMBC; ASSERT(IS_MBC_BLK(blk)); ASSERT(blk_sz >= allctr->min_block_size); HARD_CHECK_BLK_CARRIER(allctr, blk); - STAT_MBC_BLK_FREE(allctr, blk_sz, alcu_flgs); + crr = ABLK_TO_MBC(blk); + + ERTS_ALC_CPOOL_FREE_OP(allctr); + STAT_MBC_BLK_FREE(allctr, crr, busy_pcrr_pp, blk_sz, alcu_flgs); is_first_blk = IS_MBC_FIRST_ABLK(allctr, blk); is_last_blk = IS_LAST_BLK(blk); @@ -1495,7 +2255,7 @@ mbc_free(Allctr_t *allctr, void *p) ASSERT(!is_first_blk); /* Coalesce with previous block... */ blk = PREV_BLK(blk); - (*allctr->unlink_free_block)(allctr, blk, alcu_flgs); + (*allctr->unlink_free_block)(allctr, blk); blk_sz += MBC_FBLK_SZ(blk); is_first_blk = IS_MBC_FIRST_FBLK(allctr, blk); @@ -1511,7 +2271,7 @@ mbc_free(Allctr_t *allctr, void *p) nxt_blk = BLK_AFTER(blk, blk_sz); if (IS_FREE_BLK(nxt_blk)) { /* Coalesce with next block... */ - (*allctr->unlink_free_block)(allctr, nxt_blk, alcu_flgs); + (*allctr->unlink_free_block)(allctr, nxt_blk); blk_sz += MBC_FBLK_SZ(nxt_blk); SET_MBC_FBLK_SZ(blk, blk_sz); @@ -1544,19 +2304,20 @@ mbc_free(Allctr_t *allctr, void *p) if (is_first_blk && is_last_blk && allctr->main_carrier != FIRST_BLK_TO_MBC(allctr, blk)) { - if (alcu_flgs & ERTS_ALCU_FLG_SBMBC) - destroy_sbmbc(allctr, blk); - else - destroy_carrier(allctr, blk); + destroy_carrier(allctr, blk, busy_pcrr_pp); } else { - (*allctr->link_free_block)(allctr, blk, alcu_flgs); + (*allctr->link_free_block)(allctr, blk); HARD_CHECK_BLK_CARRIER(allctr, blk); +#ifdef ERTS_SMP + check_abandon_carrier(allctr, blk, busy_pcrr_pp); +#endif } } static void * -mbc_realloc(Allctr_t *allctr, void *p, Uint size, Uint32 alcu_flgs) +mbc_realloc(Allctr_t *allctr, void *p, Uint size, Uint32 alcu_flgs, + Carrier_t **busy_pcrr_pp) { void *new_p; Uint old_blk_sz; @@ -1570,11 +2331,6 @@ mbc_realloc(Allctr_t *allctr, void *p, Uint size, Uint32 alcu_flgs) Uint is_last_blk; #endif /* #ifndef MBC_REALLOC_ALWAYS_MOVES */ -#ifdef ERTS_SMP - if (allctr->dd.use) - ERTS_ALCU_HANDLE_DD_IN_OP(allctr, 1); -#endif - ASSERT(p); ASSERT(size); ASSERT(size < allctr->sbc_threshold); @@ -1588,13 +2344,13 @@ mbc_realloc(Allctr_t *allctr, void *p, Uint size, Uint32 alcu_flgs) if (alcu_flgs & ERTS_ALCU_FLG_FAIL_REALLOC_MOVE) return NULL; #else /* !MBC_REALLOC_ALWAYS_MOVES */ + +#ifdef ERTS_SMP + if (busy_pcrr_pp && *busy_pcrr_pp) + goto realloc_move; /* Don't want to use carrier in pool */ +#endif + get_blk_sz = blk_sz = UMEMSZ2BLKSZ(allctr, size); - if ((alcu_flgs & ERTS_ALCU_FLG_SBMBC) - && (blk_sz + allctr->min_block_size > allctr->sbmbc_threshold)) { - /* Since we use block size to determine if blocks are - located in sbmbc or not... */ - get_blk_sz = blk_sz + allctr->min_block_size; - } ASSERT(IS_ALLOCED_BLK(blk)); ASSERT(IS_MBC_BLK(blk)); @@ -1642,8 +2398,7 @@ mbc_realloc(Allctr_t *allctr, void *p, Uint size, Uint32 alcu_flgs) new_blk = (*allctr->get_free_block)(allctr, get_blk_sz, cand_blk, - cand_blk_sz, - alcu_flgs); + cand_blk_sz); if (new_blk || cand_blk != blk) goto move_into_new_blk; } @@ -1665,8 +2420,11 @@ mbc_realloc(Allctr_t *allctr, void *p, Uint size, Uint32 alcu_flgs) nxt_blk = BLK_AFTER(blk, blk_sz); - STAT_MBC_BLK_FREE(allctr, old_blk_sz, alcu_flgs); - STAT_MBC_BLK_ALLOC(allctr, blk_sz, alcu_flgs); + crr = ABLK_TO_MBC(blk); + + ERTS_ALC_CPOOL_REALLOC_OP(allctr); + STAT_MBC_BLK_FREE(allctr, crr, NULL, old_blk_sz, alcu_flgs); + STAT_MBC_BLK_ALLOC(allctr, crr, blk_sz, alcu_flgs); ASSERT(MBC_BLK_SZ(blk) >= allctr->min_block_size); @@ -1674,7 +2432,7 @@ mbc_realloc(Allctr_t *allctr, void *p, Uint size, Uint32 alcu_flgs) if (IS_FREE_BLK(nxt_nxt_blk)) { /* Coalesce with next free block... */ nxt_blk_sz += MBC_FBLK_SZ(nxt_nxt_blk); - (*allctr->unlink_free_block)(allctr, nxt_nxt_blk, alcu_flgs); + (*allctr->unlink_free_block)(allctr, nxt_nxt_blk); is_last_blk = GET_LAST_BLK_HDR_FLG(nxt_nxt_blk); } @@ -1684,12 +2442,11 @@ mbc_realloc(Allctr_t *allctr, void *p, Uint size, Uint32 alcu_flgs) SET_BLK_SZ_FTR(nxt_blk, nxt_blk_sz); } - crr = ABLK_TO_MBC(blk); SET_MBC_FBLK_HDR(nxt_blk, nxt_blk_sz, SBH_THIS_FREE | (is_last_blk ? SBH_LAST_BLK : 0), crr); - (*allctr->link_free_block)(allctr, nxt_blk, alcu_flgs); + (*allctr->link_free_block)(allctr, nxt_blk); ASSERT(IS_ALLOCED_BLK(blk)); @@ -1712,6 +2469,10 @@ mbc_realloc(Allctr_t *allctr, void *p, Uint size, Uint32 alcu_flgs) HARD_CHECK_BLK_CARRIER(allctr, blk); +#ifdef ERTS_SMP + check_abandon_carrier(allctr, nxt_blk, NULL); +#endif + return p; } @@ -1721,11 +2482,12 @@ mbc_realloc(Allctr_t *allctr, void *p, Uint size, Uint32 alcu_flgs) nxt_blk = BLK_AFTER(blk, old_blk_sz); nxt_blk_sz = MBC_BLK_SZ(nxt_blk); if (IS_FREE_BLK(nxt_blk) && get_blk_sz <= old_blk_sz + nxt_blk_sz) { + Carrier_t* crr = ABLK_TO_MBC(blk); /* Grow into next block... */ HARD_CHECK_BLK_CARRIER(allctr, blk); - (*allctr->unlink_free_block)(allctr, nxt_blk, alcu_flgs); + (*allctr->unlink_free_block)(allctr, nxt_blk); nxt_blk_sz -= blk_sz - old_blk_sz; is_last_blk = IS_LAST_BLK(nxt_blk); @@ -1750,7 +2512,6 @@ mbc_realloc(Allctr_t *allctr, void *p, Uint size, Uint32 alcu_flgs) } } else { - Carrier_t* crr = ABLK_TO_MBC(blk); SET_MBC_ABLK_SZ(blk, blk_sz); nxt_blk = BLK_AFTER(blk, blk_sz); @@ -1761,15 +2522,15 @@ mbc_realloc(Allctr_t *allctr, void *p, Uint size, Uint32 alcu_flgs) else SET_BLK_SZ_FTR(nxt_blk, nxt_blk_sz); - (*allctr->link_free_block)(allctr, nxt_blk, alcu_flgs); + (*allctr->link_free_block)(allctr, nxt_blk); ASSERT(IS_FREE_BLK(nxt_blk)); ASSERT(FBLK_TO_MBC(nxt_blk) == crr); } - STAT_MBC_BLK_FREE(allctr, old_blk_sz, alcu_flgs); - STAT_MBC_BLK_ALLOC(allctr, blk_sz, alcu_flgs); - + ERTS_ALC_CPOOL_REALLOC_OP(allctr); + STAT_MBC_BLK_FREE(allctr, crr, NULL, old_blk_sz, alcu_flgs); + STAT_MBC_BLK_ALLOC(allctr, crr, blk_sz, alcu_flgs); ASSERT(IS_ALLOCED_BLK(blk)); ASSERT(blk_sz == MBC_BLK_SZ(blk)); @@ -1822,13 +2583,16 @@ mbc_realloc(Allctr_t *allctr, void *p, Uint size, Uint32 alcu_flgs) if (cand_blk_sz < get_blk_sz) { /* We wont fit in cand_blk get a new one */ +#ifdef ERTS_SMP + realloc_move: +#endif #endif /* !MBC_REALLOC_ALWAYS_MOVES */ new_p = mbc_alloc(allctr, size); if (!new_p) return NULL; sys_memcpy(new_p, p, MIN(size, old_blk_sz - ABLK_HDR_SZ)); - mbc_free(allctr, p); + mbc_free(allctr, p, busy_pcrr_pp); return new_p; @@ -1841,8 +2605,7 @@ mbc_realloc(Allctr_t *allctr, void *p, Uint size, Uint32 alcu_flgs) new_blk = (*allctr->get_free_block)(allctr, get_blk_sz, cand_blk, - cand_blk_sz, - alcu_flgs); + cand_blk_sz); move_into_new_blk: /* * new_blk, and cand_blk have to be correctly set @@ -1856,11 +2619,10 @@ mbc_realloc(Allctr_t *allctr, void *p, Uint size, Uint32 alcu_flgs) GET_BLK_HDR_FLGS(new_blk), FBLK_TO_MBC(new_blk), blk_sz, - 1, - alcu_flgs); + 1); new_p = BLK2UMEM(new_blk); sys_memcpy(new_p, p, MIN(size, old_blk_sz - ABLK_HDR_SZ)); - mbc_free(allctr, p); + mbc_free(allctr, p, NULL); return new_p; } else { @@ -1880,7 +2642,7 @@ mbc_realloc(Allctr_t *allctr, void *p, Uint size, Uint32 alcu_flgs) HARD_CHECK_BLK_CARRIER(allctr, blk); - (*allctr->unlink_free_block)(allctr, new_blk, alcu_flgs); /* prev */ + (*allctr->unlink_free_block)(allctr, new_blk); /* prev */ if (is_last_blk) new_blk_flgs |= LAST_BLK_HDR_FLG; @@ -1889,7 +2651,7 @@ mbc_realloc(Allctr_t *allctr, void *p, Uint size, Uint32 alcu_flgs) if (IS_FREE_BLK(nxt_blk)) { new_blk_flgs |= GET_LAST_BLK_HDR_FLG(nxt_blk); new_blk_sz += MBC_FBLK_SZ(nxt_blk); - (*allctr->unlink_free_block)(allctr, nxt_blk, alcu_flgs); + (*allctr->unlink_free_block)(allctr, nxt_blk); } } @@ -1914,10 +2676,10 @@ mbc_realloc(Allctr_t *allctr, void *p, Uint size, Uint32 alcu_flgs) new_blk_flgs, crr, blk_sz, - 0, - alcu_flgs); + 0); - STAT_MBC_BLK_FREE(allctr, old_blk_sz, alcu_flgs); + ERTS_ALC_CPOOL_FREE_OP(allctr); + STAT_MBC_BLK_FREE(allctr, crr, NULL, old_blk_sz, alcu_flgs); return new_p; } @@ -1925,123 +2687,836 @@ mbc_realloc(Allctr_t *allctr, void *p, Uint size, Uint32 alcu_flgs) #endif /* !MBC_REALLOC_ALWAYS_MOVES */ } -#ifdef DEBUG +#ifdef ERTS_SMP -#if HAVE_ERTS_MSEG -#define ASSERT_MSEG_UNIT_SIZE_MULTIPLE(CSZ) ASSERT((CSZ) % MSEG_UNIT_SZ == 0) +#define ERTS_ALC_MAX_DEALLOC_CARRIER 10 +#define ERTS_ALC_CPOOL_MAX_FETCH_INSPECT 20 +#define ERTS_ALC_CPOOL_MAX_TRAITOR_INSPECT 10 +#define ERTS_ALC_CPOOL_CHECK_LIMIT_COUNT 100 +#define ERTS_ALC_CPOOL_MAX_FAILED_STAT_READS 3 + +#define ERTS_ALC_CPOOL_PTR_MOD_MRK (((erts_aint_t) 1) << 0) +#define ERTS_ALC_CPOOL_PTR_DEL_MRK (((erts_aint_t) 1) << 1) + +#define ERTS_ALC_CPOOL_PTR_MRKS \ + (ERTS_ALC_CPOOL_PTR_MOD_MRK | ERTS_ALC_CPOOL_PTR_DEL_MRK) + +/* + * When setting multiple mod markers we always + * set mod markers in pointer order and always + * on next pointers before prev pointers. + */ + +typedef union { + ErtsAlcCPoolData_t sentinel; + char align__[ERTS_ALC_CACHE_LINE_ALIGN_SIZE(sizeof(ErtsAlcCPoolData_t))]; +} ErtsAlcCrrPool_t; + +#if ERTS_ALC_A_INVALID != 0 +# error "Carrier pool implementation assumes ERTS_ALC_A_INVALID == 0" +#endif +#if ERTS_ALC_A_MIN <= ERTS_ALC_A_INVALID +# error "Carrier pool implementation assumes ERTS_ALC_A_MIN > ERTS_ALC_A_INVALID" +#endif + +/* + * The pool is only allowed to be manipulated by managed + * threads except in the alloc_SUITE:cpool case. In this + * test case carrier_pool[ERTS_ALC_A_INVALID] will be + * used. + */ + +static ErtsAlcCrrPool_t carrier_pool[ERTS_ALC_A_MAX+1] erts_align_attribute(ERTS_CACHE_LINE_SIZE); + +#define ERTS_ALC_CPOOL_MAX_BACKOFF (1 << 8) + +static int +backoff(int n) +{ + int i; + + for (i = 0; i < n; i++) + ERTS_SPIN_BODY; + + if (n >= ERTS_ALC_CPOOL_MAX_BACKOFF) + return ERTS_ALC_CPOOL_MAX_BACKOFF; + else + return n << 1; +} + +static int +cpool_dbg_is_in_pool(Allctr_t *allctr, Carrier_t *crr) +{ + ErtsAlcCPoolData_t *sentinel = &carrier_pool[allctr->alloc_no].sentinel; + ErtsAlcCPoolData_t *cpdp = sentinel; + Carrier_t *tmp_crr; + + while (1) { + cpdp = (ErtsAlcCPoolData_t *) (erts_atomic_read_ddrb(&cpdp->next) & ~FLG_MASK); + if (cpdp == sentinel) + return 0; + tmp_crr = (Carrier_t *) (((char *) cpdp) - offsetof(Carrier_t, cpool)); + if (tmp_crr == crr) + return 1; + } +} + +static int +cpool_is_empty(Allctr_t *allctr) +{ + ErtsAlcCPoolData_t *sentinel = &carrier_pool[allctr->alloc_no].sentinel; + return ((erts_atomic_read_rb(&sentinel->next) == (erts_aint_t) sentinel) + && (erts_atomic_read_rb(&sentinel->prev) == (erts_aint_t) sentinel)); +} + +static ERTS_INLINE ErtsAlcCPoolData_t * +cpool_aint2cpd(erts_aint_t aint) +{ + return (ErtsAlcCPoolData_t *) (aint & ~ERTS_ALC_CPOOL_PTR_MRKS); +} + +static ERTS_INLINE erts_aint_t +cpool_read(erts_atomic_t *aptr) +{ + return erts_atomic_read_acqb(aptr); +} + +static ERTS_INLINE void +cpool_init(erts_atomic_t *aptr, erts_aint_t val) +{ + erts_atomic_set_nob(aptr, val); +} + +static ERTS_INLINE void +cpool_set_mod_marked(erts_atomic_t *aptr, erts_aint_t new, erts_aint_t old) +{ +#ifdef ERTS_ALC_CPOOL_DEBUG + erts_aint_t act = erts_atomic_xchg_relb(aptr, new); + ERTS_ALC_CPOOL_ASSERT(act == (old | ERTS_ALC_CPOOL_PTR_MOD_MRK)); #else -#define ASSERT_MSEG_UNIT_SIZE_MULTIPLE(CSZ) + erts_atomic_set_relb(aptr, new); #endif +} -static void CHECK_1BLK_CARRIER(Allctr_t* A, int SBC, int MSEGED, Carrier_t* C, - UWord CSZ, Block_t* B, UWord BSZ) + +static ERTS_INLINE erts_aint_t +cpool_try_mod_mark_exp(erts_atomic_t *aptr, erts_aint_t exp) { - ASSERT(IS_LAST_BLK((B))); - ASSERT((CSZ) == CARRIER_SZ((C))); - ASSERT((BSZ) % sizeof(Unit_t) == 0); - if ((SBC)) { - ASSERT((BSZ) == SBC_BLK_SZ((B))); - ASSERT((char*)B == (char*)C + SBC_HEADER_SIZE); - ASSERT(IS_SBC_BLK((B))); - ASSERT(IS_SB_CARRIER((C))); + ERTS_ALC_CPOOL_ASSERT((exp & ERTS_ALC_CPOOL_PTR_MOD_MRK) == 0); + return erts_atomic_cmpxchg_nob(aptr, exp | ERTS_ALC_CPOOL_PTR_MOD_MRK, exp); +} + +static ERTS_INLINE erts_aint_t +cpool_mod_mark_exp(erts_atomic_t *aptr, erts_aint_t exp) +{ + int b; + erts_aint_t act; + ERTS_ALC_CPOOL_ASSERT((exp & ERTS_ALC_CPOOL_PTR_MOD_MRK) == 0); + while (1) { + act = erts_atomic_cmpxchg_nob(aptr, + exp | ERTS_ALC_CPOOL_PTR_MOD_MRK, + exp); + if (act == exp) + return exp; + b = 1; + do { + if ((act & ~ERTS_ALC_CPOOL_PTR_MOD_MRK) != exp) + return act; + b = backoff(b); + act = erts_atomic_read_nob(aptr); + } while (act != exp); } - else { - ASSERT(IS_FREE_BLK(B)); - ASSERT((BSZ) == MBC_FBLK_SZ((B))); - ASSERT(IS_MBC_FIRST_FBLK(A, (B))); - ASSERT(IS_MBC_BLK((B))); - ASSERT(IS_MB_CARRIER((C))); - ASSERT(FBLK_TO_MBC(B) == (C)); +} + +static ERTS_INLINE erts_aint_t +cpool_mod_mark(erts_atomic_t *aptr) +{ + int b; + erts_aint_t act, exp; + act = cpool_read(aptr); + while (1) { + b = 1; + while (act & ERTS_ALC_CPOOL_PTR_MOD_MRK) { + b = backoff(b); + act = erts_atomic_read_nob(aptr); + } + exp = act; + act = erts_atomic_cmpxchg_acqb(aptr, + exp | ERTS_ALC_CPOOL_PTR_MOD_MRK, + exp); + if (act == exp) + return exp; } - if ((MSEGED)) { - ASSERT(IS_MSEG_CARRIER((C))); - ASSERT_MSEG_UNIT_SIZE_MULTIPLE((CSZ)); +} + +static void +cpool_insert(Allctr_t *allctr, Carrier_t *crr) +{ + ErtsAlcCPoolData_t *cpd1p, *cpd2p; + erts_aint_t val; + ErtsAlcCPoolData_t *sentinel = &carrier_pool[allctr->alloc_no].sentinel; + + ERTS_ALC_CPOOL_ASSERT(allctr->alloc_no == ERTS_ALC_A_INVALID /* testcase */ + || erts_thr_progress_is_managed_thread()); + ERTS_ALC_CPOOL_ASSERT(erts_smp_atomic_read_nob(&crr->allctr) + == (erts_aint_t) allctr); + + erts_atomic_add_nob(&allctr->cpool.stat.blocks_size, + (erts_aint_t) crr->cpool.blocks_size); + erts_atomic_add_nob(&allctr->cpool.stat.no_blocks, + (erts_aint_t) crr->cpool.blocks); + erts_atomic_add_nob(&allctr->cpool.stat.carriers_size, + (erts_aint_t) CARRIER_SZ(crr)); + erts_atomic_inc_nob(&allctr->cpool.stat.no_carriers); + + /* + * We search in 'next' direction and begin by passing + * one element before trying to insert. This in order to + * avoid contention with threads fetching elements. + */ + + val = cpool_read(&sentinel->next); + + /* Find a predecessor to be, and set mod marker on its next ptr */ + + while (1) { + cpd1p = cpool_aint2cpd(val); + if (cpd1p == sentinel) { + val = cpool_mod_mark(&cpd1p->next); + break; + } + val = cpool_read(&cpd1p->next); + if (!(val & ERTS_ALC_CPOOL_PTR_MRKS)) { + erts_aint_t tmp = cpool_try_mod_mark_exp(&cpd1p->next, val); + if (tmp == val) { + val = tmp; + break; + } + val = tmp; + } } - else { - ASSERT(IS_SYS_ALLOC_CARRIER((C))); - ASSERT((CSZ) % sizeof(Unit_t) == 0); + + /* Set mod marker on prev ptr of the to be successor */ + + cpd2p = cpool_aint2cpd(val); + + cpool_init(&crr->cpool.next, (erts_aint_t) cpd2p); + cpool_init(&crr->cpool.prev, (erts_aint_t) cpd1p); + + val = (erts_aint_t) cpd1p; + + while (1) { + int b; + erts_aint_t tmp; + + tmp = cpool_mod_mark_exp(&cpd2p->prev, val); + if (tmp == val) + break; + b = 1; + do { + b = backoff(b); + tmp = cpool_read(&cpd2p->prev); + } while (tmp != val); } + + /* Write pointers to this element in successor and predecessor */ + + cpool_set_mod_marked(&cpd1p->next, + (erts_aint_t) &crr->cpool, + (erts_aint_t) cpd2p); + cpool_set_mod_marked(&cpd2p->prev, + (erts_aint_t) &crr->cpool, + (erts_aint_t) cpd1p); + + erts_smp_atomic_set_wb(&crr->allctr, + ((erts_aint_t) allctr)|ERTS_CRR_ALCTR_FLG_IN_POOL); } -#else -#define CHECK_1BLK_CARRIER(A, SBC, MSEGED, C, CSZ, B, BSZ) +static void +cpool_delete(Allctr_t *allctr, Allctr_t *prev_allctr, Carrier_t *crr) +{ + ErtsAlcCPoolData_t *cpd1p, *cpd2p; + erts_aint_t val; +#ifdef ERTS_ALC_CPOOL_DEBUG + ErtsAlcCPoolData_t *sentinel = &carrier_pool[allctr->alloc_no].sentinel; #endif -static Block_t * -create_sbmbc(Allctr_t *allctr, Uint umem_sz) + ERTS_ALC_CPOOL_ASSERT(allctr->alloc_no == ERTS_ALC_A_INVALID /* testcase */ + || erts_thr_progress_is_managed_thread()); + ERTS_ALC_CPOOL_ASSERT(sentinel != &crr->cpool); + + /* Set mod marker on next ptr of our predecessor */ + + val = (erts_aint_t) &crr->cpool; + while (1) { + erts_aint_t tmp; + cpd1p = cpool_aint2cpd(cpool_read(&crr->cpool.prev)); + tmp = cpool_mod_mark_exp(&cpd1p->next, val); + if (tmp == val) + break; + } + + /* Set mod marker on our next ptr */ + + val = cpool_mod_mark(&crr->cpool.next); + + /* Set mod marker on the prev ptr of our successor */ + + cpd2p = cpool_aint2cpd(val); + + val = (erts_aint_t) &crr->cpool; + + while (1) { + int b; + erts_aint_t tmp; + + tmp = cpool_mod_mark_exp(&cpd2p->prev, val); + if (tmp == val) + break; + b = 1; + do { + b = backoff(b); + tmp = cpool_read(&cpd2p->prev); + } while (tmp != val); + } + + /* Set mod marker on our prev ptr */ + + val = (erts_aint_t) cpd1p; + + while (1) { + int b; + erts_aint_t tmp; + + tmp = cpool_mod_mark_exp(&crr->cpool.prev, val); + if (tmp == val) + break; + b = 1; + do { + b = backoff(b); + tmp = cpool_read(&cpd2p->prev); + } while (tmp != val); + } + + /* Write pointers past this element in predecessor and successor */ + + cpool_set_mod_marked(&cpd1p->next, + (erts_aint_t) cpd2p, + (erts_aint_t) &crr->cpool); + cpool_set_mod_marked(&cpd2p->prev, + (erts_aint_t) cpd1p, + (erts_aint_t) &crr->cpool); + + /* Repleace mod markers with delete markers on this element */ + cpool_set_mod_marked(&crr->cpool.next, + ((erts_aint_t) cpd2p) | ERTS_ALC_CPOOL_PTR_DEL_MRK, + ((erts_aint_t) cpd2p) | ERTS_ALC_CPOOL_PTR_MOD_MRK); + cpool_set_mod_marked(&crr->cpool.prev, + ((erts_aint_t) cpd1p) | ERTS_ALC_CPOOL_PTR_DEL_MRK, + ((erts_aint_t) cpd1p) | ERTS_ALC_CPOOL_PTR_MOD_MRK); + + crr->cpool.thr_prgr = erts_thr_progress_later(NULL); + + erts_atomic_add_nob(&prev_allctr->cpool.stat.blocks_size, + -((erts_aint_t) crr->cpool.blocks_size)); + erts_atomic_add_nob(&prev_allctr->cpool.stat.no_blocks, + -((erts_aint_t) crr->cpool.blocks)); + erts_atomic_add_nob(&prev_allctr->cpool.stat.carriers_size, + -((erts_aint_t) CARRIER_SZ(crr))); + erts_atomic_dec_wb(&prev_allctr->cpool.stat.no_carriers); + +} + +static Carrier_t * +cpool_fetch(Allctr_t *allctr, UWord size) { - Block_t *blk; - Uint blk_sz; - Uint crr_sz = allctr->sbmbc_size; + int i, i_stop, has_passed_sentinel; Carrier_t *crr; + ErtsAlcCPoolData_t *cpdp; + ErtsAlcCPoolData_t *cpool_entrance; + ErtsAlcCPoolData_t *sentinel; + ErtsDoubleLink_t* dl; + ErtsDoubleLink_t* first_old_traitor; + + ERTS_ALC_CPOOL_ASSERT(allctr->alloc_no == ERTS_ALC_A_INVALID /* testcase */ + || erts_thr_progress_is_managed_thread()); + + i = ERTS_ALC_CPOOL_MAX_FETCH_INSPECT; + first_old_traitor = allctr->cpool.traitor_list.next; + cpool_entrance = NULL; + + /* + * Search my own pooled_list, + * i.e my abandoned carriers that were in the pool last time I checked. + */ + + dl = allctr->cpool.pooled_list.next; + while(dl != &allctr->cpool.pooled_list) { + erts_aint_t exp, act; + crr = (Carrier_t *) (((char *) dl) - offsetof(Carrier_t, cpool.abandoned)); + + ASSERT(!is_in_list(&allctr->cpool.traitor_list, dl)); + ASSERT(crr->cpool.orig_allctr == allctr); + dl = dl->next; + exp = erts_smp_atomic_read_rb(&crr->allctr); + if ((exp & ERTS_CRR_ALCTR_FLG_MASK) == ERTS_CRR_ALCTR_FLG_IN_POOL + && erts_atomic_read_nob(&crr->cpool.max_size) >= size) { + /* Try to fetch it... */ + act = erts_smp_atomic_cmpxchg_mb(&crr->allctr, + (erts_aint_t) allctr, + exp); + if (act == exp) { + cpool_delete(allctr, ((Allctr_t *) (act & ~ERTS_CRR_ALCTR_FLG_MASK)), crr); + unlink_abandoned_carrier(crr); + + /* Move sentinel to continue next search from here */ + relink_edl_before(dl, &allctr->cpool.pooled_list); + return crr; + } + exp = act; + } + if (exp & ERTS_CRR_ALCTR_FLG_IN_POOL) { + if (!cpool_entrance) + cpool_entrance = &crr->cpool; + } + else { /* Not in pool, move to traitor_list */ + unlink_abandoned_carrier(crr); + link_abandoned_carrier(&allctr->cpool.traitor_list, crr); + } + if (--i <= 0) { + /* Move sentinel to continue next search from here */ + relink_edl_before(dl, &allctr->cpool.pooled_list); + return NULL; + } + } -#if HALFWORD_HEAP - if (allctr->mseg_opt.low_mem) - crr = erts_alloc(ERTS_ALC_T_SBMBC_LOW, crr_sz); - else + /* Now search traitor_list. + * i.e carriers employed by other allocators last time I checked. + * They might have been abandoned since then. + */ + + i_stop = (i < ERTS_ALC_CPOOL_MAX_TRAITOR_INSPECT ? + 0 : i - ERTS_ALC_CPOOL_MAX_TRAITOR_INSPECT); + dl = first_old_traitor; + while(dl != &allctr->cpool.traitor_list) { + erts_aint_t exp, act; + crr = (Carrier_t *) (((char *) dl) - offsetof(Carrier_t, cpool.abandoned)); + ASSERT(dl != &allctr->cpool.pooled_list); + ASSERT(crr->cpool.orig_allctr == allctr); + dl = dl->next; + exp = erts_smp_atomic_read_rb(&crr->allctr); + if (exp & ERTS_CRR_ALCTR_FLG_IN_POOL) { + if (!(exp & ERTS_CRR_ALCTR_FLG_BUSY) + && erts_atomic_read_nob(&crr->cpool.max_size) >= size) { + /* Try to fetch it... */ + act = erts_smp_atomic_cmpxchg_mb(&crr->allctr, + (erts_aint_t) allctr, + exp); + if (act == exp) { + cpool_delete(allctr, ((Allctr_t *) (act & ~ERTS_CRR_ALCTR_FLG_MASK)), crr); + unlink_abandoned_carrier(crr); + + /* Move sentinel to continue next search from here */ + relink_edl_before(dl, &allctr->cpool.traitor_list); + return crr; + } + exp = act; + } + if (exp & ERTS_CRR_ALCTR_FLG_IN_POOL) { + if (!cpool_entrance) + cpool_entrance = &crr->cpool; + + /* Move to pooled_list */ + unlink_abandoned_carrier(crr); + link_abandoned_carrier(&allctr->cpool.pooled_list, crr); + } + } + if (--i <= i_stop) { + /* Move sentinel to continue next search from here */ + relink_edl_before(dl, &allctr->cpool.traitor_list); + if (i > 0) + break; + else + return NULL; + } + } + + /* + * Finally search the shared pool and try employ foreign carriers + */ + + sentinel = &carrier_pool[allctr->alloc_no].sentinel; + if (cpool_entrance) { + /* We saw a pooled carried above, use it as entrance into the pool + */ + cpdp = cpool_entrance; + } + else { + /* No pooled carried seen above. Start search at cpool sentinel, + * but begin by passing one element before trying to fetch. + * This in order to avoid contention with threads inserting elements. + */ + cpool_entrance = sentinel; + cpdp = cpool_aint2cpd(cpool_read(&cpool_entrance->prev)); + if (cpdp == sentinel) + goto check_dc_list; + } + + has_passed_sentinel = 0; + while (1) { + erts_aint_t exp; + cpdp = cpool_aint2cpd(cpool_read(&cpdp->prev)); + if (cpdp == cpool_entrance) { + if (cpool_entrance == sentinel) { + cpdp = cpool_aint2cpd(cpool_read(&cpdp->prev)); + if (cpdp == sentinel) + break; + } + i = 0; /* Last one to inspect */ + } + else if (cpdp == sentinel) { + if (has_passed_sentinel) { + /* We been here before. cpool_entrance must have been removed */ + break; + } + cpdp = cpool_aint2cpd(cpool_read(&cpdp->prev)); + if (cpdp == sentinel) + break; + has_passed_sentinel = 1; + } + crr = (Carrier_t *)(((char *)cpdp) - offsetof(Carrier_t, cpool)); + exp = erts_smp_atomic_read_rb(&crr->allctr); + if (((exp & (ERTS_CRR_ALCTR_FLG_MASK)) == ERTS_CRR_ALCTR_FLG_IN_POOL) + && (erts_atomic_read_nob(&cpdp->max_size) >= size)) { + erts_aint_t act; + /* Try to fetch it... */ + act = erts_smp_atomic_cmpxchg_mb(&crr->allctr, + (erts_aint_t) allctr, + exp); + if (act == exp) { + cpool_delete(allctr, ((Allctr_t *) (act & ~ERTS_CRR_ALCTR_FLG_MASK)), crr); + if (crr->cpool.orig_allctr == allctr) { + unlink_abandoned_carrier(crr); + } + return crr; + } + } + if (--i <= 0) + return NULL; + } + +check_dc_list: + /* Last; check our own pending dealloc carrier list... */ + crr = allctr->cpool.dc_list.last; + while (crr) { + if (erts_atomic_read_nob(&crr->cpool.max_size) >= size) { + Block_t* blk; + unlink_carrier(&allctr->cpool.dc_list, crr); +#ifdef ERTS_ALC_CPOOL_DEBUG + ERTS_ALC_CPOOL_ASSERT(erts_smp_atomic_xchg_nob(&crr->allctr, + ((erts_aint_t) allctr)) + == (((erts_aint_t) allctr) & ~ERTS_CRR_ALCTR_FLG_MASK)); +#else + erts_smp_atomic_set_nob(&crr->allctr, ((erts_aint_t) allctr)); #endif - crr = erts_alloc(ERTS_ALC_T_SBMBC, crr_sz); + blk = MBC_TO_FIRST_BLK(allctr, crr); + ASSERT(FBLK_TO_MBC(blk) == crr); + allctr->link_free_block(allctr, blk); + return crr; + } + crr = crr->prev; + if (--i <= 0) + return NULL; + } - INC_CC(allctr->calls.sbmbc_alloc); - SET_CARRIER_HDR(crr, crr_sz, SCH_SYS_ALLOC|SCH_MBC, allctr); + return NULL; +} - blk = MBC_TO_FIRST_BLK(allctr, crr); +static void +check_pending_dealloc_carrier(Allctr_t *allctr, + int *need_thr_progress, + ErtsThrPrgrVal *thr_prgr_p, + int *need_more_work) +{ + Carrier_t *crr = allctr->cpool.dc_list.first; - blk_sz = UNIT_FLOOR(crr_sz - MBC_HEADER_SIZE(allctr)); + if (crr) { + ErtsThrPrgrVal current = erts_thr_progress_current(); + int i = 0; - SET_MBC_FBLK_HDR(blk, blk_sz, SBH_THIS_FREE|SBH_LAST_BLK, crr); + do { + Carrier_t *dcrr; - link_carrier(&allctr->sbmbc_list, crr); + if (!erts_thr_progress_has_reached_this(current, crr->cpool.thr_prgr)) + break; - STAT_SBMBC_ALLOC(allctr, crr_sz); - CHECK_1BLK_CARRIER(allctr, 0, 0, crr, crr_sz, blk, blk_sz); - if (allctr->creating_mbc) - (*allctr->creating_mbc)(allctr, crr, ERTS_ALCU_FLG_SBMBC); + dcrr = crr; + crr = crr->next; + dealloc_mbc(allctr, dcrr); + i++; + } while (crr && i < ERTS_ALC_MAX_DEALLOC_CARRIER); - DEBUG_SAVE_ALIGNMENT(crr); - return blk; + allctr->cpool.dc_list.first = crr; + if (!crr) + allctr->cpool.dc_list.last = NULL; + else { + crr->prev = NULL; + + if (need_more_work) { + ERTS_ALC_CPOOL_ASSERT(need_thr_progress && thr_prgr_p); + if (erts_thr_progress_has_reached_this(current, crr->cpool.thr_prgr)) + *need_more_work = 1; + else { + *need_thr_progress = 1; + if (*thr_prgr_p == ERTS_THR_PRGR_INVALID + || erts_thr_progress_cmp(crr->cpool.thr_prgr, + *thr_prgr_p) < 0) { + *thr_prgr_p = crr->cpool.thr_prgr; + } + } + } + } + } } static void -destroy_sbmbc(Allctr_t *allctr, Block_t *blk) +schedule_dealloc_carrier(Allctr_t *allctr, Carrier_t *crr) { - Uint crr_sz; - Carrier_t *crr; + Allctr_t *orig_allctr; + Block_t *blk; + int check_pending_dealloc; + erts_aint_t max_size; - ASSERT(IS_MBC_BLK(blk)); - ASSERT(IS_MBC_FIRST_FBLK(allctr, blk)); + ASSERT(IS_MB_CARRIER(crr)); - crr = FIRST_BLK_TO_MBC(allctr, blk); - crr_sz = CARRIER_SZ(crr); + if (!ERTS_ALC_IS_CPOOL_ENABLED(allctr)) { + dealloc_mbc(allctr, crr); + return; + } -#ifdef DEBUG - if (!allctr->stopped) { - ASSERT(IS_LAST_BLK(blk)); + orig_allctr = crr->cpool.orig_allctr; -#ifdef ERTS_ALLOC_UTIL_HARD_DEBUG - (*allctr->link_free_block)(allctr, blk, ERTS_ALCU_FLG_SBMBC); - HARD_CHECK_BLK_CARRIER(allctr, blk); - (*allctr->unlink_free_block)(allctr, blk, ERTS_ALCU_FLG_SBMBC); -#endif + if (allctr != orig_allctr) { + int cinit = orig_allctr->dd.ix - allctr->dd.ix; + + /* + * We send the carrier to its origin for deallocation. + * This in order: + * - not to complicate things for the thread specific + * instances of mseg_alloc, and + * - to ensure that we always only reuse empty carriers + * originating from our own thread specific mseg_alloc + * instance which is beneficial on NUMA systems. + * + * The receiver will recognize that this is a carrier to + * deallocate (and not a block which is the common case) + * since the block is an mbc block that is free and last + * in the carrier. + */ + blk = MBC_TO_FIRST_BLK(allctr, crr); + ERTS_ALC_CPOOL_ASSERT(IS_FREE_LAST_MBC_BLK(blk)); + + ERTS_ALC_CPOOL_ASSERT(IS_MBC_FIRST_ABLK(allctr, blk)); + ERTS_ALC_CPOOL_ASSERT(crr == FBLK_TO_MBC(blk)); + ERTS_ALC_CPOOL_ASSERT(crr == FIRST_BLK_TO_MBC(allctr, blk)); + ERTS_ALC_CPOOL_ASSERT(((erts_aint_t) allctr) + == (erts_smp_atomic_read_nob(&crr->allctr) + & ~ERTS_CRR_ALCTR_FLG_MASK)); + + if (ddq_enqueue(&orig_allctr->dd.q, BLK2UMEM(blk), cinit)) + erts_alloc_notify_delayed_dealloc(orig_allctr->ix); + return; } -#endif - STAT_SBMBC_FREE(allctr, crr_sz); + if (is_abandoned(crr)) + unlink_abandoned_carrier(crr); - unlink_carrier(&allctr->sbmbc_list, crr); - if (allctr->destroying_mbc) - (*allctr->destroying_mbc)(allctr, crr, ERTS_ALCU_FLG_SBMBC); + if (crr->cpool.thr_prgr == ERTS_THR_PRGR_INVALID + || erts_thr_progress_has_reached(crr->cpool.thr_prgr)) { + dealloc_mbc(allctr, crr); + return; + } - INC_CC(allctr->calls.sbmbc_free); + blk = MBC_TO_FIRST_BLK(allctr, crr); + ASSERT(IS_FREE_LAST_MBC_BLK(blk)); + max_size = (erts_aint_t) MBC_FBLK_SZ(blk); + erts_atomic_set_nob(&crr->cpool.max_size, max_size); -#if HALFWORD_HEAP - if (allctr->mseg_opt.low_mem) - erts_free(ERTS_ALC_T_SBMBC_LOW, crr); + crr->next = NULL; + crr->prev = allctr->cpool.dc_list.last; + if (allctr->cpool.dc_list.last) { + check_pending_dealloc = 1; + allctr->cpool.dc_list.last->next = crr; + } + else { + check_pending_dealloc = 0; + allctr->cpool.dc_list.first = crr; + } + allctr->cpool.dc_list.last = crr; + if (check_pending_dealloc) + check_pending_dealloc_carrier(allctr, NULL, NULL, NULL); + erts_alloc_ensure_handle_delayed_dealloc_call(allctr->ix); +} + +static ERTS_INLINE void +cpool_init_carrier_data(Allctr_t *allctr, Carrier_t *crr) +{ + erts_atomic_init_nob(&crr->cpool.next, ERTS_AINT_NULL); + erts_atomic_init_nob(&crr->cpool.prev, ERTS_AINT_NULL); + crr->cpool.orig_allctr = allctr; + crr->cpool.thr_prgr = ERTS_THR_PRGR_INVALID; + erts_atomic_init_nob(&crr->cpool.max_size, 0); + crr->cpool.blocks = 0; + crr->cpool.blocks_size = 0; + if (!ERTS_ALC_IS_CPOOL_ENABLED(allctr)) + crr->cpool.abandon_limit = 0; + else { + UWord csz = CARRIER_SZ(crr); + UWord limit = csz*allctr->cpool.util_limit; + if (limit > csz) + limit /= 100; + else + limit = (csz/100)*allctr->cpool.util_limit; + crr->cpool.abandon_limit = limit; + } + crr->cpool.abandoned.next = NULL; + crr->cpool.abandoned.prev = NULL; +} + +static void +set_new_allctr_abandon_limit(Allctr_t *allctr) +{ + UWord limit; + UWord csz; + + allctr->cpool.check_limit_count = ERTS_ALC_CPOOL_CHECK_LIMIT_COUNT; + + csz = allctr->mbcs.curr.norm.mseg.size; + csz += allctr->mbcs.curr.norm.sys_alloc.size; + + limit = csz*allctr->cpool.util_limit; + if (limit > csz) + limit /= 100; else + limit = (csz/100)*allctr->cpool.util_limit; + + allctr->cpool.abandon_limit = limit; +} + +static void +abandon_carrier(Allctr_t *allctr, Carrier_t *crr) +{ + erts_aint_t max_size; + + STAT_MBC_CPOOL_INSERT(allctr, crr); + + unlink_carrier(&allctr->mbc_list, crr); + if (crr->cpool.orig_allctr == allctr) { + link_abandoned_carrier(&allctr->cpool.pooled_list, crr); + } + + allctr->remove_mbc(allctr, crr); + + max_size = (erts_aint_t) allctr->largest_fblk_in_mbc(allctr, crr); + erts_atomic_set_nob(&crr->cpool.max_size, max_size); + + cpool_insert(allctr, crr); + + set_new_allctr_abandon_limit(allctr); +} + +static void +cpool_read_stat(Allctr_t *allctr, UWord *nocp, UWord *cszp, UWord *nobp, UWord *bszp) +{ + int i; + UWord noc = 0, csz = 0, nob = 0, bsz = 0; + + /* + * We try to get consistent values, but after + * ERTS_ALC_CPOOL_MAX_FAILED_STAT_READS failed + * tries we give up and present what we got... + */ + for (i = 0; i <= ERTS_ALC_CPOOL_MAX_FAILED_STAT_READS; i++) { + UWord tnoc, tcsz, tnob, tbsz; + + tnoc = (UWord) (nocp + ? erts_atomic_read_nob(&allctr->cpool.stat.no_carriers) + : 0); + tcsz = (UWord) (cszp + ? erts_atomic_read_nob(&allctr->cpool.stat.carriers_size) + : 0); + tnob = (UWord) (nobp + ? erts_atomic_read_nob(&allctr->cpool.stat.no_blocks) + : 0); + tbsz = (UWord) (bszp + ? erts_atomic_read_nob(&allctr->cpool.stat.blocks_size) + : 0); + if (tnoc == noc && tcsz == csz && tnob == nob && tbsz == bsz) + break; + noc = tnoc; + csz = tcsz; + nob = tnob; + bsz = tbsz; + ERTS_THR_READ_MEMORY_BARRIER; + } + + if (nocp) + *nocp = noc; + if (cszp) + *cszp = csz; + if (nobp) + *nobp = nob; + if (bszp) + *bszp = bsz; +} + + +#endif /* ERTS_SMP */ + +#ifdef DEBUG + +#if ERTS_SA_MB_CARRIERS +#define ASSERT_ERTS_SACRR_UNIT_SIZE_MULTIPLE(CSZ) ASSERT((CSZ) % ERTS_SACRR_UNIT_SZ == 0) +#else +#define ASSERT_ERTS_SACRR_UNIT_SIZE_MULTIPLE(CSZ) #endif - erts_free(ERTS_ALC_T_SBMBC, crr); + +static void CHECK_1BLK_CARRIER(Allctr_t* A, int SBC, int MSEGED, Carrier_t* C, + UWord CSZ, Block_t* B, UWord BSZ) +{ + ASSERT(IS_LAST_BLK((B))); + ASSERT((CSZ) == CARRIER_SZ((C))); + ASSERT((BSZ) % sizeof(Unit_t) == 0); + if ((SBC)) { + ASSERT((BSZ) == SBC_BLK_SZ((B))); + ASSERT((char*)B == (char*)C + SBC_HEADER_SIZE); + ASSERT(IS_SBC_BLK((B))); + ASSERT(IS_SB_CARRIER((C))); + } + else { + ASSERT(IS_FREE_BLK(B)); + ASSERT((BSZ) == MBC_FBLK_SZ((B))); + ASSERT(IS_MBC_FIRST_FBLK(A, (B))); + ASSERT(IS_MBC_BLK((B))); + ASSERT(IS_MB_CARRIER((C))); + ASSERT(FBLK_TO_MBC(B) == (C)); + if ((MSEGED)) { + ASSERT_ERTS_SACRR_UNIT_SIZE_MULTIPLE((CSZ)); + } + } + if ((MSEGED)) { + ASSERT(IS_MSEG_CARRIER((C))); + } + else { + ASSERT(IS_SYS_ALLOC_CARRIER((C))); + ASSERT((CSZ) % sizeof(Unit_t) == 0); + } } +#else +#define CHECK_1BLK_CARRIER(A, SBC, MSEGED, C, CSZ, B, BSZ) +#endif + static Block_t * create_carrier(Allctr_t *allctr, Uint umem_sz, UWord flags) { @@ -2056,21 +3531,50 @@ create_carrier(Allctr_t *allctr, Uint umem_sz, UWord flags) int is_mseg = 0; #endif -#if HALFWORD_HEAP - flags |= CFLG_FORCE_MSEG; -#elif HAVE_SUPER_ALIGNED_MB_CARRIERS - if (flags & CFLG_MBC) { + if (HALFWORD_HEAP + || (ERTS_SUPER_ALIGNED_MSEG_ONLY && (flags & CFLG_MBC)) + || !allow_sys_alloc_carriers) { flags |= CFLG_FORCE_MSEG; - } + flags &= ~CFLG_FORCE_SYS_ALLOC; +#if !HAVE_ERTS_MSEG + return NULL; #endif + } ASSERT((flags & CFLG_SBC && !(flags & CFLG_MBC)) || (flags & CFLG_MBC && !(flags & CFLG_SBC))); ASSERT(!(flags & CFLG_FORCE_MSEG && flags & CFLG_FORCE_SYS_ALLOC)); + if (umem_sz > (ERTS_UINT_MAX - ERTS_UINT_MAX/100)) { + /* Do an overly conservative _overflow_ check here so we don't + * have to deal with it from here on. I guess we could be more accurate + * but I don't think the need to allocate over 99% of the address space + * will ever arise on any machine, neither 32 nor 64 bit. + */ + return NULL; + } + blk_sz = UMEMSZ2BLKSZ(allctr, umem_sz); +#ifdef ERTS_SMP + allctr->cpool.disable_abandon = ERTS_ALC_CPOOL_MAX_DISABLE_ABANDON; + + if ((flags & (CFLG_MBC|CFLG_NO_CPOOL)) == CFLG_MBC + && ERTS_ALC_IS_CPOOL_ENABLED(allctr) + && erts_thr_progress_is_managed_thread()) { + crr = cpool_fetch(allctr, blk_sz); + if (crr) { + STAT_MBC_CPOOL_FETCH(allctr, crr); + link_carrier(&allctr->mbc_list, crr); + (*allctr->add_mbc)(allctr, crr); + blk = (*allctr->get_free_block)(allctr, blk_sz, NULL, 0); + ASSERT(blk); + return blk; + } + } +#endif + #if HAVE_ERTS_MSEG if (flags & CFLG_FORCE_SYS_ALLOC) @@ -2083,7 +3587,7 @@ create_carrier(Allctr_t *allctr, Uint umem_sz, UWord flags) if (allctr->sbcs.curr.norm.mseg.no >= allctr->max_mseg_sbcs) goto try_sys_alloc; } -#if !HAVE_SUPER_ALIGNED_MB_CARRIERS +#if !ERTS_SUPER_ALIGNED_MSEG_ONLY else { if (allctr->mbcs.curr.norm.mseg.no >= allctr->max_mseg_mbcs) goto try_sys_alloc; @@ -2146,12 +3650,12 @@ create_carrier(Allctr_t *allctr, Uint umem_sz, UWord flags) ? UNIT_CEILING(bcrr_sz) : SYS_ALLOC_CARRIER_CEILING(bcrr_sz)); - crr = (Carrier_t *) alcu_sys_alloc(allctr, crr_sz); + crr = (Carrier_t *) alcu_sys_alloc(allctr, crr_sz, flags & CFLG_MBC); if (!crr) { if (crr_sz > UNIT_CEILING(bcrr_sz)) { crr_sz = UNIT_CEILING(bcrr_sz); - crr = (Carrier_t *) alcu_sys_alloc(allctr, crr_sz); + crr = (Carrier_t *) alcu_sys_alloc(allctr, crr_sz, flags & CFLG_MBC); } if (!crr) { #if HAVE_ERTS_MSEG @@ -2198,11 +3702,15 @@ create_carrier(Allctr_t *allctr, Uint umem_sz, UWord flags) allctr->main_carrier = crr; } +#ifdef ERTS_SMP + cpool_init_carrier_data(allctr, crr); +#endif + link_carrier(&allctr->mbc_list, crr); CHECK_1BLK_CARRIER(allctr, 0, is_mseg, crr, crr_sz, blk, blk_sz); if (allctr->creating_mbc) - (*allctr->creating_mbc)(allctr, crr, 0); + (*allctr->creating_mbc)(allctr, crr); } @@ -2245,7 +3753,7 @@ resize_carrier(Allctr_t *allctr, Block_t *old_blk, Uint umem_sz, UWord flags) if (!(flags & CFLG_FORCE_SYS_ALLOC)) { new_crr_sz = new_blk_sz + SBC_HEADER_SIZE; - new_crr_sz = MSEG_UNIT_CEILING(new_crr_sz); + new_crr_sz = ERTS_SACRR_UNIT_CEILING(new_crr_sz); new_crr = (Carrier_t *) alcu_mseg_realloc(allctr, old_crr, old_crr_sz, @@ -2295,7 +3803,9 @@ resize_carrier(Allctr_t *allctr, Block_t *old_blk, Uint umem_sz, UWord flags) new_crr = (Carrier_t *) alcu_sys_realloc(allctr, (void *) old_crr, - new_crr_sz); + new_crr_sz, + old_crr_sz, + 0); if (new_crr) { sys_realloc_success: SET_CARRIER_SZ(new_crr, new_crr_sz); @@ -2314,7 +3824,9 @@ resize_carrier(Allctr_t *allctr, Block_t *old_blk, Uint umem_sz, UWord flags) new_crr_sz = UNIT_CEILING(new_crr_sz); new_crr = (Carrier_t *) alcu_sys_realloc(allctr, (void *) old_crr, - new_crr_sz); + new_crr_sz, + old_crr_sz, + 0); if (new_crr) goto sys_realloc_success; } @@ -2333,27 +3845,36 @@ resize_carrier(Allctr_t *allctr, Block_t *old_blk, Uint umem_sz, UWord flags) (void *) BLK2UMEM(old_blk), MIN(new_blk_sz, old_blk_sz) - ABLK_HDR_SZ); unlink_carrier(&allctr->sbc_list, old_crr); - alcu_sys_free(allctr, old_crr); + alcu_sys_free(allctr, old_crr, 0); } else { /* Old carrier unchanged; restore... */ STAT_SYS_ALLOC_SBC_ALLOC(allctr, old_crr_sz, old_blk_sz); } - DEBUG_SAVE_ALIGNMENT(new_crr); return new_blk; } #endif } static void -destroy_carrier(Allctr_t *allctr, Block_t *blk) +dealloc_carrier(Allctr_t *allctr, Carrier_t *crr, int superaligned) { - Uint crr_sz; - Carrier_t *crr; #if HAVE_ERTS_MSEG - Uint is_mseg = 0; - Uint mseg_flags = ERTS_MSEG_FLG_NONE; + if (IS_MSEG_CARRIER(crr)) + alcu_mseg_dealloc(allctr, crr, CARRIER_SZ(crr), + (superaligned + ? ERTS_MSEG_FLG_2POW + : ERTS_MSEG_FLG_NONE)); + else #endif + alcu_sys_free(allctr, crr, superaligned); +} + +static void +destroy_carrier(Allctr_t *allctr, Block_t *blk, Carrier_t **busy_pcrr_pp) +{ + Uint crr_sz; + Carrier_t *crr; if (IS_SBC_BLK(blk)) { Uint blk_sz = SBC_BLK_SZ(blk); @@ -2366,8 +3887,6 @@ destroy_carrier(Allctr_t *allctr, Block_t *blk) #if HAVE_ERTS_MSEG if (IS_MSEG_CARRIER(crr)) { - is_mseg++; - ASSERT(crr_sz % MSEG_UNIT_SZ == 0); STAT_MSEG_SBC_FREE(allctr, crr_sz, blk_sz); } else @@ -2376,6 +3895,7 @@ destroy_carrier(Allctr_t *allctr, Block_t *blk) unlink_carrier(&allctr->sbc_list, crr); + dealloc_carrier(allctr, crr, 0); } else { ASSERT(IS_MBC_FIRST_FBLK(allctr, blk)); @@ -2394,30 +3914,40 @@ destroy_carrier(Allctr_t *allctr, Block_t *blk) } #endif -#if HAVE_ERTS_MSEG - if (IS_MSEG_CARRIER(crr)) { - is_mseg++; - ASSERT(crr_sz % MSEG_UNIT_SZ == 0); - STAT_MSEG_MBC_FREE(allctr, crr_sz); - mseg_flags = ERTS_MSEG_FLG_2POW; +#ifdef ERTS_SMP + if (busy_pcrr_pp && *busy_pcrr_pp) { + ERTS_ALC_CPOOL_ASSERT(*busy_pcrr_pp == crr); + *busy_pcrr_pp = NULL; + ERTS_ALC_CPOOL_ASSERT(erts_smp_atomic_read_nob(&crr->allctr) + == (((erts_aint_t) allctr) + | ERTS_CRR_ALCTR_FLG_IN_POOL + | ERTS_CRR_ALCTR_FLG_BUSY)); + erts_smp_atomic_set_nob(&crr->allctr, ((erts_aint_t) allctr)); + cpool_delete(allctr, allctr, crr); } else #endif - STAT_SYS_ALLOC_MBC_FREE(allctr, crr_sz); - - unlink_carrier(&allctr->mbc_list, crr); - if (allctr->destroying_mbc) - (*allctr->destroying_mbc)(allctr, crr, 0); - } + { + unlink_carrier(&allctr->mbc_list, crr); +#if HAVE_ERTS_MSEG + if (IS_MSEG_CARRIER(crr)) { + ASSERT(crr_sz % ERTS_SACRR_UNIT_SZ == 0); + STAT_MSEG_MBC_FREE(allctr, crr_sz); + } + else +#endif + STAT_SYS_ALLOC_MBC_FREE(allctr, crr_sz); + if (allctr->remove_mbc) + allctr->remove_mbc(allctr, crr); + } -#if HAVE_ERTS_MSEG - if (is_mseg) { - alcu_mseg_dealloc(allctr, crr, crr_sz, mseg_flags); - } - else +#ifdef ERTS_SMP + schedule_dealloc_carrier(allctr, crr); +#else + dealloc_mbc(allctr, crr); #endif - alcu_sys_free(allctr, crr); + } } @@ -2451,19 +3981,20 @@ static struct { Eterm lmbcs; Eterm smbcs; Eterm mbcgs; - Eterm sbmbcs; - Eterm sbmbct; + Eterm acul; #if HAVE_ERTS_MSEG Eterm mmc; #endif Eterm ycs; - - /* Eterm sbmbcs; */ + Eterm sac; Eterm fix_types; Eterm mbcs; +#ifdef ERTS_SMP + Eterm mbcs_pool; +#endif Eterm sbcs; Eterm sys_alloc_carriers_size; @@ -2488,8 +4019,6 @@ static struct { Eterm mseg_dealloc; Eterm mseg_realloc; #endif - Eterm sbmbc_alloc; - Eterm sbmbc_free; #ifdef DEBUG Eterm end_of_atoms; #endif @@ -2508,12 +4037,6 @@ static erts_mtx_t init_atoms_mtx; static void init_atoms(Allctr_t *allctr) { - -#ifdef USE_THREADS - if (allctr && allctr->thread_safe) - erts_mtx_unlock(&allctr->mutex); -#endif - erts_mtx_lock(&init_atoms_mtx); if (!atoms_initialized) { @@ -2551,19 +4074,20 @@ init_atoms(Allctr_t *allctr) AM_INIT(lmbcs); AM_INIT(smbcs); AM_INIT(mbcgs); - AM_INIT(sbmbcs); - AM_INIT(sbmbct); + AM_INIT(acul); #if HAVE_ERTS_MSEG AM_INIT(mmc); #endif AM_INIT(ycs); - - /*AM_INIT(sbmbcs);*/ + AM_INIT(sac); AM_INIT(fix_types); AM_INIT(mbcs); +#ifdef ERTS_SMP + AM_INIT(mbcs_pool); +#endif AM_INIT(sbcs); AM_INIT(sys_alloc_carriers_size); @@ -2588,8 +4112,6 @@ init_atoms(Allctr_t *allctr) AM_INIT(mseg_dealloc); AM_INIT(mseg_realloc); #endif - AM_INIT(sbmbc_free); - AM_INIT(sbmbc_alloc); #ifdef DEBUG for (atom = (Eterm *) &am; atom < &am.end_of_atoms; atom++) { @@ -2604,18 +4126,13 @@ init_atoms(Allctr_t *allctr) fix_type_atoms[ix] = am_atom_put(name, len); } } - - if (allctr) { + if (allctr && !allctr->atoms_initialized) { make_name_atoms(allctr); (*allctr->init_atoms)(); -#ifdef USE_THREADS - if (allctr->thread_safe) - erts_mtx_lock(&allctr->mutex); -#endif allctr->atoms_initialized = 1; } @@ -2642,19 +4159,22 @@ ensure_atoms_initialized(Allctr_t *allctr) * that would fit a small when size check is done may need to be built * as a big when the actual build is performed. Caller is required to * HRelease after build. + * + * Note, bld_unstable_uint() should have been called bld_unstable_uword() + * but we do not want to rename it... */ static ERTS_INLINE Eterm -bld_unstable_uint(Uint **hpp, Uint *szp, Uint ui) +bld_unstable_uint(Uint **hpp, Uint *szp, UWord ui) { Eterm res = THE_NON_VALUE; if (szp) - *szp += BIG_UINT_HEAP_SIZE; + *szp += BIG_UWORD_HEAP_SIZE(~((UWord) 0)); if (hpp) { if (IS_USMALL(0, ui)) res = make_small(ui); else { - res = uint_to_big(ui, *hpp); - *hpp += BIG_UINT_HEAP_SIZE; + res = uword_to_big(ui, *hpp); + *hpp += BIG_UWORD_HEAP_SIZE(ui); } } return res; @@ -2680,8 +4200,24 @@ add_4tup(Uint **hpp, Uint *szp, Eterm *lp, bld_cons(hpp, szp, bld_tuple(hpp, szp, 4, el1, el2, el3, el4), *lp); } +static ERTS_INLINE void +add_fix_types(Allctr_t *allctr, int internal, Uint **hpp, Uint *szp, + Eterm *lp, Eterm fix) +{ + if (allctr->fix) { + if (!ERTS_ALC_IS_CPOOL_ENABLED(allctr)) + add_2tup(hpp, szp, lp, am.fix_types, fix); + else if (internal) + add_3tup(hpp, szp, lp, + am.fix_types, + erts_bld_uword(hpp, szp, ~((UWord) 0)), + fix); + } +} + static Eterm sz_info_fix(Allctr_t *allctr, + int internal, int *print_to_p, void *print_to_arg, Uint **hpp, @@ -2689,36 +4225,67 @@ sz_info_fix(Allctr_t *allctr, { Eterm res; int ix; - ErtsAlcFixList_t *fix = allctr->fix; - ASSERT(fix); + ASSERT(allctr->fix); res = NIL; - for (ix = ERTS_ALC_NO_FIXED_SIZES-1; ix >= 0; ix--) { - ErtsAlcType_t n = ix + ERTS_ALC_N_MIN_A_FIXED_SIZE; - Uint alloced = (fix[ix].type_size * fix[ix].allocated); - Uint used = fix[ix].type_size*fix[ix].used; - - if (print_to_p) { - int to = *print_to_p; - void *arg = print_to_arg; - erts_print(to, - arg, - "fix type: %s %bpu %bpu\n", - (char *) ERTS_ALC_N2TD(n), - alloced, - used); - } + if (ERTS_ALC_IS_CPOOL_ENABLED(allctr)) { + + if (internal) { + for (ix = ERTS_ALC_NO_FIXED_SIZES-1; ix >= 0; ix--) { + ErtsAlcFixList_t *fix = &allctr->fix[ix]; + UWord alloced = fix->type_size * fix->u.cpool.allocated; + UWord used = fix->type_size * fix->u.cpool.used; + + if (print_to_p) { + int to = *print_to_p; + void *arg = print_to_arg; + erts_print(to, + arg, + "fix type internal: %s %bpu %bpu\n", + (char *) ERTS_ALC_N2TD(ERTS_ALC_N_MIN_A_FIXED_SIZE + + ix), + alloced, + used); + } - if (hpp || szp) { - add_3tup(hpp, szp, &res, - fix_type_atoms[ix], - bld_unstable_uint(hpp, szp, alloced), - bld_unstable_uint(hpp, szp, used)); + if (hpp || szp) { + add_3tup(hpp, szp, &res, + fix_type_atoms[ix], + bld_unstable_uint(hpp, szp, alloced), + bld_unstable_uint(hpp, szp, used)); + } + } } } + else { + + for (ix = ERTS_ALC_NO_FIXED_SIZES-1; ix >= 0; ix--) { + ErtsAlcFixList_t *fix = &allctr->fix[ix]; + UWord alloced = fix->type_size * fix->u.nocpool.allocated; + UWord used = fix->type_size*fix->u.nocpool.used; + + if (print_to_p) { + int to = *print_to_p; + void *arg = print_to_arg; + erts_print(to, + arg, + "fix type: %s %bpu %bpu\n", + (char *) ERTS_ALC_N2TD(ERTS_ALC_N_MIN_A_FIXED_SIZE + + ix), + alloced, + used); + } + if (hpp || szp) { + add_3tup(hpp, szp, &res, + fix_type_atoms[ix], + bld_unstable_uint(hpp, szp, alloced), + bld_unstable_uint(hpp, szp, used)); + } + } + } return res; } @@ -2732,9 +4299,7 @@ sz_info_carriers(Allctr_t *allctr, Uint *szp) { Eterm res = THE_NON_VALUE; - Uint curr_size = (cs == &allctr->sbmbcs - ? cs->curr.small_block.size - : cs->curr.norm.mseg.size + cs->curr.norm.sys_alloc.size); + UWord curr_size = cs->curr.norm.mseg.size + cs->curr.norm.sys_alloc.size; if (print_to_p) { int to = *print_to_p; @@ -2748,7 +4313,7 @@ sz_info_carriers(Allctr_t *allctr, cs->blocks.max_ever.size); erts_print(to, arg, - "%scarriers size: %beu %bpu %bpu\n", + "%scarriers size: %bpu %bpu %bpu\n", prefix, curr_size, cs->max.size, @@ -2772,6 +4337,62 @@ sz_info_carriers(Allctr_t *allctr, return res; } +#ifdef ERTS_SMP + +static Eterm +info_cpool(Allctr_t *allctr, + int sz_only, + char *prefix, + int *print_to_p, + void *print_to_arg, + Uint **hpp, + Uint *szp) +{ + Eterm res = THE_NON_VALUE; + UWord noc, csz, nob, bsz; + + noc = csz = nob = bsz = ~0; + if (print_to_p || hpp) { + if (sz_only) + cpool_read_stat(allctr, NULL, &csz, NULL, &bsz); + else + cpool_read_stat(allctr, &noc, &csz, &nob, &bsz); + } + + if (print_to_p) { + int to = *print_to_p; + void *arg = print_to_arg; + if (!sz_only) + erts_print(to, arg, "%sblocks: %bpu\n", prefix, nob); + erts_print(to, arg, "%sblocks size: %bpu\n", prefix, bsz); + if (!sz_only) + erts_print(to, arg, "%scarriers: %bpu\n", prefix, noc); + erts_print(to, arg, "%scarriers size: %bpu\n", prefix, csz); + } + + if (hpp || szp) { + res = NIL; + add_2tup(hpp, szp, &res, + am.carriers_size, + bld_unstable_uint(hpp, szp, csz)); + if (!sz_only) + add_2tup(hpp, szp, &res, + am.carriers, + bld_unstable_uint(hpp, szp, noc)); + add_2tup(hpp, szp, &res, + am.blocks_size, + bld_unstable_uint(hpp, szp, bsz)); + if (!sz_only) + add_2tup(hpp, szp, &res, + am.blocks, + bld_unstable_uint(hpp, szp, nob)); + } + + return res; +} + +#endif /* ERTS_SMP */ + static Eterm info_carriers(Allctr_t *allctr, CarriersStats_t *cs, @@ -2782,17 +4403,10 @@ info_carriers(Allctr_t *allctr, Uint *szp) { Eterm res = THE_NON_VALUE; - Uint curr_no, curr_size; - int small_block = cs == &allctr->sbmbcs; - - if (small_block) { - curr_no = cs->curr.small_block.no; - curr_size = cs->curr.small_block.size; - } - else { - curr_no = cs->curr.norm.mseg.no + cs->curr.norm.sys_alloc.no; - curr_size = cs->curr.norm.mseg.size + cs->curr.norm.sys_alloc.size; - } + UWord curr_no, curr_size; + + curr_no = cs->curr.norm.mseg.no + cs->curr.norm.sys_alloc.no; + curr_size = cs->curr.norm.mseg.size + cs->curr.norm.sys_alloc.size; if (print_to_p) { int to = *print_to_p; @@ -2813,75 +4427,67 @@ info_carriers(Allctr_t *allctr, cs->blocks.max_ever.size); erts_print(to, arg, - "%scarriers: %beu %bpu %bpu\n", + "%scarriers: %bpu %bpu %bpu\n", prefix, curr_no, cs->max.no, cs->max_ever.no); - if (!small_block) { #if HAVE_ERTS_MSEG - erts_print(to, - arg, - "%smseg carriers: %bpu\n", - prefix, - cs->curr.norm.mseg.no); -#endif - erts_print(to, - arg, - "%ssys_alloc carriers: %bpu\n", - prefix, - cs->curr.norm.sys_alloc.no); - } erts_print(to, arg, - "%scarriers size: %beu %bpu %bpu\n", + "%smseg carriers: %bpu\n", + prefix, + cs->curr.norm.mseg.no); +#endif + erts_print(to, + arg, + "%ssys_alloc carriers: %bpu\n", + prefix, + cs->curr.norm.sys_alloc.no); + erts_print(to, + arg, + "%scarriers size: %bpu %bpu %bpu\n", prefix, curr_size, cs->max.size, cs->max_ever.size); - if (!small_block) { #if HAVE_ERTS_MSEG - erts_print(to, - arg, - "%smseg carriers size: %bpu\n", - prefix, - cs->curr.norm.mseg.size); -#endif - erts_print(to, - arg, - "%ssys_alloc carriers size: %bpu\n", - prefix, - cs->curr.norm.sys_alloc.size); - } + erts_print(to, + arg, + "%smseg carriers size: %bpu\n", + prefix, + cs->curr.norm.mseg.size); +#endif + erts_print(to, + arg, + "%ssys_alloc carriers size: %bpu\n", + prefix, + cs->curr.norm.sys_alloc.size); } if (hpp || szp) { res = NIL; - if (!small_block) { - add_2tup(hpp, szp, &res, - am.sys_alloc_carriers_size, - bld_unstable_uint(hpp, szp, cs->curr.norm.sys_alloc.size)); + add_2tup(hpp, szp, &res, + am.sys_alloc_carriers_size, + bld_unstable_uint(hpp, szp, cs->curr.norm.sys_alloc.size)); #if HAVE_ERTS_MSEG - add_2tup(hpp, szp, &res, - am.mseg_alloc_carriers_size, - bld_unstable_uint(hpp, szp, cs->curr.norm.mseg.size)); + add_2tup(hpp, szp, &res, + am.mseg_alloc_carriers_size, + bld_unstable_uint(hpp, szp, cs->curr.norm.mseg.size)); #endif - } add_4tup(hpp, szp, &res, am.carriers_size, bld_unstable_uint(hpp, szp, curr_size), bld_unstable_uint(hpp, szp, cs->max.size), bld_unstable_uint(hpp, szp, cs->max_ever.size)); - if (!small_block) { - add_2tup(hpp, szp, &res, - am.sys_alloc_carriers, - bld_unstable_uint(hpp, szp, cs->curr.norm.sys_alloc.no)); + add_2tup(hpp, szp, &res, + am.sys_alloc_carriers, + bld_unstable_uint(hpp, szp, cs->curr.norm.sys_alloc.no)); #if HAVE_ERTS_MSEG - add_2tup(hpp, szp, &res, - am.mseg_alloc_carriers, - bld_unstable_uint(hpp, szp, cs->curr.norm.mseg.no)); + add_2tup(hpp, szp, &res, + am.mseg_alloc_carriers, + bld_unstable_uint(hpp, szp, cs->curr.norm.mseg.no)); #endif - } add_4tup(hpp, szp, &res, am.carriers, bld_unstable_uint(hpp, szp, curr_no), @@ -2912,7 +4518,7 @@ make_name_atoms(Allctr_t *allctr) size_t prefix_len = strlen(allctr->name_prefix); if (prefix_len > MAX_ATOM_CHARACTERS + sizeof(realloc) - 1) - erl_exit(1,"Too long allocator name: %salloc\n",allctr->name_prefix); + erts_exit(ERTS_ERROR_EXIT,"Too long allocator name: %salloc\n",allctr->name_prefix); memcpy((void *) buf, (void *) allctr->name_prefix, prefix_len); @@ -2940,16 +4546,10 @@ info_calls(Allctr_t *allctr, if (print_to_p) { #define PRINT_CC_4(TO, TOA, NAME, CC) \ - if ((CC).giga_no == 0) \ - erts_print(TO, TOA, "%s calls: %b32u\n", NAME, CC.no); \ - else \ - erts_print(TO, TOA, "%s calls: %b32u%09lu\n", NAME, CC.giga_no, CC.no) + erts_print(TO, TOA, "%s calls: %b64u\n", NAME, CC) #define PRINT_CC_5(TO, TOA, PRFX, NAME, CC) \ - if ((CC).giga_no == 0) \ - erts_print(TO, TOA, "%s%s calls: %b32u\n",PRFX,NAME,CC.no); \ - else \ - erts_print(TO, TOA, "%s%s calls: %b32u%09lu\n",PRFX,NAME,CC.giga_no,CC.no) + erts_print(TO, TOA, "%s%s calls: %b64u\n",PRFX,NAME,CC) char *prefix = allctr->name_prefix; int to = *print_to_p; @@ -2959,9 +4559,6 @@ info_calls(Allctr_t *allctr, PRINT_CC_5(to, arg, prefix, "free", allctr->calls.this_free); PRINT_CC_5(to, arg, prefix, "realloc", allctr->calls.this_realloc); - PRINT_CC_4(to, arg, "sbmbc_alloc", allctr->calls.sbmbc_alloc); - PRINT_CC_4(to, arg, "sbmbc_free", allctr->calls.sbmbc_free); - #if HAVE_ERTS_MSEG PRINT_CC_4(to, arg, "mseg_alloc", allctr->calls.mseg_alloc); PRINT_CC_4(to, arg, "mseg_dealloc", allctr->calls.mseg_dealloc); @@ -2988,50 +4585,42 @@ info_calls(Allctr_t *allctr, add_3tup(hpp, szp, &res, am.sys_realloc, - bld_unstable_uint(hpp, szp, allctr->calls.sys_realloc.giga_no), - bld_unstable_uint(hpp, szp, allctr->calls.sys_realloc.no)); + bld_unstable_uint(hpp, szp, ERTS_ALC_CC_GIGA_VAL(allctr->calls.sys_realloc)), + bld_unstable_uint(hpp, szp, ERTS_ALC_CC_VAL(allctr->calls.sys_realloc))); add_3tup(hpp, szp, &res, am.sys_free, - bld_unstable_uint(hpp, szp, allctr->calls.sys_free.giga_no), - bld_unstable_uint(hpp, szp, allctr->calls.sys_free.no)); + bld_unstable_uint(hpp, szp, ERTS_ALC_CC_GIGA_VAL(allctr->calls.sys_free)), + bld_unstable_uint(hpp, szp, ERTS_ALC_CC_VAL(allctr->calls.sys_free))); add_3tup(hpp, szp, &res, am.sys_alloc, - bld_unstable_uint(hpp, szp, allctr->calls.sys_alloc.giga_no), - bld_unstable_uint(hpp, szp, allctr->calls.sys_alloc.no)); + bld_unstable_uint(hpp, szp, ERTS_ALC_CC_GIGA_VAL(allctr->calls.sys_alloc)), + bld_unstable_uint(hpp, szp, ERTS_ALC_CC_VAL(allctr->calls.sys_alloc))); #if HAVE_ERTS_MSEG add_3tup(hpp, szp, &res, am.mseg_realloc, - bld_unstable_uint(hpp, szp, allctr->calls.mseg_realloc.giga_no), - bld_unstable_uint(hpp, szp, allctr->calls.mseg_realloc.no)); + bld_unstable_uint(hpp, szp, ERTS_ALC_CC_GIGA_VAL(allctr->calls.mseg_realloc)), + bld_unstable_uint(hpp, szp, ERTS_ALC_CC_VAL(allctr->calls.mseg_realloc))); add_3tup(hpp, szp, &res, am.mseg_dealloc, - bld_unstable_uint(hpp, szp, allctr->calls.mseg_dealloc.giga_no), - bld_unstable_uint(hpp, szp, allctr->calls.mseg_dealloc.no)); + bld_unstable_uint(hpp, szp, ERTS_ALC_CC_GIGA_VAL(allctr->calls.mseg_dealloc)), + bld_unstable_uint(hpp, szp, ERTS_ALC_CC_VAL(allctr->calls.mseg_dealloc))); add_3tup(hpp, szp, &res, am.mseg_alloc, - bld_unstable_uint(hpp, szp, allctr->calls.mseg_alloc.giga_no), - bld_unstable_uint(hpp, szp, allctr->calls.mseg_alloc.no)); + bld_unstable_uint(hpp, szp, ERTS_ALC_CC_GIGA_VAL(allctr->calls.mseg_alloc)), + bld_unstable_uint(hpp, szp, ERTS_ALC_CC_VAL(allctr->calls.mseg_alloc))); #endif add_3tup(hpp, szp, &res, - am.sbmbc_free, - bld_unstable_uint(hpp, szp, allctr->calls.sbmbc_free.giga_no), - bld_unstable_uint(hpp, szp, allctr->calls.sbmbc_free.no)); - add_3tup(hpp, szp, &res, - am.sbmbc_alloc, - bld_unstable_uint(hpp, szp, allctr->calls.sbmbc_alloc.giga_no), - bld_unstable_uint(hpp, szp, allctr->calls.sbmbc_alloc.no)); - add_3tup(hpp, szp, &res, allctr->name.realloc, - bld_unstable_uint(hpp, szp, allctr->calls.this_realloc.giga_no), - bld_unstable_uint(hpp, szp, allctr->calls.this_realloc.no)); + bld_unstable_uint(hpp, szp, ERTS_ALC_CC_GIGA_VAL(allctr->calls.this_realloc)), + bld_unstable_uint(hpp, szp, ERTS_ALC_CC_VAL(allctr->calls.this_realloc))); add_3tup(hpp, szp, &res, allctr->name.free, - bld_unstable_uint(hpp, szp, allctr->calls.this_free.giga_no), - bld_unstable_uint(hpp, szp, allctr->calls.this_free.no)); + bld_unstable_uint(hpp, szp, ERTS_ALC_CC_GIGA_VAL(allctr->calls.this_free)), + bld_unstable_uint(hpp, szp, ERTS_ALC_CC_VAL(allctr->calls.this_free))); add_3tup(hpp, szp, &res, allctr->name.alloc, - bld_unstable_uint(hpp, szp, allctr->calls.this_alloc.giga_no), - bld_unstable_uint(hpp, szp, allctr->calls.this_alloc.no)); + bld_unstable_uint(hpp, szp, ERTS_ALC_CC_GIGA_VAL(allctr->calls.this_alloc)), + bld_unstable_uint(hpp, szp, ERTS_ALC_CC_VAL(allctr->calls.this_alloc))); } return res; @@ -3045,6 +4634,7 @@ info_options(Allctr_t *allctr, Uint *szp) { Eterm res = THE_NON_VALUE; + int acul; if (!allctr) { if (print_to_p) @@ -3056,6 +4646,12 @@ info_options(Allctr_t *allctr, return res; } +#ifdef ERTS_SMP + acul = allctr->cpool.util_limit; +#else + acul = 0; +#endif + if (print_to_p) { char topt[21]; /* Enough for any 64-bit integer */ if (allctr->t) @@ -3085,8 +4681,7 @@ info_options(Allctr_t *allctr, "option lmbcs: %beu\n" "option smbcs: %beu\n" "option mbcgs: %beu\n" - "option sbmbcs: %beu\n" - "option sbmbct: %beu\n", + "option acul: %d\n", topt, allctr->ramv ? "true" : "false", #if HALFWORD_HEAP @@ -3107,8 +4702,7 @@ info_options(Allctr_t *allctr, allctr->largest_mbc_size, allctr->smallest_mbc_size, allctr->mbc_growth_stages, - allctr->sbmbc_size, - allctr->sbmbc_threshold); + acul); } res = (*allctr->info_options)(allctr, "option ", print_to_p, print_to_arg, @@ -3116,11 +4710,8 @@ info_options(Allctr_t *allctr, if (hpp || szp) { add_2tup(hpp, szp, &res, - am.sbmbct, - bld_uint(hpp, szp, allctr->sbmbc_threshold)); - add_2tup(hpp, szp, &res, - am.sbmbcs, - bld_uint(hpp, szp, allctr->sbmbc_size)); + am.acul, + bld_uint(hpp, szp, (UWord) acul)); add_2tup(hpp, szp, &res, am.mbcgs, bld_uint(hpp, szp, allctr->mbc_growth_stages)); @@ -3210,17 +4801,22 @@ erts_alcu_au_info_options(int *print_to_p, void *print_to_arg, #if HAVE_ERTS_MSEG "option mmc: %beu\n" #endif - "option ycs: %beu\n", + "option ycs: %beu\n" + "option sac: %s\n", #if HAVE_ERTS_MSEG max_mseg_carriers, #endif - sys_alloc_carrier_size); + sys_alloc_carrier_size, + allow_sys_alloc_carriers ? "true" : "false"); } if (hpp || szp) { res = NIL; ensure_atoms_initialized(NULL); add_2tup(hpp, szp, &res, + am.sac, + allow_sys_alloc_carriers ? am_true : am_false); + add_2tup(hpp, szp, &res, am.ycs, bld_uint(hpp, szp, sys_alloc_carrier_size)); #if HAVE_ERTS_MSEG @@ -3243,17 +4839,21 @@ erts_alcu_info_options(Allctr_t *allctr, { Eterm res; + if (hpp || szp) + ensure_atoms_initialized(allctr); #ifdef USE_THREADS - if (allctr->thread_safe) + if (allctr->thread_safe) { + erts_allctr_wrapper_pre_lock(); erts_mtx_lock(&allctr->mutex); + } #endif - if (hpp || szp) - ensure_atoms_initialized(allctr); res = info_options(allctr, print_to_p, print_to_arg, hpp, szp); #ifdef USE_THREADS - if (allctr->thread_safe) + if (allctr->thread_safe) { erts_mtx_unlock(&allctr->mutex); + erts_allctr_wrapper_pre_unlock(); + } #endif return res; } @@ -3262,13 +4862,17 @@ erts_alcu_info_options(Allctr_t *allctr, Eterm erts_alcu_sz_info(Allctr_t *allctr, + int internal, int begin_max_period, int *print_to_p, void *print_to_arg, Uint **hpp, Uint *szp) { - Eterm res, sbmbcs, mbcs, sbcs, fix = THE_NON_VALUE; + Eterm res, mbcs, sbcs, fix = THE_NON_VALUE; +#ifdef ERTS_SMP + Eterm mbcs_pool; +#endif res = THE_NON_VALUE; @@ -3280,67 +4884,81 @@ erts_alcu_sz_info(Allctr_t *allctr, return am_false; } + if (hpp || szp) + ensure_atoms_initialized(allctr); + #ifdef USE_THREADS - if (allctr->thread_safe) + if (allctr->thread_safe) { + erts_allctr_wrapper_pre_lock(); erts_mtx_lock(&allctr->mutex); + } #endif ERTS_ALCU_DBG_CHK_THR_ACCESS(allctr); - if (hpp || szp) - ensure_atoms_initialized(allctr); - /* Update sbc values not continously updated */ allctr->sbcs.blocks.curr.no = allctr->sbcs.curr.norm.mseg.no + allctr->sbcs.curr.norm.sys_alloc.no; allctr->sbcs.blocks.max.no = allctr->sbcs.max.no; - update_max_ever_values(&allctr->sbmbcs); update_max_ever_values(&allctr->mbcs); update_max_ever_values(&allctr->sbcs); if (allctr->fix) - fix = sz_info_fix(allctr, print_to_p, print_to_arg, hpp, szp); - sbmbcs = sz_info_carriers(allctr, &allctr->sbmbcs, "sbmbcs ", print_to_p, - print_to_arg, hpp, szp); + fix = sz_info_fix(allctr, internal, print_to_p, print_to_arg, hpp, szp); mbcs = sz_info_carriers(allctr, &allctr->mbcs, "mbcs ", print_to_p, print_to_arg, hpp, szp); +#ifdef ERTS_SMP + if (ERTS_ALC_IS_CPOOL_ENABLED(allctr)) + mbcs_pool = info_cpool(allctr, 1, "mbcs_pool ", print_to_p, + print_to_arg, hpp, szp); + else + mbcs_pool = THE_NON_VALUE; /* shut up annoying warning... */ +#endif sbcs = sz_info_carriers(allctr, &allctr->sbcs, "sbcs ", print_to_p, print_to_arg, hpp, szp); if (hpp || szp) { res = NIL; add_2tup(hpp, szp, &res, am.sbcs, sbcs); +#ifdef ERTS_SMP + if (ERTS_ALC_IS_CPOOL_ENABLED(allctr)) + add_2tup(hpp, szp, &res, am.mbcs_pool, mbcs_pool); +#endif add_2tup(hpp, szp, &res, am.mbcs, mbcs); - add_2tup(hpp, szp, &res, am.sbmbcs, sbmbcs); - if (allctr->fix) - add_2tup(hpp, szp, &res, am.fix_types, fix); + add_fix_types(allctr, internal, hpp, szp, &res, fix); } if (begin_max_period) { - reset_max_values(&allctr->sbmbcs); reset_max_values(&allctr->mbcs); reset_max_values(&allctr->sbcs); } #ifdef USE_THREADS - if (allctr->thread_safe) + if (allctr->thread_safe) { erts_mtx_unlock(&allctr->mutex); + erts_allctr_wrapper_pre_unlock(); + } #endif return res; } + Eterm erts_alcu_info(Allctr_t *allctr, + int internal, int begin_max_period, int *print_to_p, void *print_to_arg, Uint **hpp, Uint *szp) { - Eterm res, sett, sbmbcs, mbcs, sbcs, calls, fix = THE_NON_VALUE; + Eterm res, sett, mbcs, sbcs, calls, fix = THE_NON_VALUE; +#ifdef ERTS_SMP + Eterm mbcs_pool; +#endif res = THE_NON_VALUE; @@ -3352,22 +4970,23 @@ erts_alcu_info(Allctr_t *allctr, return am_false; } + if (hpp || szp) + ensure_atoms_initialized(allctr); + #ifdef USE_THREADS - if (allctr->thread_safe) + if (allctr->thread_safe) { + erts_allctr_wrapper_pre_lock(); erts_mtx_lock(&allctr->mutex); + } #endif ERTS_ALCU_DBG_CHK_THR_ACCESS(allctr); - if (hpp || szp) - ensure_atoms_initialized(allctr); - /* Update sbc values not continously updated */ allctr->sbcs.blocks.curr.no = allctr->sbcs.curr.norm.mseg.no + allctr->sbcs.curr.norm.sys_alloc.no; allctr->sbcs.blocks.max.no = allctr->sbcs.max.no; - update_max_ever_values(&allctr->sbmbcs); update_max_ever_values(&allctr->mbcs); update_max_ever_values(&allctr->sbcs); @@ -3381,11 +5000,16 @@ erts_alcu_info(Allctr_t *allctr, sett = info_options(allctr, print_to_p, print_to_arg, hpp, szp); if (allctr->fix) - fix = sz_info_fix(allctr, print_to_p, print_to_arg, hpp, szp); - sbmbcs = info_carriers(allctr, &allctr->sbmbcs, "sbmbcs ", print_to_p, - print_to_arg, hpp, szp); + fix = sz_info_fix(allctr, internal, print_to_p, print_to_arg, hpp, szp); mbcs = info_carriers(allctr, &allctr->mbcs, "mbcs ", print_to_p, print_to_arg, hpp, szp); +#ifdef ERTS_SMP + if (ERTS_ALC_IS_CPOOL_ENABLED(allctr)) + mbcs_pool = info_cpool(allctr, 0, "mbcs_pool ", print_to_p, + print_to_arg, hpp, szp); + else + mbcs_pool = THE_NON_VALUE; /* shut up annoying warning... */ +#endif sbcs = info_carriers(allctr, &allctr->sbcs, "sbcs ", print_to_p, print_to_arg, hpp, szp); calls = info_calls(allctr, print_to_p, print_to_arg, hpp, szp); @@ -3395,10 +5019,12 @@ erts_alcu_info(Allctr_t *allctr, add_2tup(hpp, szp, &res, am.calls, calls); add_2tup(hpp, szp, &res, am.sbcs, sbcs); +#ifdef ERTS_SMP + if (ERTS_ALC_IS_CPOOL_ENABLED(allctr)) + add_2tup(hpp, szp, &res, am.mbcs_pool, mbcs_pool); +#endif add_2tup(hpp, szp, &res, am.mbcs, mbcs); - add_2tup(hpp, szp, &res, am.sbmbcs, sbmbcs); - if (allctr->fix) - add_2tup(hpp, szp, &res, am.fix_types, fix); + add_fix_types(allctr, internal, hpp, szp, &res, fix); add_2tup(hpp, szp, &res, am.options, sett); add_3tup(hpp, szp, &res, am.versions, @@ -3407,15 +5033,16 @@ erts_alcu_info(Allctr_t *allctr, } if (begin_max_period) { - reset_max_values(&allctr->sbmbcs); reset_max_values(&allctr->mbcs); reset_max_values(&allctr->sbcs); } #ifdef USE_THREADS - if (allctr->thread_safe) + if (allctr->thread_safe) { erts_mtx_unlock(&allctr->mutex); + erts_allctr_wrapper_pre_unlock(); + } #endif return res; @@ -3433,22 +5060,37 @@ erts_alcu_current_size(Allctr_t *allctr, AllctrSize_t *size, ErtsAlcUFixInfo_t * size->carriers = allctr->mbcs.curr.norm.mseg.size; size->carriers += allctr->mbcs.curr.norm.sys_alloc.size; - size->carriers += allctr->sbmbcs.curr.small_block.size; size->carriers += allctr->sbcs.curr.norm.mseg.size; size->carriers += allctr->sbcs.curr.norm.sys_alloc.size; size->blocks = allctr->mbcs.blocks.curr.size; - size->blocks += allctr->sbmbcs.blocks.curr.size; size->blocks += allctr->sbcs.blocks.curr.size; +#ifdef ERTS_SMP + if (ERTS_ALC_IS_CPOOL_ENABLED(allctr)) { + UWord csz, bsz; + cpool_read_stat(allctr, NULL, &csz, NULL, &bsz); + size->blocks += bsz; + size->carriers += csz; + } +#endif + if (fi) { int ix; for (ix = 0; ix < fisz; ix++) { if (allctr->fix) { - fi[ix].allocated += (allctr->fix[ix].type_size - * allctr->fix[ix].allocated); - fi[ix].used += (allctr->fix[ix].type_size - * allctr->fix[ix].used); + if (ERTS_ALC_IS_CPOOL_ENABLED(allctr)) { + fi[ix].allocated += (allctr->fix[ix].type_size + * allctr->fix[ix].u.cpool.allocated); + fi[ix].used += (allctr->fix[ix].type_size + * allctr->fix[ix].u.cpool.used); + } + else { + fi[ix].allocated += (allctr->fix[ix].type_size + * allctr->fix[ix].u.nocpool.allocated); + fi[ix].used += (allctr->fix[ix].type_size + * allctr->fix[ix].u.nocpool.used); + } } } } @@ -3466,7 +5108,6 @@ do_erts_alcu_alloc(ErtsAlcType_t type, void *extra, Uint size) { Allctr_t *allctr = (Allctr_t *) extra; void *res; - ErtsAlcFixList_t *fix; ASSERT(initialized); @@ -3484,56 +5125,21 @@ do_erts_alcu_alloc(ErtsAlcType_t type, void *extra, Uint size) INC_CC(allctr->calls.this_alloc); - fix = allctr->fix; - if (fix) { - int ix = type - ERTS_ALC_N_MIN_A_FIXED_SIZE; - ERTS_DBG_CHK_FIX_LIST(allctr, fix, ix, 1); - fix[ix].used++; - res = fix[ix].list; - if (res) { - fix[ix].list_size--; - fix[ix].list = *((void **) res); - if (fix[ix].list && fix[ix].allocated > fix[ix].limit) { - void *p = fix[ix].list; - Block_t *blk; - fix[ix].list = *((void **) p); - fix[ix].list_size--; - blk = UMEM2BLK(p); - if (IS_SBC_BLK(blk)) - destroy_carrier(allctr, blk); - else - mbc_free(allctr, p); - fix[ix].allocated--; - } - ERTS_DBG_CHK_FIX_LIST(allctr, fix, ix, 0); - return res; - } - if (size < 2*sizeof(UWord)) - size += sizeof(UWord); - if (fix[ix].limit < fix[ix].used) - fix[ix].limit = fix[ix].used; - if (fix[ix].max_used < fix[ix].used) - fix[ix].max_used = fix[ix].used; - fix[ix].allocated++; + if (allctr->fix) { + if (ERTS_ALC_IS_CPOOL_ENABLED(allctr)) + return fix_cpool_alloc(allctr, type, size); + else + return fix_nocpool_alloc(allctr, type, size); } if (size >= allctr->sbc_threshold) { Block_t *blk; -#ifdef ERTS_SMP - if (allctr->dd.use) - ERTS_ALCU_HANDLE_DD_IN_OP(allctr, 1); -#endif blk = create_carrier(allctr, size, CFLG_SBC); res = blk ? BLK2UMEM(blk) : NULL; } else res = mbc_alloc(allctr, size); - if (!res && fix) { - int ix = type - ERTS_ALC_N_MIN_A_FIXED_SIZE; - fix[ix].allocated--; - fix[ix].used--; - } return res; } @@ -3602,9 +5208,22 @@ erts_alcu_alloc_thr_pref(ErtsAlcType_t type, void *extra, Uint size) if (pref_allctr->thread_safe) erts_mtx_lock(&pref_allctr->mutex); +#ifdef ERTS_SMP + ASSERT(pref_allctr->dd.use); + ERTS_ALCU_HANDLE_DD_IN_OP(pref_allctr, 1); +#endif + ERTS_ALCU_DBG_CHK_THR_ACCESS(pref_allctr); res = do_erts_alcu_alloc(type, pref_allctr, size); + +#ifdef ERTS_SMP + if (!res && ERTS_ALCU_HANDLE_DD_IN_OP(pref_allctr, 1)) { + /* Cleaned up a bit more; try one more time... */ + res = do_erts_alcu_alloc(type, pref_allctr, size); + } +#endif + if (pref_allctr->thread_safe) erts_mtx_unlock(&pref_allctr->mutex); @@ -3621,9 +5240,9 @@ erts_alcu_alloc_thr_pref(ErtsAlcType_t type, void *extra, Uint size) /* ------------------------------------------------------------------------- */ static ERTS_INLINE void -do_erts_alcu_free(ErtsAlcType_t type, void *extra, void *p) +do_erts_alcu_free(ErtsAlcType_t type, void *extra, void *p, + Carrier_t **busy_pcrr_pp) { - int ix; Allctr_t *allctr = (Allctr_t *) extra; ASSERT(initialized); @@ -3635,57 +5254,28 @@ do_erts_alcu_free(ErtsAlcType_t type, void *extra, void *p) ERTS_ALCU_DBG_CHK_THR_ACCESS(allctr); if (p) { - ErtsAlcFixList_t *fix = allctr->fix; - Block_t *blk; INC_CC(allctr->calls.this_free); - if (fix) { - ix = type - ERTS_ALC_N_MIN_A_FIXED_SIZE; - ERTS_DBG_CHK_FIX_LIST(allctr, fix, ix, 1); - fix[ix].used--; - if (fix[ix].allocated < fix[ix].limit - && fix[ix].list_size < ERTS_ALCU_FIX_MAX_LIST_SZ) { - *((void **) p) = fix[ix].list; - fix[ix].list = p; - fix[ix].list_size++; - if (!allctr->fix_shrink_scheduled) { - allctr->fix_shrink_scheduled = 1; - erts_set_aux_work_timeout( - allctr->ix, - (ERTS_SSI_AUX_WORK_FIX_ALLOC_LOWER_LIM - | ERTS_SSI_AUX_WORK_FIX_ALLOC_DEALLOC), - 1); - } - ERTS_DBG_CHK_FIX_LIST(allctr, fix, ix, 0); - return; - } - fix[ix].allocated--; - if (fix[ix].list && fix[ix].allocated > fix[ix].limit) { - blk = UMEM2BLK(p); - if (IS_SBC_BLK(blk)) - destroy_carrier(allctr, blk); - else - mbc_free(allctr, p); - p = fix[ix].list; - fix[ix].list = *((void **) p); - fix[ix].list_size--; - fix[ix].allocated--; - } + if (allctr->fix) { + if (ERTS_ALC_IS_CPOOL_ENABLED(allctr)) + fix_cpool_free(allctr, type, p, busy_pcrr_pp, 1); + else + fix_nocpool_free(allctr, type, p); + } + else { + Block_t *blk = UMEM2BLK(p); + if (IS_SBC_BLK(blk)) + destroy_carrier(allctr, blk, NULL); + else + mbc_free(allctr, p, busy_pcrr_pp); } - - blk = UMEM2BLK(p); - if (IS_SBC_BLK(blk)) - destroy_carrier(allctr, blk); - else - mbc_free(allctr, p); - ERTS_DBG_CHK_FIX_LIST(allctr, fix, ix, 0); } } void erts_alcu_free(ErtsAlcType_t type, void *extra, void *p) { - do_erts_alcu_free(type, extra, p); + do_erts_alcu_free(type, extra, p, NULL); } #ifdef USE_THREADS @@ -3695,7 +5285,7 @@ erts_alcu_free_ts(ErtsAlcType_t type, void *extra, void *p) { Allctr_t *allctr = (Allctr_t *) extra; erts_mtx_lock(&allctr->mutex); - do_erts_alcu_free(type, extra, p); + do_erts_alcu_free(type, extra, p, NULL); erts_mtx_unlock(&allctr->mutex); } @@ -3717,7 +5307,7 @@ erts_alcu_free_thr_spec(ErtsAlcType_t type, void *extra, void *p) if (allctr->thread_safe) erts_mtx_lock(&allctr->mutex); - do_erts_alcu_free(type, allctr, p); + do_erts_alcu_free(type, allctr, p, NULL); if (allctr->thread_safe) erts_mtx_unlock(&allctr->mutex); @@ -3727,10 +5317,12 @@ void erts_alcu_free_thr_pref(ErtsAlcType_t type, void *extra, void *p) { if (p) { + Carrier_t *busy_pcrr_p; Allctr_t *pref_allctr, *used_allctr; pref_allctr = get_pref_allctr(extra); - used_allctr = get_used_allctr(extra, p, NULL); + used_allctr = get_used_allctr(pref_allctr, ERTS_ALC_TS_PREF_LOCK_IF_USED, + p, NULL, &busy_pcrr_p); if (pref_allctr != used_allctr) enqueue_dealloc_other_instance(type, used_allctr, @@ -3738,12 +5330,11 @@ erts_alcu_free_thr_pref(ErtsAlcType_t type, void *extra, void *p) (used_allctr->dd.ix - pref_allctr->dd.ix)); else { - if (used_allctr->thread_safe) - erts_mtx_lock(&used_allctr->mutex); ERTS_ALCU_DBG_CHK_THR_ACCESS(used_allctr); - do_erts_alcu_free(type, used_allctr, p); - if (used_allctr->thread_safe) - erts_mtx_unlock(&used_allctr->mutex); + do_erts_alcu_free(type, used_allctr, p, &busy_pcrr_p); + clear_busy_pool_carrier(used_allctr, busy_pcrr_p); + if (pref_allctr->thread_safe) + erts_mtx_unlock(&pref_allctr->mutex); } } } @@ -3759,7 +5350,8 @@ do_erts_alcu_realloc(ErtsAlcType_t type, void *extra, void *p, Uint size, - Uint32 alcu_flgs) + Uint32 alcu_flgs, + Carrier_t **busy_pcrr_pp) { Allctr_t *allctr = (Allctr_t *) extra; Block_t *blk; @@ -3784,7 +5376,7 @@ do_erts_alcu_realloc(ErtsAlcType_t type, #if ALLOC_ZERO_EQ_NULL if (!size) { ASSERT(p); - do_erts_alcu_free(type, extra, p); + do_erts_alcu_free(type, extra, p, busy_pcrr_pp); INC_CC(allctr->calls.this_realloc); DEC_CC(allctr->calls.this_free); return NULL; @@ -3795,32 +5387,9 @@ do_erts_alcu_realloc(ErtsAlcType_t type, blk = UMEM2BLK(p); - if (allctr->sbmbc_threshold > 0) { - Uint old_sz, new_sz, lim; - lim = allctr->sbmbc_threshold; - old_sz = BLK_SZ(blk); - new_sz = UMEMSZ2BLKSZ(allctr, size); - if ((old_sz < lim && lim <= new_sz) - || (new_sz < lim && lim <= old_sz)) { - /* *Need* to move it... */ - - INC_CC(allctr->calls.this_realloc); - res = do_erts_alcu_alloc(type, extra, size); - DEC_CC(allctr->calls.this_alloc); - - sys_memcpy(res, p, MIN(size, old_sz - ABLK_HDR_SZ)); - - do_erts_alcu_free(type, extra, p); - DEC_CC(allctr->calls.this_free); - return res; - } - if (old_sz < lim) - alcu_flgs |= ERTS_ALCU_FLG_SBMBC; - } - if (size < allctr->sbc_threshold) { if (IS_MBC_BLK(blk)) - res = mbc_realloc(allctr, p, size, alcu_flgs); + res = mbc_realloc(allctr, p, size, alcu_flgs, busy_pcrr_pp); else { Uint used_sz = SBC_HEADER_SIZE + ABLK_HDR_SZ + size; Uint crr_sz; @@ -3833,7 +5402,7 @@ do_erts_alcu_realloc(ErtsAlcType_t type, crr_sz = SYS_ALLOC_CARRIER_CEILING(used_sz); #if HAVE_ERTS_MSEG else - crr_sz = MSEG_UNIT_CEILING(used_sz); + crr_sz = ERTS_SACRR_UNIT_CEILING(used_sz); #endif diff_sz_val = crr_sz - used_sz; if (diff_sz_val < (~((Uint) 0) / 100)) @@ -3859,16 +5428,12 @@ do_erts_alcu_realloc(ErtsAlcType_t type, sys_memcpy((void*) res, (void*) p, MIN(SBC_BLK_SZ(blk) - ABLK_HDR_SZ, size)); - destroy_carrier(allctr, blk); + destroy_carrier(allctr, blk, NULL); } } } else { Block_t *new_blk; -#ifdef ERTS_SMP - if (allctr->dd.use) - ERTS_ALCU_HANDLE_DD_IN_OP(allctr, 1); -#endif if(IS_SBC_BLK(blk)) { do_carrier_resize: #if HALFWORD_HEAP @@ -3887,7 +5452,7 @@ do_erts_alcu_realloc(ErtsAlcType_t type, sys_memcpy((void *) res, (void *) p, MIN(MBC_ABLK_SZ(blk) - ABLK_HDR_SZ, size)); - mbc_free(allctr, p); + mbc_free(allctr, p, busy_pcrr_pp); } else res = NULL; @@ -3901,7 +5466,7 @@ void * erts_alcu_realloc(ErtsAlcType_t type, void *extra, void *p, Uint size) { void *res; - res = do_erts_alcu_realloc(type, extra, p, size, 0); + res = do_erts_alcu_realloc(type, extra, p, size, 0, NULL); DEBUG_CHECK_ALIGNMENT(res); return res; } @@ -3922,7 +5487,7 @@ erts_alcu_realloc_mv(ErtsAlcType_t type, void *extra, void *p, Uint size) if (cpy_size > size) cpy_size = size; sys_memcpy(res, p, cpy_size); - do_erts_alcu_free(type, extra, p); + do_erts_alcu_free(type, extra, p, NULL); } DEBUG_CHECK_ALIGNMENT(res); return res; @@ -3937,7 +5502,7 @@ erts_alcu_realloc_ts(ErtsAlcType_t type, void *extra, void *ptr, Uint size) Allctr_t *allctr = (Allctr_t *) extra; void *res; erts_mtx_lock(&allctr->mutex); - res = do_erts_alcu_realloc(type, extra, ptr, size, 0); + res = do_erts_alcu_realloc(type, extra, ptr, size, 0, NULL); erts_mtx_unlock(&allctr->mutex); DEBUG_CHECK_ALIGNMENT(res); return res; @@ -3961,7 +5526,7 @@ erts_alcu_realloc_mv_ts(ErtsAlcType_t type, void *extra, void *p, Uint size) if (cpy_size > size) cpy_size = size; sys_memcpy(res, p, cpy_size); - do_erts_alcu_free(type, extra, p); + do_erts_alcu_free(type, extra, p, NULL); } erts_mtx_unlock(&allctr->mutex); DEBUG_CHECK_ALIGNMENT(res); @@ -3988,7 +5553,7 @@ erts_alcu_realloc_thr_spec(ErtsAlcType_t type, void *extra, if (allctr->thread_safe) erts_mtx_lock(&allctr->mutex); - res = do_erts_alcu_realloc(type, allctr, ptr, size, 0); + res = do_erts_alcu_realloc(type, allctr, ptr, size, 0, NULL); if (allctr->thread_safe) erts_mtx_unlock(&allctr->mutex); @@ -4031,7 +5596,7 @@ erts_alcu_realloc_mv_thr_spec(ErtsAlcType_t type, void *extra, if (cpy_size > size) cpy_size = size; sys_memcpy(res, ptr, cpy_size); - do_erts_alcu_free(type, allctr, ptr); + do_erts_alcu_free(type, allctr, ptr, NULL); if (allctr->thread_safe) erts_mtx_unlock(&allctr->mutex); } @@ -4048,40 +5613,64 @@ realloc_thr_pref(ErtsAlcType_t type, void *extra, void *p, Uint size, void *res; Allctr_t *pref_allctr, *used_allctr; UWord old_user_size; + Carrier_t *busy_pcrr_p; +#ifdef ERTS_SMP + int retried; +#endif if (!p) return erts_alcu_alloc_thr_pref(type, extra, size); pref_allctr = get_pref_allctr(extra); - used_allctr = get_used_allctr(extra, p, &old_user_size); + + if (pref_allctr->thread_safe) + erts_mtx_lock(&pref_allctr->mutex); + +#ifdef ERTS_SMP + ASSERT(pref_allctr->dd.use); + ERTS_ALCU_HANDLE_DD_IN_OP(pref_allctr, 1); + retried = 0; +restart: +#endif + + used_allctr = get_used_allctr(pref_allctr, ERTS_ALC_TS_PREF_LOCK_NO, + p, &old_user_size, &busy_pcrr_p); ASSERT(used_allctr && pref_allctr); if (!force_move && used_allctr == pref_allctr) { - if (used_allctr->thread_safe) - erts_mtx_lock(&used_allctr->mutex); ERTS_ALCU_DBG_CHK_THR_ACCESS(used_allctr); res = do_erts_alcu_realloc(type, used_allctr, p, size, - 0); - if (used_allctr->thread_safe) - erts_mtx_unlock(&used_allctr->mutex); + 0, + &busy_pcrr_p); + clear_busy_pool_carrier(used_allctr, busy_pcrr_p); +#ifdef ERTS_SMP + if (!res && !retried && ERTS_ALCU_HANDLE_DD_IN_OP(pref_allctr, 1)) { + /* Cleaned up a bit more; try one more time... */ + retried = 1; + goto restart; + } +#endif + if (pref_allctr->thread_safe) + erts_mtx_unlock(&pref_allctr->mutex); } else { - if (pref_allctr->thread_safe) - erts_mtx_lock(&pref_allctr->mutex); res = do_erts_alcu_alloc(type, pref_allctr, size); - if (pref_allctr->thread_safe && used_allctr != pref_allctr) { - erts_mtx_unlock(&pref_allctr->mutex); - } - if (res) { - DEBUG_CHECK_ALIGNMENT(res); + if (!res) + goto unlock_ts_return; + else { - sys_memcpy(res, p, MIN(size,old_user_size)); + DEBUG_CHECK_ALIGNMENT(res); if (used_allctr != pref_allctr) { + if (pref_allctr->thread_safe) + erts_mtx_unlock(&pref_allctr->mutex); + + sys_memcpy(res, p, MIN(size, old_user_size)); + enqueue_dealloc_other_instance(type, used_allctr, p, @@ -4089,8 +5678,14 @@ realloc_thr_pref(ErtsAlcType_t type, void *extra, void *p, Uint size, - pref_allctr->dd.ix)); } else { - do_erts_alcu_free(type, used_allctr, p); + + sys_memcpy(res, p, MIN(size, old_user_size)); + + do_erts_alcu_free(type, used_allctr, p, &busy_pcrr_p); ASSERT(pref_allctr == used_allctr); + clear_busy_pool_carrier(used_allctr, busy_pcrr_p); + + unlock_ts_return: if (pref_allctr->thread_safe) erts_mtx_unlock(&pref_allctr->mutex); } @@ -4124,6 +5719,11 @@ erts_alcu_start(Allctr_t *allctr, AllctrInit_t *init) { /* erts_alcu_start assumes that allctr has been zeroed */ + if (((UWord)allctr & ERTS_CRR_ALCTR_FLG_MASK) != 0) { + erts_exit(ERTS_ABORT_EXIT, "%s:%d:erts_alcu_start: Alignment error\n", + __FILE__, __LINE__); + } + if (!initialized) goto error; @@ -4175,7 +5775,7 @@ erts_alcu_start(Allctr_t *allctr, AllctrInit_t *init) allctr->mbc_move_threshold = init->rmbcmt; #if HAVE_ERTS_MSEG allctr->max_mseg_sbcs = init->mmsbc; -# if HAVE_SUPER_ALIGNED_MB_CARRIERS +# if ERTS_SUPER_ALIGNED_MSEG_ONLY allctr->max_mseg_mbcs = ~(Uint)0; # else allctr->max_mseg_mbcs = init->mmmbc; @@ -4198,13 +5798,27 @@ erts_alcu_start(Allctr_t *allctr, AllctrInit_t *init) #if ERTS_SMP if (init->tpref) { Uint sz = ABLK_HDR_SZ; - sz += ERTS_ALCU_DD_FIX_TYPE_OFFS*sizeof(UWord); - if (init->fix) - sz += sizeof(UWord); + sz += (init->fix ? + sizeof(ErtsAllctrFixDDBlock_t) : sizeof(ErtsAllctrDDBlock_t)); sz = UNIT_CEILING(sz); if (sz > allctr->min_block_size) allctr->min_block_size = sz; } + + allctr->cpool.pooled_list.next = &allctr->cpool.pooled_list; + allctr->cpool.pooled_list.prev = &allctr->cpool.pooled_list; + allctr->cpool.traitor_list.next = &allctr->cpool.traitor_list; + allctr->cpool.traitor_list.prev = &allctr->cpool.traitor_list; + allctr->cpool.dc_list.first = NULL; + allctr->cpool.dc_list.last = NULL; + allctr->cpool.abandon_limit = 0; + allctr->cpool.disable_abandon = 0; + erts_atomic_init_nob(&allctr->cpool.stat.blocks_size, 0); + erts_atomic_init_nob(&allctr->cpool.stat.no_blocks, 0); + erts_atomic_init_nob(&allctr->cpool.stat.carriers_size, 0); + erts_atomic_init_nob(&allctr->cpool.stat.no_carriers, 0); + allctr->cpool.check_limit_count = ERTS_ALC_CPOOL_CHECK_LIMIT_COUNT; + allctr->cpool.util_limit = init->ts ? 0 : init->acul; #endif allctr->sbc_threshold = init->sbct; @@ -4224,27 +5838,6 @@ erts_alcu_start(Allctr_t *allctr, AllctrInit_t *init) } #endif - - - allctr->sbmbc_threshold = init->sbmbct; - - if (!erts_have_sbmbc_alloc - || ERTS_IS_SBMBC_ALLOCATOR_NO__(allctr->alloc_no)) - allctr->sbmbc_threshold = 0; - - if (!allctr->sbmbc_threshold) - allctr->sbmbc_size = 0; - else { - Uint min_size; - allctr->sbmbc_size = init->sbmbcs; - min_size = allctr->sbmbc_threshold; - min_size += allctr->min_block_size; - min_size += MBC_HEADER_SIZE(allctr); - if (allctr->sbmbc_size < min_size) - allctr->sbmbc_size = min_size; - } - - #if HAVE_ERTS_MSEG if (allctr->mseg_opt.abs_shrink_th > ~((UWord) 0) / 100) allctr->mseg_opt.abs_shrink_th = ~((UWord) 0) / 100; @@ -4256,17 +5849,13 @@ erts_alcu_start(Allctr_t *allctr, AllctrInit_t *init) #ifdef ERTS_ENABLE_LOCK_COUNT erts_mtx_init_x_opt(&allctr->mutex, - ERTS_IS_SBMBC_ALLOCATOR_NO__(allctr->alloc_no) - ? "sbmbc_alloc" - : "alcu_allocator", + "alcu_allocator", make_small(allctr->alloc_no), - ERTS_LCNT_LT_ALLOC); + ERTS_LCNT_LT_ALLOC,1); #else erts_mtx_init_x(&allctr->mutex, - ERTS_IS_SBMBC_ALLOCATOR_NO__(allctr->alloc_no) - ? "sbmbc_alloc" - : "alcu_allocator", - make_small(allctr->alloc_no)); + "alcu_allocator", + make_small(allctr->alloc_no),1); #endif /*ERTS_ENABLE_LOCK_COUNT*/ #ifdef DEBUG @@ -4284,6 +5873,8 @@ erts_alcu_start(Allctr_t *allctr, AllctrInit_t *init) if (!allctr->get_next_mbc_size) allctr->get_next_mbc_size = get_next_mbc_size; + if (allctr->mbc_header_size < sizeof(Carrier_t)) + goto error; #ifdef ERTS_SMP allctr->dd.use = 0; if (init->tpref) { @@ -4292,6 +5883,9 @@ erts_alcu_start(Allctr_t *allctr, AllctrInit_t *init) allctr->dd.ix = init->ix; } #endif + allctr->mbc_header_size = (UNIT_CEILING(allctr->mbc_header_size + + ABLK_HDR_SZ) + - ABLK_HDR_SZ); if (allctr->main_carrier_size) { Block_t *blk; @@ -4300,14 +5894,22 @@ erts_alcu_start(Allctr_t *allctr, AllctrInit_t *init) allctr->main_carrier_size, CFLG_MBC | CFLG_FORCE_SIZE -#if !HALFWORD_HEAP && !HAVE_SUPER_ALIGNED_MB_CARRIERS + | CFLG_NO_CPOOL +#if !HALFWORD_HEAP && !ERTS_SUPER_ALIGNED_MSEG_ONLY | CFLG_FORCE_SYS_ALLOC #endif | CFLG_MAIN_CARRIER); - if (!blk) - goto error; + if (!blk) { +#ifdef USE_THREADS + if (allctr->thread_safe) + erts_mtx_destroy(&allctr->mutex); +#endif + erts_exit(ERTS_ABORT_EXIT, + "Failed to create main carrier for %salloc\n", + init->name_prefix); + } - (*allctr->link_free_block)(allctr, blk, 0); + (*allctr->link_free_block)(allctr, blk); HARD_CHECK_BLK_CARRIER(allctr, blk); @@ -4318,13 +5920,24 @@ erts_alcu_start(Allctr_t *allctr, AllctrInit_t *init) allctr->fix = init->fix; allctr->fix_shrink_scheduled = 0; for (i = 0; i < ERTS_ALC_NO_FIXED_SIZES; i++) { - allctr->fix[i].max_used = 0; - allctr->fix[i].limit = 0; allctr->fix[i].type_size = init->fix_type_size[i]; allctr->fix[i].list_size = 0; allctr->fix[i].list = NULL; - allctr->fix[i].allocated = 0; - allctr->fix[i].used = 0; +#ifdef ERTS_SMP + ASSERT(allctr->fix[i].type_size >= sizeof(ErtsAllctrFixDDBlock_t)); +#endif + if (ERTS_ALC_IS_CPOOL_ENABLED(allctr)) { + allctr->fix[i].u.cpool.min_list_size = 0; + allctr->fix[i].u.cpool.shrink_list = 0; + allctr->fix[i].u.cpool.allocated = 0; + allctr->fix[i].u.cpool.used = 0; + } + else { + allctr->fix[i].u.nocpool.max_used = 0; + allctr->fix[i].u.nocpool.limit = 0; + allctr->fix[i].u.nocpool.allocated = 0; + allctr->fix[i].u.nocpool.used = 0; + } } } @@ -4349,11 +5962,9 @@ erts_alcu_stop(Allctr_t *allctr) allctr->stopped = 1; while (allctr->sbc_list.first) - destroy_carrier(allctr, SBC2BLK(allctr, allctr->sbc_list.first)); + destroy_carrier(allctr, SBC2BLK(allctr, allctr->sbc_list.first), NULL); while (allctr->mbc_list.first) - destroy_carrier(allctr, MBC_TO_FIRST_BLK(allctr, allctr->mbc_list.first)); - while (allctr->sbmbc_list.first) - destroy_sbmbc(allctr, MBC_TO_FIRST_BLK(allctr, allctr->sbmbc_list.first)); + destroy_carrier(allctr, MBC_TO_FIRST_BLK(allctr, allctr->mbc_list.first), NULL); #ifdef USE_THREADS if (allctr->thread_safe) @@ -4367,14 +5978,23 @@ erts_alcu_stop(Allctr_t *allctr) void erts_alcu_init(AlcUInit_t *init) { - ASSERT(SBC_BLK_SZ_MASK == MBC_FBLK_SZ_MASK); /* see BLK_SZ */ +#ifdef ERTS_SMP + int i; + for (i = 0; i <= ERTS_ALC_A_MAX; i++) { + ErtsAlcCPoolData_t *sentinel = &carrier_pool[i].sentinel; + erts_atomic_init_nob(&sentinel->next, (erts_aint_t) sentinel); + erts_atomic_init_nob(&sentinel->prev, (erts_aint_t) sentinel); + } +#endif + ERTS_CT_ASSERT(SBC_BLK_SZ_MASK == MBC_FBLK_SZ_MASK); /* see BLK_SZ */ #if HAVE_ERTS_MSEG - ASSERT(erts_mseg_unit_size() == MSEG_UNIT_SZ); + ASSERT(erts_mseg_unit_size() == ERTS_SACRR_UNIT_SZ); max_mseg_carriers = init->mmc; - sys_alloc_carrier_size = MSEG_UNIT_CEILING(init->ycs); + sys_alloc_carrier_size = ERTS_SACRR_UNIT_CEILING(init->ycs); #else /* #if HAVE_ERTS_MSEG */ sys_alloc_carrier_size = ((init->ycs + 4095) / 4096) * 4096; #endif + allow_sys_alloc_carriers = init->sac; #ifdef DEBUG carrier_alignment = sizeof(Unit_t); @@ -4394,43 +6014,80 @@ erts_alcu_init(AlcUInit_t *init) * to erts_alcu_test() * \* */ -unsigned long -erts_alcu_test(unsigned long op, unsigned long a1, unsigned long a2) +UWord +erts_alcu_test(UWord op, UWord a1, UWord a2) { switch (op) { - case 0x000: return (unsigned long) BLK_SZ((Block_t *) a1); - case 0x001: return (unsigned long) BLK_UMEM_SZ((Block_t *) a1); - case 0x002: return (unsigned long) IS_PREV_BLK_FREE((Block_t *) a1); - case 0x003: return (unsigned long) IS_FREE_BLK((Block_t *) a1); - case 0x004: return (unsigned long) IS_LAST_BLK((Block_t *) a1); - case 0x005: return (unsigned long) UMEM2BLK((void *) a1); - case 0x006: return (unsigned long) BLK2UMEM((Block_t *) a1); - case 0x007: return (unsigned long) IS_SB_CARRIER((Carrier_t *) a1); - case 0x008: return (unsigned long) IS_SBC_BLK((Block_t *) a1); - case 0x009: return (unsigned long) IS_MB_CARRIER((Carrier_t *) a1); - case 0x00a: return (unsigned long) IS_MSEG_CARRIER((Carrier_t *) a1); - case 0x00b: return (unsigned long) CARRIER_SZ((Carrier_t *) a1); - case 0x00c: return (unsigned long) SBC2BLK((Allctr_t *) a1, + case 0x000: return (UWord) BLK_SZ((Block_t *) a1); + case 0x001: return (UWord) BLK_UMEM_SZ((Block_t *) a1); + case 0x002: return (UWord) IS_PREV_BLK_FREE((Block_t *) a1); + case 0x003: return (UWord) IS_FREE_BLK((Block_t *) a1); + case 0x004: return (UWord) IS_LAST_BLK((Block_t *) a1); + case 0x005: return (UWord) UMEM2BLK((void *) a1); + case 0x006: return (UWord) BLK2UMEM((Block_t *) a1); + case 0x007: return (UWord) IS_SB_CARRIER((Carrier_t *) a1); + case 0x008: return (UWord) IS_SBC_BLK((Block_t *) a1); + case 0x009: return (UWord) IS_MB_CARRIER((Carrier_t *) a1); + case 0x00a: return (UWord) IS_MSEG_CARRIER((Carrier_t *) a1); + case 0x00b: return (UWord) CARRIER_SZ((Carrier_t *) a1); + case 0x00c: return (UWord) SBC2BLK((Allctr_t *) a1, (Carrier_t *) a2); - case 0x00d: return (unsigned long) BLK_TO_SBC((Block_t *) a2); - case 0x00e: return (unsigned long) MBC_TO_FIRST_BLK((Allctr_t *) a1, + case 0x00d: return (UWord) BLK_TO_SBC((Block_t *) a2); + case 0x00e: return (UWord) MBC_TO_FIRST_BLK((Allctr_t *) a1, (Carrier_t *) a2); - case 0x00f: return (unsigned long) FIRST_BLK_TO_MBC((Allctr_t *) a1, + case 0x00f: return (UWord) FIRST_BLK_TO_MBC((Allctr_t *) a1, (Block_t *) a2); - case 0x010: return (unsigned long) ((Allctr_t *) a1)->mbc_list.first; - case 0x011: return (unsigned long) ((Allctr_t *) a1)->mbc_list.last; - case 0x012: return (unsigned long) ((Allctr_t *) a1)->sbc_list.first; - case 0x013: return (unsigned long) ((Allctr_t *) a1)->sbc_list.last; - case 0x014: return (unsigned long) ((Carrier_t *) a1)->next; - case 0x015: return (unsigned long) ((Carrier_t *) a1)->prev; - case 0x016: return (unsigned long) ABLK_HDR_SZ; - case 0x017: return (unsigned long) ((Allctr_t *) a1)->min_block_size; - case 0x018: return (unsigned long) NXT_BLK((Block_t *) a1); - case 0x019: return (unsigned long) PREV_BLK((Block_t *) a1); - case 0x01a: return (unsigned long) IS_MBC_FIRST_BLK((Allctr_t*)a1, (Block_t *) a2); - case 0x01b: return (unsigned long) sizeof(Unit_t); - default: ASSERT(0); return ~((unsigned long) 0); + case 0x010: return (UWord) ((Allctr_t *) a1)->mbc_list.first; + case 0x011: return (UWord) ((Allctr_t *) a1)->mbc_list.last; + case 0x012: return (UWord) ((Allctr_t *) a1)->sbc_list.first; + case 0x013: return (UWord) ((Allctr_t *) a1)->sbc_list.last; + case 0x014: return (UWord) ((Carrier_t *) a1)->next; + case 0x015: return (UWord) ((Carrier_t *) a1)->prev; + case 0x016: return (UWord) ABLK_HDR_SZ; + case 0x017: return (UWord) ((Allctr_t *) a1)->min_block_size; + case 0x018: return (UWord) NXT_BLK((Block_t *) a1); + case 0x019: return (UWord) PREV_BLK((Block_t *) a1); + case 0x01a: return (UWord) IS_MBC_FIRST_BLK((Allctr_t*)a1, (Block_t *) a2); + case 0x01b: return (UWord) sizeof(Unit_t); + case 0x01c: return (UWord) BLK_TO_MBC((Block_t*) a1); + case 0x01d: ((Allctr_t*) a1)->add_mbc((Allctr_t*)a1, (Carrier_t*)a2); break; + case 0x01e: ((Allctr_t*) a1)->remove_mbc((Allctr_t*)a1, (Carrier_t*)a2); break; +#ifdef ERTS_SMP + case 0x01f: return (UWord) sizeof(ErtsAlcCrrPool_t); + case 0x020: + SET_CARRIER_HDR((Carrier_t *) a2, 0, SCH_SYS_ALLOC|SCH_MBC, (Allctr_t *) a1); + cpool_init_carrier_data((Allctr_t *) a1, (Carrier_t *) a2); + return (UWord) a2; + case 0x021: + cpool_insert((Allctr_t *) a1, (Carrier_t *) a2); + return (UWord) a2; + case 0x022: + cpool_delete((Allctr_t *) a1, (Allctr_t *) a1, (Carrier_t *) a2); + return (UWord) a2; + case 0x023: return (UWord) cpool_is_empty((Allctr_t *) a1); + case 0x024: return (UWord) cpool_dbg_is_in_pool((Allctr_t *) a1, (Carrier_t *) a2); +#else + case 0x01f: return (UWord) 0; + case 0x020: return (UWord) 0; + case 0x021: return (UWord) 0; + case 0x022: return (UWord) 0; + case 0x023: return (UWord) 0; + case 0x024: return (UWord) 0; +#endif + case 0x025: /* UMEM2BLK_TEST*/ +#ifdef DEBUG +# ifdef HARD_DEBUG + return (UWord)UMEM2BLK(a1-3*sizeof(UWord)); +# else + return (UWord)UMEM2BLK(a1-2*sizeof(UWord)); +# endif +#else + return (UWord)UMEM2BLK(a1); +#endif + + default: ASSERT(0); return ~((UWord) 0); } + return 0; } /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *\ @@ -4438,6 +6095,20 @@ erts_alcu_test(unsigned long op, unsigned long a1, unsigned long a2) \* */ void +erts_alcu_assert_failed(char* expr, char* file, int line, char *func) +{ + fflush(stdout); + fprintf(stderr, "%s:%d:%s(): Assertion failed: %s\n", + file, line, func, expr); + fflush(stderr); +#if defined(__WIN__) || defined(__WIN32__) + DebugBreak(); +#else + abort(); +#endif +} + +void erts_alcu_verify_unused(Allctr_t *allctr) { UWord no; @@ -4445,13 +6116,11 @@ erts_alcu_verify_unused(Allctr_t *allctr) no = allctr->sbcs.curr.norm.mseg.no; no += allctr->sbcs.curr.norm.sys_alloc.no; no += allctr->mbcs.blocks.curr.no; - no += allctr->sbmbcs.blocks.curr.no; if (no) { UWord sz = allctr->sbcs.blocks.curr.size; sz += allctr->mbcs.blocks.curr.size; - sz += allctr->sbmbcs.blocks.curr.size; - erl_exit(ERTS_ABORT_EXIT, + erts_exit(ERTS_ABORT_EXIT, "%salloc() used when expected to be unused!\n" "Total amount of blocks allocated: %bpu\n" "Total amount of bytes allocated: %bpu\n", @@ -4493,7 +6162,7 @@ check_blk_carrier(Allctr_t *allctr, Block_t *iblk) ASSERT(CARRIER_SZ(sbc) - SBC_HEADER_SIZE >= SBC_BLK_SZ(iblk)); #if HAVE_ERTS_MSEG if (IS_MSEG_CARRIER(sbc)) { - ASSERT(CARRIER_SZ(sbc) % MSEG_UNIT_SZ == 0); + ASSERT(CARRIER_SZ(sbc) % ERTS_SACRR_UNIT_SZ == 0); } #endif crr = sbc; @@ -4578,13 +6247,13 @@ check_blk_carrier(Allctr_t *allctr, Block_t *iblk) #if HAVE_ERTS_MSEG if (IS_MSEG_CARRIER(crr)) { - ASSERT(CARRIER_SZ(crr) % MSEG_UNIT_SZ == 0); + ASSERT(CARRIER_SZ(crr) % ERTS_SACRR_UNIT_SZ == 0); } #endif cl = &allctr->mbc_list; } -#if 0 /* FIXIT sbmbc */ +#ifdef DEBUG if (cl->first == crr) { ASSERT(!crr->prev); } diff --git a/erts/emulator/beam/erl_alloc_util.h b/erts/emulator/beam/erl_alloc_util.h index e0754e7f69..f4a2ae7ff3 100644 --- a/erts/emulator/beam/erl_alloc_util.h +++ b/erts/emulator/beam/erl_alloc_util.h @@ -1,18 +1,19 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2002-2012. All Rights Reserved. + * Copyright Ericsson AB 2002-2013. All Rights Reserved. * - * The contents of this file are subject to the Erlang Public License, - * Version 1.1, (the "License"); you may not use this file except in - * compliance with the License. You should have received a copy of the - * Erlang Public License along with this software. If not, it can be - * retrieved online at http://www.erlang.org/. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Software distributed under the License is distributed on an "AS IS" - * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See - * the License for the specific language governing rights and limitations - * under the License. + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. * * %CopyrightEnd% */ @@ -32,6 +33,7 @@ typedef struct Allctr_t_ Allctr_t; typedef struct { UWord ycs; UWord mmc; + int sac; } AlcUInit_t; typedef struct { @@ -55,8 +57,7 @@ typedef struct { UWord lmbcs; UWord smbcs; UWord mbcgs; - UWord sbmbct; - UWord sbmbcs; + int acul; void *fix; size_t *fix_type_size; @@ -76,7 +77,8 @@ typedef struct { #define ERTS_DEFAULT_ALCU_INIT { \ 1024*1024, /* (bytes) ycs: sys_alloc carrier size */\ - 1024 /* (amount) mmc: max mseg carriers */\ + ~((UWord) 0), /* (amount) mmc: max mseg carriers */\ + 1 /* (bool) sac: sys_alloc carriers */\ } #define ERTS_DEFAULT_ALLCTR_INIT { \ @@ -96,12 +98,11 @@ typedef struct { 50, /* (%) rmbcmt: rel mbc move threshold */\ 1024*1024, /* (bytes) mmbcs: main multiblock carrier size */\ 256, /* (amount) mmsbc: max mseg sbcs */\ - 10, /* (amount) mmmbc: max mseg mbcs */\ + ~((UWord) 0), /* (amount) mmmbc: max mseg mbcs */ \ 10*1024*1024, /* (bytes) lmbcs: largest mbc size */\ 1024*1024, /* (bytes) smbcs: smallest mbc size */\ 10, /* (amount) mbcgs: mbc growth stages */\ - 256, /* (bytes) sbmbct: small block mbc threshold */\ - 8*1024, /* (bytes) sbmbcs: small block mbc size */ \ + 0, /* (%) acul: abandon carrier utilization limit */\ /* --- Data not options -------------------------------------------- */\ NULL, /* (ptr) fix */\ NULL /* (ptr) fix_type_size */\ @@ -111,7 +112,8 @@ typedef struct { #define ERTS_DEFAULT_ALCU_INIT { \ 128*1024, /* (bytes) ycs: sys_alloc carrier size */\ - 1024 /* (amount) mmc: max mseg carriers */\ + 1024, /* (amount) mmc: max mseg carriers */\ + 1 /* (bool) sac: sys_alloc carriers */\ } #define ERTS_DEFAULT_ALLCTR_INIT { \ @@ -130,12 +132,11 @@ typedef struct { 80, /* (%) rsbcmt: rel sbc move threshold */\ 128*1024, /* (bytes) mmbcs: main multiblock carrier size */\ 256, /* (amount) mmsbc: max mseg sbcs */\ - 10, /* (amount) mmmbc: max mseg mbcs */\ + ~((UWord) 0), /* (amount) mmmbc: max mseg mbcs */ \ 1024*1024, /* (bytes) lmbcs: largest mbc size */\ 128*1024, /* (bytes) smbcs: smallest mbc size */\ 10, /* (amount) mbcgs: mbc growth stages */\ - 256, /* (bytes) sbmbct: small block mbc threshold */\ - 8*1024, /* (bytes) sbmbcs: small block mbc size */ \ + 0, /* (%) acul: abandon carrier utilization limit */\ /* --- Data not options -------------------------------------------- */\ NULL, /* (ptr) fix */\ NULL /* (ptr) fix_type_size */\ @@ -165,8 +166,8 @@ void erts_alcu_free_thr_pref(ErtsAlcType_t, void *, void *); #endif Eterm erts_alcu_au_info_options(int *, void *, Uint **, Uint *); Eterm erts_alcu_info_options(Allctr_t *, int *, void *, Uint **, Uint *); -Eterm erts_alcu_sz_info(Allctr_t *, int, int *, void *, Uint **, Uint *); -Eterm erts_alcu_info(Allctr_t *, int, int *, void *, Uint **, Uint *); +Eterm erts_alcu_sz_info(Allctr_t *, int, int, int *, void *, Uint **, Uint *); +Eterm erts_alcu_info(Allctr_t *, int, int, int *, void *, Uint **, Uint *); void erts_alcu_init(AlcUInit_t *); void erts_alcu_current_size(Allctr_t *, AllctrSize_t *, ErtsAlcUFixInfo_t *, int); @@ -181,7 +182,6 @@ erts_aint32_t erts_alcu_fix_alloc_shrink(Allctr_t *, erts_aint32_t); #define ERL_ALLOC_UTIL_IMPL__ #define ERTS_ALCU_FLG_FAIL_REALLOC_MOVE (((Uint32) 1) << 0) -#define ERTS_ALCU_FLG_SBMBC (((Uint32) 1) << 1) #ifdef USE_THREADS #define ERL_THREADS_EMU_INTERNAL__ @@ -221,48 +221,87 @@ erts_aint32_t erts_alcu_fix_alloc_shrink(Allctr_t *, erts_aint32_t); #define MBC_FBLK_SZ_MASK UNIT_MASK #define CARRIER_SZ_MASK UNIT_MASK - -#if HAVE_ERTS_MSEG +#if ERTS_HAVE_MSEG_SUPER_ALIGNED \ + || (!HAVE_ERTS_MSEG && ERTS_HAVE_ERTS_SYS_ALIGNED_ALLOC) +# ifndef MSEG_ALIGN_BITS +# define ERTS_SUPER_ALIGN_BITS MSEG_ALIGN_BITS +# else +# define ERTS_SUPER_ALIGN_BITS 18 +# endif # ifdef ARCH_64 # define MBC_ABLK_OFFSET_BITS 24 -# elif HAVE_SUPER_ALIGNED_MB_CARRIERS +# else # define MBC_ABLK_OFFSET_BITS 9 /* Affects hard limits for sbct and lmbcs documented in erts_alloc.xml */ # endif -#endif -#ifndef MBC_ABLK_OFFSET_BITS +# define ERTS_SACRR_UNIT_SHIFT ERTS_SUPER_ALIGN_BITS +# define ERTS_SACRR_UNIT_SZ (1 << ERTS_SACRR_UNIT_SHIFT) +# define ERTS_SACRR_UNIT_MASK ((~(UWord)0) << ERTS_SACRR_UNIT_SHIFT) +# define ERTS_SACRR_UNIT_FLOOR(X) ((X) & ERTS_SACRR_UNIT_MASK) +# define ERTS_SACRR_UNIT_CEILING(X) ERTS_SACRR_UNIT_FLOOR((X) + ~ERTS_SACRR_UNIT_MASK) +# define ERTS_SA_MB_CARRIERS 1 +#else +# define ERTS_SA_MB_CARRIERS 0 # define MBC_ABLK_OFFSET_BITS 0 /* no carrier offset in block header */ #endif +#if ERTS_HAVE_MSEG_SUPER_ALIGNED && !ERTS_HAVE_ERTS_SYS_ALIGNED_ALLOC +# define ERTS_SUPER_ALIGNED_MSEG_ONLY 1 +#else +# define ERTS_SUPER_ALIGNED_MSEG_ONLY 0 +#endif #if MBC_ABLK_OFFSET_BITS # define MBC_ABLK_OFFSET_SHIFT (sizeof(UWord)*8 - MBC_ABLK_OFFSET_BITS) # define MBC_ABLK_OFFSET_MASK (~((UWord)0) << MBC_ABLK_OFFSET_SHIFT) # define MBC_ABLK_SZ_MASK (~MBC_ABLK_OFFSET_MASK & ~FLG_MASK) -# define HAVE_ERTS_SBMBC 0 #else # define MBC_ABLK_SZ_MASK (~FLG_MASK) -# define HAVE_ERTS_SBMBC 1 #endif -#define MBC_ABLK_SZ(B) (ASSERT_EXPR(!is_sbc_blk(B)), (B)->bhdr & MBC_ABLK_SZ_MASK) -#define MBC_FBLK_SZ(B) (ASSERT_EXPR(!is_sbc_blk(B)), (B)->bhdr & MBC_FBLK_SZ_MASK) -#define SBC_BLK_SZ(B) (ASSERT_EXPR(is_sbc_blk(B)), (B)->bhdr & SBC_BLK_SZ_MASK) +#define MBC_ABLK_SZ(B) (ASSERT(!is_sbc_blk(B)), (B)->bhdr & MBC_ABLK_SZ_MASK) +#define MBC_FBLK_SZ(B) (ASSERT(!is_sbc_blk(B)), (B)->bhdr & MBC_FBLK_SZ_MASK) +#define SBC_BLK_SZ(B) (ASSERT(is_sbc_blk(B)), (B)->bhdr & SBC_BLK_SZ_MASK) #define CARRIER_SZ(C) \ ((C)->chdr & CARRIER_SZ_MASK) -extern int erts_have_sbmbc_alloc; - typedef union {char c[ERTS_ALLOC_ALIGN_BYTES]; long l; double d;} Unit_t; +#ifdef ERTS_SMP + +typedef struct ErtsDoubleLink_t_ { + struct ErtsDoubleLink_t_ *next; + struct ErtsDoubleLink_t_ *prev; +}ErtsDoubleLink_t; + +typedef struct { + erts_atomic_t next; + erts_atomic_t prev; + Allctr_t *orig_allctr; /* read-only while carrier is alive */ + ErtsThrPrgrVal thr_prgr; + erts_atomic_t max_size; + UWord abandon_limit; + UWord blocks; + UWord blocks_size; + ErtsDoubleLink_t abandoned; /* node in pooled_list or traitor_list */ +} ErtsAlcCPoolData_t; + +#endif + typedef struct Carrier_t_ Carrier_t; struct Carrier_t_ { UWord chdr; Carrier_t *next; Carrier_t *prev; - Allctr_t *allctr; + erts_smp_atomic_t allctr; +#ifdef ERTS_SMP + ErtsAlcCPoolData_t cpool; /* Overwritten by block if sbc */ +#endif }; +#define ERTS_ALC_CARRIER_TO_ALLCTR(C) \ + ((Allctr_t *) (erts_smp_atomic_read_nob(&(C)->allctr) & ~FLG_MASK)) + typedef struct { Carrier_t *first; Carrier_t *last; @@ -280,12 +319,46 @@ typedef struct { #endif } Block_t; +#define THIS_FREE_BLK_HDR_FLG (((UWord) 1) << 0) +#define PREV_FREE_BLK_HDR_FLG (((UWord) 1) << 1) +#define LAST_BLK_HDR_FLG (((UWord) 1) << 2) + +#define SBC_BLK_HDR_FLG /* Special flag combo for (allocated) SBC blocks */\ + (THIS_FREE_BLK_HDR_FLG | PREV_FREE_BLK_HDR_FLG | LAST_BLK_HDR_FLG) + +/* + * FREE_LAST_MBC_BLK_HDR_FLGS is a special flag combo used for + * distinguishing empty mbc's from allocated blocks in + * handle_delayed_dealloc(). + */ +#define FREE_LAST_MBC_BLK_HDR_FLGS (THIS_FREE_BLK_HDR_FLG | LAST_BLK_HDR_FLG) + +#define IS_FREE_LAST_MBC_BLK(B) \ + (((B)->bhdr & FLG_MASK) == FREE_LAST_MBC_BLK_HDR_FLGS) + +#define IS_SBC_BLK(B) (((B)->bhdr & FLG_MASK) == SBC_BLK_HDR_FLG) +#define IS_MBC_BLK(B) (!IS_SBC_BLK((B))) +#define IS_FREE_BLK(B) (ASSERT(IS_MBC_BLK(B)), \ + (B)->bhdr & THIS_FREE_BLK_HDR_FLG) + +#if MBC_ABLK_OFFSET_BITS +# define FBLK_TO_MBC(B) (ASSERT(IS_MBC_BLK(B) && IS_FREE_BLK(B)), \ + (B)->u.carrier) +# define ABLK_TO_MBC(B) \ + (ASSERT(IS_MBC_BLK(B) && !IS_FREE_BLK(B)), \ + (Carrier_t*)((ERTS_SACRR_UNIT_FLOOR((UWord)(B)) - \ + (((B)->bhdr >> MBC_ABLK_OFFSET_SHIFT) << ERTS_SACRR_UNIT_SHIFT)))) +# define BLK_TO_MBC(B) (IS_FREE_BLK(B) ? FBLK_TO_MBC(B) : ABLK_TO_MBC(B)) +#else +# define FBLK_TO_MBC(B) ((B)->carrier) +# define ABLK_TO_MBC(B) ((B)->carrier) +# define BLK_TO_MBC(B) ((B)->carrier) +#endif +#define MBC_BLK_SZ(B) (IS_FREE_BLK(B) ? MBC_FBLK_SZ(B) : MBC_ABLK_SZ(B)) + typedef UWord FreeBlkFtr_t; /* Footer of a free block */ -typedef struct { - UWord giga_no; - UWord no; -} CallCounter_t; +typedef Uint64 CallCounter_t; typedef struct { UWord no; @@ -298,7 +371,6 @@ typedef struct { StatValues_t mseg; StatValues_t sys_alloc; } norm; - StatValues_t small_block; } curr; StatValues_t max; StatValues_t max_ever; @@ -358,10 +430,20 @@ typedef struct { size_t type_size; SWord list_size; void *list; - SWord max_used; - SWord limit; - SWord allocated; - SWord used; + union { + struct { + SWord max_used; + SWord limit; + SWord allocated; + SWord used; + } nocpool; + struct { + int min_list_size; + int shrink_list; + UWord allocated; + UWord used; + } cpool; + } u; } ErtsAlcFixList_t; struct Allctr_t_ { @@ -409,37 +491,61 @@ struct Allctr_t_ { Uint largest_mbc_size; Uint smallest_mbc_size; Uint mbc_growth_stages; - Uint sbmbc_threshold; - Uint sbmbc_size; #if HAVE_ERTS_MSEG ErtsMsegOpt_t mseg_opt; #endif /* */ + Uint mbc_header_size; Uint min_mbc_size; Uint min_mbc_first_free_size; Uint min_block_size; /* Carriers */ - CarrierList_t sbmbc_list; CarrierList_t mbc_list; CarrierList_t sbc_list; +#ifdef ERTS_SMP + struct { + /* pooled_list, traitor list and dc_list contain only + carriers _created_ by this allocator */ + ErtsDoubleLink_t pooled_list; + ErtsDoubleLink_t traitor_list; + CarrierList_t dc_list; + + UWord abandon_limit; + int disable_abandon; + int check_limit_count; + int util_limit; + struct { + erts_atomic_t blocks_size; + erts_atomic_t no_blocks; + erts_atomic_t carriers_size; + erts_atomic_t no_carriers; + } stat; + } cpool; +#endif /* Main carrier (if there is one) */ Carrier_t * main_carrier; /* Callback functions (first 4 are mandatory) */ Block_t * (*get_free_block) (Allctr_t *, Uint, - Block_t *, Uint, Uint32); - void (*link_free_block) (Allctr_t *, Block_t *, Uint32); - void (*unlink_free_block) (Allctr_t *, Block_t *, Uint32); + Block_t *, Uint); + void (*link_free_block) (Allctr_t *, Block_t *); + void (*unlink_free_block) (Allctr_t *, Block_t *); Eterm (*info_options) (Allctr_t *, char *, int *, void *, Uint **, Uint *); Uint (*get_next_mbc_size) (Allctr_t *); - void (*creating_mbc) (Allctr_t *, Carrier_t *, Uint32); - void (*destroying_mbc) (Allctr_t *, Carrier_t *, Uint32); + void (*creating_mbc) (Allctr_t *, Carrier_t *); + void (*destroying_mbc) (Allctr_t *, Carrier_t *); + + /* The three callbacks below are needed to support carrier migration */ + void (*add_mbc) (Allctr_t *, Carrier_t *); + void (*remove_mbc) (Allctr_t *, Carrier_t *); + UWord (*largest_fblk_in_mbc) (Allctr_t *, Carrier_t *); + void (*init_atoms) (void); #ifdef ERTS_ALLOC_UTIL_HARD_DEBUG @@ -471,8 +577,6 @@ struct Allctr_t_ { CallCounter_t this_alloc; CallCounter_t this_free; CallCounter_t this_realloc; - CallCounter_t sbmbc_alloc; - CallCounter_t sbmbc_free; CallCounter_t mseg_alloc; CallCounter_t mseg_dealloc; CallCounter_t mseg_realloc; @@ -483,8 +587,7 @@ struct Allctr_t_ { CarriersStats_t sbcs; CarriersStats_t mbcs; - CarriersStats_t sbmbcs; - + #ifdef DEBUG #ifdef USE_THREADS struct { @@ -501,7 +604,9 @@ void erts_alcu_stop(Allctr_t *); void erts_alcu_verify_unused(Allctr_t *); void erts_alcu_verify_unused_ts(Allctr_t *allctr); -unsigned long erts_alcu_test(unsigned long, unsigned long, unsigned long); +UWord erts_alcu_test(UWord, UWord, UWord); + +void erts_alcu_assert_failed(char* expr, char* file, int line, char *func); #ifdef DEBUG int is_sbc_blk(Block_t*); diff --git a/erts/emulator/beam/erl_ao_firstfit_alloc.c b/erts/emulator/beam/erl_ao_firstfit_alloc.c index 6ce209085c..5cd067ff54 100644 --- a/erts/emulator/beam/erl_ao_firstfit_alloc.c +++ b/erts/emulator/beam/erl_ao_firstfit_alloc.c @@ -3,16 +3,17 @@ * * Copyright Ericsson AB 2003-2013. All Rights Reserved. * - * The contents of this file are subject to the Erlang Public License, - * Version 1.1, (the "License"); you may not use this file except in - * compliance with the License. You should have received a copy of the - * Erlang Public License along with this software. If not, it can be - * retrieved online at http://www.erlang.org/. - * - * Software distributed under the License is distributed on an "AS IS" - * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See - * the License for the specific language governing rights and limitations - * under the License. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. * * %CopyrightEnd% */ @@ -28,11 +29,17 @@ * * This module is a callback-module for erl_alloc_util.c * - * Algorithm: The tree nodes are free-blocks ordered in address order. + * AOFF Algorithm: + * The tree nodes are ordered in address order. * Every node also keeps the size of the largest block in its - * sub-tree ('max_size'). By that we can start from root and keep + * sub-tree ('max_sz'). By that we can start from root and keep * left (for low addresses) while dismissing entire sub-trees with * too small blocks. + * Bestfit within carrier: + * The only difference for "bestfit within carrier" is the tree + * sorting order. Blocks within the same carrier are sorted + * wrt size instead of address. The 'max_sz' field is maintained + * in order to dismiss entire carriers with too small blocks. * * Authors: Rickard Green/Sverker Eriksson */ @@ -62,6 +69,15 @@ # define LEFT_VISITED_FLG (((Uint) 1) << 2) # define RIGHT_VISITED_FLG (((Uint) 1) << 3) #endif +#ifdef DEBUG +# define IS_BF_FLG (((Uint) 1) << 4) +#endif + +#define IS_TREE_NODE(N) (((AOFF_RBTree_t *) (N))->flags & TREE_NODE_FLG) +#define IS_LIST_ELEM(N) (!IS_TREE_NODE(((AOFF_RBTree_t *) (N)))) + +#define SET_TREE_NODE(N) (((AOFF_RBTree_t *) (N))->flags |= TREE_NODE_FLG) +#define SET_LIST_ELEM(N) (((AOFF_RBTree_t *) (N))->flags &= ~TREE_NODE_FLG) #define IS_RED(N) (((AOFF_RBTree_t *) (N)) \ && ((AOFF_RBTree_t *) (N))->flags & RED_FLG) @@ -70,9 +86,6 @@ #define SET_RED(N) (((AOFF_RBTree_t *) (N))->flags |= RED_FLG) #define SET_BLACK(N) (((AOFF_RBTree_t *) (N))->flags &= ~RED_FLG) -#undef ASSERT -#define ASSERT ASSERT_EXPR - #if 1 #define RBT_ASSERT ASSERT #else @@ -85,21 +98,55 @@ typedef struct AOFF_RBTree_t_ AOFF_RBTree_t; struct AOFF_RBTree_t_ { Block_t hdr; - Uint flags; AOFF_RBTree_t *parent; AOFF_RBTree_t *left; AOFF_RBTree_t *right; - Uint max_sz; /* of all blocks in this sub-tree */ + Uint32 flags; + Uint32 max_sz; /* of all blocks in this sub-tree */ }; #define AOFF_BLK_SZ(B) MBC_FBLK_SZ(&(B)->hdr) +/* BF block nodes keeps list of all with equal size + */ +typedef struct { + AOFF_RBTree_t t; + AOFF_RBTree_t *next; +}AOFF_RBTreeList_t; + +#define LIST_NEXT(N) (((AOFF_RBTreeList_t*) (N))->next) +#define LIST_PREV(N) (((AOFF_RBTreeList_t*) (N))->t.parent) + +typedef struct AOFF_Carrier_t_ AOFF_Carrier_t; + +struct AOFF_Carrier_t_ { + Carrier_t crr; + AOFF_RBTree_t rbt_node; /* My node in the carrier tree */ + AOFF_RBTree_t* root; /* Root of my block tree */ +}; +#define RBT_NODE_TO_MBC(PTR) ((AOFF_Carrier_t*)((char*)(PTR) - offsetof(AOFF_Carrier_t, rbt_node))) + +/* + To support carrier migration we keep two kinds of rb-trees: + 1. One tree of carriers for each allocator instance. + 2. One tree of free blocks for each carrier. + Both trees use the same node structure AOFF_RBTree_t and implementation. + Carrier nodes thus contain a phony Block_t header 'rbt_node.hdr'. + The size value of such a phony block is the size of the largest free block in + that carrier, i.e same as 'max_sz' of the root node of its block tree. +*/ + #ifdef HARD_DEBUG -static AOFF_RBTree_t * check_tree(AOFF_RBTree_t* root, Uint); +# define HARD_CHECK_IS_MEMBER(ROOT,NODE) rbt_assert_is_member(ROOT,NODE) +# define HARD_CHECK_TREE(CRR,FLV,ROOT,SZ) check_tree(CRR, FLV, ROOT, SZ) +static AOFF_RBTree_t * check_tree(Carrier_t* within_crr, enum AOFF_Flavor flavor, AOFF_RBTree_t* root, Uint); +#else +# define HARD_CHECK_IS_MEMBER(ROOT,NODE) +# define HARD_CHECK_TREE(CRR,FLV,ROOT,SZ) #endif -/* Calculate 'max_size' of tree node x by only looking at the direct children - * of x and x itself. +/* Calculate 'max_sz' of tree node x by only looking at 'max_sz' of the + * direct children of x and the size x itself. */ static ERTS_INLINE Uint node_max_size(AOFF_RBTree_t *x) { @@ -113,7 +160,7 @@ static ERTS_INLINE Uint node_max_size(AOFF_RBTree_t *x) return sz; } -/* Set new possibly lower 'max_size' of node and propagate change toward root +/* Set new possibly lower 'max_sz' of node and propagate change toward root */ static ERTS_INLINE void lower_max_size(AOFF_RBTree_t *node, AOFF_RBTree_t* stop_at) @@ -132,30 +179,53 @@ static ERTS_INLINE void lower_max_size(AOFF_RBTree_t *node, else ASSERT(new_max == old_max); } +static ERTS_INLINE SWord cmp_blocks(enum AOFF_Flavor flavor, + AOFF_RBTree_t* lhs, AOFF_RBTree_t* rhs) +{ + ASSERT(lhs != rhs); + ASSERT(flavor == AOFF_AOFF || FBLK_TO_MBC(&lhs->hdr) == FBLK_TO_MBC(&rhs->hdr)); + if (flavor != AOFF_AOFF) { + SWord diff = (SWord)AOFF_BLK_SZ(lhs) - (SWord)AOFF_BLK_SZ(rhs); + if (diff || flavor == AOFF_BF) return diff; + } + return (char*)lhs - (char*)rhs; +} -/* Prototypes of callback functions */ -static Block_t* aoff_get_free_block(Allctr_t *, Uint, Block_t *, Uint, Uint32 flags); -static void aoff_link_free_block(Allctr_t *, Block_t*, Uint32 flags); -static void aoff_unlink_free_block(Allctr_t *allctr, Block_t *del, Uint32 flags); - -static Eterm info_options(Allctr_t *, char *, int *, void *, Uint **, Uint *); -static void init_atoms(void); - +static ERTS_INLINE SWord cmp_cand_blk(enum AOFF_Flavor flavor, + Block_t* cand_blk, AOFF_RBTree_t* rhs) +{ + if (flavor != AOFF_AOFF) { + if (BLK_TO_MBC(cand_blk) == FBLK_TO_MBC(&rhs->hdr)) { + SWord diff = (SWord)MBC_BLK_SZ(cand_blk) - (SWord)MBC_FBLK_SZ(&rhs->hdr); + if (diff || flavor == AOFF_BF) return diff; + } + } + return (char*)cand_blk - (char*)rhs; +} +/* Prototypes of callback functions */ +static Block_t* aoff_get_free_block(Allctr_t *, Uint, Block_t *, Uint); +static void aoff_link_free_block(Allctr_t *, Block_t*); +static void aoff_unlink_free_block(Allctr_t *allctr, Block_t *del); +static void aoff_creating_mbc(Allctr_t*, Carrier_t*); #ifdef DEBUG - -/* Destroy all tree fields */ -#define DESTROY_TREE_NODE(N) \ - sys_memset((void *) (((Block_t *) (N)) + 1), \ - 0xff, \ - (sizeof(AOFF_RBTree_t) - sizeof(Block_t))) - -#else - -#define DESTROY_TREE_NODE(N) - +static void aoff_destroying_mbc(Allctr_t*, Carrier_t*); #endif +static void aoff_add_mbc(Allctr_t*, Carrier_t*); +static void aoff_remove_mbc(Allctr_t*, Carrier_t*); +static UWord aoff_largest_fblk_in_mbc(Allctr_t*, Carrier_t*); + +/* Generic tree functions used by both carrier and block trees. */ +static void rbt_delete(AOFF_RBTree_t** root, AOFF_RBTree_t* del); +static void rbt_insert(enum AOFF_Flavor flavor, AOFF_RBTree_t** root, AOFF_RBTree_t* blk); +static AOFF_RBTree_t* rbt_search(AOFF_RBTree_t* root, Uint size); +#ifdef HARD_DEBUG +static int rbt_assert_is_member(AOFF_RBTree_t* root, AOFF_RBTree_t* node); +#endif + +static Eterm info_options(Allctr_t *, char *, int *, void *, Uint **, Uint *); +static void init_atoms(void); static int atoms_initialized = 0; @@ -184,9 +254,12 @@ erts_aoffalc_start(AOFFAllctr_t *alc, sys_memcpy((void *) alc, (void *) &zero.allctr, sizeof(AOFFAllctr_t)); + alc->flavor = aoffinit->flavor; + allctr->mbc_header_size = sizeof(AOFF_Carrier_t); allctr->min_mbc_size = MIN_MBC_SZ; allctr->min_mbc_first_free_size = MIN_MBC_FIRST_FREE_SZ; - allctr->min_block_size = sizeof(AOFF_RBTree_t); + allctr->min_block_size = (aoffinit->flavor == AOFF_BF ? + sizeof(AOFF_RBTreeList_t):sizeof(AOFF_RBTree_t)); allctr->vsn_str = ERTS_ALC_AOFF_ALLOC_VSN_STR; @@ -195,12 +268,19 @@ erts_aoffalc_start(AOFFAllctr_t *alc, allctr->get_free_block = aoff_get_free_block; allctr->link_free_block = aoff_link_free_block; - allctr->unlink_free_block = aoff_unlink_free_block; + allctr->unlink_free_block = aoff_unlink_free_block; allctr->info_options = info_options; allctr->get_next_mbc_size = NULL; - allctr->creating_mbc = NULL; + allctr->creating_mbc = aoff_creating_mbc; +#ifdef DEBUG + allctr->destroying_mbc = aoff_destroying_mbc; +#else allctr->destroying_mbc = NULL; +#endif + allctr->add_mbc = aoff_add_mbc; + allctr->remove_mbc = aoff_remove_mbc; + allctr->largest_fblk_in_mbc = aoff_largest_fblk_in_mbc; allctr->init_atoms = init_atoms; #ifdef ERTS_ALLOC_UTIL_HARD_DEBUG @@ -303,10 +383,7 @@ replace(AOFF_RBTree_t **root, AOFF_RBTree_t *x, AOFF_RBTree_t *y) y->parent = x->parent; y->right = x->right; y->left = x->left; - - y->max_sz = x->max_sz; - lower_max_size(y, NULL); - DESTROY_TREE_NODE(x); + y->max_sz = x->max_sz; } static void @@ -403,21 +480,72 @@ tree_insert_fixup(AOFF_RBTree_t** root, AOFF_RBTree_t *blk) } static void -aoff_unlink_free_block(Allctr_t *allctr, Block_t *del, Uint32 flags) +aoff_unlink_free_block(Allctr_t *allctr, Block_t *blk) +{ + AOFFAllctr_t* alc = (AOFFAllctr_t*)allctr; + AOFF_RBTree_t* del = (AOFF_RBTree_t*)blk; + AOFF_Carrier_t *crr = (AOFF_Carrier_t*) FBLK_TO_MBC(&del->hdr); + + ASSERT(crr->rbt_node.hdr.bhdr == crr->root->max_sz); + HARD_CHECK_TREE(&crr->crr, alc->flavor, crr->root, 0); + + if (alc->flavor == AOFF_BF) { + ASSERT(del->flags & IS_BF_FLG); + if (IS_LIST_ELEM(del)) { + /* Remove from list */ + ASSERT(LIST_PREV(del)); + ASSERT(LIST_PREV(del)->flags & IS_BF_FLG); + LIST_NEXT(LIST_PREV(del)) = LIST_NEXT(del); + if (LIST_NEXT(del)) { + ASSERT(LIST_NEXT(del)->flags & IS_BF_FLG); + LIST_PREV(LIST_NEXT(del)) = LIST_PREV(del); + } + return; + } + else if (LIST_NEXT(del)) { + /* Replace tree node by next element in list... */ + + ASSERT(AOFF_BLK_SZ(LIST_NEXT(del)) == AOFF_BLK_SZ(del)); + ASSERT(IS_LIST_ELEM(LIST_NEXT(del))); + + replace(&crr->root, (AOFF_RBTree_t*)del, LIST_NEXT(del)); + + HARD_CHECK_TREE(&crr->crr, alc->flavor, crr->root, 0); + return; + } + } + + rbt_delete(&crr->root, (AOFF_RBTree_t*)del); + + HARD_CHECK_TREE(&crr->crr, alc->flavor, crr->root, 0); + + /* Update the carrier tree with a potentially new (lower) max_sz + */ + if (crr->root) { + if (crr->rbt_node.hdr.bhdr == crr->root->max_sz) { + return; + } + ASSERT(crr->rbt_node.hdr.bhdr > crr->root->max_sz); + crr->rbt_node.hdr.bhdr = crr->root->max_sz; + } + else { + crr->rbt_node.hdr.bhdr = 0; + } + lower_max_size(&crr->rbt_node, NULL); +} + + +static void +rbt_delete(AOFF_RBTree_t** root, AOFF_RBTree_t* del) { - AOFFAllctr_t *alc = (AOFFAllctr_t *) allctr; - AOFF_RBTree_t **root = ((flags & ERTS_ALCU_FLG_SBMBC) - ? &alc->sbmbc_root : &alc->mbc_root); Uint spliced_is_black; - AOFF_RBTree_t *x, *y, *z = (AOFF_RBTree_t *) del; + AOFF_RBTree_t *x, *y, *z = del; AOFF_RBTree_t null_x; /* null_x is used to get the fixup started when we splice out a node without children. */ - null_x.parent = NULL; + HARD_CHECK_IS_MEMBER(*root, del); -#ifdef HARD_DEBUG - check_tree(*root, 0); -#endif + null_x.parent = NULL; /* Remove node from tree... */ @@ -461,7 +589,9 @@ aoff_unlink_free_block(Allctr_t *allctr, Block_t *del, Uint32 flags) } if (y != z) { /* We spliced out the successor of z; replace z by the successor */ + ASSERT(z != &null_x); replace(root, z, y); + lower_max_size(y, NULL); } if (spliced_is_black) { @@ -572,28 +702,48 @@ aoff_unlink_free_block(Allctr_t *allctr, Block_t *del, Uint32 flags) RBT_ASSERT(!null_x.right); } } +} - DESTROY_TREE_NODE(del); +static void +aoff_link_free_block(Allctr_t *allctr, Block_t *block) +{ + AOFFAllctr_t* alc = (AOFFAllctr_t*) allctr; + AOFF_RBTree_t *blk = (AOFF_RBTree_t *) block; + AOFF_RBTree_t *crr_node; + AOFF_Carrier_t *blk_crr = (AOFF_Carrier_t*) FBLK_TO_MBC(block); + Uint blk_sz = AOFF_BLK_SZ(blk); -#ifdef HARD_DEBUG - check_tree(*root, 0); -#endif + ASSERT(allctr == ERTS_ALC_CARRIER_TO_ALLCTR(&blk_crr->crr)); + ASSERT(blk_crr->rbt_node.hdr.bhdr == (blk_crr->root ? blk_crr->root->max_sz : 0)); + HARD_CHECK_TREE(&blk_crr->crr, alc->flavor, blk_crr->root, 0); + + rbt_insert(alc->flavor, &blk_crr->root, blk); + + /* Update the carrier tree with a potentially new (larger) max_sz + */ + crr_node = &blk_crr->rbt_node; + if (blk_sz > crr_node->hdr.bhdr) { + ASSERT(blk_sz == blk_crr->root->max_sz); + crr_node->hdr.bhdr = blk_sz; + while (blk_sz > crr_node->max_sz) { + crr_node->max_sz = blk_sz; + crr_node = crr_node->parent; + if (!crr_node) break; + } + } + HARD_CHECK_TREE(&blk_crr->crr, alc->flavor, blk_crr->root, 0); } static void -aoff_link_free_block(Allctr_t *allctr, Block_t *block, Uint32 flags) +rbt_insert(enum AOFF_Flavor flavor, AOFF_RBTree_t** root, AOFF_RBTree_t* blk) { - AOFFAllctr_t *alc = (AOFFAllctr_t *) allctr; - AOFF_RBTree_t *blk = (AOFF_RBTree_t *) block; - AOFF_RBTree_t **root = ((flags & ERTS_ALCU_FLG_SBMBC) - ? &alc->sbmbc_root : &alc->mbc_root); Uint blk_sz = AOFF_BLK_SZ(blk); -#ifdef HARD_DEBUG - check_tree(*root, 0); +#ifdef DEBUG + blk->flags = (flavor == AOFF_BF) ? IS_BF_FLG : 0; +#else + blk->flags = 0; #endif - - blk->flags = 0; blk->left = NULL; blk->right = NULL; blk->max_sz = blk_sz; @@ -606,10 +756,12 @@ aoff_link_free_block(Allctr_t *allctr, Block_t *block, Uint32 flags) else { AOFF_RBTree_t *x = *root; while (1) { + SWord diff; if (x->max_sz < blk_sz) { x->max_sz = blk_sz; } - if (blk < x) { + diff = cmp_blocks(flavor, blk, x); + if (diff < 0) { if (!x->left) { blk->parent = x; x->left = blk; @@ -617,7 +769,7 @@ aoff_link_free_block(Allctr_t *allctr, Block_t *block, Uint32 flags) } x = x->left; } - else { + else if (diff > 0) { if (!x->right) { blk->parent = x; x->right = blk; @@ -625,7 +777,18 @@ aoff_link_free_block(Allctr_t *allctr, Block_t *block, Uint32 flags) } x = x->right; } - + else { + ASSERT(flavor == AOFF_BF); + ASSERT(blk->flags & IS_BF_FLG); + ASSERT(x->flags & IS_BF_FLG); + SET_LIST_ELEM(blk); + LIST_NEXT(blk) = LIST_NEXT(x); + LIST_PREV(blk) = x; + if (LIST_NEXT(x)) + LIST_PREV(LIST_NEXT(x)) = blk; + LIST_NEXT(x) = blk; + return; + } } /* Insert block into size tree */ @@ -635,38 +798,63 @@ aoff_link_free_block(Allctr_t *allctr, Block_t *block, Uint32 flags) if (IS_RED(blk->parent)) tree_insert_fixup(root, blk); } - -#ifdef HARD_DEBUG - check_tree(*root, 0); -#endif + if (flavor == AOFF_BF) { + SET_TREE_NODE(blk); + LIST_NEXT(blk) = NULL; + } } -static Block_t * -aoff_get_free_block(Allctr_t *allctr, Uint size, - Block_t *cand_blk, Uint cand_size, Uint32 flags) +static AOFF_RBTree_t* +rbt_search(AOFF_RBTree_t* root, Uint size) { - AOFFAllctr_t *alc = (AOFFAllctr_t *) allctr; - AOFF_RBTree_t *x = ((flags & ERTS_ALCU_FLG_SBMBC) - ? alc->sbmbc_root : alc->mbc_root); - AOFF_RBTree_t *blk = NULL; -#ifdef HARD_DEBUG - AOFF_RBTree_t* dbg_blk = check_tree(x, size); -#endif + AOFF_RBTree_t* x = root; - ASSERT(!cand_blk || cand_size >= size); - - while (x) { + ASSERT(x); + for (;;) { if (x->left && x->left->max_sz >= size) { x = x->left; } else if (AOFF_BLK_SZ(x) >= size) { - blk = x; - break; + return x; } else { x = x->right; + if (!x) { + return NULL; + } } } +} + +static Block_t * +aoff_get_free_block(Allctr_t *allctr, Uint size, + Block_t *cand_blk, Uint cand_size) +{ + AOFFAllctr_t *alc = (AOFFAllctr_t *) allctr; + AOFF_RBTree_t *crr_node = alc->mbc_root; + AOFF_Carrier_t* crr; + AOFF_RBTree_t *blk = NULL; +#ifdef HARD_DEBUG + AOFF_RBTree_t* dbg_blk; +#endif + + ASSERT(!cand_blk || cand_size >= size); + + /* Get first-fit carrier + */ + if (!crr_node || !(blk=rbt_search(crr_node, size))) { + return NULL; + } + crr = RBT_NODE_TO_MBC(blk); + + /* Get block within carrier tree + */ +#ifdef HARD_DEBUG + dbg_blk = HARD_CHECK_TREE(&crr->crr, alc->flavor, crr->root, size); +#endif + + blk = rbt_search(crr->root, size); + ASSERT(blk); #ifdef HARD_DEBUG ASSERT(blk == dbg_blk); @@ -675,15 +863,93 @@ aoff_get_free_block(Allctr_t *allctr, Uint size, if (!blk) return NULL; - if (cand_blk && cand_blk < &blk->hdr) { + if (cand_blk && cmp_cand_blk(alc->flavor, cand_blk, blk) < 0) { return NULL; /* cand_blk was better */ } - aoff_unlink_free_block(allctr, (Block_t *) blk, flags); + aoff_unlink_free_block(allctr, (Block_t *) blk); return (Block_t *) blk; } +static void aoff_creating_mbc(Allctr_t *allctr, Carrier_t *carrier) +{ + AOFFAllctr_t *alc = (AOFFAllctr_t *) allctr; + AOFF_Carrier_t *crr = (AOFF_Carrier_t*) carrier; + AOFF_RBTree_t **root = &alc->mbc_root; + + HARD_CHECK_TREE(NULL, 0, *root, 0); + + /* Link carrier in address order tree + */ + crr->rbt_node.hdr.bhdr = 0; + rbt_insert(AOFF_AOFF, root, &crr->rbt_node); + + /* aoff_link_free_block will add free block later */ + crr->root = NULL; + + HARD_CHECK_TREE(NULL, 0, *root, 0); +} + +#define IS_CRR_IN_TREE(CRR,ROOT) \ + ((CRR)->rbt_node.parent || (ROOT) == &(CRR)->rbt_node) + +#ifdef DEBUG +static void aoff_destroying_mbc(Allctr_t *allctr, Carrier_t *carrier) +{ + AOFFAllctr_t *alc = (AOFFAllctr_t *) allctr; + AOFF_Carrier_t *crr = (AOFF_Carrier_t*) carrier; + + ASSERT(!IS_CRR_IN_TREE(crr, alc->mbc_root)); +} +#endif + +static void aoff_add_mbc(Allctr_t *allctr, Carrier_t *carrier) +{ + AOFFAllctr_t *alc = (AOFFAllctr_t *) allctr; + AOFF_Carrier_t *crr = (AOFF_Carrier_t*) carrier; + AOFF_RBTree_t **root = &alc->mbc_root; + + ASSERT(!IS_CRR_IN_TREE(crr, *root)); + HARD_CHECK_TREE(NULL, 0, *root, 0); + + /* Link carrier in address order tree + */ + rbt_insert(AOFF_AOFF, root, &crr->rbt_node); + + HARD_CHECK_TREE(NULL, 0, *root, 0); +} + +static void aoff_remove_mbc(Allctr_t *allctr, Carrier_t *carrier) +{ + AOFFAllctr_t *alc = (AOFFAllctr_t *) allctr; + AOFF_Carrier_t *crr = (AOFF_Carrier_t*) carrier; + AOFF_RBTree_t **root = &alc->mbc_root; + + ASSERT(allctr == ERTS_ALC_CARRIER_TO_ALLCTR(carrier)); + + if (!IS_CRR_IN_TREE(crr,*root)) + return; + + HARD_CHECK_TREE(NULL, 0, *root, 0); + + rbt_delete(root, &crr->rbt_node); + crr->rbt_node.parent = NULL; + crr->rbt_node.left = NULL; + crr->rbt_node.right = NULL; + crr->rbt_node.max_sz = crr->rbt_node.hdr.bhdr; + + HARD_CHECK_TREE(NULL, 0, *root, 0); +} + +static UWord aoff_largest_fblk_in_mbc(Allctr_t* allctr, Carrier_t* carrier) +{ + AOFF_Carrier_t *crr = (AOFF_Carrier_t*) carrier; + + ASSERT(allctr == ERTS_ALC_CARRIER_TO_ALLCTR(carrier)); + ASSERT(crr->rbt_node.hdr.bhdr == (crr->root ? crr->root->max_sz : 0)); + return crr->rbt_node.hdr.bhdr; +} /* * info_options() @@ -692,6 +958,8 @@ aoff_get_free_block(Allctr_t *allctr, Uint size, static struct { Eterm as; Eterm aoff; + Eterm aoffcaobf; + Eterm aoffcbf; #ifdef DEBUG Eterm end_of_atoms; #endif @@ -720,6 +988,8 @@ init_atoms(void) #endif AM_INIT(as); AM_INIT(aoff); + AM_INIT(aoffcaobf); + AM_INIT(aoffcbf); #ifdef DEBUG for (atom = (Eterm *) &am; atom < &am.end_of_atoms; atom++) { @@ -749,24 +1019,27 @@ info_options(Allctr_t *allctr, Uint **hpp, Uint *szp) { + AOFFAllctr_t* alc = (AOFFAllctr_t*) allctr; Eterm res = THE_NON_VALUE; + const char* flavor_str[3] = {"aoff", "aoffcaobf", "aoffcbf"}; + Eterm flavor_atom[3] = {am.aoff, am.aoffcaobf, am.aoffcbf}; if (print_to_p) { erts_print(*print_to_p, print_to_arg, "%sas: %s\n", prefix, - "aoff"); + flavor_str[alc->flavor]); } if (hpp || szp) { if (!atoms_initialized) - erl_exit(1, "%s:%d: Internal error: Atoms not initialized", + erts_exit(ERTS_ERROR_EXIT, "%s:%d: Internal error: Atoms not initialized", __FILE__, __LINE__);; res = NIL; - add_2tup(hpp, szp, &res, am.as, am.aoff); + add_2tup(hpp, szp, &res, am.as, flavor_atom[alc->flavor]); } return res; @@ -780,19 +1053,28 @@ info_options(Allctr_t *allctr, * to erts_aoffalc_test() * \* */ -unsigned long -erts_aoffalc_test(unsigned long op, unsigned long a1, unsigned long a2) +UWord +erts_aoffalc_test(UWord op, UWord a1, UWord a2) { switch (op) { - case 0x500: return (unsigned long) 0; /* IS_AOBF */ - case 0x501: return (unsigned long) ((AOFFAllctr_t *) a1)->mbc_root; - case 0x502: return (unsigned long) ((AOFF_RBTree_t *) a1)->parent; - case 0x503: return (unsigned long) ((AOFF_RBTree_t *) a1)->left; - case 0x504: return (unsigned long) ((AOFF_RBTree_t *) a1)->right; - case 0x506: return (unsigned long) IS_BLACK((AOFF_RBTree_t *) a1); - case 0x508: return (unsigned long) 1; /* IS_AOFF */ - case 0x509: return (unsigned long) ((AOFF_RBTree_t *) a1)->max_sz; - default: ASSERT(0); return ~((unsigned long) 0); + case 0x500: return (UWord) ((AOFFAllctr_t *) a1)->flavor == AOFF_AOBF; + case 0x501: { + AOFF_RBTree_t *node = ((AOFFAllctr_t *) a1)->mbc_root; + Uint size = (Uint) a2; + node = node ? rbt_search(node, size) : NULL; + return (UWord) (node ? RBT_NODE_TO_MBC(node)->root : NULL); + } + case 0x502: return (UWord) ((AOFF_RBTree_t *) a1)->parent; + case 0x503: return (UWord) ((AOFF_RBTree_t *) a1)->left; + case 0x504: return (UWord) ((AOFF_RBTree_t *) a1)->right; + case 0x505: return (UWord) LIST_NEXT(a1); + case 0x506: return (UWord) IS_BLACK((AOFF_RBTree_t *) a1); + case 0x507: return (UWord) IS_TREE_NODE((AOFF_RBTree_t *) a1); + case 0x508: return (UWord) 0; /* IS_BF_ALGO */ + case 0x509: return (UWord) ((AOFF_RBTree_t *) a1)->max_sz; + case 0x50a: return (UWord) ((AOFFAllctr_t *) a1)->flavor == AOFF_BF; + case 0x50b: return (UWord) LIST_PREV(a1); + default: ASSERT(0); return ~((UWord) 0); } } @@ -804,6 +1086,16 @@ erts_aoffalc_test(unsigned long op, unsigned long a1, unsigned long a2) #ifdef HARD_DEBUG +static int rbt_assert_is_member(AOFF_RBTree_t* root, AOFF_RBTree_t* node) +{ + while (node != root) { + ASSERT(node->parent); + ASSERT(node->parent->left == node || node->parent->right == node); + node = node->parent; + } + return 1; +} + #define IS_LEFT_VISITED(FB) ((FB)->flags & LEFT_VISITED_FLG) #define IS_RIGHT_VISITED(FB) ((FB)->flags & RIGHT_VISITED_FLG) @@ -840,16 +1132,19 @@ static void print_tree(AOFF_RBTree_t*); */ static AOFF_RBTree_t * -check_tree(AOFF_RBTree_t* root, Uint size) +check_tree(Carrier_t* within_crr, enum AOFF_Flavor flavor, AOFF_RBTree_t* root, Uint size) { AOFF_RBTree_t *res = NULL; Sint blacks; Sint curr_blacks; AOFF_RBTree_t *x; + Carrier_t* crr; + Uint depth, max_depth, node_cnt; #ifdef PRINT_TREE print_tree(root); #endif + ASSERT(within_crr || flavor == AOFF_AOFF); if (!root) return res; @@ -859,12 +1154,16 @@ check_tree(AOFF_RBTree_t* root, Uint size) ASSERT(!x->parent); curr_blacks = 1; blacks = -1; + depth = 1; + max_depth = 0; + node_cnt = 0; while (x) { if (!IS_LEFT_VISITED(x)) { SET_LEFT_VISITED(x); if (x->left) { x = x->left; + ++depth; if (IS_BLACK(x)) curr_blacks++; continue; @@ -880,6 +1179,7 @@ check_tree(AOFF_RBTree_t* root, Uint size) SET_RIGHT_VISITED(x); if (x->right) { x = x->right; + ++depth; if (IS_BLACK(x)) curr_blacks++; continue; @@ -891,6 +1191,30 @@ check_tree(AOFF_RBTree_t* root, Uint size) } } + ++node_cnt; + if (depth > max_depth) + max_depth = depth; + + if (within_crr) { + crr = FBLK_TO_MBC(&x->hdr); + ASSERT(crr == within_crr); + ASSERT((char*)x > (char*)crr); + ASSERT(((char*)x + AOFF_BLK_SZ(x)) <= ((char*)crr + CARRIER_SZ(crr))); + + } + if (flavor == AOFF_BF) { + AOFF_RBTree_t* y = x; + AOFF_RBTree_t* nxt = LIST_NEXT(y); + ASSERT(IS_TREE_NODE(x)); + while (nxt) { + ASSERT(IS_LIST_ELEM(nxt)); + ASSERT(AOFF_BLK_SZ(nxt) == AOFF_BLK_SZ(x)); + ASSERT(FBLK_TO_MBC(&nxt->hdr) == within_crr); + ASSERT(LIST_PREV(nxt) == y); + y = nxt; + nxt = LIST_NEXT(nxt); + } + } if (IS_RED(x)) { ASSERT(IS_BLACK(x->right)); @@ -901,13 +1225,13 @@ check_tree(AOFF_RBTree_t* root, Uint size) if (x->left) { ASSERT(x->left->parent == x); - ASSERT(x->left < x); + ASSERT(cmp_blocks(flavor, x->left, x) < 0); ASSERT(x->left->max_sz <= x->max_sz); } if (x->right) { ASSERT(x->right->parent == x); - ASSERT(x->right > x); + ASSERT(cmp_blocks(flavor, x->right, x) > 0); ASSERT(x->right->max_sz <= x->max_sz); } ASSERT(x->max_sz >= AOFF_BLK_SZ(x)); @@ -916,7 +1240,7 @@ check_tree(AOFF_RBTree_t* root, Uint size) || x->max_sz == (x->right ? x->right->max_sz : 0)); if (size && AOFF_BLK_SZ(x) >= size) { - if (!res || x < res) { + if (!res || cmp_blocks(flavor, x, res) < 0) { res = x; } } @@ -926,10 +1250,11 @@ check_tree(AOFF_RBTree_t* root, Uint size) if (IS_BLACK(x)) curr_blacks--; x = x->parent; - + --depth; } - + ASSERT(depth == 0 || (!root && depth==1)); ASSERT(curr_blacks == 0); + ASSERT((1 << (max_depth/2)) <= node_cnt); UNSET_LEFT_VISITED(root); UNSET_RIGHT_VISITED(root); @@ -954,9 +1279,9 @@ print_tree_aux(AOFF_RBTree_t *x, int indent) for (i = 0; i < indent; i++) { putc(' ', stderr); } - fprintf(stderr, "%s: sz=%lu addr=0x%lx max_size=%lu\r\n", + fprintf(stderr, "%s: sz=%lu addr=0x%lx max_size=%u\r\n", IS_BLACK(x) ? "BLACK" : "RED", - AOFF_BLK_SZ(x), (Uint)x, x->max_sz); + AOFF_BLK_SZ(x), (Uint)x, (unsigned)x->max_sz); print_tree_aux(x->left, indent + INDENT_STEP); } } diff --git a/erts/emulator/beam/erl_ao_firstfit_alloc.h b/erts/emulator/beam/erl_ao_firstfit_alloc.h index 6fa626f723..4200f20622 100644 --- a/erts/emulator/beam/erl_ao_firstfit_alloc.h +++ b/erts/emulator/beam/erl_ao_firstfit_alloc.h @@ -1,18 +1,19 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2003-2011. All Rights Reserved. + * Copyright Ericsson AB 2003-2013. All Rights Reserved. * - * The contents of this file are subject to the Erlang Public License, - * Version 1.1, (the "License"); you may not use this file except in - * compliance with the License. You should have received a copy of the - * Erlang Public License along with this software. If not, it can be - * retrieved online at http://www.erlang.org/. - * - * Software distributed under the License is distributed on an "AS IS" - * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See - * the License for the specific language governing rights and limitations - * under the License. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. * * %CopyrightEnd% */ @@ -27,8 +28,14 @@ typedef struct AOFFAllctr_t_ AOFFAllctr_t; +enum AOFF_Flavor { + AOFF_AOFF = 0, + AOFF_AOBF = 1, + AOFF_BF = 2 +}; + typedef struct { - int dummy; + enum AOFF_Flavor flavor; } AOFFAllctrInit_t; #define ERTS_DEFAULT_AOFF_ALLCTR_INIT {0/*dummy*/} @@ -51,10 +58,10 @@ struct AOFFAllctr_t_ { Allctr_t allctr; /* Has to be first! */ struct AOFF_RBTree_t_* mbc_root; - struct AOFF_RBTree_t_* sbmbc_root; + enum AOFF_Flavor flavor; }; -unsigned long erts_aoffalc_test(unsigned long, unsigned long, unsigned long); +UWord erts_aoffalc_test(UWord, UWord, UWord); #endif /* #if defined(GET_ERL_AOFF_ALLOC_IMPL) && !defined(ERL_AOFF_ALLOC_IMPL__) */ diff --git a/erts/emulator/beam/erl_arith.c b/erts/emulator/beam/erl_arith.c index 5150a8a507..b8c5ef9b09 100644 --- a/erts/emulator/beam/erl_arith.c +++ b/erts/emulator/beam/erl_arith.c @@ -3,16 +3,17 @@ * * Copyright Ericsson AB 1999-2010. All Rights Reserved. * - * The contents of this file are subject to the Erlang Public License, - * Version 1.1, (the "License"); you may not use this file except in - * compliance with the License. You should have received a copy of the - * Erlang Public License along with this software. If not, it can be - * retrieved online at http://www.erlang.org/. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Software distributed under the License is distributed on an "AS IS" - * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See - * the License for the specific language governing rights and limitations - * under the License. + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. * * %CopyrightEnd% */ @@ -2048,3 +2049,8 @@ Eterm erts_gc_bnot(Process* p, Eterm* reg, Uint live) } return result; } + +/* Needed to remove compiler optimization */ +double erts_get_positive_zero_float() { + return 0.0f; +} diff --git a/erts/emulator/beam/erl_async.c b/erts/emulator/beam/erl_async.c index 054d1a48f6..f071898046 100644 --- a/erts/emulator/beam/erl_async.c +++ b/erts/emulator/beam/erl_async.c @@ -3,16 +3,17 @@ * * Copyright Ericsson AB 2000-2013. All Rights Reserved. * - * The contents of this file are subject to the Erlang Public License, - * Version 1.1, (the "License"); you may not use this file except in - * compliance with the License. You should have received a copy of the - * Erlang Public License along with this software. If not, it can be - * retrieved online at http://www.erlang.org/. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Software distributed under the License is distributed on an "AS IS" - * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See - * the License for the specific language governing rights and limitations - * under the License. + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. * * %CopyrightEnd% */ @@ -166,6 +167,7 @@ async_ready_q(Uint sched_id) #endif + void erts_init_async(void) { @@ -175,7 +177,7 @@ erts_init_async(void) ErtsThrQInit_t qinit = ERTS_THR_Q_INIT_DEFAULT; #endif erts_thr_opts_t thr_opts = ERTS_THR_OPTS_DEFAULT_INITER; - char *ptr; + char *ptr, thr_name[16]; size_t tot_size = 0; int i; @@ -226,8 +228,13 @@ erts_init_async(void) thr_opts.suggested_stack_size = erts_async_thread_suggested_stack_size; + thr_opts.name = thr_name; + for (i = 0; i < erts_async_max_threads; i++) { ErtsAsyncQ *aq = async_q(i); + + erts_snprintf(thr_opts.name, 16, "async_%d", i+1); + erts_thr_create(&aq->thr_id, async_main, (void*) aq, &thr_opts); } @@ -279,7 +286,8 @@ static ERTS_INLINE void async_add(ErtsAsync *a, ErtsAsyncQ* q) if (DTRACE_ENABLED(aio_pool_add)) { DTRACE_CHARBUF(port_str, 16); - erts_snprintf(port_str, sizeof(port_str), "%T", a->port); + erts_snprintf(port_str, sizeof(DTRACE_CHARBUF_NAME(port_str)), + "%T", a->port); /* DTRACE TODO: Get the queue length from erts_thr_q_enqueue() ? */ len = -1; DTRACE2(aio_pool_add, port_str, len); @@ -314,7 +322,8 @@ static ERTS_INLINE ErtsAsync *async_get(ErtsThrQ_t *q, if (DTRACE_ENABLED(aio_pool_get)) { DTRACE_CHARBUF(port_str, 16); - erts_snprintf(port_str, sizeof(port_str), "%T", a->port); + erts_snprintf(port_str, sizeof(DTRACE_CHARBUF_NAME(port_str)), + "%T", a->port); /* DTRACE TODO: Get the length from erts_thr_q_dequeue() ? */ len = -1; DTRACE2(aio_pool_get, port_str, len); @@ -583,12 +592,26 @@ int erts_async_ready_clean(void *varq, void *val) #endif /* +** Generate a fair async key prom an ErlDrvPort +** The port data gives a fair distribution grom port pointer +** to unsigned integer - to be used in key for driver_async below. +*/ +unsigned int driver_async_port_key(ErlDrvPort port) +{ + ErlDrvTermData td = driver_mk_port(port); + if (td == (ErlDrvTermData) NIL) { + return 0; + } + return (unsigned int) (UWord) internal_port_data(td); +} + +/* ** Schedule async_invoke on a worker thread ** NOTE will be syncrounous when threads are unsupported ** return values: ** 0 completed ** -1 error -** N handle value (used with async_cancel) +** N handle value ** arguments: ** ix driver index ** key pointer to secedule queue (NULL means round robin) @@ -673,23 +696,3 @@ long driver_async(ErlDrvPort ix, unsigned int* key, return id; } - -int driver_async_cancel(unsigned int id) -{ - /* - * Not supported anymore. Always fail (which is backward - * compatible). - * - * This functionality could be implemented again. However, - * it is (and always has been) completely useless since - * it doesn't give you any guarantees whatsoever. The user - * needs to (and always have had to) synchronize in his/her - * own code in order to get any guarantees. - */ - return 0; -} - - - - - diff --git a/erts/emulator/beam/erl_async.h b/erts/emulator/beam/erl_async.h index 95374a8fc9..65538bcef0 100644 --- a/erts/emulator/beam/erl_async.h +++ b/erts/emulator/beam/erl_async.h @@ -3,16 +3,17 @@ * * Copyright Ericsson AB 2011. All Rights Reserved. * - * The contents of this file are subject to the Erlang Public License, - * Version 1.1, (the "License"); you may not use this file except in - * compliance with the License. You should have received a copy of the - * Erlang Public License along with this software. If not, it can be - * retrieved online at http://www.erlang.org/. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Software distributed under the License is distributed on an "AS IS" - * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See - * the License for the specific language governing rights and limitations - * under the License. + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. * * %CopyrightEnd% */ diff --git a/erts/emulator/beam/erl_bestfit_alloc.c b/erts/emulator/beam/erl_bestfit_alloc.c index 5625622e3f..f39a18ac88 100644 --- a/erts/emulator/beam/erl_bestfit_alloc.c +++ b/erts/emulator/beam/erl_bestfit_alloc.c @@ -3,16 +3,17 @@ * * Copyright Ericsson AB 2003-2013. All Rights Reserved. * - * The contents of this file are subject to the Erlang Public License, - * Version 1.1, (the "License"); you may not use this file except in - * compliance with the License. You should have received a copy of the - * Erlang Public License along with this software. If not, it can be - * retrieved online at http://www.erlang.org/. - * - * Software distributed under the License is distributed on an "AS IS" - * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See - * the License for the specific language governing rights and limitations - * under the License. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. * * %CopyrightEnd% */ @@ -75,9 +76,6 @@ #define BF_BLK_SZ(B) MBC_FBLK_SZ(&(B)->hdr) -#undef ASSERT -#define ASSERT ASSERT_EXPR - #if 1 #define RBT_ASSERT ASSERT #else @@ -89,21 +87,21 @@ static RBTree_t * check_tree(RBTree_t, int, Uint); #endif -static void tree_delete(Allctr_t *allctr, Block_t *del, Uint32 flags); +static void tree_delete(Allctr_t *allctr, Block_t *del); /* Prototypes of callback functions */ /* "address order best fit" specific callback functions */ static Block_t * aobf_get_free_block (Allctr_t *, Uint, - Block_t *, Uint, Uint32); -static void aobf_link_free_block (Allctr_t *, Block_t *, Uint32); + Block_t *, Uint); +static void aobf_link_free_block (Allctr_t *, Block_t *); #define aobf_unlink_free_block tree_delete /* "best fit" specific callback functions */ static Block_t * bf_get_free_block (Allctr_t *, Uint, - Block_t *, Uint, Uint32); -static void bf_link_free_block (Allctr_t *, Block_t *, Uint32); -static ERTS_INLINE void bf_unlink_free_block (Allctr_t *, Block_t *, Uint32); + Block_t *, Uint); +static void bf_link_free_block (Allctr_t *, Block_t *); +static ERTS_INLINE void bf_unlink_free_block (Allctr_t *, Block_t *); static Eterm info_options (Allctr_t *, char *, int *, @@ -179,6 +177,7 @@ erts_bfalc_start(BFAllctr_t *bfallctr, bfallctr->address_order = bfinit->ao; + allctr->mbc_header_size = sizeof(Carrier_t); allctr->min_mbc_size = MIN_MBC_SZ; allctr->min_mbc_first_free_size = MIN_MBC_FIRST_FREE_SZ; allctr->min_block_size = (bfinit->ao @@ -207,6 +206,9 @@ erts_bfalc_start(BFAllctr_t *bfallctr, allctr->get_next_mbc_size = NULL; allctr->creating_mbc = NULL; allctr->destroying_mbc = NULL; + allctr->add_mbc = NULL; + allctr->remove_mbc = NULL; + allctr->largest_fblk_in_mbc = NULL; allctr->init_atoms = init_atoms; #ifdef ERTS_ALLOC_UTIL_HARD_DEBUG @@ -407,13 +409,11 @@ tree_insert_fixup(RBTree_t **root, RBTree_t *blk) * callback function in the address order case. */ static void -tree_delete(Allctr_t *allctr, Block_t *del, Uint32 flags) +tree_delete(Allctr_t *allctr, Block_t *del) { BFAllctr_t *bfallctr = (BFAllctr_t *) allctr; Uint spliced_is_black; - RBTree_t **root = ((flags & ERTS_ALCU_FLG_SBMBC) - ? &bfallctr->sbmbc_root - : &bfallctr->mbc_root); + RBTree_t **root = &bfallctr->mbc_root; RBTree_t *x, *y, *z = (RBTree_t *) del; RBTree_t null_x; /* null_x is used to get the fixup started when we splice out a node without children. */ @@ -586,12 +586,10 @@ tree_delete(Allctr_t *allctr, Block_t *del, Uint32 flags) \* */ static void -aobf_link_free_block(Allctr_t *allctr, Block_t *block, Uint32 flags) +aobf_link_free_block(Allctr_t *allctr, Block_t *block) { BFAllctr_t *bfallctr = (BFAllctr_t *) allctr; - RBTree_t **root = ((flags & ERTS_ALCU_FLG_SBMBC) - ? &bfallctr->sbmbc_root - : &bfallctr->mbc_root); + RBTree_t **root = &bfallctr->mbc_root; RBTree_t *blk = (RBTree_t *) block; Uint blk_sz = BF_BLK_SZ(blk); @@ -647,21 +645,18 @@ aobf_link_free_block(Allctr_t *allctr, Block_t *block, Uint32 flags) #if 0 /* tree_delete() is directly used instead */ static void -aobf_unlink_free_block(Allctr_t *allctr, Block_t *block, Uint32 flags) +aobf_unlink_free_block(Allctr_t *allctr, Block_t *block) { - tree_delete(allctr, block, flags); + tree_delete(allctr, block); } #endif static Block_t * aobf_get_free_block(Allctr_t *allctr, Uint size, - Block_t *cand_blk, Uint cand_size, - Uint32 flags) + Block_t *cand_blk, Uint cand_size) { BFAllctr_t *bfallctr = (BFAllctr_t *) allctr; - RBTree_t **root = ((flags & ERTS_ALCU_FLG_SBMBC) - ? &bfallctr->sbmbc_root - : &bfallctr->mbc_root); + RBTree_t **root = &bfallctr->mbc_root; RBTree_t *x = *root; RBTree_t *blk = NULL; Uint blk_sz; @@ -694,7 +689,7 @@ aobf_get_free_block(Allctr_t *allctr, Uint size, return NULL; /* cand_blk was better */ } - aobf_unlink_free_block(allctr, (Block_t *) blk, flags); + aobf_unlink_free_block(allctr, (Block_t *) blk); return (Block_t *) blk; } @@ -705,12 +700,10 @@ aobf_get_free_block(Allctr_t *allctr, Uint size, \* */ static void -bf_link_free_block(Allctr_t *allctr, Block_t *block, Uint32 flags) +bf_link_free_block(Allctr_t *allctr, Block_t *block) { BFAllctr_t *bfallctr = (BFAllctr_t *) allctr; - RBTree_t **root = ((flags & ERTS_ALCU_FLG_SBMBC) - ? &bfallctr->sbmbc_root - : &bfallctr->mbc_root); + RBTree_t **root = &bfallctr->mbc_root; RBTree_t *blk = (RBTree_t *) block; Uint blk_sz = BF_BLK_SZ(blk); @@ -779,12 +772,10 @@ bf_link_free_block(Allctr_t *allctr, Block_t *block, Uint32 flags) } static ERTS_INLINE void -bf_unlink_free_block(Allctr_t *allctr, Block_t *block, Uint32 flags) +bf_unlink_free_block(Allctr_t *allctr, Block_t *block) { BFAllctr_t *bfallctr = (BFAllctr_t *) allctr; - RBTree_t **root = ((flags & ERTS_ALCU_FLG_SBMBC) - ? &bfallctr->sbmbc_root - : &bfallctr->mbc_root); + RBTree_t **root = &bfallctr->mbc_root; RBTree_t *x = (RBTree_t *) block; if (IS_LIST_ELEM(x)) { @@ -812,7 +803,7 @@ bf_unlink_free_block(Allctr_t *allctr, Block_t *block, Uint32 flags) } else { /* Remove from tree */ - tree_delete(allctr, block, flags); + tree_delete(allctr, block); } DESTROY_LIST_ELEM(x); @@ -821,13 +812,10 @@ bf_unlink_free_block(Allctr_t *allctr, Block_t *block, Uint32 flags) static Block_t * bf_get_free_block(Allctr_t *allctr, Uint size, - Block_t *cand_blk, Uint cand_size, - Uint32 flags) + Block_t *cand_blk, Uint cand_size) { BFAllctr_t *bfallctr = (BFAllctr_t *) allctr; - RBTree_t **root = ((flags & ERTS_ALCU_FLG_SBMBC) - ? &bfallctr->sbmbc_root - : &bfallctr->mbc_root); + RBTree_t **root = &bfallctr->mbc_root; RBTree_t *x = *root; RBTree_t *blk = NULL; Uint blk_sz; @@ -867,7 +855,7 @@ bf_get_free_block(Allctr_t *allctr, Uint size, the tree node */ blk = LIST_NEXT(blk) ? LIST_NEXT(blk) : blk; - bf_unlink_free_block(allctr, (Block_t *) blk, flags); + bf_unlink_free_block(allctr, (Block_t *) blk); return (Block_t *) blk; } @@ -952,7 +940,7 @@ info_options(Allctr_t *allctr, if (hpp || szp) { if (!atoms_initialized) - erl_exit(1, "%s:%d: Internal error: Atoms not initialized", + erts_exit(ERTS_ERROR_EXIT, "%s:%d: Internal error: Atoms not initialized", __FILE__, __LINE__);; res = NIL; @@ -972,20 +960,22 @@ info_options(Allctr_t *allctr, * to erts_bfalc_test() * \* */ -unsigned long -erts_bfalc_test(unsigned long op, unsigned long a1, unsigned long a2) +UWord +erts_bfalc_test(UWord op, UWord a1, UWord a2) { switch (op) { - case 0x200: return (unsigned long) ((BFAllctr_t *) a1)->address_order; - case 0x201: return (unsigned long) ((BFAllctr_t *) a1)->mbc_root; - case 0x202: return (unsigned long) ((RBTree_t *) a1)->parent; - case 0x203: return (unsigned long) ((RBTree_t *) a1)->left; - case 0x204: return (unsigned long) ((RBTree_t *) a1)->right; - case 0x205: return (unsigned long) ((RBTreeList_t *) a1)->next; - case 0x206: return (unsigned long) IS_BLACK((RBTree_t *) a1); - case 0x207: return (unsigned long) IS_TREE_NODE((RBTree_t *) a1); - case 0x208: return (unsigned long) 0; /* IS_AOFF */ - default: ASSERT(0); return ~((unsigned long) 0); + case 0x200: return (UWord) ((BFAllctr_t *) a1)->address_order; /* IS_AOBF */ + case 0x201: return (UWord) ((BFAllctr_t *) a1)->mbc_root; + case 0x202: return (UWord) ((RBTree_t *) a1)->parent; + case 0x203: return (UWord) ((RBTree_t *) a1)->left; + case 0x204: return (UWord) ((RBTree_t *) a1)->right; + case 0x205: return (UWord) LIST_NEXT(a1); + case 0x206: return (UWord) IS_BLACK((RBTree_t *) a1); + case 0x207: return (UWord) IS_TREE_NODE((RBTree_t *) a1); + case 0x208: return (UWord) 1; /* IS_BF_ALGO */ + case 0x20a: return (UWord) !((BFAllctr_t *) a1)->address_order; /* IS_BF */ + case 0x20b: return (UWord) LIST_PREV(a1); + default: ASSERT(0); return ~((UWord) 0); } } diff --git a/erts/emulator/beam/erl_bestfit_alloc.h b/erts/emulator/beam/erl_bestfit_alloc.h index 0c29662852..b315518b88 100644 --- a/erts/emulator/beam/erl_bestfit_alloc.h +++ b/erts/emulator/beam/erl_bestfit_alloc.h @@ -1,18 +1,19 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2003-2011. All Rights Reserved. + * Copyright Ericsson AB 2003-2013. All Rights Reserved. * - * The contents of this file are subject to the Erlang Public License, - * Version 1.1, (the "License"); you may not use this file except in - * compliance with the License. You should have received a copy of the - * Erlang Public License along with this software. If not, it can be - * retrieved online at http://www.erlang.org/. - * - * Software distributed under the License is distributed on an "AS IS" - * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See - * the License for the specific language governing rights and limitations - * under the License. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. * * %CopyrightEnd% */ @@ -55,11 +56,10 @@ struct BFAllctr_t_ { Allctr_t allctr; /* Has to be first! */ RBTree_t * mbc_root; - RBTree_t * sbmbc_root; int address_order; }; -unsigned long erts_bfalc_test(unsigned long, unsigned long, unsigned long); +UWord erts_bfalc_test(UWord, UWord, UWord); #endif /* #if defined(GET_ERL_BF_ALLOC_IMPL) && !defined(ERL_BF_ALLOC_IMPL__) */ diff --git a/erts/emulator/beam/erl_bif_binary.c b/erts/emulator/beam/erl_bif_binary.c index 0db19a1ee6..134aa2d396 100644 --- a/erts/emulator/beam/erl_bif_binary.c +++ b/erts/emulator/beam/erl_bif_binary.c @@ -3,16 +3,17 @@ * * Copyright Ericsson AB 2010-2013. All Rights Reserved. * - * The contents of this file are subject to the Erlang Public License, - * Version 1.1, (the "License"); you may not use this file except in - * compliance with the License. You should have received a copy of the - * Erlang Public License along with this software. If not, it can be - * retrieved online at http://www.erlang.org/. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Software distributed under the License is distributed on an "AS IS" - * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See - * the License for the specific language governing rights and limitations - * under the License. + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. * * %CopyrightEnd% */ @@ -32,10 +33,13 @@ #include "global.h" #include "erl_process.h" #include "error.h" +#define ERL_WANT_HIPE_BIF_WRAPPER__ #include "bif.h" +#undef ERL_WANT_HIPE_BIF_WRAPPER__ #include "big.h" #include "erl_binary.h" #include "erl_bits.h" +#include "erl_bif_unique.h" /* @@ -927,6 +931,9 @@ static int do_binary_match_compile(Eterm argument, Eterm *tag, Binary **binp) if (binary_bitsize(b) != 0) { goto badarg; } + if (binary_size(b) == 0) { + goto badarg; + } ++words; characters += binary_size(b); } @@ -1321,9 +1328,9 @@ static int parse_match_opts_list(Eterm l, Eterm bin, Uint *posp, Uint *endp) goto badarg; } if (len < 0) { - Sint lentmp = -len; + Uint lentmp = -(Uint)len; /* overflow */ - if (lentmp == len || lentmp < 0 || -lentmp != len) { + if ((Sint)lentmp < 0) { goto badarg; } len = lentmp; @@ -1552,9 +1559,9 @@ BIF_RETTYPE erts_binary_part(Process *p, Eterm binary, Eterm epos, Eterm elen) goto badarg; } if (len < 0) { - Sint lentmp = -len; + Uint lentmp = -(Uint)len; /* overflow */ - if (lentmp == len || lentmp < 0 || -lentmp != len) { + if ((Sint)lentmp < 0) { goto badarg; } len = lentmp; @@ -1641,9 +1648,9 @@ BIF_RETTYPE erts_gc_binary_part(Process *p, Eterm *reg, Eterm live, int range_is goto badarg; } if (len < 0) { - Sint lentmp = -len; + Uint lentmp = -(Uint)len; /* overflow */ - if (lentmp == len || lentmp < 0 || -lentmp != len) { + if ((Sint)lentmp < 0) { goto badarg; } len = lentmp; @@ -2210,9 +2217,9 @@ static BIF_RETTYPE binary_bin_to_list_common(Process *p, goto badarg; } if (len < 0) { - Sint lentmp = -len; + Uint lentmp = -(Uint)len; /* overflow */ - if (lentmp == len || lentmp < 0 || -lentmp != len) { + if ((Sint)lentmp < 0) { goto badarg; } len = lentmp; @@ -2291,18 +2298,11 @@ BIF_RETTYPE binary_bin_to_list_1(BIF_ALIST_1) BIF_ERROR(BIF_P,BADARG); } -/* - * Ok, erlang:list_to_binary does not interrupt, and we really don't want - * an alternative implementation for the exact same thing, why we - * have descided to use the old non-restarting implementation for now. - * In reality, there are seldom many iterations involved in doing this, so the - * problem of long-running bifs is not really that big in this case. - * So, for now we use the old implementation also in the module binary. - */ +HIPE_WRAPPER_BIF_DISABLE_GC(binary_list_to_bin, 1) BIF_RETTYPE binary_list_to_bin_1(BIF_ALIST_1) { - return erts_list_to_binary_bif(BIF_P, BIF_ARG_1); + return erts_list_to_binary_bif(BIF_P, BIF_ARG_1, bif_export[BIF_binary_list_to_bin_1]); } typedef struct { @@ -2428,8 +2428,6 @@ static BIF_RETTYPE do_binary_copy(Process *p, Eterm bin, Eterm en) } cbs->result = erts_bin_nrml_alloc(target_size); /* Always offheap if trapping */ - cbs->result->flags = 0; - cbs->result->orig_size = target_size; erts_refc_init(&(cbs->result->refc), 1); t = (byte *) cbs->result->orig_bytes; /* No offset or anything */ pos = 0; diff --git a/erts/emulator/beam/erl_bif_chksum.c b/erts/emulator/beam/erl_bif_chksum.c index 4302fe8f79..e3074d6309 100644 --- a/erts/emulator/beam/erl_bif_chksum.c +++ b/erts/emulator/beam/erl_bif_chksum.c @@ -3,16 +3,17 @@ * * Copyright Ericsson AB 2008-2013. All Rights Reserved. * - * The contents of this file are subject to the Erlang Public License, - * Version 1.1, (the "License"); you may not use this file except in - * compliance with the License. You should have received a copy of the - * Erlang Public License along with this software. If not, it can be - * retrieved online at http://www.erlang.org/. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Software distributed under the License is distributed on an "AS IS" - * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See - * the License for the specific language governing rights and limitations - * under the License. + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. * * %CopyrightEnd% */ diff --git a/erts/emulator/beam/erl_bif_ddll.c b/erts/emulator/beam/erl_bif_ddll.c index 1c3e955f47..f4ef59993a 100644 --- a/erts/emulator/beam/erl_bif_ddll.c +++ b/erts/emulator/beam/erl_bif_ddll.c @@ -3,16 +3,17 @@ * * Copyright Ericsson AB 2006-2013. All Rights Reserved. * - * The contents of this file are subject to the Erlang Public License, - * Version 1.1, (the "License"); you may not use this file except in - * compliance with the License. You should have received a copy of the - * Erlang Public License along with this software. If not, it can be - * retrieved online at http://www.erlang.org/. - * - * Software distributed under the License is distributed on an "AS IS" - * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See - * the License for the specific language governing rights and limitations - * under the License. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. * * %CopyrightEnd% */ @@ -45,6 +46,7 @@ #include "big.h" #include "dist.h" #include "erl_version.h" +#include "erl_bif_unique.h" #include "dtrace-wrapper.h" #ifdef ERTS_SMP @@ -182,7 +184,7 @@ BIF_RETTYPE erl_ddll_try_load_3(BIF_ALIST_3) Eterm name_term = BIF_ARG_2; Eterm options = BIF_ARG_3; char *path = NULL; - ErlDrvSizeT path_len; + Sint path_len; char *name = NULL; DE_Handle *dh; erts_driver_t *drv; @@ -198,6 +200,7 @@ BIF_RETTYPE erl_ddll_try_load_3(BIF_ALIST_3) int kill_ports = 0; int do_build_load_error = 0; int build_this_load_error = 0; + int encoding; for(l = options; is_list(l); l = CDR(list_val(l))) { Eterm opt = CAR(list_val(l)); @@ -257,18 +260,23 @@ BIF_RETTYPE erl_ddll_try_load_3(BIF_ALIST_3) goto error; } - if (erts_iolist_size(path_term, &path_len)) { - goto error; + encoding = erts_get_native_filename_encoding(); + if (encoding == ERL_FILENAME_WIN_WCHAR) { + /* Do not convert the lib name to utf-16le yet, do that in win32 specific code */ + /* since lib_name is used in error messages */ + encoding = ERL_FILENAME_UTF8; } - path = erts_alloc(ERTS_ALC_T_DDLL_TMP_BUF, path_len + 1 /* might need path separator */ + sys_strlen(name) + 1); - if (erts_iolist_to_buf(path_term, path, path_len) != 0) { + path = erts_convert_filename_to_encoding(path_term, NULL, 0, + ERTS_ALC_T_DDLL_TMP_BUF, 1, 0, + encoding, &path_len, + sys_strlen(name) + 2); /* might need path separator */ + if (!path) { goto error; } - while (path_len > 0 && (path[path_len-1] == '\\' || path[path_len-1] == '/')) { - --path_len; - } + ASSERT(path_len > 0 && path[path_len-1] == 0); + while (--path_len > 0 && (path[path_len-1] == '\\' || path[path_len-1] == '/')) + ; path[path_len++] = '/'; - /*path[path_len] = '\0';*/ sys_strcpy(path+path_len,name); #if DDLL_SMP @@ -1301,7 +1309,7 @@ static Eterm notify_when_loaded(Process *p, Eterm name_term, char *name, ErtsPro case ERL_DE_FORCE_RELOAD: break; default: - erl_exit(1,"Internal error, unknown state %u in dynamic driver.", drv->handle->status); + erts_exit(ERTS_ERROR_EXIT,"Internal error, unknown state %u in dynamic driver.", drv->handle->status); } p->flags |= F_USING_DDLL; r = add_monitor(p, drv->handle, ERL_DE_PROC_AWAIT_LOAD); @@ -1524,7 +1532,7 @@ static int do_load_driver_entry(DE_Handle *dh, char *path, char *name) assert_drv_list_rwlocked(); - if ((res = erts_sys_ddll_open(path, &(dh->handle))) != ERL_DE_NO_ERROR) { + if ((res = erts_sys_ddll_open(path, &(dh->handle), NULL)) != ERL_DE_NO_ERROR) { return res; } @@ -1542,8 +1550,10 @@ static int do_load_driver_entry(DE_Handle *dh, char *path, char *name) switch (dp->extended_marker) { case ERL_DRV_EXTENDED_MARKER: - if (ERL_DRV_EXTENDED_MAJOR_VERSION != dp->major_version - || ERL_DRV_EXTENDED_MINOR_VERSION < dp->minor_version) { + if (dp->major_version < ERL_DRV_MIN_REQUIRED_MAJOR_VERSION_ON_LOAD + || (ERL_DRV_EXTENDED_MAJOR_VERSION < dp->major_version + || (ERL_DRV_EXTENDED_MAJOR_VERSION == dp->major_version + && ERL_DRV_EXTENDED_MINOR_VERSION < dp->minor_version))) { /* Incompatible driver version */ res = ERL_DE_LOAD_ERROR_INCORRECT_VERSION; goto error; @@ -1722,11 +1732,7 @@ static void notify_proc(Process *proc, Eterm ref, Eterm driver_name, Eterm type, hp += REF_THING_SIZE; mess = TUPLE5(hp,type,r,am_driver,driver_name,tag); } - erts_queue_message(proc, &rp_locks, bp, mess, am_undefined -#ifdef USE_VM_PROBES - , NIL -#endif - ); + erts_queue_message(proc, &rp_locks, bp, mess, am_undefined); erts_smp_proc_unlock(proc, rp_locks); ERTS_SMP_CHK_NO_PROC_LOCKS; } diff --git a/erts/emulator/beam/erl_bif_guard.c b/erts/emulator/beam/erl_bif_guard.c index a715756c15..4a9a6a5fcd 100644 --- a/erts/emulator/beam/erl_bif_guard.c +++ b/erts/emulator/beam/erl_bif_guard.c @@ -3,16 +3,17 @@ * * Copyright Ericsson AB 2006-2011. All Rights Reserved. * - * The contents of this file are subject to the Erlang Public License, - * Version 1.1, (the "License"); you may not use this file except in - * compliance with the License. You should have received a copy of the - * Erlang Public License along with this software. If not, it can be - * retrieved online at http://www.erlang.org/. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Software distributed under the License is distributed on an "AS IS" - * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See - * the License for the specific language governing rights and limitations - * under the License. + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. * * %CopyrightEnd% */ @@ -33,6 +34,7 @@ #include "bif.h" #include "big.h" #include "erl_binary.h" +#include "erl_map.h" static Eterm gc_double_to_integer(Process* p, double x, Eterm* reg, Uint live); @@ -455,6 +457,30 @@ Eterm erts_gc_byte_size_1(Process* p, Eterm* reg, Uint live) } } +Eterm erts_gc_map_size_1(Process* p, Eterm* reg, Uint live) +{ + Eterm arg = reg[live]; + if (is_flatmap(arg)) { + flatmap_t *mp = (flatmap_t*)flatmap_val(arg); + return make_small(flatmap_get_size(mp)); + } else if (is_hashmap(arg)) { + Eterm* hp; + Uint size; + size = hashmap_size(arg); + if (IS_USMALL(0, size)) { + return make_small(size); + } + if (ERTS_NEED_GC(p, BIG_UINT_HEAP_SIZE)) { + erts_garbage_collect(p, BIG_UINT_HEAP_SIZE, reg, live); + } + hp = p->htop; + p->htop += BIG_UINT_HEAP_SIZE; + return uint_to_big(size, hp); + } + p->fvalue = arg; + BIF_ERROR(p, BADMAP); +} + Eterm erts_gc_abs_1(Process* p, Eterm* reg, Uint live) { Eterm arg; diff --git a/erts/emulator/beam/erl_bif_info.c b/erts/emulator/beam/erl_bif_info.c index 8582a8954b..ac7a70c642 100755..100644 --- a/erts/emulator/beam/erl_bif_info.c +++ b/erts/emulator/beam/erl_bif_info.c @@ -3,16 +3,17 @@ * * Copyright Ericsson AB 1999-2013. All Rights Reserved. * - * The contents of this file are subject to the Erlang Public License, - * Version 1.1, (the "License"); you may not use this file except in - * compliance with the License. You should have received a copy of the - * Erlang Public License along with this software. If not, it can be - * retrieved online at http://www.erlang.org/. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Software distributed under the License is distributed on an "AS IS" - * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See - * the License for the specific language governing rights and limitations - * under the License. + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. * * %CopyrightEnd% */ @@ -27,9 +28,11 @@ #include "erl_process.h" #include "error.h" #include "erl_driver.h" +#include "erl_nif.h" #include "bif.h" #include "big.h" #include "erl_version.h" +#include "erl_compile_flags.h" #include "erl_db_util.h" #include "erl_message.h" #include "erl_binary.h" @@ -40,6 +43,7 @@ #include "erl_cpu_topology.h" #include "erl_async.h" #include "erl_thr_progress.h" +#include "erl_bif_unique.h" #define ERTS_PTAB_WANT_DEBUG_FUNCS__ #include "erl_ptab.h" #ifdef HIPE @@ -57,14 +61,19 @@ static Export* alloc_info_trap = NULL; static Export* alloc_sizes_trap = NULL; +static Export* gather_io_bytes_trap = NULL; static Export *gather_sched_wall_time_res_trap; +static Export *gather_gc_info_res_trap; +static Export *gather_system_check_res_trap; #define DECL_AM(S) Eterm AM_ ## S = am_atom_put(#S, sizeof(#S) - 1) +static char otp_version[] = ERLANG_OTP_VERSION; /* Keep erts_system_version as a global variable for easy access from a core */ -static char erts_system_version[] = ("Erlang " ERLANG_OTP_RELEASE - " (erts-" ERLANG_VERSION ")" +static char erts_system_version[] = ("Erlang/OTP " ERLANG_OTP_RELEASE + "%s" + " [erts-" ERLANG_VERSION "]" #if !HEAP_ON_C_STACK && !HALFWORD_HEAP " [no-c-stack-objects]" #endif @@ -86,6 +95,9 @@ static char erts_system_version[] = ("Erlang " ERLANG_OTP_RELEASE " [smp:%beu:%beu]" #endif #ifdef USE_THREADS +#if defined(ERTS_DIRTY_SCHEDULERS) && defined(ERTS_SMP) + " [ds:%beu:%beu:%beu]" +#endif " [async-threads:%d]" #endif #ifdef HIPE @@ -108,12 +120,18 @@ static char erts_system_version[] = ("Erlang " ERLANG_OTP_RELEASE #ifdef ERTS_ENABLE_LOCK_COUNT " [lock-counting]" #endif +#ifdef ERTS_OPCODE_COUNTER_SUPPORT + " [instruction-counting]" +#endif #ifdef PURIFY " [purify-compiled]" #endif #ifdef VALGRIND " [valgrind-compiled]" #endif +#ifdef ERTS_FRMPTR + " [frame-pointer]" +#endif #ifdef USE_DTRACE " [dtrace]" #endif @@ -299,13 +317,39 @@ make_link_list(Process *p, ErtsLink *root, Eterm tail) int erts_print_system_version(int to, void *arg, Process *c_p) { + int i, rc = -1; + char *rc_str = ""; + char rc_buf[100]; + char *ov = otp_version; #ifdef ERTS_SMP Uint total, online, active; - (void) erts_schedulers_state(&total, &online, &active, 0); +#ifdef ERTS_DIRTY_SCHEDULERS + Uint dirty_cpu, dirty_cpu_onln, dirty_io; + + (void) erts_schedulers_state(&total, &online, &active, &dirty_cpu, &dirty_cpu_onln, &dirty_io, 0); +#else + (void) erts_schedulers_state(&total, &online, &active, NULL, NULL, NULL, 0); +#endif #endif - return erts_print(to, arg, erts_system_version + for (i = 0; i < sizeof(otp_version)-4; i++) { + if (ov[i] == '-' && ov[i+1] == 'r' && ov[i+2] == 'c') + rc = atoi(&ov[i+3]); + } + if (rc >= 0) { + if (rc == 0) + rc_str = " [DEVELOPMENT]"; + else { + erts_snprintf(rc_buf, sizeof(rc_buf), " [RELEASE CANDIDATE %d]", rc); + rc_str = rc_buf; + } + } + return erts_print(to, arg, erts_system_version, + rc_str #ifdef ERTS_SMP , total, online +#ifdef ERTS_DIRTY_SCHEDULERS + , dirty_cpu, dirty_cpu_onln, dirty_io +#endif #endif #ifdef USE_THREADS , erts_async_max_threads @@ -486,27 +530,6 @@ collect_one_suspend_monitor(ErtsSuspendMonitor *smon, void *vsmicp) } } - -static void one_link_size(ErtsLink *lnk, void *vpu) -{ - Uint *pu = vpu; - *pu += ERTS_LINK_SIZE*sizeof(Uint); - if(!IS_CONST(lnk->pid)) - *pu += NC_HEAP_SIZE(lnk->pid)*sizeof(Uint); - if (lnk->type != LINK_NODE && ERTS_LINK_ROOT(lnk) != NULL) { - erts_doforall_links(ERTS_LINK_ROOT(lnk),&one_link_size,vpu); - } -} -static void one_mon_size(ErtsMonitor *mon, void *vpu) -{ - Uint *pu = vpu; - *pu += ERTS_MONITOR_SIZE*sizeof(Uint); - if(!IS_CONST(mon->pid)) - *pu += NC_HEAP_SIZE(mon->pid)*sizeof(Uint); - if(!IS_CONST(mon->ref)) - *pu += NC_HEAP_SIZE(mon->ref)*sizeof(Uint); -} - /* * process_info/[1,2] */ @@ -521,6 +544,7 @@ pi_locks(Eterm info) switch (info) { case am_status: case am_priority: + case am_trap_exit: return ERTS_PROC_LOCK_STATUS; case am_links: case am_monitors: @@ -573,7 +597,7 @@ static Eterm pi_args[] = { am_min_bin_vheap_size, am_current_location, am_current_stacktrace, -}; +}; #define ERTS_PI_ARGS ((int) (sizeof(pi_args)/sizeof(Eterm))) @@ -891,7 +915,7 @@ BIF_RETTYPE process_info_1(BIF_ALIST_1) case ERTS_PI_FAIL_TYPE_AWAIT_EXIT: ERTS_BIF_AWAIT_X_DATA_TRAP(BIF_P, BIF_ARG_1, am_undefined); default: - erl_exit(ERTS_ABORT_EXIT, "%s:%d: Internal error", __FILE__, __LINE__); + erts_exit(ERTS_ABORT_EXIT, "%s:%d: Internal error", __FILE__, __LINE__); } } @@ -931,7 +955,7 @@ BIF_RETTYPE process_info_2(BIF_ALIST_2) case ERTS_PI_FAIL_TYPE_AWAIT_EXIT: ERTS_BIF_AWAIT_X_DATA_TRAP(BIF_P, BIF_ARG_1, am_undefined); default: - erl_exit(ERTS_ABORT_EXIT, "%s:%d: Internal error", + erts_exit(ERTS_ABORT_EXIT, "%s:%d: Internal error", __FILE__, __LINE__); } } @@ -1033,9 +1057,9 @@ process_info_aux(Process *BIF_P, case am_initial_call: hp = HAlloc(BIF_P, 3+4); res = TUPLE3(hp, - rp->initial[INITIAL_MOD], - rp->initial[INITIAL_FUN], - make_small(rp->initial[INITIAL_ARI])); + rp->u.initial[INITIAL_MOD], + rp->u.initial[INITIAL_FUN], + make_small(rp->u.initial[INITIAL_ARI])); hp += 4; break; @@ -1085,19 +1109,18 @@ process_info_aux(Process *BIF_P, heap_need += mq[i].copy_struct_size; } else { - mq[i].copy_struct_size = 0; - if (mp->data.attached) - heap_need += erts_msg_attached_data_size(mp); + mq[i].copy_struct_size = mp->data.attached ? + erts_msg_attached_data_size(mp) : 0; } i++; } - hp = HAlloc(BIF_P, heap_need); - hp_end = hp + heap_need; - ASSERT(i == n); - for (i--; i >= 0; i--) { - Eterm msg = ERL_MESSAGE_TERM(mq[i].msgp); - if (rp != BIF_P) { + if (rp != BIF_P) { + hp = HAlloc(BIF_P, heap_need); + hp_end = hp + heap_need; + ASSERT(i == n); + for (i--; i >= 0; i--) { + Eterm msg = ERL_MESSAGE_TERM(mq[i].msgp); if (is_value(msg)) { if (mq[i].copy_struct_size) msg = copy_struct(msg, @@ -1131,9 +1154,9 @@ process_info_aux(Process *BIF_P, } else { /* Make our copy of the message */ - ASSERT(size_object(msg) == hfp->used_size); + ASSERT(size_object(msg) == erts_used_frag_sz(hfp)); msg = copy_struct(msg, - hfp->used_size, + erts_used_frag_sz(hfp), &hp, &MSO(BIF_P)); } @@ -1143,27 +1166,38 @@ process_info_aux(Process *BIF_P, remove_bad_messages = 1; continue; } + res = CONS(hp, msg, res); + hp += 2; } - else { + HRelease(BIF_P, hp_end, hp+3); + } + else { + for (i--; i >= 0; i--) { + ErtsHeapFactory factory; + Eterm msg = ERL_MESSAGE_TERM(mq[i].msgp); + + erts_factory_proc_prealloc_init(&factory, BIF_P, + mq[i].copy_struct_size+2); if (mq[i].msgp->data.attached) { /* Decode it on the heap */ - erts_move_msg_attached_data_to_heap(&hp, - &MSO(BIF_P), + erts_move_msg_attached_data_to_heap(&factory, mq[i].msgp); msg = ERL_MESSAGE_TERM(mq[i].msgp); ASSERT(!mq[i].msgp->data.attached); - if (is_non_value(msg)) { - /* Bad distribution message; ignore */ - remove_bad_messages = 1; - continue; - } - } + } + if (is_value(msg)) { + hp = erts_produce_heap(&factory, 2, 0); + res = CONS(hp, msg, res); + } + else { + /* Bad distribution message; ignore */ + remove_bad_messages = 1; + continue; + } + erts_factory_close(&factory); } - - res = CONS(hp, msg, res); - hp += 2; + hp = HAlloc(BIF_P, 3); } - HRelease(BIF_P, hp_end, hp+3); erts_free(ERTS_ALC_T_TMP, mq); if (remove_bad_messages) { ErlMessage **mpp; @@ -1420,41 +1454,8 @@ process_info_aux(Process *BIF_P, } case am_memory: { /* Memory consumed in bytes */ - ErlMessage *mp; - Uint size = 0; Uint hsz = 3; - struct saved_calls *scb; - size += sizeof(Process); - - ERTS_SMP_MSGQ_MV_INQ2PRIVQ(rp); - - erts_doforall_links(ERTS_P_LINKS(rp), &one_link_size, &size); - erts_doforall_monitors(ERTS_P_MONITORS(rp), &one_mon_size, &size); - size += (rp->heap_sz + rp->mbuf_sz) * sizeof(Eterm); - if (rp->old_hend && rp->old_heap) - size += (rp->old_hend - rp->old_heap) * sizeof(Eterm); - - size += rp->msg.len * sizeof(ErlMessage); - - for (mp = rp->msg.first; mp; mp = mp->next) - if (mp->data.attached) - size += erts_msg_attached_data_size(mp)*sizeof(Eterm); - - if (rp->arg_reg != rp->def_arg_reg) { - size += rp->arity * sizeof(rp->arg_reg[0]); - } - - if (rp->psd) - size += sizeof(ErtsPSD); - - scb = ERTS_PROC_GET_SAVED_CALLS_BUF(rp); - if (scb) { - size += (sizeof(struct saved_calls) - + (scb->len-1) * sizeof(scb->ct[0])); - } - - size += erts_dicts_mem_size(rp); - + Uint size = erts_process_memory(rp); (void) erts_bld_uint(NULL, &hsz, size); hp = HAlloc(BIF_P, hsz); res = erts_bld_uint(&hp, NULL, size); @@ -1533,7 +1534,7 @@ process_info_aux(Process *BIF_P, } case am_last_calls: { - struct saved_calls *scb = ERTS_PROC_GET_SAVED_CALLS_BUF(BIF_P); + struct saved_calls *scb = ERTS_PROC_GET_SAVED_CALLS_BUF(rp); if (!scb) { hp = HAlloc(BIF_P, 3); res = am_false; @@ -1721,13 +1722,22 @@ info_1_tuple(Process* BIF_P, /* Pointer to current process. */ sel = *tp++; - if (sel == am_allocator_sizes) { + if (sel == am_memory_internal) { + switch (arity) { + case 3: + if (erts_request_alloc_info(BIF_P, tp[0], tp[1], 1, 1)) + return am_true; + default: + goto badarg; + } + } + else if (sel == am_allocator_sizes) { switch (arity) { case 2: ERTS_BIF_PREP_TRAP1(ret, alloc_sizes_trap, BIF_P, *tp); return ret; case 3: - if (erts_request_alloc_info(BIF_P, tp[0], tp[1], 1)) + if (erts_request_alloc_info(BIF_P, tp[0], tp[1], 1, 0)) return am_true; default: goto badarg; @@ -1750,7 +1760,7 @@ info_1_tuple(Process* BIF_P, /* Pointer to current process. */ return res; buf = (char *) erts_alloc(ERTS_ALC_T_TMP, len+1); if (intlist_to_buf(*tp, buf, len) != len) - erl_exit(1, "%s:%d: Internal error\n", __FILE__, __LINE__); + erts_exit(ERTS_ERROR_EXIT, "%s:%d: Internal error\n", __FILE__, __LINE__); buf[len] = '\0'; res = erts_instr_dump_memory_map(buf) ? am_true : am_false; erts_free(ERTS_ALC_T_TMP, (void *) buf); @@ -1769,7 +1779,7 @@ info_1_tuple(Process* BIF_P, /* Pointer to current process. */ return res; buf = (char *) erts_alloc(ERTS_ALC_T_TMP, len+1); if (intlist_to_buf(tp[1], buf, len) != len) - erl_exit(1, "%s:%d: Internal error\n", __FILE__, __LINE__); + erts_exit(ERTS_ERROR_EXIT, "%s:%d: Internal error\n", __FILE__, __LINE__); buf[len] = '\0'; res = erts_instr_dump_stat(buf, 1) ? am_true : am_false; erts_free(ERTS_ALC_T_TMP, (void *) buf); @@ -1786,7 +1796,7 @@ info_1_tuple(Process* BIF_P, /* Pointer to current process. */ ERTS_BIF_PREP_TRAP1(ret, alloc_info_trap, BIF_P, *tp); return ret; case 3: - if (erts_request_alloc_info(BIF_P, tp[0], tp[1], 0)) + if (erts_request_alloc_info(BIF_P, tp[0], tp[1], 0, 0)) return am_true; default: goto badarg; @@ -1811,7 +1821,11 @@ info_1_tuple(Process* BIF_P, /* Pointer to current process. */ #if defined(PURIFY) BIF_RET(erts_make_integer(purify_new_leaks(), BIF_P)); #elif defined(VALGRIND) +# ifdef VALGRIND_DO_ADDED_LEAK_CHECK + VALGRIND_DO_ADDED_LEAK_CHECK; +# else VALGRIND_DO_LEAK_CHECK; +# endif BIF_RET(make_small(0)); #endif } else if (*tp == am_fd) { @@ -2096,10 +2110,57 @@ BIF_RETTYPE system_info_1(BIF_ALIST_1) #elif defined(ERTS_ENABLE_LOCK_COUNT) ERTS_DECL_AM(lcnt); BIF_RET(AM_lcnt); +#elif defined(ERTS_FRMPTR) + ERTS_DECL_AM(frmptr); + BIF_RET(AM_frmptr); #else BIF_RET(am_opt); #endif BIF_RET(res); + } else if (BIF_ARG_1 == am_time_offset) { + switch (erts_time_offset_state()) { + case ERTS_TIME_OFFSET_PRELIMINARY: { + ERTS_DECL_AM(preliminary); + BIF_RET(AM_preliminary); + } + case ERTS_TIME_OFFSET_FINAL: { + ERTS_DECL_AM(final); + BIF_RET(AM_final); + } + case ERTS_TIME_OFFSET_VOLATILE: { + ERTS_DECL_AM(volatile); + BIF_RET(AM_volatile); + } + default: + ERTS_INTERNAL_ERROR("Invalid time offset state"); + } + } else if (ERTS_IS_ATOM_STR("os_monotonic_time_source", BIF_ARG_1)) { + BIF_RET(erts_monotonic_time_source(BIF_P)); + } else if (ERTS_IS_ATOM_STR("os_system_time_source", BIF_ARG_1)) { + BIF_RET(erts_system_time_source(BIF_P)); + } else if (ERTS_IS_ATOM_STR("time_correction", BIF_ARG_1)) { + BIF_RET(erts_has_time_correction() ? am_true : am_false); + } else if (ERTS_IS_ATOM_STR("start_time", BIF_ARG_1)) { + BIF_RET(erts_get_monotonic_start_time(BIF_P)); + } else if (ERTS_IS_ATOM_STR("end_time", BIF_ARG_1)) { + BIF_RET(erts_get_monotonic_end_time(BIF_P)); + } else if (ERTS_IS_ATOM_STR("time_warp_mode", BIF_ARG_1)) { + switch (erts_time_warp_mode()) { + case ERTS_NO_TIME_WARP_MODE: { + ERTS_DECL_AM(no_time_warp); + BIF_RET(AM_no_time_warp); + } + case ERTS_SINGLE_TIME_WARP_MODE: { + ERTS_DECL_AM(single_time_warp); + BIF_RET(AM_single_time_warp); + } + case ERTS_MULTI_TIME_WARP_MODE: { + ERTS_DECL_AM(multi_time_warp); + BIF_RET(AM_multi_time_warp); + } + default: + ERTS_INTERNAL_ERROR("Invalid time warp mode"); + } } else if (BIF_ARG_1 == am_allocated_areas) { res = erts_allocated_areas(NULL, NULL, BIF_P); BIF_RET(res); @@ -2128,7 +2189,7 @@ BIF_RETTYPE system_info_1(BIF_ALIST_1) BIF_RET(res); } else if (BIF_ARG_1 == am_sequential_tracer) { val = erts_get_system_seq_tracer(); - ASSERT(is_internal_pid(val) || is_internal_port(val) || val==am_false) + ASSERT(is_internal_pid(val) || is_internal_port(val) || val==am_false); hp = HAlloc(BIF_P, 3); res = TUPLE2(hp, am_sequential_tracer, val); BIF_RET(res); @@ -2302,7 +2363,7 @@ BIF_RETTYPE system_info_1(BIF_ALIST_1) for (i = num_instructions-1; i >= 0; i--) { res = erts_bld_cons(hpp, hszp, erts_bld_tuple(hpp, hszp, 2, - erts_atom_put(opc[i].name, + erts_atom_put((byte *)opc[i].name, strlen(opc[i].name), ERTS_ATOM_ENC_LATIN1, 1), @@ -2461,6 +2522,13 @@ BIF_RETTYPE system_info_1(BIF_ALIST_1) ERL_DRV_EXTENDED_MINOR_VERSION); hp = HAlloc(BIF_P, 2*n); BIF_RET(buf_to_intlist(&hp, buf, n, NIL)); + } else if (ERTS_IS_ATOM_STR("nif_version", BIF_ARG_1)) { + char buf[42]; + int n = erts_snprintf(buf, 42, "%d.%d", + ERL_NIF_MAJOR_VERSION, + ERL_NIF_MINOR_VERSION); + hp = HAlloc(BIF_P, 2*n); + BIF_RET(buf_to_intlist(&hp, buf, n, NIL)); } else if (ERTS_IS_ATOM_STR("smp_support", BIF_ARG_1)) { #ifdef ERTS_SMP BIF_RET(am_true); @@ -2487,6 +2555,9 @@ BIF_RETTYPE system_info_1(BIF_ALIST_1) switch (erts_schedulers_state(&total, &online, &active, + NULL, + NULL, + NULL, 1)) { case ERTS_SCHDLR_SSPND_DONE: { Eterm *hp = HAlloc(BIF_P, 4); @@ -2510,7 +2581,7 @@ BIF_RETTYPE system_info_1(BIF_ALIST_1) BIF_RET(make_small(1)); #else Uint total, online, active; - switch (erts_schedulers_state(&total, &online, &active, 1)) { + switch (erts_schedulers_state(&total, &online, &active, NULL, NULL, NULL, 1)) { case ERTS_SCHDLR_SSPND_DONE: BIF_RET(make_small(online)); case ERTS_SCHDLR_SSPND_YIELD_RESTART: @@ -2527,7 +2598,7 @@ BIF_RETTYPE system_info_1(BIF_ALIST_1) BIF_RET(make_small(1)); #else Uint total, online, active; - switch (erts_schedulers_state(&total, &online, &active, 1)) { + switch (erts_schedulers_state(&total, &online, &active, NULL, NULL, NULL, 1)) { case ERTS_SCHDLR_SSPND_DONE: BIF_RET(make_small(active)); case ERTS_SCHDLR_SSPND_YIELD_RESTART: @@ -2539,6 +2610,20 @@ BIF_RETTYPE system_info_1(BIF_ALIST_1) BIF_ERROR(BIF_P, EXC_INTERNAL_ERROR); } #endif +#if defined(ERTS_SMP) && defined(ERTS_DIRTY_SCHEDULERS) + } else if (ERTS_IS_ATOM_STR("dirty_cpu_schedulers", BIF_ARG_1)) { + Uint dirty_cpu; + erts_schedulers_state(NULL, NULL, NULL, &dirty_cpu, NULL, NULL, 1); + BIF_RET(make_small(dirty_cpu)); + } else if (ERTS_IS_ATOM_STR("dirty_cpu_schedulers_online", BIF_ARG_1)) { + Uint dirty_cpu_onln; + erts_schedulers_state(NULL, NULL, NULL, NULL, &dirty_cpu_onln, NULL, 1); + BIF_RET(make_small(dirty_cpu_onln)); + } else if (ERTS_IS_ATOM_STR("dirty_io_schedulers", BIF_ARG_1)) { + Uint dirty_io; + erts_schedulers_state(NULL, NULL, NULL, NULL, NULL, &dirty_io, 1); + BIF_RET(make_small(dirty_io)); +#endif } else if (ERTS_IS_ATOM_STR("run_queues", BIF_ARG_1)) { res = make_small(erts_no_run_queues); BIF_RET(res); @@ -2615,74 +2700,23 @@ BIF_RETTYPE system_info_1(BIF_ALIST_1) hp = hsz ? HAlloc(BIF_P, hsz) : NULL; res = erts_bld_uint(&hp, NULL, erts_dist_buf_busy_limit); BIF_RET(res); - } else if (ERTS_IS_ATOM_STR("print_ethread_info", BIF_ARG_1)) { -#if defined(ETHR_NATIVE_ATOMIC32_IMPL) \ - || defined(ETHR_NATIVE_ATOMIC64_IMPL) \ - || defined(ETHR_NATIVE_DW_ATOMIC_IMPL) - int i; - char **str; -#endif -#ifdef ETHR_NATIVE_ATOMIC32_IMPL - erts_printf("32-bit native atomics: %s\n", - ETHR_NATIVE_ATOMIC32_IMPL); - str = ethr_native_atomic32_ops(); - for (i = 0; str[i]; i++) - erts_printf("ethr_native_atomic32_%s()\n", str[i]); -#endif -#ifdef ETHR_NATIVE_ATOMIC64_IMPL - erts_printf("64-bit native atomics: %s\n", - ETHR_NATIVE_ATOMIC64_IMPL); - str = ethr_native_atomic64_ops(); - for (i = 0; str[i]; i++) - erts_printf("ethr_native_atomic64_%s()\n", str[i]); -#endif -#ifdef ETHR_NATIVE_DW_ATOMIC_IMPL - if (ethr_have_native_dw_atomic()) { - erts_printf("Double word native atomics: %s\n", - ETHR_NATIVE_DW_ATOMIC_IMPL); - str = ethr_native_dw_atomic_ops(); - for (i = 0; str[i]; i++) - erts_printf("ethr_native_dw_atomic_%s()\n", str[i]); - str = ethr_native_su_dw_atomic_ops(); - for (i = 0; str[i]; i++) - erts_printf("ethr_native_su_dw_atomic_%s()\n", str[i]); - } -#endif -#ifdef ETHR_NATIVE_SPINLOCK_IMPL - erts_printf("Native spin-locks: %s\n", ETHR_NATIVE_SPINLOCK_IMPL); -#endif -#ifdef ETHR_NATIVE_RWSPINLOCK_IMPL - erts_printf("Native rwspin-locks: %s\n", ETHR_NATIVE_RWSPINLOCK_IMPL); -#endif -#ifdef ETHR_X86_RUNTIME_CONF_HAVE_SSE2__ - erts_printf("SSE2 support: %s\n", (ETHR_X86_RUNTIME_CONF_HAVE_SSE2__ - ? "yes" : "no")); -#endif -#ifdef ETHR_X86_OUT_OF_ORDER - erts_printf("x86" -#ifdef ARCH_64 - "_64" -#endif - " out of order\n"); -#endif -#ifdef ETHR_SPARC_TSO - erts_printf("Sparc TSO\n"); -#endif -#ifdef ETHR_SPARC_PSO - erts_printf("Sparc PSO\n"); -#endif -#ifdef ETHR_SPARC_RMO - erts_printf("Sparc RMO\n"); -#endif -#if defined(ETHR_PPC_HAVE_LWSYNC) - erts_printf("Have lwsync instruction: yes\n"); -#elif defined(ETHR_PPC_HAVE_NO_LWSYNC) - erts_printf("Have lwsync instruction: no\n"); -#elif defined(ETHR_PPC_RUNTIME_CONF_HAVE_LWSYNC__) - erts_printf("Have lwsync instruction: %s (runtime test)\n", - ETHR_PPC_RUNTIME_CONF_HAVE_LWSYNC__ ? "yes" : "no"); -#endif - BIF_RET(am_true); + } else if (ERTS_IS_ATOM_STR("delayed_node_table_gc", BIF_ARG_1)) { + Uint hsz = 0; + Uint dntgc = erts_delayed_node_table_gc(); + if (dntgc == ERTS_NODE_TAB_DELAY_GC_INFINITY) + BIF_RET(am_infinity); + (void) erts_bld_uint(NULL, &hsz, dntgc); + hp = hsz ? HAlloc(BIF_P, hsz) : NULL; + res = erts_bld_uint(&hp, NULL, dntgc); + BIF_RET(res); + } else if (ERTS_IS_ATOM_STR("ethread_info", BIF_ARG_1)) { + BIF_RET(erts_get_ethread_info(BIF_P)); + } + else if (ERTS_IS_ATOM_STR("emu_args", BIF_ARG_1)) { + BIF_RET(erts_get_emu_args(BIF_P)); + } + else if (ERTS_IS_ATOM_STR("beam_jump_table", BIF_ARG_1)) { + BIF_RET(erts_beam_jump_table() ? am_true : am_false); } else if (ERTS_IS_ATOM_STR("dynamic_trace", BIF_ARG_1)) { #if defined(USE_DTRACE) @@ -2708,6 +2742,44 @@ BIF_RETTYPE system_info_1(BIF_ALIST_1) BIF_RET(am_true); } #endif + else if (ERTS_IS_ATOM_STR("compile_info",BIF_ARG_1)) { + Uint sz; + Eterm res = NIL, tup, text; + Eterm *hp = HAlloc(BIF_P, 3*(2 + 3) + /* three 2-tuples and three cons */ + 2*(strlen(erts_build_flags_CONFIG_H) + + strlen(erts_build_flags_CFLAGS) + + strlen(erts_build_flags_LDFLAGS))); + + sz = strlen(erts_build_flags_CONFIG_H); + text = buf_to_intlist(&hp, erts_build_flags_CONFIG_H, sz, NIL); + tup = TUPLE2(hp, am_config_h, text); hp += 3; + res = CONS(hp, tup, res); hp += 2; + + sz = strlen(erts_build_flags_CFLAGS); + text = buf_to_intlist(&hp, erts_build_flags_CFLAGS, sz, NIL); + tup = TUPLE2(hp, am_cflags, text); hp += 3; + res = CONS(hp, tup, res); hp += 2; + + sz = strlen(erts_build_flags_LDFLAGS); + text = buf_to_intlist(&hp, erts_build_flags_LDFLAGS, sz, NIL); + tup = TUPLE2(hp, am_ldflags, text); hp += 3; + res = CONS(hp, tup, res); hp += 2; + + BIF_RET(res); + } + else if (ERTS_IS_ATOM_STR("ets_limit",BIF_ARG_1)) { + BIF_RET(make_small(erts_db_get_max_tabs())); + } + else if (ERTS_IS_ATOM_STR("tolerant_timeofday",BIF_ARG_1)) { + if (erts_has_time_correction() + && erts_time_offset_state() == ERTS_TIME_OFFSET_FINAL) { + BIF_RET(am_enabled); + } + BIF_RET(am_disabled); + } + else if (ERTS_IS_ATOM_STR("eager_check_io",BIF_ARG_1)) { + BIF_RET(erts_eager_check_io ? am_true : am_false); + } BIF_ERROR(BIF_P, BADARG); } @@ -2861,12 +2933,10 @@ erts_bld_port_info(Eterm **hpp, ErlOffHeap *ohp, Uint *szp, Port *prt, Eterm ite included though). */ Uint size = 0; - ErlHeapFragment* bp; - erts_doforall_links(ERTS_P_LINKS(prt), &one_link_size, &size); + erts_doforall_links(ERTS_P_LINKS(prt), &erts_one_link_size, &size); - for (bp = prt->bp; bp; bp = bp->next) - size += sizeof(ErlHeapFragment) + (bp->alloc_size - 1)*sizeof(Eterm); + size += erts_port_data_size(prt); if (prt->linebuf) size += sizeof(LineBuf) + prt->linebuf->ovsiz; @@ -3069,6 +3139,25 @@ fun_info_2(BIF_ALIST_2) return TUPLE2(hp, what, val); } +BIF_RETTYPE +fun_info_mfa_1(BIF_ALIST_1) +{ + Process* p = BIF_P; + Eterm fun = BIF_ARG_1; + Eterm* hp; + + if (is_fun(fun)) { + ErlFunThing* funp = (ErlFunThing *) fun_val(fun); + hp = HAlloc(p, 4); + BIF_RET(TUPLE3(hp,funp->fe->module,funp->fe->address[-2],make_small(funp->arity))); + } else if (is_export(fun)) { + Export* exp = (Export *) ((UWord) (export_val(fun))[1]); + hp = HAlloc(p, 4); + BIF_RET(TUPLE3(hp,exp->code[0],exp->code[1],make_small(exp->code[2]))); + } + BIF_ERROR(p, BADARG); +} + BIF_RETTYPE is_process_alive_1(BIF_ALIST_1) { if(is_internal_pid(BIF_ARG_1)) { @@ -3077,7 +3166,7 @@ BIF_RETTYPE is_process_alive_1(BIF_ALIST_1) if (BIF_ARG_1 == BIF_P->common.id) BIF_RET(am_true); - rp = erts_proc_lookup(BIF_ARG_1); + rp = erts_proc_lookup_raw(BIF_ARG_1); if (!rp) { BIF_RET(am_false); } @@ -3134,7 +3223,6 @@ BIF_RETTYPE process_display_2(BIF_ALIST_2) BIF_RET(am_true); } - /* this is a general call which return some possibly useful information */ BIF_RETTYPE statistics_1(BIF_ALIST_1) @@ -3147,24 +3235,49 @@ BIF_RETTYPE statistics_1(BIF_ALIST_1) if (is_non_value(res)) BIF_RET(am_undefined); BIF_TRAP1(gather_sched_wall_time_res_trap, BIF_P, res); + } else if (BIF_ARG_1 == am_total_active_tasks + || BIF_ARG_1 == am_total_run_queue_lengths) { + Uint no = erts_run_queues_len(NULL, 0, BIF_ARG_1 == am_total_active_tasks); + if (IS_USMALL(0, no)) + res = make_small(no); + else { + Eterm *hp = HAlloc(BIF_P, BIG_UINT_HEAP_SIZE); + res = uint_to_big(no, hp); + } + BIF_RET(res); + } else if (BIF_ARG_1 == am_active_tasks + || BIF_ARG_1 == am_run_queue_lengths) { + Eterm res, *hp, **hpp; + Uint sz, *szp; + int no_qs = erts_no_run_queues; + Uint *qszs = erts_alloc(ERTS_ALC_T_TMP,sizeof(Uint)*no_qs*2); + (void) erts_run_queues_len(qszs, 0, BIF_ARG_1 == am_active_tasks); + sz = 0; + szp = &sz; + hpp = NULL; + while (1) { + int i; + for (i = 0; i < no_qs; i++) + qszs[no_qs+i] = erts_bld_uint(hpp, szp, qszs[i]); + res = erts_bld_list(hpp, szp, no_qs, &qszs[no_qs]); + if (hpp) { + erts_free(ERTS_ALC_T_TMP, qszs); + BIF_RET(res); + } + hp = HAlloc(BIF_P, sz); + szp = NULL; + hpp = &hp; + } } else if (BIF_ARG_1 == am_context_switches) { Eterm cs = erts_make_integer(erts_get_total_context_switches(), BIF_P); hp = HAlloc(BIF_P, 3); res = TUPLE2(hp, cs, SMALL_ZERO); BIF_RET(res); } else if (BIF_ARG_1 == am_garbage_collection) { - Uint hsz = 4; - ErtsGCInfo gc_info; - Eterm gcs; - Eterm recl; - erts_gc_info(&gc_info); - (void) erts_bld_uint(NULL, &hsz, gc_info.garbage_collections); - (void) erts_bld_uint(NULL, &hsz, gc_info.reclaimed); - hp = HAlloc(BIF_P, hsz); - gcs = erts_bld_uint(&hp, NULL, gc_info.garbage_collections); - recl = erts_bld_uint(&hp, NULL, gc_info.reclaimed); - res = TUPLE3(hp, gcs, recl, SMALL_ZERO); - BIF_RET(res); + res = erts_gc_info_request(BIF_P); + if (is_non_value(res)) + BIF_RET(am_undefined); + BIF_TRAP1(gather_gc_info_res_trap, BIF_P, res); } else if (BIF_ARG_1 == am_reductions) { Uint reds; Uint diff; @@ -3203,7 +3316,7 @@ BIF_RETTYPE statistics_1(BIF_ALIST_1) res = TUPLE2(hp, b1, b2); BIF_RET(res); } else if (BIF_ARG_1 == am_run_queue) { - res = erts_run_queues_len(NULL); + res = erts_run_queues_len(NULL, 1, 0); BIF_RET(make_small(res)); } else if (BIF_ARG_1 == am_wall_clock) { UWord w1, w2; @@ -3215,30 +3328,15 @@ BIF_RETTYPE statistics_1(BIF_ALIST_1) res = TUPLE2(hp, b1, b2); BIF_RET(res); } else if (BIF_ARG_1 == am_io) { - Eterm r1, r2; - Eterm in, out; - Uint hsz = 9; - Uint bytes_in = (Uint) erts_smp_atomic_read_nob(&erts_bytes_in); - Uint bytes_out = (Uint) erts_smp_atomic_read_nob(&erts_bytes_out); - - (void) erts_bld_uint(NULL, &hsz, bytes_in); - (void) erts_bld_uint(NULL, &hsz, bytes_out); - hp = HAlloc(BIF_P, hsz); - in = erts_bld_uint(&hp, NULL, bytes_in); - out = erts_bld_uint(&hp, NULL, bytes_out); - - r1 = TUPLE2(hp, am_input, in); - hp += 3; - r2 = TUPLE2(hp, am_output, out); - hp += 3; - BIF_RET(TUPLE2(hp, r1, r2)); + Eterm ref = erts_request_io_bytes(BIF_P); + BIF_TRAP2(gather_io_bytes_trap, BIF_P, ref, make_small(erts_no_schedulers)); } else if (ERTS_IS_ATOM_STR("run_queues", BIF_ARG_1)) { Eterm res, *hp, **hpp; Uint sz, *szp; int no_qs = erts_no_run_queues; Uint *qszs = erts_alloc(ERTS_ALC_T_TMP,sizeof(Uint)*no_qs*2); - (void) erts_run_queues_len(qszs); + (void) erts_run_queues_len(qszs, 0, 0); sz = 0; szp = &sz; hpp = NULL; @@ -3307,17 +3405,38 @@ BIF_RETTYPE erts_debug_get_internal_state_1(BIF_ALIST_1) BIF_RET(make_small((Uint) words)); } else if (ERTS_IS_ATOM_STR("check_io_debug", BIF_ARG_1)) { - /* Used by (emulator) */ - int res; + /* Used by driver_SUITE (emulator) */ + Uint sz, *szp; + Eterm res, *hp, **hpp; + int no_errors; + ErtsCheckIoDebugInfo ciodi = {0}; #ifdef HAVE_ERTS_CHECK_IO_DEBUG erts_smp_proc_unlock(BIF_P,ERTS_PROC_LOCK_MAIN); - res = erts_check_io_debug(); + no_errors = erts_check_io_debug(&ciodi); erts_smp_proc_lock(BIF_P,ERTS_PROC_LOCK_MAIN); #else - res = 0; + no_errors = 0; #endif - ASSERT(res >= 0); - BIF_RET(erts_make_integer((Uint) res, BIF_P)); + sz = 0; + szp = &sz; + hpp = NULL; + while (1) { + res = erts_bld_tuple(hpp, szp, 4, + erts_bld_uint(hpp, szp, + (Uint) no_errors), + erts_bld_uint(hpp, szp, + (Uint) ciodi.no_used_fds), + erts_bld_uint(hpp, szp, + (Uint) ciodi.no_driver_select_structs), + erts_bld_uint(hpp, szp, + (Uint) ciodi.no_driver_event_structs)); + if (hpp) + break; + hp = HAlloc(BIF_P, sz); + szp = NULL; + hpp = &hp; + } + BIF_RET(res); } else if (ERTS_IS_ATOM_STR("process_info_args", BIF_ARG_1)) { /* Used by process_SUITE (emulator) */ @@ -3368,6 +3487,32 @@ BIF_RETTYPE erts_debug_get_internal_state_1(BIF_ALIST_1) erts_smp_thr_progress_unblock(); BIF_RET(res); } + else if (ERTS_IS_ATOM_STR("mmap", BIF_ARG_1)) { + BIF_RET(erts_mmap_debug_info(BIF_P)); + } + else if (ERTS_IS_ATOM_STR("unique_monotonic_integer_state", BIF_ARG_1)) { + BIF_RET(erts_debug_get_unique_monotonic_integer_state(BIF_P)); + } + else if (ERTS_IS_ATOM_STR("min_unique_monotonic_integer", BIF_ARG_1)) { + Sint64 value = erts_get_min_unique_monotonic_integer(); + if (IS_SSMALL(value)) + BIF_RET(make_small(value)); + else { + Uint hsz = ERTS_SINT64_HEAP_SIZE(value); + Eterm *hp = HAlloc(BIF_P, hsz); + BIF_RET(erts_sint64_to_big(value, &hp)); + } + } + else if (ERTS_IS_ATOM_STR("min_unique_integer", BIF_ARG_1)) { + Sint64 value = erts_get_min_unique_integer(); + if (IS_SSMALL(value)) + BIF_RET(make_small(value)); + else { + Uint hsz = ERTS_SINT64_HEAP_SIZE(value); + Eterm *hp = HAlloc(BIF_P, hsz); + BIF_RET(erts_sint64_to_big(value, &hp)); + } + } } else if (is_tuple(BIF_ARG_1)) { Eterm* tp = tuple_val(BIF_ARG_1); @@ -3562,6 +3707,58 @@ BIF_RETTYPE erts_debug_get_internal_state_1(BIF_ALIST_1) BIF_RET(erts_debug_reader_groups_map(BIF_P, (int) groups)); } + else if (ERTS_IS_ATOM_STR("internal_hash", tp[1])) { + Uint hash = (Uint) make_internal_hash(tp[2]); + Uint hsz = 0; + Eterm* hp; + erts_bld_uint(NULL, &hsz, hash); + hp = HAlloc(BIF_P,hsz); + return erts_bld_uint(&hp, NULL, hash); + } + else if (ERTS_IS_ATOM_STR("atom", tp[1])) { + Uint ix; + if (!term_to_Uint(tp[2], &ix)) + BIF_ERROR(BIF_P, BADARG); + while (ix >= atom_table_size()) { + char tmp[20]; + erts_snprintf(tmp, sizeof(tmp), "am%x", atom_table_size()); + erts_atom_put((byte *) tmp, strlen(tmp), ERTS_ATOM_ENC_LATIN1, 1); + } + return make_atom(ix); + } + + break; + } + case 3: { + if (ERTS_IS_ATOM_STR("check_time_config", tp[1])) { + int res, time_correction; + ErtsTimeWarpMode time_warp_mode; + if (tp[2] == am_true) + time_correction = !0; + else if (tp[2] == am_false) + time_correction = 0; + else + break; + if (ERTS_IS_ATOM_STR("no_time_warp", tp[3])) + time_warp_mode = ERTS_NO_TIME_WARP_MODE; + else if (ERTS_IS_ATOM_STR("single_time_warp", tp[3])) + time_warp_mode = ERTS_SINGLE_TIME_WARP_MODE; + else if (ERTS_IS_ATOM_STR("multi_time_warp", tp[3])) + time_warp_mode = ERTS_MULTI_TIME_WARP_MODE; + else + break; + res = erts_check_time_adj_support(time_correction, + time_warp_mode); + BIF_RET(res ? am_true : am_false); + } + else if (ERTS_IS_ATOM_STR("make_unique_integer", tp[1])) { + Eterm res = erts_debug_make_unique_integer(BIF_P, + tp[2], + tp[3]); + if (is_non_value(res)) + break; + BIF_RET(res); + } break; } default: @@ -3571,8 +3768,52 @@ BIF_RETTYPE erts_debug_get_internal_state_1(BIF_ALIST_1) BIF_ERROR(BIF_P, BADARG); } +BIF_RETTYPE erts_internal_is_system_process_1(BIF_ALIST_1) +{ + if (is_internal_pid(BIF_ARG_1)) { + Process *rp = erts_proc_lookup(BIF_ARG_1); + if (rp && (rp->static_flags & ERTS_STC_FLG_SYSTEM_PROC)) + BIF_RET(am_true); + BIF_RET(am_false); + } + + if (is_external_pid(BIF_ARG_1) + && external_pid_dist_entry(BIF_ARG_1) == erts_this_dist_entry) { + BIF_RET(am_false); + } + + BIF_ERROR(BIF_P, BADARG); +} + +BIF_RETTYPE erts_internal_system_check_1(BIF_ALIST_1) +{ + Eterm res; + if (ERTS_IS_ATOM_STR("schedulers", BIF_ARG_1)) { + res = erts_system_check_request(BIF_P); + if (is_non_value(res)) + BIF_RET(am_undefined); + BIF_TRAP1(gather_system_check_res_trap, BIF_P, res); + } + + BIF_ERROR(BIF_P, BADARG); +} + static erts_smp_atomic_t hipe_test_reschedule_flag; +#if defined(VALGRIND) && defined(__GNUC__) +/* Force noinline for valgrind suppression */ +static void broken_halt_test(Eterm bif_arg_2) __attribute__((noinline)); +#endif + +static void broken_halt_test(Eterm bif_arg_2) +{ + /* Ugly ugly code used by bif_SUITE:erlang_halt/1 */ +#if defined(ERTS_HAVE_TRY_CATCH) + erts_get_scheduler_data()->run_queue = NULL; +#endif + erts_exit(ERTS_DUMP_EXIT, "%T", bif_arg_2); +} + BIF_RETTYPE erts_debug_set_internal_state_2(BIF_ALIST_2) { @@ -3675,6 +3916,20 @@ BIF_RETTYPE erts_debug_set_internal_state_2(BIF_ALIST_2) BIF_RET(am_true); } } + else if (ERTS_IS_ATOM_STR("gc_state", BIF_ARG_1)) { + /* Used by process_SUITE (emulator) */ + int res, enable; + + switch (BIF_ARG_2) { + case am_true: enable = 1; break; + case am_false: enable = 0; break; + default: BIF_ERROR(BIF_P, BADARG); break; + } + + res = (BIF_P->flags & F_DISABLE_GC) ? am_false : am_true; + erts_set_gc_state(BIF_P, enable); + BIF_RET(res); + } else if (ERTS_IS_ATOM_STR("send_fake_exit_signal", BIF_ARG_1)) { /* Used by signal_SUITE (emulator) */ @@ -3803,7 +4058,7 @@ BIF_RETTYPE erts_debug_set_internal_state_2(BIF_ALIST_2) BIF_RET(am_true); } else if (ERTS_IS_ATOM_STR("abort", BIF_ARG_1)) { - erl_exit(ERTS_ABORT_EXIT, "%T\n", BIF_ARG_2); + erts_exit(ERTS_ABORT_EXIT, "%T\n", BIF_ARG_2); } else if (ERTS_IS_ATOM_STR("kill_dist_connection", BIF_ARG_1)) { DistEntry *dep = erts_sysname_to_connected_dist_entry(BIF_ARG_2); @@ -3846,10 +4101,35 @@ BIF_RETTYPE erts_debug_set_internal_state_2(BIF_ALIST_2) } else if (ERTS_IS_ATOM_STR("wait", BIF_ARG_1)) { if (ERTS_IS_ATOM_STR("deallocations", BIF_ARG_2)) { - if (erts_debug_wait_deallocations(BIF_P)) { + int flag = ERTS_DEBUG_WAIT_COMPLETED_DEALLOCATIONS; + if (erts_debug_wait_completed(BIF_P, flag)) { ERTS_BIF_YIELD_RETURN(BIF_P, am_ok); } } + if (ERTS_IS_ATOM_STR("timer_cancellations", BIF_ARG_2)) { + int flag = ERTS_DEBUG_WAIT_COMPLETED_TIMER_CANCELLATIONS; + if (erts_debug_wait_completed(BIF_P, flag)) { + ERTS_BIF_YIELD_RETURN(BIF_P, am_ok); + } + } + } + else if (ERTS_IS_ATOM_STR("broken_halt", BIF_ARG_1)) { + broken_halt_test(BIF_ARG_2); + } + else if (ERTS_IS_ATOM_STR("unique_monotonic_integer_state", BIF_ARG_1)) { + int res = erts_debug_set_unique_monotonic_integer_state(BIF_ARG_2); + BIF_RET(res ? am_true : am_false); + } + else if (ERTS_IS_ATOM_STR("node_tab_delayed_delete", BIF_ARG_1)) { + /* node_container_SUITE */ + Sint64 msecs; + if (term_to_Sint64(BIF_ARG_2, &msecs)) { + /* Negative value restore original value... */ + erts_smp_proc_unlock(BIF_P, ERTS_PROC_LOCK_MAIN); + erts_debug_test_node_tab_delayed_delete(msecs); + erts_smp_proc_lock(BIF_P, ERTS_PROC_LOCK_MAIN); + BIF_RET(am_ok); + } } } @@ -3861,16 +4141,19 @@ static Eterm lcnt_build_lock_stats_term(Eterm **hpp, Uint *szp, erts_lcnt_lock_s Uint tries = 0, colls = 0; unsigned long timer_s = 0, timer_ns = 0, timer_n = 0; unsigned int line = 0; + unsigned int i; Eterm af, uil; Eterm uit, uic; Eterm uits, uitns, uitn; Eterm tt, tstat, tloc, t; + Eterm thist, vhist[ERTS_LCNT_HISTOGRAM_SLOT_SIZE]; /* term: - * [{{file, line}, {tries, colls, {seconds, nanoseconds, n_blocks}}}] + * [{{file, line}, {tries, colls, {seconds, nanoseconds, n_blocks}}, + * { .. histogram .. }] */ - + tries = (Uint) ethr_atomic_read(&stats->tries); colls = (Uint) ethr_atomic_read(&stats->colls); @@ -3879,23 +4162,27 @@ static Eterm lcnt_build_lock_stats_term(Eterm **hpp, Uint *szp, erts_lcnt_lock_s timer_ns = stats->timer.ns; timer_n = stats->timer_n; - af = erts_atom_put(stats->file, strlen(stats->file), ERTS_ATOM_ENC_LATIN1, 1); + af = erts_atom_put((byte *)stats->file, strlen(stats->file), ERTS_ATOM_ENC_LATIN1, 1); uil = erts_bld_uint( hpp, szp, line); tloc = erts_bld_tuple(hpp, szp, 2, af, uil); - uit = erts_bld_uint( hpp, szp, tries); - uic = erts_bld_uint( hpp, szp, colls); - + uit = erts_bld_uint( hpp, szp, tries); + uic = erts_bld_uint( hpp, szp, colls); + uits = erts_bld_uint( hpp, szp, timer_s); uitns = erts_bld_uint( hpp, szp, timer_ns); uitn = erts_bld_uint( hpp, szp, timer_n); tt = erts_bld_tuple(hpp, szp, 3, uits, uitns, uitn); tstat = erts_bld_tuple(hpp, szp, 3, uit, uic, tt); - - t = erts_bld_tuple(hpp, szp, 2, tloc, tstat); - - res = erts_bld_cons( hpp, szp, t, res); + + for(i = 0; i < ERTS_LCNT_HISTOGRAM_SLOT_SIZE; i++) { + vhist[i] = erts_bld_uint(hpp, szp, stats->hist.ns[i]); + } + thist = erts_bld_tuplev(hpp, szp, ERTS_LCNT_HISTOGRAM_SLOT_SIZE, vhist); + + t = erts_bld_tuple(hpp, szp, 3, tloc, tstat, thist); + res = erts_bld_cons( hpp, szp, t, res); return res; } @@ -3916,13 +4203,13 @@ static Eterm lcnt_build_lock_term(Eterm **hpp, Uint *szp, erts_lcnt_lock_t *lock ASSERT(ltype); - type = erts_atom_put(ltype, strlen(ltype), ERTS_ATOM_ENC_LATIN1, 1); - name = erts_atom_put(lock->name, strlen(lock->name), ERTS_ATOM_ENC_LATIN1, 1); + type = erts_atom_put((byte *)ltype, strlen(ltype), ERTS_ATOM_ENC_LATIN1, 1); + name = erts_atom_put((byte *)lock->name, strlen(lock->name), ERTS_ATOM_ENC_LATIN1, 1); if (lock->flag & ERTS_LCNT_LT_ALLOC) { /* use allocator types names as id's for allocator locks */ ltype = (char *) ERTS_ALC_A2AD(signed_val(lock->id)); - id = erts_atom_put(ltype, strlen(ltype), ERTS_ATOM_ENC_LATIN1, 1); + id = erts_atom_put((byte *)ltype, strlen(ltype), ERTS_ATOM_ENC_LATIN1, 1); } else if (lock->flag & ERTS_LCNT_LT_PROCLOCK) { /* use registered names as id's for process locks if available */ proc = erts_proc_lookup(lock->id); @@ -3933,16 +4220,15 @@ static Eterm lcnt_build_lock_term(Eterm **hpp, Uint *szp, erts_lcnt_lock_t *lock id = lock->id; } } else { - id = lock->id; + id = lock->id; } - + for (i = 0; i < lock->n_stats; i++) { stats = lcnt_build_lock_stats_term(hpp, szp, &(lock->stats[i]), stats); } - - t = erts_bld_tuple(hpp, szp, 4, name, id, type, stats); - - res = erts_bld_cons( hpp, szp, t, res); + + t = erts_bld_tuple(hpp, szp, 4, name, id, type, stats); + res = erts_bld_cons( hpp, szp, t, res); return res; } @@ -3962,12 +4248,12 @@ static Eterm lcnt_build_result_term(Eterm **hpp, Uint *szp, erts_lcnt_data_t *da dtns = erts_bld_uint( hpp, szp, data->duration.ns); tdt = erts_bld_tuple(hpp, szp, 2, dts, dtns); - adur = erts_atom_put(str_duration, strlen(str_duration), ERTS_ATOM_ENC_LATIN1, 1); + adur = erts_atom_put((byte *)str_duration, strlen(str_duration), ERTS_ATOM_ENC_LATIN1, 1); tdur = erts_bld_tuple(hpp, szp, 2, adur, tdt); /* lock tuple */ - aloc = erts_atom_put(str_locks, strlen(str_locks), ERTS_ATOM_ENC_LATIN1, 1); + aloc = erts_atom_put((byte *)str_locks, strlen(str_locks), ERTS_ATOM_ENC_LATIN1, 1); for (lock = data->current_locks->head; lock != NULL ; lock = lock->next ) { lloc = lcnt_build_lock_term(hpp, szp, lock, lloc); @@ -4042,59 +4328,41 @@ BIF_RETTYPE erts_debug_lock_counters_1(BIF_ALIST_1) BIF_RET(am_ok); } else if (is_tuple(BIF_ARG_1)) { - Eterm* tp = tuple_val(BIF_ARG_1); - - switch (arityval(tp[0])) { - case 2: { - int opt = 0; - int val = 0; - if (ERTS_IS_ATOM_STR("copy_save", tp[1])) { - opt = ERTS_LCNT_OPT_COPYSAVE; - } else if (ERTS_IS_ATOM_STR("process_locks", tp[1])) { - opt = ERTS_LCNT_OPT_PROCLOCK; - } else if (ERTS_IS_ATOM_STR("port_locks", tp[1])) { - opt = ERTS_LCNT_OPT_PORTLOCK; - } else if (ERTS_IS_ATOM_STR("suspend", tp[1])) { - opt = ERTS_LCNT_OPT_SUSPEND; - } else if (ERTS_IS_ATOM_STR("location", tp[1])) { - opt = ERTS_LCNT_OPT_LOCATION; - } else { - BIF_ERROR(BIF_P, BADARG); - } - if (tp[2] == am_true) { - val = 1; - } else if (tp[2] == am_false) { - val = 0; - } else { - BIF_ERROR(BIF_P, BADARG); - } - - erts_smp_proc_unlock(BIF_P, ERTS_PROC_LOCK_MAIN); - erts_smp_thr_progress_block(); - - if (val) { - res = erts_lcnt_set_rt_opt(opt) ? am_true : am_false; - } else { - res = erts_lcnt_clear_rt_opt(opt) ? am_true : am_false; - } + Eterm* ptr = tuple_val(BIF_ARG_1); + + if ((arityval(ptr[0]) == 2) && (ptr[2] == am_false || ptr[2] == am_true)) { + int lock_opt = 0, enable = (ptr[2] == am_true) ? 1 : 0; + if (ERTS_IS_ATOM_STR("copy_save", ptr[1])) { + lock_opt = ERTS_LCNT_OPT_COPYSAVE; + } else if (ERTS_IS_ATOM_STR("process_locks", ptr[1])) { + lock_opt = ERTS_LCNT_OPT_PROCLOCK; + } else if (ERTS_IS_ATOM_STR("port_locks", ptr[1])) { + lock_opt = ERTS_LCNT_OPT_PORTLOCK; + } else if (ERTS_IS_ATOM_STR("suspend", ptr[1])) { + lock_opt = ERTS_LCNT_OPT_SUSPEND; + } else if (ERTS_IS_ATOM_STR("location", ptr[1])) { + lock_opt = ERTS_LCNT_OPT_LOCATION; + } else { + BIF_ERROR(BIF_P, BADARG); + } + + erts_smp_proc_unlock(BIF_P, ERTS_PROC_LOCK_MAIN); + erts_smp_thr_progress_block(); + + if (enable) res = erts_lcnt_set_rt_opt(lock_opt) ? am_true : am_false; + else res = erts_lcnt_clear_rt_opt(lock_opt) ? am_true : am_false; + #ifdef ERTS_SMP - if (res != tp[2]) { - if (opt == ERTS_LCNT_OPT_PORTLOCK) { - erts_lcnt_enable_io_lock_count(val); - } else if (opt == ERTS_LCNT_OPT_PROCLOCK) { - erts_lcnt_enable_proc_lock_count(val); - } - } + if (res != ptr[2] && lock_opt == ERTS_LCNT_OPT_PORTLOCK) { + erts_lcnt_enable_io_lock_count(enable); + } else if (res != ptr[2] && lock_opt == ERTS_LCNT_OPT_PROCLOCK) { + erts_lcnt_enable_proc_lock_count(enable); + } #endif - erts_smp_thr_progress_unblock(); - erts_smp_proc_lock(BIF_P, ERTS_PROC_LOCK_MAIN); - BIF_RET(res); - break; - } - - default: - break; - } + erts_smp_thr_progress_unblock(); + erts_smp_proc_lock(BIF_P, ERTS_PROC_LOCK_MAIN); + BIF_RET(res); + } } #endif @@ -4132,6 +4400,12 @@ erts_bif_info_init(void) alloc_sizes_trap = erts_export_put(am_erlang, am_alloc_sizes, 1); gather_sched_wall_time_res_trap = erts_export_put(am_erlang, am_gather_sched_wall_time_result, 1); + gather_gc_info_res_trap + = erts_export_put(am_erlang, am_gather_gc_info_result, 1); + gather_io_bytes_trap + = erts_export_put(am_erts_internal, am_gather_io_bytes, 2); + gather_system_check_res_trap + = erts_export_put(am_erts_internal, am_gather_system_check_result, 1); process_info_init(); os_info_init(); } diff --git a/erts/emulator/beam/erl_bif_lists.c b/erts/emulator/beam/erl_bif_lists.c index 1805366cfe..5583dcb371 100644 --- a/erts/emulator/beam/erl_bif_lists.c +++ b/erts/emulator/beam/erl_bif_lists.c @@ -3,16 +3,17 @@ * * Copyright Ericsson AB 1999-2011. All Rights Reserved. * - * The contents of this file are subject to the Erlang Public License, - * Version 1.1, (the "License"); you may not use this file except in - * compliance with the License. You should have received a copy of the - * Erlang Public License along with this software. If not, it can be - * retrieved online at http://www.erlang.org/. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Software distributed under the License is distributed on an "AS IS" - * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See - * the License for the specific language governing rights and limitations - * under the License. + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. * * %CopyrightEnd% */ @@ -43,7 +44,7 @@ static BIF_RETTYPE append(Process* p, Eterm A, Eterm B) Eterm* hp; int i; - if ((i = list_length(A)) < 0) { + if ((i = erts_list_length(A)) < 0) { BIF_ERROR(p, BADARG); } if (i == 0) { @@ -102,10 +103,10 @@ static Eterm subtract(Process* p, Eterm A, Eterm B) int n; int m; - if ((n = list_length(A)) < 0) { + if ((n = erts_list_length(A)) < 0) { BIF_ERROR(p, BADARG); } - if ((m = list_length(B)) < 0) { + if ((m = erts_list_length(B)) < 0) { BIF_ERROR(p, BADARG); } @@ -390,7 +391,7 @@ keyfind(int Bif, Process* p, Eterm Key, Eterm Pos, Eterm List) Eterm *tuple_ptr = tuple_val(term); if (pos <= arityval(*tuple_ptr)) { Eterm element = tuple_ptr[pos]; - if (CMP(Key, element) == 0) { + if (CMP_EQ(Key, element)) { return term; } } diff --git a/erts/emulator/beam/erl_bif_op.c b/erts/emulator/beam/erl_bif_op.c index adac0052d6..c9192fc420 100644 --- a/erts/emulator/beam/erl_bif_op.c +++ b/erts/emulator/beam/erl_bif_op.c @@ -3,16 +3,17 @@ * * Copyright Ericsson AB 1999-2013. All Rights Reserved. * - * The contents of this file are subject to the Erlang Public License, - * Version 1.1, (the "License"); you may not use this file except in - * compliance with the License. You should have received a copy of the - * Erlang Public License along with this software. If not, it can be - * retrieved online at http://www.erlang.org/. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Software distributed under the License is distributed on an "AS IS" - * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See - * the License for the specific language governing rights and limitations - * under the License. + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. * * %CopyrightEnd% */ @@ -36,6 +37,7 @@ #include "dist.h" #include "erl_version.h" #include "erl_binary.h" +#include "erl_map.h" BIF_RETTYPE and_2(BIF_ALIST_2) { @@ -321,7 +323,10 @@ BIF_RETTYPE is_record_3(BIF_ALIST_3) BIF_RET(am_false); } - - - - +BIF_RETTYPE is_map_1(BIF_ALIST_1) +{ + if (is_map(BIF_ARG_1)) { + BIF_RET(am_true); + } + BIF_RET(am_false); +} diff --git a/erts/emulator/beam/erl_bif_os.c b/erts/emulator/beam/erl_bif_os.c index 1062d4379b..2333ca0851 100644 --- a/erts/emulator/beam/erl_bif_os.c +++ b/erts/emulator/beam/erl_bif_os.c @@ -3,16 +3,17 @@ * * Copyright Ericsson AB 1999-2012. All Rights Reserved. * - * The contents of this file are subject to the Erlang Public License, - * Version 1.1, (the "License"); you may not use this file except in - * compliance with the License. You should have received a copy of the - * Erlang Public License along with this software. If not, it can be - * retrieved online at http://www.erlang.org/. - * - * Software distributed under the License is distributed on an "AS IS" - * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See - * the License for the specific language governing rights and limitations - * under the License. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. * * %CopyrightEnd% */ @@ -180,3 +181,25 @@ BIF_RETTYPE os_putenv_2(BIF_ALIST_2) BIF_RET(am_true); } +BIF_RETTYPE os_unsetenv_1(BIF_ALIST_1) +{ + char *key_buf; + char buf[STATIC_BUF_SIZE]; + + key_buf = erts_convert_filename_to_native(BIF_ARG_1,buf,STATIC_BUF_SIZE, + ERTS_ALC_T_TMP,0,0,NULL); + if (!key_buf) { + BIF_ERROR(BIF_P, BADARG); + } + + if (erts_sys_unsetenv(key_buf)) { + if (key_buf != buf) { + erts_free(ERTS_ALC_T_TMP, key_buf); + } + BIF_ERROR(BIF_P, BADARG); + } + if (key_buf != buf) { + erts_free(ERTS_ALC_T_TMP, key_buf); + } + BIF_RET(am_true); +} diff --git a/erts/emulator/beam/erl_bif_port.c b/erts/emulator/beam/erl_bif_port.c index 44fa41c7b6..27c24197ea 100644 --- a/erts/emulator/beam/erl_bif_port.c +++ b/erts/emulator/beam/erl_bif_port.c @@ -3,16 +3,17 @@ * * Copyright Ericsson AB 2001-2013. All Rights Reserved. * - * The contents of this file are subject to the Erlang Public License, - * Version 1.1, (the "License"); you may not use this file except in - * compliance with the License. You should have received a copy of the - * Erlang Public License along with this software. If not, it can be - * retrieved online at http://www.erlang.org/. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Software distributed under the License is distributed on an "AS IS" - * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See - * the License for the specific language governing rights and limitations - * under the License. + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. * * %CopyrightEnd% */ @@ -84,7 +85,7 @@ BIF_RETTYPE open_port_2(BIF_ALIST_2) } static ERTS_INLINE Port * -lookup_port(Process *c_p, Eterm id_or_name) +lookup_port(Process *c_p, Eterm id_or_name, Uint32 invalid_flags) { /* TODO: Implement nicer lookup in register... */ Eterm id; @@ -92,7 +93,19 @@ lookup_port(Process *c_p, Eterm id_or_name) id = erts_whereis_name_to_id(c_p, id_or_name); else id = id_or_name; - return erts_port_lookup(id, ERTS_PORT_SFLGS_INVALID_LOOKUP); + return erts_port_lookup(id, invalid_flags); +} + +static ERTS_INLINE Port * +sig_lookup_port(Process *c_p, Eterm id_or_name) +{ + return lookup_port(c_p, id_or_name, ERTS_PORT_SFLGS_INVALID_DRIVER_LOOKUP); +} + +static ERTS_INLINE Port * +data_lookup_port(Process *c_p, Eterm id_or_name) +{ + return lookup_port(c_p, id_or_name, ERTS_PORT_SFLGS_INVALID_LOOKUP); } /* @@ -125,7 +138,7 @@ BIF_RETTYPE erts_internal_port_command_3(BIF_ALIST_3) BIF_RET(am_badarg); } - prt = lookup_port(BIF_P, BIF_ARG_1); + prt = sig_lookup_port(BIF_P, BIF_ARG_1); if (!prt) BIF_RET(am_badarg); @@ -185,7 +198,7 @@ BIF_RETTYPE erts_internal_port_call_3(BIF_ALIST_3) unsigned int op; erts_aint32_t state; - prt = lookup_port(BIF_P, BIF_ARG_1); + prt = sig_lookup_port(BIF_P, BIF_ARG_1); if (!prt) BIF_RET(am_badarg); @@ -235,7 +248,7 @@ BIF_RETTYPE erts_internal_port_control_3(BIF_ALIST_3) unsigned int op; erts_aint32_t state; - prt = lookup_port(BIF_P, BIF_ARG_1); + prt = sig_lookup_port(BIF_P, BIF_ARG_1); if (!prt) BIF_RET(am_badarg); @@ -290,7 +303,7 @@ BIF_RETTYPE erts_internal_port_close_1(BIF_ALIST_1) ref = NIL; #endif - prt = lookup_port(BIF_P, BIF_ARG_1); + prt = sig_lookup_port(BIF_P, BIF_ARG_1); if (!prt) BIF_RET(am_badarg); @@ -320,7 +333,7 @@ BIF_RETTYPE erts_internal_port_connect_2(BIF_ALIST_2) Eterm ref; Port* prt; - prt = lookup_port(BIF_P, BIF_ARG_1); + prt = sig_lookup_port(BIF_P, BIF_ARG_1); if (!prt) BIF_RET(am_badarg); @@ -352,7 +365,7 @@ BIF_RETTYPE erts_internal_port_info_1(BIF_ALIST_1) Port* prt; if (is_internal_port(BIF_ARG_1) || is_atom(BIF_ARG_1)) { - prt = lookup_port(BIF_P, BIF_ARG_1); + prt = sig_lookup_port(BIF_P, BIF_ARG_1); if (!prt) BIF_RET(am_undefined); } @@ -391,7 +404,7 @@ BIF_RETTYPE erts_internal_port_info_2(BIF_ALIST_2) Port* prt; if (is_internal_port(BIF_ARG_1) || is_atom(BIF_ARG_1)) { - prt = lookup_port(BIF_P, BIF_ARG_1); + prt = sig_lookup_port(BIF_P, BIF_ARG_1); if (!prt) BIF_RET(am_undefined); } @@ -423,57 +436,173 @@ BIF_RETTYPE erts_internal_port_info_2(BIF_ALIST_2) } } +/* + * The erlang:port_set_data()/erlang:port_get_data() operations should + * be viewed as operations on a table (inet_db) with data values + * associated with port identifier keys. That is, these operations are + * *not* signals to/from ports. + */ + +#if (TAG_PRIMARY_IMMED1 & 0x3) == 0 +# error "erlang:port_set_data()/erlang:port_get_data() needs to be rewritten!" +#endif + +typedef struct { + ErtsThrPrgrLaterOp later_op; + Uint hsize; + Eterm data; + ErlOffHeap off_heap; + Eterm heap[1]; +} ErtsPortDataHeap; -BIF_RETTYPE erts_internal_port_set_data_2(BIF_ALIST_2) +static void +free_port_data_heap(void *vpdhp) { - Eterm ref; + erts_cleanup_offheap(&((ErtsPortDataHeap *) vpdhp)->off_heap); + erts_free(ERTS_ALC_T_PORT_DATA_HEAP, vpdhp); +} + +static ERTS_INLINE void +cleanup_old_port_data(erts_aint_t data) +{ + if ((data & 0x3) != 0) { + ASSERT(is_immed((Eterm) data)); + } + else { +#ifdef ERTS_SMP + ErtsPortDataHeap *pdhp = (ErtsPortDataHeap *) data; + size_t size; + ERTS_SMP_DATA_DEPENDENCY_READ_MEMORY_BARRIER; + size = sizeof(ErtsPortDataHeap) + (pdhp->hsize-1)*sizeof(Eterm); + erts_schedule_thr_prgr_later_cleanup_op(free_port_data_heap, + (void *) pdhp, + &pdhp->later_op, + size); +#else + free_port_data_heap((void *) data); +#endif + } +} + +void +erts_init_port_data(Port *prt) +{ + erts_smp_atomic_init_nob(&prt->data, (erts_aint_t) am_undefined); +} + +void +erts_cleanup_port_data(Port *prt) +{ + ASSERT(erts_atomic32_read_nob(&prt->state) & ERTS_PORT_SFLGS_INVALID_LOOKUP); + cleanup_old_port_data(erts_smp_atomic_xchg_nob(&prt->data, + (erts_aint_t) NULL)); +} + +Uint +erts_port_data_size(Port *prt) +{ + erts_aint_t data = erts_smp_atomic_read_ddrb(&prt->data); + + if ((data & 0x3) != 0) { + ASSERT(is_immed((Eterm) (UWord) data)); + return (Uint) 0; + } + else { + ErtsPortDataHeap *pdhp = (ErtsPortDataHeap *) data; + return (Uint) sizeof(ErtsPortDataHeap) + (pdhp->hsize-1)*sizeof(Eterm); + } +} + +ErlOffHeap * +erts_port_data_offheap(Port *prt) +{ + erts_aint_t data = erts_smp_atomic_read_ddrb(&prt->data); + + if ((data & 0x3) != 0) { + ASSERT(is_immed((Eterm) (UWord) data)); + return NULL; + } + else { + ErtsPortDataHeap *pdhp = (ErtsPortDataHeap *) data; + return &pdhp->off_heap; + } +} + +BIF_RETTYPE port_set_data_2(BIF_ALIST_2) +{ + /* + * This is not a signal. See comment above. + */ + erts_aint_t data; Port* prt; - prt = lookup_port(BIF_P, BIF_ARG_1); + prt = data_lookup_port(BIF_P, BIF_ARG_1); if (!prt) - BIF_RET(am_badarg); + BIF_ERROR(BIF_P, BADARG); - switch (erts_port_set_data(BIF_P, prt, BIF_ARG_2, &ref)) { - case ERTS_PORT_OP_CALLER_EXIT: - case ERTS_PORT_OP_BADARG: - case ERTS_PORT_OP_DROPPED: - BIF_RET(am_badarg); - case ERTS_PORT_OP_SCHEDULED: - ASSERT(is_internal_ref(ref)); - BIF_RET(ref); - case ERTS_PORT_OP_DONE: - BIF_RET(am_true); - default: - ERTS_INTERNAL_ERROR("Unexpected erts_port_set_data() result"); - BIF_RET(am_internal_error); + if (is_immed(BIF_ARG_2)) { + data = (erts_aint_t) BIF_ARG_2; + ASSERT((data & 0x3) != 0); + } + else { + ErtsPortDataHeap *pdhp; + Uint hsize; + Eterm *hp; + + hsize = size_object(BIF_ARG_2); + pdhp = erts_alloc(ERTS_ALC_T_PORT_DATA_HEAP, + sizeof(ErtsPortDataHeap) + (hsize-1)*sizeof(Eterm)); + hp = &pdhp->heap[0]; + pdhp->off_heap.first = NULL; + pdhp->off_heap.overhead = 0; + pdhp->hsize = hsize; + pdhp->data = copy_struct(BIF_ARG_2, hsize, &hp, &pdhp->off_heap); + data = (erts_aint_t) pdhp; + ASSERT((data & 0x3) == 0); } + + data = erts_smp_atomic_xchg_wb(&prt->data, data); + + if (data == (erts_aint_t)NULL) { + /* Port terminated by racing thread */ + data = erts_smp_atomic_xchg_wb(&prt->data, data); + ASSERT(data != (erts_aint_t)NULL); + cleanup_old_port_data(data); + BIF_ERROR(BIF_P, BADARG); + } + cleanup_old_port_data(data); + BIF_RET(am_true); } -BIF_RETTYPE erts_internal_port_get_data_1(BIF_ALIST_1) +BIF_RETTYPE port_get_data_1(BIF_ALIST_1) { - Eterm retval; + /* + * This is not a signal. See comment above. + */ + Eterm res; + erts_aint_t data; Port* prt; - prt = lookup_port(BIF_P, BIF_ARG_1); + prt = data_lookup_port(BIF_P, BIF_ARG_1); if (!prt) - BIF_RET(am_badarg); + BIF_ERROR(BIF_P, BADARG); - switch (erts_port_get_data(BIF_P, prt, &retval)) { - case ERTS_PORT_OP_CALLER_EXIT: - case ERTS_PORT_OP_BADARG: - case ERTS_PORT_OP_DROPPED: - BIF_RET(am_badarg); - case ERTS_PORT_OP_SCHEDULED: - ASSERT(is_internal_ref(retval)); - BIF_RET(retval); - case ERTS_PORT_OP_DONE: - ASSERT(is_not_internal_ref(retval)); - BIF_RET(retval); - default: - ERTS_INTERNAL_ERROR("Unexpected erts_port_get_data() result"); - BIF_RET(am_internal_error); + data = erts_smp_atomic_read_ddrb(&prt->data); + if (data == (erts_aint_t)NULL) + BIF_ERROR(BIF_P, BADARG); /* Port terminated by racing thread */ + + if ((data & 0x3) != 0) { + res = (Eterm) (UWord) data; + ASSERT(is_immed(res)); + } + else { + ErtsPortDataHeap *pdhp = (ErtsPortDataHeap *) data; + Eterm *hp = HAlloc(BIF_P, pdhp->hsize); + res = copy_struct(pdhp->data, pdhp->hsize, &hp, &MSO(BIF_P)); } + + BIF_RET(res); } /* @@ -662,7 +791,7 @@ open_port(Process* p, Eterm name, Eterm settings, int *err_typep, int *err_nump) } else { name_buf = (char *) erts_alloc(ERTS_ALC_T_TMP, i + 1); if (intlist_to_buf(name, name_buf, i) != i) - erl_exit(1, "%s:%d: Internal error\n", __FILE__, __LINE__); + erts_exit(ERTS_ERROR_EXIT, "%s:%d: Internal error\n", __FILE__, __LINE__); name_buf[i] = '\0'; } driver = &vanilla_driver; @@ -677,43 +806,29 @@ open_port(Process* p, Eterm name, Eterm settings, int *err_typep, int *err_nump) goto badarg; } - if (*tp == am_spawn || *tp == am_spawn_driver) { /* A process port */ + if (*tp == am_spawn || *tp == am_spawn_driver || *tp == am_spawn_executable) { /* A process port */ + int encoding; if (arity != make_arityval(2)) { goto badarg; } name = tp[1]; - if (is_atom(name)) { - name_buf = (char *) erts_alloc(ERTS_ALC_T_TMP, - atom_tab(atom_val(name))->len+1); - sys_memcpy((void *) name_buf, - (void *) atom_tab(atom_val(name))->name, - atom_tab(atom_val(name))->len); - name_buf[atom_tab(atom_val(name))->len] = '\0'; - } else if ((i = is_string(name))) { - name_buf = (char *) erts_alloc(ERTS_ALC_T_TMP, i + 1); - if (intlist_to_buf(name, name_buf, i) != i) - erl_exit(1, "%s:%d: Internal error\n", __FILE__, __LINE__); - name_buf[i] = '\0'; - } else { + encoding = erts_get_native_filename_encoding(); + /* Do not convert the command to utf-16le yet, do that in win32 specific code */ + /* since the cmd is used for comparsion with drivers names and copied to port info */ + if (encoding == ERL_FILENAME_WIN_WCHAR) { + encoding = ERL_FILENAME_UTF8; + } + if ((name_buf = erts_convert_filename_to_encoding(name, NULL, 0, ERTS_ALC_T_TMP,0,1, encoding, NULL, 0)) + == NULL) { goto badarg; } + if (*tp == am_spawn_driver) { opts.spawn_type = ERTS_SPAWN_DRIVER; + } else if (*tp == am_spawn_executable) { + opts.spawn_type = ERTS_SPAWN_EXECUTABLE; } - driver = &spawn_driver; - } else if (*tp == am_spawn_executable) { /* A program */ - /* - * {spawn_executable,Progname} - */ - - if (arity != make_arityval(2)) { - goto badarg; - } - name = tp[1]; - if ((name_buf = erts_convert_filename_to_native(name, NULL, 0, ERTS_ALC_T_TMP,0,1, NULL)) == NULL) { - goto badarg; - } - opts.spawn_type = ERTS_SPAWN_EXECUTABLE; + driver = &spawn_driver; } else if (*tp == am_fd) { /* An fd port */ int n; @@ -754,29 +869,8 @@ open_port(Process* p, Eterm name, Eterm settings, int *err_typep, int *err_nump) } if (edir != NIL) { - /* A working directory is expressed differently if spawn_executable, i.e. Unicode is handles - for spawn_executable... */ - if (opts.spawn_type != ERTS_SPAWN_EXECUTABLE) { - Eterm iolist; - DeclareTmpHeap(heap,4,p); - int r; - - UseTmpHeap(4,p); - heap[0] = edir; - heap[1] = make_list(heap+2); - heap[2] = make_small(0); - heap[3] = NIL; - iolist = make_list(heap); - r = erts_iolist_to_buf(iolist, (char*) dir, MAXPATHLEN); - UnUseTmpHeap(4,p); - if (ERTS_IOLIST_TO_BUF_FAILED(r)) { - goto badarg; - } - opts.wd = (char *) dir; - } else { - if ((opts.wd = erts_convert_filename_to_native(edir, NULL, 0, ERTS_ALC_T_TMP,0,1,NULL)) == NULL) { - goto badarg; - } + if ((opts.wd = erts_convert_filename_to_native(edir, NULL, 0, ERTS_ALC_T_TMP,0,1,NULL)) == NULL) { + goto badarg; } } @@ -798,7 +892,7 @@ open_port(Process* p, Eterm name, Eterm settings, int *err_typep, int *err_nump) DTRACE_CHARBUF(port_str, DTRACE_TERM_BUF_SIZE); dtrace_proc_str(p, process_str); - erts_snprintf(port_str, sizeof(port_str), "%T", port->common.id); + erts_snprintf(port_str, sizeof(DTRACE_CHARBUF_NAME(port_str)), "%T", port->common.id); DTRACE3(port_open, process_str, name_buf, port_str); } #endif @@ -854,11 +948,12 @@ static char **convert_args(Eterm l) int n; int i = 0; Eterm str; - /* We require at least one element in list (argv[0]) */ if (is_not_list(l) && is_not_nil(l)) { return NULL; } - n = list_length(l); + + n = erts_list_length(l); + /* We require at least one element in argv[0] + NULL at end */ pp = erts_alloc(ERTS_ALC_T_TMP, (n + 2) * sizeof(char **)); pp[i++] = erts_default_arg0; while (is_list(l)) { @@ -902,7 +997,7 @@ static byte* convert_environment(Process* p, Eterm env) byte* bytes; int encoding = erts_get_native_filename_encoding(); - if ((n = list_length(env)) < 0) { + if ((n = erts_list_length(env)) < 0) { return NULL; } heap_size = 2*(5*n+1); @@ -1074,7 +1169,7 @@ static Eterm http_bld_uri(struct packet_callback_args* pca, return erts_bld_tuple(hpp, szp, 3, am_scheme, s1, s2); default: - erl_exit(1, "%s, line %d: type=%u\n", __FILE__, __LINE__, uri->type); + erts_exit(ERTS_ERROR_EXIT, "%s, line %d: type=%u\n", __FILE__, __LINE__, uri->type); } } @@ -1234,7 +1329,8 @@ BIF_RETTYPE decode_packet_3(BIF_ALIST_3) ErlSubBin* rest; Eterm res; Eterm options; - int code; + int code; + char delimiter = '\n'; if (!is_binary(BIF_ARG_2) || (!is_list(BIF_ARG_3) && !is_nil(BIF_ARG_3))) { @@ -1275,6 +1371,11 @@ BIF_RETTYPE decode_packet_3(BIF_ALIST_3) case am_line_length: trunc_len = val; goto next_option; + case am_line_delimiter: + if (type == TCP_PB_LINE_LF && val >= 0 && val <= 255) { + delimiter = (char)val; + goto next_option; + } } } } @@ -1295,7 +1396,7 @@ BIF_RETTYPE decode_packet_3(BIF_ALIST_3) pca.aligned_ptr = bin_ptr; } packet_sz = packet_get_length(type, (char*)pca.aligned_ptr, pca.bin_sz, - max_plen, trunc_len, &http_state); + max_plen, trunc_len, delimiter, &http_state); if (!(packet_sz > 0 && packet_sz <= pca.bin_sz)) { if (packet_sz < 0) { goto error; diff --git a/erts/emulator/beam/erl_bif_re.c b/erts/emulator/beam/erl_bif_re.c index 3d34c2a77f..86951f32b0 100644 --- a/erts/emulator/beam/erl_bif_re.c +++ b/erts/emulator/beam/erl_bif_re.c @@ -3,16 +3,17 @@ * * Copyright Ericsson AB 2008-2013. All Rights Reserved. * - * The contents of this file are subject to the Erlang Public License, - * Version 1.1, (the "License"); you may not use this file except in - * compliance with the License. You should have received a copy of the - * Erlang Public License along with this software. If not, it can be - * retrieved online at http://www.erlang.org/. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Software distributed under the License is distributed on an "AS IS" - * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See - * the License for the specific language governing rights and limitations - * under the License. + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. * * %CopyrightEnd% */ @@ -180,6 +181,9 @@ static Eterm make_signed_integer(int x, Process *p) #define PARSE_FLAG_STARTOFFSET 8 #define PARSE_FLAG_CAPTURE_OPT 16 #define PARSE_FLAG_GLOBAL 32 +#define PARSE_FLAG_REPORT_ERRORS 64 +#define PARSE_FLAG_MATCH_LIMIT 128 +#define PARSE_FLAG_MATCH_LIMIT_RECURSION 256 #define CAPSPEC_VALUES 0 #define CAPSPEC_TYPE 1 @@ -192,7 +196,9 @@ parse_options(Eterm listp, /* in */ int *exec_options, /* out */ int *flags,/* out */ int *startoffset, /* out */ - Eterm *capture_spec) /* capture_spec[CAPSPEC_SIZE] */ /* out */ + Eterm *capture_spec, /* capture_spec[CAPSPEC_SIZE] */ /* out */ + int *match_limit, /* out */ + int *match_limit_recursion) /* out */ { int copt,eopt,fl; Eterm item; @@ -234,7 +240,7 @@ parse_options(Eterm listp, /* in */ case am_offset: { int tmp; - if (!term_to_int(tp[2],&tmp)) { + if (!term_to_int(tp[2],&tmp) || tmp < 0) { return -1; } if (startoffset != NULL) { @@ -243,6 +249,31 @@ parse_options(Eterm listp, /* in */ } fl |= (PARSE_FLAG_UNIQUE_EXEC_OPT|PARSE_FLAG_STARTOFFSET); break; + case am_match_limit: + { + int tmp; + if (!term_to_int(tp[2],&tmp) || tmp < 0) { + return -1; + } + if (match_limit != NULL) { + *match_limit = tmp; + } + } + fl |= (PARSE_FLAG_UNIQUE_EXEC_OPT|PARSE_FLAG_MATCH_LIMIT); + break; + case am_match_limit_recursion: + { + int tmp; + if (!term_to_int(tp[2],&tmp) || tmp < 0) { + return -1; + } + if (match_limit_recursion != NULL) { + *match_limit_recursion = tmp; + } + } + fl |= (PARSE_FLAG_UNIQUE_EXEC_OPT| + PARSE_FLAG_MATCH_LIMIT_RECURSION); + break; case am_newline: if (!is_atom(tp[2])) { return -1; @@ -276,7 +307,7 @@ parse_options(Eterm listp, /* in */ default: return -1; } - }else if (is_not_atom(item)) { + } else if (is_not_atom(item)) { return -1; } else { switch(item) { @@ -288,6 +319,10 @@ parse_options(Eterm listp, /* in */ eopt |= PCRE_NOTEMPTY; fl |= PARSE_FLAG_UNIQUE_EXEC_OPT; break; + case am_notempty_atstart: + eopt |= PCRE_NOTEMPTY_ATSTART; + fl |= PARSE_FLAG_UNIQUE_EXEC_OPT; + break; case am_notbol: eopt |= PCRE_NOTBOL; fl |= PARSE_FLAG_UNIQUE_EXEC_OPT; @@ -296,6 +331,10 @@ parse_options(Eterm listp, /* in */ eopt |= PCRE_NOTEOL; fl |= PARSE_FLAG_UNIQUE_EXEC_OPT; break; + case am_no_start_optimize: + copt |= PCRE_NO_START_OPTIMIZE; + fl |= PARSE_FLAG_UNIQUE_COMPILE_OPT; + break; case am_caseless: copt |= PCRE_CASELESS; fl |= PARSE_FLAG_UNIQUE_COMPILE_OPT; @@ -332,6 +371,18 @@ parse_options(Eterm listp, /* in */ copt |= PCRE_UNGREEDY; fl |= PARSE_FLAG_UNIQUE_COMPILE_OPT; break; + case am_ucp: + copt |= PCRE_UCP; + fl |= PARSE_FLAG_UNIQUE_COMPILE_OPT; + break; + case am_never_utf: + copt |= PCRE_NEVER_UTF; + fl |= PARSE_FLAG_UNIQUE_COMPILE_OPT; + break; + case am_report_errors: + fl |= (PARSE_FLAG_UNIQUE_EXEC_OPT | + PARSE_FLAG_REPORT_ERRORS); + break; case am_unicode: copt |= PCRE_UTF8; fl |= (PARSE_FLAG_UNIQUE_COMPILE_OPT | PARSE_FLAG_UNICODE); @@ -359,7 +410,7 @@ parse_options(Eterm listp, /* in */ if (compile_options != NULL) { *compile_options = copt; } - if (exec_options != NULL) { + if (exec_options != NULL) { *exec_options = eopt; } if (flags != NULL) { @@ -373,34 +424,49 @@ parse_options(Eterm listp, /* in */ */ static Eterm -build_compile_result(Process *p, Eterm error_tag, pcre *result, int errcode, const char *errstr, int errofset, int unicode, int with_ok) +build_compile_result(Process *p, Eterm error_tag, pcre *result, int errcode, const char *errstr, int errofset, int unicode, int with_ok, Eterm extra_err_tag) { Eterm *hp; Eterm ret; size_t pattern_size; int capture_count; + int use_crlf; + unsigned long options; if (!result) { /* Return {error_tag, {Code, String, Offset}} */ int elen = sys_strlen(errstr); int need = 3 /* tuple of 2 */ + 3 /* tuple of 2 */ + - (2 * elen) /* The error string list */; + (2 * elen) /* The error string list */ + + ((extra_err_tag != NIL) ? 3 : 0); hp = HAlloc(p, need); ret = buf_to_intlist(&hp, (char *) errstr, elen, NIL); ret = TUPLE2(hp, ret, make_small(errofset)); hp += 3; + if (extra_err_tag != NIL) { + /* Return {error_tag, {extra_tag, + {Code, String, Offset}}} instead */ + ret = TUPLE2(hp, extra_err_tag, ret); + hp += 3; + } ret = TUPLE2(hp, error_tag, ret); } else { erts_pcre_fullinfo(result, NULL, PCRE_INFO_SIZE, &pattern_size); erts_pcre_fullinfo(result, NULL, PCRE_INFO_CAPTURECOUNT, &capture_count); + erts_pcre_fullinfo(result, NULL, PCRE_INFO_OPTIONS, &options); + options &= PCRE_NEWLINE_CR|PCRE_NEWLINE_LF | PCRE_NEWLINE_CRLF | + PCRE_NEWLINE_ANY | PCRE_NEWLINE_ANYCRLF; + use_crlf = (options == PCRE_NEWLINE_ANY || + options == PCRE_NEWLINE_CRLF || + options == PCRE_NEWLINE_ANYCRLF); /* XXX: Optimize - keep in offheap binary to allow this to be kept across traps w/o need of copying */ ret = new_binary(p, (byte *) result, pattern_size); erts_pcre_free(result); - hp = HAlloc(p, (with_ok) ? (3+5) : 5); - ret = TUPLE4(hp,am_re_pattern, make_small(capture_count), make_small(unicode),ret); + hp = HAlloc(p, (with_ok) ? (3+6) : 6); + ret = TUPLE5(hp,am_re_pattern, make_small(capture_count), make_small(unicode),make_small(use_crlf),ret); if (with_ok) { - hp += 5; + hp += 6; ret = TUPLE2(hp,am_ok,ret); } } @@ -424,9 +490,12 @@ re_compile(Process* p, Eterm arg1, Eterm arg2) int options = 0; int pflags = 0; int unicode = 0; +#ifdef DEBUG + int buffres; +#endif - if (parse_options(arg2,&options,NULL,&pflags,NULL,NULL) + if (parse_options(arg2,&options,NULL,&pflags,NULL,NULL,NULL,NULL) < 0) { BIF_ERROR(p,BADARG); } @@ -445,16 +514,19 @@ re_compile(Process* p, Eterm arg1, Eterm arg2) BIF_ERROR(p,BADARG); } expr = erts_alloc(ERTS_ALC_T_RE_TMP_BUF, slen + 1); - if (erts_iolist_to_buf(arg1, expr, slen) != 0) { - erts_free(ERTS_ALC_T_RE_TMP_BUF, expr); - BIF_ERROR(p,BADARG); - } +#ifdef DEBUG + buffres = +#endif + erts_iolist_to_buf(arg1, expr, slen); + + ASSERT(buffres >= 0); + expr[slen]='\0'; result = erts_pcre_compile2(expr, options, &errcode, &errstr, &errofset, default_table); ret = build_compile_result(p, am_error, result, errcode, - errstr, errofset, unicode, 1); + errstr, errofset, unicode, 1, NIL); erts_free(ERTS_ALC_T_RE_TMP_BUF, expr); BIF_RET(ret); } @@ -492,7 +564,7 @@ typedef struct _return_info { } ReturnInfo; typedef struct _restart_context { - pcre_extra extra; + erts_pcre_extra extra; void *restart_data; Uint32 flags; char *subject; /* to be able to free it when done */ @@ -502,6 +574,7 @@ typedef struct _restart_context { } RestartContext; #define RESTART_FLAG_SUBJECT_IN_BINARY 0x1 +#define RESTART_FLAG_REPORT_MATCH_LIMIT 0x2 static void cleanup_restart_context(RestartContext *rc) { @@ -542,13 +615,29 @@ static Eterm build_exec_return(Process *p, int rc, RestartContext *restartp, Ete Eterm res; Eterm *hp; if (rc <= 0) { - res = am_nomatch; + if (restartp->flags & RESTART_FLAG_REPORT_MATCH_LIMIT) { + if (rc == PCRE_ERROR_MATCHLIMIT) { + hp = HAlloc(p,3); + res = TUPLE2(hp,am_error,am_match_limit); + } else if (rc == PCRE_ERROR_RECURSIONLIMIT) { + hp = HAlloc(p,3); + res = TUPLE2(hp,am_error,am_match_limit_recursion); + } else { + res = am_nomatch; + } + } else { + res = am_nomatch; + } } else { - ReturnInfo *ri = restartp->ret_info; + ReturnInfo *ri; ReturnInfo defri = {RetIndex,0,{0}}; - if (ri == NULL) { + + if (restartp->ret_info == NULL) { ri = &defri; + } else { + ri = restartp->ret_info; } + if (ri->type == RetNone) { res = am_match; } else if (ri->type == RetIndex){ @@ -577,6 +666,17 @@ static Eterm build_exec_return(Process *p, int rc, RestartContext *restartp, Ete ri->num_spec * 2 * sizeof(Eterm)); for (i = 0; i < ri->num_spec; ++i) { x = ri->v[i]; + if (x < -1) { + int n = i-x+1; + int j; + for (j = i+1; j < ri->num_spec && j < n; ++j) { + if (restartp->ovector[(ri->v[j])*2] >= 0) { + x = ri->v[j]; + break; + } + } + i = n-1; + } if (x < rc && x >= 0) { tmp_vect[n*2] = make_signed_integer(restartp->ovector[x*2],p); tmp_vect[n*2+1] = make_signed_integer(restartp->ovector[x*2+1]-restartp->ovector[x*2],p); @@ -658,6 +758,17 @@ static Eterm build_exec_return(Process *p, int rc, RestartContext *restartp, Ete ri->num_spec * sizeof(Eterm)); for (i = 0; i < ri->num_spec; ++i) { x = ri->v[i]; + if (x < -1) { + int n = i-x+1; + int j; + for (j = i+1; j < ri->num_spec && j < n; ++j) { + if (restartp->ovector[(ri->v[j])*2] >= 0) { + x = ri->v[j]; + break; + } + } + i = n-1; + } if (x < rc && x >= 0) { char *cp; int len; @@ -722,6 +833,49 @@ static Eterm build_exec_return(Process *p, int rc, RestartContext *restartp, Ete */ #define RINFO_SIZ(Num) (sizeof(ReturnInfo) + (sizeof(int) * (Num - 1))) +#define PICK_INDEX(NameEntry) \ + ((int) ((((unsigned) ((unsigned char *) (NameEntry))[0]) << 8) + \ + ((unsigned) ((unsigned char *) (NameEntry))[1]))) + + +static void build_one_capture(const pcre *code, ReturnInfo **ri, int *sallocated, int has_dupnames, char *name) +{ + ReturnInfo *r = (*ri); + if (has_dupnames) { + /* Build a sequence of positions, starting with -size if + more than one, otherwise just put the index there... */ + char *first,*last; + int esize = erts_pcre_get_stringtable_entries(code,name,&first,&last); + if (esize == PCRE_ERROR_NOSUBSTRING) { + r->v[r->num_spec - 1] = -1; + } else if(last == first) { + r->v[r->num_spec - 1] = PICK_INDEX(first); + } else { + int num = ((last - first) / esize) + 1; + int i; + ASSERT(num > 1); + r->v[r->num_spec - 1] = -num; /* A value less than -1 means + multiple indexes for same name */ + for (i = 0; i < num; ++i) { + ++(r->num_spec); + if(r->num_spec > (*sallocated)) { + (*sallocated) += 10; + r = erts_realloc(ERTS_ALC_T_RE_SUBJECT, r, + RINFO_SIZ((*sallocated))); + } + r->v[r->num_spec - 1] = PICK_INDEX(first); + first += esize; + } + } + } else { + /* Use the faster binary search if no duplicate names are present */ + if ((r->v[r->num_spec - 1] = erts_pcre_get_stringnumber(code,name)) == + PCRE_ERROR_NOSUBSTRING) { + r->v[r->num_spec - 1] = -1; + } + } + *ri = r; +} static ReturnInfo * build_capture(Eterm capture_spec[CAPSPEC_SIZE], const pcre *code) @@ -770,13 +924,58 @@ build_capture(Eterm capture_spec[CAPSPEC_SIZE], const pcre *code) } ri->v[ri->num_spec - 1] = 0; break; + case am_all_names: + { + int rc,i,top; + int entrysize; + unsigned char *nametable, *last = NULL; + int has_dupnames; + unsigned long options; + + if (erts_pcre_fullinfo(code, NULL, PCRE_INFO_OPTIONS, &options) != 0) + goto error; + if ((rc = erts_pcre_fullinfo(code, NULL, PCRE_INFO_NAMECOUNT, &top)) != 0) + goto error; + if (top <= 0) { + ri->num_spec = 0; + ri->type = RetNone; + break; + } + if (erts_pcre_fullinfo(code, NULL, PCRE_INFO_NAMEENTRYSIZE, &entrysize) != 0) + goto error; + if (erts_pcre_fullinfo(code, NULL, PCRE_INFO_NAMETABLE, &nametable) != 0) + goto error; + + has_dupnames = ((options & PCRE_DUPNAMES) != 0); + + for(i=0;i<top;++i) { + if (last == NULL || !has_dupnames || strcmp((char *) last+2,(char *) nametable+2)) { + ASSERT(ri->num_spec >= 0); + ++(ri->num_spec); + if(ri->num_spec > sallocated) { + sallocated += 10; + ri = erts_realloc(ERTS_ALC_T_RE_SUBJECT, ri, RINFO_SIZ(sallocated)); + } + if (has_dupnames) { + /* This could be more effective, we actually have + the names and could fill in the vector + immediately. Now we lookup the name again. */ + build_one_capture(code,&ri,&sallocated,has_dupnames,(char *) nametable+2); + } else { + ri->v[ri->num_spec - 1] = PICK_INDEX(nametable); + } + } + last = nametable; + nametable += entrysize; + } + break; + } default: if (is_list(capture_spec[CAPSPEC_VALUES])) { for(l=capture_spec[CAPSPEC_VALUES];is_list(l);l = CDR(list_val(l))) { int x; Eterm val = CAR(list_val(l)); - if (ri->num_spec < 0) - ri->num_spec = 0; + ASSERT(ri->num_spec >= 0); ++(ri->num_spec); if(ri->num_spec > sallocated) { sallocated += 10; @@ -785,6 +984,11 @@ build_capture(Eterm capture_spec[CAPSPEC_SIZE], const pcre *code) if (term_to_int(val,&x)) { ri->v[ri->num_spec - 1] = x; } else if (is_atom(val) || is_binary(val) || is_list(val)) { + int has_dupnames; + unsigned long options; + if (erts_pcre_fullinfo(code, NULL, PCRE_INFO_OPTIONS, &options) != 0) + goto error; + has_dupnames = ((options & PCRE_DUPNAMES) != 0); if (is_atom(val)) { Atom *ap = atom_tab(atom_val(val)); if ((ap->len + 1) > tmpbsiz) { @@ -799,6 +1003,10 @@ build_capture(Eterm capture_spec[CAPSPEC_SIZE], const pcre *code) tmpb[ap->len] = '\0'; } else { ErlDrvSizeT slen; +#ifdef DEBUG + int buffres; +#endif + if (erts_iolist_size(val, &slen)) { goto error; } @@ -810,15 +1018,15 @@ build_capture(Eterm capture_spec[CAPSPEC_SIZE], const pcre *code) (tmpbsiz = slen + 1)); } } - if (erts_iolist_to_buf(val, tmpb, slen) != 0) { - goto error; - } + +#ifdef DEBUG + buffres = +#endif + erts_iolist_to_buf(val, tmpb, slen); + ASSERT(buffres >= 0); tmpb[slen] = '\0'; } - if ((ri->v[ri->num_spec - 1] = erts_pcre_get_stringnumber(code,tmpb)) == - PCRE_ERROR_NOSUBSTRING) { - ri->v[ri->num_spec - 1] = -1; - } + build_one_capture(code,&ri,&sallocated,has_dupnames,tmpb); } else { goto error; } @@ -867,15 +1075,18 @@ re_run(Process *p, Eterm arg1, Eterm arg2, Eterm arg3) unsigned long loop_count; Eterm capture[CAPSPEC_SIZE] = CAPSPEC_INIT; int is_list_cap; + int match_limit = 0; + int match_limit_recursion = 0; - if (parse_options(arg3,&comp_options,&options,&pflags,&startoffset,capture) + if (parse_options(arg3,&comp_options,&options,&pflags,&startoffset,capture, + &match_limit,&match_limit_recursion) < 0) { BIF_ERROR(p,BADARG); } is_list_cap = ((pflags & PARSE_FLAG_CAPTURE_OPT) && (capture[CAPSPEC_TYPE] == am_list)); - if (is_not_tuple(arg2) || (arityval(*tuple_val(arg2)) != 4)) { + if (is_not_tuple(arg2) || (arityval(*tuple_val(arg2)) != 5)) { if (is_binary(arg2) || is_list(arg2) || is_nil(arg2)) { /* Compile from textual RE */ ErlDrvSizeT slen; @@ -885,6 +1096,9 @@ re_run(Process *p, Eterm arg1, Eterm arg2, Eterm arg3) const char *errstr = ""; int errofset = 0; int capture_count; +#ifdef DEBUG + int buffres; +#endif if (pflags & PARSE_FLAG_UNICODE && (!is_binary(arg2) || !is_binary(arg1) || @@ -897,18 +1111,32 @@ re_run(Process *p, Eterm arg1, Eterm arg2, Eterm arg3) } expr = erts_alloc(ERTS_ALC_T_RE_TMP_BUF, slen + 1); - if (erts_iolist_to_buf(arg2, expr, slen) != 0) { - erts_free(ERTS_ALC_T_RE_TMP_BUF, expr); - BIF_ERROR(p,BADARG); - } + +#ifdef DEBUG + buffres = +#endif + erts_iolist_to_buf(arg2, expr, slen); + + ASSERT(buffres >= 0); + expr[slen]='\0'; result = erts_pcre_compile2(expr, comp_options, &errcode, &errstr, &errofset, default_table); if (!result) { - erts_free(ERTS_ALC_T_RE_TMP_BUF, expr); /* Compilation error gives badarg except in the compile - function */ - BIF_ERROR(p,BADARG); + function or if we have PARSE_FLAG_REPORT_ERRORS */ + if (pflags & PARSE_FLAG_REPORT_ERRORS) { + res = build_compile_result(p, am_error, result, errcode, + errstr, errofset, + (pflags & + PARSE_FLAG_UNICODE) ? 1 : 0, + 1, am_compile); + erts_free(ERTS_ALC_T_RE_TMP_BUF, expr); + BIF_RET(res); + } else { + erts_free(ERTS_ALC_T_RE_TMP_BUF, expr); + BIF_ERROR(p,BADARG); + } } if (pflags & PARSE_FLAG_GLOBAL) { Eterm precompiled = @@ -917,7 +1145,7 @@ re_run(Process *p, Eterm arg1, Eterm arg2, Eterm arg3) errstr, errofset, (pflags & PARSE_FLAG_UNICODE) ? 1 : 0, - 0); + 0, NIL); Eterm *hp,r; erts_free(ERTS_ALC_T_RE_TMP_BUF, expr); hp = HAlloc(p,4); @@ -947,7 +1175,8 @@ re_run(Process *p, Eterm arg1, Eterm arg2, Eterm arg3) tp = tuple_val(arg2); if (tp[1] != am_re_pattern || is_not_small(tp[2]) || - is_not_small(tp[3]) || is_not_binary(tp[4])) { + is_not_small(tp[3]) || is_not_small(tp[4]) || + is_not_binary(tp[5])) { BIF_ERROR(p,BADARG); } @@ -967,9 +1196,9 @@ re_run(Process *p, Eterm arg1, Eterm arg2, Eterm arg3) } ovsize = 3*(unsigned_val(tp[2])+1); - code_size = binary_size(tp[4]); - if ((code_tmp = (const pcre *) - erts_get_aligned_binary_bytes(tp[4], &temp_alloc)) == NULL) { + code_size = binary_size(tp[5]); + code_tmp = (const pcre *) erts_get_aligned_binary_bytes(tp[5], &temp_alloc); + if (code_tmp == NULL || code_size < 4) { erts_free_aligned_binary_bytes(temp_alloc); BIF_ERROR(p, BADARG); } @@ -994,6 +1223,16 @@ re_run(Process *p, Eterm arg1, Eterm arg2, Eterm arg3) restart.extra.restart_flags = 0; restart.extra.loop_counter_return = &loop_count; restart.ret_info = NULL; + + if (pflags & PARSE_FLAG_MATCH_LIMIT) { + restart.extra.flags |= PCRE_EXTRA_MATCH_LIMIT; + restart.extra.match_limit = match_limit; + } + + if (pflags & PARSE_FLAG_MATCH_LIMIT_RECURSION) { + restart.extra.flags |= PCRE_EXTRA_MATCH_LIMIT_RECURSION; + restart.extra.match_limit_recursion = match_limit_recursion; + } if (pflags & PARSE_FLAG_CAPTURE_OPT) { if ((restart.ret_info = build_capture(capture,restart.code)) == NULL) { @@ -1002,7 +1241,7 @@ re_run(Process *p, Eterm arg1, Eterm arg2, Eterm arg3) BIF_ERROR(p,BADARG); } } - + /* Optimized - if already in binary off heap, keep that and avoid copying, also binary returns can be sub binaries in that case */ @@ -1029,6 +1268,9 @@ re_run(Process *p, Eterm arg1, Eterm arg2, Eterm arg3) restart.subject = (char *) (pb->bytes+offset); restart.flags |= RESTART_FLAG_SUBJECT_IN_BINARY; } else { +#ifdef DEBUG + int buffres; +#endif handle_iolist: if (erts_iolist_size(arg1, &slength)) { erts_free(ERTS_ALC_T_RE_SUBJECT, restart.ovector); @@ -1040,24 +1282,30 @@ handle_iolist: } restart.subject = erts_alloc(ERTS_ALC_T_RE_SUBJECT, slength); - if (erts_iolist_to_buf(arg1, restart.subject, slength) != 0) { - erts_free(ERTS_ALC_T_RE_SUBJECT, restart.ovector); - erts_free(ERTS_ALC_T_RE_SUBJECT, restart.code); - erts_free(ERTS_ALC_T_RE_SUBJECT, restart.subject); - if (restart.ret_info != NULL) { - erts_free(ERTS_ALC_T_RE_SUBJECT, restart.ret_info); - } - BIF_ERROR(p,BADARG); - } +#ifdef DEBUG + buffres = +#endif + erts_iolist_to_buf(arg1, restart.subject, slength); + ASSERT(buffres >= 0); } + if (pflags & PARSE_FLAG_REPORT_ERRORS) { + restart.flags |= RESTART_FLAG_REPORT_MATCH_LIMIT; + } #ifdef DEBUG loop_count = 0xFFFFFFFF; #endif + + rc = erts_pcre_exec(restart.code, &(restart.extra), restart.subject, + slength, startoffset, + options, restart.ovector, ovsize); + + if (rc == PCRE_ERROR_BADENDIANNESS || rc == PCRE_ERROR_BADMAGIC) { + cleanup_restart_context(&restart); + BIF_ERROR(p,BADARG); + } - rc = erts_pcre_exec(restart.code, &(restart.extra), restart.subject, slength, startoffset, - options, restart.ovector, ovsize); ASSERT(loop_count != 0xFFFFFFFF); BUMP_REDS(p, loop_count / LOOP_FACTOR); if (rc == PCRE_ERROR_LOOP_LIMIT) { @@ -1077,7 +1325,7 @@ handle_iolist: arg2 /* To avoid GC of precompiled code, XXX: not utilized yet */, magic_bin); } - + res = build_exec_return(p, rc, &restart, arg1); cleanup_restart_context(&restart); @@ -1149,6 +1397,120 @@ static BIF_RETTYPE re_exec_trap(BIF_ALIST_3) BIF_RET(res); } +BIF_RETTYPE +re_inspect_2(BIF_ALIST_2) +{ + Eterm *tp,*tmp_vec,*hp; + int i,top,j; + int entrysize; + unsigned char *nametable, *last,*name; + int has_dupnames; + unsigned long options; + int num_names; + Eterm res; + const pcre *code; + byte *temp_alloc = NULL; +#ifdef DEBUG + int infores; +#endif + + + if (is_not_tuple(BIF_ARG_1) || (arityval(*tuple_val(BIF_ARG_1)) != 5)) { + goto error; + } + tp = tuple_val(BIF_ARG_1); + if (tp[1] != am_re_pattern || is_not_small(tp[2]) || + is_not_small(tp[3]) || is_not_small(tp[4]) || + is_not_binary(tp[5])) { + goto error; + } + if (BIF_ARG_2 != am_namelist) { + goto error; + } + if ((code = (const pcre *) + erts_get_aligned_binary_bytes(tp[5], &temp_alloc)) == NULL) { + goto error; + } + + /* OK, so let's try to get some info */ + + if (erts_pcre_fullinfo(code, NULL, PCRE_INFO_OPTIONS, &options) != 0) + goto error; + +#ifdef DEBUG + infores = +#endif + erts_pcre_fullinfo(code, NULL, PCRE_INFO_NAMECOUNT, &top); + + ASSERT(infores == 0); + + if (top <= 0) { + hp = HAlloc(BIF_P, 3); + res = TUPLE2(hp,am_namelist,NIL); + erts_free_aligned_binary_bytes(temp_alloc); + BIF_RET(res); + } +#ifdef DEBUG + infores = +#endif + erts_pcre_fullinfo(code, NULL, PCRE_INFO_NAMEENTRYSIZE, &entrysize); + + ASSERT(infores == 0); + +#ifdef DEBUG + infores = +#endif + erts_pcre_fullinfo(code, NULL, PCRE_INFO_NAMETABLE, &nametable); + + ASSERT(infores == 0); + + has_dupnames = ((options & PCRE_DUPNAMES) != 0); + /* First, count the names */ + num_names = 0; + last = NULL; + name = nametable; + for(i=0;i<top;++i) { + if (last == NULL || !has_dupnames || strcmp((char *) last+2, + (char *) name+2)) { + ++num_names; + } + last = name; + name += entrysize; + } + tmp_vec = erts_alloc(ERTS_ALC_T_RE_TMP_BUF, + num_names * sizeof(Eterm)); + /* Re-iterate and fill tmp_vec */ + last = NULL; + name = nametable; + j = 0; + for(i=0;i<top;++i) { + if (last == NULL || !has_dupnames || strcmp((char *) last+2, + (char *) name+2)) { + tmp_vec[j++] = new_binary(BIF_P, (byte *) name+2, strlen((char *) name+2)); + } + last = name; + name += entrysize; + } + ASSERT(j == num_names); + hp = HAlloc(BIF_P, 3+2*j); + res = NIL; + for(i = j-1 ;i >= 0; --i) { + res = CONS(hp,tmp_vec[i],res); + hp += 2; + } + res = TUPLE2(hp,am_namelist,res); + erts_free_aligned_binary_bytes(temp_alloc); + erts_free(ERTS_ALC_T_RE_TMP_BUF, tmp_vec); + BIF_RET(res); + + error: + /* tmp_vec never allocated when we reach here */ + erts_free_aligned_binary_bytes(temp_alloc); + BIF_ERROR(BIF_P,BADARG); +} + + + diff --git a/erts/emulator/beam/erl_bif_timer.c b/erts/emulator/beam/erl_bif_timer.c deleted file mode 100644 index d67695e533..0000000000 --- a/erts/emulator/beam/erl_bif_timer.c +++ /dev/null @@ -1,705 +0,0 @@ -/* - * %CopyrightBegin% - * - * Copyright Ericsson AB 2005-2012. All Rights Reserved. - * - * The contents of this file are subject to the Erlang Public License, - * Version 1.1, (the "License"); you may not use this file except in - * compliance with the License. You should have received a copy of the - * Erlang Public License along with this software. If not, it can be - * retrieved online at http://www.erlang.org/. - * - * Software distributed under the License is distributed on an "AS IS" - * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See - * the License for the specific language governing rights and limitations - * under the License. - * - * %CopyrightEnd% - */ - -#ifdef HAVE_CONFIG_H -# include "config.h" -#endif - -#include "erl_bif_timer.h" -#include "global.h" -#include "bif.h" -#include "error.h" -#include "big.h" -#include "erl_thr_progress.h" - -/**************************************************************************** -** BIF Timer support -****************************************************************************/ - -#define BTM_FLG_SL_TIMER (((Uint32) 1) << 0) -#define BTM_FLG_CANCELED (((Uint32) 1) << 1) -#define BTM_FLG_HEAD (((Uint32) 1) << 2) -#define BTM_FLG_BYNAME (((Uint32) 1) << 3) -#define BTM_FLG_WRAP (((Uint32) 1) << 4) - -struct ErtsBifTimer_ { - struct { - union { - ErtsBifTimer **head; - ErtsBifTimer *prev; - } u; - ErtsBifTimer *next; - } tab; - union { - Eterm name; - struct { - ErtsBifTimer *prev; - ErtsBifTimer *next; - Process *ess; - } proc; - } receiver; - ErlTimer tm; - ErlHeapFragment* bp; - Uint32 flags; - Eterm message; - Uint32 ref_numbers[ERTS_REF_NUMBERS]; -}; - -#ifdef SMALL_MEMORY -#define TIMER_HASH_VEC_SZ 3331 -#define BTM_PREALC_SZ 10 -#else -#define TIMER_HASH_VEC_SZ 10007 -#define BTM_PREALC_SZ 100 -#endif -static ErtsBifTimer **bif_timer_tab; -static Uint no_bif_timers; - - -static erts_smp_rwmtx_t bif_timer_lock; - -#define erts_smp_safe_btm_rwlock(P, L) \ - safe_btm_lock((P), (L), 1) -#define erts_smp_safe_btm_rlock(P, L) \ - safe_btm_lock((P), (L), 0) -#define erts_smp_btm_rwlock() \ - erts_smp_rwmtx_rwlock(&bif_timer_lock) -#define erts_smp_btm_tryrwlock() \ - erts_smp_rwmtx_tryrwlock(&bif_timer_lock) -#define erts_smp_btm_rwunlock() \ - erts_smp_rwmtx_rwunlock(&bif_timer_lock) -#define erts_smp_btm_rlock() \ - erts_smp_rwmtx_rlock(&bif_timer_lock) -#define erts_smp_btm_tryrlock() \ - erts_smp_rwmtx_tryrlock(&bif_timer_lock) -#define erts_smp_btm_runlock() \ - erts_smp_rwmtx_runlock(&bif_timer_lock) -#define erts_smp_btm_lock_init() \ - erts_smp_rwmtx_init(&bif_timer_lock, "bif_timers") - - -static ERTS_INLINE int -safe_btm_lock(Process *c_p, ErtsProcLocks c_p_locks, int rw_lock) -{ - ASSERT(c_p && c_p_locks); -#ifdef ERTS_SMP - if ((rw_lock ? erts_smp_btm_tryrwlock() : erts_smp_btm_tryrlock()) != EBUSY) - return 0; - erts_smp_proc_unlock(c_p, c_p_locks); - if (rw_lock) - erts_smp_btm_rwlock(); - else - erts_smp_btm_rlock(); - erts_smp_proc_lock(c_p, c_p_locks); - if (ERTS_PROC_IS_EXITING(c_p)) { - if (rw_lock) - erts_smp_btm_rwunlock(); - else - erts_smp_btm_runlock(); - return 1; - } -#endif - return 0; -} - -ERTS_SCHED_PREF_PALLOC_IMPL(btm_pre, ErtsBifTimer, BTM_PREALC_SZ) - -static ERTS_INLINE int -get_index(Uint32 *ref_numbers, Uint32 len) -{ - Uint32 hash; - /* len can potentially be larger than ERTS_REF_NUMBERS - if it has visited another node... */ - if (len > ERTS_REF_NUMBERS) - len = ERTS_REF_NUMBERS; - -#if ERTS_REF_NUMBERS != 3 -#error "ERTS_REF_NUMBERS changed. Update me..." -#endif - switch (len) { - case 3: if (!ref_numbers[2]) len = 2; - case 2: if (!ref_numbers[1]) len = 1; - default: break; - } - - ASSERT(1 <= len && len <= ERTS_REF_NUMBERS); - - hash = block_hash((byte *) ref_numbers, len * sizeof(Uint32), 0x08d12e65); - return (int) (hash % ((Uint32) TIMER_HASH_VEC_SZ)); -} - -static Eterm -create_ref(Uint *hp, Uint32 *ref_numbers, Uint32 len) -{ - Uint32 *datap; - int i; - - - if (len > ERTS_MAX_REF_NUMBERS) { - /* Such large refs should no be able to appear in the emulator */ - erl_exit(1, "%s:%d: Internal error\n", __FILE__, __LINE__); - } - -#if defined(ARCH_64) && !HALFWORD_HEAP - hp[0] = make_ref_thing_header(len/2 + 1); - datap = (Uint32 *) &hp[1]; - *(datap++) = len; -#else - hp[0] = make_ref_thing_header(len); - datap = (Uint32 *) &hp[1]; -#endif - - for (i = 0; i < len; i++) - datap[i] = ref_numbers[i]; - - return make_internal_ref(hp); -} - -static int -eq_non_standard_ref_numbers(Uint32 *rn1, Uint32 len1, Uint32 *rn2, Uint32 len2) -{ -#if defined(ARCH_64) && !HALFWORD_HEAP -#define MAX_REF_HEAP_SZ (1+(ERTS_MAX_REF_NUMBERS/2+1)) -#else -#define MAX_REF_HEAP_SZ (1+ERTS_MAX_REF_NUMBERS) -#endif - DeclareTmpHeapNoproc(r1_hp,(MAX_REF_HEAP_SZ * 2)); - Eterm *r2_hp = r1_hp +MAX_REF_HEAP_SZ; - - return eq(create_ref(r1_hp, rn1, len1), create_ref(r2_hp, rn2, len2)); -#undef MAX_REF_HEAP_SZ -} - -static ERTS_INLINE int -eq_ref_numbers(Uint32 *rn1, Uint32 len1, Uint32 *rn2, Uint32 len2) -{ - int res; - if (len1 != ERTS_REF_NUMBERS || len2 != ERTS_REF_NUMBERS) { - /* Can potentially happen, but will never... */ - return eq_non_standard_ref_numbers(rn1, len1, rn2, len2); - } - -#if ERTS_REF_NUMBERS != 3 -#error "ERTS_REF_NUMBERS changed. Update me..." -#endif - res = rn1[0] == rn2[0] && rn1[1] == rn2[1] && rn1[2] == rn2[2]; - - ASSERT(res - ? eq_non_standard_ref_numbers(rn1, len1, rn2, len2) - : !eq_non_standard_ref_numbers(rn1, len1, rn2, len2)); - - return res; -} - -static ERTS_INLINE ErtsBifTimer * -tab_find(Eterm ref) -{ - Uint32 *ref_numbers = internal_ref_numbers(ref); - Uint32 ref_numbers_len = internal_ref_no_of_numbers(ref); - int ix = get_index(ref_numbers, ref_numbers_len); - ErtsBifTimer* btm; - - for (btm = bif_timer_tab[ix]; btm; btm = btm->tab.next) - if (eq_ref_numbers(ref_numbers, ref_numbers_len, - btm->ref_numbers, ERTS_REF_NUMBERS)) - return btm; - return NULL; -} - -static ERTS_INLINE void -tab_remove(ErtsBifTimer* btm) -{ - if (btm->flags & BTM_FLG_HEAD) { - *btm->tab.u.head = btm->tab.next; - if (btm->tab.next) { - btm->tab.next->flags |= BTM_FLG_HEAD; - btm->tab.next->tab.u.head = btm->tab.u.head; - } - } - else { - btm->tab.u.prev->tab.next = btm->tab.next; - if (btm->tab.next) - btm->tab.next->tab.u.prev = btm->tab.u.prev; - } - btm->flags |= BTM_FLG_CANCELED; - ASSERT(no_bif_timers > 0); - no_bif_timers--; -} - -static ERTS_INLINE void -tab_insert(ErtsBifTimer* btm) -{ - int ix = get_index(btm->ref_numbers, ERTS_REF_NUMBERS); - ErtsBifTimer* btm_list = bif_timer_tab[ix]; - - if (btm_list) { - btm_list->flags &= ~BTM_FLG_HEAD; - btm_list->tab.u.prev = btm; - } - - btm->flags |= BTM_FLG_HEAD; - btm->tab.u.head = &bif_timer_tab[ix]; - btm->tab.next = btm_list; - bif_timer_tab[ix] = btm; - no_bif_timers++; -} - -static ERTS_INLINE void -link_proc(Process *p, ErtsBifTimer* btm) -{ - btm->receiver.proc.ess = p; - btm->receiver.proc.prev = NULL; - btm->receiver.proc.next = p->u.bif_timers; - if (p->u.bif_timers) - p->u.bif_timers->receiver.proc.prev = btm; - p->u.bif_timers = btm; -} - -static ERTS_INLINE void -unlink_proc(ErtsBifTimer* btm) -{ - if (btm->receiver.proc.prev) - btm->receiver.proc.prev->receiver.proc.next = btm->receiver.proc.next; - else - btm->receiver.proc.ess->u.bif_timers = btm->receiver.proc.next; - if (btm->receiver.proc.next) - btm->receiver.proc.next->receiver.proc.prev = btm->receiver.proc.prev; -} - -static void -bif_timer_cleanup(ErtsBifTimer* btm) -{ - ASSERT(btm); - - if (btm->bp) - free_message_buffer(btm->bp); - - if (!btm_pre_free(btm)) { - if (btm->flags & BTM_FLG_SL_TIMER) - erts_free(ERTS_ALC_T_SL_BIF_TIMER, (void *) btm); - else - erts_free(ERTS_ALC_T_LL_BIF_TIMER, (void *) btm); - } -} - -static void -bif_timer_timeout(ErtsBifTimer* btm) -{ - ASSERT(btm); - - - erts_smp_btm_rwlock(); - - if (btm->flags & BTM_FLG_CANCELED) { - /* - * A concurrent cancel is ongoing. Do not send the timeout message, - * but cleanup here since the cancel call-back won't be called. - */ -#ifndef ERTS_SMP - ASSERT(0); -#endif - } - else { - ErtsProcLocks rp_locks = 0; - Process* rp; - - tab_remove(btm); - - ASSERT(!erts_get_current_process()); - - if (btm->flags & BTM_FLG_BYNAME) - rp = erts_whereis_process(NULL, 0, btm->receiver.name, 0, 0); - else { - rp = btm->receiver.proc.ess; - unlink_proc(btm); - } - - if (rp) { - Eterm message; - ErlHeapFragment *bp; - - bp = btm->bp; - btm->bp = NULL; /* Prevent cleanup of message buffer... */ - - if (!(btm->flags & BTM_FLG_WRAP)) - message = btm->message; - else { -#if ERTS_REF_NUMBERS != 3 -#error "ERTS_REF_NUMBERS changed. Update me..." -#endif - Eterm ref; - Uint *hp; - Uint wrap_size = REF_THING_SIZE + 4; - message = btm->message; - - if (!bp) { - ErlOffHeap *ohp; - ASSERT(is_immed(message)); - hp = erts_alloc_message_heap(wrap_size, - &bp, - &ohp, - rp, - &rp_locks); - } else { - Eterm old_size = bp->used_size; - bp = erts_resize_message_buffer(bp, old_size + wrap_size, - &message, 1); - hp = &bp->mem[0] + old_size; - } - - write_ref_thing(hp, - btm->ref_numbers[0], - btm->ref_numbers[1], - btm->ref_numbers[2]); - ref = make_internal_ref(hp); - hp += REF_THING_SIZE; - message = TUPLE3(hp, am_timeout, ref, message); - } - - erts_queue_message(rp, &rp_locks, bp, message, NIL -#ifdef USE_VM_PROBES - , NIL -#endif - ); - erts_smp_proc_unlock(rp, rp_locks); - } - } - - erts_smp_btm_rwunlock(); - - bif_timer_cleanup(btm); -} - -static Eterm -setup_bif_timer(Uint32 xflags, - Process *c_p, - Eterm time, - Eterm receiver, - Eterm message) -{ - Process *rp; - ErtsBifTimer* btm; - Uint timeout; - Eterm ref; - Uint32 *ref_numbers; - - if (!term_to_Uint(time, &timeout)) - return THE_NON_VALUE; -#if defined(ARCH_64) && !HALFWORD_HEAP - if ((timeout >> 32) != 0) - return THE_NON_VALUE; -#endif - if (is_not_internal_pid(receiver) && is_not_atom(receiver)) - return THE_NON_VALUE; - - ref = erts_make_ref(c_p); - - if (is_atom(receiver)) - rp = NULL; - else { - rp = erts_pid2proc(c_p, ERTS_PROC_LOCK_MAIN, - receiver, ERTS_PROC_LOCK_MSGQ); - if (!rp) - return ref; - } - - if (timeout < ERTS_ALC_MIN_LONG_LIVED_TIME) { - if (timeout < 1000) { - btm = btm_pre_alloc(); - if (!btm) - goto sl_timer_alloc; - btm->flags = 0; - } - else { - sl_timer_alloc: - btm = (ErtsBifTimer *) erts_alloc(ERTS_ALC_T_SL_BIF_TIMER, - sizeof(ErtsBifTimer)); - btm->flags = BTM_FLG_SL_TIMER; - } - } - else { - btm = (ErtsBifTimer *) erts_alloc(ERTS_ALC_T_LL_BIF_TIMER, - sizeof(ErtsBifTimer)); - btm->flags = 0; - } - - if (rp) { - link_proc(rp, btm); - erts_smp_proc_unlock(rp, ERTS_PROC_LOCK_MSGQ); - } - else { - ASSERT(is_atom(receiver)); - btm->receiver.name = receiver; - btm->flags |= BTM_FLG_BYNAME; - } - - btm->flags |= xflags; - - ref_numbers = internal_ref_numbers(ref); - ASSERT(internal_ref_no_of_numbers(ref) == 3); -#if ERTS_REF_NUMBERS != 3 -#error "ERTS_REF_NUMBERS changed. Update me..." -#endif - btm->ref_numbers[0] = ref_numbers[0]; - btm->ref_numbers[1] = ref_numbers[1]; - btm->ref_numbers[2] = ref_numbers[2]; - - ASSERT(eq_ref_numbers(btm->ref_numbers, ERTS_REF_NUMBERS, - ref_numbers, ERTS_REF_NUMBERS)); - - if (is_immed(message)) { - btm->bp = NULL; - btm->message = message; - } - else { - ErlHeapFragment* bp; - Eterm* hp; - Uint size; - - size = size_object(message); - btm->bp = bp = new_message_buffer(size); - hp = bp->mem; - btm->message = copy_struct(message, size, &hp, &bp->off_heap); - } - - tab_insert(btm); - ASSERT(btm == tab_find(ref)); - btm->tm.active = 0; /* MUST be initalized */ - erts_set_timer(&btm->tm, - (ErlTimeoutProc) bif_timer_timeout, - (ErlCancelProc) bif_timer_cleanup, - (void *) btm, - timeout); - return ref; -} - -/* send_after(Time, Pid, Message) -> Ref */ -BIF_RETTYPE send_after_3(BIF_ALIST_3) -{ - Eterm res; - - if (erts_smp_safe_btm_rwlock(BIF_P, ERTS_PROC_LOCK_MAIN)) - ERTS_BIF_EXITED(BIF_P); - - res = setup_bif_timer(0, BIF_P, BIF_ARG_1, BIF_ARG_2, BIF_ARG_3); - - erts_smp_btm_rwunlock(); - - if (is_non_value(res)) { - BIF_ERROR(BIF_P, BADARG); - } - else { - ASSERT(is_internal_ref(res)); - BIF_RET(res); - } -} - -/* start_timer(Time, Pid, Message) -> Ref */ -BIF_RETTYPE start_timer_3(BIF_ALIST_3) -{ - Eterm res; - - if (erts_smp_safe_btm_rwlock(BIF_P, ERTS_PROC_LOCK_MAIN)) - ERTS_BIF_EXITED(BIF_P); - - res = setup_bif_timer(BTM_FLG_WRAP, BIF_P, BIF_ARG_1, BIF_ARG_2, BIF_ARG_3); - - erts_smp_btm_rwunlock(); - - if (is_non_value(res)) { - BIF_ERROR(BIF_P, BADARG); - } - else { - ASSERT(is_internal_ref(res)); - BIF_RET(res); - } -} - -/* cancel_timer(Ref) -> false | RemainingTime */ -BIF_RETTYPE cancel_timer_1(BIF_ALIST_1) -{ - Eterm res; - ErtsBifTimer *btm; - - if (is_not_internal_ref(BIF_ARG_1)) { - if (is_ref(BIF_ARG_1)) { - BIF_RET(am_false); - } - BIF_ERROR(BIF_P, BADARG); - } - - if (erts_smp_safe_btm_rwlock(BIF_P, ERTS_PROC_LOCK_MAIN)) - ERTS_BIF_EXITED(BIF_P); - - btm = tab_find(BIF_ARG_1); - if (!btm || btm->flags & BTM_FLG_CANCELED) { - erts_smp_btm_rwunlock(); - res = am_false; - } - else { - Uint left = erts_time_left(&btm->tm); - if (!(btm->flags & BTM_FLG_BYNAME)) { - erts_smp_proc_lock(btm->receiver.proc.ess, ERTS_PROC_LOCK_MSGQ); - unlink_proc(btm); - erts_smp_proc_unlock(btm->receiver.proc.ess, ERTS_PROC_LOCK_MSGQ); - } - tab_remove(btm); - ASSERT(!tab_find(BIF_ARG_1)); - erts_cancel_timer(&btm->tm); - erts_smp_btm_rwunlock(); - res = erts_make_integer(left, BIF_P); - } - - BIF_RET(res); -} - -/* read_timer(Ref) -> false | RemainingTime */ -BIF_RETTYPE read_timer_1(BIF_ALIST_1) -{ - Eterm res; - ErtsBifTimer *btm; - - if (is_not_internal_ref(BIF_ARG_1)) { - if (is_ref(BIF_ARG_1)) { - BIF_RET(am_false); - } - BIF_ERROR(BIF_P, BADARG); - } - - if (erts_smp_safe_btm_rlock(BIF_P, ERTS_PROC_LOCK_MAIN)) - ERTS_BIF_EXITED(BIF_P); - - btm = tab_find(BIF_ARG_1); - if (!btm || btm->flags & BTM_FLG_CANCELED) { - res = am_false; - } - else { - Uint left = erts_time_left(&btm->tm); - res = erts_make_integer(left, BIF_P); - } - - erts_smp_btm_runlock(); - - BIF_RET(res); -} - -void -erts_print_bif_timer_info(int to, void *to_arg) -{ - int i; - int lock = !ERTS_IS_CRASH_DUMPING; - - if (lock) - erts_smp_btm_rlock(); - - for (i = 0; i < TIMER_HASH_VEC_SZ; i++) { - ErtsBifTimer *btm; - for (btm = bif_timer_tab[i]; btm; btm = btm->tab.next) { - Eterm receiver = (btm->flags & BTM_FLG_BYNAME - ? btm->receiver.name - : btm->receiver.proc.ess->common.id); - erts_print(to, to_arg, "=timer:%T\n", receiver); - erts_print(to, to_arg, "Message: %T\n", btm->message); - erts_print(to, to_arg, "Time left: %u ms\n", - erts_time_left(&btm->tm)); - } - } - - if (lock) - erts_smp_btm_runlock(); -} - - -void -erts_cancel_bif_timers(Process *p, ErtsProcLocks plocks) -{ - ErtsBifTimer *btm; - - if (erts_smp_btm_tryrwlock() == EBUSY) { - erts_smp_proc_unlock(p, plocks); - erts_smp_btm_rwlock(); - erts_smp_proc_lock(p, plocks); - } - - btm = p->u.bif_timers; - while (btm) { - ErtsBifTimer *tmp_btm; - ASSERT(!(btm->flags & BTM_FLG_CANCELED)); - tab_remove(btm); - tmp_btm = btm; - btm = btm->receiver.proc.next; - erts_cancel_timer(&tmp_btm->tm); - } - - p->u.bif_timers = NULL; - - erts_smp_btm_rwunlock(); -} - -void erts_bif_timer_init(void) -{ - int i; - no_bif_timers = 0; - init_btm_pre_alloc(); - erts_smp_btm_lock_init(); - bif_timer_tab = erts_alloc(ERTS_ALC_T_BIF_TIMER_TABLE, - sizeof(ErtsBifTimer *)*TIMER_HASH_VEC_SZ); - for (i = 0; i < TIMER_HASH_VEC_SZ; ++i) - bif_timer_tab[i] = NULL; -} - -Uint -erts_bif_timer_memory_size(void) -{ - Uint res; - int lock = !ERTS_IS_CRASH_DUMPING; - - if (lock) - erts_smp_btm_rlock(); - - res = (sizeof(ErtsBifTimer *)*TIMER_HASH_VEC_SZ - + no_bif_timers*sizeof(ErtsBifTimer)); - - if (lock) - erts_smp_btm_runlock(); - - return res; -} - - -void -erts_bif_timer_foreach(void (*func)(Eterm, Eterm, ErlHeapFragment *, void *), - void *arg) -{ - int i; - - ERTS_SMP_LC_ASSERT(erts_smp_thr_progress_is_blocking()); - - for (i = 0; i < TIMER_HASH_VEC_SZ; i++) { - ErtsBifTimer *btm; - for (btm = bif_timer_tab[i]; btm; btm = btm->tab.next) { - (*func)((btm->flags & BTM_FLG_BYNAME - ? btm->receiver.name - : btm->receiver.proc.ess->common.id), - btm->message, - btm->bp, - arg); - } - } -} diff --git a/erts/emulator/beam/erl_bif_timer.h b/erts/emulator/beam/erl_bif_timer.h deleted file mode 100644 index 1197c176f5..0000000000 --- a/erts/emulator/beam/erl_bif_timer.h +++ /dev/null @@ -1,36 +0,0 @@ -/* - * %CopyrightBegin% - * - * Copyright Ericsson AB 2005-2009. All Rights Reserved. - * - * The contents of this file are subject to the Erlang Public License, - * Version 1.1, (the "License"); you may not use this file except in - * compliance with the License. You should have received a copy of the - * Erlang Public License along with this software. If not, it can be - * retrieved online at http://www.erlang.org/. - * - * Software distributed under the License is distributed on an "AS IS" - * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See - * the License for the specific language governing rights and limitations - * under the License. - * - * %CopyrightEnd% - */ - - -#ifndef ERL_BIF_TIMER_H__ -#define ERL_BIF_TIMER_H__ - -typedef struct ErtsBifTimer_ ErtsBifTimer; - -#include "sys.h" -#include "erl_process.h" -#include "erl_message.h" - -Uint erts_bif_timer_memory_size(void); -void erts_print_bif_timer_info(int to, void *to_arg); -void erts_cancel_bif_timers(Process *p, ErtsProcLocks plocks); -void erts_bif_timer_init(void); -void erts_bif_timer_foreach(void (*func)(Eterm,Eterm,ErlHeapFragment *,void *), - void *arg); -#endif diff --git a/erts/emulator/beam/erl_bif_trace.c b/erts/emulator/beam/erl_bif_trace.c index 41c98bbffb..08807d72c9 100644 --- a/erts/emulator/beam/erl_bif_trace.c +++ b/erts/emulator/beam/erl_bif_trace.c @@ -3,16 +3,17 @@ * * Copyright Ericsson AB 1999-2013. All Rights Reserved. * - * The contents of this file are subject to the Erlang Public License, - * Version 1.1, (the "License"); you may not use this file except in - * compliance with the License. You should have received a copy of the - * Erlang Public License along with this software. If not, it can be - * retrieved online at http://www.erlang.org/. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Software distributed under the License is distributed on an "AS IS" - * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See - * the License for the specific language governing rights and limitations - * under the License. + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. * * %CopyrightEnd% */ @@ -38,6 +39,7 @@ #include "beam_bp.h" #include "erl_binary.h" #include "erl_thr_progress.h" +#include "erl_bif_unique.h" #define DECL_AM(S) Eterm AM_ ## S = am_atom_put(#S, sizeof(#S) - 1) @@ -358,7 +360,7 @@ trace_pattern(Process* p, Eterm MFA, Eterm Pattern, Eterm flaglist) ASSERT(finish_bp.stager == NULL); finish_bp.stager = p; erts_schedule_thr_prgr_later_op(smp_bp_finisher, NULL, &finish_bp.lop); - erts_smp_proc_inc_refc(p); + erts_proc_inc_refc(p); erts_suspend(p, ERTS_PROC_LOCK_MAIN, NULL); ERTS_BIF_YIELD_RETURN(p, make_small(matches)); } @@ -392,7 +394,7 @@ static void smp_bp_finisher(void* null) erts_resume(p, ERTS_PROC_LOCK_STATUS); } erts_smp_proc_unlock(p, ERTS_PROC_LOCK_STATUS); - erts_smp_proc_dec_refc(p); + erts_proc_dec_refc(p); } } #endif /* ERTS_SMP */ @@ -429,6 +431,9 @@ Uint erts_trace_flag2bit(Eterm flag) { switch (flag) { + case am_timestamp: return F_NOW_TS; + case am_strict_monotonic_timestamp: return F_STRICT_MON_TS; + case am_monotonic_timestamp: return F_MON_TS; case am_all: return TRACEE_FLAGS; case am_send: return F_TRACE_SEND; case am_receive: return F_TRACE_RECEIVE; @@ -437,7 +442,6 @@ erts_trace_flag2bit(Eterm flag) case am_set_on_first_spawn: return F_TRACE_SOS1; case am_set_on_link: return F_TRACE_SOL; case am_set_on_first_link: return F_TRACE_SOL1; - case am_timestamp: return F_TIMESTAMP; case am_running: return F_TRACE_SCHED; case am_exiting: return F_TRACE_SCHED_EXIT; case am_garbage_collection: return F_TRACE_GC; @@ -590,7 +594,7 @@ Eterm trace_3(BIF_ALIST_3) ERTS_TRACE_FLAGS(tracee_port) |= mask; else ERTS_TRACE_FLAGS(tracee_port) &= ~mask; - + if (!ERTS_TRACE_FLAGS(tracee_port)) ERTS_TRACER_PROC(tracee_port) = NIL; else if (tracer != NIL) @@ -651,7 +655,7 @@ Eterm trace_3(BIF_ALIST_3) if (pid_spec == am_all) { if (on) { if (!erts_cpu_timestamp) { -#ifdef HAVE_CLOCK_GETTIME +#ifdef HAVE_CLOCK_GETTIME_CPU_TIME /* Perhaps clock_gettime was found during config on a different machine than this. We check @@ -678,7 +682,7 @@ Eterm trace_3(BIF_ALIST_3) if (erts_start_now_cpu() < 0) { goto error; } -#endif /* HAVE_CLOCK_GETTIME */ +#endif /* HAVE_CLOCK_GETTIME_CPU_TIME */ erts_cpu_timestamp = !0; } } @@ -976,7 +980,7 @@ trace_info_pid(Process* p, Eterm pid_spec, Eterm key) } if (key == am_flags) { - int num_flags = 19; /* MAXIMUM number of flags. */ + int num_flags = 21; /* MAXIMUM number of flags. */ Uint needed = 3+2*num_flags; Eterm flag_list = NIL; Eterm* limit; @@ -994,6 +998,9 @@ trace_info_pid(Process* p, Eterm pid_spec, Eterm key) #endif hp = HAlloc(p, needed); limit = hp+needed; + FLAG(F_NOW_TS, am_timestamp); + FLAG(F_STRICT_MON_TS, am_strict_monotonic_timestamp); + FLAG(F_MON_TS, am_monotonic_timestamp); FLAG(F_TRACE_SEND, am_send); FLAG(F_TRACE_RECEIVE, am_receive); FLAG(F_TRACE_SOS, am_set_on_spawn); @@ -1005,7 +1012,6 @@ trace_info_pid(Process* p, Eterm pid_spec, Eterm key) FLAG(F_TRACE_SCHED, am_running); FLAG(F_TRACE_SCHED_EXIT, am_exiting); FLAG(F_TRACE_GC, am_garbage_collection); - FLAG(F_TIMESTAMP, am_timestamp); FLAG(F_TRACE_ARITY_ONLY, am_arity); FLAG(F_TRACE_RETURN_TO, am_return_to); FLAG(F_TRACE_SILENT, am_silent); @@ -1224,9 +1230,7 @@ trace_info_func(Process* p, Eterm func_spec, Eterm key) match_spec_meta = NIL; } if (r & FUNC_TRACE_COUNT_TRACE) { - c = count < 0 ? - erts_make_integer(-count-1, p) : - erts_make_integer(count, p); + c = erts_make_integer(count, p); } if (r & FUNC_TRACE_TIME_TRACE) { ct = call_time; @@ -1798,7 +1802,11 @@ Eterm erts_seq_trace(Process *p, Eterm arg1, Eterm arg2, } else if (arg1 == am_print) { current_flag = SEQ_TRACE_PRINT; } else if (arg1 == am_timestamp) { - current_flag = SEQ_TRACE_TIMESTAMP; + current_flag = SEQ_TRACE_NOW_TS; + } else if (arg1 == am_strict_monotonic_timestamp) { + current_flag = SEQ_TRACE_STRICT_MON_TS; + } else if (arg1 == am_monotonic_timestamp) { + current_flag = SEQ_TRACE_MON_TS; } else current_flag = 0; @@ -1909,7 +1917,9 @@ BIF_RETTYPE erl_seq_trace_info(Process *p, Eterm item) #endif ) { if ((item == am_send) || (item == am_receive) || - (item == am_print) || (item == am_timestamp)) { + (item == am_print) || (item == am_timestamp) + || (item == am_monotonic_timestamp) + || (item == am_strict_monotonic_timestamp)) { hp = HAlloc(p,3); res = TUPLE2(hp, item, am_false); BIF_RET(res); @@ -1927,7 +1937,11 @@ BIF_RETTYPE erl_seq_trace_info(Process *p, Eterm item) } else if (item == am_print) { current_flag = SEQ_TRACE_PRINT; } else if (item == am_timestamp) { - current_flag = SEQ_TRACE_TIMESTAMP; + current_flag = SEQ_TRACE_NOW_TS; + } else if (item == am_strict_monotonic_timestamp) { + current_flag = SEQ_TRACE_STRICT_MON_TS; + } else if (item == am_monotonic_timestamp) { + current_flag = SEQ_TRACE_MON_TS; } else { current_flag = 0; } @@ -2014,6 +2028,7 @@ void erts_system_monitor_clear(Process *c_p) { #endif erts_set_system_monitor(NIL); erts_system_monitor_long_gc = 0; + erts_system_monitor_long_schedule = 0; erts_system_monitor_large_heap = 0; erts_system_monitor_flags.busy_port = 0; erts_system_monitor_flags.busy_dist_port = 0; @@ -2038,12 +2053,17 @@ static Eterm system_monitor_get(Process *p) Uint hsz = 3 + (erts_system_monitor_flags.busy_dist_port ? 2 : 0) + (erts_system_monitor_flags.busy_port ? 2 : 0); Eterm long_gc = NIL; + Eterm long_schedule = NIL; Eterm large_heap = NIL; if (erts_system_monitor_long_gc != 0) { hsz += 2+3; (void) erts_bld_uint(NULL, &hsz, erts_system_monitor_long_gc); } + if (erts_system_monitor_long_schedule != 0) { + hsz += 2+3; + (void) erts_bld_uint(NULL, &hsz, erts_system_monitor_long_schedule); + } if (erts_system_monitor_large_heap != 0) { hsz += 2+3; (void) erts_bld_uint(NULL, &hsz, erts_system_monitor_large_heap); @@ -2053,6 +2073,10 @@ static Eterm system_monitor_get(Process *p) if (erts_system_monitor_long_gc != 0) { long_gc = erts_bld_uint(&hp, NULL, erts_system_monitor_long_gc); } + if (erts_system_monitor_long_schedule != 0) { + long_schedule = erts_bld_uint(&hp, NULL, + erts_system_monitor_long_schedule); + } if (erts_system_monitor_large_heap != 0) { large_heap = erts_bld_uint(&hp, NULL, erts_system_monitor_large_heap); } @@ -2061,6 +2085,10 @@ static Eterm system_monitor_get(Process *p) Eterm t = TUPLE2(hp, am_long_gc, long_gc); hp += 3; res = CONS(hp, t, res); hp += 2; } + if (long_schedule != NIL) { + Eterm t = TUPLE2(hp, am_long_schedule, long_schedule); hp += 3; + res = CONS(hp, t, res); hp += 2; + } if (large_heap != NIL) { Eterm t = TUPLE2(hp, am_large_heap, large_heap); hp += 3; res = CONS(hp, t, res); hp += 2; @@ -2115,7 +2143,7 @@ system_monitor(Process *p, Eterm monitor_pid, Eterm list) } if (is_not_list(list)) goto error; else { - Uint long_gc, large_heap; + Uint long_gc, long_schedule, large_heap; int busy_port, busy_dist_port; system_blocked = 1; @@ -2125,7 +2153,8 @@ system_monitor(Process *p, Eterm monitor_pid, Eterm list) if (!erts_pid2proc(p, ERTS_PROC_LOCK_MAIN, monitor_pid, 0)) goto error; - for (long_gc = 0, large_heap = 0, busy_port = 0, busy_dist_port = 0; + for (long_gc = 0, long_schedule = 0, large_heap = 0, + busy_port = 0, busy_dist_port = 0; is_list(list); list = CDR(list_val(list))) { Eterm t = CAR(list_val(list)); @@ -2135,6 +2164,9 @@ system_monitor(Process *p, Eterm monitor_pid, Eterm list) if (tp[1] == am_long_gc) { if (! term_to_Uint(tp[2], &long_gc)) goto error; if (long_gc < 1) long_gc = 1; + } else if (tp[1] == am_long_schedule) { + if (! term_to_Uint(tp[2], &long_schedule)) goto error; + if (long_schedule < 1) long_schedule = 1; } else if (tp[1] == am_large_heap) { if (! term_to_Uint(tp[2], &large_heap)) goto error; if (large_heap < 16384) large_heap = 16384; @@ -2150,6 +2182,7 @@ system_monitor(Process *p, Eterm monitor_pid, Eterm list) prev = system_monitor_get(p); erts_set_system_monitor(monitor_pid); erts_system_monitor_long_gc = long_gc; + erts_system_monitor_long_schedule = long_schedule; erts_system_monitor_large_heap = large_heap; erts_system_monitor_flags.busy_port = !!busy_port; erts_system_monitor_flags.busy_dist_port = !!busy_dist_port; @@ -2218,6 +2251,7 @@ static Eterm system_profile_get(Process *p) { if (erts_system_profile_flags.exclusive) { res = CONS(hp, am_exclusive, res); hp += 2; } + return TUPLE2(hp, system_profile, res); } } @@ -2236,6 +2270,7 @@ BIF_RETTYPE system_profile_2(BIF_ALIST_2) int system_blocked = 0; Process *profiler_p = NULL; Port *profiler_port = NULL; + int ts; if (profiler == am_undefined || list == NIL) { prev = system_profile_get(p); @@ -2267,7 +2302,8 @@ BIF_RETTYPE system_profile_2(BIF_ALIST_2) goto error; } - for (scheduler = 0, runnable_ports = 0, runnable_procs = 0, exclusive = 0; + for (ts = ERTS_TRACE_FLG_NOW_TIMESTAMP, scheduler = 0, + runnable_ports = 0, runnable_procs = 0, exclusive = 0; is_list(list); list = CDR(list_val(list))) { @@ -2280,6 +2316,12 @@ BIF_RETTYPE system_profile_2(BIF_ALIST_2) exclusive = !0; } else if (t == am_scheduler) { scheduler = !0; + } else if (t == am_timestamp) { + ts = ERTS_TRACE_FLG_NOW_TIMESTAMP; + } else if (t == am_strict_monotonic_timestamp) { + ts = ERTS_TRACE_FLG_STRICT_MONOTONIC_TIMESTAMP; + } else if (t == am_monotonic_timestamp) { + ts = ERTS_TRACE_FLG_MONOTONIC_TIMESTAMP; } else goto error; } if (is_not_nil(list)) goto error; @@ -2292,7 +2334,7 @@ BIF_RETTYPE system_profile_2(BIF_ALIST_2) erts_system_profile_flags.runnable_ports = !!runnable_ports; erts_system_profile_flags.runnable_procs = !!runnable_procs; erts_system_profile_flags.exclusive = !!exclusive; - + erts_system_profile_ts_type = ts; erts_smp_thr_progress_unblock(); erts_smp_proc_lock(p, ERTS_PROC_LOCK_MAIN); diff --git a/erts/emulator/beam/erl_bif_unique.c b/erts/emulator/beam/erl_bif_unique.c new file mode 100644 index 0000000000..5eca09c5a6 --- /dev/null +++ b/erts/emulator/beam/erl_bif_unique.c @@ -0,0 +1,557 @@ +/* + * %CopyrightBegin% + * + * Copyright Ericsson AB 2014. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * %CopyrightEnd% + */ + +#ifdef HAVE_CONFIG_H +# include "config.h" +#endif + +#include "sys.h" +#include "erl_vm.h" +#include "erl_alloc.h" +#include "export.h" +#include "bif.h" +#include "erl_bif_unique.h" + +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *\ + * Reference * +\* */ + +static union { + erts_atomic64_t count; + char align__[ERTS_CACHE_LINE_SIZE]; +} global_reference erts_align_attribute(ERTS_CACHE_LINE_SIZE); + + +/* + * ref[0] indicate thread creating reference as follows: + * + * - ref[0] == 0 => Non-scheduler thread; + * - else; ref[0] <= erts_no_schedulers => + * ordinary scheduler with id == ref[0]; + * - else; ref[0] <= erts_no_schedulers + * + erts_no_dirty_cpu_schedulers => + * dirty cpu scheduler with id == 'ref[0] - erts_no_schedulers'; + * - else => + * dirty io scheduler with id == 'ref[0] + * - erts_no_schedulers + * - erts_no_dirty_cpu_schedulers' + */ + +#ifdef DEBUG +static Uint32 max_thr_id; +#endif + +static void +init_reference(void) +{ +#ifdef DEBUG + max_thr_id = (Uint32) erts_no_schedulers; +#ifdef ERTS_DIRTY_SCHEDULERS + max_thr_id += (Uint32) erts_no_dirty_cpu_schedulers; + max_thr_id += (Uint32) erts_no_dirty_io_schedulers; +#endif +#endif + erts_atomic64_init_nob(&global_reference.count, 0); +} + +static ERTS_INLINE void +global_make_ref_in_array(Uint32 thr_id, Uint32 ref[ERTS_MAX_REF_NUMBERS]) +{ + Uint64 value; + + value = (Uint64) erts_atomic64_inc_read_mb(&global_reference.count); + + erts_set_ref_numbers(ref, thr_id, value); +} + +static ERTS_INLINE void +make_ref_in_array(Uint32 ref[ERTS_MAX_REF_NUMBERS]) +{ + ErtsSchedulerData *esdp = erts_get_scheduler_data(); + if (esdp) + erts_sched_make_ref_in_array(esdp, ref); + else + global_make_ref_in_array(0, ref); +} + +void +erts_make_ref_in_array(Uint32 ref[ERTS_MAX_REF_NUMBERS]) +{ + make_ref_in_array(ref); +} + +Eterm erts_make_ref_in_buffer(Eterm buffer[REF_THING_SIZE]) +{ + Eterm* hp = buffer; + Uint32 ref[ERTS_MAX_REF_NUMBERS]; + + make_ref_in_array(ref); + write_ref_thing(hp, ref[0], ref[1], ref[2]); + return make_internal_ref(hp); +} + +Eterm erts_make_ref(Process *c_p) +{ + Eterm* hp; + Uint32 ref[ERTS_MAX_REF_NUMBERS]; + + ERTS_SMP_LC_ASSERT(ERTS_PROC_LOCK_MAIN & erts_proc_lc_my_proc_locks(c_p)); + + hp = HAlloc(c_p, REF_THING_SIZE); + + make_ref_in_array(ref); + write_ref_thing(hp, ref[0], ref[1], ref[2]); + + return make_internal_ref(hp); +} + +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *\ + * Unique Integer * +\* */ + +static struct { + union { + struct { + int left_shift; + int right_shift; + Uint64 mask; + Uint64 val0_max; + } o; + char align__[ERTS_CACHE_LINE_SIZE]; + } r; + union { + erts_atomic64_t val1; + char align__[ERTS_CACHE_LINE_SIZE]; + } w; +} unique_data erts_align_attribute(ERTS_CACHE_LINE_SIZE); + +static void +init_unique_integer(void) +{ + int bits; + unique_data.r.o.val0_max = (Uint64) erts_no_schedulers; +#ifdef ERTS_DIRTY_SCHEDULERS + unique_data.r.o.val0_max += (Uint64) erts_no_dirty_cpu_schedulers; + unique_data.r.o.val0_max += (Uint64) erts_no_dirty_io_schedulers; +#endif + bits = erts_fit_in_bits_int64(unique_data.r.o.val0_max); + unique_data.r.o.left_shift = bits; + unique_data.r.o.right_shift = 64 - bits; + unique_data.r.o.mask = (((Uint64) 1) << bits) - 1; + erts_atomic64_init_nob(&unique_data.w.val1, -1); +} + +#define ERTS_MAX_UNIQUE_INT_HEAP_SIZE ERTS_UINT64_ARRAY_TO_BIG_MAX_HEAP_SZ(2) + +static ERTS_INLINE Eterm +bld_unique_integer_term(Eterm **hpp, Uint *szp, + Uint64 val0, Uint64 val1, + int positive) +{ + Uint hsz; + Uint64 unique_val[2]; + + unique_val[0] = ((Uint64) val0); + unique_val[0] |= ((Uint64) val1) << unique_data.r.o.left_shift; + unique_val[1] = ((Uint64) val1) >> unique_data.r.o.right_shift; + unique_val[1] &= unique_data.r.o.mask; + + if (positive) { + unique_val[0]++; + if (unique_val[0] == 0) + unique_val[1]++; + } + else { + ASSERT(MIN_SMALL < 0); + if (unique_val[1] == 0 + && unique_val[0] < ((Uint64) -1*((Sint64) MIN_SMALL))) { + Sint64 s_unique_val = (Sint64) unique_val[0]; + s_unique_val += MIN_SMALL; + ASSERT(MIN_SMALL <= s_unique_val && s_unique_val < 0); + if (szp) + *szp = 0; + if (!hpp) + return THE_NON_VALUE; + return make_small((Sint) s_unique_val); + } + if (unique_val[0] < ((Uint64) -1*((Sint64) MIN_SMALL))) { + ASSERT(unique_val[1] != 0); + unique_val[1] -= 1; + } + unique_val[0] += MIN_SMALL; + } + + if (!unique_val[1]) { + if (unique_val[0] <= MAX_SMALL) { + if (szp) + *szp = 0; + if (!hpp) + return THE_NON_VALUE; + return make_small((Uint) unique_val[0]); + } + + if (szp) + *szp = ERTS_UINT64_HEAP_SIZE(unique_val[0]); + if (!hpp) + return THE_NON_VALUE; + return erts_uint64_to_big(unique_val[0], hpp); + } + else { + Eterm tmp, *tmp_hp, res; + DeclareTmpHeapNoproc(local_heap, 2*ERTS_MAX_UNIQUE_INT_HEAP_SIZE); + + UseTmpHeapNoproc(2*ERTS_MAX_UNIQUE_INT_HEAP_SIZE); + + tmp_hp = local_heap; + + tmp = erts_uint64_array_to_big(&tmp_hp, 0, 2, unique_val); + ASSERT(is_big(tmp)); + + hsz = big_arity(tmp) + 1; + + ASSERT(hsz <= ERTS_MAX_UNIQUE_INT_HEAP_SIZE); + + if (szp) + *szp = hsz; + + if (!hpp) + res = THE_NON_VALUE; + else { + int hix; + Eterm *hp = *hpp; + tmp_hp = big_val(tmp); + for (hix = 0; hix < hsz; hix++) + hp[hix] = tmp_hp[hix]; + + *hpp = hp + hsz; + res = make_big(hp); + } + + UnUseTmpHeapNoproc(2*ERTS_MAX_UNIQUE_INT_HEAP_SIZE); + + return res; + } +} + +static ERTS_INLINE Eterm unique_integer_bif(Process *c_p, int positive) +{ + ErtsSchedulerData *esdp; + Uint64 thr_id, unique; + Uint hsz; + Eterm *hp; + + esdp = ERTS_PROC_GET_SCHDATA(c_p); + thr_id = (Uint64) esdp->thr_id; + unique = esdp->unique++; + bld_unique_integer_term(NULL, &hsz, thr_id, unique, positive); + hp = hsz ? HAlloc(c_p, hsz) : NULL; + return bld_unique_integer_term(&hp, NULL, thr_id, unique, positive); +} + +Uint +erts_raw_unique_integer_heap_size(Uint64 val[ERTS_UNIQUE_INT_RAW_VALUES]) +{ + Uint sz; + bld_unique_integer_term(NULL, &sz, val[0], val[1], 0); + return sz; +} + +Eterm +erts_raw_make_unique_integer(Eterm **hpp, Uint64 val[ERTS_UNIQUE_INT_RAW_VALUES]) +{ + return bld_unique_integer_term(hpp, NULL, val[0], val[1], 0); +} + +void +erts_raw_get_unique_integer(Uint64 val[ERTS_UNIQUE_INT_RAW_VALUES]) +{ + ErtsSchedulerData *esdp = erts_get_scheduler_data(); + if (esdp) { + val[0] = (Uint64) esdp->thr_id; + val[1] = esdp->unique++; + } + else { + val[0] = (Uint64) 0; + val[1] = (Uint64) erts_atomic64_inc_read_nob(&unique_data.w.val1); + } +} + + +Sint64 +erts_get_min_unique_integer(void) +{ + return (Sint64) MIN_SMALL; +} + +/* --- Debug --- */ + +Eterm +erts_debug_make_unique_integer(Process *c_p, Eterm etval0, Eterm etval1) +{ + Uint64 val0, val1; + Uint hsz; + Eterm res, *hp, *end_hp; + + if (!term_to_Uint64(etval0, &val0)) + return THE_NON_VALUE; + + if (!term_to_Uint64(etval1, &val1)) + return THE_NON_VALUE; + + bld_unique_integer_term(NULL, &hsz, val0, val1, 0); + + hp = HAlloc(c_p, hsz); + end_hp = hp + hsz; + + res = bld_unique_integer_term(&hp, NULL, val0, val1, 0); + if (hp != end_hp) + ERTS_INTERNAL_ERROR("Heap allocation error"); + + return res; +} + +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *\ + * Strict Monotonic Counter * +\* */ + +static struct { + union { + erts_atomic64_t value; + char align__[ERTS_CACHE_LINE_SIZE]; + } w; +} raw_unique_monotonic_integer erts_align_attribute(ERTS_CACHE_LINE_SIZE); + +#if defined(ARCH_32) || HALFWORD_HEAP +# define ERTS_UNIQUE_MONOTONIC_OFFSET ERTS_SINT64_MIN +#else +# define ERTS_UNIQUE_MONOTONIC_OFFSET MIN_SMALL +#endif + +static void +init_unique_monotonic_integer(void) +{ + erts_atomic64_init_nob(&raw_unique_monotonic_integer.w.value, + (erts_aint64_t) -1); +} + +static ERTS_INLINE Uint64 +get_raw_unique_monotonic_integer(void) +{ + return (Uint64) erts_atomic64_inc_read_mb(&raw_unique_monotonic_integer.w.value); +} + +static ERTS_INLINE Uint +get_unique_monotonic_integer_heap_size(Uint64 raw, int positive) +{ + if (positive) { + Uint64 value = raw+1; + return ERTS_UINT64_HEAP_SIZE(value); + } + else { + Sint64 value = ((Sint64) raw) + ERTS_UNIQUE_MONOTONIC_OFFSET; + if (IS_SSMALL(value)) + return 0; +#if defined(ARCH_32) || HALFWORD_HEAP + return ERTS_SINT64_HEAP_SIZE(value); +#else + return ERTS_UINT64_HEAP_SIZE((Uint64) value); +#endif + } +} + +static ERTS_INLINE Eterm +make_unique_monotonic_integer_value(Eterm *hp, Uint hsz, Uint64 raw, int positive) +{ + Eterm res; +#ifdef DEBUG + Eterm *end_hp = hp + hsz; +#endif + + if (positive) { + Uint64 value = raw+1; + res = hsz ? erts_uint64_to_big(value, &hp) : make_small(value); + } + else { + Sint64 value = ((Sint64) raw) + ERTS_UNIQUE_MONOTONIC_OFFSET; + if (hsz == 0) + res = make_small(value); + else { +#if defined(ARCH_32) || HALFWORD_HEAP + res = erts_sint64_to_big(value, &hp); +#else + res = erts_uint64_to_big((Uint64) value, &hp); +#endif + } + } + + ASSERT(end_hp == hp); + + return res; +} + +static ERTS_INLINE Eterm +unique_monotonic_integer_bif(Process *c_p, int positive) +{ + Uint64 raw; + Uint hsz; + Eterm *hp; + + raw = get_raw_unique_monotonic_integer(); + hsz = get_unique_monotonic_integer_heap_size(raw, positive); + hp = hsz ? HAlloc(c_p, hsz) : NULL; + return make_unique_monotonic_integer_value(hp, hsz, raw, positive); +} + +Sint64 +erts_raw_get_unique_monotonic_integer(void) +{ + return get_raw_unique_monotonic_integer(); +} + +Uint +erts_raw_unique_monotonic_integer_heap_size(Sint64 raw) +{ + return get_unique_monotonic_integer_heap_size(raw, 0); +} + +Eterm +erts_raw_make_unique_monotonic_integer_value(Eterm **hpp, Sint64 raw) +{ + Uint hsz = get_unique_monotonic_integer_heap_size(raw, 0); + Eterm res = make_unique_monotonic_integer_value(*hpp, hsz, raw, 0); + *hpp += hsz; + return res; +} + +Sint64 +erts_get_min_unique_monotonic_integer(void) +{ + return ERTS_UNIQUE_MONOTONIC_OFFSET; +} + +/* --- Debug --- */ + +int +erts_debug_set_unique_monotonic_integer_state(Eterm et_value) +{ + Sint64 value; + + if (!term_to_Sint64(et_value, &value)) { + Uint64 uvalue; + if (!term_to_Uint64(et_value, &uvalue)) + return 0; + value = (Sint64) uvalue; + } + + erts_atomic64_set_mb(&raw_unique_monotonic_integer.w.value, + (erts_aint64_t) value); + return 1; +} + +Eterm +erts_debug_get_unique_monotonic_integer_state(Process *c_p) +{ + Uint64 value; + Eterm hsz, *hp; + + value = (Uint64) erts_atomic64_read_mb(&raw_unique_monotonic_integer.w.value); + + if (IS_USMALL(0, value)) + return make_small(value); + hsz = ERTS_UINT64_HEAP_SIZE(value); + hp = HAlloc(c_p, hsz); + return erts_uint64_to_big(value, &hp); +} + +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *\ + * Initilazation * +\* */ + +void +erts_bif_unique_init(void) +{ + init_reference(); + init_unique_monotonic_integer(); + init_unique_integer(); +} + +void +erts_sched_bif_unique_init(ErtsSchedulerData *esdp) +{ + esdp->unique = (Uint64) 0; + esdp->ref = (Uint64) 0; +} + +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *\ + * The BIFs * +\* */ + + +BIF_RETTYPE make_ref_0(BIF_ALIST_0) +{ + BIF_RETTYPE res; + Eterm* hp; + + ERTS_SMP_LC_ASSERT(ERTS_PROC_LOCK_MAIN & erts_proc_lc_my_proc_locks(BIF_P)); + + hp = HAlloc(BIF_P, REF_THING_SIZE); + + res = erts_sched_make_ref_in_buffer(ERTS_PROC_GET_SCHDATA(BIF_P), hp); + + BIF_RET(res); +} + +BIF_RETTYPE unique_integer_0(BIF_ALIST_0) +{ + BIF_RET(unique_integer_bif(BIF_P, 0)); +} + +BIF_RETTYPE unique_integer_1(BIF_ALIST_1) +{ + Eterm modlist = BIF_ARG_1; + int monotonic = 0; + int positive = 0; + BIF_RETTYPE res; + + while (is_list(modlist)) { + Eterm *consp = list_val(modlist); + switch (CAR(consp)) { + case am_monotonic: + monotonic = 1; + break; + case am_positive: + positive = 1; + break; + default: + BIF_ERROR(BIF_P, BADARG); + } + modlist = CDR(consp); + } + + if (is_not_nil(modlist)) + BIF_ERROR(BIF_P, BADARG); + + if (monotonic) + res = unique_monotonic_integer_bif(BIF_P, positive); + else + res = unique_integer_bif(BIF_P, positive); + + BIF_RET(res); +} diff --git a/erts/emulator/beam/erl_bif_unique.h b/erts/emulator/beam/erl_bif_unique.h new file mode 100644 index 0000000000..37d5d91c39 --- /dev/null +++ b/erts/emulator/beam/erl_bif_unique.h @@ -0,0 +1,132 @@ +/* + * %CopyrightBegin% + * + * Copyright Ericsson AB 2014. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * %CopyrightEnd% + */ + +#ifndef ERTS_BIF_UNIQUE_H__ +#define ERTS_BIF_UNIQUE_H__ + +#include "erl_process.h" +#include "big.h" + +void erts_bif_unique_init(void); +void erts_sched_bif_unique_init(ErtsSchedulerData *esdp); + +/* reference */ +Eterm erts_make_ref(Process *); +Eterm erts_make_ref_in_buffer(Eterm buffer[REF_THING_SIZE]); +void erts_make_ref_in_array(Uint32 ref[ERTS_MAX_REF_NUMBERS]); + +/* strict monotonic counter */ + +#define ERTS_MAX_UNIQUE_MONOTONIC_INTEGER_HEAP_SIZE ERTS_MAX_UINT64_HEAP_SIZE + +/* + * Note that a raw value is an intermediate value that + * not necessarily correspond to the end result. + */ +Sint64 erts_raw_get_unique_monotonic_integer(void); +Uint erts_raw_unique_monotonic_integer_heap_size(Sint64 raw); +Eterm erts_raw_make_unique_monotonic_integer_value(Eterm **hpp, Sint64 raw); + +Sint64 erts_get_min_unique_monotonic_integer(void); + +int erts_debug_set_unique_monotonic_integer_state(Eterm et_value); +Eterm erts_debug_get_unique_monotonic_integer_state(Process *c_p); + +/* unique integer */ +#define ERTS_UNIQUE_INT_RAW_VALUES 2 +#define ERTS_MAX_UNIQUE_INT_HEAP_SIZE ERTS_UINT64_ARRAY_TO_BIG_MAX_HEAP_SZ(2) + +Uint erts_raw_unique_integer_heap_size(Uint64 val[ERTS_UNIQUE_INT_RAW_VALUES]); +Eterm erts_raw_make_unique_integer(Eterm **hpp, Uint64 val[ERTS_UNIQUE_INT_RAW_VALUES]); +void erts_raw_get_unique_integer(Uint64 val[ERTS_UNIQUE_INT_RAW_VALUES]); +Sint64 erts_get_min_unique_integer(void); + +Eterm erts_debug_make_unique_integer(Process *c_p, + Eterm etval0, + Eterm etval1); + + +ERTS_GLB_INLINE void erts_set_ref_numbers(Uint32 ref[ERTS_MAX_REF_NUMBERS], + Uint32 thr_id, Uint64 value); +ERTS_GLB_INLINE Uint32 erts_get_ref_numbers_thr_id(Uint32 ref[ERTS_MAX_REF_NUMBERS]); +ERTS_GLB_INLINE Uint64 erts_get_ref_numbers_value(Uint32 ref[ERTS_MAX_REF_NUMBERS]); +ERTS_GLB_INLINE void erts_sched_make_ref_in_array(ErtsSchedulerData *esdp, + Uint32 ref[ERTS_MAX_REF_NUMBERS]); +ERTS_GLB_INLINE Eterm erts_sched_make_ref_in_buffer(ErtsSchedulerData *esdp, + Eterm buffer[REF_THING_SIZE]); + +#if ERTS_GLB_INLINE_INCL_FUNC_DEF + +ERTS_GLB_INLINE void +erts_set_ref_numbers(Uint32 ref[ERTS_MAX_REF_NUMBERS], Uint32 thr_id, Uint64 value) +{ + /* + * We cannot use thread id in the first 18-bit word since + * the hash/phash/phash2 BIFs only hash on this word. If + * we did, we would get really poor hash values. Instead + * we have to shuffle the bits a bit. + */ + ASSERT(thr_id == (thr_id & ((Uint32) 0x3ffff))); + ref[0] = (Uint32) (value & ((Uint64) 0x3ffff)); + ref[1] = (((Uint32) (value & ((Uint64) 0xfffc0000))) + | (thr_id & ((Uint32) 0x3ffff))); + ref[2] = (Uint32) ((value >> 32) & ((Uint64) 0xffffffff)); +} + +ERTS_GLB_INLINE Uint32 +erts_get_ref_numbers_thr_id(Uint32 ref[ERTS_MAX_REF_NUMBERS]) +{ + return ref[1] & ((Uint32) 0x3ffff); +} + +ERTS_GLB_INLINE Uint64 +erts_get_ref_numbers_value(Uint32 ref[ERTS_MAX_REF_NUMBERS]) +{ + return (((((Uint64) ref[2]) & ((Uint64) 0xffffffff)) << 32) + | (((Uint64) ref[1]) & ((Uint64) 0xfffc0000)) + | (((Uint64) ref[0]) & ((Uint64) 0x3ffff))); +} + +ERTS_GLB_INLINE void +erts_sched_make_ref_in_array(ErtsSchedulerData *esdp, + Uint32 ref[ERTS_MAX_REF_NUMBERS]) +{ + Uint64 value; + + ASSERT(esdp); + value = esdp->ref++; + erts_set_ref_numbers(ref, (Uint32) esdp->thr_id, value); +} + +ERTS_GLB_INLINE Eterm +erts_sched_make_ref_in_buffer(ErtsSchedulerData *esdp, + Eterm buffer[REF_THING_SIZE]) +{ + Eterm* hp = buffer; + Uint32 ref[ERTS_MAX_REF_NUMBERS]; + + erts_sched_make_ref_in_array(esdp, ref); + write_ref_thing(hp, ref[0], ref[1], ref[2]); + return make_internal_ref(hp); +} + +#endif /* ERTS_GLB_INLINE_INCL_FUNC_DEF */ + +#endif /* ERTS_BIF_UNIQUE_H__ */ diff --git a/erts/emulator/beam/erl_binary.h b/erts/emulator/beam/erl_binary.h index 506c4813fa..ea01bf08f0 100644 --- a/erts/emulator/beam/erl_binary.h +++ b/erts/emulator/beam/erl_binary.h @@ -1,18 +1,19 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2000-2011. All Rights Reserved. + * Copyright Ericsson AB 2000-2013. All Rights Reserved. * - * The contents of this file are subject to the Erlang Public License, - * Version 1.1, (the "License"); you may not use this file except in - * compliance with the License. You should have received a copy of the - * Erlang Public License along with this software. If not, it can be - * retrieved online at http://www.erlang.org/. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Software distributed under the License is distributed on an "AS IS" - * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See - * the License for the specific language governing rights and limitations - * under the License. + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. * * %CopyrightEnd% */ @@ -153,7 +154,7 @@ do { \ #define binary_bytes(Bin) \ (*binary_val(Bin) == HEADER_PROC_BIN ? \ ((ProcBin *) binary_val(Bin))->bytes : \ - (ASSERT_EXPR(thing_subtag(*binary_val(Bin)) == HEAP_BINARY_SUBTAG), \ + (ASSERT(thing_subtag(*binary_val(Bin)) == HEAP_BINARY_SUBTAG), \ (byte *)(&(((ErlHeapBin *) binary_val(Bin))->data)))) void erts_init_binary(void); @@ -166,7 +167,7 @@ Eterm erts_bin_bytes_to_list(Eterm previous, Eterm* hp, byte* bytes, Uint size, * Common implementation for erlang:list_to_binary/1 and binary:list_to_bin/1 */ -BIF_RETTYPE erts_list_to_binary_bif(Process *p, Eterm arg); +BIF_RETTYPE erts_list_to_binary_bif(Process *p, Eterm arg, Export *bif); BIF_RETTYPE erts_gc_binary_part(Process *p, Eterm *reg, Eterm live, int range_is_tuple); BIF_RETTYPE erts_binary_part(Process *p, Eterm binary, Eterm epos, Eterm elen); @@ -183,7 +184,7 @@ BIF_RETTYPE erts_binary_part(Process *p, Eterm binary, Eterm epos, Eterm elen); #endif #define ERTS_CHK_BIN_ALIGNMENT(B) \ - do { ASSERT(!(B) || (((UWord) &((Binary *)(B))->orig_bytes[0]) & ERTS_BIN_ALIGNMENT_MASK) == ((UWord) 0)) } while(0) + do { ASSERT(!(B) || (((UWord) &((Binary *)(B))->orig_bytes[0]) & ERTS_BIN_ALIGNMENT_MASK) == ((UWord) 0)); } while(0) ERTS_GLB_INLINE byte* erts_get_aligned_binary_bytes(Eterm bin, byte** base_ptr); ERTS_GLB_INLINE void erts_free_aligned_binary_bytes(byte* buf); @@ -194,6 +195,9 @@ ERTS_GLB_INLINE Binary *erts_bin_nrml_alloc(Uint size); ERTS_GLB_INLINE Binary *erts_bin_realloc_fnf(Binary *bp, Uint size); ERTS_GLB_INLINE Binary *erts_bin_realloc(Binary *bp, Uint size); ERTS_GLB_INLINE void erts_bin_free(Binary *bp); +ERTS_GLB_INLINE Binary *erts_create_magic_binary_x(Uint size, + void (*destructor)(Binary *), + int unaligned); ERTS_GLB_INLINE Binary *erts_create_magic_binary(Uint size, void (*destructor)(Binary *)); @@ -225,41 +229,64 @@ erts_free_aligned_binary_bytes(byte* buf) ** These extra bytes where earlier (< R13B04) added by an alignment-bug ** in this code. Do we dare remove this in some major release (R14?) maybe? */ -#ifdef DEBUG +#if defined(DEBUG) || defined(VALGRIND) # define CHICKEN_PAD 0 #else # define CHICKEN_PAD (sizeof(void*) - 1) #endif +/* Caller must initialize 'refc' +*/ ERTS_GLB_INLINE Binary * erts_bin_drv_alloc_fnf(Uint size) { Uint bsize = ERTS_SIZEOF_Binary(size) + CHICKEN_PAD; - void *res; + Binary *res; + + if (bsize < size) /* overflow */ + return NULL; res = erts_alloc_fnf(ERTS_ALC_T_DRV_BINARY, bsize); ERTS_CHK_BIN_ALIGNMENT(res); - return (Binary *) res; + if (res) { + res->orig_size = size; + res->flags = BIN_FLAG_DRV; + } + return res; } +/* Caller must initialize 'refc' +*/ ERTS_GLB_INLINE Binary * erts_bin_drv_alloc(Uint size) { Uint bsize = ERTS_SIZEOF_Binary(size) + CHICKEN_PAD; - void *res; + Binary *res; + + if (bsize < size) /* overflow */ + erts_alloc_enomem(ERTS_ALC_T_DRV_BINARY, size); res = erts_alloc(ERTS_ALC_T_DRV_BINARY, bsize); ERTS_CHK_BIN_ALIGNMENT(res); - return (Binary *) res; + res->orig_size = size; + res->flags = BIN_FLAG_DRV; + return res; } +/* Caller must initialize 'refc' +*/ ERTS_GLB_INLINE Binary * erts_bin_nrml_alloc(Uint size) { Uint bsize = ERTS_SIZEOF_Binary(size) + CHICKEN_PAD; - void *res; + Binary *res; + + if (bsize < size) /* overflow */ + erts_alloc_enomem(ERTS_ALC_T_BINARY, size); res = erts_alloc(ERTS_ALC_T_BINARY, bsize); ERTS_CHK_BIN_ALIGNMENT(res); - return (Binary *) res; + res->orig_size = size; + res->flags = 0; + return res; } ERTS_GLB_INLINE Binary * @@ -267,12 +294,15 @@ erts_bin_realloc_fnf(Binary *bp, Uint size) { Binary *nbp; Uint bsize = ERTS_SIZEOF_Binary(size) + CHICKEN_PAD; + ErtsAlcType_t type = (bp->flags & BIN_FLAG_DRV) ? ERTS_ALC_T_DRV_BINARY + : ERTS_ALC_T_BINARY; ASSERT((bp->flags & BIN_FLAG_MAGIC) == 0); - if (bp->flags & BIN_FLAG_DRV) - nbp = erts_realloc_fnf(ERTS_ALC_T_DRV_BINARY, (void *) bp, bsize); - else - nbp = erts_realloc_fnf(ERTS_ALC_T_BINARY, (void *) bp, bsize); + if (bsize < size) /* overflow */ + return NULL; + nbp = erts_realloc_fnf(type, (void *) bp, bsize); ERTS_CHK_BIN_ALIGNMENT(nbp); + if (nbp) + nbp->orig_size = size; return nbp; } @@ -281,18 +311,16 @@ erts_bin_realloc(Binary *bp, Uint size) { Binary *nbp; Uint bsize = ERTS_SIZEOF_Binary(size) + CHICKEN_PAD; + ErtsAlcType_t type = (bp->flags & BIN_FLAG_DRV) ? ERTS_ALC_T_DRV_BINARY + : ERTS_ALC_T_BINARY; ASSERT((bp->flags & BIN_FLAG_MAGIC) == 0); - if (bp->flags & BIN_FLAG_DRV) - nbp = erts_realloc_fnf(ERTS_ALC_T_DRV_BINARY, (void *) bp, bsize); - else - nbp = erts_realloc_fnf(ERTS_ALC_T_BINARY, (void *) bp, bsize); + if (bsize < size) /* overflow */ + erts_realloc_enomem(type, bp, size); + nbp = erts_realloc_fnf(type, (void *) bp, bsize); if (!nbp) - erts_realloc_n_enomem(ERTS_ALC_T2N(bp->flags & BIN_FLAG_DRV - ? ERTS_ALC_T_DRV_BINARY - : ERTS_ALC_T_BINARY), - bp, - bsize); + erts_realloc_enomem(type, bp, bsize); ERTS_CHK_BIN_ALIGNMENT(nbp); + nbp->orig_size = size; return nbp; } @@ -308,20 +336,30 @@ erts_bin_free(Binary *bp) } ERTS_GLB_INLINE Binary * -erts_create_magic_binary(Uint size, void (*destructor)(Binary *)) +erts_create_magic_binary_x(Uint size, void (*destructor)(Binary *), + int unaligned) { - Uint bsize = ERTS_MAGIC_BIN_SIZE(size); + Uint bsize = unaligned ? ERTS_MAGIC_BIN_UNALIGNED_SIZE(size) + : ERTS_MAGIC_BIN_SIZE(size); Binary* bptr = erts_alloc_fnf(ERTS_ALC_T_BINARY, bsize); + ASSERT(bsize > size); if (!bptr) erts_alloc_n_enomem(ERTS_ALC_T2N(ERTS_ALC_T_BINARY), bsize); ERTS_CHK_BIN_ALIGNMENT(bptr); bptr->flags = BIN_FLAG_MAGIC; - bptr->orig_size = ERTS_MAGIC_BIN_ORIG_SIZE(size); + bptr->orig_size = unaligned ? ERTS_MAGIC_BIN_UNALIGNED_ORIG_SIZE(size) + : ERTS_MAGIC_BIN_ORIG_SIZE(size); erts_refc_init(&bptr->refc, 0); ERTS_MAGIC_BIN_DESTRUCTOR(bptr) = destructor; return bptr; } +ERTS_GLB_INLINE Binary * +erts_create_magic_binary(Uint size, void (*destructor)(Binary *)) +{ + return erts_create_magic_binary_x(size, destructor, 0); +} + #endif /* #if ERTS_GLB_INLINE_INCL_FUNC_DEF */ -#endif +#endif /* !__ERL_BINARY_H */ diff --git a/erts/emulator/beam/erl_bits.c b/erts/emulator/beam/erl_bits.c index 3753b618e1..01734c55d7 100644 --- a/erts/emulator/beam/erl_bits.c +++ b/erts/emulator/beam/erl_bits.c @@ -1,18 +1,19 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 1999-2012. All Rights Reserved. + * Copyright Ericsson AB 1999-2013. All Rights Reserved. * - * The contents of this file are subject to the Erlang Public License, - * Version 1.1, (the "License"); you may not use this file except in - * compliance with the License. You should have received a copy of the - * Erlang Public License along with this software. If not, it can be - * retrieved online at http://www.erlang.org/. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Software distributed under the License is distributed on an "AS IS" - * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See - * the License for the specific language governing rights and limitations - * under the License. + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. * * %CopyrightEnd% */ @@ -107,6 +108,14 @@ erts_bits_destroy_state(ERL_BITS_PROTO_0) void erts_init_bits(void) { + ERTS_CT_ASSERT(offsetof(Binary,orig_bytes) % 8 == 0); + ERTS_CT_ASSERT(offsetof(ErtsMagicBinary,u.aligned.data) % 8 == 0); + ERTS_CT_ASSERT(ERTS_MAGIC_BIN_BYTES_TO_ALIGN == + (offsetof(ErtsMagicBinary,u.aligned.data) + - offsetof(ErtsMagicBinary,u.unaligned.data))); + ERTS_CT_ASSERT(offsetof(ErtsBinary,driver.binary.orig_bytes) + == offsetof(Binary,orig_bytes)); + erts_smp_atomic_init_nob(&bits_bufs_size, 0); #if defined(ERTS_SMP) /* erl_process.c calls erts_bits_init_state() on all state instances */ @@ -165,6 +174,26 @@ erts_bs_start_match_2(Process *p, Eterm Binary, Uint Max) return make_matchstate(ms); } +#ifdef DEBUG +# define CHECK_MATCH_BUFFER(MB) check_match_buffer(MB) + +static void check_match_buffer(ErlBinMatchBuffer* mb) +{ + Eterm realbin; + Uint byteoffs; + byte* bytes, bitoffs, bitsz; + ProcBin* pb; + ERTS_GET_REAL_BIN(mb->orig, realbin, byteoffs, bitoffs, bitsz); + bytes = binary_bytes(realbin) + byteoffs; + ERTS_ASSERT(mb->base >= bytes && mb->base <= (bytes + binary_size(mb->orig))); + pb = (ProcBin *) boxed_val(realbin); + if (pb->thing_word == HEADER_PROC_BIN) + ERTS_ASSERT(pb->flags == 0); +} +#else +# define CHECK_MATCH_BUFFER(MB) +#endif + Eterm erts_bs_get_integer_2(Process *p, Uint num_bits, unsigned flags, ErlBinMatchBuffer* mb) { @@ -185,6 +214,7 @@ erts_bs_get_integer_2(Process *p, Uint num_bits, unsigned flags, ErlBinMatchBuff return SMALL_ZERO; } + CHECK_MATCH_BUFFER(mb); if (mb->size - mb->offset < num_bits) { /* Asked for too many bits. */ return THE_NON_VALUE; } @@ -403,7 +433,10 @@ erts_bs_get_integer_2(Process *p, Uint num_bits, unsigned flags, ErlBinMatchBuff words_needed = 1+WSIZE(bytes); hp = HeapOnlyAlloc(p, words_needed); res = bytes_to_big(LSB, bytes, sgn, hp); - if (is_small(res)) { + if (is_nil(res)) { + p->htop = hp; + res = THE_NON_VALUE; + } else if (is_small(res)) { p->htop = hp; } else if ((actual = bignum_header_arity(*hp)+1) < words_needed) { p->htop = hp + actual; @@ -422,6 +455,7 @@ erts_bs_get_binary_2(Process *p, Uint num_bits, unsigned flags, ErlBinMatchBuffe { ErlSubBin* sb; + CHECK_MATCH_BUFFER(mb); if (mb->size - mb->offset < num_bits) { /* Asked for too many bits. */ return THE_NON_VALUE; } @@ -453,6 +487,7 @@ erts_bs_get_float_2(Process *p, Uint num_bits, unsigned flags, ErlBinMatchBuffer byte* fptr; FloatDef f; + CHECK_MATCH_BUFFER(mb); if (num_bits == 0) { f.fd = 0.0; hp = HeapOnlyAlloc(p, FLOAT_SIZE_OBJECT); @@ -484,8 +519,16 @@ erts_bs_get_float_2(Process *p, Uint num_bits, unsigned flags, ErlBinMatchBuffer ERTS_FP_ERROR_THOROUGH(p, f32, return THE_NON_VALUE); f.fd = f32; } else { +#ifdef DOUBLE_MIDDLE_ENDIAN + FloatDef ftmp; + ftmp.fd = f64; + f.fw[0] = ftmp.fw[1]; + f.fw[1] = ftmp.fw[0]; + ERTS_FP_ERROR_THOROUGH(p, f.fd, return THE_NON_VALUE); +#else ERTS_FP_ERROR_THOROUGH(p, f64, return THE_NON_VALUE); f.fd = f64; +#endif } mb->offset += num_bits; hp = HeapOnlyAlloc(p, FLOAT_SIZE_OBJECT); @@ -498,6 +541,8 @@ erts_bs_get_binary_all_2(Process *p, ErlBinMatchBuffer* mb) { ErlSubBin* sb; Uint size; + + CHECK_MATCH_BUFFER(mb); size = mb->size-mb->offset; sb = (ErlSubBin *) HeapOnlyAlloc(p, ERL_SUB_BIN_SIZE); sb->thing_word = HEADER_SUB_BIN; @@ -1014,8 +1059,13 @@ erts_new_bs_put_float(Process *c_p, Eterm arg, Uint num_bits, int flags) #endif } else if (is_small(arg)) { u.f64 = (double) signed_val(arg); +#ifdef DOUBLE_MIDDLE_ENDIAN + a = u.i32[1]; + b = u.i32[0]; +#else a = u.i32[0]; b = u.i32[1]; +#endif } else if (is_big(arg)) { if (big_to_double(arg, &u.f64) < 0) { return 0; @@ -1118,21 +1168,42 @@ erts_new_bs_put_float(Process *c_p, Eterm arg, Uint num_bits, int flags) byte *bptr; double f64; float f32; +#ifdef DOUBLE_MIDDLE_ENDIAN + FloatDef fbuf, ftmp; +#endif if (num_bits == 64) { if (is_float(arg)) { +#ifdef DOUBLE_MIDDLE_ENDIAN + FloatDef *fdp = (FloatDef*)(float_val(arg) + 1); + ftmp = *fdp; +#else bptr = (byte *) (float_val(arg) + 1); +#endif } else if (is_small(arg)) { f64 = (double) signed_val(arg); +#ifdef DOUBLE_MIDDLE_ENDIAN + ftmp.fd = f64; +#else bptr = (byte *) &f64; +#endif } else if (is_big(arg)) { if (big_to_double(arg, &f64) < 0) { return 0; } +#ifdef DOUBLE_MIDDLE_ENDIAN + ftmp.fd = f64; +#else bptr = (byte *) &f64; +#endif } else { return 0; } +#ifdef DOUBLE_MIDDLE_ENDIAN + fbuf.fw[0] = ftmp.fw[1]; + fbuf.fw[1] = ftmp.fw[0]; + bptr = fbuf.fb; +#endif } else if (num_bits == 32) { if (is_float(arg)) { FloatDef f; @@ -1265,7 +1336,6 @@ erts_bs_append(Process* c_p, Eterm* reg, Uint live, Eterm build_size_term, if (binp->orig_size < pb->size) { Uint new_size = 2*pb->size; binp = erts_bin_realloc(binp, new_size); - binp->orig_size = new_size; pb->val = binp; pb->bytes = (byte *) binp->orig_bytes; } @@ -1337,8 +1407,6 @@ erts_bs_append(Process* c_p, Eterm* reg, Uint live, Eterm build_size_term, * Allocate the binary data struct itself. */ bptr = erts_bin_nrml_alloc(bin_size); - bptr->flags = 0; - bptr->orig_size = bin_size; erts_refc_init(&bptr->refc, 1); erts_current_bin = (byte *) bptr->orig_bytes; @@ -1441,7 +1509,6 @@ erts_bs_private_append(Process* p, Eterm bin, Eterm build_size_term, Uint unit) * is safe to reallocate it. */ binp = erts_bin_realloc(binp, new_size); - binp->orig_size = new_size; pb->val = binp; pb->bytes = (byte *) binp->orig_bytes; } else { @@ -1454,10 +1521,8 @@ erts_bs_private_append(Process* p, Eterm bin, Eterm build_size_term, Uint unit) * binary and copy the contents of the old binary into it. */ Binary* bptr = erts_bin_nrml_alloc(new_size); - bptr->flags = 0; - bptr->orig_size = new_size; erts_refc_init(&bptr->refc, 1); - sys_memcpy(bptr->orig_bytes, binp->orig_bytes, pb->size); + sys_memcpy(bptr->orig_bytes, binp->orig_bytes, binp->orig_size); pb->flags |= PB_IS_WRITABLE | PB_ACTIVE_WRITER; pb->val = bptr; pb->bytes = (byte *) bptr->orig_bytes; @@ -1503,8 +1568,6 @@ erts_bs_init_writable(Process* p, Eterm sz) * Allocate the binary data struct itself. */ bptr = erts_bin_nrml_alloc(bin_size); - bptr->flags = 0; - bptr->orig_size = bin_size; erts_refc_init(&bptr->refc, 1); /* @@ -1551,9 +1614,7 @@ erts_emasculate_writable_binary(ProcBin* pb) /* Our allocators are 8 byte aligned, i.e., shrinking with less than 8 bytes will have no real effect */ if (unused >= 8) { - Uint new_size = pb->size; binp = erts_bin_realloc(binp, pb->size); - binp->orig_size = new_size; pb->val = binp; pb->bytes = (byte *) binp->orig_bytes; } @@ -1568,6 +1629,7 @@ erts_bs_get_unaligned_uint32(ErlBinMatchBuffer* mb) byte* LSB; byte* MSB; + CHECK_MATCH_BUFFER(mb); ASSERT((mb->offset & 7) != 0); ASSERT(mb->size - mb->offset >= 32); @@ -1627,6 +1689,8 @@ erts_bs_get_utf8(ErlBinMatchBuffer* mb) 2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2, 3,3,3,3,3,3,3,3,9,9,9,9,9,9,9,9 }; + CHECK_MATCH_BUFFER(mb); + if ((remaining_bits = mb->size - mb->offset) < 8) { return THE_NON_VALUE; } @@ -1711,6 +1775,7 @@ erts_bs_get_utf16(ErlBinMatchBuffer* mb, Uint flags) return THE_NON_VALUE; } + CHECK_MATCH_BUFFER(mb); /* * Set up the pointer to the source bytes. */ @@ -1776,6 +1841,11 @@ erts_cmp_bits(byte* a_ptr, size_t a_offs, byte* b_ptr, size_t b_offs, size_t siz Uint rshift; int cmp; + ASSERT(a_offs < 8 && b_offs < 8); + + if (size == 0) + return 0; + if (((a_offs | b_offs | size) & 7) == 0) { int byte_size = size >> 3; return sys_memcmp(a_ptr, b_ptr, byte_size); @@ -1784,58 +1854,72 @@ erts_cmp_bits(byte* a_ptr, size_t a_offs, byte* b_ptr, size_t b_offs, size_t siz /* Compare bit by bit until a_ptr is aligned on byte boundary */ a = *a_ptr++; b = *b_ptr++; - while (size > 0) { - a_bit = get_bit(a, a_offs); - b_bit = get_bit(b, b_offs); - if ((cmp = (a_bit-b_bit)) != 0) { - return cmp; - } - size--; - b_offs++; - if (b_offs == 8) { - b_offs = 0; - b = *b_ptr++; - } - a_offs++; - if (a_offs == 8) { - a_offs = 0; - a = *a_ptr++; - break; + if (a_offs) { + for (;;) { + a_bit = get_bit(a, a_offs); + b_bit = get_bit(b, b_offs); + if ((cmp = (a_bit-b_bit)) != 0) { + return cmp; + } + if (--size == 0) + return 0; + + b_offs++; + if (b_offs == 8) { + b_offs = 0; + b = *b_ptr++; + } + a_offs++; + if (a_offs == 8) { + a_offs = 0; + a = *a_ptr++; + break; + } } } /* Compare byte by byte as long as at least 8 bits remain */ - lshift = b_offs; - rshift = 8 - lshift; - while (size >= 8) { - byte b_cmp = (b << lshift); - b = *b_ptr++; - b_cmp |= b >> rshift; - if ((cmp = (a - b_cmp)) != 0) { - return cmp; - } + if (size >= 8) { + lshift = b_offs; + rshift = 8 - lshift; + for (;;) { + byte b_cmp = (b << lshift); + b = *b_ptr++; + b_cmp |= b >> rshift; + if ((cmp = (a - b_cmp)) != 0) { + return cmp; + } + size -= 8; + if (size < 8) + break; + a = *a_ptr++; + } + + if (size == 0) + return 0; a = *a_ptr++; - size -= 8; } /* Compare the remaining bits bit by bit */ - while (size > 0) { - a_bit = get_bit(a, a_offs); - b_bit = get_bit(b, b_offs); - if ((cmp = (a_bit-b_bit)) != 0) { - return cmp; - } - a_offs++; - if (a_offs == 8) { - a_offs = 0; - a = *a_ptr++; - } - b_offs++; - if (b_offs == 8) { - b_offs = 0; - b = *b_ptr++; - } - size--; + if (size > 0) { + for (;;) { + a_bit = get_bit(a, a_offs); + b_bit = get_bit(b, b_offs); + if ((cmp = (a_bit-b_bit)) != 0) { + return cmp; + } + if (--size == 0) + return 0; + + a_offs++; + ASSERT(a_offs < 8); + + b_offs++; + if (b_offs == 8) { + b_offs = 0; + b = *b_ptr++; + } + } } return 0; diff --git a/erts/emulator/beam/erl_bits.h b/erts/emulator/beam/erl_bits.h index 388d943755..8b7807fbd9 100644 --- a/erts/emulator/beam/erl_bits.h +++ b/erts/emulator/beam/erl_bits.h @@ -3,16 +3,17 @@ * * Copyright Ericsson AB 1999-2011. All Rights Reserved. * - * The contents of this file are subject to the Erlang Public License, - * Version 1.1, (the "License"); you may not use this file except in - * compliance with the License. You should have received a copy of the - * Erlang Public License along with this software. If not, it can be - * retrieved online at http://www.erlang.org/. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Software distributed under the License is distributed on an "AS IS" - * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See - * the License for the specific language governing rights and limitations - * under the License. + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. * * %CopyrightEnd% */ diff --git a/erts/emulator/beam/erl_cpu_topology.c b/erts/emulator/beam/erl_cpu_topology.c index 88c6c34881..c263eebfcc 100644 --- a/erts/emulator/beam/erl_cpu_topology.c +++ b/erts/emulator/beam/erl_cpu_topology.c @@ -3,16 +3,17 @@ * * Copyright Ericsson AB 2010-2013. All Rights Reserved. * - * The contents of this file are subject to the Erlang Public License, - * Version 1.1, (the "License"); you may not use this file except in - * compliance with the License. You should have received a copy of the - * Erlang Public License along with this software. If not, it can be - * retrieved online at http://www.erlang.org/. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Software distributed under the License is distributed on an "AS IS" - * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See - * the License for the specific language governing rights and limitations - * under the License. + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. * * %CopyrightEnd% */ @@ -401,7 +402,7 @@ cpu_bind_order_sort(erts_cpu_topology_t *cpudata, break; default: cmp_func = NULL; - erl_exit(ERTS_ABORT_EXIT, + erts_exit(ERTS_ABORT_EXIT, "Bad cpu bind type: %d\n", (int) cpu_bind_order); break; @@ -1589,7 +1590,7 @@ get_cpu_topology_term(Process *c_p, int type) } break; default: - erl_exit(ERTS_ABORT_EXIT, "Bad cpu topology type: %d\n", type); + erts_exit(ERTS_ABORT_EXIT, "Bad cpu topology type: %d\n", type); break; } @@ -1699,7 +1700,7 @@ erts_early_init_cpu_topology(int no_schedulers, } max_main_threads = erts_get_cpu_configured(cpuinfo); - if (max_main_threads > no_schedulers) + if (max_main_threads > no_schedulers || max_main_threads < 0) max_main_threads = no_schedulers; *max_main_threads_p = max_main_threads; @@ -1966,7 +1967,7 @@ cpu_group_insert(erts_cpu_groups_map_t *map, ix = 0; } while (ix != start); - erl_exit(ERTS_ABORT_EXIT, "Reader groups map full\n"); + erts_exit(ERTS_ABORT_EXIT, "Reader groups map full\n"); } @@ -2289,7 +2290,7 @@ remove_cpu_groups(erts_cpu_groups_callback_t callback, void *arg) prev_cgm = cgm; } - erl_exit(ERTS_ABORT_EXIT, "Cpu groups not found\n"); + erts_exit(ERTS_ABORT_EXIT, "Cpu groups not found\n"); } static int @@ -2319,7 +2320,7 @@ cpu_groups_lookup(erts_cpu_groups_map_t *map, ix = 0; } while (ix != start); - erl_exit(ERTS_ABORT_EXIT, "Logical cpu id %d not found\n", logical); + erts_exit(ERTS_ABORT_EXIT, "Logical cpu id %d not found\n", logical); } static void diff --git a/erts/emulator/beam/erl_cpu_topology.h b/erts/emulator/beam/erl_cpu_topology.h index b502258dae..f3fbfc6da0 100644 --- a/erts/emulator/beam/erl_cpu_topology.h +++ b/erts/emulator/beam/erl_cpu_topology.h @@ -3,16 +3,17 @@ * * Copyright Ericsson AB 2010-2013. All Rights Reserved. * - * The contents of this file are subject to the Erlang Public License, - * Version 1.1, (the "License"); you may not use this file except in - * compliance with the License. You should have received a copy of the - * Erlang Public License along with this software. If not, it can be - * retrieved online at http://www.erlang.org/. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Software distributed under the License is distributed on an "AS IS" - * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See - * the License for the specific language governing rights and limitations - * under the License. + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. * * %CopyrightEnd% */ diff --git a/erts/emulator/beam/erl_db.c b/erts/emulator/beam/erl_db.c index d0afc8ed8d..870b556851 100644 --- a/erts/emulator/beam/erl_db.c +++ b/erts/emulator/beam/erl_db.c @@ -1,18 +1,19 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 1996-2013. All Rights Reserved. + * Copyright Ericsson AB 1996-2014. All Rights Reserved. * - * The contents of this file are subject to the Erlang Public License, - * Version 1.1, (the "License"); you may not use this file except in - * compliance with the License. You should have received a copy of the - * Erlang Public License along with this software. If not, it can be - * retrieved online at http://www.erlang.org/. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Software distributed under the License is distributed on an "AS IS" - * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See - * the License for the specific language governing rights and limitations - * under the License. + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. * * %CopyrightEnd% */ @@ -125,6 +126,7 @@ get_meta_main_tab_lock(unsigned slot) static erts_smp_spinlock_t meta_main_tab_main_lock; static Uint meta_main_tab_first_free; /* Index of first free slot */ static int meta_main_tab_cnt; /* Number of active tables */ +static int meta_main_tab_top; /* Highest ever used slot + 1 */ static Uint meta_main_tab_slot_mask; /* The slot index part of an unnamed table id */ static Uint meta_main_tab_seq_incr; static Uint meta_main_tab_seq_cnt = 0; /* To give unique(-ish) table identifiers */ @@ -258,15 +260,17 @@ static void schedule_free_dbtable(DbTable* tb) /* * NON-SMP case: Caller is *not* allowed to access the *tb * structure after this function has returned! - * SMP case: Caller is allowed to access the *tb structure - * until the bif has returned (we typically - * need to unlock the table lock after this - * function has returned). + * SMP case: Caller is allowed to access the *common* part of the *tb + * structure until the bif has returned (we typically need to + * unlock the table lock after this function has returned). + * Caller is *not* allowed to access the specialized part + * (hash or tree) of *tb after this function has returned. */ ASSERT(erts_refc_read(&tb->common.ref, 0) == 0); - erts_schedule_thr_prgr_later_op(free_dbtable, - (void *) tb, - &tb->release.data); + erts_schedule_thr_prgr_later_cleanup_op(free_dbtable, + (void *) tb, + &tb->release.data, + sizeof(DbTable)); } static ERTS_INLINE void db_init_lock(DbTable* tb, int use_frequent_read_lock, @@ -276,6 +280,8 @@ static ERTS_INLINE void db_init_lock(DbTable* tb, int use_frequent_read_lock, erts_smp_rwmtx_opt_t rwmtx_opt = ERTS_SMP_RWMTX_OPT_DEFAULT_INITER; if (use_frequent_read_lock) rwmtx_opt.type = ERTS_SMP_RWMTX_TYPE_FREQUENT_READ; + if (erts_ets_rwmtx_spin_count >= 0) + rwmtx_opt.main_spincount = erts_ets_rwmtx_spin_count; #endif #ifdef ERTS_SMP erts_smp_rwmtx_init_opt_x(&tb->common.rwlock, &rwmtx_opt, @@ -617,7 +623,7 @@ BIF_RETTYPE ets_safe_fixtable_2(BIF_ALIST_2) erts_fprintf(stderr, "ets:safe_fixtable(%T,%T); Process: %T, initial: %T:%T/%bpu\n", BIF_ARG_1, BIF_ARG_2, BIF_P->common.id, - BIF_P->initial[0], BIF_P->initial[1], BIF_P->initial[2]); + BIF_P->u.initial[0], BIF_P->u.initial[1], BIF_P->u.initial[2]); #endif kind = (BIF_ARG_2 == am_true) ? LCK_READ : LCK_WRITE_REC; @@ -750,6 +756,31 @@ BIF_RETTYPE ets_prev_2(BIF_ALIST_2) BIF_RET(ret); } +/* +** take(Tab, Key) +*/ +BIF_RETTYPE ets_take_2(BIF_ALIST_2) +{ + DbTable* tb; +#ifdef DEBUG + int cret; +#endif + Eterm ret; + CHECK_TABLES(); + + tb = db_get_table(BIF_P, BIF_ARG_1, DB_WRITE, LCK_WRITE_REC); + if (!tb) { + BIF_ERROR(BIF_P, BADARG); + } +#ifdef DEBUG + cret = +#endif + tb->common.meth->db_take(BIF_P, tb, BIF_ARG_2, &ret); + ASSERT(cret == DB_ERROR_NONE); + db_unlock(tb, LCK_WRITE_REC); + BIF_RET(ret); +} + /* ** update_element(Tab, Key, {Pos, Value}) ** update_element(Tab, Key, [{Pos, Value}]) @@ -777,7 +808,7 @@ BIF_RETTYPE ets_update_element_3(BIF_ALIST_3) list = BIF_ARG_3; } - if (!tb->common.meth->db_lookup_dbterm(tb, BIF_ARG_2, &handle)) { + if (!tb->common.meth->db_lookup_dbterm(BIF_P, tb, BIF_ARG_2, THE_NON_VALUE, &handle)) { cret = DB_ERROR_BADKEY; goto bail_out; } @@ -816,7 +847,7 @@ BIF_RETTYPE ets_update_element_3(BIF_ALIST_3) } finalize: - tb->common.meth->db_finalize_dbterm(&handle); + tb->common.meth->db_finalize_dbterm(cret, &handle); bail_out: UnUseTmpHeap(2,BIF_P); @@ -835,14 +866,8 @@ bail_out: } } -/* -** update_counter(Tab, Key, Incr) -** update_counter(Tab, Key, {Upop}) -** update_counter(Tab, Key, [{Upop}]) -** Upop = {Pos,Incr} | {Pos,Incr,Threshold,WarpTo} -** Returns new value(s) (integer or [integer]) -*/ -BIF_RETTYPE ets_update_counter_3(BIF_ALIST_3) +static BIF_RETTYPE +do_update_counter(Process *p, Eterm arg1, Eterm arg2, Eterm arg3, Eterm arg4) { DbTable* tb; int cret = DB_ERROR_BADITEM; @@ -852,7 +877,7 @@ BIF_RETTYPE ets_update_counter_3(BIF_ALIST_3) Eterm* ret_list_currp = NULL; Eterm* ret_list_prevp = NULL; Eterm iter; - DeclareTmpHeap(cell,5,BIF_P); + DeclareTmpHeap(cell, 5, p); Eterm *tuple = cell+2; DbUpdateHandle handle; Uint halloc_size = 0; /* overestimated heap usage */ @@ -860,28 +885,29 @@ BIF_RETTYPE ets_update_counter_3(BIF_ALIST_3) Eterm* hstart; Eterm* hend; - if ((tb = db_get_table(BIF_P, BIF_ARG_1, DB_WRITE, LCK_WRITE_REC)) == NULL) { - BIF_ERROR(BIF_P, BADARG); + if ((tb = db_get_table(p, arg1, DB_WRITE, LCK_WRITE_REC)) == NULL) { + BIF_ERROR(p, BADARG); } - UseTmpHeap(5,BIF_P); + UseTmpHeap(5, p); if (!(tb->common.status & (DB_SET | DB_ORDERED_SET))) { goto bail_out; } - if (is_integer(BIF_ARG_3)) { /* Incr */ - upop_list = CONS(cell, TUPLE2(tuple, make_small(tb->common.keypos+1), - BIF_ARG_3), NIL); + if (is_integer(arg3)) { /* Incr */ + upop_list = CONS(cell, + TUPLE2(tuple, make_small(tb->common.keypos+1), arg3), + NIL); } - else if (is_tuple(BIF_ARG_3)) { /* {Upop} */ - upop_list = CONS(cell, BIF_ARG_3, NIL); + else if (is_tuple(arg3)) { /* {Upop} */ + upop_list = CONS(cell, arg3, NIL); } else { /* [{Upop}] (probably) */ - upop_list = BIF_ARG_3; + upop_list = arg3; ret_list_prevp = &ret; } - if (!tb->common.meth->db_lookup_dbterm(tb, BIF_ARG_2, &handle)) { + if (!tb->common.meth->db_lookup_dbterm(p, tb, arg2, arg4, &handle)) { goto bail_out; /* key not found */ } @@ -954,13 +980,13 @@ BIF_RETTYPE ets_update_counter_3(BIF_ALIST_3) if (ret_list_prevp) { /* Prepare to return a list */ ret = NIL; halloc_size += list_size; - hstart = HAlloc(BIF_P, halloc_size); + hstart = HAlloc(p, halloc_size); ret_list_currp = hstart; htop = hstart + list_size; hend = hstart + halloc_size; } else { - hstart = htop = HAlloc(BIF_P, halloc_size); + hstart = htop = HAlloc(p, halloc_size); } hend = hstart + halloc_size; @@ -1007,26 +1033,54 @@ BIF_RETTYPE ets_update_counter_3(BIF_ALIST_3) (is_list(ret) && (list_val(ret)+list_size)==ret_list_currp)); ASSERT(htop <= hend); - HRelease(BIF_P,hend,htop); + HRelease(p, hend, htop); finalize: - tb->common.meth->db_finalize_dbterm(&handle); + tb->common.meth->db_finalize_dbterm(cret, &handle); bail_out: - UnUseTmpHeap(5,BIF_P); + UnUseTmpHeap(5, p); db_unlock(tb, LCK_WRITE_REC); switch (cret) { case DB_ERROR_NONE: BIF_RET(ret); case DB_ERROR_SYSRES: - BIF_ERROR(BIF_P, SYSTEM_LIMIT); + BIF_ERROR(p, SYSTEM_LIMIT); default: - BIF_ERROR(BIF_P, BADARG); + BIF_ERROR(p, BADARG); break; } } +/* +** update_counter(Tab, Key, Incr) +** update_counter(Tab, Key, Upop) +** update_counter(Tab, Key, [{Upop}]) +** Upop = {Pos,Incr} | {Pos,Incr,Threshold,WarpTo} +** Returns new value(s) (integer or [integer]) +*/ +BIF_RETTYPE ets_update_counter_3(BIF_ALIST_3) +{ + return do_update_counter(BIF_P, BIF_ARG_1, BIF_ARG_2, BIF_ARG_3, THE_NON_VALUE); +} + +/* +** update_counter(Tab, Key, Incr, Default) +** update_counter(Tab, Key, Upop, Default) +** update_counter(Tab, Key, [{Upop}], Default) +** Upop = {Pos,Incr} | {Pos,Incr,Threshold,WarpTo} +** Returns new value(s) (integer or [integer]) +*/ +BIF_RETTYPE ets_update_counter_4(BIF_ALIST_4) +{ + if (is_not_tuple(BIF_ARG_4)) { + BIF_ERROR(BIF_P, BADARG); + } + return do_update_counter(BIF_P, BIF_ARG_1, BIF_ARG_2, BIF_ARG_3, BIF_ARG_4); +} + + /* ** The put BIF */ @@ -1196,7 +1250,7 @@ BIF_RETTYPE ets_rename_2(BIF_ALIST_2) erts_fprintf(stderr, "ets:rename(%T,%T); Process: %T, initial: %T:%T/%bpu\n", BIF_ARG_1, BIF_ARG_2, BIF_P->common.id, - BIF_P->initial[0], BIF_P->initial[1], BIF_P->initial[2]); + BIF_P->u.initial[0], BIF_P->u.initial[1], BIF_P->u.initial[2]); #endif @@ -1241,7 +1295,7 @@ BIF_RETTYPE ets_rename_2(BIF_ALIST_2) goto badarg; if (!remove_named_tab(tb, 1)) - erl_exit(1,"Could not find named tab %s", tb->common.id); + erts_exit(ERTS_ERROR_EXIT,"Could not find named tab %s", tb->common.id); tb->common.id = tb->common.the_name = BIF_ARG_2; @@ -1322,7 +1376,6 @@ BIF_RETTYPE ets_new_2(BIF_ALIST_2) status |= DB_ORDERED_SET; status &= ~(DB_SET | DB_BAG | DB_DUPLICATE_BAG); } - /*TT*/ else if (is_tuple(val)) { Eterm *tp = tuple_val(val); if (arityval(tp[0]) == 2) { @@ -1468,6 +1521,10 @@ BIF_RETTYPE ets_new_2(BIF_ALIST_2) ASSERT(slot>=0 && slot<db_max_tabs); meta_main_tab_first_free = GET_NEXT_FREE_SLOT(slot); meta_main_tab_cnt++; + if (slot >= meta_main_tab_top) { + ASSERT(slot == meta_main_tab_top); + meta_main_tab_top = slot + 1; + } if (is_named) { ret = BIF_ARG_1; @@ -1508,7 +1565,7 @@ BIF_RETTYPE ets_new_2(BIF_ALIST_2) erts_fprintf(stderr, "ets:new(%T,%T)=%T; Process: %T, initial: %T:%T/%bpu\n", BIF_ARG_1, BIF_ARG_2, ret, BIF_P->common.id, - BIF_P->initial[0], BIF_P->initial[1], BIF_P->initial[2]); + BIF_P->u.initial[0], BIF_P->u.initial[1], BIF_P->u.initial[2]); erts_fprintf(stderr, "ets: new: meta_pid_to_tab common.memory_size = %ld\n", erts_smp_atomic_read_nob(&meta_pid_to_tab->common.memory_size)); erts_fprintf(stderr, "ets: new: meta_pid_to_fixed_tab common.memory_size = %ld\n", @@ -1523,7 +1580,7 @@ BIF_RETTYPE ets_new_2(BIF_ALIST_2) BIF_P->common.id, make_small(slot)), 0) != DB_ERROR_NONE) { - erl_exit(1,"Could not update ets metadata."); + erts_exit(ERTS_ERROR_EXIT,"Could not update ets metadata."); } db_meta_unlock(meta_pid_to_tab, LCK_WRITE_REC); @@ -1641,7 +1698,7 @@ BIF_RETTYPE ets_delete_1(BIF_ALIST_1) erts_fprintf(stderr, "ets:delete(%T); Process: %T, initial: %T:%T/%bpu\n", BIF_ARG_1, BIF_P->common.id, - BIF_P->initial[0], BIF_P->initial[1], BIF_P->initial[2]); + BIF_P->u.initial[0], BIF_P->u.initial[1], BIF_P->u.initial[2]); #endif CHECK_TABLES(); @@ -2057,27 +2114,31 @@ BIF_RETTYPE ets_all_0(BIF_ALIST_0) { DbTable* tb; Eterm previous; - int i, j; + int i; Eterm* hp; Eterm* hendp; int t_tabs_cnt; - int t_max_tabs; + int t_top; erts_smp_spin_lock(&meta_main_tab_main_lock); t_tabs_cnt = meta_main_tab_cnt; - t_max_tabs = db_max_tabs; + t_top = meta_main_tab_top; erts_smp_spin_unlock(&meta_main_tab_main_lock); hp = HAlloc(BIF_P, 2*t_tabs_cnt); hendp = hp + 2*t_tabs_cnt; previous = NIL; - j = 0; - for(i = 0; (i < t_max_tabs && j < t_tabs_cnt); i++) { + for(i = 0; i < t_top; i++) { erts_smp_rwmtx_t *mmtl = get_meta_main_tab_lock(i); erts_smp_rwmtx_rlock(mmtl); if (IS_SLOT_ALIVE(i)) { - j++; + if (hp == hendp) { + /* Racing table creator, grab some more heap space */ + t_tabs_cnt = 10; + hp = HAlloc(BIF_P, 2*t_tabs_cnt); + hendp = hp + 2*t_tabs_cnt; + } tb = meta_main_tab[i].u.tb; previous = CONS(hp, tb->common.id, previous); hp += 2; @@ -2235,7 +2296,7 @@ static BIF_RETTYPE ets_select_trap_1(BIF_ALIST_1) CHECK_TABLES(); tptr = tuple_val(a1); - ASSERT(arityval(*tptr) >= 1) + ASSERT(arityval(*tptr) >= 1); if ((tb = db_get_table(p, tptr[1], DB_READ, kind)) == NULL) { BIF_ERROR(p, BADARG); @@ -2402,7 +2463,7 @@ static BIF_RETTYPE ets_select_count_1(BIF_ALIST_1) CHECK_TABLES(); tptr = tuple_val(a1); - ASSERT(arityval(*tptr) >= 1) + ASSERT(arityval(*tptr) >= 1); if ((tb = db_get_table(p, tptr[1], DB_READ, kind)) == NULL) { BIF_ERROR(p, BADARG); } @@ -2632,7 +2693,9 @@ BIF_RETTYPE ets_match_object_3(BIF_ALIST_3) BIF_RETTYPE ets_info_1(BIF_ALIST_1) { static Eterm fields[] = {am_protection, am_keypos, am_type, am_named_table, - am_node, am_size, am_name, am_heir, am_owner, am_memory, am_compressed}; + am_node, am_size, am_name, am_heir, am_owner, am_memory, am_compressed, + am_write_concurrency, + am_read_concurrency}; Eterm results[sizeof(fields)/sizeof(Eterm)]; DbTable* tb; Eterm res; @@ -2795,10 +2858,11 @@ BIF_RETTYPE ets_match_spec_run_r_3(BIF_ALIST_3) ** External interface (NOT BIF's) */ +int erts_ets_rwmtx_spin_count = -1; /* Init the db */ -void init_db(void) +void init_db(ErtsDbSpinCount db_spin_count) { DbTable init_tb; int i; @@ -2807,10 +2871,48 @@ void init_db(void) size_t size; #ifdef ERTS_SMP + int max_spin_count = (1 << 15) - 1; /* internal limit */ erts_smp_rwmtx_opt_t rwmtx_opt = ERTS_SMP_RWMTX_OPT_DEFAULT_INITER; rwmtx_opt.type = ERTS_SMP_RWMTX_TYPE_FREQUENT_READ; rwmtx_opt.lived = ERTS_SMP_RWMTX_LONG_LIVED; + switch (db_spin_count) { + case ERTS_DB_SPNCNT_NONE: + erts_ets_rwmtx_spin_count = 0; + break; + case ERTS_DB_SPNCNT_VERY_LOW: + erts_ets_rwmtx_spin_count = 100; + break; + case ERTS_DB_SPNCNT_LOW: + erts_ets_rwmtx_spin_count = 200; + erts_ets_rwmtx_spin_count += erts_no_schedulers * 50; + if (erts_ets_rwmtx_spin_count > 1000) + erts_ets_rwmtx_spin_count = 1000; + break; + case ERTS_DB_SPNCNT_HIGH: + erts_ets_rwmtx_spin_count = 2000; + erts_ets_rwmtx_spin_count += erts_no_schedulers * 100; + if (erts_ets_rwmtx_spin_count > 15000) + erts_ets_rwmtx_spin_count = 15000; + break; + case ERTS_DB_SPNCNT_VERY_HIGH: + erts_ets_rwmtx_spin_count = 15000; + erts_ets_rwmtx_spin_count += erts_no_schedulers * 500; + if (erts_ets_rwmtx_spin_count > max_spin_count) + erts_ets_rwmtx_spin_count = max_spin_count; + break; + case ERTS_DB_SPNCNT_EXTREMELY_HIGH: + erts_ets_rwmtx_spin_count = max_spin_count; + break; + case ERTS_DB_SPNCNT_NORMAL: + default: + erts_ets_rwmtx_spin_count = -1; + break; + } + + if (erts_ets_rwmtx_spin_count >= 0) + rwmtx_opt.main_spincount = erts_ets_rwmtx_spin_count; + meta_main_tab_locks = erts_alloc_permanent_cache_aligned(ERTS_ALC_T_DB_TABLES, sizeof(erts_meta_main_tab_lock_t) @@ -2837,7 +2939,7 @@ void init_db(void) bits = erts_fit_in_bits_int32(db_max_tabs-1); if (bits > SMALL_BITS) { - erl_exit(1,"Max limit for ets tabled too high %u (max %u).", + erts_exit(ERTS_ERROR_EXIT,"Max limit for ets tabled too high %u (max %u).", db_max_tabs, ((Uint)1)<<SMALL_BITS); } meta_main_tab_slot_mask = (((Uint)1)<<bits) - 1; @@ -2848,6 +2950,7 @@ void init_db(void) ERTS_ETS_MISC_MEM_ADD(size); meta_main_tab_cnt = 0; + meta_main_tab_top = 0; for (i=1; i<db_max_tabs; i++) { SET_NEXT_FREE_SLOT(i-1,i); } @@ -2897,7 +3000,7 @@ void init_db(void) db_init_lock(meta_pid_to_tab, "meta_pid_to_tab", "meta_pid_to_tab_FIX");*/ if (db_create_hash(NULL, meta_pid_to_tab) != DB_ERROR_NONE) { - erl_exit(1,"Unable to create ets metadata tables."); + erts_exit(ERTS_ERROR_EXIT,"Unable to create ets metadata tables."); } erts_smp_atomic_set_nob(&init_tb.common.memory_size, 0); @@ -2928,7 +3031,7 @@ void init_db(void) db_init_lock(meta_pid_to_fixed_tab, "meta_pid_to_fixed_tab", "meta_pid_to_fixed_tab_FIX");*/ if (db_create_hash(NULL, meta_pid_to_fixed_tab) != DB_ERROR_NONE) { - erl_exit(1,"Unable to create ets metadata tables."); + erts_exit(ERTS_ERROR_EXIT,"Unable to create ets metadata tables."); } /* Non visual BIF to trap to. */ @@ -3125,7 +3228,7 @@ retry: * yielding. */ #define ERTS_DB_INTERNAL_ERROR(LSTR) \ - erl_exit(ERTS_ABORT_EXIT, "%s:%d:erts_db_process_exiting(): " LSTR "\n", \ + erts_exit(ERTS_ABORT_EXIT, "%s:%d:erts_db_process_exiting(): " LSTR "\n", \ __FILE__, __LINE__) int @@ -3268,34 +3371,37 @@ erts_db_process_exiting(Process *c_p, ErtsProcLocks c_p_locks) } erts_smp_rwmtx_runlock(mmtl); if (tb) { - int reds; - DbFixation** pp; + int reds = 0; db_lock(tb, LCK_WRITE_REC); - #ifdef ERTS_SMP - erts_smp_mtx_lock(&tb->common.fixlock); - #endif - reds = 10; - - for (pp = &tb->common.fixations; *pp != NULL; - pp = &(*pp)->next) { - if ((*pp)->pid == pid) { - DbFixation* fix = *pp; - erts_aint_t diff = -((erts_aint_t) fix->counter); - erts_refc_add(&tb->common.ref,diff,0); - *pp = fix->next; - erts_db_free(ERTS_ALC_T_DB_FIXATION, - tb, fix, sizeof(DbFixation)); - ERTS_ETS_MISC_MEM_ADD(-sizeof(DbFixation)); - break; + if (!(tb->common.status & DB_DELETE)) { + DbFixation** pp; + + #ifdef ERTS_SMP + erts_smp_mtx_lock(&tb->common.fixlock); + #endif + reds = 10; + + for (pp = &tb->common.fixations; *pp != NULL; + pp = &(*pp)->next) { + if ((*pp)->pid == pid) { + DbFixation* fix = *pp; + erts_aint_t diff = -((erts_aint_t) fix->counter); + erts_refc_add(&tb->common.ref,diff,0); + *pp = fix->next; + erts_db_free(ERTS_ALC_T_DB_FIXATION, + tb, fix, sizeof(DbFixation)); + ERTS_ETS_MISC_MEM_ADD(-sizeof(DbFixation)); + break; + } + } + #ifdef ERTS_SMP + erts_smp_mtx_unlock(&tb->common.fixlock); + #endif + if (!IS_FIXED(tb) && IS_HASH_TABLE(tb->common.status)) { + db_unfix_table_hash(&(tb->hash)); + reds += 40; } - } - #ifdef ERTS_SMP - erts_smp_mtx_unlock(&tb->common.fixlock); - #endif - if (!IS_FIXED(tb) && IS_HASH_TABLE(tb->common.status)) { - db_unfix_table_hash(&(tb->hash)); - reds += 40; } db_unlock(tb, LCK_WRITE_REC); BUMP_REDS(c_p, reds); @@ -3359,10 +3465,10 @@ static void fix_table_locked(Process* p, DbTable* tb) #endif erts_refc_inc(&tb->common.ref,1); fix = tb->common.fixations; - if (fix == NULL) { - get_now(&(tb->common.megasec), - &(tb->common.sec), - &(tb->common.microsec)); + if (fix == NULL) { + tb->common.time.monotonic + = erts_get_monotonic_time(ERTS_PROC_GET_SCHDATA(p)); + tb->common.time.offset = erts_get_time_offset(); } else { for (; fix != NULL; fix = fix->next) { @@ -3394,7 +3500,7 @@ static void fix_table_locked(Process* p, DbTable* tb) make_small(tb->common.slot)), 0) != DB_ERROR_NONE) { UnUseTmpHeap(3,p); - erl_exit(1,"Could not insert ets metadata in safe_fixtable."); + erts_exit(ERTS_ERROR_EXIT,"Could not insert ets metadata in safe_fixtable."); } UnUseTmpHeap(3,p); db_meta_unlock(meta_pid_to_fixed_tab, LCK_WRITE_REC); @@ -3624,6 +3730,7 @@ static int free_table_cont(Process *p, static Eterm table_info(Process* p, DbTable* tb, Eterm What) { Eterm ret = THE_NON_VALUE; + int use_monotonic; if (What == am_size) { ret = make_small(erts_smp_atomic_read_nob(&tb->common.nitems)); @@ -3655,6 +3762,10 @@ static Eterm table_info(Process* p, DbTable* tb, Eterm What) ret = am_protected; else if (tb->common.status & DB_PUBLIC) ret = am_public; + } else if (What == am_write_concurrency) { + ret = tb->common.status & DB_FINE_LOCKED ? am_true : am_false; + } else if (What == am_read_concurrency) { + ret = tb->common.status & DB_FREQ_READ ? am_true : am_false; } else if (What == am_name) { ret = tb->common.the_name; } else if (What == am_keypos) { @@ -3677,7 +3788,10 @@ static Eterm table_info(Process* p, DbTable* tb, Eterm What) ret = am_true; else ret = am_false; - } else if (What == am_atom_put("safe_fixed",10)) { + } else if ((use_monotonic + = ERTS_IS_ATOM_STR("safe_fixed_monotonic_time", + What)) + || ERTS_IS_ATOM_STR("safe_fixed", What)) { #ifdef ERTS_SMP erts_smp_mtx_lock(&tb->common.fixlock); #endif @@ -3686,7 +3800,19 @@ static Eterm table_info(Process* p, DbTable* tb, Eterm What) Eterm *hp; Eterm tpl, lst; DbFixation *fix; - need = 7; + Sint64 mtime; + + need = 3; + if (use_monotonic) { + mtime = (Sint64) tb->common.time.monotonic; + mtime += ERTS_MONOTONIC_OFFSET_NATIVE; + if (!IS_SSMALL(mtime)) + need += ERTS_SINT64_HEAP_SIZE(mtime); + } + else { + mtime = 0; + need += 4; + } for (fix = tb->common.fixations; fix != NULL; fix = fix->next) { need += 5; } @@ -3698,11 +3824,18 @@ static Eterm table_info(Process* p, DbTable* tb, Eterm What) lst = CONS(hp,tpl,lst); hp += 2; } - tpl = TUPLE3(hp, - make_small(tb->common.megasec), - make_small(tb->common.sec), - make_small(tb->common.microsec)); - hp += 4; + if (use_monotonic) + tpl = (IS_SSMALL(mtime) + ? make_small(mtime) + : erts_sint64_to_big(mtime, &hp)); + else { + Uint ms, s, us; + erts_make_timestamp_value(&ms, &s, &us, + tb->common.time.monotonic, + tb->common.time.offset); + tpl = TUPLE3(hp, make_small(ms), make_small(s), make_small(us)); + hp += 4; + } ret = TUPLE2(hp, tpl, lst); } else { ret = am_false; @@ -3737,7 +3870,7 @@ static Eterm table_info(Process* p, DbTable* tb, Eterm What) avg, std_dev_real, std_dev_exp, make_small(stats.min_chain_len), make_small(stats.max_chain_len), - make_small(db_kept_items_hash(&tb->hash))); + make_small(stats.kept_items)); } else { ret = am_false; @@ -3759,6 +3892,11 @@ static void print_table(int to, void *to_arg, int show, DbTable* tb) + sizeof(Uint) - 1) / sizeof(Uint))); + erts_print(to, to_arg, "Type: %T\n", table_info(NULL, tb, am_type)); + erts_print(to, to_arg, "Protection: %T\n", table_info(NULL, tb, am_protection)); + erts_print(to, to_arg, "Compressed: %T\n", table_info(NULL, tb, am_compressed)); + erts_print(to, to_arg, "Write Concurrency: %T\n", table_info(NULL, tb, am_write_concurrency)); + erts_print(to, to_arg, "Read Concurrency: %T\n", table_info(NULL, tb, am_read_concurrency)); } void db_info(int to, void *to_arg, int show) /* Called by break handler */ @@ -3810,6 +3948,13 @@ erts_db_foreach_offheap(DbTable *tb, tb->common.meth->db_foreach_offheap(tb, func, arg); } +/* retrieve max number of ets tables */ +Uint +erts_db_get_max_tabs() +{ + return db_max_tabs; +} + /* * For testing of meta tables only. * diff --git a/erts/emulator/beam/erl_db.h b/erts/emulator/beam/erl_db.h index 6b62e10eb7..a589af784c 100644 --- a/erts/emulator/beam/erl_db.h +++ b/erts/emulator/beam/erl_db.h @@ -3,16 +3,17 @@ * * Copyright Ericsson AB 1996-2013. All Rights Reserved. * - * The contents of this file are subject to the Erlang Public License, - * Version 1.1, (the "License"); you may not use this file except in - * compliance with the License. You should have received a copy of the - * Erlang Public License along with this software. If not, it can be - * retrieved online at http://www.erlang.org/. - * - * Software distributed under the License is distributed on an "AS IS" - * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See - * the License for the specific language governing rights and limitations - * under the License. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. * * %CopyrightEnd% */ @@ -61,7 +62,17 @@ union db_table { "ERL_MAX_ETS_TABLES" */ #define ERL_MAX_ETS_TABLES_ENV "ERL_MAX_ETS_TABLES" -void init_db(void); +typedef enum { + ERTS_DB_SPNCNT_NONE, + ERTS_DB_SPNCNT_VERY_LOW, + ERTS_DB_SPNCNT_LOW, + ERTS_DB_SPNCNT_NORMAL, + ERTS_DB_SPNCNT_HIGH, + ERTS_DB_SPNCNT_VERY_HIGH, + ERTS_DB_SPNCNT_EXTREMELY_HIGH +} ErtsDbSpinCount; + +void init_db(ErtsDbSpinCount); int erts_db_process_exiting(Process *, ErtsProcLocks); void db_info(int, void *, int); void erts_db_foreach_table(void (*)(DbTable *, void *), void *); @@ -69,6 +80,7 @@ void erts_db_foreach_offheap(DbTable *, void (*func)(ErlOffHeap *, void *), void *); +extern int erts_ets_rwmtx_spin_count; extern int user_requested_db_max_tabs; /* set in erl_init */ extern int erts_ets_realloc_always_moves; /* set in erl_init */ extern int erts_ets_always_compress; /* set in erl_init */ @@ -79,6 +91,8 @@ extern erts_smp_atomic_t erts_ets_misc_mem_size; Eterm erts_ets_colliding_names(Process*, Eterm name, Uint cnt); +Uint erts_db_get_max_tabs(void); + #endif #if defined(ERTS_WANT_DB_INTERNAL__) && !defined(ERTS_HAVE_DB_INTERNAL__) diff --git a/erts/emulator/beam/erl_db_hash.c b/erts/emulator/beam/erl_db_hash.c index 2fea4671e1..fa925c94a5 100644 --- a/erts/emulator/beam/erl_db_hash.c +++ b/erts/emulator/beam/erl_db_hash.c @@ -3,16 +3,17 @@ * * Copyright Ericsson AB 1998-2012. All Rights Reserved. * - * The contents of this file are subject to the Erlang Public License, - * Version 1.1, (the "License"); you may not use this file except in - * compliance with the License. You should have received a copy of the - * Erlang Public License along with this software. If not, it can be - * retrieved online at http://www.erlang.org/. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Software distributed under the License is distributed on an "AS IS" - * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See - * the License for the specific language governing rights and limitations - * under the License. + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. * * %CopyrightEnd% */ @@ -147,8 +148,11 @@ static ERTS_INLINE Uint hash_to_ix(DbTableHash* tb, HashValue hval) } /* Remember a slot containing a pseudo-deleted item (INVALID_HASH) -*/ -static ERTS_INLINE void add_fixed_deletion(DbTableHash* tb, int ix) + * Return false if we got raced by unfixing thread + * and the object should be deleted for real. + */ +static ERTS_INLINE int add_fixed_deletion(DbTableHash* tb, int ix, + erts_aint_t fixated_by_me) { erts_aint_t was_next; erts_aint_t exp_next; @@ -159,12 +163,18 @@ static ERTS_INLINE void add_fixed_deletion(DbTableHash* tb, int ix) fixd->slot = ix; was_next = erts_smp_atomic_read_acqb(&tb->fixdel); do { /* Lockless atomic insertion in linked list: */ - exp_next = was_next; + if (NFIXED(tb) <= fixated_by_me) { + erts_db_free(ERTS_ALC_T_DB_FIX_DEL, (DbTable*)tb, + fixd, sizeof(FixedDeletion)); + return 0; /* raced by unfixer */ + } + exp_next = was_next; fixd->next = (FixedDeletion*) exp_next; - was_next = erts_smp_atomic_cmpxchg_relb(&tb->fixdel, - (erts_aint_t) fixd, - exp_next); + was_next = erts_smp_atomic_cmpxchg_mb(&tb->fixdel, + (erts_aint_t) fixd, + exp_next); }while (was_next != exp_next); + return 1; } @@ -174,7 +184,7 @@ static ERTS_INLINE void add_fixed_deletion(DbTableHash* tb, int ix) /* optimised version of make_hash (normal case? atomic key) */ #define MAKE_HASH(term) \ ((is_atom(term) ? (atom_tab(atom_val(term))->slot.bucket.hvalue) : \ - make_hash2(term)) % MAX_HASH) + make_internal_hash(term)) % MAX_HASH) #ifdef ERTS_SMP # define DB_HASH_LOCK_MASK (DB_HASH_LOCK_CNT-1) @@ -382,7 +392,7 @@ static HashDbTerm* search_list(DbTableHash* tb, Eterm key, static void shrink(DbTableHash* tb, int nactive); static void grow(DbTableHash* tb, int nactive); static Eterm build_term_list(Process* p, HashDbTerm* ptr1, HashDbTerm* ptr2, - DbTableHash*); + Uint sz, DbTableHash*); static int analyze_pattern(DbTableHash *tb, Eterm pattern, struct mp_info *mpi); @@ -426,6 +436,7 @@ static int db_select_count_continue_hash(Process *p, DbTable *tbl, static int db_select_delete_continue_hash(Process *p, DbTable *tbl, Eterm continuation, Eterm *ret); +static int db_take_hash(Process *, DbTable *, Eterm, Eterm *); static void db_print_hash(int to, void *to_arg, int show, @@ -443,8 +454,11 @@ static int db_delete_all_objects_hash(Process* p, DbTable* tbl); #ifdef HARDDEBUG static void db_check_table_hash(DbTableHash *tb); #endif -static int db_lookup_dbterm_hash(DbTable *tbl, Eterm key, DbUpdateHandle* handle); -static void db_finalize_dbterm_hash(DbUpdateHandle* handle); +static int +db_lookup_dbterm_hash(Process *p, DbTable *tbl, Eterm key, Eterm obj, + DbUpdateHandle* handle); +static void +db_finalize_dbterm_hash(int cret, DbUpdateHandle* handle); static ERTS_INLINE void try_shrink(DbTableHash* tb) { @@ -536,6 +550,7 @@ DbTableMethod db_hash = db_select_delete_continue_hash, db_select_count_hash, db_select_count_continue_hash, + db_take_hash, db_delete_all_objects_hash, db_free_table_hash, db_free_table_continue_hash, @@ -601,8 +616,8 @@ void db_unfix_table_hash(DbTableHash *tb) || (erts_smp_lc_rwmtx_is_rlocked(&tb->common.rwlock) && !tb->common.is_thread_safe)); restart: - fixdel = (FixedDeletion*) erts_smp_atomic_xchg_acqb(&tb->fixdel, - (erts_aint_t) NULL); + fixdel = (FixedDeletion*) erts_smp_atomic_xchg_mb(&tb->fixdel, + (erts_aint_t) NULL); while (fixdel != NULL) { FixedDeletion *fx = fixdel; int ix = fx->slot; @@ -646,25 +661,6 @@ restart: /* ToDo: Maybe try grow/shrink the table as well */ } -/* Only used by tests -*/ -Uint db_kept_items_hash(DbTableHash *tb) -{ - Uint kept_items = 0; - Uint ix = 0; - erts_smp_rwmtx_t* lck = RLOCK_HASH(tb,ix); - HashDbTerm* b; - do { - for (b = BUCKET(tb, ix); b != NULL; b = b->next) { - if (b->hvalue == INVALID_HASH) { - ++kept_items; - } - } - ix = next_slot(tb, ix, &lck); - }while (ix); - return kept_items; -} - int db_create_hash(Process *p, DbTable *tbl) { DbTableHash *tb = &tbl->hash; @@ -684,6 +680,8 @@ int db_create_hash(Process *p, DbTable *tbl) int i; if (tb->common.type & DB_FREQ_READ) rwmtx_opt.type = ERTS_SMP_RWMTX_TYPE_FREQUENT_READ; + if (erts_ets_rwmtx_spin_count >= 0) + rwmtx_opt.main_spincount = erts_ets_rwmtx_spin_count; tb->locks = (DbTableHashFineLocks*) erts_db_alloc_fnf(ERTS_ALC_T_DB_SEG, /* Other type maybe? */ (DbTable *) tb, sizeof(DbTableHashFineLocks)); @@ -879,34 +877,49 @@ Ldone: return ret; } +static Eterm +get_term_list(Process *p, DbTableHash *tb, Eterm key, HashValue hval, + HashDbTerm *b1, HashDbTerm **bend) +{ + HashDbTerm* b2 = b1->next; + Eterm copy; + Uint sz = b1->dbterm.size + 2; + + if (tb->common.status & (DB_BAG | DB_DUPLICATE_BAG)) { + while (b2 && has_key(tb, b2, key, hval)) { + if (b2->hvalue != INVALID_HASH) + sz += b2->dbterm.size + 2; + + b2 = b2->next; + } + } + copy = build_term_list(p, b1, b2, sz, tb); + CHECK_TABLES(); + if (bend) { + *bend = b2; + } + return copy; +} + int db_get_hash(Process *p, DbTable *tbl, Eterm key, Eterm *ret) { DbTableHash *tb = &tbl->hash; HashValue hval; int ix; - HashDbTerm* b1; + HashDbTerm* b; erts_smp_rwmtx_t* lck; hval = MAKE_HASH(key); lck = RLOCK_HASH(tb,hval); ix = hash_to_ix(tb, hval); - b1 = BUCKET(tb, ix); + b = BUCKET(tb, ix); - while(b1 != 0) { - if (has_live_key(tb,b1,key,hval)) { - HashDbTerm* b2 = b1->next; - Eterm copy; - - if (tb->common.status & (DB_BAG | DB_DUPLICATE_BAG)) { - while(b2 != NULL && has_key(tb,b2,key,hval)) - b2 = b2->next; - } - copy = build_term_list(p, b1, b2, tb); - CHECK_TABLES(); - *ret = copy; + while(b != 0) { + if (has_live_key(tb, b, key, hval)) { + *ret = get_term_list(p, tb, key, hval, b, NULL); goto done; } - b1 = b1->next; + b = b->next; } *ret = NIL; done: @@ -1048,7 +1061,6 @@ static int db_get_element_hash(Process *p, DbTable *tbl, Eterm copy = db_copy_element_from_ets(&tb->common, p, &b->dbterm, ndex, &hp, 2); elem_list = CONS(hp, copy, elem_list); - hp += 2; } b = b->next; } @@ -1139,9 +1151,9 @@ int db_erase_hash(DbTable *tbl, Eterm key, Eterm *ret) while(b != 0) { if (has_live_key(tb,b,key,hval)) { --nitems_diff; - if (nitems_diff == -1 && IS_FIXED(tb)) { + if (nitems_diff == -1 && IS_FIXED(tb) + && add_fixed_deletion(tb, ix, 0)) { /* Pseudo remove (no need to keep several of same key) */ - add_fixed_deletion(tb, ix); b->hvalue = INVALID_HASH; } else { *bp = b->next; @@ -1193,9 +1205,8 @@ static int db_erase_object_hash(DbTable *tbl, Eterm object, Eterm *ret) ++nkeys; if (db_eq(&tb->common,object, &b->dbterm)) { --nitems_diff; - if (nkeys==1 && IS_FIXED(tb)) { /* Pseudo remove */ - add_fixed_deletion(tb,ix); - b->hvalue = INVALID_HASH; + if (nkeys==1 && IS_FIXED(tb) && add_fixed_deletion(tb,ix,0)) { + b->hvalue = INVALID_HASH; /* Pseudo remove */ bp = &b->next; b = b->next; } else { @@ -1240,7 +1251,7 @@ static int db_slot_hash(Process *p, DbTable *tbl, Eterm slot_term, Eterm *ret) lck = RLOCK_HASH(tb, slot); nactive = NACTIVE(tb); if (slot < nactive) { - *ret = build_term_list(p, BUCKET(tb, slot), 0, tb); + *ret = build_term_list(p, BUCKET(tb, slot), NULL, 0, tb); retval = DB_ERROR_NONE; } else if (slot == nactive) { @@ -1817,14 +1828,17 @@ static int db_select_delete_hash(Process *p, int did_erase = 0; if (db_match_dbterm(&tb->common, p, mpi.mp, 0, &(*current)->dbterm, NULL, 0) == am_true) { + HashDbTerm *del; if (NFIXED(tb) > fixated_by_me) { /* fixated by others? */ if (slot_ix != last_pseudo_delete) { - add_fixed_deletion(tb, slot_ix); - last_pseudo_delete = slot_ix; + if (!add_fixed_deletion(tb, slot_ix, fixated_by_me)) + goto do_erase; + last_pseudo_delete = slot_ix; } (*current)->hvalue = INVALID_HASH; } else { - HashDbTerm *del = *current; + do_erase: + del = *current; *current = (*current)->next; free_term(tb, del); did_erase = 1; @@ -1928,14 +1942,17 @@ static int db_select_delete_continue_hash(Process *p, int did_erase = 0; if (db_match_dbterm(&tb->common, p, mp, 0, &(*current)->dbterm, NULL, 0) == am_true) { + HashDbTerm *del; if (NFIXED(tb) > fixated_by_me) { /* fixated by others? */ if (slot_ix != last_pseudo_delete) { - add_fixed_deletion(tb, slot_ix); + if (!add_fixed_deletion(tb, slot_ix, fixated_by_me)) + goto do_erase; last_pseudo_delete = slot_ix; } (*current)->hvalue = INVALID_HASH; } else { - HashDbTerm *del = *current; + do_erase: + del = *current; *current = (*current)->next; free_term(tb, del); did_erase = 1; @@ -2069,6 +2086,46 @@ trap: } +static int db_take_hash(Process *p, DbTable *tbl, Eterm key, Eterm *ret) +{ + DbTableHash *tb = &tbl->hash; + HashDbTerm **bp, *b; + HashValue hval = MAKE_HASH(key); + erts_smp_rwmtx_t *lck = WLOCK_HASH(tb, hval); + int ix = hash_to_ix(tb, hval); + int nitems_diff = 0; + + *ret = NIL; + for (bp = &BUCKET(tb, ix), b = *bp; b; bp = &b->next, b = b->next) { + if (has_live_key(tb, b, key, hval)) { + HashDbTerm *bend; + + *ret = get_term_list(p, tb, key, hval, b, &bend); + while (b != bend) { + --nitems_diff; + if (nitems_diff == -1 && IS_FIXED(tb) + && add_fixed_deletion(tb, ix, 0)) { + /* Pseudo remove (no need to keep several of same key) */ + bp = &b->next; + b->hvalue = INVALID_HASH; + b = b->next; + } else { + *bp = b->next; + free_term(tb, b); + b = *bp; + } + } + break; + } + } + WUNLOCK_HASH(lck); + if (nitems_diff) { + erts_smp_atomic_add_nob(&tb->common.nitems, nitems_diff); + try_shrink(tb); + } + return DB_ERROR_NONE; +} + /* ** Other interface routines (not directly coupled to one bif) */ @@ -2088,7 +2145,7 @@ int db_mark_all_deleted_hash(DbTable *tbl) for (i = 0; i < NACTIVE(tb); i++) { if ((list = BUCKET(tb,i)) != NULL) { - add_fixed_deletion(tb, i); + add_fixed_deletion(tb, i, 0); do { list->hvalue = INVALID_HASH; list = list->next; @@ -2104,10 +2161,38 @@ int db_mark_all_deleted_hash(DbTable *tbl) static void db_print_hash(int to, void *to_arg, int show, DbTable *tbl) { DbTableHash *tb = &tbl->hash; + DbHashStats stats; int i; - erts_print(to, to_arg, "Buckets: %d \n", NACTIVE(tb)); - + erts_print(to, to_arg, "Buckets: %d\n", NACTIVE(tb)); + +#ifdef ERTS_SMP + i = tbl->common.is_thread_safe; + /* If crash dumping we set table to thread safe in order to + avoid taking any locks */ + if (ERTS_IS_CRASH_DUMPING) + tbl->common.is_thread_safe = 1; + + db_calc_stats_hash(&tbl->hash, &stats); + + tbl->common.is_thread_safe = i; +#else + db_calc_stats_hash(&tbl->hash, &stats); +#endif + + erts_print(to, to_arg, "Chain Length Avg: %f\n", stats.avg_chain_len); + erts_print(to, to_arg, "Chain Length Max: %d\n", stats.max_chain_len); + erts_print(to, to_arg, "Chain Length Min: %d\n", stats.min_chain_len); + erts_print(to, to_arg, "Chain Length Std Dev: %f\n", + stats.std_dev_chain_len); + erts_print(to, to_arg, "Chain Length Expected Std Dev: %f\n", + stats.std_dev_expected); + + if (IS_FIXED(tb)) + erts_print(to, to_arg, "Fixed: %d\n", stats.kept_items); + else + erts_print(to, to_arg, "Fixed: false\n"); + if (show) { for (i = 0; i < NACTIVE(tb); i++) { HashDbTerm* list = BUCKET(tb,i); @@ -2402,10 +2487,10 @@ static int alloc_seg(DbTableHash *tb) */ static int free_seg(DbTableHash *tb, int free_records) { - int seg_ix = (tb->nslots >> SEGSZ_EXP) - 1; + const int seg_ix = (tb->nslots >> SEGSZ_EXP) - 1; + struct segment** const segtab = SEGTAB(tb); + struct ext_segment* const top = (struct ext_segment*) segtab[seg_ix]; int bytes; - struct segment** segtab = SEGTAB(tb); - struct ext_segment* top = (struct ext_segment*) segtab[seg_ix]; int nrecords = 0; ASSERT(top != NULL); @@ -2468,7 +2553,7 @@ static int free_seg(DbTableHash *tb, int free_records) (void*)top, bytes); #ifdef DEBUG if (seg_ix > 0) { - if (seg_ix < tb->nsegs) SEGTAB(tb)[seg_ix] = NULL; + segtab[seg_ix] = NULL; } else { SET_SEGTAB(tb, NULL); } @@ -2483,23 +2568,23 @@ static int free_seg(DbTableHash *tb, int free_records) ** Copy terms from ptr1 until ptr2 ** works for ptr1 == ptr2 == 0 => [] ** or ptr2 == 0 +** sz is either precalculated heap size or 0 if not known */ static Eterm build_term_list(Process* p, HashDbTerm* ptr1, HashDbTerm* ptr2, - DbTableHash* tb) + Uint sz, DbTableHash* tb) { - int sz = 0; HashDbTerm* ptr; Eterm list = NIL; Eterm copy; Eterm *hp, *hend; - ptr = ptr1; - while(ptr != ptr2) { - - if (ptr->hvalue != INVALID_HASH) - sz += ptr->dbterm.size + 2; - - ptr = ptr->next; + if (!sz) { + ptr = ptr1; + while(ptr != ptr2) { + if (ptr->hvalue != INVALID_HASH) + sz += ptr->dbterm.size + 2; + ptr = ptr->next; + } } hp = HAlloc(p, sz); @@ -2730,59 +2815,129 @@ static HashDbTerm* next(DbTableHash *tb, Uint *iptr, erts_smp_rwmtx_t** lck_ptr, return NULL; } -static int db_lookup_dbterm_hash(DbTable *tbl, Eterm key, DbUpdateHandle* handle) +static int +db_lookup_dbterm_hash(Process *p, DbTable *tbl, Eterm key, Eterm obj, + DbUpdateHandle* handle) { DbTableHash *tb = &tbl->hash; - HashDbTerm* b; - HashDbTerm** prevp; - int ix; HashValue hval; + HashDbTerm **bp, *b; erts_smp_rwmtx_t* lck; + int flags = 0; + + ASSERT(tb->common.status & DB_SET); hval = MAKE_HASH(key); - lck = WLOCK_HASH(tb,hval); - ix = hash_to_ix(tb, hval); - prevp = &BUCKET(tb, ix); - b = *prevp; + lck = WLOCK_HASH(tb, hval); + bp = &BUCKET(tb, hash_to_ix(tb, hval)); + b = *bp; - while (b != 0) { - if (has_live_key(tb,b,key,hval)) { - handle->tb = tbl; - handle->bp = (void**) prevp; - handle->dbterm = &b->dbterm; - handle->mustResize = 0; - handle->new_size = b->dbterm.size; - #if HALFWORD_HEAP - handle->abs_vec = NULL; - #endif - handle->lck = lck; - /* KEEP hval WLOCKED, db_finalize_dbterm_hash will WUNLOCK */ - return 1; - } - prevp = &b->next; - b = *prevp; + for (;;) { + if (b == NULL) { + break; + } + if (has_key(tb, b, key, hval)) { + if (b->hvalue != INVALID_HASH) { + goto Ldone; + } + break; + } + bp = &b->next; + b = *bp; } - WUNLOCK_HASH(lck); - return 0; + + if (obj == THE_NON_VALUE) { + WUNLOCK_HASH(lck); + return 0; + } + + { + Eterm *objp = tuple_val(obj); + int arity = arityval(*objp); + Eterm *htop, *hend; + + ASSERT(arity >= tb->common.keypos); + htop = HAlloc(p, arity + 1); + hend = htop + arity + 1; + sys_memcpy(htop, objp, sizeof(Eterm) * (arity + 1)); + htop[tb->common.keypos] = key; + obj = make_tuple(htop); + + if (b == NULL) { + HashDbTerm *q = new_dbterm(tb, obj); + + q->hvalue = hval; + q->next = NULL; + *bp = b = q; + + { + int nitems = erts_smp_atomic_inc_read_nob(&tb->common.nitems); + int nactive = NACTIVE(tb); + + if (nitems > nactive * (CHAIN_LEN + 1) && !IS_FIXED(tb)) { + grow(tb, nactive); + } + } + } else { + HashDbTerm *q, *next = b->next; + + ASSERT(b->hvalue == INVALID_HASH); + q = replace_dbterm(tb, b, obj); + q->next = next; + q->hvalue = hval; + *bp = b = q; + erts_smp_atomic_inc_nob(&tb->common.nitems); + } + + HRelease(p, hend, htop); + flags |= DB_NEW_OBJECT; + } + +Ldone: + handle->tb = tbl; + handle->bp = (void **)bp; + handle->dbterm = &b->dbterm; + handle->flags = flags; + handle->new_size = b->dbterm.size; +#if HALFWORD_HEAP + handle->abs_vec = NULL; +#endif + handle->lck = lck; + return 1; } /* Must be called after call to db_lookup_dbterm */ -static void db_finalize_dbterm_hash(DbUpdateHandle* handle) +static void +db_finalize_dbterm_hash(int cret, DbUpdateHandle* handle) { DbTable* tbl = handle->tb; - HashDbTerm* oldp = (HashDbTerm*) *(handle->bp); + DbTableHash *tb = &tbl->hash; + HashDbTerm **bp = (HashDbTerm **) handle->bp; + HashDbTerm *b = *bp; erts_smp_rwmtx_t* lck = (erts_smp_rwmtx_t*) handle->lck; - ERTS_SMP_LC_ASSERT(IS_HASH_WLOCKED(&tbl->hash,lck)); /* locked by db_lookup_dbterm_hash */ + ERTS_SMP_LC_ASSERT(IS_HASH_WLOCKED(tb, lck)); /* locked by db_lookup_dbterm_hash */ + + ASSERT((&b->dbterm == handle->dbterm) == !(tb->common.compress && handle->flags & DB_MUST_RESIZE)); - ASSERT((&oldp->dbterm == handle->dbterm) == !(tbl->common.compress && handle->mustResize)); + if (handle->flags & DB_NEW_OBJECT && cret != DB_ERROR_NONE) { + if (IS_FIXED(tb) && add_fixed_deletion(tb, hash_to_ix(tb, b->hvalue), + 0)) { + b->hvalue = INVALID_HASH; + } else { + *bp = b->next; + free_term(tb, b); + } - if (handle->mustResize) { + WUNLOCK_HASH(lck); + erts_smp_atomic_dec_nob(&tb->common.nitems); + try_shrink(tb); + } else if (handle->flags & DB_MUST_RESIZE) { db_finalize_resize(handle, offsetof(HashDbTerm,dbterm)); WUNLOCK_HASH(lck); - free_term(&tbl->hash, oldp); + free_term(tb, b); } else { WUNLOCK_HASH(lck); @@ -2833,6 +2988,7 @@ void db_calc_stats_hash(DbTableHash* tb, DbHashStats* stats) erts_smp_rwmtx_t* lck; int sum = 0; int sq_sum = 0; + int kept_items = 0; int ix; int len; @@ -2844,6 +3000,8 @@ void db_calc_stats_hash(DbTableHash* tb, DbHashStats* stats) len = 0; for (b = BUCKET(tb,ix); b!=NULL; b=b->next) { len++; + if (b->hvalue == INVALID_HASH) + ++kept_items; } sum += len; sq_sum += len*len; @@ -2855,7 +3013,8 @@ void db_calc_stats_hash(DbTableHash* tb, DbHashStats* stats) stats->std_dev_chain_len = sqrt((sq_sum - stats->avg_chain_len*sum) / NACTIVE(tb)); /* Expected standard deviation from a good uniform hash function, ie binomial distribution (not taking the linear hashing into acount) */ - stats->std_dev_expected = sqrt(stats->avg_chain_len * (1 - 1.0/NACTIVE(tb))); + stats->std_dev_expected = sqrt(stats->avg_chain_len * (1 - 1.0/NACTIVE(tb))); + stats->kept_items = kept_items; } #ifdef HARDDEBUG @@ -2869,7 +3028,7 @@ void db_check_table_hash(DbTable *tbl) if ((list = BUCKET(tb,j)) != 0) { while (list != 0) { if (!is_tuple(make_tuple(list->dbterm.tpl))) { - erl_exit(1, "Bad term in slot %d of ets table", j); + erts_exit(ERTS_ERROR_EXIT, "Bad term in slot %d of ets table", j); } list = list->next; } diff --git a/erts/emulator/beam/erl_db_hash.h b/erts/emulator/beam/erl_db_hash.h index d17bd9f693..66d8ec71d9 100644 --- a/erts/emulator/beam/erl_db_hash.h +++ b/erts/emulator/beam/erl_db_hash.h @@ -1,18 +1,19 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 1998-2013. All Rights Reserved. + * Copyright Ericsson AB 1998-2014. All Rights Reserved. * - * The contents of this file are subject to the Erlang Public License, - * Version 1.1, (the "License"); you may not use this file except in - * compliance with the License. You should have received a copy of the - * Erlang Public License along with this software. If not, it can be - * retrieved online at http://www.erlang.org/. - * - * Software distributed under the License is distributed on an "AS IS" - * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See - * the License for the specific language governing rights and limitations - * under the License. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. * * %CopyrightEnd% */ @@ -33,11 +34,16 @@ typedef struct hash_db_term { DbTerm dbterm; /* The actual term */ } HashDbTerm; +#ifdef ERTS_DB_HASH_LOCK_CNT +#define DB_HASH_LOCK_CNT ERTS_DB_HASH_LOCK_CNT +#else #define DB_HASH_LOCK_CNT 64 +#endif + typedef struct db_table_hash_fine_locks { union { erts_smp_rwmtx_t lck; - byte _cache_line_alignment[64]; + byte _cache_line_alignment[ERTS_ALC_CACHE_LINE_ALIGN_SIZE(sizeof(erts_smp_rwmtx_t))]; }lck_vec[DB_HASH_LOCK_CNT]; } DbTableHashFineLocks; @@ -99,6 +105,7 @@ typedef struct { float std_dev_expected; int max_chain_len; int min_chain_len; + int kept_items; }DbHashStats; void db_calc_stats_hash(DbTableHash* tb, DbHashStats*); diff --git a/erts/emulator/beam/erl_db_tree.c b/erts/emulator/beam/erl_db_tree.c index 25029ba90f..61293fbf9a 100644 --- a/erts/emulator/beam/erl_db_tree.c +++ b/erts/emulator/beam/erl_db_tree.c @@ -3,16 +3,17 @@ * * Copyright Ericsson AB 1998-2013. All Rights Reserved. * - * The contents of this file are subject to the Erlang Public License, - * Version 1.1, (the "License"); you may not use this file except in - * compliance with the License. You should have received a copy of the - * Erlang Public License along with this software. If not, it can be - * retrieved online at http://www.erlang.org/. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Software distributed under the License is distributed on an "AS IS" - * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See - * the License for the specific language governing rights and limitations - * under the License. + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. * * %CopyrightEnd% */ @@ -383,6 +384,7 @@ static int db_select_delete_tree(Process *p, DbTable *tbl, Eterm pattern, Eterm *ret); static int db_select_delete_continue_tree(Process *p, DbTable *tbl, Eterm continuation, Eterm *ret); +static int db_take_tree(Process *, DbTable *, Eterm, Eterm *); static void db_print_tree(int to, void *to_arg, int show, DbTable *tbl); static int db_free_table_tree(DbTable *tbl); @@ -398,8 +400,11 @@ static int db_delete_all_objects_tree(Process* p, DbTable* tbl); #ifdef HARDDEBUG static void db_check_table_tree(DbTable *tbl); #endif -static int db_lookup_dbterm_tree(DbTable *, Eterm key, DbUpdateHandle*); -static void db_finalize_dbterm_tree(DbUpdateHandle*); +static int +db_lookup_dbterm_tree(Process *, DbTable *, Eterm key, Eterm obj, + DbUpdateHandle*); +static void +db_finalize_dbterm_tree(int cret, DbUpdateHandle *); /* ** Static variables @@ -431,6 +436,7 @@ DbTableMethod db_tree = db_select_delete_continue_tree, db_select_count_tree, db_select_count_continue_tree, + db_take_tree, db_delete_all_objects_tree, db_free_table_tree, db_free_table_continue_tree, @@ -485,7 +491,7 @@ static int db_first_tree(Process *p, DbTable *tbl, Eterm *ret) *ret = am_EOT; return DB_ERROR_NONE; } - /* Walk down to the tree to the left */ + /* Walk down the tree to the left */ if ((stack = get_static_stack(tb)) != NULL) { stack->pos = stack->slot = 0; } @@ -531,7 +537,7 @@ static int db_last_tree(Process *p, DbTable *tbl, Eterm *ret) *ret = am_EOT; return DB_ERROR_NONE; } - /* Walk down to the tree to the left */ + /* Walk down the tree to the right */ if ((stack = get_static_stack(tb)) != NULL) { stack->pos = stack->slot = 0; } @@ -1111,7 +1117,7 @@ static int db_select_tree(Process *p, DbTable *tbl, sc.all_objects = mpi.all_objects; if (!mpi.got_partial && mpi.some_limitation && - CMP(mpi.least,mpi.most) == 0) { + CMP_EQ(mpi.least,mpi.most)) { doit_select(tb,mpi.save_term,&sc,0 /* direction doesn't matter */); RET_TO_BIF(sc.accum,DB_ERROR_NONE); } @@ -1207,7 +1213,7 @@ static int db_select_count_continue_tree(Process *p, tptr = tuple_val(continuation); if (arityval(*tptr) != 5) - erl_exit(1,"Internal error in ets:select_count/1"); + erts_exit(ERTS_ERROR_EXIT,"Internal error in ets:select_count/1"); lastkey = tptr[2]; end_condition = tptr[3]; @@ -1319,7 +1325,7 @@ static int db_select_count_tree(Process *p, DbTable *tbl, sc.all_objects = mpi.all_objects; if (!mpi.got_partial && mpi.some_limitation && - CMP(mpi.least,mpi.most) == 0) { + CMP_EQ(mpi.least,mpi.most)) { doit_select_count(tb,mpi.save_term,&sc,0 /* dummy */); RET_TO_BIF(erts_make_integer(sc.got,p),DB_ERROR_NONE); } @@ -1424,7 +1430,7 @@ static int db_select_chunk_tree(Process *p, DbTable *tbl, sc.all_objects = mpi.all_objects; if (!mpi.got_partial && mpi.some_limitation && - CMP(mpi.least,mpi.most) == 0) { + CMP_EQ(mpi.least,mpi.most)) { doit_select(tb,mpi.save_term,&sc, 0 /* direction doesn't matter */); if (sc.accum != NIL) { hp=HAlloc(p, 3); @@ -1668,7 +1674,7 @@ static int db_select_delete_tree(Process *p, DbTable *tbl, sc.mp = mpi.mp; if (!mpi.got_partial && mpi.some_limitation && - CMP(mpi.least,mpi.most) == 0) { + CMP_EQ(mpi.least,mpi.most)) { doit_select_delete(tb,mpi.save_term,&sc, 0 /* direction doesn't matter */); RET_TO_BIF(erts_make_integer(sc.accum,p),DB_ERROR_NONE); @@ -1722,6 +1728,28 @@ static int db_select_delete_tree(Process *p, DbTable *tbl, } +static int db_take_tree(Process *p, DbTable *tbl, Eterm key, Eterm *ret) +{ + DbTableTree *tb = &tbl->tree; + TreeDbTerm *this; + + *ret = NIL; + this = linkout_tree(tb, key, NULL); + if (this) { + Eterm copy, *hp, *hend; + + hp = HAlloc(p, this->dbterm.size + 2); + hend = hp + this->dbterm.size + 2; + copy = db_copy_object_from_ets(&tb->common, + &this->dbterm, &hp, &MSO(p)); + *ret = CONS(hp, copy, NIL); + hp += 2; + HRelease(p, hend, hp); + free_term(tb, this); + } + return DB_ERROR_NONE; +} + /* ** Other interface routines (not directly coupled to one bif) */ @@ -2522,16 +2550,43 @@ static TreeDbTerm **find_node2(DbTableTree *tb, Eterm key) return this; } -static int db_lookup_dbterm_tree(DbTable *tbl, Eterm key, DbUpdateHandle* handle) +static int +db_lookup_dbterm_tree(Process *p, DbTable *tbl, Eterm key, Eterm obj, + DbUpdateHandle* handle) { DbTableTree *tb = &tbl->tree; TreeDbTerm **pp = find_node2(tb, key); - - if (pp == NULL) return 0; + int flags = 0; + + if (pp == NULL) { + if (obj == THE_NON_VALUE) { + return 0; + } else { + Eterm *objp = tuple_val(obj); + int arity = arityval(*objp); + Eterm *htop, *hend; + + ASSERT(arity >= tb->common.keypos); + htop = HAlloc(p, arity + 1); + hend = htop + arity + 1; + sys_memcpy(htop, objp, sizeof(Eterm) * (arity + 1)); + htop[tb->common.keypos] = key; + obj = make_tuple(htop); + + if (db_put_tree(tbl, obj, 1) != DB_ERROR_NONE) { + return 0; + } + + pp = find_node2(tb, key); + ASSERT(pp != NULL); + HRelease(p, hend, htop); + flags |= DB_NEW_OBJECT; + } + } handle->tb = tbl; handle->dbterm = &(*pp)->dbterm; - handle->mustResize = 0; + handle->flags = flags; handle->bp = (void**) pp; handle->new_size = (*pp)->dbterm.size; #if HALFWORD_HEAP @@ -2540,15 +2595,21 @@ static int db_lookup_dbterm_tree(DbTable *tbl, Eterm key, DbUpdateHandle* handle return 1; } -static void db_finalize_dbterm_tree(DbUpdateHandle* handle) +static void +db_finalize_dbterm_tree(int cret, DbUpdateHandle *handle) { - if (handle->mustResize) { - TreeDbTerm* oldp = (TreeDbTerm*) *handle->bp; + DbTable *tbl = handle->tb; + DbTableTree *tb = &tbl->tree; + TreeDbTerm *bp = (TreeDbTerm *) *handle->bp; + if (handle->flags & DB_NEW_OBJECT && cret != DB_ERROR_NONE) { + Eterm ret; + db_erase_tree(tbl, GETKEY(tb, bp->dbterm.tpl), &ret); + } else if (handle->flags & DB_MUST_RESIZE) { db_finalize_resize(handle, offsetof(TreeDbTerm,dbterm)); - reset_static_stack(&handle->tb->tree); + reset_static_stack(tb); - free_term(&handle->tb->tree, oldp); + free_term(tb, bp); } #ifdef DEBUG handle->dbterm = 0; @@ -2656,7 +2717,7 @@ static int key_given(DbTableTree *tb, Eterm pattern, TreeDbTerm **ret, *ret = this; return 1; } else if (partly_bound != NULL && key != am_Underscore && - db_is_variable(key) < 0) + db_is_variable(key) < 0 && !db_has_map(key)) *partly_bound = key; return 0; diff --git a/erts/emulator/beam/erl_db_tree.h b/erts/emulator/beam/erl_db_tree.h index 7bc235e135..6098387f5d 100644 --- a/erts/emulator/beam/erl_db_tree.h +++ b/erts/emulator/beam/erl_db_tree.h @@ -3,16 +3,17 @@ * * Copyright Ericsson AB 1998-2009. All Rights Reserved. * - * The contents of this file are subject to the Erlang Public License, - * Version 1.1, (the "License"); you may not use this file except in - * compliance with the License. You should have received a copy of the - * Erlang Public License along with this software. If not, it can be - * retrieved online at http://www.erlang.org/. - * - * Software distributed under the License is distributed on an "AS IS" - * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See - * the License for the specific language governing rights and limitations - * under the License. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. * * %CopyrightEnd% */ diff --git a/erts/emulator/beam/erl_db_util.c b/erts/emulator/beam/erl_db_util.c index 713ac0ba18..af5b611afd 100644 --- a/erts/emulator/beam/erl_db_util.c +++ b/erts/emulator/beam/erl_db_util.c @@ -1,18 +1,19 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 1998-2013. All Rights Reserved. + * Copyright Ericsson AB 1998-2014. All Rights Reserved. * - * The contents of this file are subject to the Erlang Public License, - * Version 1.1, (the "License"); you may not use this file except in - * compliance with the License. You should have received a copy of the - * Erlang Public License along with this software. If not, it can be - * retrieved online at http://www.erlang.org/. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Software distributed under the License is distributed on an "AS IS" - * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See - * the License for the specific language governing rights and limitations - * under the License. + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. * * %CopyrightEnd% */ @@ -35,6 +36,7 @@ #include "bif.h" #include "big.h" #include "erl_binary.h" +#include "erl_map.h" #include "erl_thr_progress.h" #include "erl_db_util.h" @@ -197,11 +199,6 @@ set_match_trace(Process *tracee_p, Eterm fail_term, Eterm tracer, return ret; } - -/* Type checking... */ - -#define BOXED_IS_TUPLE(Boxed) is_arity_value(*boxed_val((Boxed))) - /* ** ** Types and enum's (compiled matches) @@ -217,7 +214,9 @@ typedef enum { matchTuple, matchPushT, matchPushL, + matchPushM, matchPop, + matchSwap, matchBind, matchCmp, matchEqBin, @@ -226,11 +225,15 @@ typedef enum { matchEqRef, matchEq, matchList, + matchMap, + matchKey, matchSkip, matchPushC, matchConsA, /* Car is below Cdr */ matchConsB, /* Cdr is below Car (unusual) */ matchMkTuple, + matchMkFlatMap, + matchMkHashMap, matchCall0, matchCall1, matchCall2, @@ -482,7 +485,8 @@ void match_pseudo_process_init(void) { #ifdef ERTS_SMP - erts_smp_tsd_key_create(&match_pseudo_process_key); + erts_smp_tsd_key_create(&match_pseudo_process_key, + "erts_match_pseudo_process_key"); erts_smp_install_exit_handler(destroy_match_pseudo_process); #else match_pseudo_process = create_match_pseudo_process(); @@ -565,6 +569,12 @@ static DMCGuardBif guard_tab[] = DBIF_ALL }, { + am_is_map, + &is_map_1, + 1, + DBIF_ALL + }, + { am_is_binary, &is_binary_1, 1, @@ -631,6 +641,12 @@ static DMCGuardBif guard_tab[] = DBIF_ALL }, { + am_map_size, + &map_size_1, + 1, + DBIF_ALL + }, + { am_bit_size, &bit_size_1, 1, @@ -842,6 +858,13 @@ static int match_compact(ErlHeapFragment *expr, DMCErrInfo *err_info); static Uint my_size_object(Eterm t); static Eterm my_copy_struct(Eterm t, Eterm **hp, ErlOffHeap* off_heap); +/* Guard subroutines */ +static void +dmc_rearrange_constants(DMCContext *context, DMC_STACK_TYPE(UWord) *text, + int textpos, Eterm *p, Uint nelems); +static DMCRet +dmc_array(DMCContext *context, DMCHeap *heap, DMC_STACK_TYPE(UWord) *text, + Eterm *p, Uint nelems, int *constant); /* Guard compilation */ static void do_emit_constant(DMCContext *context, DMC_STACK_TYPE(UWord) *text, Eterm t); @@ -855,6 +878,9 @@ static DMCRet dmc_tuple(DMCContext *context, DMC_STACK_TYPE(UWord) *text, Eterm t, int *constant); +static DMCRet +dmc_map(DMCContext *context, DMCHeap *heap, DMC_STACK_TYPE(UWord) *text, + Eterm t, int *constant); static DMCRet dmc_variable(DMCContext *context, DMCHeap *heap, DMC_STACK_TYPE(UWord) *text, @@ -874,12 +900,14 @@ static DMCRet compile_guard_expr(DMCContext *context, DMCHeap *heap, DMC_STACK_TYPE(UWord) *text, Eterm t); -/* match expression subroutine */ +/* match expression subroutines */ static DMCRet dmc_one_term(DMCContext *context, DMCHeap *heap, DMC_STACK_TYPE(Eterm) *stack, DMC_STACK_TYPE(UWord) *text, Eterm c); +static Eterm +dmc_private_copy(DMCContext *context, Eterm c); #ifdef DMC_DEBUG @@ -1350,7 +1378,112 @@ restart: for (;;) { switch (t & _TAG_PRIMARY_MASK) { case TAG_PRIMARY_BOXED: - if (!BOXED_IS_TUPLE(t)) { + if (is_flatmap(t)) { + num_iters = flatmap_get_size(flatmap_val(t)); + if (!structure_checked) { + DMC_PUSH(text, matchMap); + DMC_PUSH(text, num_iters); + } + structure_checked = 0; + for (i = 0; i < num_iters; ++i) { + Eterm key = flatmap_get_keys(flatmap_val(t))[i]; + if (db_is_variable(key) >= 0) { + if (context.err_info) { + add_dmc_err(context.err_info, + "Variable found in map key.", + -1, 0UL, dmcError); + } + goto error; + } else if (key == am_Underscore) { + if (context.err_info) { + add_dmc_err(context.err_info, + "Underscore found in map key.", + -1, 0UL, dmcError); + } + goto error; + } + DMC_PUSH(text, matchKey); + DMC_PUSH(text, dmc_private_copy(&context, key)); + { + int old_stack = ++(context.stack_used); + Eterm value = flatmap_get_values(flatmap_val(t))[i]; + res = dmc_one_term(&context, &heap, &stack, &text, + value); + ASSERT(res != retFail); + if (res == retRestart) { + goto restart; + } + if (old_stack != context.stack_used) { + ASSERT(old_stack + 1 == context.stack_used); + DMC_PUSH(text, matchSwap); + } + if (context.stack_used > context.stack_need) { + context.stack_need = context.stack_used; + } + DMC_PUSH(text, matchPop); + --(context.stack_used); + } + } + break; + } + if (is_hashmap(t)) { + DECLARE_WSTACK(wstack); + Eterm *kv; + num_iters = hashmap_size(t); + if (!structure_checked) { + DMC_PUSH(text, matchMap); + DMC_PUSH(text, num_iters); + } + structure_checked = 0; + + hashmap_iterator_init(&wstack, t, 0); + + while ((kv=hashmap_iterator_next(&wstack)) != NULL) { + Eterm key = CAR(kv); + Eterm value = CDR(kv); + if (db_is_variable(key) >= 0) { + if (context.err_info) { + add_dmc_err(context.err_info, + "Variable found in map key.", + -1, 0UL, dmcError); + } + DESTROY_WSTACK(wstack); + goto error; + } else if (key == am_Underscore) { + if (context.err_info) { + add_dmc_err(context.err_info, + "Underscore found in map key.", + -1, 0UL, dmcError); + } + DESTROY_WSTACK(wstack); + goto error; + } + DMC_PUSH(text, matchKey); + DMC_PUSH(text, dmc_private_copy(&context, key)); + { + int old_stack = ++(context.stack_used); + res = dmc_one_term(&context, &heap, &stack, &text, + value); + ASSERT(res != retFail); + if (res == retRestart) { + DESTROY_WSTACK(wstack); + goto restart; + } + if (old_stack != context.stack_used) { + ASSERT(old_stack + 1 == context.stack_used); + DMC_PUSH(text, matchSwap); + } + if (context.stack_used > context.stack_need) { + context.stack_need = context.stack_used; + } + DMC_PUSH(text, matchPop); + --(context.stack_used); + } + } + DESTROY_WSTACK(wstack); + break; + } + if (!is_tuple(t)) { goto simple_term; } num_iters = arityval(*tuple_val(t)); @@ -1701,10 +1834,8 @@ Eterm db_prog_match(Process *c_p, Binary *bprog, Uint32 *return_flags) { MatchProg *prog = Binary2MatchProg(bprog); - Eterm *ep; - Eterm *tp; + const Eterm *ep, *tp, **sp; Eterm t; - Eterm **sp; Eterm *esp; MatchVariable* variables; BeamInstr *cp; @@ -1794,7 +1925,7 @@ Eterm db_prog_match(Process *c_p, Binary *bprog, restart: ep = &term; esp = (Eterm*)((char*)mpsp->u.heap + prog->stack_offset); - sp = (Eterm **) esp; + sp = (const Eterm **)esp; ret = am_true; do_catch = 0; fail_label = -1; @@ -1815,11 +1946,11 @@ restart: #ifdef DMC_DEBUG if (*heap_fence != FENCE_PATTERN) { - erl_exit(1, "Heap fence overwritten in db_prog_match after op " + erts_exit(ERTS_ERROR_EXIT, "Heap fence overwritten in db_prog_match after op " "0x%08x, overwritten with 0x%08x.", save_op, *heap_fence); } if (*stack_fence != FENCE_PATTERN) { - erl_exit(1, "Stack fence overwritten in db_prog_match after op " + erts_exit(ERTS_ERROR_EXIT, "Stack fence overwritten in db_prog_match after op " "0x%08x, overwritten with 0x%08x.", save_op, *stack_fence); } @@ -1838,7 +1969,7 @@ restart: ep = termp; break; case matchArrayBind: /* When the array size is unknown. */ - ASSERT(termp); + ASSERT(termp || arity==0); n = *pc++; variables[n].term = dpm_array_to_list(psp, termp, arity); break; @@ -1873,9 +2004,57 @@ restart: *sp++ = list_val_rel(*ep,base); ++ep; break; + case matchMap: + if (!is_map_rel(*ep, base)) { + FAIL(); + } + n = *pc++; + if (is_flatmap_rel(*ep,base)) { + if (flatmap_get_size(flatmap_val_rel(*ep, base)) < n) { + FAIL(); + } + } else { + ASSERT(is_hashmap_rel(*ep,base)); + if (hashmap_size_rel(*ep, base) < n) { + FAIL(); + } + } + ep = flatmap_val_rel(*ep, base); + break; + case matchPushM: + if (!is_map_rel(*ep, base)) { + FAIL(); + } + n = *pc++; + if (is_flatmap_rel(*ep,base)) { + if (flatmap_get_size(flatmap_val_rel(*ep, base)) < n) { + FAIL(); + } + } else { + ASSERT(is_hashmap_rel(*ep,base)); + if (hashmap_size_rel(*ep, base) < n) { + FAIL(); + } + } + *sp++ = flatmap_val_rel(*ep++, base); + break; + case matchKey: + t = (Eterm) *pc++; + tp = erts_maps_get_rel(t, make_boxed_rel(ep, base), base); + if (!tp) { + FAIL(); + } + *sp++ = ep; + ep = tp; + break; case matchPop: ep = *(--sp); break; + case matchSwap: + tp = sp[-1]; + sp[-1] = sp[-2]; + sp[-2] = tp; + break; case matchBind: n = *pc++; variables[n].term = *ep++; @@ -1973,6 +2152,39 @@ restart: } *esp++ = t; break; + case matchMkFlatMap: + n = *pc++; + ehp = HAllocX(build_proc, MAP_HEADER_FLATMAP_SZ + n, HEAP_XTRA); + t = *--esp; + { + flatmap_t *m = (flatmap_t *)ehp; + m->thing_word = MAP_HEADER_FLATMAP; + m->size = n; + m->keys = t; + } + t = make_flatmap(ehp); + ehp += MAP_HEADER_FLATMAP_SZ; + while (n--) { + *ehp++ = *--esp; + } + *esp++ = t; + break; + case matchMkHashMap: + n = *pc++; + esp -= 2*n; + ehp = HAllocX(build_proc, 2*n, HEAP_XTRA); + { + ErtsHeapFactory factory; + Uint ix; + for (ix = 0; ix < 2*n; ix++){ + ehp[ix] = esp[ix]; + } + erts_factory_proc_init(&factory, build_proc); + t = erts_hashmap_from_array(&factory, ehp, n, 0); + erts_factory_close(&factory); + } + *esp++ = t; + break; case matchCall0: bif = (Eterm (*)(Process*, ...)) *pc++; t = (*bif)(build_proc, bif_args); @@ -2319,6 +2531,8 @@ restart: break; case matchSilent: --esp; + if (in_flags & ERTS_PAM_IGNORE_TRACE_SILENT) + break; if (*esp == am_true) { erts_smp_proc_lock(c_p, ERTS_PROC_LOCKS_ALL_MINOR); ERTS_TRACE_FLAGS(c_p) |= F_TRACE_SILENT; @@ -2401,7 +2615,7 @@ restart: case matchHalt: goto success; default: - erl_exit(1, "Internal error: unexpected opcode in match program."); + erts_exit(ERTS_ERROR_EXIT, "Internal error: unexpected opcode in match program."); } } fail: @@ -2425,11 +2639,11 @@ success: #ifdef DMC_DEBUG if (*heap_fence != FENCE_PATTERN) { - erl_exit(1, "Heap fence overwritten in db_prog_match after op " + erts_exit(ERTS_ERROR_EXIT, "Heap fence overwritten in db_prog_match after op " "0x%08x, overwritten with 0x%08x.", save_op, *heap_fence); } if (*stack_fence != FENCE_PATTERN) { - erl_exit(1, "Stack fence overwritten in db_prog_match after op " + erts_exit(ERTS_ERROR_EXIT, "Stack fence overwritten in db_prog_match after op " "0x%08x, overwritten with 0x%08x.", save_op, *stack_fence); } @@ -2585,10 +2799,10 @@ Wterm db_do_read_element(DbUpdateHandle* handle, Sint position) } ASSERT(((DbTableCommon*)handle->tb)->compress); - ASSERT(!handle->mustResize); + ASSERT(!(handle->flags & DB_MUST_RESIZE)); handle->dbterm = db_alloc_tmp_uncompressed(&handle->tb->common, handle->dbterm); - handle->mustResize = 1; + handle->flags |= DB_MUST_RESIZE; return handle->dbterm->tpl[position]; } @@ -2621,11 +2835,11 @@ void db_do_update_element(DbUpdateHandle* handle, #endif return; } - if (!handle->mustResize) { + if (!(handle->flags & DB_MUST_RESIZE)) { if (handle->tb->common.compress) { handle->dbterm = db_alloc_tmp_uncompressed(&handle->tb->common, handle->dbterm); - handle->mustResize = 1; + handle->flags |= DB_MUST_RESIZE; oldval = handle->dbterm->tpl[position]; #if HALFWORD_HEAP old_base = NULL; @@ -2685,7 +2899,7 @@ both_size_set: /* write new value in old dbterm, finalize will make a flat copy */ handle->dbterm->tpl[position] = newval; - handle->mustResize = 1; + handle->flags |= DB_MUST_RESIZE; #if HALFWORD_HEAP if (old_base && newval_sz > 0) { @@ -2980,6 +3194,7 @@ Eterm db_copy_from_comp(DbTableCommon* tb, DbTerm* bp, Eterm** hpp, { Eterm* hp = *hpp; int i, arity = arityval(bp->tpl[0]); + ErtsHeapFactory factory; hp[0] = bp->tpl[0]; *hpp += arity + 1; @@ -2987,17 +3202,23 @@ Eterm db_copy_from_comp(DbTableCommon* tb, DbTerm* bp, Eterm** hpp, hp[tb->keypos] = copy_struct_rel(bp->tpl[tb->keypos], size_object_rel(bp->tpl[tb->keypos], bp->tpl), hpp, off_heap, bp->tpl, NULL); + + erts_factory_static_init(&factory, *hpp, bp->size - (arity+1), off_heap); + for (i=arity; i>0; i--) { if (i != tb->keypos) { if (is_immed(bp->tpl[i])) { hp[i] = bp->tpl[i]; } else { - hp[i] = erts_decode_ext_ets(hpp, off_heap, + hp[i] = erts_decode_ext_ets(&factory, elem2ext(bp->tpl, i)); } } } + *hpp = factory.hp; + erts_factory_close(&factory); + ASSERT((*hpp - hp) <= bp->size); #ifdef DEBUG_CLONE ASSERT(eq_rel(make_tuple(hp),NULL,make_tuple(bp->debug_clone),bp->debug_clone)); @@ -3016,12 +3237,13 @@ Eterm db_copy_element_from_ets(DbTableCommon* tb, Process* p, if (tb->compress && pos != tb->keypos) { byte* ext = elem2ext(obj->tpl, pos); Sint sz = erts_decode_ext_size_ets(ext, db_alloced_size_comp(obj)) + extra; - Eterm* hp = HAlloc(p, sz); - Eterm* endp = hp + sz; - Eterm copy = erts_decode_ext_ets(&hp, &MSO(p), ext); - *hpp = hp; - hp += extra; - HRelease(p, endp, hp); + Eterm copy; + ErtsHeapFactory factory; + + erts_factory_proc_prealloc_init(&factory, p, sz); + copy = erts_decode_ext_ets(&factory, ext); + *hpp = erts_produce_heap(&factory, extra, 0); + erts_factory_close(&factory); #ifdef DEBUG_CLONE ASSERT(eq_rel(copy, NULL, obj->debug_clone[pos], obj->debug_clone)); #endif @@ -3135,39 +3357,86 @@ int db_is_variable(Eterm obj) return N; } +/* check if node is (or contains) a map + * return 1 if node contains a map + * return 0 otherwise + */ + +int db_has_map(Eterm node) { + DECLARE_ESTACK(s); + + ESTACK_PUSH(s,node); + while (!ESTACK_ISEMPTY(s)) { + node = ESTACK_POP(s); + if (is_list(node)) { + while (is_list(node)) { + ESTACK_PUSH(s,CAR(list_val(node))); + node = CDR(list_val(node)); + } + ESTACK_PUSH(s,node); /* Non wellformed list or [] */ + } else if (is_tuple(node)) { + Eterm *tuple = tuple_val(node); + int arity = arityval(*tuple); + while(arity--) { + ESTACK_PUSH(s,*(++tuple)); + } + } else if is_map(node) { + DESTROY_ESTACK(s); + return 1; + } + } + DESTROY_ESTACK(s); + return 0; +} /* check if obj is (or contains) a variable */ /* return 1 if obj contains a variable or underscore */ /* return 0 if obj is fully ground */ -int db_has_variable(Eterm obj) -{ - switch(obj & _TAG_PRIMARY_MASK) { - case TAG_PRIMARY_LIST: { - while (is_list(obj)) { - if (db_has_variable(CAR(list_val(obj)))) +int db_has_variable(Eterm node) { + DECLARE_ESTACK(s); + + ESTACK_PUSH(s,node); + while (!ESTACK_ISEMPTY(s)) { + node = ESTACK_POP(s); + switch(node & _TAG_PRIMARY_MASK) { + case TAG_PRIMARY_LIST: + while (is_list(node)) { + ESTACK_PUSH(s,CAR(list_val(node))); + node = CDR(list_val(node)); + } + ESTACK_PUSH(s,node); /* Non wellformed list or [] */ + break; + case TAG_PRIMARY_BOXED: + if (is_tuple(node)) { + Eterm *tuple = tuple_val(node); + int arity = arityval(*tuple); + while(arity--) { + ESTACK_PUSH(s,*(++tuple)); + } + } else if (is_flatmap(node)) { + Eterm *values = flatmap_get_values(flatmap_val(node)); + Uint size = flatmap_get_size(flatmap_val(node)); + ESTACK_PUSH(s, ((flatmap_t *) flatmap_val(node))->keys); + while (size--) { + ESTACK_PUSH(s, *(values++)); + } + } else if (is_map(node)) { /* other map-nodes or map-heads */ + Eterm *ptr = hashmap_val(node); + int i = hashmap_bitcount(MAP_HEADER_VAL(*ptr)); + ptr += MAP_HEADER_ARITY(*ptr); + while(i--) { ESTACK_PUSH(s, *++ptr); } + } + break; + case TAG_PRIMARY_IMMED1: + if (node == am_Underscore || db_is_variable(node) >= 0) { + DESTROY_ESTACK(s); return 1; - obj = CDR(list_val(obj)); - } - return(db_has_variable(obj)); /* Non wellformed list or [] */ - } - case TAG_PRIMARY_BOXED: - if (!BOXED_IS_TUPLE(obj)) { - return 0; - } else { - Eterm *tuple = tuple_val(obj); - int arity = arityval(*tuple++); - while(arity--) { - if (db_has_variable(*tuple)) - return 1; - tuple++; } - return(0); + break; } - case TAG_PRIMARY_IMMED1: - if (obj == am_Underscore || db_is_variable(obj) >= 0) - return 1; } + DESTROY_ESTACK(s); return 0; } @@ -3227,11 +3496,9 @@ static DMCRet dmc_one_term(DMCContext *context, { Sint n; Eterm *hp; - ErlHeapFragment *tmp_mb; Uint sz, sz2, sz3; Uint i, j; - switch (c & _TAG_PRIMARY_MASK) { case TAG_PRIMARY_IMMED1: if ((n = db_is_variable(c)) >= 0) { /* variable */ @@ -3318,6 +3585,16 @@ static DMCRet dmc_one_term(DMCContext *context, DMC_PUSH(*text, n); DMC_PUSH(*stack, c); break; + case (_TAG_HEADER_MAP >> _TAG_PRIMARY_SIZE): + if (is_flatmap(c)) + n = flatmap_get_size(flatmap_val(c)); + else + n = hashmap_size(c); + DMC_PUSH(*text, matchPushM); + ++(context->stack_used); + DMC_PUSH(*text, n); + DMC_PUSH(*stack, c); + break; case (_TAG_HEADER_REF >> _TAG_PRIMARY_SIZE): { Eterm* ref_val = internal_ref_val(c); @@ -3399,28 +3676,36 @@ static DMCRet dmc_one_term(DMCContext *context, #endif break; default: /* BINARY, FUN, VECTOR, or EXTERNAL */ - /* - ** Make a private copy... - */ - n = size_object(c); - tmp_mb = new_message_buffer(n); - hp = tmp_mb->mem; DMC_PUSH(*text, matchEqBin); - DMC_PUSH(*text, copy_struct(c, n, &hp, &(tmp_mb->off_heap))); - tmp_mb->next = context->save; - context->save = tmp_mb; + DMC_PUSH(*text, dmc_private_copy(context, c)); break; } break; } default: - erl_exit(1, "db_match_compile: " + erts_exit(ERTS_ERROR_EXIT, "db_match_compile: " "Bad object on heap: 0x%bex\n", c); } return retOk; } /* +** Make a private copy of a term in a context. +*/ + +static Eterm +dmc_private_copy(DMCContext *context, Eterm c) +{ + Uint n = size_object(c); + ErlHeapFragment *tmp_mb = new_message_buffer(n); + Eterm *hp = tmp_mb->mem; + Eterm copy = copy_struct(c, n, &hp, &(tmp_mb->off_heap)); + tmp_mb->next = context->save; + context->save = tmp_mb; + return copy; +} + +/* ** Match guard compilation */ @@ -3511,57 +3796,78 @@ static DMCRet dmc_list(DMCContext *context, return retOk; } -static DMCRet dmc_tuple(DMCContext *context, - DMCHeap *heap, - DMC_STACK_TYPE(UWord) *text, - Eterm t, - int *constant) +static void +dmc_rearrange_constants(DMCContext *context, DMC_STACK_TYPE(UWord) *text, + int textpos, Eterm *p, Uint nelems) { DMC_STACK_TYPE(UWord) instr_save; + Uint i; + + DMC_INIT_STACK(instr_save); + while (DMC_STACK_NUM(*text) > textpos) { + DMC_PUSH(instr_save, DMC_POP(*text)); + } + for (i = nelems; i--;) { + do_emit_constant(context, text, p[i]); + } + while(!DMC_EMPTY(instr_save)) { + DMC_PUSH(*text, DMC_POP(instr_save)); + } + DMC_FREE(instr_save); +} + +static DMCRet +dmc_array(DMCContext *context, DMCHeap *heap, DMC_STACK_TYPE(UWord) *text, + Eterm *p, Uint nelems, int *constant) +{ int all_constant = 1; int textpos = DMC_STACK_NUM(*text); - Eterm *p = tuple_val(t); - Uint nelems = arityval(*p); Uint i; - int c; - DMCRet ret; /* - ** We remember where we started to layout code, + ** We remember where we started to layout code, ** assume all is constant and back up and restart if not so. - ** The tuple should be laid out with the last element first, - ** so we can memcpy the tuple to the eheap. + ** The array should be laid out with the last element first, + ** so we can memcpy it to the eheap. */ - for (i = nelems; i > 0; --i) { - if ((ret = dmc_expr(context, heap, text, p[i], &c)) != retOk) - return ret; - if (!c && all_constant) { - all_constant = 0; - if (i < nelems) { - Uint j; + for (i = nelems; i--;) { + DMCRet ret; + int c; - /* - * Oops, we need to relayout the constants. - * Save the already laid out instructions. - */ - DMC_INIT_STACK(instr_save); - while (DMC_STACK_NUM(*text) > textpos) - DMC_PUSH(instr_save, DMC_POP(*text)); - for (j = nelems; j > i; --j) - do_emit_constant(context, text, p[j]); - while(!DMC_EMPTY(instr_save)) - DMC_PUSH(*text, DMC_POP(instr_save)); - DMC_FREE(instr_save); - } - } else if (c && !all_constant) { - /* push a constant */ - do_emit_constant(context, text, p[i]); - } + ret = dmc_expr(context, heap, text, p[i], &c); + if (ret != retOk) { + return ret; + } + if (!c && all_constant) { + all_constant = 0; + if (i < nelems - 1) { + dmc_rearrange_constants(context, text, textpos, + p + i + 1, nelems - i - 1); + } + } else if (c && !all_constant) { + do_emit_constant(context, text, p[i]); + } + } + *constant = all_constant; + return retOk; +} + +static DMCRet +dmc_tuple(DMCContext *context, DMCHeap *heap, DMC_STACK_TYPE(UWord) *text, + Eterm t, int *constant) +{ + int all_constant; + Eterm *p = tuple_val(t); + Uint nelems = arityval(*p); + DMCRet ret; + + ret = dmc_array(context, heap, text, p + 1, nelems, &all_constant); + if (ret != retOk) { + return ret; } - if (all_constant) { - *constant = 1; - return retOk; + *constant = 1; + return retOk; } DMC_PUSH(*text, matchMkTuple); DMC_PUSH(*text, nelems); @@ -3570,6 +3876,93 @@ static DMCRet dmc_tuple(DMCContext *context, return retOk; } +static DMCRet +dmc_map(DMCContext *context, DMCHeap *heap, DMC_STACK_TYPE(UWord) *text, + Eterm t, int *constant) +{ + int nelems; + int constant_values; + DMCRet ret; + if (is_flatmap(t)) { + flatmap_t *m = (flatmap_t *)flatmap_val(t); + Eterm *values = flatmap_get_values(m); + + nelems = flatmap_get_size(m); + ret = dmc_array(context, heap, text, values, nelems, &constant_values); + + if (ret != retOk) { + return ret; + } + if (constant_values) { + *constant = 1; + return retOk; + } + DMC_PUSH(*text, matchPushC); + DMC_PUSH(*text, dmc_private_copy(context, m->keys)); + if (++context->stack_used > context->stack_need) { + context->stack_need = context->stack_used; + } + DMC_PUSH(*text, matchMkFlatMap); + DMC_PUSH(*text, nelems); + context->stack_used -= nelems; + *constant = 0; + return retOk; + } else { + DECLARE_WSTACK(wstack); + Eterm *kv; + int c; + + ASSERT(is_hashmap(t)); + + hashmap_iterator_init(&wstack, t, 1); + constant_values = 1; + nelems = hashmap_size(t); + + while ((kv=hashmap_iterator_prev(&wstack)) != NULL) { + if ((ret = dmc_expr(context, heap, text, CDR(kv), &c)) != retOk) { + DESTROY_WSTACK(wstack); + return ret; + } + if (!c) + constant_values = 0; + } + + if (constant_values) { + *constant = 1; + DESTROY_WSTACK(wstack); + return retOk; + } + + *constant = 0; + + hashmap_iterator_init(&wstack, t, 1); + + while ((kv=hashmap_iterator_prev(&wstack)) != NULL) { + /* push key */ + if ((ret = dmc_expr(context, heap, text, CAR(kv), &c)) != retOk) { + DESTROY_WSTACK(wstack); + return ret; + } + if (c) { + do_emit_constant(context, text, CAR(kv)); + } + /* push value */ + if ((ret = dmc_expr(context, heap, text, CDR(kv), &c)) != retOk) { + DESTROY_WSTACK(wstack); + return ret; + } + if (c) { + do_emit_constant(context, text, CDR(kv)); + } + } + DMC_PUSH(*text, matchMkHashMap); + DMC_PUSH(*text, nelems); + context->stack_used -= nelems; + DESTROY_WSTACK(wstack); + return retOk; + } +} + static DMCRet dmc_whole_expression(DMCContext *context, DMCHeap *heap, DMC_STACK_TYPE(UWord) *text, @@ -4537,7 +4930,7 @@ static DMCRet dmc_fun(DMCContext *context, DMC_PUSH(*text, matchCall3); break; default: - erl_exit(1,"ets:match() internal error, " + erts_exit(ERTS_ERROR_EXIT,"ets:match() internal error, " "guard with more than 3 arguments."); } DMC_PUSH(*text, (UWord) b->biff); @@ -4564,7 +4957,10 @@ static DMCRet dmc_expr(DMCContext *context, return ret; break; case TAG_PRIMARY_BOXED: - if (!BOXED_IS_TUPLE(t)) { + if (is_map(t)) { + return dmc_map(context, heap, text, t, constant); + } + if (!is_tuple(t)) { goto simple_term; } p = tuple_val(t); @@ -4814,7 +5210,7 @@ static Uint my_size_object(Eterm t) tmp == am_const) { sum += size_object(tuple_val(t)[2]); } else { - erl_exit(1,"Internal error, sizing unrecognized object in " + erts_exit(ERTS_ERROR_EXIT,"Internal error, sizing unrecognized object in " "(d)ets:match compilation."); } break; @@ -4839,7 +5235,7 @@ static Eterm my_copy_struct(Eterm t, Eterm **hp, ErlOffHeap* off_heap) *hp += 2; break; case TAG_PRIMARY_BOXED: - if (BOXED_IS_TUPLE(t)) { + if (is_tuple(t)) { if (arityval(*tuple_val(t)) == 1 && is_tuple(a = tuple_val(t)[1])) { Uint i,n; @@ -4859,7 +5255,7 @@ static Eterm my_copy_struct(Eterm t, Eterm **hp, ErlOffHeap* off_heap) sz = size_object(b); ret = copy_struct(b,sz,hp,off_heap); } else { - erl_exit(1, "Trying to constant-copy non constant expression " + erts_exit(ERTS_ERROR_EXIT, "Trying to constant-copy non constant expression " "0x%bex in (d)ets:match compilation.", t); } } else { @@ -4971,7 +5367,8 @@ static Eterm match_spec_test(Process *p, Eterm against, Eterm spec, int trace) save_cp = p->cp; p->cp = NULL; res = erts_match_set_run(p, mps, arr, n, - ERTS_PAM_COPY_RESULT, &ret_flags); + ERTS_PAM_COPY_RESULT|ERTS_PAM_IGNORE_TRACE_SILENT, + &ret_flags); p->cp = save_cp; } else { n = 0; @@ -5109,6 +5506,18 @@ void db_match_dis(Binary *bp) ++t; erts_printf("Tuple\t%beu\n", n); break; + case matchMap: + ++t; + n = *t; + ++t; + erts_printf("Map\t%beu\n", n); + break; + case matchKey: + ++t; + p = (Eterm) *t; + ++t; + erts_printf("Key\t%p (%T)\n", t, p); + break; case matchPushT: ++t; n = *t; @@ -5119,10 +5528,20 @@ void db_match_dis(Binary *bp) ++t; erts_printf("PushL\n"); break; + case matchPushM: + ++t; + n = *t; + ++t; + erts_printf("PushM\t%beu\n", n); + break; case matchPop: ++t; erts_printf("Pop\n"); break; + case matchSwap: + ++t; + erts_printf("Swap\n"); + break; case matchBind: ++t; n = *t; @@ -5235,6 +5654,18 @@ void db_match_dis(Binary *bp) ++t; erts_printf("MkTuple\t%beu\n", n); break; + case matchMkFlatMap: + ++t; + n = *t; + ++t; + erts_printf("MkFlatMap\t%beu\n", n); + break; + case matchMkHashMap: + ++t; + n = *t; + ++t; + erts_printf("MkHashMap\t%beu\n", n); + break; case matchOr: ++t; n = *t; diff --git a/erts/emulator/beam/erl_db_util.h b/erts/emulator/beam/erl_db_util.h index 90b79e6044..0903a40460 100644 --- a/erts/emulator/beam/erl_db_util.h +++ b/erts/emulator/beam/erl_db_util.h @@ -3,16 +3,17 @@ * * Copyright Ericsson AB 1998-2013. All Rights Reserved. * - * The contents of this file are subject to the Erlang Public License, - * Version 1.1, (the "License"); you may not use this file except in - * compliance with the License. You should have received a copy of the - * Erlang Public License along with this software. If not, it can be - * retrieved online at http://www.erlang.org/. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Software distributed under the License is distributed on an "AS IS" - * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See - * the License for the specific language governing rights and limitations - * under the License. + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. * * %CopyrightEnd% */ @@ -76,6 +77,9 @@ typedef struct db_term { union db_table; typedef union db_table DbTable; +#define DB_MUST_RESIZE 1 +#define DB_NEW_OBJECT 2 + /* Info about a database entry while it's being updated * (by update_counter or update_element) */ @@ -84,7 +88,7 @@ typedef struct { DbTerm* dbterm; void** bp; /* {Hash|Tree}DbTerm** */ Uint new_size; - int mustResize; + int flags; void* lck; #if HALFWORD_HEAP unsigned char* abs_vec; /* [i] true if dbterm->tpl[i] is absolute Eterm */ @@ -165,6 +169,7 @@ typedef struct db_table_method DbTable* tb, /* [in out] */ Eterm continuation, Eterm* ret); + int (*db_take)(Process *, DbTable *, Eterm, Eterm *); int (*db_delete_all_objects)(Process* p, DbTable* db /* [in out] */ ); @@ -182,15 +187,14 @@ typedef struct db_table_method void *arg); void (*db_check_table)(DbTable* tb); - /* Lookup a dbterm for updating. Return false if not found. - */ - int (*db_lookup_dbterm)(DbTable*, Eterm key, - DbUpdateHandle* handle); /* [out] */ + /* Lookup a dbterm for updating. Return false if not found. */ + int (*db_lookup_dbterm)(Process *, DbTable *, Eterm key, Eterm obj, + DbUpdateHandle* handle); - /* Must be called for each db_lookup_dbterm that returned true, - ** even if dbterm was not updated. - */ - void (*db_finalize_dbterm)(DbUpdateHandle* handle); + /* Must be called for each db_lookup_dbterm that returned true, even if + ** dbterm was not updated. If the handle was of a new object and cret is + ** not DB_ERROR_NONE, the object is removed from the table. */ + void (*db_finalize_dbterm)(int cret, DbUpdateHandle* handle); } DbTableMethod; @@ -225,7 +229,10 @@ typedef struct db_table_common { DbTableMethod* meth; /* table methods */ erts_smp_atomic_t nitems; /* Total number of items in table */ erts_smp_atomic_t memory_size;/* Total memory size. NOTE: in bytes! */ - Uint megasec,sec,microsec; /* Last fixation time */ + struct { /* Last fixation time */ + ErtsMonotonicTime monotonic; + ErtsMonotonicTime offset; + } time; DbFixation* fixations; /* List of processes who have done safe_fixtable, "local" fixations not included. */ /* All 32-bit fields */ @@ -339,6 +346,7 @@ void* db_store_term(DbTableCommon *tb, DbTerm* old, Uint offset, Eterm obj); void* db_store_term_comp(DbTableCommon *tb, DbTerm* old, Uint offset, Eterm obj); Eterm db_copy_element_from_ets(DbTableCommon* tb, Process* p, DbTerm* obj, Uint pos, Eterm** hpp, Uint extra); +int db_has_map(Eterm obj); int db_has_variable(Eterm obj); int db_is_variable(Eterm obj); void db_do_update_element(DbUpdateHandle* handle, @@ -457,7 +465,7 @@ int erts_db_is_compiled_ms(Eterm term); && ERTS_MAGIC_BIN_DESTRUCTOR((BP)) == erts_db_match_prog_destructor) #define Binary2MatchProg(BP) \ - (ASSERT_EXPR(IsMatchProgBinary((BP))), \ + (ASSERT(IsMatchProgBinary((BP))), \ ((MatchProg *) ERTS_MAGIC_BIN_DATA((BP)))) /* ** Debugging diff --git a/erts/emulator/beam/erl_debug.c b/erts/emulator/beam/erl_debug.c index b90d00f236..4928aae9c2 100644 --- a/erts/emulator/beam/erl_debug.c +++ b/erts/emulator/beam/erl_debug.c @@ -1,18 +1,19 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 1998-2012. All Rights Reserved. + * Copyright Ericsson AB 1998-2013. All Rights Reserved. * - * The contents of this file are subject to the Erlang Public License, - * Version 1.1, (the "License"); you may not use this file except in - * compliance with the License. You should have received a copy of the - * Erlang Public License along with this software. If not, it can be - * retrieved online at http://www.erlang.org/. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Software distributed under the License is distributed on an "AS IS" - * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See - * the License for the specific language governing rights and limitations - * under the License. + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. * * %CopyrightEnd% */ @@ -29,6 +30,7 @@ #include "bif.h" #include "beam_catches.h" #include "erl_debug.h" +#include "erl_map.h" #define WITHIN(ptr, x, y) ((x) <= (ptr) && (ptr) < (y)) @@ -187,6 +189,9 @@ pdisplay1(int to, void *to_arg, Process* p, Eterm obj) case BINARY_DEF: erts_print(to, to_arg, "#Bin"); break; + case MATCHSTATE_DEF: + erts_print(to, to_arg, "#Matchstate"); + break; default: erts_print(to, to_arg, "unknown object %x", obj); } @@ -250,14 +255,14 @@ void erts_check_stack(Process *p) Eterm *stack_end = p->htop; if (p->stop > stack_start) - erl_exit(1, + erts_exit(ERTS_ERROR_EXIT, "<%lu.%lu.%lu>: Stack underflow\n", internal_pid_channel_no(p->common.id), internal_pid_number(p->common.id), internal_pid_serial(p->common.id)); if (p->stop < stack_end) - erl_exit(1, + erts_exit(ERTS_ERROR_EXIT, "<%lu.%lu.%lu>: Stack overflow\n", internal_pid_channel_no(p->common.id), internal_pid_number(p->common.id), @@ -282,7 +287,7 @@ void erts_check_stack(Process *p) if (in_mbuf) continue; - erl_exit(1, + erts_exit(ERTS_ERROR_EXIT, "<%lu.%lu.%lu>: Wild stack pointer\n", internal_pid_channel_no(p->common.id), internal_pid_number(p->common.id), @@ -299,6 +304,9 @@ void erts_check_for_holes(Process* p) ErlHeapFragment* hf; Eterm* start; + if (p->flags & F_DISABLE_GC) + return; + start = p->last_htop ? p->last_htop : HEAP_START(p); check_memory(start, HEAP_TOP(p)); p->last_htop = HEAP_TOP(p); @@ -364,7 +372,7 @@ void erts_check_memory(Process *p, Eterm *start, Eterm *end) #ifdef DEBUG if (hval == DEBUG_BAD_WORD) { print_untagged_memory(start, end); - erl_exit(1, "Uninitialized HAlloc'ed memory found @ 0x%0*lx!\n", + erts_exit(ERTS_ERROR_EXIT, "Uninitialized HAlloc'ed memory found @ 0x%0*lx!\n", PTR_SIZE,(unsigned long)(pos - 1)); } #endif @@ -377,7 +385,7 @@ void erts_check_memory(Process *p, Eterm *start, Eterm *end) if (verify_eterm(p,hval)) continue; - erl_exit(1, "Wild pointer found @ 0x%0*lx!\n", + erts_exit(ERTS_ERROR_EXIT, "Wild pointer found @ 0x%0*lx!\n", PTR_SIZE,(unsigned long)(pos - 1)); } } @@ -387,11 +395,11 @@ void verify_process(Process *p) #define VERIFY_AREA(name,ptr,sz) { \ int n = (sz); \ while (n--) if(!verify_eterm(p,*(ptr+n))) \ - erl_exit(1,"Wild pointer found in " name " of %T!\n",p->common.id); } + erts_exit(ERTS_ERROR_EXIT,"Wild pointer found in " name " of %T!\n",p->common.id); } #define VERIFY_ETERM(name,eterm) { \ if(!verify_eterm(p,eterm)) \ - erl_exit(1,"Wild pointer found in " name " of %T!\n",p->common.id); } + erts_exit(ERTS_ERROR_EXIT,"Wild pointer found in " name " of %T!\n",p->common.id); } ErlMessage* mp = p->msg.first; diff --git a/erts/emulator/beam/erl_debug.h b/erts/emulator/beam/erl_debug.h index af51212281..4905e64f07 100644 --- a/erts/emulator/beam/erl_debug.h +++ b/erts/emulator/beam/erl_debug.h @@ -3,16 +3,17 @@ * * Copyright Ericsson AB 2004-2012. All Rights Reserved. * - * The contents of this file are subject to the Erlang Public License, - * Version 1.1, (the "License"); you may not use this file except in - * compliance with the License. You should have received a copy of the - * Erlang Public License along with this software. If not, it can be - * retrieved online at http://www.erlang.org/. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Software distributed under the License is distributed on an "AS IS" - * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See - * the License for the specific language governing rights and limitations - * under the License. + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. * * %CopyrightEnd% */ diff --git a/erts/emulator/beam/erl_driver.h b/erts/emulator/beam/erl_driver.h index e280563de1..7ab58e336b 100644 --- a/erts/emulator/beam/erl_driver.h +++ b/erts/emulator/beam/erl_driver.h @@ -1,18 +1,19 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 1999-2013. All Rights Reserved. + * Copyright Ericsson AB 1999-2014. All Rights Reserved. * - * The contents of this file are subject to the Erlang Public License, - * Version 1.1, (the "License"); you may not use this file except in - * compliance with the License. You should have received a copy of the - * Erlang Public License along with this software. If not, it can be - * retrieved online at http://www.erlang.org/. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Software distributed under the License is distributed on an "AS IS" - * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See - * the License for the specific language governing rights and limitations - * under the License. + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. * * %CopyrightEnd% */ @@ -36,47 +37,6 @@ # endif #endif -#ifdef SIZEOF_CHAR -# define SIZEOF_CHAR_SAVED__ SIZEOF_CHAR -# undef SIZEOF_CHAR -#endif -#ifdef SIZEOF_SHORT -# define SIZEOF_SHORT_SAVED__ SIZEOF_SHORT -# undef SIZEOF_SHORT -#endif -#ifdef SIZEOF_INT -# define SIZEOF_INT_SAVED__ SIZEOF_INT -# undef SIZEOF_INT -#endif -#ifdef SIZEOF_LONG -# define SIZEOF_LONG_SAVED__ SIZEOF_LONG -# undef SIZEOF_LONG -#endif -#ifdef SIZEOF_LONG_LONG -# define SIZEOF_LONG_LONG_SAVED__ SIZEOF_LONG_LONG -# undef SIZEOF_LONG_LONG -#endif -#ifdef HALFWORD_HEAP_EMULATOR -# define HALFWORD_HEAP_EMULATOR_SAVED__ HALFWORD_HEAP_EMULATOR -# undef HALFWORD_HEAP_EMULATOR -#endif -#include "erl_int_sizes_config.h" -#if defined(SIZEOF_CHAR_SAVED__) && SIZEOF_CHAR_SAVED__ != SIZEOF_CHAR -# error SIZEOF_CHAR mismatch -#endif -#if defined(SIZEOF_SHORT_SAVED__) && SIZEOF_SHORT_SAVED__ != SIZEOF_SHORT -# error SIZEOF_SHORT mismatch -#endif -#if defined(SIZEOF_INT_SAVED__) && SIZEOF_INT_SAVED__ != SIZEOF_INT -# error SIZEOF_INT mismatch -#endif -#if defined(SIZEOF_LONG_SAVED__) && SIZEOF_LONG_SAVED__ != SIZEOF_LONG -# error SIZEOF_LONG mismatch -#endif -#if defined(SIZEOF_LONG_LONG_SAVED__) && SIZEOF_LONG_LONG_SAVED__ != SIZEOF_LONG_LONG -# error SIZEOF_LONG_LONG mismatch -#endif - /* This is OK to override by the NIF/driver implementor */ #if defined(HALFWORD_HEAP_EMULATOR_SAVED__) && !defined(HALFWORD_HEAP_EMULATOR) #define HALFWORD_HEAP_EMULATOR HALFWORD_HEAP_EMULATOR_SAVED__ @@ -132,8 +92,24 @@ typedef struct { #define DO_WRITE ERL_DRV_WRITE #define ERL_DRV_EXTENDED_MARKER (0xfeeeeeed) -#define ERL_DRV_EXTENDED_MAJOR_VERSION 2 -#define ERL_DRV_EXTENDED_MINOR_VERSION 1 +#define ERL_DRV_EXTENDED_MAJOR_VERSION 3 +#define ERL_DRV_EXTENDED_MINOR_VERSION 3 + +/* + * The emulator will refuse to load a driver with a major version + * lower than ERL_DRV_MIN_REQUIRED_MAJOR_VERSION_ON_LOAD. The load + * may however fail if user have not removed use of deprecated + * symbols. + * + * The ERL_DRV_MIN_REQUIRED_MAJOR_VERSION_ON_LOAD have to allow + * loading of drivers built at least two major OTP releases + * ago. + * + * Bump of major version to 3 happened in OTP 17. That is, in + * OTP 19 we can increase ERL_DRV_MIN_REQUIRED_MAJOR_VERSION_ON_LOAD + * to 3. + */ +#define ERL_DRV_MIN_REQUIRED_MAJOR_VERSION_ON_LOAD 2 /* * The emulator will refuse to load a driver with different major @@ -159,30 +135,14 @@ typedef struct { /* * Integer types */ -#if defined(__WIN32__) && (SIZEOF_VOID_P == 8) -typedef unsigned __int64 ErlDrvTermData; -typedef unsigned __int64 ErlDrvUInt; -typedef signed __int64 ErlDrvSInt; -#else -typedef unsigned long ErlDrvTermData; -typedef unsigned long ErlDrvUInt; -typedef signed long ErlDrvSInt; -#endif -#if defined(__WIN32__) -typedef unsigned __int64 ErlDrvUInt64; -typedef __int64 ErlDrvSInt64; -#elif SIZEOF_LONG == 8 -typedef unsigned long ErlDrvUInt64; -typedef long ErlDrvSInt64; -#elif SIZEOF_LONG_LONG == 8 -typedef unsigned long long ErlDrvUInt64; -typedef long long ErlDrvSInt64; -#else -#error No 64-bit integer type -#endif +typedef ErlNapiUInt64 ErlDrvUInt64; +typedef ErlNapiSInt64 ErlDrvSInt64; +typedef ErlNapiUInt ErlDrvUInt; +typedef ErlNapiSInt ErlDrvSInt; +typedef ErlNapiUInt ErlDrvTermData; -#if defined(__WIN32__) +#if defined(__WIN32__) || defined(_WIN32) typedef ErlDrvUInt ErlDrvSizeT; typedef ErlDrvSInt ErlDrvSSizeT; #else @@ -233,6 +193,17 @@ typedef struct { unsigned long microsecs; } ErlDrvNowData; +typedef ErlDrvSInt64 ErlDrvTime; + +#define ERL_DRV_TIME_ERROR ((ErlDrvSInt64) ERTS_NAPI_TIME_ERROR__) + +typedef enum { + ERL_DRV_SEC = ERTS_NAPI_SEC__, + ERL_DRV_MSEC = ERTS_NAPI_MSEC__, + ERL_DRV_USEC = ERTS_NAPI_USEC__, + ERL_DRV_NSEC = ERTS_NAPI_NSEC__ +} ErlDrvTimeUnit; + /* * Error codes that can be return from driver. */ @@ -271,7 +242,6 @@ typedef struct ErlDrvCond_ ErlDrvCond; typedef struct ErlDrvRWLock_ ErlDrvRWLock; typedef int ErlDrvTSDKey; - /* * */ @@ -346,6 +316,9 @@ typedef struct erl_drv_entry { /* Called on behalf of driver_select when it is safe to release 'event'. A typical unix driver would call close(event) */ + void (*emergency_close)(ErlDrvData drv_data); + /* called when the port is closed abruptly. + specifically when erl_crash_dump is called. */ /* When adding entries here, dont forget to pad in obsolete/driver.h */ } ErlDrvEntry; @@ -365,17 +338,23 @@ typedef struct erl_drv_entry { * It must initialize a ErlDrvEntry structure and return a pointer to it. */ +#ifdef STATIC_ERLANG_DRIVER +# define ERLANG_DRIVER_NAME(NAME) NAME ## _driver_init +#else +# define ERLANG_DRIVER_NAME(NAME) driver_init +#endif + /* For windows dynamic drivers */ #ifndef ERL_DRIVER_TYPES_ONLY #if defined(__WIN32__) # define DRIVER_INIT(DRIVER_NAME) \ - __declspec(dllexport) ErlDrvEntry* driver_init(void); \ - __declspec(dllexport) ErlDrvEntry* driver_init(void) + __declspec(dllexport) ErlDrvEntry* ERLANG_DRIVER_NAME(DRIVER_NAME)(void); \ + __declspec(dllexport) ErlDrvEntry* ERLANG_DRIVER_NAME(DRIVER_NAME)(void) #else # define DRIVER_INIT(DRIVER_NAME) \ - ErlDrvEntry* driver_init(void); \ - ErlDrvEntry* driver_init(void) + ErlDrvEntry* ERLANG_DRIVER_NAME(DRIVER_NAME)(void); \ + ErlDrvEntry* ERLANG_DRIVER_NAME(DRIVER_NAME)(void) #endif #define ERL_DRV_BUSY_MSGQ_DISABLED (~((ErlDrvSizeT) 0)) @@ -546,6 +525,11 @@ EXTERN int erl_drv_equal_tids(ErlDrvTid tid1, ErlDrvTid tid2); EXTERN void erl_drv_thread_exit(void *resp); EXTERN int erl_drv_thread_join(ErlDrvTid, void **respp); +EXTERN char* erl_drv_mutex_name(ErlDrvMutex *mtx); +EXTERN char* erl_drv_cond_name(ErlDrvCond *cnd); +EXTERN char* erl_drv_rwlock_name(ErlDrvRWLock *rwlck); +EXTERN char* erl_drv_thread_name(ErlDrvTid tid); + /* * Misc. */ @@ -594,6 +578,8 @@ EXTERN int null_func(void); #define ERL_DRV_INT64 ((ErlDrvTermData) 15) /* ErlDrvSInt64 * */ #define ERL_DRV_UINT64 ((ErlDrvTermData) 16) /* ErlDrvUInt64 * */ +#define ERL_DRV_MAP ((ErlDrvTermData) 17) /* ErlDrvUInt */ + #ifndef ERL_DRIVER_TYPES_ONLY /* make terms for driver_output_term and driver_send_term */ @@ -638,18 +624,14 @@ EXTERN int erl_drv_send_term(ErlDrvTermData port, int len); /* Async IO functions */ +EXTERN unsigned int driver_async_port_key(ErlDrvPort port); + EXTERN long driver_async(ErlDrvPort ix, unsigned int* key, void (*async_invoke)(void*), void* async_data, void (*async_free)(void*)); -/* - * driver_async_cancel() is deprecated. It is scheduled for removal - * in OTP-R16. For more information see the erl_driver(3) documentation. - */ -EXTERN int driver_async_cancel(unsigned int key) ERL_DRV_DEPRECATED_FUNC; - /* Locks the driver in the machine "forever", there is no unlock function. Note that this is almost never useful, as an open port towards the driver locks it until the port is closed, why unexpected @@ -657,8 +639,16 @@ EXTERN int driver_async_cancel(unsigned int key) ERL_DRV_DEPRECATED_FUNC; EXTERN int driver_lock_driver(ErlDrvPort ix); /* Get the current 'now' timestamp (analogue to erlang:now()) */ -EXTERN int driver_get_now(ErlDrvNowData *now); +EXTERN int driver_get_now(ErlDrvNowData *now) ERL_DRV_DEPRECATED_FUNC; +/* Erlang Monotonic Time */ +EXTERN ErlDrvTime erl_drv_monotonic_time(ErlDrvTimeUnit time_unit); +/* Time offset between Erlang Monotonic Time and Erlang System Time */ +EXTERN ErlDrvTime erl_drv_time_offset(ErlDrvTimeUnit time_unit); +/* Time unit conversion */ +EXTERN ErlDrvTime erl_drv_convert_time_unit(ErlDrvTime val, + ErlDrvTimeUnit from, + ErlDrvTimeUnit to); /* These were removed from the ANSI version, now they're back. */ @@ -668,8 +658,18 @@ EXTERN int driver_dl_close(void *); EXTERN char *driver_dl_error(void); /* environment */ -EXTERN int erl_drv_putenv(char *key, char *value); -EXTERN int erl_drv_getenv(char *key, char *value, size_t *value_size); +EXTERN int erl_drv_putenv(const char *key, char *value); +EXTERN int erl_drv_getenv(const char *key, char *value, size_t *value_size); + +#ifdef __OSE__ +typedef ErlDrvUInt ErlDrvOseEventId; +EXTERN union SIGNAL *erl_drv_ose_get_signal(ErlDrvEvent ev); +EXTERN ErlDrvEvent erl_drv_ose_event_alloc(SIGSELECT sig, ErlDrvOseEventId handle, + ErlDrvOseEventId (*resolve_signal)(union SIGNAL *sig), void *extra); +EXTERN void erl_drv_ose_event_free(ErlDrvEvent ev); +EXTERN void erl_drv_ose_event_fetch(ErlDrvEvent ev, SIGSELECT *sig, + ErlDrvOseEventId *handle, void **extra); +#endif #endif /* !ERL_DRIVER_TYPES_ONLY */ @@ -681,6 +681,3 @@ EXTERN int erl_drv_getenv(char *key, char *value, size_t *value_size); /* also in global.h, but driver's can't include global.h */ void dtrace_drvport_str(ErlDrvPort port, char *port_buf); - - - diff --git a/erts/emulator/beam/erl_drv_nif.h b/erts/emulator/beam/erl_drv_nif.h index ea013a49a3..f6b946ae82 100644 --- a/erts/emulator/beam/erl_drv_nif.h +++ b/erts/emulator/beam/erl_drv_nif.h @@ -3,16 +3,17 @@ * * Copyright Ericsson AB 2010. All Rights Reserved. * - * The contents of this file are subject to the Erlang Public License, - * Version 1.1, (the "License"); you may not use this file except in - * compliance with the License. You should have received a copy of the - * Erlang Public License along with this software. If not, it can be - * retrieved online at http://www.erlang.org/. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Software distributed under the License is distributed on an "AS IS" - * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See - * the License for the specific language governing rights and limitations - * under the License. + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. * * %CopyrightEnd% */ @@ -35,12 +36,104 @@ typedef struct { int scheduler_threads; int nif_major_version; int nif_minor_version; + int dirty_scheduler_support; } ErlDrvSysInfo; typedef struct { int suggested_stack_size; } ErlDrvThreadOpts; +#if defined(ERL_DRV_DIRTY_SCHEDULER_SUPPORT) || defined(ERL_NIF_DIRTY_SCHEDULER_SUPPORT) +typedef enum { + ERL_DRV_DIRTY_JOB_CPU_BOUND = 1, + ERL_DRV_DIRTY_JOB_IO_BOUND = 2 +} ErlDrvDirtyJobFlags; +#endif + +#ifdef SIZEOF_CHAR +# define SIZEOF_CHAR_SAVED__ SIZEOF_CHAR +# undef SIZEOF_CHAR +#endif +#ifdef SIZEOF_SHORT +# define SIZEOF_SHORT_SAVED__ SIZEOF_SHORT +# undef SIZEOF_SHORT +#endif +#ifdef SIZEOF_INT +# define SIZEOF_INT_SAVED__ SIZEOF_INT +# undef SIZEOF_INT +#endif +#ifdef SIZEOF_LONG +# define SIZEOF_LONG_SAVED__ SIZEOF_LONG +# undef SIZEOF_LONG +#endif +#ifdef SIZEOF_LONG_LONG +# define SIZEOF_LONG_LONG_SAVED__ SIZEOF_LONG_LONG +# undef SIZEOF_LONG_LONG +#endif +#ifdef HALFWORD_HEAP_EMULATOR +# define HALFWORD_HEAP_EMULATOR_SAVED__ HALFWORD_HEAP_EMULATOR +# undef HALFWORD_HEAP_EMULATOR +#endif +#include "erl_int_sizes_config.h" +#if defined(SIZEOF_CHAR_SAVED__) && SIZEOF_CHAR_SAVED__ != SIZEOF_CHAR +# error SIZEOF_CHAR mismatch +#endif +#if defined(SIZEOF_SHORT_SAVED__) && SIZEOF_SHORT_SAVED__ != SIZEOF_SHORT +# error SIZEOF_SHORT mismatch +#endif +#if defined(SIZEOF_INT_SAVED__) && SIZEOF_INT_SAVED__ != SIZEOF_INT +# error SIZEOF_INT mismatch +#endif +#if defined(SIZEOF_LONG_SAVED__) && SIZEOF_LONG_SAVED__ != SIZEOF_LONG +# error SIZEOF_LONG mismatch +#endif +#if defined(SIZEOF_LONG_LONG_SAVED__) && SIZEOF_LONG_LONG_SAVED__ != SIZEOF_LONG_LONG +# error SIZEOF_LONG_LONG mismatch +#endif + +#if !defined(__GNUC__) && (defined(__WIN32__) || defined(_WIN32) || defined(_WIN32_)) +typedef unsigned __int64 ErlNapiUInt64; +typedef signed __int64 ErlNapiSInt64; +#define ERL_NAPI_SINT64_MAX__ 9223372036854775807i64 +#define ERL_NAPI_SINT64_MIN__ (-ERL_NAPI_SINT64_MAX__ - 1i64) +#elif SIZEOF_LONG == 8 +typedef unsigned long ErlNapiUInt64; +typedef signed long ErlNapiSInt64; +#define ERL_NAPI_SINT64_MAX__ 9223372036854775807L +#define ERL_NAPI_SINT64_MIN__ (-ERL_NAPI_SINT64_MAX__ - 1L) +#elif SIZEOF_LONG_LONG == 8 +typedef unsigned long long ErlNapiUInt64; +typedef signed long long ErlNapiSInt64; +#define ERL_NAPI_SINT64_MAX__ 9223372036854775807LL +#define ERL_NAPI_SINT64_MIN__ (-ERL_NAPI_SINT64_MAX__ - 1LL) +#else +# error No 64-bit integer type +#endif + +#if SIZEOF_VOID_P == 8 +typedef ErlNapiUInt64 ErlNapiUInt; +typedef ErlNapiSInt64 ErlNapiSInt; +#elif SIZEOF_VOID_P == 4 +# if SIZEOF_LONG == SIZEOF_VOID_P +typedef unsigned long ErlNapiUInt; +typedef signed long ErlNapiSInt; +# elif SIZEOF_INT == SIZEOF_VOID_P +typedef unsigned int ErlNapiUInt; +typedef signed int ErlNapiSInt; +# else +# error No 32-bit integer type +# endif +#else +# error Not support arch +#endif + +#define ERTS_NAPI_TIME_ERROR__ ERL_NAPI_SINT64_MIN__ + +#define ERTS_NAPI_SEC__ 0 +#define ERTS_NAPI_MSEC__ 1 +#define ERTS_NAPI_USEC__ 2 +#define ERTS_NAPI_NSEC__ 3 + #endif /* __ERL_DRV_NIF_H__ */ diff --git a/erts/emulator/beam/erl_drv_thread.c b/erts/emulator/beam/erl_drv_thread.c index a49a155701..184f8e8931 100644 --- a/erts/emulator/beam/erl_drv_thread.c +++ b/erts/emulator/beam/erl_drv_thread.c @@ -3,16 +3,17 @@ * * Copyright Ericsson AB 2007-2011. All Rights Reserved. * - * The contents of this file are subject to the Erlang Public License, - * Version 1.1, (the "License"); you may not use this file except in - * compliance with the License. You should have received a copy of the - * Erlang Public License along with this software. If not, it can be - * retrieved online at http://www.erlang.org/. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Software distributed under the License is distributed on an "AS IS" - * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See - * the License for the specific language governing rights and limitations - * under the License. + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. * * %CopyrightEnd% */ @@ -42,7 +43,7 @@ fatal_error(int err, char *func) else estr = "Unknown error"; } - erl_exit(ERTS_ABORT_EXIT, "Fatal error in %s: %s [%d]\n", func, estr, err); + erts_exit(ERTS_ABORT_EXIT, "Fatal error in %s: %s [%d]\n", func, estr, err); } #define ERL_DRV_TSD_KEYS_INC 10 @@ -78,8 +79,6 @@ struct ErlDrvTid_ { static ethr_tsd_key tid_key; -static ethr_thr_opts def_ethr_opts = ETHR_THR_OPTS_DEFAULT_INITER; - #else /* USE_THREADS */ static Uint tsd_len; static void **tsd; @@ -123,7 +122,7 @@ void erl_drv_thr_init(void) { int i; #ifdef USE_THREADS - int res = ethr_tsd_key_create(&tid_key); + int res = ethr_tsd_key_create(&tid_key,"erts_tid_key"); if (res == 0) res = ethr_install_exit_handler(thread_exit_handler); if (res != 0) @@ -188,6 +187,17 @@ erl_drv_mutex_destroy(ErlDrvMutex *dmtx) #endif } + +char * +erl_drv_mutex_name(ErlDrvMutex *dmtx) +{ +#ifdef USE_THREADS + return dmtx ? dmtx->name : NULL; +#else + return NULL; +#endif +} + int erl_drv_mutex_trylock(ErlDrvMutex *dmtx) { @@ -258,6 +268,15 @@ erl_drv_cond_destroy(ErlDrvCond *dcnd) #endif } +char * +erl_drv_cond_name(ErlDrvCond *dcnd) +{ +#ifdef USE_THREADS + return dcnd ? dcnd->name : NULL; +#else + return NULL; +#endif +} void erl_drv_cond_signal(ErlDrvCond *dcnd) @@ -331,6 +350,16 @@ erl_drv_rwlock_destroy(ErlDrvRWLock *drwlck) #endif } +char * +erl_drv_rwlock_name(ErlDrvRWLock *drwlck) +{ +#ifdef USE_THREADS + return drwlck ? drwlck->name : NULL; +#else + return NULL; +#endif +} + int erl_drv_rwlock_tryrlock(ErlDrvRWLock *drwlck) { @@ -573,16 +602,16 @@ erl_drv_thread_create(char *name, #ifdef USE_THREADS int res; struct ErlDrvTid_ *dtid; - ethr_thr_opts ethr_opts; + ethr_thr_opts ethr_opts = ETHR_THR_OPTS_DEFAULT_INITER; ethr_thr_opts *use_opts; - if (!opts) + if (!opts && !name) use_opts = NULL; else { - sys_memcpy((void *) ðr_opts, - (void *) &def_ethr_opts, - sizeof(ethr_thr_opts)); - ethr_opts.suggested_stack_size = opts->suggested_stack_size; + if(opts) + ethr_opts.suggested_stack_size = opts->suggested_stack_size; + + ethr_opts.name = name; use_opts = ðr_opts; } @@ -617,6 +646,18 @@ erl_drv_thread_create(char *name, #endif } +char * +erl_drv_thread_name(ErlDrvTid tid) +{ +#ifdef USE_THREADS + struct ErlDrvTid_ *dtid = (struct ErlDrvTid_ *) tid; + return dtid ? dtid->name : NULL; +#else + return NULL; +#endif +} + + ErlDrvTid erl_drv_thread_self(void) { diff --git a/erts/emulator/beam/erl_fun.c b/erts/emulator/beam/erl_fun.c index 88947b5536..2f13aa364f 100644 --- a/erts/emulator/beam/erl_fun.c +++ b/erts/emulator/beam/erl_fun.c @@ -3,16 +3,17 @@ * * Copyright Ericsson AB 2000-2010. All Rights Reserved. * - * The contents of this file are subject to the Erlang Public License, - * Version 1.1, (the "License"); you may not use this file except in - * compliance with the License. You should have received a copy of the - * Erlang Public License along with this software. If not, it can be - * retrieved online at http://www.erlang.org/. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Software distributed under the License is distributed on an "AS IS" - * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See - * the License for the specific language governing rights and limitations - * under the License. + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. * * %CopyrightEnd% */ @@ -184,7 +185,7 @@ erts_erase_fun_entry(ErlFunEntry* fe) #endif { if (fe->address != unloaded_fun) - erl_exit(1, + erts_exit(ERTS_ERROR_EXIT, "Internal error: " "Invalid reference count found on #Fun<%T.%d.%d>: " " About to erase fun still referred by code.\n", diff --git a/erts/emulator/beam/erl_fun.h b/erts/emulator/beam/erl_fun.h index b673ef6b3c..0024b1ff71 100644 --- a/erts/emulator/beam/erl_fun.h +++ b/erts/emulator/beam/erl_fun.h @@ -3,16 +3,17 @@ * * Copyright Ericsson AB 2000-2012. All Rights Reserved. * - * The contents of this file are subject to the Erlang Public License, - * Version 1.1, (the "License"); you may not use this file except in - * compliance with the License. You should have received a copy of the - * Erlang Public License along with this software. If not, it can be - * retrieved online at http://www.erlang.org/. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Software distributed under the License is distributed on an "AS IS" - * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See - * the License for the specific language governing rights and limitations - * under the License. + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. * * %CopyrightEnd% */ diff --git a/erts/emulator/beam/erl_gc.c b/erts/emulator/beam/erl_gc.c index 298909c921..f48c46ca33 100644 --- a/erts/emulator/beam/erl_gc.c +++ b/erts/emulator/beam/erl_gc.c @@ -1,18 +1,19 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2002-2013. All Rights Reserved. + * Copyright Ericsson AB 2002-2014. All Rights Reserved. * - * The contents of this file are subject to the Erlang Public License, - * Version 1.1, (the "License"); you may not use this file except in - * compliance with the License. You should have received a copy of the - * Erlang Public License along with this software. If not, it can be - * retrieved online at http://www.erlang.org/. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Software distributed under the License is distributed on an "AS IS" - * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See - * the License for the specific language governing rights and limitations - * under the License. + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. * * %CopyrightEnd% */ @@ -20,6 +21,8 @@ # include "config.h" #endif +#define ERL_WANT_GC_INTERNALS__ + #include "sys.h" #include "erl_vm.h" #include "global.h" @@ -28,6 +31,7 @@ #include "beam_catches.h" #include "erl_binary.h" #include "erl_bits.h" +#include "erl_map.h" #include "error.h" #include "big.h" #include "erl_gc.h" @@ -36,6 +40,7 @@ #include "hipe_mode_switch.h" #endif #include "dtrace-wrapper.h" +#include "erl_bif_unique.h" #define ERTS_INACT_WR_PB_LEAVE_MUCH_LIMIT 1 #define ERTS_INACT_WR_PB_LEAVE_MUCH_PERCENTAGE 20 @@ -47,10 +52,6 @@ */ #define ALENGTH(a) (sizeof(a)/sizeof(a[0])) -static erts_smp_spinlock_t info_lck; -static Uint garbage_cols; /* no of garbage collections */ -static Uint reclaimed; /* no of words reclaimed in GCs */ - # define STACK_SZ_ON_HEAP(p) ((p)->hend - (p)->stop) # define OverRunCheck(P) \ if ((P)->stop < (P)->htop) { \ @@ -58,7 +59,7 @@ static Uint reclaimed; /* no of words reclaimed in GCs */ erts_fprintf(stderr, "stop=%p\n", (p)->stop); \ erts_fprintf(stderr, "htop=%p\n", (p)->htop); \ erts_fprintf(stderr, "heap=%p\n", (p)->heap); \ - erl_exit(ERTS_ABORT_EXIT, "%s, line %d: %T: Overrun stack and heap\n", \ + erts_exit(ERTS_ABORT_EXIT, "%s, line %d: %T: Overrun stack and heap\n", \ __FILE__,__LINE__,(P)->common.id); \ } @@ -108,8 +109,7 @@ static Eterm* sweep_one_heap(Eterm* heap_ptr, Eterm* heap_end, Eterm* htop, char* src, Uint src_size); static Eterm* collect_heap_frags(Process* p, Eterm* heap, Eterm* htop, Eterm* objv, int nobj); -static Uint adjust_after_fullsweep(Process *p, Uint size_before, - int need, Eterm *objv, int nobj); +static void adjust_after_fullsweep(Process *p, int need, Eterm *objv, int nobj); static void shrink_new_heap(Process *p, Uint new_sz, Eterm *objv, int nobj); static void grow_new_heap(Process *p, Uint new_sz, Eterm* objv, int nobj); static void sweep_off_heap(Process *p, int fullsweep); @@ -120,6 +120,8 @@ static void offset_rootset(Process *p, Sint offs, char* area, Uint area_size, static void offset_off_heap(Process* p, Sint offs, char* area, Uint area_size); static void offset_mqueue(Process *p, Sint offs, char* area, Uint area_size); +static void init_gc_info(ErtsGCInfo *gcip); + #ifdef HARDDEBUG static void disallow_heap_frag_ref_in_heap(Process* p); static void disallow_heap_frag_ref_in_old_heap(Process* p); @@ -137,28 +139,53 @@ static int num_heap_sizes; /* Number of heap sizes. */ Uint erts_test_long_gc_sleep; /* Only used for testing... */ +typedef struct { + Process *proc; + Eterm ref; + Eterm ref_heap[REF_THING_SIZE]; + Uint req_sched; + erts_smp_atomic32_t refc; +} ErtsGCInfoReq; + +#if !HALFWORD_HEAP +ERTS_SCHED_PREF_QUICK_ALLOC_IMPL(gcireq, + ErtsGCInfoReq, + 5, + ERTS_ALC_T_GC_INFO_REQ) +#else +static ERTS_INLINE ErtsGCInfoReq * +gcireq_alloc(void) +{ + return erts_alloc(ERTS_ALC_T_GC_INFO_REQ, + sizeof(ErtsGCInfoReq)); +} + +static ERTS_INLINE void +gcireq_free(ErtsGCInfoReq *ptr) +{ + erts_free(ERTS_ALC_T_GC_INFO_REQ, ptr); +} +#endif + /* * Initialize GC global data. */ void erts_init_gc(void) { - int i = 0; + int i = 0, ix; Sint max_heap_size = 0; - ASSERT(offsetof(ProcBin,thing_word) == offsetof(struct erl_off_heap_header,thing_word)); - ASSERT(offsetof(ProcBin,thing_word) == offsetof(ErlFunThing,thing_word)); - ASSERT(offsetof(ProcBin,thing_word) == offsetof(ExternalThing,header)); - ASSERT(offsetof(ProcBin,size) == offsetof(struct erl_off_heap_header,size)); - ASSERT(offsetof(ProcBin,size) == offsetof(ErlSubBin,size)); - ASSERT(offsetof(ProcBin,size) == offsetof(ErlHeapBin,size)); - ASSERT(offsetof(ProcBin,next) == offsetof(struct erl_off_heap_header,next)); - ASSERT(offsetof(ProcBin,next) == offsetof(ErlFunThing,next)); - ASSERT(offsetof(ProcBin,next) == offsetof(ExternalThing,next)); - - erts_smp_spinlock_init(&info_lck, "gc_info"); - garbage_cols = 0; - reclaimed = 0; + ERTS_CT_ASSERT(offsetof(ProcBin,thing_word) == offsetof(struct erl_off_heap_header,thing_word)); + ERTS_CT_ASSERT(offsetof(ProcBin,thing_word) == offsetof(ErlFunThing,thing_word)); + ERTS_CT_ASSERT(offsetof(ProcBin,thing_word) == offsetof(ExternalThing,header)); + ERTS_CT_ASSERT(offsetof(ProcBin,size) == offsetof(struct erl_off_heap_header,size)); + ERTS_CT_ASSERT(offsetof(ProcBin,size) == offsetof(ErlSubBin,size)); + ERTS_CT_ASSERT(offsetof(ProcBin,size) == offsetof(ErlHeapBin,size)); + ERTS_CT_ASSERT(offsetof(ProcBin,next) == offsetof(struct erl_off_heap_header,next)); + ERTS_CT_ASSERT(offsetof(ProcBin,next) == offsetof(ErlFunThing,next)); + ERTS_CT_ASSERT(offsetof(ProcBin,next) == offsetof(ExternalThing,next)); + erts_test_long_gc_sleep = 0; /* @@ -199,6 +226,16 @@ erts_init_gc(void) } } num_heap_sizes = i; + + for (ix = 0; ix < erts_no_schedulers; ix++) { + ErtsSchedulerData *esdp = ERTS_SCHEDULER_IX(ix); + init_gc_info(&esdp->gc_info); + } + +#if !HALFWORD_HEAP + init_gcireq_alloc(); +#endif + } /* @@ -231,7 +268,7 @@ erts_next_heap_size(Uint size, Uint offset) low = mid + 1; } } - erl_exit(1, "no next heap size found: %beu, offset %beu\n", size, offset); + erts_exit(ERTS_ERROR_EXIT, "no next heap size found: %beu, offset %beu\n", size, offset); } return 0; } @@ -287,17 +324,6 @@ erts_heap_sizes(Process* p) return res; } -void -erts_gc_info(ErtsGCInfo *gcip) -{ - if (gcip) { - erts_smp_spin_lock(&info_lck); - gcip->garbage_collections = garbage_cols; - gcip->reclaimed = reclaimed; - erts_smp_spin_unlock(&info_lck); - } -} - void erts_offset_heap(Eterm* hp, Uint sz, Sint offs, Eterm* low, Eterm* high) { @@ -377,18 +403,26 @@ erts_garbage_collect(Process* p, int need, Eterm* objv, int nobj) { Uint reclaimed_now = 0; int done = 0; - Uint ms1, s1, us1; + ErtsMonotonicTime start_time = 0; /* Shut up faulty warning... */ + ErtsSchedulerData *esdp; #ifdef USE_VM_PROBES DTRACE_CHARBUF(pidbuf, DTRACE_TERM_BUF_SIZE); #endif + + if (p->flags & F_DISABLE_GC) { + ASSERT(need == 0); + return 1; + } + + esdp = erts_get_scheduler_data(); + if (IS_TRACED_FL(p, F_TRACE_GC)) { trace_gc(p, am_gc_start); } - erts_smp_atomic32_read_bor_nob(&p->state, ERTS_PSFLG_GC); - if (erts_system_monitor_long_gc != 0) { - get_now(&ms1, &s1, &us1); - } + (void) erts_smp_atomic32_read_bor_nob(&p->state, ERTS_PSFLG_GC); + if (erts_system_monitor_long_gc != 0) + start_time = erts_get_monotonic_time(esdp); ERTS_CHK_OFFHEAP(p); @@ -436,16 +470,14 @@ erts_garbage_collect(Process* p, int need, Eterm* objv, int nobj) } if (erts_system_monitor_long_gc != 0) { - Uint ms2, s2, us2; - Sint t; + ErtsMonotonicTime end_time; + Uint gc_time; if (erts_test_long_gc_sleep) while (0 != erts_milli_sleep(erts_test_long_gc_sleep)); - get_now(&ms2, &s2, &us2); - t = ms2 - ms1; - t = t*1000000 + s2 - s1; - t = t*1000 + ((Sint) (us2 - us1))/1000; - if (t > 0 && (Uint)t > erts_system_monitor_long_gc) { - monitor_long_gc(p, t); + end_time = erts_get_monotonic_time(esdp); + gc_time = (Uint) ERTS_MONOTONIC_TO_MSEC(end_time - start_time); + if (gc_time && gc_time > erts_system_monitor_long_gc) { + monitor_long_gc(p, gc_time); } } if (erts_system_monitor_large_heap != 0) { @@ -455,11 +487,9 @@ erts_garbage_collect(Process* p, int need, Eterm* objv, int nobj) monitor_large_heap(p); } - erts_smp_spin_lock(&info_lck); - garbage_cols++; - reclaimed += reclaimed_now; - erts_smp_spin_unlock(&info_lck); - + esdp->gc_info.garbage_cols++; + esdp->gc_info.reclaimed += reclaimed_now; + FLAGS(p) &= ~F_FORCE_GC; #ifdef CHECK_FOR_HOLES @@ -511,6 +541,9 @@ erts_garbage_collect_hibernate(Process* p) Uint area_size; Sint offs; + if (p->flags & F_DISABLE_GC) + ERTS_INTERNAL_ERROR("GC disabled"); + /* * Preliminaries. */ @@ -644,8 +677,10 @@ erts_garbage_collect_literals(Process* p, Eterm* literals, Uint area_size; Eterm* old_htop; Uint n; - struct erl_off_heap_header** prev; + struct erl_off_heap_header** prev = NULL; + if (p->flags & F_DISABLE_GC) + return; /* * Set GC state. */ @@ -751,10 +786,10 @@ erts_garbage_collect_literals(Process* p, Eterm* literals, */ if (oh) { - prev = &MSO(p).first; - while (*prev) { - prev = &(*prev)->next; - } + prev = &MSO(p).first; + while (*prev) { + prev = &(*prev)->next; + } } /* @@ -783,6 +818,10 @@ erts_garbage_collect_literals(Process* p, Eterm* literals, oh = oh->next; } + if (prev) { + *prev = NULL; + } + /* * We no longer need this temporary area. */ @@ -833,29 +872,37 @@ minor_collection(Process* p, int need, Eterm* objv, int nobj, Uint *recl) ErlMessage *msgp; Uint size_after; Uint need_after; - Uint stack_size = STACK_SZ_ON_HEAP(p); - Uint fragments = MBUF_SIZE(p) + combined_message_size(p); - Uint size_before = fragments + (HEAP_TOP(p) - HEAP_START(p)); - Uint new_sz = next_heap_size(p, HEAP_SIZE(p) + fragments, 0); + const Uint stack_size = STACK_SZ_ON_HEAP(p); + const Uint size_before = MBUF_SIZE(p) + (HEAP_TOP(p) - HEAP_START(p)); + Uint new_sz = HEAP_SIZE(p) + MBUF_SIZE(p) + combined_message_size(p); + new_sz = next_heap_size(p, new_sz, 0); do_minor(p, new_sz, objv, nobj); - /* + size_after = HEAP_TOP(p) - HEAP_START(p); + *recl += (size_before - size_after); + + /* * Copy newly received message onto the end of the new heap. */ - ErtsGcQuickSanityCheck(p); - for (msgp = p->msg.first; msgp; msgp = msgp->next) { - if (msgp->data.attached) { - erts_move_msg_attached_data_to_heap(&p->htop, &p->off_heap, msgp); - ErtsGcQuickSanityCheck(p); - } - } + ErtsGcQuickSanityCheck(p); + for (msgp = p->msg.first; msgp; msgp = msgp->next) { + if (msgp->data.attached) { + ErtsHeapFactory factory; + erts_factory_proc_prealloc_init(&factory, p, + erts_msg_attached_data_size(msgp)); + erts_move_msg_attached_data_to_heap(&factory, msgp); + erts_factory_close(&factory); + ErtsGcQuickSanityCheck(p); + } + } ErtsGcQuickSanityCheck(p); GEN_GCS(p)++; - size_after = HEAP_TOP(p) - HEAP_START(p); - need_after = size_after + need + stack_size; - *recl += (size_before - size_after); + need_after = ((HEAP_TOP(p) - HEAP_START(p)) + + erts_used_frag_sz(MBUF(p)) + + need + + stack_size); /* * Excessively large heaps should be shrunk, but @@ -890,6 +937,7 @@ minor_collection(Process* p, int need, Eterm* objv, int nobj, Uint *recl) } ASSERT(HEAP_SIZE(p) == next_heap_size(p, HEAP_SIZE(p), 0)); + ASSERT(MBUF(p) == NULL); return 1; /* We are done. */ } @@ -898,6 +946,7 @@ minor_collection(Process* p, int need, Eterm* objv, int nobj, Uint *recl) * The heap size turned out to be just right. We are done. */ ASSERT(HEAP_SIZE(p) == next_heap_size(p, HEAP_SIZE(p), 0)); + ASSERT(MBUF(p) == NULL); return 1; } } @@ -1125,7 +1174,7 @@ do_minor(Process *p, Uint new_sz, Eterm* objv, int nobj) old_htop = sweep_one_area(OLD_HTOP(p), old_htop, heap, heap_size); } OLD_HTOP(p) = old_htop; - HIGH_WATER(p) = (HEAP_START(p) != HIGH_WATER(p)) ? n_heap : n_htop; + HIGH_WATER(p) = n_htop; if (MSO(p).first) { sweep_off_heap(p, 0); @@ -1177,7 +1226,9 @@ major_collection(Process* p, int need, Eterm* objv, int nobj, Uint *recl) { Rootset rootset; Roots* roots; - Uint size_before; + const Uint size_before = ((HEAP_TOP(p) - HEAP_START(p)) + + (OLD_HTOP(p) - OLD_HEAP(p)) + + MBUF_SIZE(p)); Eterm* n_heap; Eterm* n_htop; char* src = (char *) HEAP_START(p); @@ -1186,25 +1237,16 @@ major_collection(Process* p, int need, Eterm* objv, int nobj, Uint *recl) Uint oh_size = (char *) OLD_HTOP(p) - oh; Uint n; Uint new_sz; - Uint fragments = MBUF_SIZE(p) + combined_message_size(p); - ErlMessage *msgp; - - size_before = fragments + (HEAP_TOP(p) - HEAP_START(p)); + int done; /* * Do a fullsweep GC. First figure out the size of the heap * to receive all live data. */ - new_sz = HEAP_SIZE(p) + fragments + (OLD_HTOP(p) - OLD_HEAP(p)); - /* - * We used to do - * - * new_sz += STACK_SZ_ON_HEAP(p); - * - * here for no obvious reason. (The stack size is already counted once - * in HEAP_SIZE(p).) - */ + new_sz = (HEAP_SIZE(p) + MBUF_SIZE(p) + + combined_message_size(p) + + (OLD_HTOP(p) - OLD_HEAP(p))); new_sz = next_heap_size(p, new_sz, 0); /* @@ -1397,42 +1439,56 @@ major_collection(Process* p, int need, Eterm* objv, int nobj, Uint *recl) ErtsGcQuickSanityCheck(p); - /* - * Copy newly received message onto the end of the new heap. - */ - for (msgp = p->msg.first; msgp; msgp = msgp->next) { - if (msgp->data.attached) { - erts_move_msg_attached_data_to_heap(&p->htop, &p->off_heap, msgp); - ErtsGcQuickSanityCheck(p); + *recl += size_before - (HEAP_TOP(p) - HEAP_START(p)); + + remove_message_buffers(p); + + { + ErlMessage *msgp; + + /* + * Copy newly received message onto the end of the new heap. + */ + for (msgp = p->msg.first; msgp; msgp = msgp->next) { + if (msgp->data.attached) { + ErtsHeapFactory factory; + erts_factory_proc_prealloc_init(&factory, p, + erts_msg_attached_data_size(msgp)); + erts_move_msg_attached_data_to_heap(&factory, msgp); + erts_factory_close(&factory); + ErtsGcQuickSanityCheck(p); + } } } - *recl += adjust_after_fullsweep(p, size_before, need, objv, nobj); - -#ifdef HARDDEBUG - disallow_heap_frag_ref_in_heap(p); -#endif - remove_message_buffers(p); + if (MBUF(p)) { + /* This is a very rare case when distributed messages copied above + * contained maps so big they did not fit on the heap causing the + * factory to create heap frags. + * Solution: Trigger a minor gc (without tenuring) + */ + HIGH_WATER(p) = HEAP_START(p); + done = 0; + } else { + adjust_after_fullsweep(p, need, objv, nobj); + done = 1; + } ErtsGcQuickSanityCheck(p); - return 1; /* We are done. */ + return done; } -static Uint -adjust_after_fullsweep(Process *p, Uint size_before, int need, Eterm *objv, int nobj) +static void +adjust_after_fullsweep(Process *p, int need, Eterm *objv, int nobj) { - Uint wanted, sz, size_after, need_after; + Uint wanted, sz, need_after; Uint stack_size = STACK_SZ_ON_HEAP(p); - Uint reclaimed_now; - - size_after = (HEAP_TOP(p) - HEAP_START(p)); - reclaimed_now = (size_before - size_after); /* * Resize the heap if needed. */ - need_after = size_after + need + stack_size; + need_after = (HEAP_TOP(p) - HEAP_START(p)) + need + stack_size; if (HEAP_SIZE(p) < need_after) { /* Too small - grow to match requested need */ sz = next_heap_size(p, need_after, 0); @@ -1455,8 +1511,6 @@ adjust_after_fullsweep(Process *p, Uint size_before, int need, Eterm *objv, int shrink_new_heap(p, sz, objv, nobj); } } - - return reclaimed_now; } /* @@ -1466,13 +1520,12 @@ adjust_after_fullsweep(Process *p, Uint size_before, int need, Eterm *objv, int static Uint combined_message_size(Process* p) { - Uint sz = 0; + Uint sz; ErlMessage *msgp; - for (msgp = p->msg.first; msgp; msgp = msgp->next) { - if (msgp->data.attached) { + for (sz = 0, msgp = p->msg.first; msgp; msgp = msgp->next) { + if (msgp->data.attached) sz += erts_msg_attached_data_size(msgp); - } } return sz; } @@ -1834,6 +1887,21 @@ sweep_one_heap(Eterm* heap_ptr, Eterm* heap_end, Eterm* htop, char* src, Uint sr if (!header_is_thing(gval)) { heap_ptr++; } else { + if (header_is_bin_matchstate(gval)) { + ErlBinMatchState *ms = (ErlBinMatchState*) heap_ptr; + ErlBinMatchBuffer *mb = &(ms->mb); + Eterm* origptr; + origptr = &(mb->orig); + ptr = boxed_val(*origptr); + val = *ptr; + if (IS_MOVED_BOXED(val)) { + *origptr = val; + mb->base = binary_bytes(*origptr); + } else if (in_area(ptr, src, src_size)) { + MOVE_BOXED(ptr,val,htop,origptr); + mb->base = binary_bytes(*origptr); + } + } heap_ptr += (thing_arityval(gval)+1); } break; @@ -1896,7 +1964,18 @@ collect_heap_frags(Process* p, Eterm* n_hstart, Eterm* n_htop, if (p->dictionary != NULL) { disallow_heap_frag_ref(p, n_htop, p->dictionary->data, p->dictionary->used); } - disallow_heap_frag_ref_in_heap(p); + /* OTP-18: Actually we do allow references from heap to heap fragments now. + This can happen when doing "binary_to_term" with a "fat" map contained + in another term. A "fat" map is a hashmap with higher heap demand than + first estimated by "binary_to_term" causing the factory to allocate + additional heap (fragments) for the hashmap tree nodes. + Run map_SUITE:t_gc_rare_map_overflow to provoke this. + + Inverted references like this does not matter however. The copy done + below by move_one_area() with move markers in the fragments and the + sweeping done later by the GC should make everything ok in the end. + */ + /***disallow_heap_frag_ref_in_heap(p);***/ #endif /* @@ -1905,7 +1984,8 @@ collect_heap_frags(Process* p, Eterm* n_hstart, Eterm* n_htop, * until next GC. */ qb = MBUF(p); - while (qb != NULL) { + while (qb != NULL) { + ASSERT(!qb->off_heap.first); /* process fragments use the MSO(p) list */ frag_size = qb->used_size * sizeof(Eterm); if (frag_size != 0) { frag_begin = (char *) qb->mem; @@ -1983,6 +2063,20 @@ setup_rootset(Process *p, Eterm *objv, int nobj, Rootset *rootset) roots[n].sz = 1; n++; } + + /* + * If a NIF has saved arguments, they need to be added + */ + if (ERTS_PROC_GET_NIF_TRAP_EXPORT(p)) { + Eterm* argv; + int argc; + if (erts_setup_nif_gc(p, &argv, &argc)) { + roots[n].v = argv; + roots[n].sz = argc; + n++; + } + } + ASSERT(n <= rootset->size); mp = p->msg.first; @@ -2351,7 +2445,6 @@ sweep_off_heap(Process *p, int fullsweep) } pb->val = erts_bin_realloc(pb->val, new_size); - pb->val->orig_size = new_size; pb->bytes = (byte *) pb->val->orig_bytes; } } @@ -2520,6 +2613,7 @@ offset_one_rootset(Process *p, Sint offs, char* area, Uint area_size, p->dictionary->used, offs, area, area_size); } + offset_heap_ptr(&p->fvalue, 1, offs, area, area_size); offset_heap_ptr(&p->ftrace, 1, offs, area, area_size); offset_heap_ptr(&p->seq_trace_token, 1, offs, area, area_size); @@ -2543,6 +2637,106 @@ offset_rootset(Process *p, Sint offs, char* area, Uint area_size, offset_one_rootset(p, offs, area, area_size, objv, nobj); } +static void +init_gc_info(ErtsGCInfo *gcip) +{ + gcip->reclaimed = 0; + gcip->garbage_cols = 0; +} + +static void +reply_gc_info(void *vgcirp) +{ + Uint64 reclaimed = 0, garbage_cols = 0; + ErtsSchedulerData *esdp = erts_get_scheduler_data(); + ErtsGCInfoReq *gcirp = (ErtsGCInfoReq *) vgcirp; + ErtsProcLocks rp_locks = (gcirp->req_sched == esdp->no + ? ERTS_PROC_LOCK_MAIN + : 0); + Process *rp = gcirp->proc; + Eterm ref_copy = NIL, msg; + Eterm *hp = NULL; + Eterm **hpp; + Uint sz, *szp; + ErlOffHeap *ohp = NULL; + ErlHeapFragment *bp = NULL; + + ASSERT(esdp); + + reclaimed = esdp->gc_info.reclaimed; + garbage_cols = esdp->gc_info.garbage_cols; + + sz = 0; + hpp = NULL; + szp = &sz; + + while (1) { + if (hpp) + ref_copy = STORE_NC(hpp, ohp, gcirp->ref); + else + *szp += REF_THING_SIZE; + + msg = erts_bld_tuple(hpp, szp, 3, + make_small(esdp->no), + erts_bld_uint64(hpp, szp, garbage_cols), + erts_bld_uint64(hpp, szp, reclaimed)); + + msg = erts_bld_tuple(hpp, szp, 2, ref_copy, msg); + if (hpp) + break; + + hp = erts_alloc_message_heap(sz, &bp, &ohp, rp, &rp_locks); + szp = NULL; + hpp = &hp; + } + + erts_queue_message(rp, &rp_locks, bp, msg, NIL); + + if (gcirp->req_sched == esdp->no) + rp_locks &= ~ERTS_PROC_LOCK_MAIN; + + if (rp_locks) + erts_smp_proc_unlock(rp, rp_locks); + + erts_proc_dec_refc(rp); + + if (erts_smp_atomic32_dec_read_nob(&gcirp->refc) == 0) + gcireq_free(vgcirp); +} + +Eterm +erts_gc_info_request(Process *c_p) +{ + ErtsSchedulerData *esdp = ERTS_PROC_GET_SCHDATA(c_p); + Eterm ref; + ErtsGCInfoReq *gcirp; + Eterm *hp; + + gcirp = gcireq_alloc(); + ref = erts_make_ref(c_p); + hp = &gcirp->ref_heap[0]; + + gcirp->proc = c_p; + gcirp->ref = STORE_NC(&hp, NULL, ref); + gcirp->req_sched = esdp->no; + erts_smp_atomic32_init_nob(&gcirp->refc, + (erts_aint32_t) erts_no_schedulers); + + erts_proc_add_refc(c_p, (Sint) erts_no_schedulers); + +#ifdef ERTS_SMP + if (erts_no_schedulers > 1) + erts_schedule_multi_misc_aux_work(1, + erts_no_schedulers, + reply_gc_info, + (void *) gcirp); +#endif + + reply_gc_info((void *) gcirp); + + return ref; +} + #if defined(DEBUG) || defined(ERTS_OFFHEAP_DEBUG) static int @@ -2594,7 +2788,7 @@ within(Eterm *ptr, Process *p) #define ERTS_CHK_OFFHEAP_ASSERT(EXP) \ do { \ if (!(EXP)) \ - erl_exit(ERTS_ABORT_EXIT, \ + erts_exit(ERTS_ABORT_EXIT, \ "%s:%d: Assertion failed: %s\n", \ __FILE__, __LINE__, #EXP); \ } while (0) @@ -2628,7 +2822,7 @@ erts_check_off_heap2(Process *p, Eterm *htop) refc = erts_refc_read(&u.ext->node->refc, 1); break; default: - ASSERT(!!"erts_check_off_heap2: Invalid thing_word"); + ASSERT(!"erts_check_off_heap2: Invalid thing_word"); } ERTS_CHK_OFFHEAP_ASSERT(refc >= 1); #ifdef ERTS_OFFHEAP_DEBUG_CHK_CIRCULAR_LIST diff --git a/erts/emulator/beam/erl_gc.h b/erts/emulator/beam/erl_gc.h index 1801df359a..ecd1bf4d22 100644 --- a/erts/emulator/beam/erl_gc.h +++ b/erts/emulator/beam/erl_gc.h @@ -3,16 +3,17 @@ * * Copyright Ericsson AB 2007-2011. All Rights Reserved. * - * The contents of this file are subject to the Erlang Public License, - * Version 1.1, (the "License"); you may not use this file except in - * compliance with the License. You should have received a copy of the - * Erlang Public License along with this software. If not, it can be - * retrieved online at http://www.erlang.org/. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Software distributed under the License is distributed on an "AS IS" - * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See - * the License for the specific language governing rights and limitations - * under the License. + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. * * %CopyrightEnd% */ @@ -20,8 +21,12 @@ #ifndef __ERL_GC_H__ #define __ERL_GC_H__ +#if defined(ERL_WANT_GC_INTERNALS__) || defined(ERTS_DO_INCL_GLB_INLINE_FUNC_DEF) + /* GC declarations shared by beam/erl_gc.c and hipe/hipe_gc.c */ +#include "erl_map.h" + #if defined(DEBUG) && !ERTS_GLB_INLINE_INCL_FUNC_DEF # define HARDDEBUG 1 #endif @@ -42,30 +47,31 @@ do { \ HTOP += 2; /* update tospace htop */ \ } while(0) -#define MOVE_BOXED(PTR,HDR,HTOP,ORIG) \ -do { \ - Eterm gval; \ - Sint nelts; \ - \ - ASSERT(is_header(HDR)); \ - gval = make_boxed(HTOP); \ - *ORIG = gval; \ - *HTOP++ = HDR; \ - *PTR++ = gval; \ - nelts = header_arity(HDR); \ - switch ((HDR) & _HEADER_SUBTAG_MASK) { \ - case SUB_BINARY_SUBTAG: nelts++; break; \ - case FUN_SUBTAG: nelts+=((ErlFunThing*)(PTR-1))->num_free+1; break; \ - } \ - while (nelts--) \ - *HTOP++ = *PTR++; \ +#define MOVE_BOXED(PTR,HDR,HTOP,ORIG) \ +do { \ + Eterm gval; \ + Sint nelts; \ + \ + ASSERT(is_header(HDR)); \ + nelts = header_arity(HDR); \ + switch ((HDR) & _HEADER_SUBTAG_MASK) { \ + case SUB_BINARY_SUBTAG: nelts++; break; \ + case MAP_SUBTAG: \ + if (is_flatmap_header(HDR)) nelts+=flatmap_get_size(PTR) + 1; \ + else nelts += hashmap_bitcount(MAP_HEADER_VAL(HDR)); \ + break; \ + case FUN_SUBTAG: nelts+=((ErlFunThing*)(PTR))->num_free+1; break; \ + } \ + gval = make_boxed(HTOP); \ + *ORIG = gval; \ + *HTOP++ = HDR; \ + *PTR++ = gval; \ + while (nelts--) *HTOP++ = *PTR++; \ } while(0) #define in_area(ptr,start,nbytes) \ ((UWord)((char*)(ptr) - (char*)(start)) < (nbytes)) -extern Uint erts_test_long_gc_sleep; - #if defined(DEBUG) || defined(ERTS_OFFHEAP_DEBUG) int within(Eterm *ptr, Process *p); #endif @@ -94,4 +100,33 @@ ERTS_GLB_INLINE Eterm follow_moved(Eterm term) } #endif +#endif /* ERL_GC_C__ || HIPE_GC_C__ */ + +/* + * Global exported + */ + +extern Uint erts_test_long_gc_sleep; + +typedef struct { + Uint64 reclaimed; + Uint64 garbage_cols; +} ErtsGCInfo; + +void erts_gc_info(ErtsGCInfo *gcip); +void erts_init_gc(void); +int erts_garbage_collect(struct process*, int, Eterm*, int); +void erts_garbage_collect_hibernate(struct process* p); +Eterm erts_gc_after_bif_call(struct process* p, Eterm result, Eterm* regs, Uint arity); +void erts_garbage_collect_literals(struct process* p, Eterm* literals, + Uint lit_size, + struct erl_off_heap_header* oh); +Uint erts_next_heap_size(Uint, Uint); +Eterm erts_heap_sizes(struct process* p); + +void erts_offset_off_heap(struct erl_off_heap*, Sint, Eterm*, Eterm*); +void erts_offset_heap_ptr(Eterm*, Uint, Sint, Eterm*, Eterm*); +void erts_offset_heap(Eterm*, Uint, Sint, Eterm*, Eterm*); +void erts_free_heap_frags(struct process* p); + #endif /* __ERL_GC_H__ */ diff --git a/erts/emulator/beam/erl_goodfit_alloc.c b/erts/emulator/beam/erl_goodfit_alloc.c index c2088929e9..9b4aad9d91 100644 --- a/erts/emulator/beam/erl_goodfit_alloc.c +++ b/erts/emulator/beam/erl_goodfit_alloc.c @@ -3,16 +3,17 @@ * * Copyright Ericsson AB 2003-2013. All Rights Reserved. * - * The contents of this file are subject to the Erlang Public License, - * Version 1.1, (the "License"); you may not use this file except in - * compliance with the License. You should have received a copy of the - * Erlang Public License along with this software. If not, it can be - * retrieved online at http://www.erlang.org/. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Software distributed under the License is distributed on an "AS IS" - * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See - * the License for the specific language governing rights and limitations - * under the License. + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. * * %CopyrightEnd% */ @@ -163,10 +164,10 @@ BKT_MIN_SZ(GFAllctr_t *gfallctr, int ix) /* Prototypes of callback functions */ static Block_t * get_free_block (Allctr_t *, Uint, - Block_t *, Uint, Uint32); -static void link_free_block (Allctr_t *, Block_t *, Uint32); -static void unlink_free_block (Allctr_t *, Block_t *, Uint32); -static void update_last_aux_mbc (Allctr_t *, Carrier_t *, Uint32); + Block_t *, Uint); +static void link_free_block (Allctr_t *, Block_t *); +static void unlink_free_block (Allctr_t *, Block_t *); +static void update_last_aux_mbc (Allctr_t *, Carrier_t *); static Eterm info_options (Allctr_t *, char *, int *, void *, Uint **, Uint *); static void init_atoms (void); @@ -203,8 +204,7 @@ erts_gfalc_start(GFAllctr_t *gfallctr, sys_memcpy((void *) gfallctr, (void *) &zero.allctr, sizeof(GFAllctr_t)); - init->sbmbct = 0; /* Small mbc not yet supported by goodfit */ - + allctr->mbc_header_size = sizeof(Carrier_t); allctr->min_mbc_size = MIN_MBC_SZ; allctr->min_mbc_first_free_size = MIN_MBC_FIRST_FREE_SZ; allctr->min_block_size = sizeof(GFFreeBlock_t); @@ -222,7 +222,9 @@ erts_gfalc_start(GFAllctr_t *gfallctr, allctr->get_next_mbc_size = NULL; allctr->creating_mbc = update_last_aux_mbc; allctr->destroying_mbc = update_last_aux_mbc; - + allctr->add_mbc = NULL; + allctr->remove_mbc = NULL; + allctr->largest_fblk_in_mbc = NULL; allctr->init_atoms = init_atoms; #ifdef ERTS_ALLOC_UTIL_HARD_DEBUG @@ -384,7 +386,7 @@ search_bucket(Allctr_t *allctr, int ix, Uint size) static Block_t * get_free_block(Allctr_t *allctr, Uint size, - Block_t *cand_blk, Uint cand_size, Uint32 flags) + Block_t *cand_blk, Uint cand_size) { GFAllctr_t *gfallctr = (GFAllctr_t *) allctr; int unsafe_bi, min_bi; @@ -403,7 +405,7 @@ get_free_block(Allctr_t *allctr, Uint size, if (blk) { if (cand_blk && cand_size <= MBC_FBLK_SZ(blk)) return NULL; /* cand_blk was better */ - unlink_free_block(allctr, blk, flags); + unlink_free_block(allctr, blk); return blk; } if (min_bi < NO_OF_BKTS - 1) { @@ -423,14 +425,14 @@ get_free_block(Allctr_t *allctr, Uint size, ASSERT(blk); if (cand_blk && cand_size <= MBC_FBLK_SZ(blk)) return NULL; /* cand_blk was better */ - unlink_free_block(allctr, blk, flags); + unlink_free_block(allctr, blk); return blk; } static void -link_free_block(Allctr_t *allctr, Block_t *block, Uint32 flags) +link_free_block(Allctr_t *allctr, Block_t *block) { GFAllctr_t *gfallctr = (GFAllctr_t *) allctr; GFFreeBlock_t *blk = (GFFreeBlock_t *) block; @@ -451,7 +453,7 @@ link_free_block(Allctr_t *allctr, Block_t *block, Uint32 flags) } static void -unlink_free_block(Allctr_t *allctr, Block_t *block, Uint32 flags) +unlink_free_block(Allctr_t *allctr, Block_t *block) { GFAllctr_t *gfallctr = (GFAllctr_t *) allctr; GFFreeBlock_t *blk = (GFFreeBlock_t *) block; @@ -472,7 +474,7 @@ unlink_free_block(Allctr_t *allctr, Block_t *block, Uint32 flags) } static void -update_last_aux_mbc(Allctr_t *allctr, Carrier_t *mbc, Uint32 flags) +update_last_aux_mbc(Allctr_t *allctr, Carrier_t *mbc) { GFAllctr_t *gfallctr = (GFAllctr_t *) allctr; @@ -569,8 +571,8 @@ info_options(Allctr_t *allctr, if (hpp || szp) { if (!atoms_initialized) - erl_exit(1, "%s:%d: Internal error: Atoms not initialized", - __FILE__, __LINE__);; + erts_exit(ERTS_ERROR_EXIT, "%s:%d: Internal error: Atoms not initialized", + __FILE__, __LINE__); res = NIL; add_2tup(hpp, szp, &res, am.as, am.gf); @@ -589,16 +591,16 @@ info_options(Allctr_t *allctr, * to erts_gfalc_test() * \* */ -unsigned long -erts_gfalc_test(unsigned long op, unsigned long a1, unsigned long a2) +UWord +erts_gfalc_test(UWord op, UWord a1, UWord a2) { switch (op) { - case 0x100: return (unsigned long) BKT_IX((GFAllctr_t *) a1, (Uint) a2); - case 0x101: return (unsigned long) BKT_MIN_SZ((GFAllctr_t *) a1, (int) a2); - case 0x102: return (unsigned long) NO_OF_BKTS; - case 0x103: return (unsigned long) + case 0x100: return (UWord) BKT_IX((GFAllctr_t *) a1, (Uint) a2); + case 0x101: return (UWord) BKT_MIN_SZ((GFAllctr_t *) a1, (int) a2); + case 0x102: return (UWord) NO_OF_BKTS; + case 0x103: return (UWord) find_bucket(&((GFAllctr_t *) a1)->bucket_mask, (int) a2); - default: ASSERT(0); return ~((unsigned long) 0); + default: ASSERT(0); return ~((UWord) 0); } } diff --git a/erts/emulator/beam/erl_goodfit_alloc.h b/erts/emulator/beam/erl_goodfit_alloc.h index a554a6f466..ababdbd0a1 100644 --- a/erts/emulator/beam/erl_goodfit_alloc.h +++ b/erts/emulator/beam/erl_goodfit_alloc.h @@ -1,18 +1,19 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2003-2010. All Rights Reserved. + * Copyright Ericsson AB 2003-2013. All Rights Reserved. * - * The contents of this file are subject to the Erlang Public License, - * Version 1.1, (the "License"); you may not use this file except in - * compliance with the License. You should have received a copy of the - * Erlang Public License along with this software. If not, it can be - * retrieved online at http://www.erlang.org/. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Software distributed under the License is distributed on an "AS IS" - * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See - * the License for the specific language governing rights and limitations - * under the License. + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. * * %CopyrightEnd% */ @@ -82,7 +83,7 @@ struct GFAllctr_t_ { }; -unsigned long erts_gfalc_test(unsigned long, unsigned long, unsigned long); +UWord erts_gfalc_test(UWord, UWord, UWord); #endif /* #if defined(GET_ERL_GF_ALLOC_IMPL) && !defined(ERL_GF_ALLOC_IMPL__) */ diff --git a/erts/emulator/beam/erl_hl_timer.c b/erts/emulator/beam/erl_hl_timer.c new file mode 100644 index 0000000000..fb6d249145 --- /dev/null +++ b/erts/emulator/beam/erl_hl_timer.c @@ -0,0 +1,3179 @@ +/* + * %CopyrightBegin% + * + * Copyright Ericsson AB 2015. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * %CopyrightEnd% + */ + +/* + * Description: High level timers implementing BIF timers + * as well as process and port timers. + * + * Author: Rickard Green + */ + +#ifdef HAVE_CONFIG_H +# include "config.h" +#endif + +#include "sys.h" +#include "global.h" +#include "bif.h" +#include "erl_bif_unique.h" +#define ERTS_WANT_TIMER_WHEEL_API +#include "erl_time.h" +#include "erl_hl_timer.h" + +#define ERTS_TMR_CHECK_CANCEL_ON_CREATE 0 + +#if 0 +# define ERTS_HLT_HARD_DEBUG +#endif +#if 0 +# define ERTS_HLT_DEBUG +#endif + +#if defined(ERTS_HLT_HARD_DEBUG) || defined(DEBUG) +# if defined(ERTS_HLT_HARD_DEBUG) +# undef ERTS_RBT_HARD_DEBUG +# define ERTS_RBT_HARD_DEBUG 1 +# endif +# ifndef ERTS_HLT_DEBUG +# define ERTS_HLT_DEBUG 1 +# endif +#endif + +#undef ERTS_HLT_ASSERT +#if defined(ERTS_HLT_DEBUG) +# define ERTS_HLT_ASSERT(E) ERTS_ASSERT(E) +# undef ERTS_RBT_DEBUG +# define ERTS_RBT_DEBUG +#else +# define ERTS_HLT_ASSERT(E) ((void) 1) +#endif + +#if defined(ERTS_HLT_HARD_DEBUG) && defined(__GNUC__) +#warning "* * * * * * * * * * * * * * * * * *" +#warning "* ERTS_HLT_HARD_DEBUG IS ENABLED! *" +#warning "* * * * * * * * * * * * * * * * * *" +#endif + +#ifdef ERTS_HLT_HARD_DEBUG +# define ERTS_HLT_HDBG_CHK_SRV(SRV) hdbg_chk_srv((SRV)) +static void hdbg_chk_srv(ErtsHLTimerService *srv); +#else +# define ERTS_HLT_HDBG_CHK_SRV(SRV) ((void) 1) +#endif + +#if ERTS_REF_NUMBERS != 3 +#error "ERTS_REF_NUMBERS changed. Update me..." +#endif + +typedef enum { + ERTS_TMR_BIF, + ERTS_TMR_PROC, + ERTS_TMR_PORT, + ERTS_TMR_CALLBACK +} ErtsTmrType; + +#define ERTS_BIF_TIMER_SHORT_TIME 5000 + +#ifdef ERTS_SMP +# define ERTS_HLT_SMP_MEMBAR_LoadLoad_LoadStore \ + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore) +#else +# define ERTS_HLT_SMP_MEMBAR_LoadLoad_LoadStore +#endif + +/* Bit 0 to 9 contains scheduler id (see mask below) */ +#define ERTS_TMR_ROFLG_HLT (((Uint32) 1) << 10) +#define ERTS_TMR_ROFLG_BIF_TMR (((Uint32) 1) << 11) +#define ERTS_TMR_ROFLG_PRE_ALC (((Uint32) 1) << 12) +#define ERTS_TMR_ROFLG_REG_NAME (((Uint32) 1) << 13) +#define ERTS_TMR_ROFLG_PROC (((Uint32) 1) << 14) +#define ERTS_TMR_ROFLG_PORT (((Uint32) 1) << 15) +#define ERTS_TMR_ROFLG_CALLBACK (((Uint32) 1) << 16) +#ifdef ERTS_BTM_ACCESSOR_SUPPORT +#define ERTS_TMR_ROFLG_ABIF_TMR (((Uint32) 1) << 17) +#endif + +#define ERTS_TMR_ROFLG_SID_MASK \ + (ERTS_TMR_ROFLG_HLT - (Uint32) 1) + +#define ERTS_TMR_STATE_ACTIVE ((erts_aint32_t) 0) +#define ERTS_TMR_STATE_CANCELED ((erts_aint32_t) 1) +#define ERTS_TMR_STATE_TIMED_OUT ((erts_aint32_t) 2) + +typedef struct ErtsHLTimer_ ErtsHLTimer; + +#define ERTS_HLT_PFLG_RED (((UWord) 1) << 0) +#define ERTS_HLT_PFLG_SAME_TIME (((UWord) 1) << 1) + +#define ERTS_HLT_PFLGS_MASK \ + (ERTS_HLT_PFLG_RED|ERTS_HLT_PFLG_SAME_TIME) + +#define ERTS_HLT_PFIELD_NOT_IN_TABLE (~((UWord) 0)) + +typedef struct { + UWord parent; /* parent pointer and flags... */ + union { + struct { + ErtsHLTimer *right; + ErtsHLTimer *left; + } t; + struct { + ErtsHLTimer *prev; + ErtsHLTimer *next; + } l; + } u; + ErtsHLTimer *same_time; +} ErtsHLTimerTimeTree; + +typedef struct { + UWord parent; /* parent pointer and flags... */ + ErtsHLTimer *right; + ErtsHLTimer *left; +} ErtsHLTimerTree; + +typedef struct { + Uint32 roflgs; + erts_smp_atomic32_t refc; + union { + void *arg; + erts_atomic_t next; + } u; +} ErtsTmrHead; + +struct ErtsHLTimer_ { + ErtsTmrHead head; /* NEED to be first! */ + union { + ErtsThrPrgrLaterOp cleanup; + ErtsHLTimerTimeTree tree; + } time; + ErtsMonotonicTime timeout; + union { + Process *proc; + Port *port; + Eterm name; + void (*callback)(void *); + } receiver; + +#ifdef ERTS_HLT_HARD_DEBUG + int pending_timeout; +#endif + + erts_smp_atomic32_t state; + + /* BIF timer only fields follow... */ + struct { + Uint32 refn[ERTS_REF_NUMBERS]; + ErtsHLTimerTree proc_tree; + ErtsHLTimerTree tree; + Eterm message; + ErlHeapFragment *bp; + } btm; +#ifdef ERTS_BTM_ACCESSOR_SUPPORT + struct { + Eterm accessor; + ErtsHLTimerTree tree; + } abtm; +#endif +}; + +#define ERTS_HL_PTIMER_SIZE offsetof(ErtsHLTimer, btm) +#ifdef ERTS_BTM_ACCESSOR_SUPPORT +#define ERTS_BIF_TIMER_SIZE offsetof(ErtsHLTimer, abtm) +#define ERTS_ABIF_TIMER_SIZE sizeof(ErtsHLTimer) +#else +#define ERTS_BIF_TIMER_SIZE sizeof(ErtsHLTimer) +#endif + +typedef struct { + ErtsTmrHead head; /* NEED to be first! */ + union { + void *p; + void (*callback)(void *); + } u; + ErtsTWheelTimer tw_tmr; +} ErtsTWTimer; + +typedef union { + ErtsTmrHead head; + ErtsHLTimer hlt; + ErtsTWTimer twt; +} ErtsTimer; + +#ifdef SMALL_MEMORY +#define BIF_TIMER_PREALC_SZ 10 +#define PTIMER_PREALC_SZ 10 +#else +#define BIF_TIMER_PREALC_SZ 100 +#define PTIMER_PREALC_SZ 100 +#endif + +ERTS_SCHED_PREF_PALLOC_IMPL(bif_timer_pre, + ErtsHLTimer, + BIF_TIMER_PREALC_SZ) + +ERTS_SCHED_PREF_QUICK_ALLOC_IMPL(tw_timer, + ErtsTWTimer, + PTIMER_PREALC_SZ, + ERTS_ALC_T_LL_PTIMER) + +#ifdef ERTS_HLT_DEBUG +#define ERTS_TMR_TIMEOUT_YIELD_LIMIT 5 +#else +#define ERTS_TMR_TIMEOUT_YIELD_LIMIT 100 +#endif +#define ERTS_TMR_CANCELED_TIMER_LIMIT 100 +#define ERTS_TMR_CANCELED_TIMER_SMALL_LIMIT 5 + +#define ERTS_TMR_TIMEOUT_YIELD_STATE_T same_time_list_yield_state_t +#define ERTS_TMR_YIELDING_TIMEOUT_STATE_INITER {NULL, {0}} +typedef struct { + int dummy; +} ERTS_TMR_TIMEOUT_YIELD_STATE_T; + +typedef struct { + ErtsTmrHead marker; + erts_atomic_t last; +} ErtsHLTCncldTmrQTail; + +#ifdef ERTS_SMP + +typedef struct { + /* + * This structure needs to be cache line aligned for best + * performance. + */ + union { + /* + * Modified by threads returning canceled + * timers to this timer service. + */ + ErtsHLTCncldTmrQTail data; + char align__[ERTS_ALC_CACHE_LINE_ALIGN_SIZE( + sizeof(ErtsHLTCncldTmrQTail))]; + } tail; + /* + * Everything below this point is *only* accessed by the + * thread managing this timer service. + */ + struct { + ErtsTimer *first; + ErtsTimer *unref_end; + struct { + ErtsThrPrgrVal thr_progress; + int thr_progress_reached; + ErtsTimer *unref_end; + } next; + int used_marker; + } head; +} ErtsHLTCncldTmrQ; + +#endif /* ERTS_SMP */ + +typedef struct { + ErtsHLTimer *root; + ERTS_TMR_TIMEOUT_YIELD_STATE_T state; +} ErtsYieldingTimeoutState; + +struct ErtsHLTimerService_ { +#ifdef ERTS_SMP + ErtsHLTCncldTmrQ canceled_queue; +#endif + ErtsHLTimer *time_tree; + ErtsHLTimer *btm_tree; + ErtsHLTimer *next_timeout; + ErtsYieldingTimeoutState yield; + ErtsTWheelTimer service_timer; +}; + +static ERTS_INLINE int +refn_is_lt(Uint32 *x, Uint32 *y) +{ + /* !0 if x < y */ + if (x[2] < y[2]) + return 1; + if (x[2] != y[2]) + return 0; + if (x[1] < y[1]) + return 1; + if (x[1] != y[1]) + return 0; + return x[0] < y[0]; +} + +#define ERTS_RBT_PREFIX time +#define ERTS_RBT_T ErtsHLTimer +#define ERTS_RBT_KEY_T ErtsMonotonicTime +#define ERTS_RBT_FLAGS_T UWord +#define ERTS_RBT_INIT_EMPTY_TNODE(T) \ + do { \ + (T)->time.tree.parent = (UWord) NULL; \ + (T)->time.tree.u.t.right = NULL; \ + (T)->time.tree.u.t.left = NULL; \ + } while (0) +#define ERTS_RBT_IS_RED(T) \ + ((int) ((T)->time.tree.parent & ERTS_HLT_PFLG_RED)) +#define ERTS_RBT_SET_RED(T) \ + ((T)->time.tree.parent |= ERTS_HLT_PFLG_RED) +#define ERTS_RBT_IS_BLACK(T) \ + (!ERTS_RBT_IS_RED((T))) +#define ERTS_RBT_SET_BLACK(T) \ + ((T)->time.tree.parent &= ~ERTS_HLT_PFLG_RED) +#define ERTS_RBT_GET_FLAGS(T) \ + ((T)->time.tree.parent & ERTS_HLT_PFLGS_MASK) +#define ERTS_RBT_SET_FLAGS(T, F) \ + do { \ + ERTS_HLT_ASSERT((((UWord) (F)) & ~ERTS_HLT_PFLGS_MASK) == 0); \ + (T)->time.tree.parent &= ~ERTS_HLT_PFLGS_MASK; \ + (T)->time.tree.parent |= (F); \ + } while (0) +#define ERTS_RBT_GET_PARENT(T) \ + ((ErtsHLTimer *) ((T)->time.tree.parent & ~ERTS_HLT_PFLGS_MASK)) +#define ERTS_RBT_SET_PARENT(T, P) \ + do { \ + ERTS_HLT_ASSERT((((UWord) (P)) & ERTS_HLT_PFLGS_MASK) == 0); \ + (T)->time.tree.parent &= ERTS_HLT_PFLGS_MASK; \ + (T)->time.tree.parent |= (UWord) (P); \ + } while (0) +#define ERTS_RBT_GET_RIGHT(T) ((T)->time.tree.u.t.right) +#define ERTS_RBT_SET_RIGHT(T, R) ((T)->time.tree.u.t.right = (R)) +#define ERTS_RBT_GET_LEFT(T) ((T)->time.tree.u.t.left) +#define ERTS_RBT_SET_LEFT(T, L) ((T)->time.tree.u.t.left = (L)) +#define ERTS_RBT_GET_KEY(T) ((T)->timeout) +#define ERTS_RBT_IS_LT(KX, KY) ((KX) < (KY)) +#define ERTS_RBT_IS_EQ(KX, KY) ((KX) == (KY)) +#define ERTS_RBT_WANT_DELETE +#define ERTS_RBT_WANT_SMALLEST +#define ERTS_RBT_WANT_LOOKUP_INSERT +#define ERTS_RBT_WANT_REPLACE +#define ERTS_RBT_WANT_FOREACH +#ifdef ERTS_HLT_HARD_DEBUG +# define ERTS_RBT_WANT_LOOKUP +#endif +#define ERTS_RBT_UNDEF + +#include "erl_rbtree.h" + +/* Use circular list for timers at same time */ + +static ERTS_INLINE void +same_time_list_insert(ErtsHLTimer **root, ErtsHLTimer *tmr) +{ + ErtsHLTimer *first = *root; + if (!first) { + ERTS_HLT_ASSERT((((UWord) root) & ERTS_HLT_PFLG_SAME_TIME) == 0); + tmr->time.tree.parent = ((UWord) root) | ERTS_HLT_PFLG_SAME_TIME; + tmr->time.tree.u.l.next = tmr; + tmr->time.tree.u.l.prev = tmr; + *root = tmr; + } + else { + tmr->time.tree.parent = ERTS_HLT_PFLG_SAME_TIME; + tmr->time.tree.u.l.next = first; + tmr->time.tree.u.l.prev = first->time.tree.u.l.prev; + first->time.tree.u.l.prev = tmr; + tmr->time.tree.u.l.prev->time.tree.u.l.next = tmr; + } +} + +static ERTS_INLINE void +same_time_list_delete(ErtsHLTimer *tmr) +{ + ErtsHLTimer **root, *next; + + root = (ErtsHLTimer **) (tmr->time.tree.parent & ~ERTS_HLT_PFLG_SAME_TIME); + next = tmr->time.tree.u.l.next; + + ERTS_HLT_ASSERT((tmr->time.tree.parent + == (((UWord) root) | ERTS_HLT_PFLG_SAME_TIME)) + || (tmr->time.tree.parent + == ERTS_HLT_PFLG_SAME_TIME)); + + if (next == tmr) { + ERTS_HLT_ASSERT(root && *root == tmr); + ERTS_HLT_ASSERT(tmr->time.tree.u.l.prev == tmr); + *root = NULL; + } + else { + if (root) { + ERTS_HLT_ASSERT(*root == tmr); + *root = next; + next->time.tree.parent = ((UWord) root) | ERTS_HLT_PFLG_SAME_TIME; + } + tmr->time.tree.u.l.next->time.tree.u.l.prev = tmr->time.tree.u.l.prev; + tmr->time.tree.u.l.prev->time.tree.u.l.next = next; + } +} + +static ERTS_INLINE void +same_time_list_new_root(ErtsHLTimer **root) +{ + ErtsHLTimer *tmr = *root; + if (tmr) { + ERTS_HLT_ASSERT(root); + tmr->time.tree.parent = ((UWord) root) | ERTS_HLT_PFLG_SAME_TIME; + } +} + +static ERTS_INLINE int +same_time_list_foreach_destroy_yielding(ErtsHLTimer **root, + void (*op)(ErtsHLTimer *, void *), + void *arg, + ERTS_TMR_TIMEOUT_YIELD_STATE_T *ys, + Sint ylimit) +{ + Sint ycnt = ylimit; + ErtsHLTimer *end, *tmr = *root; + if (!tmr) + return 0; + + ERTS_HLT_ASSERT(tmr->time.tree.parent + == (((UWord) root) | ERTS_HLT_PFLG_SAME_TIME)); + + end = tmr->time.tree.u.l.prev; + end->time.tree.u.l.next = NULL; + + while (1) { + ErtsHLTimer *op_tmr = tmr; + + ERTS_HLT_ASSERT((tmr->time.tree.parent + == (((UWord) root) | ERTS_HLT_PFLG_SAME_TIME)) + || (tmr->time.tree.parent + == ERTS_HLT_PFLG_SAME_TIME)); + + tmr = tmr->time.tree.u.l.next; + (*op)(op_tmr, arg); + if (!tmr) { + *root = NULL; + return 0; + } + if (--ycnt <= 0) { + /* Make new circle of timers left to process... */ + *root = tmr; + end->time.tree.u.l.next = tmr; + tmr->time.tree.u.l.prev = end; + tmr->time.tree.parent = ((UWord) root) | ERTS_HLT_PFLG_SAME_TIME; + return 1; + } + } +} + +static ERTS_INLINE void +same_time_list_foreach(ErtsHLTimer *root, + void (*op)(ErtsHLTimer *, void *), + void *arg) +{ + if (root) { + ErtsHLTimer *tmr = root; + do { + (*op)(tmr, arg); + tmr = tmr->time.tree.u.l.next; + } while (root != tmr); + } +} + +#ifdef ERTS_HLT_HARD_DEBUG + +static ERTS_INLINE ErtsHLTimer * +same_time_list_lookup(ErtsHLTimer *root, ErtsHLTimer *x) +{ + if (root) { + ErtsHLTimer *tmr = root; + do { + if (tmr == x) + return tmr; + tmr = tmr->time.tree.u.l.next; + } while (root != tmr); + } + return NULL; +} + +#endif /* ERTS_HLT_HARD_DEBUG */ + +#define ERTS_RBT_PREFIX btm +#define ERTS_RBT_T ErtsHLTimer +#define ERTS_RBT_KEY_T Uint32 * +#define ERTS_RBT_FLAGS_T UWord +#define ERTS_RBT_INIT_EMPTY_TNODE(T) \ + do { \ + (T)->btm.tree.parent = (UWord) NULL; \ + (T)->btm.tree.right = NULL; \ + (T)->btm.tree.left = NULL; \ + } while (0) +#define ERTS_RBT_IS_RED(T) \ + ((int) ((T)->btm.tree.parent & ERTS_HLT_PFLG_RED)) +#define ERTS_RBT_SET_RED(T) \ + ((T)->btm.tree.parent |= ERTS_HLT_PFLG_RED) +#define ERTS_RBT_IS_BLACK(T) \ + (!ERTS_RBT_IS_RED((T))) +#define ERTS_RBT_SET_BLACK(T) \ + ((T)->btm.tree.parent &= ~ERTS_HLT_PFLG_RED) +#define ERTS_RBT_GET_FLAGS(T) \ + ((T)->btm.tree.parent & ERTS_HLT_PFLGS_MASK) +#define ERTS_RBT_SET_FLAGS(T, F) \ + do { \ + ERTS_HLT_ASSERT((((UWord) (F)) & ~ERTS_HLT_PFLGS_MASK) == 0); \ + (T)->btm.tree.parent &= ~ERTS_HLT_PFLGS_MASK; \ + (T)->btm.tree.parent |= (F); \ + } while (0) +#define ERTS_RBT_GET_PARENT(T) \ + ((ErtsHLTimer *) ((T)->btm.tree.parent & ~ERTS_HLT_PFLGS_MASK)) +#define ERTS_RBT_SET_PARENT(T, P) \ + do { \ + ERTS_HLT_ASSERT((((UWord) (P)) & ERTS_HLT_PFLGS_MASK) == 0); \ + (T)->btm.tree.parent &= ERTS_HLT_PFLGS_MASK; \ + (T)->btm.tree.parent |= (UWord) (P); \ + } while (0) +#define ERTS_RBT_GET_RIGHT(T) ((T)->btm.tree.right) +#define ERTS_RBT_SET_RIGHT(T, R) ((T)->btm.tree.right = (R)) +#define ERTS_RBT_GET_LEFT(T) ((T)->btm.tree.left) +#define ERTS_RBT_SET_LEFT(T, L) ((T)->btm.tree.left = (L)) +#define ERTS_RBT_GET_KEY(T) ((T)->btm.refn) +#define ERTS_RBT_IS_LT(KX, KY) refn_is_lt((KX), (KY)) +#define ERTS_RBT_IS_EQ(KX, KY) \ + (((KX)[0] == (KY)[0]) & ((KX)[1] == (KY)[1]) & ((KX)[2] == (KY)[2])) +#define ERTS_RBT_WANT_DELETE +#define ERTS_RBT_WANT_INSERT +#define ERTS_RBT_WANT_LOOKUP +#define ERTS_RBT_WANT_FOREACH +#define ERTS_RBT_UNDEF + +#include "erl_rbtree.h" + +#define ERTS_RBT_PREFIX proc_btm +#define ERTS_RBT_T ErtsHLTimer +#define ERTS_RBT_KEY_T Uint32 * +#define ERTS_RBT_FLAGS_T UWord +#define ERTS_RBT_INIT_EMPTY_TNODE(T) \ + do { \ + (T)->btm.proc_tree.parent = (UWord) NULL; \ + (T)->btm.proc_tree.right = NULL; \ + (T)->btm.proc_tree.left = NULL; \ + } while (0) +#define ERTS_RBT_IS_RED(T) \ + ((int) ((T)->btm.proc_tree.parent & ERTS_HLT_PFLG_RED)) +#define ERTS_RBT_SET_RED(T) \ + ((T)->btm.proc_tree.parent |= ERTS_HLT_PFLG_RED) +#define ERTS_RBT_IS_BLACK(T) \ + (!ERTS_RBT_IS_RED((T))) +#define ERTS_RBT_SET_BLACK(T) \ + ((T)->btm.proc_tree.parent &= ~ERTS_HLT_PFLG_RED) +#define ERTS_RBT_GET_FLAGS(T) \ + ((T)->btm.proc_tree.parent & ERTS_HLT_PFLGS_MASK) +#define ERTS_RBT_SET_FLAGS(T, F) \ + do { \ + ERTS_HLT_ASSERT((((UWord) (F)) & ~ERTS_HLT_PFLGS_MASK) == 0); \ + (T)->btm.proc_tree.parent &= ~ERTS_HLT_PFLGS_MASK; \ + (T)->btm.proc_tree.parent |= (F); \ + } while (0) +#define ERTS_RBT_GET_PARENT(T) \ + ((ErtsHLTimer *) ((T)->btm.proc_tree.parent & ~ERTS_HLT_PFLGS_MASK)) +#define ERTS_RBT_SET_PARENT(T, P) \ + do { \ + ERTS_HLT_ASSERT((((UWord) (P)) & ERTS_HLT_PFLGS_MASK) == 0); \ + (T)->btm.proc_tree.parent &= ERTS_HLT_PFLGS_MASK; \ + (T)->btm.proc_tree.parent |= (UWord) (P); \ + } while (0) +#define ERTS_RBT_GET_RIGHT(T) ((T)->btm.proc_tree.right) +#define ERTS_RBT_SET_RIGHT(T, R) ((T)->btm.proc_tree.right = (R)) +#define ERTS_RBT_GET_LEFT(T) ((T)->btm.proc_tree.left) +#define ERTS_RBT_SET_LEFT(T, L) ((T)->btm.proc_tree.left = (L)) +#define ERTS_RBT_GET_KEY(T) ((T)->btm.refn) +#define ERTS_RBT_IS_LT(KX, KY) refn_is_lt((KX), (KY)) +#define ERTS_RBT_IS_EQ(KX, KY) \ + (((KX)[0] == (KY)[0]) & ((KX)[1] == (KY)[1]) & ((KX)[2] == (KY)[2])) +#define ERTS_RBT_WANT_DELETE +#define ERTS_RBT_WANT_INSERT +#define ERTS_RBT_WANT_LOOKUP +#define ERTS_RBT_WANT_FOREACH_DESTROY_YIELDING +#define ERTS_RBT_UNDEF + +#include "erl_rbtree.h" + +#ifdef ERTS_BTM_ACCESSOR_SUPPORT + +#define ERTS_RBT_PREFIX abtm +#define ERTS_RBT_T ErtsHLTimer +#define ERTS_RBT_KEY_T Uint32 * +#define ERTS_RBT_FLAGS_T UWord +#define ERTS_RBT_INIT_EMPTY_TNODE(T) \ + do { \ + (T)->abtm.tree.parent = (UWord) NULL; \ + (T)->abtm.tree.right = NULL; \ + (T)->abtm.tree.left = NULL; \ + } while (0) +#define ERTS_RBT_IS_RED(T) \ + ((int) ((T)->abtm.tree.parent & ERTS_HLT_PFLG_RED)) +#define ERTS_RBT_SET_RED(T) \ + ((T)->abtm.tree.parent |= ERTS_HLT_PFLG_RED) +#define ERTS_RBT_IS_BLACK(T) \ + (!ERTS_RBT_IS_RED((T))) +#define ERTS_RBT_SET_BLACK(T) \ + ((T)->abtm.tree.parent &= ~ERTS_HLT_PFLG_RED) +#define ERTS_RBT_GET_FLAGS(T) \ + ((T)->abtm.tree.parent & ERTS_HLT_PFLGS_MASK) +#define ERTS_RBT_SET_FLAGS(T, F) \ + do { \ + ERTS_HLT_ASSERT((((UWord) (F)) & ~ERTS_HLT_PFLGS_MASK) == 0); \ + (T)->abtm.tree.parent &= ~ERTS_HLT_PFLGS_MASK; \ + (T)->abtm.tree.parent |= (F); \ + } while (0) +#define ERTS_RBT_GET_PARENT(T) \ + ((ErtsHLTimer *) ((T)->abtm.tree.parent & ~ERTS_HLT_PFLGS_MASK)) +#define ERTS_RBT_SET_PARENT(T, P) \ + do { \ + ERTS_HLT_ASSERT((((UWord) (P)) & ERTS_HLT_PFLGS_MASK) == 0); \ + (T)->abtm.tree.parent &= ERTS_HLT_PFLGS_MASK; \ + (T)->abtm.tree.parent |= (UWord) (P); \ + } while (0) +#define ERTS_RBT_GET_RIGHT(T) ((T)->abtm.tree.right) +#define ERTS_RBT_SET_RIGHT(T, R) ((T)->abtm.tree.right = (R)) +#define ERTS_RBT_GET_LEFT(T) ((T)->abtm.tree.left) +#define ERTS_RBT_SET_LEFT(T, L) ((T)->abtm.tree.left = (L)) +#define ERTS_RBT_GET_KEY(T) ((T)->btm.refn) +#define ERTS_RBT_IS_LT(KX, KY) refn_is_lt((KX), (KY)) +#define ERTS_RBT_IS_EQ(KX, KY) \ + (((KX)[0] == (KY)[0]) & ((KX)[1] == (KY)[1]) & ((KX)[2] == (KY)[2])) +#define ERTS_RBT_WANT_DELETE +#define ERTS_RBT_WANT_INSERT +#define ERTS_RBT_WANT_LOOKUP +#define ERTS_RBT_WANT_FOREACH_DESTROY_YIELDING +#define ERTS_RBT_UNDEF + +#include "erl_rbtree.h" + +#endif /* ERTS_BTM_ACCESSOR_SUPPORT */ + +#ifdef ERTS_SMP +static void init_canceled_queue(ErtsHLTCncldTmrQ *cq); +#endif + +void +erts_hl_timer_init(void) +{ + init_tw_timer_alloc(); + init_bif_timer_pre_alloc(); +} + +ErtsHLTimerService * +erts_create_timer_service(void) +{ + ErtsYieldingTimeoutState init_yield = ERTS_TMR_YIELDING_TIMEOUT_STATE_INITER; + ErtsHLTimerService *srv; + + srv = erts_alloc_permanent_cache_aligned(ERTS_ALC_T_TIMER_SERVICE, + sizeof(ErtsHLTimerService)); + srv->time_tree = NULL; + srv->btm_tree = NULL; + srv->next_timeout = NULL; + srv->yield = init_yield; + erts_twheel_init_timer(&srv->service_timer); + +#ifdef ERTS_SMP + init_canceled_queue(&srv->canceled_queue); +#endif + + return srv; +} + +size_t +erts_timer_type_size(ErtsAlcType_t type) +{ + switch (type) { + case ERTS_ALC_T_LL_PTIMER: return sizeof(ErtsTWTimer); + case ERTS_ALC_T_HL_PTIMER: return ERTS_HL_PTIMER_SIZE; + case ERTS_ALC_T_BIF_TIMER: return ERTS_BIF_TIMER_SIZE; +#ifdef ERTS_BTM_ACCESSOR_SUPPORT + case ERTS_ALC_T_ABIF_TIMER: return ERTS_ABIF_TIMER_SIZE; +#endif + default: ERTS_INTERNAL_ERROR("Unknown type"); + } + return 0; +} + +static ERTS_INLINE ErtsMonotonicTime +get_timeout_pos(ErtsMonotonicTime now, ErtsMonotonicTime msec) +{ + ErtsMonotonicTime timeout_pos; + if (msec <= 0) + return ERTS_MONOTONIC_TO_CLKTCKS(now); + timeout_pos = ERTS_MONOTONIC_TO_CLKTCKS(now-1); + timeout_pos += ERTS_MSEC_TO_CLKTCKS(msec) + 1; + return timeout_pos; +} + +static ERTS_INLINE Sint64 +get_time_left(ErtsSchedulerData *esdp, ErtsMonotonicTime timeout_pos) +{ + ErtsMonotonicTime now = erts_get_monotonic_time(esdp); + + now = ERTS_MONOTONIC_TO_CLKTCKS(now-1)+1; + if (timeout_pos <= now) + return (Sint64) 0; + return (Sint64) ERTS_CLKTCKS_TO_MSEC(timeout_pos - now); +} + +static ERTS_INLINE int +proc_timeout_common(Process *proc, void *tmr) +{ + if (tmr == (void *) erts_smp_atomic_cmpxchg_mb(&proc->common.timer, + ERTS_PTMR_TIMEDOUT, + (erts_aint_t) tmr)) { + erts_aint32_t state = erts_smp_atomic32_read_acqb(&proc->state); + if (!(state & (ERTS_PSFLG_ACTIVE|ERTS_PSFLG_EXITING))) + erts_schedule_process(proc, state, 0); + return 1; + } + return 0; +} + +static ERTS_INLINE int +port_timeout_common(Port *port, void *tmr) +{ + if (tmr == (void *) erts_smp_atomic_cmpxchg_mb(&port->common.timer, + ERTS_PTMR_TIMEDOUT, + (erts_aint_t) tmr)) { + erts_port_task_schedule(port->common.id, + &port->timeout_task, + ERTS_PORT_TASK_TIMEOUT); + return 1; + } + return 0; +} + +/* + * Basic timer wheel timer stuff + */ + +static void +scheduled_tw_timer_destroy(void *vtmr) +{ + tw_timer_free((ErtsTWTimer *) vtmr); +} + +static void +schedule_tw_timer_destroy(ErtsTWTimer *tmr) +{ + /* + * Reference to process/port can be + * dropped at once... + */ + if (tmr->head.roflgs & ERTS_TMR_ROFLG_PROC) + erts_proc_dec_refc((Process *) tmr->u.p); + else if (tmr->head.roflgs & ERTS_TMR_ROFLG_PORT) + erts_port_dec_refc((Port *) tmr->u.p); + + erts_schedule_thr_prgr_later_cleanup_op( + scheduled_tw_timer_destroy, + (void *) tmr, + &tmr->tw_tmr.u.cleanup, + sizeof(ErtsTWTimer)); +} + +static ERTS_INLINE void +tw_timer_dec_refc(ErtsTWTimer *tmr) +{ + if (erts_smp_atomic32_dec_read_relb(&tmr->head.refc) == 0) { + ERTS_HLT_SMP_MEMBAR_LoadLoad_LoadStore; + schedule_tw_timer_destroy(tmr); + } +} + +static void +tw_proc_timeout(void *vtwtp) +{ + ErtsTWTimer *twtp = (ErtsTWTimer *) vtwtp; + Process *proc = (Process *) twtp->u.p; + if (proc_timeout_common(proc, vtwtp)) + tw_timer_dec_refc(twtp); + tw_timer_dec_refc(twtp); +} + +static void +tw_port_timeout(void *vtwtp) +{ + ErtsTWTimer *twtp = (ErtsTWTimer *) vtwtp; + Port *port = (Port *) twtp->u.p; + if (port_timeout_common(port, vtwtp)) + tw_timer_dec_refc(twtp); + tw_timer_dec_refc(twtp); +} + +static void +tw_ptimer_cancel(void *vtwtp) +{ + tw_timer_dec_refc((ErtsTWTimer *) vtwtp); +} + +static void +cancel_tw_timer(ErtsSchedulerData *esdp, ErtsTWTimer *tmr) +{ + ERTS_HLT_ASSERT((tmr->head.roflgs & ERTS_TMR_ROFLG_SID_MASK) + == (Uint32) esdp->no); + erts_twheel_cancel_timer(esdp->timer_wheel, &tmr->tw_tmr); +} + +static void +tw_callback_timeout(void *vtwtp) +{ + ErtsTWTimer *twtp = (ErtsTWTimer *) vtwtp; + void (*callback)(void *) = twtp->u.callback; + void *arg = twtp->head.u.arg; + tw_timer_dec_refc(twtp); + (*callback)(arg); +} + +static ErtsTWTimer * +create_tw_timer(ErtsSchedulerData *esdp, + ErtsTmrType type, void *p, + void (*callback)(void *), void *arg, + ErtsMonotonicTime timeout_pos) +{ + ErtsTWTimer *tmr; + void (*timeout_func)(void *); + void (*cancel_func)(void *); + erts_aint32_t refc; + + tmr = tw_timer_alloc(); + erts_twheel_init_timer(&tmr->tw_tmr); + + tmr->head.roflgs = (Uint32) esdp->no; + ERTS_HLT_ASSERT((tmr->head.roflgs + & ~ERTS_TMR_ROFLG_SID_MASK) == 0); + + switch (type) { + + case ERTS_TMR_PROC: + tmr->u.p = p; + tmr->head.roflgs |= ERTS_TMR_ROFLG_PROC; + timeout_func = tw_proc_timeout; + cancel_func = tw_ptimer_cancel; + erts_proc_inc_refc((Process *) p); + refc = 2; + break; + + case ERTS_TMR_PORT: + tmr->u.p = p; + tmr->head.roflgs |= ERTS_TMR_ROFLG_PORT; + timeout_func = tw_port_timeout; + cancel_func = tw_ptimer_cancel; + erts_port_inc_refc((Port *) p); + refc = 2; + break; + + case ERTS_TMR_CALLBACK: + tmr->head.u.arg = arg; + tmr->u.callback = callback; + + tmr->head.roflgs |= ERTS_TMR_ROFLG_CALLBACK; + timeout_func = tw_callback_timeout; + cancel_func = NULL; + refc = 1; + break; + + default: + ERTS_INTERNAL_ERROR("Unsupported timer type"); + return NULL; + } + + erts_smp_atomic32_init_nob(&tmr->head.refc, refc); + + erts_twheel_set_timer(esdp->timer_wheel, + &tmr->tw_tmr, + timeout_func, + cancel_func, + tmr, + timeout_pos); + + return tmr; +} + +/* + * Basic high level timer stuff + */ + +static ERTS_INLINE void +hl_timer_destroy(ErtsHLTimer *tmr) +{ + Uint32 roflgs = tmr->head.roflgs; + if (!(roflgs & ERTS_TMR_ROFLG_BIF_TMR)) + erts_free(ERTS_ALC_T_HL_PTIMER, tmr); + else { + if (roflgs & ERTS_TMR_ROFLG_PRE_ALC) + bif_timer_pre_free(tmr); +#ifdef ERTS_BTM_ACCESSOR_SUPPORT + else if (roflgs & ERTS_TMR_ROFLG_ABIF_TMR) + erts_free(ERTS_ALC_T_ABIF_TIMER, tmr); +#endif + else + erts_free(ERTS_ALC_T_BIF_TIMER, tmr); + } +} + +static void +scheduled_hl_timer_destroy(void *vtmr) +{ + hl_timer_destroy((ErtsHLTimer *) vtmr); +} + +static void +schedule_hl_timer_destroy(ErtsHLTimer *tmr, Uint32 roflgs) +{ + UWord size; + + /* + * Reference to process/port can be dropped + * at once... + */ + + ERTS_HLT_ASSERT(erts_smp_atomic32_read_nob(&tmr->head.refc) == 0); + + if (roflgs & ERTS_TMR_ROFLG_REG_NAME) { + ERTS_HLT_ASSERT(is_atom(tmr->receiver.name)); + } + else if (roflgs & ERTS_TMR_ROFLG_PROC) { + ERTS_HLT_ASSERT(tmr->receiver.proc); + erts_proc_dec_refc(tmr->receiver.proc); + } + else if (roflgs & ERTS_TMR_ROFLG_PORT) { + ERTS_HLT_ASSERT(tmr->receiver.port); + erts_port_dec_refc(tmr->receiver.port); + } + + if (!(roflgs & ERTS_TMR_ROFLG_BIF_TMR)) + size = ERTS_HL_PTIMER_SIZE; + else { + /* + * Message buffer can be dropped at + * once... + */ + size = sizeof(ErtsHLTimer); + } + + erts_schedule_thr_prgr_later_cleanup_op( + scheduled_hl_timer_destroy, tmr, + &tmr->time.cleanup, size); +} + +static ERTS_INLINE void +hl_timer_pre_dec_refc(ErtsHLTimer *tmr) +{ +#ifdef ERTS_HLT_DEBUG + erts_aint_t refc; + refc = erts_smp_atomic32_dec_read_nob(&tmr->head.refc); + ERTS_HLT_ASSERT(refc > 0); +#else + erts_smp_atomic32_dec_nob(&tmr->head.refc); +#endif +} + +static ERTS_INLINE void +hl_timer_dec_refc(ErtsHLTimer *tmr, Uint32 roflgs) +{ + if (erts_smp_atomic32_dec_read_relb(&tmr->head.refc) == 0) { + ERTS_HLT_SMP_MEMBAR_LoadLoad_LoadStore; + schedule_hl_timer_destroy(tmr, roflgs); + } +} + +static void hlt_service_timeout(void *vesdp); +#ifdef ERTS_SMP +static void handle_canceled_queue(ErtsSchedulerData *esdp, + ErtsHLTCncldTmrQ *cq, + int use_limit, + int ops_limit, + int *need_thr_progress, + ErtsThrPrgrVal *thr_prgr_p, + int *need_more_work); +#endif + +static ERTS_INLINE void +check_canceled_queue(ErtsSchedulerData *esdp, ErtsHLTimerService *srv) +{ +#if defined(ERTS_SMP) && ERTS_TMR_CHECK_CANCEL_ON_CREATE + ErtsHLTCncldTmrQ *cq = &srv->canceled_queue; + if (cq->head.first != cq->head.unref_end) + handle_canceled_queue(esdp, cq, 1, + ERTS_TMR_CANCELED_TIMER_SMALL_LIMIT, + NULL, NULL, NULL); +#endif +} + +#ifdef ERTS_BTM_ACCESSOR_SUPPORT + +static void +hlt_delete_abtm(ErtsHLTimer *tmr) +{ + Process *proc; + + ERTS_HLT_ASSERT(tmr->head.roflgs & ERTS_TMR_ROFLG_ABIF_TMR); + + proc = erts_proc_lookup(tmr->abtm.accessor); + + if (proc) { + int deref = 0; + erts_smp_proc_lock(proc, ERTS_PROC_LOCK_BTM); + if (tmr->abtm.tree.parent != ERTS_HLT_PFIELD_NOT_IN_TABLE) { + abtm_rbt_delete(&proc->accessor_bif_timers, tmr); + deref = 1; + tmr->abtm.tree.parent = ERTS_HLT_PFIELD_NOT_IN_TABLE; + } + erts_smp_proc_unlock(proc, ERTS_PROC_LOCK_BTM); + if (deref) + hl_timer_pre_dec_refc(tmr); + } +} + +#endif + +static ErtsHLTimer * +create_hl_timer(ErtsSchedulerData *esdp, + ErtsMonotonicTime timeout_pos, + int short_time, ErtsTmrType type, + void *rcvrp, Eterm rcvr, Eterm acsr, + Eterm msg, Uint32 *refn, + void (*callback)(void *), void *arg) +{ + ErtsHLTimerService *srv = esdp->timer_service; + ErtsHLTimer *tmr, *st_tmr; + erts_aint32_t refc; + Uint32 roflgs; + + ERTS_HLT_HDBG_CHK_SRV(srv); + + check_canceled_queue(esdp, srv); + + ERTS_HLT_ASSERT((esdp->no & ~ERTS_TMR_ROFLG_SID_MASK) == 0); + + roflgs = ((Uint32) esdp->no) | ERTS_TMR_ROFLG_HLT; + + if (type != ERTS_TMR_BIF) { + + tmr = erts_alloc(ERTS_ALC_T_HL_PTIMER, + ERTS_HL_PTIMER_SIZE); + tmr->timeout = timeout_pos; + + switch (type) { + + case ERTS_TMR_PROC: + ERTS_HLT_ASSERT(is_internal_pid(rcvr)); + + erts_proc_inc_refc((Process *) rcvrp); + tmr->receiver.proc = (Process *) rcvrp; + roflgs |= ERTS_TMR_ROFLG_PROC; + refc = 2; + break; + + case ERTS_TMR_PORT: + ERTS_HLT_ASSERT(is_internal_port(rcvr)); + erts_port_inc_refc((Port *) rcvrp); + tmr->receiver.port = (Port *) rcvrp; + roflgs |= ERTS_TMR_ROFLG_PORT; + refc = 2; + break; + + case ERTS_TMR_CALLBACK: + roflgs |= ERTS_TMR_ROFLG_CALLBACK; + tmr->receiver.callback = callback; + tmr->head.u.arg = arg; + refc = 1; + break; + + default: + ERTS_INTERNAL_ERROR("Unsupported timer type"); + return NULL; + } + + } + else { /* ERTS_TMR_BIF */ + Uint hsz; +#ifdef ERTS_BTM_ACCESSOR_SUPPORT + int is_abif_tmr = is_value(acsr) && acsr != rcvr; +#endif + + if (short_time) { + tmr = bif_timer_pre_alloc(); + if (!tmr) + goto alloc_bif_timer; + roflgs |= ERTS_TMR_ROFLG_PRE_ALC; + } + else { + alloc_bif_timer: +#ifdef ERTS_BTM_ACCESSOR_SUPPORT + if (is_abif_tmr) + tmr = erts_alloc(ERTS_ALC_T_ABIF_TIMER, + ERTS_ABIF_TIMER_SIZE); + else +#endif + tmr = erts_alloc(ERTS_ALC_T_BIF_TIMER, + ERTS_BIF_TIMER_SIZE); + } + + tmr->timeout = timeout_pos; + + roflgs |= ERTS_TMR_ROFLG_BIF_TMR; + if (is_internal_pid(rcvr)) { + roflgs |= ERTS_TMR_ROFLG_PROC; + tmr->receiver.proc = (Process *) rcvrp; + refc = 2; + } + else { + ERTS_HLT_ASSERT(is_atom(rcvr)); + roflgs |= ERTS_TMR_ROFLG_REG_NAME; + tmr->receiver.name = rcvr; + refc = 1; + } + + hsz = is_immed(msg) ? ((Uint) 0) : size_object(msg); + if (!hsz) { + tmr->btm.message = msg; + tmr->btm.bp = NULL; + } + else { + ErlHeapFragment *bp = new_message_buffer(hsz); + Eterm *hp = bp->mem; + tmr->btm.message = copy_struct(msg, hsz, &hp, &bp->off_heap); + tmr->btm.bp = bp; + } + tmr->btm.refn[0] = refn[0]; + tmr->btm.refn[1] = refn[1]; + tmr->btm.refn[2] = refn[2]; + + tmr->btm.proc_tree.parent = ERTS_HLT_PFIELD_NOT_IN_TABLE; + +#ifdef ERTS_BTM_ACCESSOR_SUPPORT + if (is_abif_tmr) { + Process *aproc; + roflgs |= ERTS_TMR_ROFLG_ABIF_TMR; + tmr->abtm.accessor = acsr; + aproc = erts_proc_lookup(acsr); + if (!aproc) + tmr->abtm.tree.parent = ERTS_HLT_PFIELD_NOT_IN_TABLE; + else { + refc++; + erts_smp_proc_lock(aproc, ERTS_PROC_LOCK_BTM); + abtm_rbt_insert(&aproc->accessor_bif_timers, tmr); + erts_smp_proc_unlock(aproc, ERTS_PROC_LOCK_BTM); + } + } +#endif + + btm_rbt_insert(&srv->btm_tree, tmr); + } + + tmr->head.roflgs = roflgs; + erts_smp_atomic32_init_nob(&tmr->head.refc, refc); + erts_smp_atomic32_init_nob(&tmr->state, ERTS_TMR_STATE_ACTIVE); + + if (!srv->next_timeout + || tmr->timeout < srv->next_timeout->timeout) { + if (srv->next_timeout) + erts_twheel_cancel_timer(esdp->timer_wheel, + &srv->service_timer); + erts_twheel_set_timer(esdp->timer_wheel, + &srv->service_timer, + hlt_service_timeout, + NULL, + (void *) esdp, + tmr->timeout); + srv->next_timeout = tmr; + } + + st_tmr = time_rbt_lookup_insert(&srv->time_tree, tmr); + tmr->time.tree.same_time = st_tmr; + if (st_tmr) + same_time_list_insert(&st_tmr->time.tree.same_time, tmr); + +#ifdef ERTS_HLT_HARD_DEBUG + tmr->pending_timeout = 0; +#endif + + ERTS_HLT_HDBG_CHK_SRV(srv); + + return tmr; +} + +static ERTS_INLINE void +hlt_bif_timer_timeout(ErtsHLTimer *tmr, Uint32 roflgs) +{ + ErtsProcLocks proc_locks = ERTS_PROC_LOCKS_MSG_SEND; + Process *proc; + int queued_message = 0; + int dec_refc = 0; + Uint32 is_reg_name = (roflgs & ERTS_TMR_ROFLG_REG_NAME); + ERTS_HLT_ASSERT(roflgs & ERTS_TMR_ROFLG_BIF_TMR); + +#ifdef ERTS_BTM_ACCESSOR_SUPPORT + if (tmr->head.roflgs & ERTS_TMR_ROFLG_ABIF_TMR) + hlt_delete_abtm(tmr); +#endif + + if (is_reg_name) { + Eterm pid; + ERTS_HLT_ASSERT(is_atom(tmr->receiver.name)); + pid = erts_whereis_name_to_id(NULL, tmr->receiver.name); + proc = erts_proc_lookup(pid); + } + else { + ERTS_HLT_ASSERT(roflgs & ERTS_TMR_ROFLG_PROC); + ERTS_HLT_ASSERT(tmr->receiver.proc); + + proc = tmr->receiver.proc; + proc_locks |= ERTS_PROC_LOCK_BTM; + } + if (proc) { + erts_smp_proc_lock(proc, proc_locks); + /* + * If process is exiting, let it clean up + * the btm tree by itself (it may be in + * the middle of tree destruction). + */ + if (!ERTS_PROC_IS_EXITING(proc)) { + erts_queue_message(proc, &proc_locks, tmr->btm.bp, + tmr->btm.message, NIL); + erts_smp_proc_unlock(proc, ERTS_PROC_LOCKS_MSG_SEND); + queued_message = 1; + proc_locks &= ~ERTS_PROC_LOCKS_MSG_SEND; + tmr->btm.bp = NULL; + if (tmr->btm.proc_tree.parent != ERTS_HLT_PFIELD_NOT_IN_TABLE) { + proc_btm_rbt_delete(&proc->bif_timers, tmr); + tmr->btm.proc_tree.parent = ERTS_HLT_PFIELD_NOT_IN_TABLE; + dec_refc = 1; + } + } + if (proc_locks) + erts_smp_proc_unlock(proc, proc_locks); + if (dec_refc) + hl_timer_pre_dec_refc(tmr); + } + if (!queued_message && tmr->btm.bp) + free_message_buffer(tmr->btm.bp); +} + +static ERTS_INLINE void +hlt_proc_timeout(ErtsHLTimer *tmr) +{ + if (proc_timeout_common(tmr->receiver.proc, (void *) tmr)) + hl_timer_dec_refc(tmr, tmr->head.roflgs); +} + +static ERTS_INLINE void +hlt_port_timeout(ErtsHLTimer *tmr) +{ + if (port_timeout_common(tmr->receiver.port, (void *) tmr)) + hl_timer_dec_refc(tmr, tmr->head.roflgs); +} + +static void hlt_timeout(ErtsHLTimer *tmr, void *vsrv) +{ + ErtsHLTimerService *srv = (ErtsHLTimerService *) vsrv; + Uint32 roflgs; + erts_aint32_t state; + + ERTS_HLT_HDBG_CHK_SRV(srv); + + roflgs = tmr->head.roflgs; + ERTS_HLT_ASSERT(roflgs & ERTS_TMR_ROFLG_HLT); + + state = erts_smp_atomic32_cmpxchg_acqb(&tmr->state, + ERTS_TMR_STATE_TIMED_OUT, + ERTS_TMR_STATE_ACTIVE); + + ERTS_HLT_ASSERT(state == ERTS_TMR_STATE_CANCELED + || state == ERTS_TMR_STATE_ACTIVE); + + if (state == ERTS_TMR_STATE_ACTIVE) { + + if (roflgs & ERTS_TMR_ROFLG_BIF_TMR) + hlt_bif_timer_timeout(tmr, roflgs); + else if (roflgs & ERTS_TMR_ROFLG_PROC) + hlt_proc_timeout(tmr); + else if (roflgs & ERTS_TMR_ROFLG_PORT) + hlt_port_timeout(tmr); + else { + ERTS_HLT_ASSERT(roflgs & ERTS_TMR_ROFLG_CALLBACK); + (*tmr->receiver.callback)(tmr->head.u.arg); + } + + } + + tmr->time.tree.parent = ERTS_HLT_PFIELD_NOT_IN_TABLE; + if ((roflgs & ERTS_TMR_ROFLG_BIF_TMR) + && tmr->btm.tree.parent != ERTS_HLT_PFIELD_NOT_IN_TABLE) { + btm_rbt_delete(&srv->btm_tree, tmr); + tmr->btm.tree.parent = ERTS_HLT_PFIELD_NOT_IN_TABLE; + } + + ERTS_HLT_HDBG_CHK_SRV(srv); + + hl_timer_dec_refc(tmr, roflgs); +} + +#ifdef ERTS_HLT_HARD_DEBUG +static void +set_pending_timeout(ErtsHLTimer *tmr, void *unused) +{ + tmr->pending_timeout = -1; +} +#endif + +static void +hlt_service_timeout(void *vesdp) +{ + ErtsSchedulerData *esdp = (ErtsSchedulerData *) vesdp; + ErtsHLTimerService *srv = esdp->timer_service; + ErtsHLTimer *tmr = srv->next_timeout; + int yield; + + ERTS_HLT_HDBG_CHK_SRV(srv); + + ERTS_HLT_ASSERT(esdp == erts_get_scheduler_data()); + + ERTS_HLT_ASSERT(!srv->yield.root || srv->yield.root == tmr); + ERTS_HLT_ASSERT(tmr); + ERTS_HLT_ASSERT(tmr->timeout <= erts_get_monotonic_time(esdp)); + + if (!srv->yield.root) { + ERTS_HLT_ASSERT(tmr->time.tree.parent + != ERTS_HLT_PFIELD_NOT_IN_TABLE); + time_rbt_delete(&srv->time_tree, tmr); + tmr->time.tree.parent = ERTS_HLT_PFIELD_NOT_IN_TABLE; +#ifdef ERTS_HLT_HARD_DEBUG + tmr->pending_timeout = 1; + if (tmr->time.tree.same_time) + same_time_list_foreach(tmr->time.tree.same_time, set_pending_timeout, NULL); +#endif + } + + if (!tmr->time.tree.same_time && !srv->yield.root) + yield = 0; + else { + yield = same_time_list_foreach_destroy_yielding( + &tmr->time.tree.same_time, hlt_timeout, (void *) srv, + &srv->yield.state, ERTS_TMR_TIMEOUT_YIELD_LIMIT); + } + + if (yield) + srv->yield.root = tmr; + else { + srv->yield.root = NULL; + hlt_timeout(tmr, (void *) srv); + + tmr = time_rbt_smallest(srv->time_tree); + srv->next_timeout = tmr; + } + + ERTS_HLT_HDBG_CHK_SRV(srv); + + if (tmr) + erts_twheel_set_timer(esdp->timer_wheel, + &srv->service_timer, + hlt_service_timeout, + NULL, + vesdp, + tmr->timeout); +} + +static void +hlt_delete_timer(ErtsSchedulerData *esdp, ErtsHLTimer *tmr) +{ + ErtsHLTimerService *srv = esdp->timer_service; + + ERTS_HLT_HDBG_CHK_SRV(srv); + + if (tmr->head.roflgs & ERTS_TMR_ROFLG_BIF_TMR) { + + if (tmr->btm.tree.parent != ERTS_HLT_PFIELD_NOT_IN_TABLE) { + btm_rbt_delete(&srv->btm_tree, tmr); + tmr->btm.tree.parent = ERTS_HLT_PFIELD_NOT_IN_TABLE; + } + +#ifdef ERTS_BTM_ACCESSOR_SUPPORT + if (tmr->head.roflgs & ERTS_TMR_ROFLG_ABIF_TMR) + hlt_delete_abtm(tmr); +#endif + } + + if (tmr->time.tree.parent == ERTS_HLT_PFIELD_NOT_IN_TABLE) { + /* Already removed... */ + ERTS_HLT_HDBG_CHK_SRV(srv); + return; + } + + if (tmr->time.tree.parent & ERTS_HLT_PFLG_SAME_TIME) { + same_time_list_delete(tmr); + } + else if (tmr->time.tree.same_time) { + ErtsHLTimer *st_container; + + ERTS_HLT_ASSERT((tmr->time.tree.parent & ERTS_HLT_PFLG_SAME_TIME) == 0); + st_container = tmr->time.tree.same_time->time.tree.u.l.prev; + + ERTS_HLT_ASSERT(st_container); + ERTS_HLT_ASSERT(st_container->time.tree.parent + & ERTS_HLT_PFLG_SAME_TIME); + ERTS_HLT_ASSERT(tmr->timeout == st_container->timeout); + + same_time_list_delete(st_container); + st_container->time.tree.same_time = tmr->time.tree.same_time; + same_time_list_new_root(&st_container->time.tree.same_time); + + time_rbt_replace(&srv->time_tree, tmr, st_container); + ERTS_HLT_ASSERT((st_container->time.tree.parent + & ERTS_HLT_PFLG_SAME_TIME) == 0); + + if (srv->next_timeout == tmr) + srv->next_timeout = st_container; + } + else { + ERTS_HLT_ASSERT((tmr->time.tree.parent & ERTS_HLT_PFLG_SAME_TIME) == 0); + time_rbt_delete(&srv->time_tree, tmr); + if (tmr == srv->next_timeout) { + ErtsHLTimer *smlst; + erts_twheel_cancel_timer(esdp->timer_wheel, + &srv->service_timer); + smlst = time_rbt_smallest(srv->time_tree); + srv->next_timeout = smlst; + if (smlst) { + ERTS_HLT_ASSERT(smlst->timeout > tmr->timeout); + erts_twheel_set_timer(esdp->timer_wheel, + &srv->service_timer, + hlt_service_timeout, + NULL, + (void *) esdp, + smlst->timeout); + } + } + } + tmr->time.tree.parent = ERTS_HLT_PFIELD_NOT_IN_TABLE; + + hl_timer_dec_refc(tmr, tmr->head.roflgs); + + ERTS_HLT_HDBG_CHK_SRV(srv); +} + +/* + * Pass canceled timers back to originating scheduler + */ + +static ERTS_INLINE void +cleanup_sched_local_canceled_timer(ErtsSchedulerData *esdp, + ErtsTimer *tmr) +{ + Uint32 roflgs = tmr->head.roflgs; + ERTS_HLT_ASSERT(esdp == erts_get_scheduler_data()); + ERTS_HLT_ASSERT((tmr->head.roflgs & ERTS_TMR_ROFLG_SID_MASK) + == (Uint32) esdp->no); + if (roflgs & ERTS_TMR_ROFLG_HLT) { + hlt_delete_timer(esdp, &tmr->hlt); + hl_timer_dec_refc(&tmr->hlt, roflgs); + } + else { + cancel_tw_timer(esdp, &tmr->twt); + tw_timer_dec_refc(&tmr->twt); + } +} + +#ifdef ERTS_SMP + +static void +init_canceled_queue(ErtsHLTCncldTmrQ *cq) +{ + erts_atomic_init_nob(&cq->tail.data.marker.u.next, ERTS_AINT_NULL); + erts_atomic_init_nob(&cq->tail.data.last, + (erts_aint_t) &cq->tail.data.marker); + cq->head.first = (ErtsTimer *) &cq->tail.data.marker; + cq->head.unref_end = (ErtsTimer *) &cq->tail.data.marker; + cq->head.next.thr_progress = erts_thr_progress_current(); + cq->head.next.thr_progress_reached = 1; + cq->head.next.unref_end = (ErtsTimer *) &cq->tail.data.marker; + cq->head.used_marker = 1; +} + +static ERTS_INLINE int +cq_enqueue(ErtsHLTCncldTmrQ *cq, ErtsTimer *tmr, int cinit) +{ + erts_aint_t itmp; + ErtsTimer *enq, *this = tmr; + + erts_atomic_init_nob(&this->head.u.next, ERTS_AINT_NULL); + /* Enqueue at end of list... */ + + enq = (ErtsTimer *) erts_atomic_read_nob(&cq->tail.data.last); + itmp = erts_atomic_cmpxchg_relb(&enq->head.u.next, + (erts_aint_t) this, + ERTS_AINT_NULL); + if (itmp == ERTS_AINT_NULL) { + /* We are required to move last pointer */ +#ifdef DEBUG + ASSERT(ERTS_AINT_NULL == erts_atomic_read_nob(&this->head.u.next)); + ASSERT(((erts_aint_t) enq) + == erts_atomic_xchg_relb(&cq->tail.data.last, + (erts_aint_t) this)); +#else + erts_atomic_set_relb(&cq->tail.data.last, (erts_aint_t) this); +#endif + return 1; + } + else { + /* + * We *need* to insert element somewhere in between the + * last element we read earlier and the actual last element. + */ + int i = cinit; + + while (1) { + erts_aint_t itmp2; + erts_atomic_set_nob(&this->head.u.next, itmp); + itmp2 = erts_atomic_cmpxchg_relb(&enq->head.u.next, + (erts_aint_t) this, + itmp); + if (itmp == itmp2) + return 0; /* inserted this */ + if ((i & 1) == 0) + itmp = itmp2; + else { + enq = (ErtsTimer *) itmp2; + itmp = erts_atomic_read_acqb(&enq->head.u.next); + ASSERT(itmp != ERTS_AINT_NULL); + } + i++; + } + } +} + +static ERTS_INLINE erts_aint_t +check_insert_marker(ErtsHLTCncldTmrQ *cq, erts_aint_t ilast) +{ + if (!cq->head.used_marker + && cq->head.unref_end == (ErtsTimer *) ilast) { + erts_aint_t itmp; + ErtsTimer *last = (ErtsTimer *) ilast; + + erts_atomic_init_nob(&cq->tail.data.marker.u.next, ERTS_AINT_NULL); + itmp = erts_atomic_cmpxchg_relb(&last->head.u.next, + (erts_aint_t) &cq->tail.data.marker, + ERTS_AINT_NULL); + if (itmp == ERTS_AINT_NULL) { + ilast = (erts_aint_t) &cq->tail.data.marker; + cq->head.used_marker = !0; + erts_atomic_set_relb(&cq->tail.data.last, ilast); + } + } + return ilast; +} + +static ERTS_INLINE ErtsTimer * +cq_dequeue(ErtsHLTCncldTmrQ *cq) +{ + ErtsTimer *tmr; + + if (cq->head.first == cq->head.unref_end) + return NULL; + + tmr = cq->head.first; + if (tmr == (ErtsTimer *) &cq->tail.data.marker) { + ASSERT(cq->head.used_marker); + cq->head.used_marker = 0; + tmr = (ErtsTimer *) erts_atomic_read_nob(&tmr->head.u.next); + if (tmr == cq->head.unref_end) { + cq->head.first = tmr; + return NULL; + } + } + + cq->head.first = (ErtsTimer *) erts_atomic_read_nob(&tmr->head.u.next); + + ASSERT(cq->head.first); + + return tmr; +} + +static int +cq_check_incoming(ErtsSchedulerData *esdp, ErtsHLTCncldTmrQ *cq) +{ + erts_aint_t ilast = erts_atomic_read_nob(&cq->tail.data.last); + if (((ErtsTimer *) ilast) == (ErtsTimer *) &cq->tail.data.marker + && cq->head.first == (ErtsTimer *) &cq->tail.data.marker) { + /* Nothing more to do... */ + return 0; + } + + if (cq->head.next.thr_progress_reached + || erts_thr_progress_has_reached(cq->head.next.thr_progress)) { + cq->head.next.thr_progress_reached = 1; + /* Move unreferenced end pointer forward... */ + + ERTS_HLT_SMP_MEMBAR_LoadLoad_LoadStore; + + cq->head.unref_end = cq->head.next.unref_end; + + ilast = check_insert_marker(cq, ilast); + + if (cq->head.unref_end != (ErtsTimer *) ilast) { + cq->head.next.unref_end = (ErtsTimer *) ilast; + cq->head.next.thr_progress = erts_thr_progress_later(esdp); + cq->head.next.thr_progress_reached = 0; + } + } + return 1; +} + +static ERTS_INLINE void +store_earliest_thr_prgr(ErtsThrPrgrVal *prev_val, ErtsHLTCncldTmrQ *cq) +{ + if (!cq->head.next.thr_progress_reached + && (*prev_val == ERTS_THR_PRGR_INVALID + || erts_thr_progress_cmp(cq->head.next.thr_progress, + *prev_val) < 0)) { + *prev_val = cq->head.next.thr_progress; + } +} + +static void +handle_canceled_queue(ErtsSchedulerData *esdp, + ErtsHLTCncldTmrQ *cq, + int use_limit, + int ops_limit, + int *need_thr_progress, + ErtsThrPrgrVal *thr_prgr_p, + int *need_more_work) +{ + int need_thr_prgr = 0; + int need_mr_wrk = 0; + int have_checked_incoming = 0; + int ops = 0; + + ERTS_HLT_ASSERT(cq == &esdp->timer_service->canceled_queue); + + while (1) { + ErtsTimer *tmr = cq_dequeue(cq); + + if (tmr) + cleanup_sched_local_canceled_timer(esdp, tmr); + else { + if (have_checked_incoming) + break; + need_thr_prgr = cq_check_incoming(esdp, cq); + if (need_thr_progress) { + *need_thr_progress |= need_thr_prgr; + if (need_thr_prgr) + store_earliest_thr_prgr(thr_prgr_p, cq); + } + have_checked_incoming = 1; + continue; + } + + if (use_limit && ++ops >= ops_limit) { + if (cq->head.first != cq->head.unref_end) { + need_mr_wrk = 1; + if (need_more_work) + *need_more_work |= 1; + } + break; + } + } + + if (need_thr_progress && !(need_thr_prgr | need_mr_wrk)) { + need_thr_prgr = cq_check_incoming(esdp, cq); + *need_thr_progress |= need_thr_prgr; + if (need_thr_prgr) + store_earliest_thr_prgr(thr_prgr_p, cq); + } +} + +void +erts_handle_canceled_timers(void *vesdp, + int *need_thr_progress, + ErtsThrPrgrVal *thr_prgr_p, + int *need_more_work) +{ + ErtsSchedulerData *esdp = (ErtsSchedulerData *) vesdp; + ERTS_HLT_ASSERT(esdp == erts_get_scheduler_data()); + + handle_canceled_queue(esdp, &esdp->timer_service->canceled_queue, + 1, ERTS_TMR_CANCELED_TIMER_LIMIT, + need_thr_progress, thr_prgr_p, + need_more_work); +} + +#endif /* ERTS_SMP */ + +static void +queue_canceled_timer(ErtsSchedulerData *esdp, int rsched_id, ErtsTimer *tmr) +{ +#ifdef ERTS_SMP + ErtsHLTCncldTmrQ *cq; + cq = &ERTS_SCHEDULER_IX(rsched_id-1)->timer_service->canceled_queue; + if (cq_enqueue(cq, tmr, rsched_id - (int) esdp->no)) + erts_notify_canceled_timer(esdp, rsched_id); +#else + ERTS_INTERNAL_ERROR("Unexpected enqueue of canceled timer"); +#endif +} + +static void +continue_cancel_ptimer(ErtsSchedulerData *esdp, ErtsTimer *tmr) +{ +#ifdef ERTS_SMP + Uint32 sid = (tmr->head.roflgs & ERTS_TMR_ROFLG_SID_MASK); + + if (esdp->no != sid) + queue_canceled_timer(esdp, sid, tmr); + else +#endif + cleanup_sched_local_canceled_timer(esdp, tmr); +} + +/* + * BIF timer specific + */ + +Uint erts_bif_timer_memory_size(void) +{ + return (Uint) 0; +} + +static BIF_RETTYPE +setup_bif_timer(Process *c_p, ErtsMonotonicTime timeout_pos, + int short_time, Eterm rcvr, Eterm acsr, + Eterm msg, int wrap) +{ + BIF_RETTYPE ret; + Eterm ref, tmo_msg, *hp; + ErtsHLTimer *tmr; + ErtsSchedulerData *esdp; + DeclareTmpHeap(tmp_hp, 4, c_p); + + if (is_not_internal_pid(rcvr) && is_not_atom(rcvr)) + goto badarg; + + esdp = ERTS_PROC_GET_SCHDATA(c_p); + + hp = HAlloc(c_p, REF_THING_SIZE); + ref = erts_sched_make_ref_in_buffer(esdp, hp); + + ASSERT(erts_get_ref_numbers_thr_id( + internal_ref_numbers(ref)) == (Uint32) esdp->no); + + UseTmpHeap(4, c_p); + + tmo_msg = wrap ? TUPLE3(tmp_hp, am_timeout, ref, msg) : msg; + + tmr = create_hl_timer(esdp, timeout_pos, short_time, + ERTS_TMR_BIF, NULL, rcvr, acsr, tmo_msg, + internal_ref_numbers(ref), NULL, NULL); + + UnUseTmpHeap(4, c_p); + + if (is_internal_pid(rcvr)) { + Process *proc = erts_pid2proc_opt(c_p, ERTS_PROC_LOCK_MAIN, + rcvr, ERTS_PROC_LOCK_BTM, + ERTS_P2P_FLG_INC_REFC); + if (!proc) { + if (tmr->btm.bp) + free_message_buffer(tmr->btm.bp); + hlt_delete_timer(esdp, tmr); + hl_timer_destroy(tmr); + } + else { + proc_btm_rbt_insert(&proc->bif_timers, tmr); + erts_smp_proc_unlock(proc, ERTS_PROC_LOCK_BTM); + tmr->receiver.proc = proc; + } + } + + ERTS_BIF_PREP_RET(ret, ref); + return ret; + +badarg: + + ERTS_BIF_PREP_ERROR(ret, c_p, BADARG); + return ret; +} + +static int +cancel_bif_timer(ErtsHLTimer *tmr) +{ + erts_aint_t state; + Uint32 roflgs; + int res; + + state = erts_smp_atomic32_cmpxchg_acqb(&tmr->state, + ERTS_TMR_STATE_CANCELED, + ERTS_TMR_STATE_ACTIVE); + if (state != ERTS_TMR_STATE_ACTIVE) + return 0; + + if (tmr->btm.bp) + free_message_buffer(tmr->btm.bp); + + res = -1; + + roflgs = tmr->head.roflgs; + if (roflgs & ERTS_TMR_ROFLG_PROC) { + Process *proc = tmr->receiver.proc; + ERTS_HLT_ASSERT(!(tmr->head.roflgs & ERTS_TMR_ROFLG_REG_NAME)); + + erts_smp_proc_lock(proc, ERTS_PROC_LOCK_BTM); + /* + * If process is exiting, let it clean up + * the btm tree by itself (it may be in + * the middle of tree destruction). + */ + if (!ERTS_PROC_IS_EXITING(proc) + && tmr->btm.proc_tree.parent != ERTS_HLT_PFIELD_NOT_IN_TABLE) { + proc_btm_rbt_delete(&proc->bif_timers, tmr); + tmr->btm.proc_tree.parent = ERTS_HLT_PFIELD_NOT_IN_TABLE; + res = 1; + } + erts_smp_proc_unlock(proc, ERTS_PROC_LOCK_BTM); + } + + return res; +} + +static ERTS_INLINE Eterm +access_sched_local_btm(Process *c_p, Eterm pid, + Eterm tref, Uint32 *trefn, + Uint32 *rrefn, + int async, int cancel, + int return_res, + int info) +{ + ErtsSchedulerData *esdp; + ErtsHLTimerService *srv; + ErtsHLTimer *tmr; + Sint64 time_left; + Process *proc; + ErtsProcLocks proc_locks; + + time_left = -1; + + if (!c_p) + esdp = erts_get_scheduler_data(); + else { + esdp = ERTS_PROC_GET_SCHDATA(c_p); + ERTS_HLT_ASSERT(esdp == erts_get_scheduler_data()); + } + + ERTS_HLT_ASSERT(erts_get_ref_numbers_thr_id(trefn) + == (Uint32) esdp->no); + + srv = esdp->timer_service; + + tmr = btm_rbt_lookup(srv->btm_tree, trefn); + if (tmr) { + if (!cancel) { + erts_aint32_t state = erts_smp_atomic32_read_acqb(&tmr->state); + if (state == ERTS_TMR_STATE_ACTIVE) + time_left = get_time_left(esdp, tmr->timeout); + } + else { + int cncl_res = cancel_bif_timer(tmr); + if (cncl_res) { + + time_left = get_time_left(esdp, tmr->timeout); + + if (cncl_res > 0) + hl_timer_dec_refc(tmr, tmr->head.roflgs); + + hlt_delete_timer(esdp, tmr); + } + } + } + + if (!info) + return am_ok; + + if (return_res) { + ERTS_HLT_ASSERT(c_p); + if (time_left < 0) + return am_false; + else if (time_left <= (Sint64) MAX_SMALL) + return make_small((Sint) time_left); + else { + Uint hsz = ERTS_SINT64_HEAP_SIZE(time_left); + Eterm *hp = HAlloc(c_p, hsz); + return erts_sint64_to_big(time_left, &hp); + } + } + + if (c_p) { + proc = c_p; + proc_locks = ERTS_PROC_LOCK_MAIN; + } + else { + proc = erts_proc_lookup(pid); + proc_locks = 0; + } + + if (proc) { + Uint hsz; + ErlOffHeap *ohp; + ErlHeapFragment* bp; + Eterm *hp, msg, ref, result; +#ifdef ERTS_HLT_DEBUG + Eterm *hp_end; +#endif + + hsz = 3; /* 2-tuple */ + if (!async) + hsz += REF_THING_SIZE; + else { + if (is_non_value(tref) || proc != c_p) + hsz += REF_THING_SIZE; + hsz += 1; /* upgrade to 3-tuple */ + } + if (time_left > (Sint64) MAX_SMALL) + hsz += ERTS_SINT64_HEAP_SIZE(time_left); + + if (proc == c_p) { + bp = NULL; + ohp = NULL; + hp = HAlloc(c_p, hsz); + } + else { + hp = erts_alloc_message_heap(hsz, + &bp, + &ohp, + proc, + &proc_locks); + } + +#ifdef ERTS_HLT_DEBUG + hp_end = hp + hsz; +#endif + + if (time_left < 0) + result = am_false; + else if (time_left <= (Sint64) MAX_SMALL) + result = make_small((Sint) time_left); + else + result = erts_sint64_to_big(time_left, &hp); + + if (!async) { + write_ref_thing(hp, + rrefn[0], + rrefn[1], + rrefn[2]); + ref = make_internal_ref(hp); + hp += REF_THING_SIZE; + msg = TUPLE2(hp, ref, result); + + ERTS_HLT_ASSERT(hp + 3 == hp_end); + } + else { + Eterm tag = cancel ? am_cancel_timer : am_read_timer; + if (is_value(tref) && proc == c_p) + ref = tref; + else { + write_ref_thing(hp, + trefn[0], + trefn[1], + trefn[2]); + ref = make_internal_ref(hp); + hp += REF_THING_SIZE; + } + msg = TUPLE3(hp, tag, ref, result); + + ERTS_HLT_ASSERT(hp + 4 == hp_end); + + } + erts_queue_message(proc, &proc_locks, bp, msg, NIL); + + if (c_p) + proc_locks &= ~ERTS_PROC_LOCK_MAIN; + if (proc_locks) + erts_smp_proc_unlock(proc, proc_locks); + } + + return am_ok; +} + +#define ERTS_BTM_REQ_FLG_ASYNC (((Uint32) 1) << 0) +#define ERTS_BTM_REQ_FLG_CANCEL (((Uint32) 1) << 1) +#define ERTS_BTM_REQ_FLG_INFO (((Uint32) 1) << 2) + +typedef struct { + Eterm pid; + Uint32 trefn[ERTS_REF_NUMBERS]; + Uint32 rrefn[ERTS_REF_NUMBERS]; + Uint32 flags; +} ErtsBifTimerRequest; + +static void +bif_timer_access_request(void *vreq) +{ + ErtsBifTimerRequest *req = (ErtsBifTimerRequest *) vreq; + int async = (int) (req->flags & ERTS_BTM_REQ_FLG_ASYNC); + int cancel = (int) (req->flags & ERTS_BTM_REQ_FLG_CANCEL); + int info = (int) (req->flags & ERTS_BTM_REQ_FLG_INFO); + (void) access_sched_local_btm(NULL, req->pid, THE_NON_VALUE, + req->trefn, req->rrefn, async, + cancel, 0, info); + erts_free(ERTS_ALC_T_TIMER_REQUEST, vreq); +} + +static int +try_access_sched_remote_btm(ErtsSchedulerData *esdp, + Process *c_p, Uint32 sid, + Eterm tref, Uint32 *trefn, + int async, int cancel, + int info, Eterm *resp) +{ + ErtsHLTimer *tmr; + Sint64 time_left; + + ERTS_HLT_ASSERT(c_p); + + /* + * Check if the timer is aimed at current + * process of if this process is an accessor + * of the timer... + */ + erts_smp_proc_lock(c_p, ERTS_PROC_LOCK_BTM); + tmr = proc_btm_rbt_lookup(c_p->bif_timers, trefn); +#ifdef ERTS_BTM_ACCESSOR_SUPPORT + if (!tmr) + tmr = abtm_rbt_lookup(c_p->accessor_bif_timers, trefn); +#endif + erts_smp_proc_unlock(c_p, ERTS_PROC_LOCK_BTM); + if (!tmr) + return 0; + + if (!cancel) { + erts_aint32_t state = erts_smp_atomic32_read_acqb(&tmr->state); + if (state == ERTS_TMR_STATE_ACTIVE) + time_left = get_time_left(esdp, tmr->timeout); + else + time_left = -1; + } + else { + int cncl_res = cancel_bif_timer(tmr); + if (!cncl_res) + time_left = -1; + else { + time_left = get_time_left(esdp, tmr->timeout); + if (cncl_res > 0) + queue_canceled_timer(esdp, sid, (ErtsTimer *) tmr); + } + } + + if (!info) { + *resp = am_ok; + return 1; + } + + if (!async) { + if (time_left < 0) + *resp = am_false; + else if (time_left <= (Sint64) MAX_SMALL) + *resp = make_small((Sint) time_left); + else { + Uint hsz = ERTS_SINT64_HEAP_SIZE(time_left); + Eterm *hp = HAlloc(c_p, hsz); + *resp = erts_sint64_to_big(time_left, &hp); + } + } + else { + Eterm tag, res, msg; + Uint hsz; + Eterm *hp; + ErtsProcLocks proc_locks = ERTS_PROC_LOCK_MAIN; + + hsz = 4; + if (time_left > (Sint64) MAX_SMALL) + hsz += ERTS_SINT64_HEAP_SIZE(time_left); + + hp = HAlloc(c_p, hsz); + if (cancel) + tag = am_cancel_timer; + else + tag = am_read_timer; + + if (time_left < 0) + res = am_false; + else if (time_left <= (Sint64) MAX_SMALL) + res = make_small((Sint) time_left); + else + res = erts_sint64_to_big(time_left, &hp); + + msg = TUPLE3(hp, tag, tref, res); + + erts_queue_message(c_p, &proc_locks, NULL, msg, NIL); + + proc_locks &= ~ERTS_PROC_LOCK_MAIN; + if (proc_locks) + erts_smp_proc_unlock(c_p, proc_locks); + + *resp = am_ok; + } + return 1; +} + +static BIF_RETTYPE +access_bif_timer(Process *c_p, Eterm tref, int cancel, int async, int info) +{ + BIF_RETTYPE ret; + ErtsSchedulerData *esdp; + Uint32 sid; + Uint32 *trefn; + Eterm res; + + if (is_not_internal_ref(tref)) { + if (is_not_ref(tref)) + goto badarg; + else + goto no_timer; + } + + esdp = ERTS_PROC_GET_SCHDATA(c_p); + + trefn = internal_ref_numbers(tref); + sid = erts_get_ref_numbers_thr_id(trefn); + if (sid < 1 || erts_no_schedulers < sid) + goto no_timer; + + if (sid == (Uint32) esdp->no) { + res = access_sched_local_btm(c_p, c_p->common.id, + tref, trefn, NULL, + async, cancel, !async, + info); + ERTS_BIF_PREP_RET(ret, res); + } + else if (try_access_sched_remote_btm(esdp, c_p, sid, + tref, trefn, + async, cancel, + info, &res)) { + ERTS_BIF_PREP_RET(ret, res); + } + else { + /* + * Schedule access for execution on + * remote scheduler... + */ + ErtsBifTimerRequest *req = erts_alloc(ERTS_ALC_T_TIMER_REQUEST, + sizeof(ErtsBifTimerRequest)); + + req->flags = 0; + if (cancel) + req->flags |= ERTS_BTM_REQ_FLG_CANCEL; + if (async) + req->flags |= ERTS_BTM_REQ_FLG_ASYNC; + if (info) + req->flags |= ERTS_BTM_REQ_FLG_INFO; + + req->pid = c_p->common.id; + + req->trefn[0] = trefn[0]; + req->trefn[1] = trefn[1]; + req->trefn[2] = trefn[2]; + + if (async) + ERTS_BIF_PREP_RET(ret, am_ok); + else { + Eterm *hp, rref; + Uint32 *rrefn; + + hp = HAlloc(c_p, REF_THING_SIZE); + rref = erts_sched_make_ref_in_buffer(esdp, hp); + rrefn = internal_ref_numbers(rref); + + req->rrefn[0] = rrefn[0]; + req->rrefn[1] = rrefn[1]; + req->rrefn[2] = rrefn[2]; + + erts_smp_proc_lock(c_p, ERTS_PROC_LOCKS_MSG_RECEIVE); + + if (ERTS_PROC_PENDING_EXIT(c_p)) + ERTS_VBUMP_ALL_REDS(c_p); + else { + /* + * Caller needs to wait for a message containing + * the ref that we just created. No such message + * can exist in callers message queue at this time. + * We therefore move the save pointer of the + * callers message queue to the end of the queue. + * + * NOTE: It is of vital importance that the caller + * immediately do a receive unconditionaly + * waiting for the message with the reference; + * otherwise, next receive will *not* work + * as expected! + */ + ERTS_SMP_MSGQ_MV_INQ2PRIVQ(c_p); + c_p->msg.save = c_p->msg.last; + } + erts_smp_proc_unlock(c_p, ERTS_PROC_LOCKS_MSG_RECEIVE); + + ERTS_BIF_PREP_TRAP1(ret, erts_await_result, c_p, rref); + } + + erts_schedule_misc_aux_work(sid, + bif_timer_access_request, + (void *) req); + } + + return ret; + +badarg: + ERTS_BIF_PREP_ERROR(ret, c_p, BADARG); + return ret; + +no_timer: + ERTS_BIF_PREP_RET(ret, am_false); + return ret; + +} + +static ERTS_INLINE int +bool_arg(Eterm val, int *argp) +{ + switch (val) { + case am_true: *argp = 1; return 1; + case am_false: *argp = 0; return 1; + default: return 0; + } +} + +static ERTS_INLINE int +parse_bif_timer_options(Eterm option_list, int *async, int *info, + int *abs, Eterm *accessor) +{ + Eterm list = option_list; + + if (async) + *async = 0; + if (info) + *info = 1; + if (abs) + *abs = 0; + if (accessor) + *accessor = THE_NON_VALUE; + + while (is_list(list)) { + Eterm *consp, *tp, opt; + + consp = list_val(list); + opt = CAR(consp); + if (is_not_tuple(opt)) + return 0; + + tp = tuple_val(opt); + if (arityval(tp[0]) != 2) + return 0; + + switch (tp[1]) { + case am_async: + if (!async || !bool_arg(tp[2], async)) + return 0; + break; + case am_info: + if (!info || !bool_arg(tp[2], info)) + return 0; + break; + case am_abs: + if (!abs || !bool_arg(tp[2], abs)) + return 0; + break; +#ifdef ERTS_BTM_ACCESSOR_SUPPORT + case am_accessor: + if (!accessor || is_not_internal_pid(tp[2])) + return 0; + *accessor = tp[2]; + break; +#endif + default: + return 0; + } + + list = CDR(consp); + } + + if (is_not_nil(list)) + return 0; + return 1; +} + +static void +exit_cancel_bif_timer(ErtsHLTimer *tmr, void *vesdp) +{ + ErtsSchedulerData *esdp = (ErtsSchedulerData *) vesdp; + Uint32 sid, roflgs; + erts_aint_t state; + + state = erts_smp_atomic32_cmpxchg_acqb(&tmr->state, + ERTS_TMR_STATE_CANCELED, + ERTS_TMR_STATE_ACTIVE); + + roflgs = tmr->head.roflgs; + sid = roflgs & ERTS_TMR_ROFLG_SID_MASK; + + ERTS_HLT_ASSERT(sid == erts_get_ref_numbers_thr_id(tmr->btm.refn)); + ERTS_HLT_ASSERT(tmr->btm.proc_tree.parent + != ERTS_HLT_PFIELD_NOT_IN_TABLE); + + tmr->btm.proc_tree.parent = ERTS_HLT_PFIELD_NOT_IN_TABLE; + + if (sid == (Uint32) esdp->no) { + if (state == ERTS_TMR_STATE_ACTIVE) { + if (tmr->btm.bp) + free_message_buffer(tmr->btm.bp); + hlt_delete_timer(esdp, tmr); + } + hl_timer_dec_refc(tmr, roflgs); + } + else { + if (state == ERTS_TMR_STATE_ACTIVE) { + if (tmr->btm.bp) + free_message_buffer(tmr->btm.bp); + queue_canceled_timer(esdp, sid, (ErtsTimer *) tmr); + } + else + hl_timer_dec_refc(tmr, roflgs); + } +} + +#ifdef ERTS_HLT_DEBUG +# define ERTS_BTM_MAX_DESTROY_LIMIT 2 +#else +# define ERTS_BTM_MAX_DESTROY_LIMIT 50 +#endif + +typedef struct { + ErtsBifTimers *bif_timers; + union { + proc_btm_rbt_yield_state_t proc_btm_yield_state; +#ifdef ERTS_BTM_ACCESSOR_SUPPORT + abtm_rbt_yield_state_t abtm_yield_state; +#endif + } u; +} ErtsBifTimerYieldState; + +int erts_cancel_bif_timers(Process *p, ErtsBifTimers *btm, void **vyspp) +{ + ErtsSchedulerData *esdp = ERTS_PROC_GET_SCHDATA(p); + ErtsBifTimerYieldState ys = {btm, {ERTS_RBT_YIELD_STAT_INITER}}; + ErtsBifTimerYieldState *ysp; + int res; + + ysp = (ErtsBifTimerYieldState *) *vyspp; + if (!ysp) + ysp = &ys; + + res = proc_btm_rbt_foreach_destroy_yielding(&ysp->bif_timers, + exit_cancel_bif_timer, + (void *) esdp, + &ysp->u.proc_btm_yield_state, + ERTS_BTM_MAX_DESTROY_LIMIT); + + if (res == 0) { + if (ysp != &ys) + erts_free(ERTS_ALC_T_BTM_YIELD_STATE, ysp); + *vyspp = NULL; + } + else { + + if (ysp == &ys) { + ysp = erts_alloc(ERTS_ALC_T_BTM_YIELD_STATE, + sizeof(ErtsBifTimerYieldState)); + sys_memcpy((void *) ysp, (void *) &ys, + sizeof(ErtsBifTimerYieldState)); + } + + *vyspp = (void *) ysp; + } + + return res; +} + +#ifdef ERTS_BTM_ACCESSOR_SUPPORT + +static void +detach_bif_timer(ErtsHLTimer *tmr, void *vesdp) +{ + tmr->abtm.tree.parent = ERTS_HLT_PFIELD_NOT_IN_TABLE; + hl_timer_dec_refc(tmr, tmr->head.roflgs); +} + +int erts_detach_accessor_bif_timers(Process *p, ErtsBifTimers *btm, void **vyspp) +{ + ErtsSchedulerData *esdp = ERTS_PROC_GET_SCHDATA(p); + ErtsBifTimerYieldState ys = {btm, {ERTS_RBT_YIELD_STAT_INITER}}; + ErtsBifTimerYieldState *ysp; + int res; + + ysp = (ErtsBifTimerYieldState *) *vyspp; + if (!ysp) + ysp = &ys; + + res = abtm_rbt_foreach_destroy_yielding(&ysp->bif_timers, + detach_bif_timer, + (void *) esdp, + &ysp->u.abtm_yield_state, + ERTS_BTM_MAX_DESTROY_LIMIT); + + if (res == 0) { + if (ysp != &ys) + erts_free(ERTS_ALC_T_BTM_YIELD_STATE, ysp); + *vyspp = NULL; + } + else { + + if (ysp == &ys) { + ysp = erts_alloc(ERTS_ALC_T_BTM_YIELD_STATE, + sizeof(ErtsBifTimerYieldState)); + sys_memcpy((void *) ysp, (void *) &ys, + sizeof(ErtsBifTimerYieldState)); + } + + *vyspp = (void *) ysp; + } + + return res; +} + +#endif /* ERTS_BTM_ACCESSOR_SUPPORT */ + +static ERTS_INLINE int +parse_timeout_pos(ErtsSchedulerData *esdp, Eterm arg, + ErtsMonotonicTime *conv_arg, int abs, + ErtsMonotonicTime *tposp, int *stimep) +{ + ErtsMonotonicTime t; + + if (!term_to_Sint64(arg, &t)) { + ERTS_HLT_ASSERT(!is_small(arg)); + if (!is_big(arg)) + return -1; + + if (abs || !big_sign(arg)) + return 1; + + return -1; + } + + if (conv_arg) + *conv_arg = t; + + if (abs) { + t += -1*ERTS_MONOTONIC_OFFSET_MSEC; /* external to internal */ + if (t < ERTS_MONOTONIC_TO_MSEC(ERTS_MONOTONIC_BEGIN)) + return 1; + if (t > ERTS_MONOTONIC_TO_MSEC(ERTS_MONOTONIC_END)) + return 1; + *stimep = (t - ERTS_MONOTONIC_TO_MSEC(esdp->last_monotonic_time) + < ERTS_BIF_TIMER_SHORT_TIME); + *tposp = ERTS_MSEC_TO_CLKTCKS(t); + } + else { + ErtsMonotonicTime now, ticks; + + if (t < 0) + return -1; + + ticks = ERTS_MSEC_TO_CLKTCKS(t); + + if (ERTS_CLKTCK_RESOLUTION > 1000 && ticks < 0) + return 1; + + ERTS_HLT_ASSERT(ticks >= 0); + + now = erts_get_monotonic_time(esdp); + ticks += ERTS_MONOTONIC_TO_CLKTCKS(now-1); + ticks += 1; + + if (ticks < ERTS_MONOTONIC_TO_CLKTCKS(ERTS_MONOTONIC_BEGIN)) + return 1; + if (ticks > ERTS_MONOTONIC_TO_CLKTCKS(ERTS_MONOTONIC_END)) + return 1; + + *stimep = (t < ERTS_BIF_TIMER_SHORT_TIME); + *tposp = ticks; + } + + return 0; +} + +/* + * + * The BIF timer BIFs... + */ + +BIF_RETTYPE send_after_3(BIF_ALIST_3) +{ + ErtsMonotonicTime timeout_pos; + int short_time, tres; + + tres = parse_timeout_pos(ERTS_PROC_GET_SCHDATA(BIF_P), BIF_ARG_1, NULL, + 0, &timeout_pos, &short_time); + if (tres != 0) + BIF_ERROR(BIF_P, BADARG); + + return setup_bif_timer(BIF_P, timeout_pos, short_time, + BIF_ARG_2, BIF_ARG_2, BIF_ARG_3, 0); +} + +BIF_RETTYPE send_after_4(BIF_ALIST_4) +{ + ErtsMonotonicTime timeout_pos; + Eterm accessor; + int short_time, abs, tres; + + if (!parse_bif_timer_options(BIF_ARG_4, NULL, NULL, &abs, &accessor)) + BIF_ERROR(BIF_P, BADARG); + + tres = parse_timeout_pos(ERTS_PROC_GET_SCHDATA(BIF_P), BIF_ARG_1, NULL, + abs, &timeout_pos, &short_time); + if (tres != 0) + BIF_ERROR(BIF_P, BADARG); + + return setup_bif_timer(BIF_P, timeout_pos, short_time, + BIF_ARG_2, accessor, BIF_ARG_3, 0); +} + +BIF_RETTYPE start_timer_3(BIF_ALIST_3) +{ + ErtsMonotonicTime timeout_pos; + int short_time, tres; + + tres = parse_timeout_pos(ERTS_PROC_GET_SCHDATA(BIF_P), BIF_ARG_1, NULL, + 0, &timeout_pos, &short_time); + if (tres != 0) + BIF_ERROR(BIF_P, BADARG); + + return setup_bif_timer(BIF_P, timeout_pos, short_time, + BIF_ARG_2, BIF_ARG_2, BIF_ARG_3, !0); +} + +BIF_RETTYPE start_timer_4(BIF_ALIST_4) +{ + ErtsMonotonicTime timeout_pos; + Eterm accessor; + int short_time, abs, tres; + + if (!parse_bif_timer_options(BIF_ARG_4, NULL, NULL, &abs, &accessor)) + BIF_ERROR(BIF_P, BADARG); + + tres = parse_timeout_pos(ERTS_PROC_GET_SCHDATA(BIF_P), BIF_ARG_1, NULL, + abs, &timeout_pos, &short_time); + if (tres != 0) + BIF_ERROR(BIF_P, BADARG); + + return setup_bif_timer(BIF_P, timeout_pos, short_time, + BIF_ARG_2, accessor, BIF_ARG_3, !0); +} + +BIF_RETTYPE cancel_timer_1(BIF_ALIST_1) +{ + return access_bif_timer(BIF_P, BIF_ARG_1, 1, 0, 1); +} + +BIF_RETTYPE cancel_timer_2(BIF_ALIST_2) +{ + BIF_RETTYPE ret; + int async, info; + + if (parse_bif_timer_options(BIF_ARG_2, &async, &info, NULL, NULL)) + return access_bif_timer(BIF_P, BIF_ARG_1, 1, async, info); + + ERTS_BIF_PREP_ERROR(ret, BIF_P, BADARG); + return ret; +} + +BIF_RETTYPE read_timer_1(BIF_ALIST_1) +{ + return access_bif_timer(BIF_P, BIF_ARG_1, 0, 0, 1); +} + +BIF_RETTYPE read_timer_2(BIF_ALIST_2) +{ + BIF_RETTYPE ret; + int async; + + if (parse_bif_timer_options(BIF_ARG_2, &async, NULL, NULL, NULL)) + return access_bif_timer(BIF_P, BIF_ARG_1, 0, async, 1); + + ERTS_BIF_PREP_ERROR(ret, BIF_P, BADARG); + return ret; +} + +static void +start_callback_timer(ErtsSchedulerData *esdp, + int twt, + ErtsMonotonicTime timeout_pos, + void (*callback)(void *), + void *arg) + +{ + if (twt) + create_tw_timer(esdp, ERTS_TMR_CALLBACK, NULL, + callback, arg, timeout_pos); + else + create_hl_timer(esdp, timeout_pos, 0, + ERTS_TMR_CALLBACK, NULL, + NIL, THE_NON_VALUE, NIL, + NULL, callback, arg); +} + +typedef struct { + int twt; + ErtsMonotonicTime timeout_pos; + void (*callback)(void *); + void *arg; +} ErtsStartCallbackTimerRequest; + +static void +scheduled_start_callback_timer(void *vsctr) +{ + ErtsStartCallbackTimerRequest *sctr + = (ErtsStartCallbackTimerRequest *) vsctr; + + start_callback_timer(erts_get_scheduler_data(), + sctr->twt, + sctr->timeout_pos, + sctr->callback, + sctr->arg); + + erts_free(ERTS_ALC_T_TIMER_REQUEST, vsctr); +} + +void +erts_start_timer_callback(ErtsMonotonicTime tmo, + void (*callback)(void *), + void *arg) +{ + ErtsSchedulerData *esdp; + ErtsMonotonicTime timeout_pos; + int twt; + + esdp = erts_get_scheduler_data(); + timeout_pos = get_timeout_pos(erts_get_monotonic_time(esdp), + tmo); + twt = tmo < ERTS_TIMER_WHEEL_MSEC; + + if (esdp) + start_callback_timer(esdp, + twt, + timeout_pos, + callback, + arg); + else { + ErtsStartCallbackTimerRequest *sctr; + sctr = erts_alloc(ERTS_ALC_T_TIMER_REQUEST, + sizeof(ErtsStartCallbackTimerRequest)); + sctr->twt = twt; + sctr->timeout_pos = timeout_pos; + sctr->callback = callback; + sctr->arg = arg; + erts_schedule_misc_aux_work(1, + scheduled_start_callback_timer, + (void *) sctr); + } +} + +/* + * Process and Port timer functionality. + * + * NOTE! These are only allowed to be called by a + * scheduler thread that currently is + * executing the process or port. + */ + +static ERTS_INLINE void +set_proc_timer_common(Process *c_p, ErtsSchedulerData *esdp, Sint64 tmo, + ErtsMonotonicTime timeout_pos, int short_time) +{ + void *tmr; + check_canceled_queue(esdp, esdp->timer_service); + + if (tmo == 0) + c_p->flags |= F_TIMO; + else { + + c_p->flags |= F_INSLPQUEUE; + c_p->flags &= ~F_TIMO; + + if (tmo < ERTS_TIMER_WHEEL_MSEC) + tmr = (void *) create_tw_timer(esdp, ERTS_TMR_PROC, (void *) c_p, + NULL, NULL, timeout_pos); + else + tmr = (void *) create_hl_timer(esdp, timeout_pos, short_time, + ERTS_TMR_PROC, (void *) c_p, + c_p->common.id, THE_NON_VALUE, + NIL, NULL, NULL, NULL); + erts_smp_atomic_set_relb(&c_p->common.timer, (erts_aint_t) tmr); + } +} + +int +erts_set_proc_timer_term(Process *c_p, Eterm etmo) +{ + ErtsSchedulerData *esdp = ERTS_PROC_GET_SCHDATA(c_p); + ErtsMonotonicTime tmo, timeout_pos; + int short_time, tres; + + ERTS_HLT_ASSERT(erts_smp_atomic_read_nob(&c_p->common.timer) + == ERTS_PTMR_NONE); + + tres = parse_timeout_pos(esdp, etmo, &tmo, 0, + &timeout_pos, &short_time); + if (tres != 0) + return tres; + + if ((tmo >> 32) != 0) + return 1; + + set_proc_timer_common(c_p, esdp, tmo, timeout_pos, short_time); + return 0; +} + +void +erts_set_proc_timer_uword(Process *c_p, UWord tmo) +{ + ErtsSchedulerData *esdp = ERTS_PROC_GET_SCHDATA(c_p); + + ERTS_HLT_ASSERT(erts_smp_atomic_read_nob(&c_p->common.timer) + == ERTS_PTMR_NONE); + +#ifndef ARCH_32 + ERTS_HLT_ASSERT((tmo >> 32) == (UWord) 0); +#endif + + if (tmo == 0) + c_p->flags |= F_TIMO; + else { + ErtsMonotonicTime timeout_pos; + timeout_pos = get_timeout_pos(erts_get_monotonic_time(esdp), + (ErtsMonotonicTime) tmo); + set_proc_timer_common(c_p, esdp, (ErtsMonotonicTime) tmo, + timeout_pos, + tmo < ERTS_BIF_TIMER_SHORT_TIME); + } +} + +void +erts_cancel_proc_timer(Process *c_p) +{ + erts_aint_t tval; + tval = erts_smp_atomic_xchg_acqb(&c_p->common.timer, + ERTS_PTMR_NONE); + c_p->flags &= ~(F_INSLPQUEUE|F_TIMO); + if (tval == ERTS_PTMR_NONE) + return; + if (tval == ERTS_PTMR_TIMEDOUT) { + erts_smp_atomic_set_nob(&c_p->common.timer, ERTS_PTMR_NONE); + return; + } + continue_cancel_ptimer(ERTS_PROC_GET_SCHDATA(c_p), + (ErtsTimer *) tval); +} + +void +erts_set_port_timer(Port *c_prt, Sint64 tmo) +{ + void *tmr; + ErtsSchedulerData *esdp = erts_get_scheduler_data(); + ErtsMonotonicTime timeout_pos; + + if (erts_smp_atomic_read_nob(&c_prt->common.timer) != ERTS_PTMR_NONE) + erts_cancel_port_timer(c_prt); + + check_canceled_queue(esdp, esdp->timer_service); + + timeout_pos = get_timeout_pos(erts_get_monotonic_time(esdp), tmo); + + if (tmo < ERTS_TIMER_WHEEL_MSEC) + tmr = (void *) create_tw_timer(esdp, ERTS_TMR_PORT, (void *) c_prt, + NULL, NULL, timeout_pos); + else + tmr = (void *) create_hl_timer(esdp, timeout_pos, 0, ERTS_TMR_PORT, + (void *) c_prt, c_prt->common.id, + THE_NON_VALUE, NIL, NULL, NULL, NULL); + erts_smp_atomic_set_relb(&c_prt->common.timer, (erts_aint_t) tmr); +} + +void +erts_cancel_port_timer(Port *c_prt) +{ + erts_aint_t tval; + tval = erts_smp_atomic_xchg_acqb(&c_prt->common.timer, + ERTS_PTMR_NONE); + if (tval == ERTS_PTMR_NONE) + return; + if (tval == ERTS_PTMR_TIMEDOUT) { + while (!erts_port_task_is_scheduled(&c_prt->timeout_task)) + erts_thr_yield(); + erts_port_task_abort(&c_prt->timeout_task); + erts_smp_atomic_set_nob(&c_prt->common.timer, ERTS_PTMR_NONE); + return; + } + continue_cancel_ptimer(erts_get_scheduler_data(), + (ErtsTimer *) tval); +} + +Sint64 +erts_read_port_timer(Port *c_prt) +{ + ErtsTimer *tmr; + erts_aint_t itmr; + ErtsMonotonicTime timeout_pos; + + itmr = erts_smp_atomic_read_acqb(&c_prt->common.timer); + if (itmr == ERTS_PTMR_NONE) + return (Sint64) -1; + if (itmr == ERTS_PTMR_TIMEDOUT) + return (Sint64) 0; + tmr = (ErtsTimer *) itmr; + if (tmr->head.roflgs & ERTS_TMR_ROFLG_HLT) + timeout_pos = tmr->hlt.timeout; + else + timeout_pos = tmr->twt.tw_tmr.timeout_pos; + return get_time_left(NULL, timeout_pos); +} + +/* + * Debug stuff... + */ + +typedef struct { + int to; + void *to_arg; + ErtsMonotonicTime now; +} ErtsBTMPrint; + +static void +btm_print(ErtsHLTimer *tmr, void *vbtmp) +{ + ErtsBTMPrint *btmp = (ErtsBTMPrint *) vbtmp; + ErtsMonotonicTime left; + Eterm receiver; + + if (tmr->timeout <= btmp->now) + left = 0; + left = ERTS_CLKTCKS_TO_MSEC(tmr->timeout - btmp->now); + + receiver = ((tmr->head.roflgs & ERTS_TMR_ROFLG_REG_NAME) + ? tmr->receiver.name + : tmr->receiver.proc->common.id); + + erts_print(btmp->to, btmp->to_arg, + "=timer:%T\n" + "Message: %T\n" + "Time left: %b64d\n", + receiver, + tmr->btm.message, + (Sint64) left); +} + +void +erts_print_bif_timer_info(int to, void *to_arg) +{ + ErtsBTMPrint btmp; + int six; + + if (!ERTS_IS_CRASH_DUMPING) + ERTS_INTERNAL_ERROR("Not crash dumping"); + + btmp.to = to; + btmp.to_arg = to_arg; + btmp.now = erts_get_monotonic_time(NULL); + btmp.now = ERTS_MONOTONIC_TO_CLKTCKS(btmp.now); + + for (six = 0; six < erts_no_schedulers; six++) { + ErtsHLTimerService *srv = + erts_aligned_scheduler_data[six].esd.timer_service; + btm_rbt_foreach(srv->btm_tree, btm_print, (void *) &btmp); + } +} + +typedef struct { + void (*func)(Eterm, + Eterm, + ErlHeapFragment *, + void *); + void *arg; +} ErtsBTMForeachDebug; + +static void +debug_btm_foreach(ErtsHLTimer *tmr, void *vbtmfd) +{ + if (erts_smp_atomic32_read_nob(&tmr->state) == ERTS_TMR_STATE_ACTIVE) { + ErtsBTMForeachDebug *btmfd = (ErtsBTMForeachDebug *) vbtmfd; + (*btmfd->func)(((tmr->head.roflgs & ERTS_TMR_ROFLG_REG_NAME) + ? tmr->receiver.name + : tmr->receiver.proc->common.id), + tmr->btm.message, + tmr->btm.bp, + btmfd->arg); + } +} + +void +erts_debug_bif_timer_foreach(void (*func)(Eterm, + Eterm, + ErlHeapFragment *, + void *), + void *arg) +{ + ErtsBTMForeachDebug btmfd; + int six; + + btmfd.func = func; + btmfd.arg = arg; + + if (!erts_smp_thr_progress_is_blocking()) + ERTS_INTERNAL_ERROR("Not blocking thread progress"); + + for (six = 0; six < erts_no_schedulers; six++) { + ErtsHLTimerService *srv = + erts_aligned_scheduler_data[six].esd.timer_service; + btm_rbt_foreach(srv->btm_tree, + debug_btm_foreach, + (void *) &btmfd); + } +} + +typedef struct { + void (*tclbk)(void *); + void (*func)(void *, + ErtsMonotonicTime, + void *); + void *arg; +} ErtsDebugForeachCallbackTimer; + +static void +debug_callback_timer_foreach_list(ErtsHLTimer *tmr, void *vdfct) +{ + ErtsDebugForeachCallbackTimer *dfct + = (ErtsDebugForeachCallbackTimer *) vdfct; + + if ((tmr->head.roflgs & ERTS_TMR_ROFLG_CALLBACK) + && (tmr->receiver.callback == dfct->tclbk)) + (*dfct->func)(dfct->arg, + tmr->timeout, + tmr->head.u.arg); +} + +static void +debug_callback_timer_foreach(ErtsHLTimer *tmr, void *vdfct) +{ + ErtsDebugForeachCallbackTimer *dfct + = (ErtsDebugForeachCallbackTimer *) vdfct; + + if (tmr->time.tree.same_time) + same_time_list_foreach(tmr->time.tree.same_time, + debug_callback_timer_foreach_list, + vdfct); + + if ((tmr->head.roflgs & ERTS_TMR_ROFLG_CALLBACK) + && (tmr->receiver.callback == dfct->tclbk)) + (*dfct->func)(dfct->arg, + tmr->timeout, + tmr->head.u.arg); +} + +static void +debug_tw_callback_timer(void *vdfct, + ErtsMonotonicTime timeout_pos, + void *vtwtp) +{ + ErtsTWTimer *twtp = (ErtsTWTimer *) vtwtp; + ErtsDebugForeachCallbackTimer *dfct + = (ErtsDebugForeachCallbackTimer *) vdfct; + + if (twtp->u.callback == dfct->tclbk) + (*dfct->func)(dfct->arg, + timeout_pos, + twtp->head.u.arg); +} + +void +erts_debug_callback_timer_foreach(void (*tclbk)(void *), + void (*func)(void *, + ErtsMonotonicTime, + void *), + void *arg) +{ + int six; + ErtsDebugForeachCallbackTimer dfct; + + dfct.tclbk = tclbk; + dfct.func = func; + dfct.arg = arg; + + if (!erts_smp_thr_progress_is_blocking()) + ERTS_INTERNAL_ERROR("Not blocking thread progress"); + + for (six = 0; six < erts_no_schedulers; six++) { + ErtsHLTimerService *srv = + erts_aligned_scheduler_data[six].esd.timer_service; + ErtsTimerWheel *twheel = + erts_aligned_scheduler_data[six].esd.timer_wheel; + + erts_twheel_debug_foreach(twheel, + tw_callback_timeout, + debug_tw_callback_timer, + (void *) &dfct); + + if (srv->yield.root) + debug_callback_timer_foreach(srv->yield.root, + (void *) &dfct); + + time_rbt_foreach(srv->time_tree, + debug_callback_timer_foreach, + (void *) &dfct); + } +} + +#ifdef ERTS_HLT_HARD_DEBUG + +typedef struct { + ErtsHLTimerService *srv; + int found_root; + ErtsHLTimer **rootpp; +} ErtsHdbgHLT; + +static void +st_hdbg_func(ErtsHLTimer *tmr, void *vhdbg) +{ + ErtsHdbgHLT *hdbg = (ErtsHdbgHLT *) vhdbg; + ErtsHLTimer **rootpp; + ERTS_HLT_ASSERT(tmr->time.tree.parent & ERTS_HLT_PFLG_SAME_TIME); + if (tmr->time.tree.parent == ERTS_HLT_PFLG_SAME_TIME) { + ERTS_HLT_ASSERT(tmr != *hdbg->rootpp); + } + else { + rootpp = (ErtsHLTimer **) (tmr->time.tree.parent + & ~ERTS_HLT_PFLG_SAME_TIME); + ERTS_HLT_ASSERT(rootpp == hdbg->rootpp); + ERTS_HLT_ASSERT(tmr == *rootpp); + ERTS_HLT_ASSERT(!hdbg->found_root); + hdbg->found_root = 1; + } + ERTS_HLT_ASSERT(tmr->time.tree.u.l.next->time.tree.u.l.prev == tmr); + ERTS_HLT_ASSERT(tmr->time.tree.u.l.prev->time.tree.u.l.next == tmr); + ERTS_HLT_ASSERT(btm_rbt_lookup(hdbg->srv->btm_tree, tmr->btm.refn) == tmr); +} + +static void +tt_hdbg_func(ErtsHLTimer *tmr, void *vhdbg) +{ + ErtsHdbgHLT *hdbg = (ErtsHdbgHLT *) vhdbg; + ErtsHLTimer *prnt; + ERTS_HLT_ASSERT((tmr->time.tree.parent & ERTS_HLT_PFLG_SAME_TIME) == 0); + prnt = (ErtsHLTimer *) (tmr->time.tree.parent & ~ERTS_HLT_PFLGS_MASK); + if (prnt) { + ERTS_HLT_ASSERT(prnt->time.tree.u.t.left == tmr + || prnt->time.tree.u.t.right == tmr); + } + else { + ERTS_HLT_ASSERT(!hdbg->found_root); + hdbg->found_root = 1; + ERTS_HLT_ASSERT(tmr == *hdbg->rootpp); + } + if (tmr->time.tree.u.t.left) { + prnt = (ErtsHLTimer *) (tmr->time.tree.u.t.left->time.tree.parent + & ~ERTS_HLT_PFLGS_MASK); + ERTS_HLT_ASSERT(tmr == prnt); + } + if (tmr->time.tree.u.t.right) { + prnt = (ErtsHLTimer *) (tmr->time.tree.u.t.right->time.tree.parent + & ~ERTS_HLT_PFLGS_MASK); + ERTS_HLT_ASSERT(tmr == prnt); + } + if (tmr->head.roflgs & ERTS_TMR_ROFLG_BIF_TMR) + ERTS_HLT_ASSERT(btm_rbt_lookup(hdbg->srv->btm_tree, tmr->btm.refn) == tmr); + if (tmr->time.tree.same_time) { + ErtsHdbgHLT st_hdbg; + st_hdbg.srv = hdbg->srv; + st_hdbg.found_root = 0; + st_hdbg.rootpp = &tmr->time.tree.same_time; + same_time_list_foreach(tmr->time.tree.same_time, st_hdbg_func, (void *) &st_hdbg); + ERTS_HLT_ASSERT(st_hdbg.found_root); + } +} + +static void +bt_hdbg_func(ErtsHLTimer *tmr, void *vhdbg) +{ + ErtsHdbgHLT *hdbg = (ErtsHdbgHLT *) vhdbg; + ErtsHLTimer *prnt; + ERTS_HLT_ASSERT((tmr->btm.tree.parent & ERTS_HLT_PFLG_SAME_TIME) == 0); + prnt = (ErtsHLTimer *) (tmr->btm.tree.parent & ~ERTS_HLT_PFLGS_MASK); + if (prnt) { + ERTS_HLT_ASSERT(prnt->btm.tree.left == tmr + || prnt->btm.tree.right == tmr); + } + else { + ERTS_HLT_ASSERT(!hdbg->found_root); + hdbg->found_root = 1; + ERTS_HLT_ASSERT(tmr == *hdbg->rootpp); + } + if (tmr->btm.tree.left) { + prnt = (ErtsHLTimer *) (tmr->btm.tree.left->btm.tree.parent + & ~ERTS_HLT_PFLGS_MASK); + ERTS_HLT_ASSERT(tmr == prnt); + } + if (tmr->btm.tree.right) { + prnt = (ErtsHLTimer *) (tmr->btm.tree.right->btm.tree.parent + & ~ERTS_HLT_PFLGS_MASK); + ERTS_HLT_ASSERT(tmr == prnt); + } + if (tmr->pending_timeout) { + if (tmr->pending_timeout > 0) /* container > 0 */ + ERTS_HLT_ASSERT(tmr->time.tree.parent == ERTS_HLT_PFIELD_NOT_IN_TABLE); + else { + ERTS_HLT_ASSERT(tmr->time.tree.parent != ERTS_HLT_PFIELD_NOT_IN_TABLE); + ERTS_HLT_ASSERT(tmr->time.tree.parent & ERTS_HLT_PFLG_SAME_TIME); + } + } + else { + ErtsHLTimer *ttmr = time_rbt_lookup(hdbg->srv->time_tree, tmr->timeout); + ERTS_HLT_ASSERT(ttmr); + if (ttmr != tmr) { + ERTS_HLT_ASSERT(ttmr->time.tree.same_time); + ERTS_HLT_ASSERT(tmr == same_time_list_lookup(ttmr->time.tree.same_time, tmr)); + } + } +} + +static void +hdbg_chk_srv(ErtsHLTimerService *srv) +{ + if (srv->time_tree) { + ErtsHdbgHLT hdbg; + hdbg.srv = srv; + hdbg.found_root = 0; + hdbg.rootpp = &srv->time_tree; + time_rbt_foreach(srv->time_tree, tt_hdbg_func, (void *) &hdbg); + ERTS_HLT_ASSERT(hdbg.found_root); + } + if (srv->btm_tree) { + ErtsHdbgHLT hdbg; + hdbg.srv = srv; + hdbg.found_root = 0; + hdbg.rootpp = &srv->btm_tree; + btm_rbt_foreach(srv->btm_tree, bt_hdbg_func, (void *) &hdbg); + ERTS_HLT_ASSERT(hdbg.found_root); + } +} + +#endif /* ERTS_HLT_HARD_DEBUG */ diff --git a/erts/emulator/beam/erl_hl_timer.h b/erts/emulator/beam/erl_hl_timer.h new file mode 100644 index 0000000000..0931bb8965 --- /dev/null +++ b/erts/emulator/beam/erl_hl_timer.h @@ -0,0 +1,88 @@ +/* + * %CopyrightBegin% + * + * Copyright Ericsson AB 2015. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * %CopyrightEnd% + */ + +#ifndef ERL_HL_TIMER_H__ +#define ERL_HL_TIMER_H__ + +typedef struct ErtsHLTimer_ ErtsBifTimers; +typedef struct ErtsHLTimerService_ ErtsHLTimerService; + +#include "sys.h" +#include "erl_process.h" +#define ERL_PORT_GET_PORT_TYPE_ONLY__ +#include "erl_port.h" +#undef ERL_PORT_GET_PORT_TYPE_ONLY__ +#include "erl_message.h" +#include "erl_alloc_types.h" + +#define ERTS_PTMR_NONE ((erts_aint_t) NULL) +#define ERTS_PTMR_TIMEDOUT (ERTS_PTMR_NONE + ((erts_aint_t) 1)) + +#define ERTS_PTMR_INIT(P) \ + erts_smp_atomic_init_nob(&(P)->common.timer, ERTS_PTMR_NONE) +#define ERTS_PTMR_IS_SET(P) \ + (ERTS_PTMR_NONE != erts_smp_atomic_read_nob(&(P)->common.timer)) +#define ERTS_PTMR_IS_TIMED_OUT(P) \ + (ERTS_PTMR_TIMEDOUT == erts_smp_atomic_read_nob(&(P)->common.timer)) + +#define ERTS_PTMR_CLEAR(P) \ + do { \ + ASSERT(ERTS_PTMR_IS_TIMED_OUT((P))); \ + erts_smp_atomic_set_nob(&(P)->common.timer, \ + ERTS_PTMR_NONE); \ + } while (0) + +size_t erts_timer_type_size(ErtsAlcType_t type); +int erts_set_proc_timer_term(Process *, Eterm); +void erts_set_proc_timer_uword(Process *, UWord); +void erts_cancel_proc_timer(Process *); +void erts_set_port_timer(Port *, Sint64); +void erts_cancel_port_timer(Port *); +Sint64 erts_read_port_timer(Port *); +int erts_cancel_bif_timers(Process *, ErtsBifTimers *, void **); +int erts_detach_accessor_bif_timers(Process *, ErtsBifTimers *, void **); +ErtsHLTimerService *erts_create_timer_service(void); +void erts_hl_timer_init(void); +void erts_start_timer_callback(ErtsMonotonicTime, + void (*)(void *), + void *); +#ifdef ERTS_SMP +void +erts_handle_canceled_timers(void *vesdp, + int *need_thr_progress, + ErtsThrPrgrVal *thr_prgr_p, + int *need_more_work); +#endif + +Uint erts_bif_timer_memory_size(void); +void erts_print_bif_timer_info(int to, void *to_arg); + +void erts_debug_bif_timer_foreach(void (*func)(Eterm, + Eterm, + ErlHeapFragment *, + void *), + void *arg); +void +erts_debug_callback_timer_foreach(void (*tclbk)(void *), + void (*func)(void *, + ErtsMonotonicTime, + void *), + void *arg); +#endif /* ERL_HL_TIMER_H__ */ diff --git a/erts/emulator/beam/erl_init.c b/erts/emulator/beam/erl_init.c index e7e4030900..e729574ec7 100644 --- a/erts/emulator/beam/erl_init.c +++ b/erts/emulator/beam/erl_init.c @@ -3,16 +3,17 @@ * * Copyright Ericsson AB 1997-2013. All Rights Reserved. * - * The contents of this file are subject to the Erlang Public License, - * Version 1.1, (the "License"); you may not use this file except in - * compliance with the License. You should have received a copy of the - * Erlang Public License along with this software. If not, it can be - * retrieved online at http://www.erlang.org/. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Software distributed under the License is distributed on an "AS IS" - * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See - * the License for the specific language governing rights and limitations - * under the License. + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. * * %CopyrightEnd% */ @@ -35,7 +36,7 @@ #include "dist.h" #include "erl_mseg.h" #include "erl_threads.h" -#include "erl_bif_timer.h" +#include "erl_hl_timer.h" #include "erl_instrument.h" #include "erl_printf_term.h" #include "erl_misc_utils.h" @@ -45,6 +46,9 @@ #include "erl_thr_queue.h" #include "erl_async.h" #include "erl_ptab.h" +#include "erl_bif_unique.h" +#define ERTS_WANT_TIMER_WHEEL_API +#include "erl_time.h" #ifdef HIPE #include "hipe_mode_switch.h" /* for hipe_mode_switch_init() */ @@ -117,6 +121,8 @@ const int etp_big_endian = 1; #else const int etp_big_endian = 0; #endif +const Eterm etp_the_non_value = THE_NON_VALUE; + /* * Note about VxWorks: All variables must be initialized by executable code, * not by an initializer. Otherwise a new instance of the emulator will @@ -131,8 +137,14 @@ extern void ConWaitForExit(void); static void erl_init(int ncpu, int proc_tab_sz, + int legacy_proc_tab, int port_tab_sz, - int port_tab_sz_ignore_files); + int port_tab_sz_ignore_files, + int legacy_port_tab, + int time_correction, + ErtsTimeWarpMode time_warp_mode, + int node_tab_delete_delay, + ErtsDbSpinCount db_spin_count); static erts_atomic_t exiting; @@ -159,9 +171,6 @@ int H_MIN_SIZE; /* The minimum heap grain */ int BIN_VH_MIN_SIZE; /* The minimum binary virtual*/ Uint32 erts_debug_flags; /* Debug flags. */ -#ifdef ERTS_OPCODE_COUNTER_SUPPORT -int count_instructions; -#endif int erts_backtrace_depth; /* How many functions to show in a backtrace * in error codes. */ @@ -176,17 +185,20 @@ int erts_compat_rel; static int no_schedulers; static int no_schedulers_online; +#ifdef ERTS_DIRTY_SCHEDULERS +static int no_dirty_cpu_schedulers; +static int no_dirty_cpu_schedulers_online; +static int no_dirty_io_schedulers; +#endif #ifdef DEBUG Uint32 verbose; /* See erl_debug.h for information about verbose */ #endif -int erts_disable_tolerant_timeofday; /* Time correction can be disabled it is - * not and/or it is too slow. - */ - int erts_atom_table_size = ATOM_LIMIT; /* Maximum number of atoms */ +int erts_pd_initial_size = 10; + int erts_modified_timing_level; int erts_no_crash_dump = 0; /* Use -d to suppress crash dump. */ @@ -257,11 +269,24 @@ this_rel_num(void) i++; this_rel = atoi(&this_rel_str[i]); if (this_rel < 1) - erl_exit(-1, "Unexpected ERLANG_OTP_RELEASE format\n"); + erts_exit(1, "Unexpected ERLANG_OTP_RELEASE format\n"); } return this_rel; } +static ERTS_INLINE void +set_default_time_adj(int *time_correction_p, ErtsTimeWarpMode *time_warp_mode_p) +{ + *time_correction_p = 1; + *time_warp_mode_p = ERTS_NO_TIME_WARP_MODE; + if (!erts_check_time_adj_support(*time_correction_p, + *time_warp_mode_p)) { + *time_correction_p = 0; + ASSERT(erts_check_time_adj_support(*time_correction_p, + *time_warp_mode_p)); + } +} + /* * Common error printout function, all error messages * that don't go to the error logger go through here. @@ -277,37 +302,63 @@ static int early_init(int *argc, char **argv); void erts_short_init(void) { - int ncpu = early_init(NULL, NULL); + + int ncpu; + int time_correction; + ErtsTimeWarpMode time_warp_mode; + + set_default_time_adj(&time_correction, + &time_warp_mode); + ncpu = early_init(NULL, NULL); erl_init(ncpu, ERTS_DEFAULT_MAX_PROCESSES, + 0, ERTS_DEFAULT_MAX_PORTS, - 0); + 0, + 0, + time_correction, + time_warp_mode, + ERTS_NODE_TAB_DELAY_GC_DEFAULT, + ERTS_DB_SPNCNT_NORMAL); erts_initialized = 1; } static void erl_init(int ncpu, int proc_tab_sz, + int legacy_proc_tab, int port_tab_sz, - int port_tab_sz_ignore_files) + int port_tab_sz_ignore_files, + int legacy_port_tab, + int time_correction, + ErtsTimeWarpMode time_warp_mode, + int node_tab_delete_delay, + ErtsDbSpinCount db_spin_count) { init_benchmarking(); + erts_bif_unique_init(); erts_init_monitors(); - erts_init_gc(); - erts_init_time(); + erts_init_time(time_correction, time_warp_mode); erts_init_sys_common_misc(); - erts_init_process(ncpu, proc_tab_sz); + erts_init_process(ncpu, proc_tab_sz, legacy_proc_tab); erts_init_scheduling(no_schedulers, - no_schedulers_online); + no_schedulers_online +#ifdef ERTS_DIRTY_SCHEDULERS + , no_dirty_cpu_schedulers, + no_dirty_cpu_schedulers_online, + no_dirty_io_schedulers +#endif + ); + erts_late_init_time_sup(); erts_init_cpu_topology(); /* Must be after init_scheduling */ + erts_init_gc(); /* Must be after init_scheduling */ erts_alloc_late_init(); H_MIN_SIZE = erts_next_heap_size(H_MIN_SIZE, 0); BIN_VH_MIN_SIZE = erts_next_heap_size(BIN_VH_MIN_SIZE, 0); erts_init_trace(); - erts_init_binary(); erts_init_bits(); erts_code_ix_init(); erts_init_fun_table(); @@ -320,20 +371,22 @@ erl_init(int ncpu, erts_ddll_init(); init_emulator(); erts_ptab_init(); /* Must be after init_emulator() */ + erts_init_binary(); /* Must be after init_emulator() */ erts_bp_init(); - init_db(); /* Must be after init_emulator */ - erts_bif_timer_init(); - erts_init_node_tables(); + init_db(db_spin_count); /* Must be after init_emulator */ + erts_init_node_tables(node_tab_delete_delay); init_dist(); erl_drv_thr_init(); erts_init_async(); - erts_init_io(port_tab_sz, port_tab_sz_ignore_files); + erts_init_io(port_tab_sz, port_tab_sz_ignore_files, legacy_port_tab); init_load(); erts_init_bif(); erts_init_bif_chksum(); erts_init_bif_binary(); erts_init_bif_re(); erts_init_unicode(); /* after RE to get access to PCRE unicode */ + erts_init_external(); + erts_init_map(); erts_delay_trap = erts_export_put(am_erlang, am_delay_trap, 2); erts_late_init_process(); #if HAVE_ERTS_MSEG @@ -347,12 +400,13 @@ erl_init(int ncpu, erl_nif_init(); } -static void +static Eterm erl_first_process_otp(char* modname, void* code, unsigned size, int argc, char** argv) { int i; Eterm start_mod; Eterm args; + Eterm res; Eterm* hp; Process parent; ErlSpawnOpts so; @@ -361,7 +415,7 @@ erl_first_process_otp(char* modname, void* code, unsigned size, int argc, char** start_mod = erts_atom_put((byte *) modname, sys_strlen(modname), ERTS_ATOM_ENC_LATIN1, 1); if (erts_find_function(start_mod, am_start, 2, erts_active_code_ix()) == NULL) { - erl_exit(5, "No function %s:start/2\n", modname); + erts_exit(ERTS_ERROR_EXIT, "No function %s:start/2\n", modname); } /* @@ -382,10 +436,11 @@ erl_first_process_otp(char* modname, void* code, unsigned size, int argc, char** hp += 2; args = CONS(hp, env, args); - so.flags = 0; - (void) erl_create_process(&parent, start_mod, am_start, args, &so); + so.flags = SPO_SYSTEM_PROC; + res = erl_create_process(&parent, start_mod, am_start, args, &so); erts_smp_proc_unlock(&parent, ERTS_PROC_LOCK_MAIN); erts_cleanup_empty_process(&parent); + return res; } Eterm @@ -457,12 +512,12 @@ load_preloaded(void) length = preload_p[i].size; module_name = erts_atom_put((byte *) name, sys_strlen(name), ERTS_ATOM_ENC_LATIN1, 1); if ((code = sys_preload_begin(&preload_p[i])) == 0) - erl_exit(1, "Failed to find preloaded code for module %s\n", + erts_exit(ERTS_ERROR_EXIT, "Failed to find preloaded code for module %s\n", name); res = erts_preload_module(NULL, 0, NIL, &module_name, code, length); sys_preload_end(&preload_p[i]); if (res != NIL) - erl_exit(1,"Failed loading preloaded module %s (%T)\n", + erts_exit(ERTS_ERROR_EXIT,"Failed loading preloaded module %s (%T)\n", name, res); i++; } @@ -477,92 +532,123 @@ void erts_usage(void) /* erts_fprintf(stderr, "-# number set the number of items to be used in traces etc\n"); */ - erts_fprintf(stderr, "-a size suggested stack size in kilo words for threads\n"); - erts_fprintf(stderr, " in the async-thread pool, valid range is [%d-%d]\n", + erts_fprintf(stderr, "-a size suggested stack size in kilo words for threads\n"); + erts_fprintf(stderr, " in the async-thread pool, valid range is [%d-%d]\n", ERTS_ASYNC_THREAD_MIN_STACK_SIZE, ERTS_ASYNC_THREAD_MAX_STACK_SIZE); - erts_fprintf(stderr, "-A number set number of threads in async thread pool,\n"); - erts_fprintf(stderr, " valid range is [0-%d]\n", + erts_fprintf(stderr, "-A number set number of threads in async thread pool,\n"); + erts_fprintf(stderr, " valid range is [0-%d]\n", ERTS_MAX_NO_OF_ASYNC_THREADS); - erts_fprintf(stderr, "-B[c|d|i] c to have Ctrl-c interrupt the Erlang shell,\n"); - erts_fprintf(stderr, " d (or no extra option) to disable the break\n"); - erts_fprintf(stderr, " handler, i to ignore break signals\n"); + erts_fprintf(stderr, "-B[c|d|i] c to have Ctrl-c interrupt the Erlang shell,\n"); + erts_fprintf(stderr, " d (or no extra option) to disable the break\n"); + erts_fprintf(stderr, " handler, i to ignore break signals\n"); /* erts_fprintf(stderr, "-b func set the boot function (default boot)\n"); */ - erts_fprintf(stderr, "-c disable continuous date/time correction with\n"); - erts_fprintf(stderr, " respect to uptime\n"); - - erts_fprintf(stderr, "-d don't write a crash dump for internally detected errors\n"); - erts_fprintf(stderr, " (halt(String) will still produce a crash dump)\n"); - erts_fprintf(stderr, "-fn[u|a|l] Control how filenames are interpreted\n"); - erts_fprintf(stderr, "-hms size set minimum heap size in words (default %d)\n", + erts_fprintf(stderr, "-c bool enable or disable time correction\n"); + erts_fprintf(stderr, "-C mode set time warp mode; valid modes are:\n"); + erts_fprintf(stderr, " no_time_warp|single_time_warp|multi_time_warp\n"); + erts_fprintf(stderr, "-d don't write a crash dump for internally detected errors\n"); + erts_fprintf(stderr, " (halt(String) will still produce a crash dump)\n"); + erts_fprintf(stderr, "-fn[u|a|l] Control how filenames are interpreted\n"); + erts_fprintf(stderr, "-hms size set minimum heap size in words (default %d)\n", H_DEFAULT_SIZE); - erts_fprintf(stderr, "-hmbs size set minimum binary virtual heap size in words (default %d)\n", + erts_fprintf(stderr, "-hmbs size set minimum binary virtual heap size in words (default %d)\n", VH_DEFAULT_SIZE); + erts_fprintf(stderr, "-hpds size initial process dictionary size (default %d)\n", + erts_pd_initial_size); /* erts_fprintf(stderr, "-i module set the boot module (default init)\n"); */ - erts_fprintf(stderr, "-K boolean enable or disable kernel poll\n"); - erts_fprintf(stderr, "-n[s|a|d] Control behavior of signals to ports\n"); - erts_fprintf(stderr, " Note that this flag is deprecated!\n"); - erts_fprintf(stderr, "-M<X> <Y> memory allocator switches,\n"); - erts_fprintf(stderr, " see the erts_alloc(3) documentation for more info.\n"); - erts_fprintf(stderr, "-pc <set> Control what characters are considered printable (default latin1)\n"); - erts_fprintf(stderr, "-P number set maximum number of processes on this node,\n"); - erts_fprintf(stderr, " valid range is [%d-%d]\n", + erts_fprintf(stderr, "-K boolean enable or disable kernel poll\n"); + erts_fprintf(stderr, "-n[s|a|d] Control behavior of signals to ports\n"); + erts_fprintf(stderr, " Note that this flag is deprecated!\n"); + erts_fprintf(stderr, "-M<X> <Y> memory allocator switches,\n"); + erts_fprintf(stderr, " see the erts_alloc(3) documentation for more info.\n"); + erts_fprintf(stderr, "-pc <set> Control what characters are considered printable (default latin1)\n"); + erts_fprintf(stderr, "-P number set maximum number of processes on this node,\n"); + erts_fprintf(stderr, " valid range is [%d-%d]\n", ERTS_MIN_PROCESSES, ERTS_MAX_PROCESSES); - erts_fprintf(stderr, "-Q number set maximum number of ports on this node,\n"); - erts_fprintf(stderr, " valid range is [%d-%d]\n", + erts_fprintf(stderr, "-Q number set maximum number of ports on this node,\n"); + erts_fprintf(stderr, " valid range is [%d-%d]\n", ERTS_MIN_PORTS, ERTS_MAX_PORTS); - erts_fprintf(stderr, "-R number set compatibility release number,\n"); - erts_fprintf(stderr, " valid range [%d-%d]\n", + erts_fprintf(stderr, "-R number set compatibility release number,\n"); + erts_fprintf(stderr, " valid range [%d-%d]\n", this_rel-2, this_rel); - erts_fprintf(stderr, "-r force ets memory block to be moved on realloc\n"); - erts_fprintf(stderr, "-rg amount set reader groups limit\n"); - erts_fprintf(stderr, "-sbt type set scheduler bind type, valid types are:\n"); - erts_fprintf(stderr, "-stbt type u|ns|ts|ps|s|nnts|nnps|tnnps|db\n"); - erts_fprintf(stderr, "-sbwt val set scheduler busy wait threshold, valid values are:\n"); - erts_fprintf(stderr, " none|very_short|short|medium|long|very_long.\n"); - erts_fprintf(stderr, "-scl bool enable/disable compaction of scheduler load,\n"); - erts_fprintf(stderr, " see the erl(1) documentation for more info.\n"); - erts_fprintf(stderr, "-sct cput set cpu topology,\n"); - erts_fprintf(stderr, " see the erl(1) documentation for more info.\n"); - erts_fprintf(stderr, "-sws val set scheduler wakeup strategy, valid values are:\n"); - erts_fprintf(stderr, " default|legacy.\n"); - erts_fprintf(stderr, "-swt val set scheduler wakeup threshold, valid values are:\n"); - erts_fprintf(stderr, " very_low|low|medium|high|very_high.\n"); - erts_fprintf(stderr, "-sss size suggested stack size in kilo words for scheduler threads,\n"); - erts_fprintf(stderr, " valid range is [%d-%d]\n", + erts_fprintf(stderr, "-r force ets memory block to be moved on realloc\n"); + erts_fprintf(stderr, "-rg amount set reader groups limit\n"); + erts_fprintf(stderr, "-sbt type set scheduler bind type, valid types are:\n"); + erts_fprintf(stderr, "-stbt type u|ns|ts|ps|s|nnts|nnps|tnnps|db\n"); + erts_fprintf(stderr, "-sbwt val set scheduler busy wait threshold, valid values are:\n"); + erts_fprintf(stderr, " none|very_short|short|medium|long|very_long.\n"); + erts_fprintf(stderr, "-scl bool enable/disable compaction of scheduler load,\n"); + erts_fprintf(stderr, " see the erl(1) documentation for more info.\n"); + erts_fprintf(stderr, "-sct cput set cpu topology,\n"); + erts_fprintf(stderr, " see the erl(1) documentation for more info.\n"); + erts_fprintf(stderr, "-secio bool enable/disable eager check I/O scheduling,\n"); + erts_fprintf(stderr, " see the erl(1) documentation for more info.\n"); +#if ERTS_HAVE_SCHED_UTIL_BALANCING_SUPPORT_OPT + erts_fprintf(stderr, "-sub bool enable/disable scheduler utilization balancing,\n"); +#else + erts_fprintf(stderr, "-sub false disable scheduler utilization balancing,\n"); +#endif + erts_fprintf(stderr, " see the erl(1) documentation for more info.\n"); + erts_fprintf(stderr, "-sws val set scheduler wakeup strategy, valid values are:\n"); + erts_fprintf(stderr, " default|legacy.\n"); + erts_fprintf(stderr, "-swct val set scheduler wake cleanup threshold, valid values are:\n"); + erts_fprintf(stderr, " very_lazy|lazy|medium|eager|very_eager.\n"); + erts_fprintf(stderr, "-swt val set scheduler wakeup threshold, valid values are:\n"); + erts_fprintf(stderr, " very_low|low|medium|high|very_high.\n"); + erts_fprintf(stderr, "-sss size suggested stack size in kilo words for scheduler threads,\n"); + erts_fprintf(stderr, " valid range is [%d-%d]\n", ERTS_SCHED_THREAD_MIN_STACK_SIZE, ERTS_SCHED_THREAD_MAX_STACK_SIZE); - erts_fprintf(stderr, "-spp Bool set port parallelism scheduling hint\n"); - erts_fprintf(stderr, "-S n1:n2 set number of schedulers (n1), and number of\n"); - erts_fprintf(stderr, " schedulers online (n2), valid range for both\n"); - erts_fprintf(stderr, " numbers are [1-%d]\n", + erts_fprintf(stderr, "-spp Bool set port parallelism scheduling hint\n"); + erts_fprintf(stderr, "-S n1:n2 set number of schedulers (n1), and number of\n"); + erts_fprintf(stderr, " schedulers online (n2), maximum for both\n"); + erts_fprintf(stderr, " numbers is %d\n", ERTS_MAX_NO_OF_SCHEDULERS); - erts_fprintf(stderr, "-t size set the maximum number of atoms the " - "emulator can handle\n"); - erts_fprintf(stderr, " valid range is [%d-%d]\n", + erts_fprintf(stderr, "-SP p1:p2 specify schedulers (p1) and schedulers online (p2)\n"); + erts_fprintf(stderr, " as percentages of logical processors configured and logical\n"); + erts_fprintf(stderr, " processors available, respectively\n"); +#ifdef ERTS_DIRTY_SCHEDULERS + erts_fprintf(stderr, "-SDcpu n1:n2 set number of dirty CPU schedulers (n1), and number of\n"); + erts_fprintf(stderr, " dirty CPU schedulers online (n2), valid range for both\n"); + erts_fprintf(stderr, " numbers is [1-%d], and n2 must be less than or equal to n1\n", + ERTS_MAX_NO_OF_DIRTY_CPU_SCHEDULERS); + erts_fprintf(stderr, "-SDPcpu p1:p2 specify dirty CPU schedulers (p1) and dirty CPU schedulers\n"); + erts_fprintf(stderr, " online (p2) as percentages of logical processors configured\n"); + erts_fprintf(stderr, " and logical processors available, respectively\n"); + erts_fprintf(stderr, "-SDio n set number of dirty I/O schedulers, valid range is [0-%d]\n", + ERTS_MAX_NO_OF_DIRTY_IO_SCHEDULERS); +#endif + erts_fprintf(stderr, "-t size set the maximum number of atoms the emulator can handle\n"); + erts_fprintf(stderr, " valid range is [%d-%d]\n", MIN_ATOM_TABLE_SIZE, MAX_ATOM_TABLE_SIZE); - erts_fprintf(stderr, "-T number set modified timing level,\n"); - erts_fprintf(stderr, " valid range is [0-%d]\n", + erts_fprintf(stderr, "-T number set modified timing level, valid range is [0-%d]\n", ERTS_MODIFIED_TIMING_LEVELS-1); - erts_fprintf(stderr, "-V print Erlang version\n"); - - erts_fprintf(stderr, "-v turn on chatty mode (GCs will be reported etc)\n"); - - erts_fprintf(stderr, "-W<i|w> set error logger warnings mapping,\n"); - erts_fprintf(stderr, " see error_logger documentation for details\n"); - erts_fprintf(stderr, "-zdbbl size set the distribution buffer busy limit in kilobytes\n"); - erts_fprintf(stderr, " valid range is [1-%d]\n", INT_MAX/1024); + erts_fprintf(stderr, "-V print Erlang version\n"); + + erts_fprintf(stderr, "-v turn on chatty mode (GCs will be reported etc)\n"); + + erts_fprintf(stderr, "-W<i|w|e> set error logger warnings mapping,\n"); + erts_fprintf(stderr, " see error_logger documentation for details\n"); + erts_fprintf(stderr, "-zdbbl size set the distribution buffer busy limit in kilobytes\n"); + erts_fprintf(stderr, " valid range is [1-%d]\n", INT_MAX/1024); + erts_fprintf(stderr, "-zdntgc time set delayed node table gc in seconds\n"); + erts_fprintf(stderr, " valid values are infinity or intergers in the range [0-%d]\n", + ERTS_NODE_TAB_DELAY_GC_MAX); +#if 0 + erts_fprintf(stderr, "-zebwt val set ets busy wait threshold, valid values are:\n"); + erts_fprintf(stderr, " none|very_short|short|medium|long|very_long|extremely_long\n"); +#endif erts_fprintf(stderr, "\n"); erts_fprintf(stderr, "Note that if the emulator is started with erlexec (typically\n"); erts_fprintf(stderr, "from the erl script), these flags should be specified with +.\n"); erts_fprintf(stderr, "\n\n"); - erl_exit(-1, ""); + erts_exit(1, ""); } #ifdef USE_THREADS @@ -622,15 +708,25 @@ early_init(int *argc, char **argv) /* int ncpuavail; int schdlrs; int schdlrs_onln; + int schdlrs_percentage = 100; + int schdlrs_onln_percentage = 100; int max_main_threads; +#ifdef ERTS_DIRTY_SCHEDULERS + int dirty_cpu_scheds; + int dirty_cpu_scheds_online; + int dirty_cpu_scheds_pctg = 100; + int dirty_cpu_scheds_onln_pctg = 100; + int dirty_io_scheds; +#endif int max_reader_groups; int reader_groups; char envbuf[21]; /* enough for any 64-bit integer */ size_t envbufsz; + erts_save_emu_args(*argc, argv); + erts_sched_compact_load = 1; erts_printf_eterm_func = erts_printf_term; - erts_disable_tolerant_timeofday = 0; display_items = 200; erts_backtrace_depth = DEFAULT_BACKTRACE_SIZE; erts_async_max_threads = ERTS_DEFAULT_NO_ASYNC_THREADS; @@ -671,7 +767,7 @@ early_init(int *argc, char **argv) /* #endif #ifdef ERTS_SMP erts_smp_atomic32_init_nob(&erts_writing_erl_crash_dump, 0L); - erts_tsd_key_create(&erts_is_crash_dumping_key); + erts_tsd_key_create(&erts_is_crash_dumping_key,"erts_is_crash_dumping_key"); #else erts_writing_erl_crash_dump = 0; #endif @@ -696,6 +792,12 @@ early_init(int *argc, char **argv) /* schdlrs = no_schedulers; schdlrs_onln = no_schedulers_online; +#ifdef ERTS_DIRTY_SCHEDULERS + dirty_cpu_scheds = no_schedulers; + dirty_cpu_scheds_online = no_schedulers_online; + dirty_io_scheds = 10; +#endif + envbufsz = sizeof(envbuf); /* erts_sys_getenv(_raw)() not initialized yet; need erts_sys_getenv__() */ @@ -736,7 +838,7 @@ early_init(int *argc, char **argv) /* case 'A': { /* set number of threads in thread pool */ char *arg = get_arg(argv[i]+2, argv[i+1], &i); - if (((erts_async_max_threads = atoi(arg)) < 0) || + if (((erts_async_max_threads = atoi(arg)) < ERTS_MIN_NO_OF_ASYNC_THREADS) || (erts_async_max_threads > ERTS_MAX_NO_OF_ASYNC_THREADS)) { erts_fprintf(stderr, "bad number of async threads %s\n", @@ -747,63 +849,257 @@ early_init(int *argc, char **argv) /* } break; } - case 'S' : { - int tot, onln; - char *arg = get_arg(argv[i]+2, argv[i+1], &i); - switch (sscanf(arg, "%d:%d", &tot, &onln)) { - case 0: - switch (sscanf(arg, ":%d", &onln)) { + case 'S' : + if (argv[i][2] == 'P') { + int ptot, ponln; + char *arg = get_arg(argv[i]+3, argv[i+1], &i); + switch (sscanf(arg, "%d:%d", &ptot, &ponln)) { + case 0: + switch (sscanf(arg, ":%d", &ponln)) { + case 1: + if (ponln < 0) + goto bad_SP; + ptot = 100; + goto chk_SP; + default: + goto bad_SP; + } case 1: - tot = no_schedulers; - goto chk_S; + if (ptot < 0) + goto bad_SP; + ponln = ptot < 100 ? ptot : 100; + goto chk_SP; + case 2: + if (ptot < 0 || ponln < 0) + goto bad_SP; + chk_SP: + schdlrs_percentage = ptot; + schdlrs_onln_percentage = ponln; + break; default: - goto bad_S; - } - case 1: - onln = tot < schdlrs_onln ? tot : schdlrs_onln; - case 2: - chk_S: - if (tot > 0) - schdlrs = tot; - else - schdlrs = no_schedulers + tot; - if (onln > 0) - schdlrs_onln = onln; - else - schdlrs_onln = no_schedulers_online + onln; - if (schdlrs < 1 || ERTS_MAX_NO_OF_SCHEDULERS < schdlrs) { + bad_SP: + erts_fprintf(stderr, + "bad schedulers percentage specifier %s\n", + arg); + erts_usage(); + break; + } + + VERBOSE(DEBUG_SYSTEM, + ("using %d:%d scheduler percentages\n", + schdlrs_percentage, schdlrs_onln_percentage)); + } +#ifdef ERTS_DIRTY_SCHEDULERS + else if (argv[i][2] == 'D') { + char *arg; + char *type = argv[i]+3; + if (strncmp(type, "Pcpu", 4) == 0) { + int ptot, ponln; + arg = get_arg(argv[i]+7, argv[i+1], &i); + switch (sscanf(arg, "%d:%d", &ptot, &ponln)) { + case 0: + switch (sscanf(arg, ":%d", &ponln)) { + case 1: + if (ponln < 0) + goto bad_SDPcpu; + ptot = 100; + goto chk_SDPcpu; + default: + goto bad_SDPcpu; + } + case 1: + if (ptot < 0) + goto bad_SDPcpu; + ponln = ptot < 100 ? ptot : 100; + goto chk_SDPcpu; + case 2: + if (ptot < 0 || ponln < 0) + goto bad_SDPcpu; + chk_SDPcpu: + dirty_cpu_scheds_pctg = ptot; + dirty_cpu_scheds_onln_pctg = ponln; + break; + default: + bad_SDPcpu: + erts_fprintf(stderr, + "bad dirty CPU schedulers percentage specifier %s\n", + arg); + erts_usage(); + break; + } + VERBOSE(DEBUG_SYSTEM, + ("using %d:%d dirty CPU scheduler percentages\n", + dirty_cpu_scheds_pctg, dirty_cpu_scheds_onln_pctg)); + } else if (strncmp(type, "cpu", 3) == 0) { + int tot, onln; + arg = get_arg(argv[i]+6, argv[i+1], &i); + switch (sscanf(arg, "%d:%d", &tot, &onln)) { + case 0: + switch (sscanf(arg, ":%d", &onln)) { + case 1: + tot = no_schedulers; + goto chk_SDcpu; + default: + goto bad_SDcpu; + } + case 1: + onln = tot < dirty_cpu_scheds_online ? + tot : dirty_cpu_scheds_online; + case 2: + chk_SDcpu: + if (tot > 0) + dirty_cpu_scheds = tot; + else + dirty_cpu_scheds = no_schedulers + tot; + if (onln > 0) + dirty_cpu_scheds_online = onln; + else + dirty_cpu_scheds_online = no_schedulers_online + onln; + if (dirty_cpu_scheds < 1 || + ERTS_MAX_NO_OF_DIRTY_CPU_SCHEDULERS < dirty_cpu_scheds) { + erts_fprintf(stderr, + "bad amount of dirty CPU schedulers %d\n", + tot); + erts_usage(); + } + if (dirty_cpu_scheds_online < 1 || + dirty_cpu_scheds < dirty_cpu_scheds_online) { + erts_fprintf(stderr, + "bad amount of dirty CPU schedulers online %d " + "(total amount of dirty CPU schedulers %d)\n", + dirty_cpu_scheds_online, dirty_cpu_scheds); + erts_usage(); + } + break; + default: + bad_SDcpu: + erts_fprintf(stderr, + "bad amount of dirty CPU schedulers %s\n", + arg); + erts_usage(); + break; + } + VERBOSE(DEBUG_SYSTEM, + ("using %d:%d dirty CPU scheduler(s)\n", tot, onln)); + } else if (strncmp(type, "io", 2) == 0) { + arg = get_arg(argv[i]+5, argv[i+1], &i); + dirty_io_scheds = atoi(arg); + if (dirty_io_scheds < 0 || + dirty_io_scheds > ERTS_MAX_NO_OF_DIRTY_IO_SCHEDULERS) { + erts_fprintf(stderr, + "bad number of dirty I/O schedulers %s\n", + arg); + erts_usage(); + } + VERBOSE(DEBUG_SYSTEM, + ("using %d dirty I/O scheduler(s)\n", dirty_io_scheds)); + } else { erts_fprintf(stderr, - "bad amount of schedulers %d\n", - tot); + "bad or missing dirty scheduler specifier: %s\n", + argv[i]); erts_usage(); + break; } - if (schdlrs_onln < 1 || schdlrs < schdlrs_onln) { + } +#endif + else { + int tot, onln; + char *arg = get_arg(argv[i]+2, argv[i+1], &i); + switch (sscanf(arg, "%d:%d", &tot, &onln)) { + case 0: + switch (sscanf(arg, ":%d", &onln)) { + case 1: + tot = no_schedulers; + goto chk_S; + default: + goto bad_S; + } + case 1: + onln = tot < schdlrs_onln ? tot : schdlrs_onln; + case 2: + chk_S: + if (tot > 0) + schdlrs = tot; + else + schdlrs = no_schedulers + tot; + if (onln > 0) + schdlrs_onln = onln; + else + schdlrs_onln = no_schedulers_online + onln; + if (schdlrs < 1 || ERTS_MAX_NO_OF_SCHEDULERS < schdlrs) { + erts_fprintf(stderr, + "bad amount of schedulers %d\n", + tot); + erts_usage(); + } + if (schdlrs_onln < 1 || schdlrs < schdlrs_onln) { + erts_fprintf(stderr, + "bad amount of schedulers online %d " + "(total amount of schedulers %d)\n", + schdlrs_onln, schdlrs); + erts_usage(); + } + break; + default: + bad_S: erts_fprintf(stderr, - "bad amount of schedulers online %d " - "(total amount of schedulers %d)\n", - schdlrs_onln, schdlrs); + "bad amount of schedulers %s\n", + arg); erts_usage(); + break; } - break; - default: - bad_S: - erts_fprintf(stderr, - "bad amount of schedulers %s\n", - arg); - erts_usage(); - break; - } - VERBOSE(DEBUG_SYSTEM, - ("using %d:%d scheduler(s)\n", tot, onln)); - break; - } + VERBOSE(DEBUG_SYSTEM, + ("using %d:%d scheduler(s)\n", tot, onln)); + } + break; default: break; } } i++; } + +#ifdef ERTS_SMP + /* apply any scheduler percentages */ + if (schdlrs_percentage != 100 || schdlrs_onln_percentage != 100) { + schdlrs = schdlrs * schdlrs_percentage / 100; + schdlrs_onln = schdlrs_onln * schdlrs_onln_percentage / 100; + if (schdlrs < 1) + schdlrs = 1; + if (ERTS_MAX_NO_OF_SCHEDULERS < schdlrs) { + erts_fprintf(stderr, + "bad schedulers percentage %d " + "(total amount of schedulers %d)\n", + schdlrs_percentage, schdlrs); + erts_usage(); + } + if (schdlrs_onln < 1) + schdlrs_onln = 1; + if (schdlrs < schdlrs_onln) { + erts_fprintf(stderr, + "bad schedulers online percentage %d " + "(total amount of schedulers %d, online %d)\n", + schdlrs_onln_percentage, schdlrs, schdlrs_onln); + erts_usage(); + } + } +#else + /* Silence gcc warnings */ + (void)schdlrs_percentage; + (void)schdlrs_onln_percentage; +#endif +#ifdef ERTS_DIRTY_SCHEDULERS + /* apply any dirty scheduler precentages */ + if (dirty_cpu_scheds_pctg != 100 || dirty_cpu_scheds_onln_pctg != 100) { + dirty_cpu_scheds = dirty_cpu_scheds * dirty_cpu_scheds_pctg / 100; + dirty_cpu_scheds_online = dirty_cpu_scheds_online * dirty_cpu_scheds_onln_pctg / 100; + } + if (dirty_cpu_scheds > schdlrs) + dirty_cpu_scheds = schdlrs; + if (dirty_cpu_scheds_online > schdlrs_onln) + dirty_cpu_scheds_online = schdlrs_onln; +#endif } #ifndef USE_THREADS @@ -816,6 +1112,11 @@ early_init(int *argc, char **argv) /* erts_no_schedulers = (Uint) no_schedulers; #endif +#ifdef ERTS_DIRTY_SCHEDULERS + erts_no_dirty_cpu_schedulers = no_dirty_cpu_schedulers = dirty_cpu_scheds; + no_dirty_cpu_schedulers_online = dirty_cpu_scheds_online; + erts_no_dirty_io_schedulers = no_dirty_io_schedulers = dirty_io_scheds; +#endif erts_early_init_scheduling(no_schedulers); alloc_opts.ncpu = ncpu; @@ -833,10 +1134,18 @@ early_init(int *argc, char **argv) /* * * * Unmanaged threads that need to register: * ** Async threads (see erl_async.c) + * ** Dirty scheduler threads */ erts_thr_progress_init(no_schedulers, no_schedulers+2, - erts_async_max_threads); +#ifndef ERTS_DIRTY_SCHEDULERS + erts_async_max_threads +#else + erts_async_max_threads + + erts_no_dirty_cpu_schedulers + + erts_no_dirty_io_schedulers +#endif + ); #endif erts_thr_q_init(); erts_init_utils(); @@ -921,6 +1230,15 @@ erl_start(int argc, char **argv) int proc_tab_sz = ERTS_DEFAULT_MAX_PROCESSES; int port_tab_sz = ERTS_DEFAULT_MAX_PORTS; int port_tab_sz_ignore_files = 0; + int legacy_proc_tab = 0; + int legacy_port_tab = 0; + int time_correction; + ErtsTimeWarpMode time_warp_mode; + int node_tab_delete_delay = ERTS_NODE_TAB_DELAY_GC_DEFAULT; + ErtsDbSpinCount db_spin_count = ERTS_DB_SPNCNT_NORMAL; + + set_default_time_adj(&time_correction, + &time_warp_mode); envbufsz = sizeof(envbuf); if (erts_sys_getenv_raw(ERL_MAX_ETS_TABLES_ENV, envbuf, &envbufsz) == 0) @@ -952,7 +1270,7 @@ erl_start(int argc, char **argv) verbose = DEBUG_DEFAULT; #endif - erts_error_logger_warnings = am_error; + erts_error_logger_warnings = am_warning; while (i < argc) { if (argv[i][0] != '-') { @@ -1127,7 +1445,7 @@ erl_start(int argc, char **argv) } erts_fprintf(stderr, "(" EMULATOR ") emulator version " ERLANG_VERSION "\n"); - erl_exit(0, ""); + erts_exit(0, ""); } break; @@ -1141,6 +1459,7 @@ erl_start(int argc, char **argv) * * h|ms - min_heap_size * h|mbs - min_bin_vheap_size + * h|pds - erts_pd_initial_size * */ if (has_prefix("mbs", sub_param)) { @@ -1158,6 +1477,14 @@ erl_start(int argc, char **argv) erts_usage(); } VERBOSE(DEBUG_SYSTEM, ("using minimum heap size %d\n", H_MIN_SIZE)); + } else if (has_prefix("pds", sub_param)) { + arg = get_arg(sub_param+3, argv[i+1], &i); + if ((erts_pd_initial_size = atoi(arg)) <= 0) { + erts_fprintf(stderr, "bad initial process dictionary size %s\n", arg); + erts_usage(); + } + VERBOSE(DEBUG_SYSTEM, ("using initial process dictionary size %d\n", + erts_pd_initial_size)); } else { /* backward compatibility */ arg = get_arg(argv[i]+2, argv[i+1], &i); @@ -1266,31 +1593,50 @@ erl_start(int argc, char **argv) case 'P': /* set maximum number of processes */ arg = get_arg(argv[i]+2, argv[i+1], &i); - errno = 0; - proc_tab_sz = strtol(arg, NULL, 10); - if (errno != 0 - || proc_tab_sz < ERTS_MIN_PROCESSES - || ERTS_MAX_PROCESSES < proc_tab_sz) { - erts_fprintf(stderr, "bad number of processes %s\n", arg); - erts_usage(); + if (strcmp(arg, "legacy") == 0) + legacy_proc_tab = 1; + else { + errno = 0; + proc_tab_sz = strtol(arg, NULL, 10); + if (errno != 0 + || proc_tab_sz < ERTS_MIN_PROCESSES + || ERTS_MAX_PROCESSES < proc_tab_sz) { + erts_fprintf(stderr, "bad number of processes %s\n", arg); + erts_usage(); + } } break; case 'Q': /* set maximum number of ports */ arg = get_arg(argv[i]+2, argv[i+1], &i); - errno = 0; - port_tab_sz = strtol(arg, NULL, 10); - if (errno != 0 - || port_tab_sz < ERTS_MIN_PROCESSES - || ERTS_MAX_PROCESSES < port_tab_sz) { - erts_fprintf(stderr, "bad number of ports %s\n", arg); - erts_usage(); + if (strcmp(arg, "legacy") == 0) + legacy_port_tab = 1; + else { + errno = 0; + port_tab_sz = strtol(arg, NULL, 10); + if (errno != 0 + || port_tab_sz < ERTS_MIN_PROCESSES + || ERTS_MAX_PROCESSES < port_tab_sz) { + erts_fprintf(stderr, "bad number of ports %s\n", arg); + erts_usage(); + } + port_tab_sz_ignore_files = 1; } - port_tab_sz_ignore_files = 1; break; case 'S' : /* Was handled in early_init() just read past it */ - (void) get_arg(argv[i]+2, argv[i+1], &i); + if (argv[i][2] == 'D') { + char* type = argv[i]+3; + if (strcmp(type, "Pcpu") == 0) + (void) get_arg(argv[i]+7, argv[i+1], &i); + if (strcmp(type, "cpu") == 0) + (void) get_arg(argv[i]+6, argv[i+1], &i); + else if (strcmp(type, "io") == 0) + (void) get_arg(argv[i]+5, argv[i+1], &i); + } else if (argv[i][2] == 'P') + (void) get_arg(argv[i]+3, argv[i+1], &i); + else + (void) get_arg(argv[i]+2, argv[i+1], &i); break; case 's' : { @@ -1334,8 +1680,10 @@ erl_start(int argc, char **argv) } else if (has_prefix("cl", sub_param)) { arg = get_arg(sub_param+2, argv[i+1], &i); - if (sys_strcmp("true", arg) == 0) + if (sys_strcmp("true", arg) == 0) { erts_sched_compact_load = 1; + erts_sched_balance_util = 0; + } else if (sys_strcmp("false", arg) == 0) erts_sched_compact_load = 0; else { @@ -1388,6 +1736,22 @@ erl_start(int argc, char **argv) erts_usage(); } } + else if (has_prefix("ecio", sub_param)) { + arg = get_arg(sub_param+4, argv[i+1], &i); +#ifndef __OSE__ + if (sys_strcmp("true", arg) == 0) + erts_eager_check_io = 1; + else +#endif + if (sys_strcmp("false", arg) == 0) + erts_eager_check_io = 0; + else { + erts_fprintf(stderr, + "bad schedule eager check I/O value '%s'\n", + arg); + erts_usage(); + } + } else if (has_prefix("pp", sub_param)) { arg = get_arg(sub_param+2, argv[i+1], &i); if (sys_strcmp(arg, "true") == 0) @@ -1413,7 +1777,37 @@ erl_start(int argc, char **argv) erts_usage(); } } - else if (sys_strcmp("wt", sub_param) == 0) { + else if (has_prefix("ub", sub_param)) { + arg = get_arg(sub_param+2, argv[i+1], &i); + if (sys_strcmp("true", arg) == 0) { +#if ERTS_HAVE_SCHED_UTIL_BALANCING_SUPPORT_OPT + erts_sched_balance_util = 1; +#else + erts_fprintf(stderr, + "scheduler utilization balancing not " + "supported on this system\n"); + erts_usage(); +#endif + } + else if (sys_strcmp("false", arg) == 0) + erts_sched_balance_util = 0; + else { + erts_fprintf(stderr, "bad scheduler utilization balancing " + " value '%s'\n", arg); + erts_usage(); + } + } + else if (has_prefix("wct", sub_param)) { + arg = get_arg(sub_param+3, argv[i+1], &i); + if (erts_sched_set_wake_cleanup_threshold(arg) != 0) { + erts_fprintf(stderr, "scheduler wake cleanup threshold: %s\n", + arg); + erts_usage(); + } + VERBOSE(DEBUG_SYSTEM, + ("scheduler wake cleanup threshold: %s\n", arg)); + } + else if (has_prefix("wt", sub_param)) { arg = get_arg(sub_param+2, argv[i+1], &i); if (erts_sched_set_wakeup_other_thresold(arg) != 0) { erts_fprintf(stderr, "scheduler wakeup threshold: %s\n", @@ -1423,7 +1817,7 @@ erl_start(int argc, char **argv) VERBOSE(DEBUG_SYSTEM, ("scheduler wakeup threshold: %s\n", arg)); } - else if (sys_strcmp("ws", sub_param) == 0) { + else if (has_prefix("ws", sub_param)) { arg = get_arg(sub_param+2, argv[i+1], &i); if (erts_sched_set_wakeup_other_type(arg) != 0) { erts_fprintf(stderr, "scheduler wakeup strategy: %s\n", @@ -1450,6 +1844,22 @@ erl_start(int argc, char **argv) ("suggested scheduler thread stack size %d kilo words\n", erts_sched_thread_suggested_stack_size)); } + else if (has_prefix("fwi", sub_param)) { + long val; + arg = get_arg(sub_param+3, argv[i+1], &i); + errno = 0; + val = strtol(arg, NULL, 10); + if (errno != 0 || val < 0) { + erts_fprintf(stderr, + "bad scheduler forced wakeup " + "interval %s\n", + arg); + erts_usage(); + } +#ifdef ERTS_SMP + erts_runq_supervision_interval = val; +#endif + } else { erts_fprintf(stderr, "bad scheduling option %s\n", argv[i]); erts_usage(); @@ -1546,15 +1956,51 @@ erl_start(int argc, char **argv) } break; } + case 'C': + arg = get_arg(argv[i]+2, argv[i+1], &i); + if (sys_strcmp(arg, "no_time_warp") == 0) + time_warp_mode = ERTS_NO_TIME_WARP_MODE; + else if (sys_strcmp(arg, "single_time_warp") == 0) + time_warp_mode = ERTS_SINGLE_TIME_WARP_MODE; + else if (sys_strcmp(arg, "multi_time_warp") == 0) + time_warp_mode = ERTS_MULTI_TIME_WARP_MODE; + else { + erts_fprintf(stderr, + "Invalid time warp mode: %s\n", arg); + erts_usage(); + } + break; case 'c': - if (argv[i][2] == 0) { /* -c: documented option */ - erts_disable_tolerant_timeofday = 1; + if (sys_strcmp(argv[i]+2, "false") == 0) + goto time_correction_false; + else if (sys_strcmp(argv[i]+2, "true") == 0) + goto time_correction_true; + else if (argv[i][2] == '\0') { + if (i + 1 >= argc) + goto time_correction_false; + else { + if (sys_strcmp(argv[i+1], "false") == 0) { + (void) get_arg(argv[i]+2, argv[i+1], &i); + goto time_correction_false; + } + else if (sys_strcmp(argv[i+1], "true") == 0) { + (void) get_arg(argv[i]+2, argv[i+1], &i); + time_correction_true: + time_correction = 1; + break; + } + else { + time_correction_false: + time_correction = 0; + break; + } + } } -#ifdef ERTS_OPCODE_COUNTER_SUPPORT - else if (argv[i][2] == 'i') { /* -ci: undcoumented option*/ - count_instructions = 1; + else { + arg = get_arg(argv[i]+2, argv[i+1], &i); + erts_fprintf(stderr, "Invalid time correnction value: %s\n", arg); + erts_usage(); } -#endif break; case 'W': arg = get_arg(argv[i]+2, argv[i+1], &i); @@ -1562,11 +2008,12 @@ erl_start(int argc, char **argv) case 'i': erts_error_logger_warnings = am_info; break; + case 'e': + erts_error_logger_warnings = am_error; + break; case 'w': erts_error_logger_warnings = am_warning; break; - case 'e': /* The default */ - erts_error_logger_warnings = am_error; default: erts_fprintf(stderr, "unrecognized warning_map option %s\n", arg); erts_usage(); @@ -1575,9 +2022,9 @@ erl_start(int argc, char **argv) case 'z': { char *sub_param = argv[i]+2; - int new_limit; if (has_prefix("dbbl", sub_param)) { + int new_limit; arg = get_arg(sub_param+4, argv[i+1], &i); new_limit = atoi(arg); if (new_limit < 1 || INT_MAX/1024 < new_limit) { @@ -1586,6 +2033,46 @@ erl_start(int argc, char **argv) } else { erts_dist_buf_busy_limit = new_limit*1024; } + } + else if (has_prefix("dntgc", sub_param)) { + long secs; + + arg = get_arg(sub_param+5, argv[i+1], &i); + if (sys_strcmp(arg, "infinity") == 0) + secs = ERTS_NODE_TAB_DELAY_GC_INFINITY; + else { + char *endptr; + errno = 0; + secs = strtol(arg, &endptr, 10); + if (errno != 0 || *arg == '\0' || *endptr != '\0' + || secs < 0 || ERTS_NODE_TAB_DELAY_GC_MAX < secs) { + erts_fprintf(stderr, "Invalid delayed node table gc: %s\n", arg); + erts_usage(); + } + } + node_tab_delete_delay = (int) secs; + } + else if (has_prefix("ebwt", sub_param)) { + arg = get_arg(sub_param+4, argv[i+1], &i); + if (sys_strcmp(arg, "none") == 0) + db_spin_count = ERTS_DB_SPNCNT_NONE; + else if (sys_strcmp(arg, "very_short") == 0) + db_spin_count = ERTS_DB_SPNCNT_VERY_LOW; + else if (sys_strcmp(arg, "short") == 0) + db_spin_count = ERTS_DB_SPNCNT_LOW; + else if (sys_strcmp(arg, "medium") == 0) + db_spin_count = ERTS_DB_SPNCNT_NORMAL; + else if (sys_strcmp(arg, "long") == 0) + db_spin_count = ERTS_DB_SPNCNT_HIGH; + else if (sys_strcmp(arg, "very_long") == 0) + db_spin_count = ERTS_DB_SPNCNT_VERY_HIGH; + else if (sys_strcmp(arg, "extremely_long") == 0) + db_spin_count = ERTS_DB_SPNCNT_EXTREMELY_HIGH; + else { + erts_fprintf(stderr, + "Invalid ets busy wait threshold: %s\n", arg); + erts_usage(); + } } else { erts_fprintf(stderr, "bad -z option %s\n", argv[i]); erts_usage(); @@ -1600,6 +2087,30 @@ erl_start(int argc, char **argv) i++; } + if (!erts_check_time_adj_support(time_correction, time_warp_mode)) { + char *time_correction_str = time_correction ? "Enabled" : "Disabled"; + char *time_warp_str = "undefined"; + switch (time_warp_mode) { + case ERTS_NO_TIME_WARP_MODE: + time_warp_str = "no"; + break; + case ERTS_SINGLE_TIME_WARP_MODE: + time_warp_str = "single"; + break; + case ERTS_MULTI_TIME_WARP_MODE: + time_warp_str = "multi"; + break; + default: + time_warp_str = "undefined"; + break; + } + erts_fprintf(stderr, "%s time correction with %s time warp mode " + "is not supported on this platform\n", + time_correction_str, + time_warp_str); + erts_usage(); + } + /* Output format on windows for sprintf defaults to three exponents. * We use two-exponent to mimic normal sprintf behaviour. */ @@ -1630,8 +2141,14 @@ erl_start(int argc, char **argv) erl_init(ncpu, proc_tab_sz, + legacy_proc_tab, port_tab_sz, - port_tab_sz_ignore_files); + port_tab_sz_ignore_files, + legacy_port_tab, + time_correction, + time_warp_mode, + node_tab_delete_delay, + db_spin_count); load_preloaded(); erts_end_staging_code_ix(); @@ -1639,7 +2156,8 @@ erl_start(int argc, char **argv) erts_initialized = 1; - erl_first_process_otp("otp_ring0", NULL, 0, boot_argc, boot_argv); + (void) erl_first_process_otp("otp_ring0", NULL, 0, + boot_argc, boot_argv); #ifdef ERTS_SMP erts_start_schedulers(); @@ -1647,13 +2165,17 @@ erl_start(int argc, char **argv) erts_sys_main_thread(); /* May or may not return! */ #else - erts_thr_set_main_status(1, 1); + { + ErtsSchedulerData *esdp = erts_get_scheduler_data(); + erts_thr_set_main_status(1, 1); #if ERTS_USE_ASYNC_READY_Q - erts_get_scheduler_data()->aux_work_data.async_ready.queue - = erts_get_async_ready_queue(1); + esdp->aux_work_data.async_ready.queue + = erts_get_async_ready_queue(1); #endif - set_main_stack_size(); - process_main(); + set_main_stack_size(); + erts_sched_init_time_sup(esdp); + process_main(); + } #endif } @@ -1725,21 +2247,17 @@ system_cleanup(int flush_async) } static __decl_noreturn void __noreturn -erl_exit_vv(int n, int flush_async, char *fmt, va_list args1, va_list args2) +erts_exit_vv(int n, int flush_async, char *fmt, va_list args1, va_list args2) { - unsigned int an; - system_cleanup(flush_async); save_statistics(); - an = abs(n); - if (erts_mtrace_enabled) - erts_mtrace_exit((Uint32) an); + erts_mtrace_exit((Uint32) n); /* Produce an Erlang core dump if error */ - if (((n > 0 && erts_no_crash_dump == 0) || n == ERTS_DUMP_EXIT) + if (((n == ERTS_ERROR_EXIT && erts_no_crash_dump == 0) || n == ERTS_DUMP_EXIT) && erts_initialized) { erl_crash_dump_v((char*) NULL, 0, fmt, args1); } @@ -1752,29 +2270,29 @@ erl_exit_vv(int n, int flush_async, char *fmt, va_list args1, va_list args2) exit(0); else if (n == ERTS_DUMP_EXIT) ERTS_EXIT_AFTER_DUMP(1); - else if (n > 0 || n == ERTS_ABORT_EXIT) + else if (n == ERTS_ERROR_EXIT || n == ERTS_ABORT_EXIT) abort(); - exit(an); + exit(n); } /* Exit without flushing async threads */ -__decl_noreturn void __noreturn erl_exit(int n, char *fmt, ...) +__decl_noreturn void __noreturn erts_exit(int n, char *fmt, ...) { va_list args1, args2; va_start(args1, fmt); va_start(args2, fmt); - erl_exit_vv(n, 0, fmt, args1, args2); + erts_exit_vv(n, 0, fmt, args1, args2); va_end(args2); va_end(args1); } /* Exit after flushing async threads */ -__decl_noreturn void __noreturn erl_exit_flush_async(int n, char *fmt, ...) +__decl_noreturn void __noreturn erts_flush_async_exit(int n, char *fmt, ...) { va_list args1, args2; va_start(args1, fmt); va_start(args2, fmt); - erl_exit_vv(n, 1, fmt, args1, args2); + erts_exit_vv(n, 1, fmt, args1, args2); va_end(args2); va_end(args1); } diff --git a/erts/emulator/beam/erl_instrument.c b/erts/emulator/beam/erl_instrument.c index b5b245288b..12a72ad839 100644 --- a/erts/emulator/beam/erl_instrument.c +++ b/erts/emulator/beam/erl_instrument.c @@ -3,16 +3,17 @@ * * Copyright Ericsson AB 2003-2013. All Rights Reserved. * - * The contents of this file are subject to the Erlang Public License, - * Version 1.1, (the "License"); you may not use this file except in - * compliance with the License. You should have received a copy of the - * Erlang Public License along with this software. If not, it can be - * retrieved online at http://www.erlang.org/. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Software distributed under the License is distributed on an "AS IS" - * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See - * the License for the specific language governing rights and limitations - * under the License. + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. * * %CopyrightEnd% */ @@ -271,6 +272,18 @@ stat_upd_realloc(ErtsAlcType_t n, Uint size, Uint old_size) * stat instrumentation callback functions */ +static void stat_pre_lock(void) +{ + erts_mtx_lock(&instr_mutex); +} + +static void stat_pre_unlock(void) +{ + erts_mtx_unlock(&instr_mutex); +} + +static ErtsAllocatorWrapper_t instr_wrapper; + static void * stat_alloc(ErtsAlcType_t n, void *extra, Uint size) { @@ -278,7 +291,9 @@ stat_alloc(ErtsAlcType_t n, void *extra, Uint size) Uint ssize; void *res; - erts_mtx_lock(&instr_mutex); + if (!erts_is_allctr_wrapper_prelocked()) { + erts_mtx_lock(&instr_mutex); + } ssize = size + STAT_BLOCK_HEADER_SIZE; res = (*real_af->alloc)(n, real_af->extra, ssize); @@ -293,7 +308,9 @@ stat_alloc(ErtsAlcType_t n, void *extra, Uint size) res = (void *) ((StatBlock_t *) res)->mem; } - erts_mtx_unlock(&instr_mutex); + if (!erts_is_allctr_wrapper_prelocked()) { + erts_mtx_unlock(&instr_mutex); + } return res; } @@ -307,7 +324,9 @@ stat_realloc(ErtsAlcType_t n, void *extra, void *ptr, Uint size) void *sptr; void *res; - erts_mtx_lock(&instr_mutex); + if (!erts_is_allctr_wrapper_prelocked()) { + erts_mtx_lock(&instr_mutex); + } if (ptr) { sptr = (void *) (((char *) ptr) - STAT_BLOCK_HEADER_SIZE); @@ -329,7 +348,9 @@ stat_realloc(ErtsAlcType_t n, void *extra, void *ptr, Uint size) res = (void *) ((StatBlock_t *) res)->mem; } - erts_mtx_unlock(&instr_mutex); + if (!erts_is_allctr_wrapper_prelocked()) { + erts_mtx_unlock(&instr_mutex); + } return res; } @@ -340,7 +361,9 @@ stat_free(ErtsAlcType_t n, void *extra, void *ptr) ErtsAllocatorFunctions_t *real_af = (ErtsAllocatorFunctions_t *) extra; void *sptr; - erts_mtx_lock(&instr_mutex); + if (!erts_is_allctr_wrapper_prelocked()) { + erts_mtx_lock(&instr_mutex); + } if (ptr) { sptr = (void *) (((char *) ptr) - STAT_BLOCK_HEADER_SIZE); @@ -352,7 +375,9 @@ stat_free(ErtsAlcType_t n, void *extra, void *ptr) (*real_af->free)(n, real_af->extra, sptr); - erts_mtx_unlock(&instr_mutex); + if (!erts_is_allctr_wrapper_prelocked()) { + erts_mtx_unlock(&instr_mutex); + } } @@ -360,6 +385,18 @@ stat_free(ErtsAlcType_t n, void *extra, void *ptr) * map stat instrumentation callback functions */ +static void map_stat_pre_lock(void) +{ + erts_mtx_lock(&instr_x_mutex); + erts_mtx_lock(&instr_mutex); +} + +static void map_stat_pre_unlock(void) +{ + erts_mtx_unlock(&instr_mutex); + erts_mtx_unlock(&instr_x_mutex); +} + static void * map_stat_alloc(ErtsAlcType_t n, void *extra, Uint size) { @@ -367,7 +404,9 @@ map_stat_alloc(ErtsAlcType_t n, void *extra, Uint size) Uint msize; void *res; - erts_mtx_lock(&instr_mutex); + if (!erts_is_allctr_wrapper_prelocked()) { + erts_mtx_lock(&instr_mutex); + } msize = size + MAP_STAT_BLOCK_HEADER_SIZE; res = (*real_af->alloc)(n, real_af->extra, msize); @@ -388,7 +427,9 @@ map_stat_alloc(ErtsAlcType_t n, void *extra, Uint size) res = (void *) mb->mem; } - erts_mtx_unlock(&instr_mutex); + if (!erts_is_allctr_wrapper_prelocked()) { + erts_mtx_unlock(&instr_mutex); + } return res; } @@ -402,8 +443,10 @@ map_stat_realloc(ErtsAlcType_t n, void *extra, void *ptr, Uint size) void *mptr; void *res; - erts_mtx_lock(&instr_x_mutex); - erts_mtx_lock(&instr_mutex); + if (!erts_is_allctr_wrapper_prelocked()) { + erts_mtx_lock(&instr_x_mutex); + erts_mtx_lock(&instr_mutex); + } if (ptr) { mptr = (void *) (((char *) ptr) - MAP_STAT_BLOCK_HEADER_SIZE); @@ -449,9 +492,10 @@ map_stat_realloc(ErtsAlcType_t n, void *extra, void *ptr, Uint size) res = (void *) mb->mem; } - - erts_mtx_unlock(&instr_mutex); - erts_mtx_unlock(&instr_x_mutex); + if (!erts_is_allctr_wrapper_prelocked()) { + erts_mtx_unlock(&instr_mutex); + erts_mtx_unlock(&instr_x_mutex); + } return res; } @@ -462,8 +506,10 @@ map_stat_free(ErtsAlcType_t n, void *extra, void *ptr) ErtsAllocatorFunctions_t *real_af = (ErtsAllocatorFunctions_t *) extra; void *mptr; - erts_mtx_lock(&instr_x_mutex); - erts_mtx_lock(&instr_mutex); + if (!erts_is_allctr_wrapper_prelocked()) { + erts_mtx_lock(&instr_x_mutex); + erts_mtx_lock(&instr_mutex); + } if (ptr) { MapStatBlock_t *mb; @@ -486,8 +532,10 @@ map_stat_free(ErtsAlcType_t n, void *extra, void *ptr) (*real_af->free)(n, real_af->extra, mptr); - erts_mtx_unlock(&instr_mutex); - erts_mtx_unlock(&instr_x_mutex); + if (!erts_is_allctr_wrapper_prelocked()) { + erts_mtx_unlock(&instr_mutex); + erts_mtx_unlock(&instr_x_mutex); + } } @@ -496,8 +544,10 @@ static void dump_memory_map_to_stream(FILE *fp) ErtsAlcType_t n; MapStatBlock_t *bp; int lock = !ERTS_IS_CRASH_DUMPING; - if (lock) + if (lock) { + ASSERT(!erts_is_allctr_wrapper_prelocked()); erts_mtx_lock(&instr_mutex); + } /* Write header */ @@ -1155,6 +1205,7 @@ erts_instr_get_type_info(Process *proc) Uint erts_instr_init(int stat, int map_stat) { + Uint extra_sz; int i; am_tot = NULL; @@ -1176,7 +1227,7 @@ erts_instr_init(int stat, int map_stat) mem_anchor = NULL; /* Install instrumentation functions */ - ASSERT(sizeof(erts_allctrs) == sizeof(real_allctrs)); + ERTS_CT_ASSERT(sizeof(erts_allctrs) == sizeof(real_allctrs)); sys_memcpy((void *)real_allctrs,(void *)erts_allctrs,sizeof(erts_allctrs)); @@ -1186,8 +1237,6 @@ erts_instr_init(int stat, int map_stat) sys_memzero((void *) stats->n, sizeof(Stat_t)*(ERTS_ALC_N_MAX+1)); for (i = ERTS_ALC_A_MIN; i <= ERTS_ALC_A_MAX; i++) { - if (ERTS_IS_SBMBC_ALLOCATOR_NO__(i)) - continue; if (erts_allctrs_info[i].enabled) stats->ap[i] = &stats->a[i]; else @@ -1201,27 +1250,28 @@ erts_instr_init(int stat, int map_stat) erts_instr_memory_map = 1; erts_instr_stat = 1; for (i = ERTS_ALC_A_MIN; i <= ERTS_ALC_A_MAX; i++) { - if (ERTS_IS_SBMBC_ALLOCATOR_NO__(i)) - continue; erts_allctrs[i].alloc = map_stat_alloc; erts_allctrs[i].realloc = map_stat_realloc; erts_allctrs[i].free = map_stat_free; erts_allctrs[i].extra = (void *) &real_allctrs[i]; } - return MAP_STAT_BLOCK_HEADER_SIZE; + instr_wrapper.lock = map_stat_pre_lock; + instr_wrapper.unlock = map_stat_pre_unlock; + extra_sz = MAP_STAT_BLOCK_HEADER_SIZE; } else { erts_instr_stat = 1; for (i = ERTS_ALC_A_MIN; i <= ERTS_ALC_A_MAX; i++) { - if (ERTS_IS_SBMBC_ALLOCATOR_NO__(i)) - continue; erts_allctrs[i].alloc = stat_alloc; erts_allctrs[i].realloc = stat_realloc; erts_allctrs[i].free = stat_free; erts_allctrs[i].extra = (void *) &real_allctrs[i]; } - return STAT_BLOCK_HEADER_SIZE; + instr_wrapper.lock = stat_pre_lock; + instr_wrapper.unlock = stat_pre_unlock; + extra_sz = STAT_BLOCK_HEADER_SIZE; } - + erts_allctr_wrapper_prelock_init(&instr_wrapper); + return extra_sz; } diff --git a/erts/emulator/beam/erl_instrument.h b/erts/emulator/beam/erl_instrument.h index 37b9b67139..cb3b1920d3 100644 --- a/erts/emulator/beam/erl_instrument.h +++ b/erts/emulator/beam/erl_instrument.h @@ -3,16 +3,17 @@ * * Copyright Ericsson AB 2003-2009. All Rights Reserved. * - * The contents of this file are subject to the Erlang Public License, - * Version 1.1, (the "License"); you may not use this file except in - * compliance with the License. You should have received a copy of the - * Erlang Public License along with this software. If not, it can be - * retrieved online at http://www.erlang.org/. - * - * Software distributed under the License is distributed on an "AS IS" - * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See - * the License for the specific language governing rights and limitations - * under the License. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. * * %CopyrightEnd% */ diff --git a/erts/emulator/beam/erl_lock_check.c b/erts/emulator/beam/erl_lock_check.c index 69bb4be717..3b3b247020 100644 --- a/erts/emulator/beam/erl_lock_check.c +++ b/erts/emulator/beam/erl_lock_check.c @@ -1,18 +1,19 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2005-2012. All Rights Reserved. + * Copyright Ericsson AB 2005-2013. All Rights Reserved. * - * The contents of this file are subject to the Erlang Public License, - * Version 1.1, (the "License"); you may not use this file except in - * compliance with the License. You should have received a copy of the - * Erlang Public License along with this software. If not, it can be - * retrieved online at http://www.erlang.org/. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Software distributed under the License is distributed on an "AS IS" - * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See - * the License for the specific language governing rights and limitations - * under the License. + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. * * %CopyrightEnd% */ @@ -91,6 +92,7 @@ static erts_lc_lock_order_t erts_lock_order[] = { { "driver_list", NULL }, { "proc_link", "pid" }, { "proc_msgq", "pid" }, + { "proc_btm", "pid" }, { "dist_entry", "address" }, { "dist_entry_links", "address" }, { "code_write_permission", NULL }, @@ -123,6 +125,9 @@ static erts_lc_lock_order_t erts_lock_order[] = { { "schdlr_sspnd", NULL }, { "migration_info_update", NULL }, { "run_queue", "address" }, +#ifdef ERTS_DIRTY_SCHEDULERS + { "dirty_run_queue_sleep_list", "address" }, +#endif { "process_table", NULL }, { "cpu_info", NULL }, { "pollset", "address" }, @@ -132,32 +137,31 @@ static erts_lc_lock_order_t erts_lock_order[] = { #endif /* __WIN32__ */ { "alcu_init_atoms", NULL }, { "mseg_init_atoms", NULL }, + { "mmap_init_atoms", NULL }, { "drv_tsd", NULL }, { "async_enq_mtx", NULL }, #ifdef ERTS_SMP - { "sys_msg_q", NULL }, { "atom_tab", NULL }, - { "make_ref", NULL }, { "misc_op_list_pre_alloc_lock", "address" }, { "message_pre_alloc_lock", "address" }, { "ptimer_pre_alloc_lock", "address", }, { "btm_pre_alloc_lock", NULL, }, { "dist_entry_out_queue", "address" }, + { "port_sched_lock", "port_id" }, + { "sys_msg_q", NULL }, + { "port_table", NULL }, #endif { "mtrace_op", NULL }, { "instr_x", NULL }, { "instr", NULL }, { "alcu_allocator", "index" }, - { "sbmbc_alloc", "index" }, { "mseg", NULL }, #if HALFWORD_HEAP { "pmmap", NULL }, #endif #ifdef ERTS_SMP - { "port_sched_lock", "port_id" }, { "port_task_pre_alloc_lock", "address" }, { "proclist_pre_alloc_lock", "address" }, - { "port_table", NULL }, { "xports_list_pre_alloc_lock", "address" }, { "inet_buffer_stack_lock", NULL }, { "gc_info", NULL }, @@ -165,6 +169,8 @@ static erts_lc_lock_order_t erts_lock_order[] = { { "timer_wheel", NULL }, { "system_block", NULL }, { "timeofday", NULL }, + { "get_time", NULL }, + { "get_corrected_time", NULL }, { "breakpoints", NULL }, { "pollsets_lock", NULL }, { "pix_lock", "address" }, @@ -181,7 +187,12 @@ static erts_lc_lock_order_t erts_lock_order[] = { { "efile_drv dtrace mutex", NULL }, #endif { "mtrace_buf", NULL }, - { "erts_alloc_hard_debug", NULL } +#ifdef ERTS_SMP + { "os_monotonic_time", NULL }, +#endif + { "erts_alloc_hard_debug", NULL }, + { "hard_dbg_mseg", NULL }, + { "erts_mmap", NULL } }; #define ERTS_LOCK_ORDER_SIZE \ @@ -217,8 +228,7 @@ rw_op_str(Uint16 flags) case ERTS_LC_FLG_LO_READ: return " (r)"; case ERTS_LC_FLG_LO_WRITE: - erts_fprintf(stderr, "\nInternal error\n"); - lc_abort(); + ERTS_INTERNAL_ERROR("Only write flag present"); default: break; } @@ -231,6 +241,8 @@ struct erts_lc_locked_lock_t_ { erts_lc_locked_lock_t *prev; UWord extra; Sint16 id; + char *file; + unsigned int line; Uint16 flags; }; @@ -258,9 +270,9 @@ union erts_lc_free_block_t_ { static ethr_tsd_key locks_key; -static erts_lc_locked_locks_t *erts_locked_locks; +static erts_lc_locked_locks_t *erts_locked_locks = NULL; -static erts_lc_free_block_t *free_blocks; +static erts_lc_free_block_t *free_blocks = NULL; #ifdef ERTS_LC_STATIC_ALLOC #define ERTS_LC_FB_CHUNK_SIZE 10000 @@ -299,8 +311,7 @@ static ERTS_INLINE void lc_free(void *p) static void *lc_core_alloc(void) { lc_unlock(); - erts_fprintf(stderr, "Lock checker out of memory!\n"); - lc_abort(); + ERTS_INTERNAL_ERROR("Lock checker out of memory!\n"); } #else @@ -313,8 +324,7 @@ static void *lc_core_alloc(void) fbs = (erts_lc_free_block_t *) malloc(sizeof(erts_lc_free_block_t) * ERTS_LC_FB_CHUNK_SIZE); if (!fbs) { - erts_fprintf(stderr, "Lock checker failed to allocate memory!\n"); - lc_abort(); + ERTS_INTERNAL_ERROR("Lock checker failed to allocate memory!"); } for (i = 1; i < ERTS_LC_FB_CHUNK_SIZE - 1; i++) { #ifdef DEBUG @@ -354,11 +364,11 @@ create_locked_locks(char *thread_name) { erts_lc_locked_locks_t *l_lcks = malloc(sizeof(erts_lc_locked_locks_t)); if (!l_lcks) - lc_abort(); + ERTS_INTERNAL_ERROR("Lock checker failed to allocate memory!"); l_lcks->thread_name = strdup(thread_name ? thread_name : "unknown"); if (!l_lcks->thread_name) - lc_abort(); + ERTS_INTERNAL_ERROR("Lock checker failed to allocate memory!"); l_lcks->emu_thread = 0; l_lcks->tid = erts_thr_self(); @@ -420,47 +430,51 @@ make_my_locked_locks(void) } static ERTS_INLINE erts_lc_locked_lock_t * -new_locked_lock(erts_lc_lock_t *lck, Uint16 op_flags) +new_locked_lock(erts_lc_lock_t *lck, Uint16 op_flags, + char *file, unsigned int line) { erts_lc_locked_lock_t *l_lck = (erts_lc_locked_lock_t *) lc_alloc(); l_lck->next = NULL; l_lck->prev = NULL; l_lck->id = lck->id; l_lck->extra = lck->extra; + l_lck->file = file; + l_lck->line = line; l_lck->flags = lck->flags | op_flags; return l_lck; } static void -print_lock2(char *prefix, Sint16 id, Wterm extra, Uint16 flags, char *suffix) +raw_print_lock(char *prefix, Sint16 id, Wterm extra, Uint16 flags, + char* file, unsigned int line, char *suffix) { char *lname = (0 <= id && id < ERTS_LOCK_ORDER_SIZE ? erts_lock_order[id].name : "unknown"); + erts_fprintf(stderr,"%s'%s:",prefix,lname); + if (is_not_immed(extra)) - erts_fprintf(stderr, - "%s'%s:%p%s'%s%s", - prefix, - lname, - _unchecked_boxed_val(extra), - lock_type(flags), - rw_op_str(flags), - suffix); + erts_fprintf(stderr,"%p",_unchecked_boxed_val(extra)); else - erts_fprintf(stderr, - "%s'%s:%T%s'%s%s", - prefix, - lname, - extra, - lock_type(flags), - rw_op_str(flags), - suffix); + erts_fprintf(stderr,"%T",extra); + erts_fprintf(stderr,"%s",lock_type(flags)); + + if (file) + erts_fprintf(stderr,"(%s:%d)",file,line); + + erts_fprintf(stderr,"'%s%s",rw_op_str(flags),suffix); +} + +static void +print_lock2(char *prefix, Sint16 id, Wterm extra, Uint16 flags, char *suffix) +{ + raw_print_lock(prefix, id, extra, flags, NULL, 0, suffix); } static void print_lock(char *prefix, erts_lc_lock_t *lck, char *suffix) { - print_lock2(prefix, lck->id, lck->extra, lck->flags, suffix); + raw_print_lock(prefix, lck->id, lck->extra, lck->flags, NULL, 0, suffix); } static void @@ -476,7 +490,8 @@ print_curr_locks(erts_lc_locked_locks_t *l_lcks) "Currently these locks are locked by the %s thread:\n", l_lcks->thread_name); for (l_lck = l_lcks->locked.first; l_lck; l_lck = l_lck->next) - print_lock2(" ", l_lck->id, l_lck->extra, l_lck->flags, "\n"); + raw_print_lock(" ", l_lck->id, l_lck->extra, l_lck->flags, + l_lck->file, l_lck->line, "\n"); } } @@ -674,7 +689,7 @@ erts_lc_set_thread_name(char *thread_name) free((void *) l_lcks->thread_name); l_lcks->thread_name = strdup(thread_name ? thread_name : "unknown"); if (!l_lcks->thread_name) - lc_abort(); + ERTS_INTERNAL_ERROR("strdup failed"); } l_lcks->emu_thread = 1; } @@ -992,7 +1007,8 @@ erts_lc_trylock_force_busy_flg(erts_lc_lock_t *lck, Uint16 op_flags) #endif } -void erts_lc_trylock_flg(int locked, erts_lc_lock_t *lck, Uint16 op_flags) +void erts_lc_trylock_flg_x(int locked, erts_lc_lock_t *lck, Uint16 op_flags, + char *file, unsigned int line) { erts_lc_locked_locks_t *l_lcks; erts_lc_locked_lock_t *l_lck; @@ -1004,7 +1020,7 @@ void erts_lc_trylock_flg(int locked, erts_lc_lock_t *lck, Uint16 op_flags) return; l_lcks = make_my_locked_locks(); - l_lck = locked ? new_locked_lock(lck, op_flags) : NULL; + l_lck = locked ? new_locked_lock(lck, op_flags, file, line) : NULL; if (!l_lcks->locked.last) { ASSERT(!l_lcks->locked.first); @@ -1045,13 +1061,14 @@ void erts_lc_trylock_flg(int locked, erts_lc_lock_t *lck, Uint16 op_flags) } -void erts_lc_require_lock_flg(erts_lc_lock_t *lck, Uint16 op_flags) +void erts_lc_require_lock_flg(erts_lc_lock_t *lck, Uint16 op_flags, + char *file, unsigned int line) { erts_lc_locked_locks_t *l_lcks = make_my_locked_locks(); erts_lc_locked_lock_t *l_lck = l_lcks->locked.first; if (!find_lock(&l_lck, lck)) required_not_locked(l_lcks, lck); - l_lck = new_locked_lock(lck, op_flags); + l_lck = new_locked_lock(lck, op_flags, file, line); if (!l_lcks->required.last) { ASSERT(!l_lcks->required.first); l_lck->next = l_lck->prev = NULL; @@ -1119,7 +1136,8 @@ void erts_lc_unrequire_lock_flg(erts_lc_lock_t *lck, Uint16 op_flags) lc_free((void *) l_lck); } -void erts_lc_lock_flg(erts_lc_lock_t *lck, Uint16 op_flags) +void erts_lc_lock_flg_x(erts_lc_lock_t *lck, Uint16 op_flags, + char *file, unsigned int line) { erts_lc_locked_locks_t *l_lcks; erts_lc_locked_lock_t *l_lck; @@ -1131,7 +1149,7 @@ void erts_lc_lock_flg(erts_lc_lock_t *lck, Uint16 op_flags) return; l_lcks = make_my_locked_locks(); - l_lck = new_locked_lock(lck, op_flags); + l_lck = new_locked_lock(lck, op_flags, file, line); if (!l_lcks->locked.last) { ASSERT(!l_lcks->locked.first); @@ -1222,15 +1240,15 @@ erts_lc_trylock_force_busy(erts_lc_lock_t *lck) } void -erts_lc_trylock(int locked, erts_lc_lock_t *lck) +erts_lc_trylock_x(int locked, erts_lc_lock_t *lck, char *file, unsigned int line) { - erts_lc_trylock_flg(locked, lck, 0); + erts_lc_trylock_flg_x(locked, lck, 0, file, line); } void -erts_lc_lock(erts_lc_lock_t *lck) +erts_lc_lock_x(erts_lc_lock_t *lck, char *file, unsigned int line) { - erts_lc_lock_flg(lck, 0); + erts_lc_lock_flg_x(lck, 0, file, line); } void @@ -1244,9 +1262,9 @@ void erts_lc_might_unlock(erts_lc_lock_t *lck) erts_lc_might_unlock_flg(lck, 0); } -void erts_lc_require_lock(erts_lc_lock_t *lck) +void erts_lc_require_lock(erts_lc_lock_t *lck, char *file, unsigned int line) { - erts_lc_require_lock_flg(lck, 0); + erts_lc_require_lock_flg(lck, 0, file, line); } void erts_lc_unrequire_lock(erts_lc_lock_t *lck) @@ -1310,9 +1328,9 @@ erts_lc_init(void) #endif /* #ifdef ERTS_LC_STATIC_ALLOC */ if (ethr_spinlock_init(&free_blocks_lock) != 0) - lc_abort(); + ERTS_INTERNAL_ERROR("spinlock_init failed"); - erts_tsd_key_create(&locks_key); + erts_tsd_key_create(&locks_key,"erts_lock_check_key"); } void diff --git a/erts/emulator/beam/erl_lock_check.h b/erts/emulator/beam/erl_lock_check.h index 068340abe7..66251ef4e8 100644 --- a/erts/emulator/beam/erl_lock_check.h +++ b/erts/emulator/beam/erl_lock_check.h @@ -3,16 +3,17 @@ * * Copyright Ericsson AB 2005-2012. All Rights Reserved. * - * The contents of this file are subject to the Erlang Public License, - * Version 1.1, (the "License"); you may not use this file except in - * compliance with the License. You should have received a copy of the - * Erlang Public License along with this software. If not, it can be - * retrieved online at http://www.erlang.org/. - * - * Software distributed under the License is distributed on an "AS IS" - * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See - * the License for the specific language governing rights and limitations - * under the License. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. * * %CopyrightEnd% */ @@ -35,6 +36,11 @@ #ifdef ERTS_ENABLE_LOCK_CHECK +#ifndef ERTS_ENABLE_LOCK_POSITION +/* Enable in order for _x variants of mtx functions to be used. */ +#define ERTS_ENABLE_LOCK_POSITION 1 +#endif + typedef struct { int inited; Sint16 id; @@ -79,13 +85,16 @@ void erts_lc_have_locks(int *resv, erts_lc_lock_t *lcks, int len); void erts_lc_have_lock_ids(int *resv, int *ids, int len); void erts_lc_check_no_locked_of_type(Uint16 flags); int erts_lc_trylock_force_busy_flg(erts_lc_lock_t *lck, Uint16 op_flags); -void erts_lc_trylock_flg(int locked, erts_lc_lock_t *lck, Uint16 op_flags); -void erts_lc_lock_flg(erts_lc_lock_t *lck, Uint16 op_flags); +void erts_lc_trylock_flg_x(int locked, erts_lc_lock_t *lck, Uint16 op_flags, + char *file, unsigned int line); +void erts_lc_lock_flg_x(erts_lc_lock_t *lck, Uint16 op_flags, + char *file, unsigned int line); void erts_lc_unlock_flg(erts_lc_lock_t *lck, Uint16 op_flags); void erts_lc_might_unlock_flg(erts_lc_lock_t *lck, Uint16 op_flags); int erts_lc_trylock_force_busy(erts_lc_lock_t *lck); -void erts_lc_trylock(int locked, erts_lc_lock_t *lck); -void erts_lc_lock(erts_lc_lock_t *lck); +void erts_lc_trylock_x(int locked, erts_lc_lock_t *lck, + char* file, unsigned int line); +void erts_lc_lock_x(erts_lc_lock_t *lck, char* file, unsigned int line); void erts_lc_unlock(erts_lc_lock_t *lck); void erts_lc_might_unlock(erts_lc_lock_t *lck); void erts_lc_init_lock(erts_lc_lock_t *lck, char *name, Uint16 flags); @@ -96,10 +105,11 @@ int erts_lc_assert_failed(char *file, int line, char *assertion); void erts_lc_set_thread_name(char *thread_name); void erts_lc_pll(void); -void erts_lc_require_lock_flg(erts_lc_lock_t *lck, Uint16 op_flags); +void erts_lc_require_lock_flg(erts_lc_lock_t *lck, Uint16 op_flags, + char *file, unsigned int line); void erts_lc_unrequire_lock_flg(erts_lc_lock_t *lck, Uint16 op_flags); -void erts_lc_require_lock(erts_lc_lock_t *lck); +void erts_lc_require_lock(erts_lc_lock_t *lck, char *file, unsigned int line); void erts_lc_unrequire_lock(erts_lc_lock_t *lck); int erts_lc_is_emu_thr(void); @@ -116,4 +126,9 @@ int erts_lc_is_emu_thr(void); #define ERTS_LC_ASSERT(A) ((void) 1) #endif /* #ifdef ERTS_ENABLE_LOCK_CHECK */ +#define erts_lc_lock(lck) erts_lc_lock_x(lck,__FILE__,__LINE__) +#define erts_lc_trylock(res,lck) erts_lc_trylock_x(res,lck,__FILE__,__LINE__) +#define erts_lc_lock_flg(lck) erts_lc_lock_flg_x(lck,__FILE__,__LINE__) +#define erts_lc_trylock_flg(res,lck) erts_lc_trylock_flg_x(res,lck,__FILE__,__LINE__) + #endif /* #ifndef ERTS_LOCK_CHECK_H__ */ diff --git a/erts/emulator/beam/erl_lock_count.c b/erts/emulator/beam/erl_lock_count.c index 5f75b0ac0b..bd00480ba2 100644 --- a/erts/emulator/beam/erl_lock_count.c +++ b/erts/emulator/beam/erl_lock_count.c @@ -3,16 +3,17 @@ * * Copyright Ericsson AB 2008-2012. All Rights Reserved. * - * The contents of this file are subject to the Erlang Public License, - * Version 1.1, (the "License"); you may not use this file except in - * compliance with the License. You should have received a copy of the - * Erlang Public License along with this software. If not, it can be - * retrieved online at http://www.erlang.org/. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Software distributed under the License is distributed on an "AS IS" - * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See - * the License for the specific language governing rights and limitations - * under the License. + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. * * %CopyrightEnd% */ @@ -20,7 +21,7 @@ /* * Description: Statistics for locks. * - * Author: Bj�rn-Egil Dahlberg + * Author: Björn-Egil Dahlberg * Date: 2008-07-03 */ @@ -49,7 +50,7 @@ const char *str_undefined = "undefined"; static ethr_tsd_key lcnt_thr_data_key; static int lcnt_n_thr; -static erts_lcnt_thread_data_t *lcnt_thread_data[4096]; +static erts_lcnt_thread_data_t *lcnt_thread_data[2048]; /* local functions */ @@ -61,15 +62,34 @@ static ERTS_INLINE void lcnt_unlock(void) { ethr_mutex_unlock(&lcnt_data_lock); } +const int log2_tab64[64] = { + 63, 0, 58, 1, 59, 47, 53, 2, + 60, 39, 48, 27, 54, 33, 42, 3, + 61, 51, 37, 40, 49, 18, 28, 20, + 55, 30, 34, 11, 43, 14, 22, 4, + 62, 57, 46, 52, 38, 26, 32, 41, + 50, 36, 17, 19, 29, 10, 13, 21, + 56, 45, 25, 31, 35, 16, 9, 12, + 44, 24, 15, 8, 23, 7, 6, 5}; + +static ERTS_INLINE int lcnt_log2(Uint64 v) { + v |= v >> 1; + v |= v >> 2; + v |= v >> 4; + v |= v >> 8; + v |= v >> 16; + v |= v >> 32; + return log2_tab64[((Uint64)((v - (v >> 1))*0x07EDD5E59A4E28C2)) >> 58]; +} static char* lcnt_lock_type(Uint16 flag) { switch(flag & ERTS_LCNT_LT_ALL) { - case ERTS_LCNT_LT_SPINLOCK: return "spinlock"; - case ERTS_LCNT_LT_RWSPINLOCK: return "rw_spinlock"; - case ERTS_LCNT_LT_MUTEX: return "mutex"; - case ERTS_LCNT_LT_RWMUTEX: return "rw_mutex"; - case ERTS_LCNT_LT_PROCLOCK: return "proclock"; - default: return ""; + case ERTS_LCNT_LT_SPINLOCK: return "spinlock"; + case ERTS_LCNT_LT_RWSPINLOCK: return "rw_spinlock"; + case ERTS_LCNT_LT_MUTEX: return "mutex"; + case ERTS_LCNT_LT_RWMUTEX: return "rw_mutex"; + case ERTS_LCNT_LT_PROCLOCK: return "proclock"; + default: return ""; } } @@ -81,58 +101,56 @@ static void lcnt_clear_stats(erts_lcnt_lock_stats_t *stats) { stats->timer_n = 0; stats->file = (char *)str_undefined; stats->line = 0; + sys_memzero(stats->hist.ns, sizeof(stats->hist.ns)); } static void lcnt_time(erts_lcnt_time_t *time) { -#ifdef HAVE_GETHRTIME - SysHrTime hr_time; - hr_time = sys_gethrtime(); - time->s = (unsigned long)(hr_time / 1000000000LL); - time->ns = (unsigned long)(hr_time - 1000000000LL*time->s); -#else - SysTimeval tv; - sys_gettimeofday(&tv); - time->s = tv.tv_sec; - time->ns = tv.tv_usec*1000LL; -#endif + /* + * erts_sys_hrtime() is the highest resolution + * we could find, it may or may not be monotonic... + */ + ErtsMonotonicTime mtime = erts_sys_hrtime(); + time->s = (unsigned long) (mtime / 1000000000LL); + time->ns = (unsigned long) (mtime - 1000000000LL*time->s); } static void lcnt_time_diff(erts_lcnt_time_t *d, erts_lcnt_time_t *t1, erts_lcnt_time_t *t0) { long ds; long dns; - + ds = t1->s - t0->s; dns = t1->ns - t0->ns; - + /* the difference should not be able to get bigger than 1 sec in ns*/ - + if (dns < 0) { - ds -= 1; - dns += 1000000000LL; + ds -= 1; + dns += 1000000000LL; } + ASSERT(ds >= 0); + d->s = ds; d->ns = dns; } -/* difference d must be positive */ +/* difference d must be non-negative */ static void lcnt_time_add(erts_lcnt_time_t *t, erts_lcnt_time_t *d) { - unsigned long ngns = 0; - t->s += d->s; t->ns += d->ns; - ngns = t->ns / 1000000000LL; + t->s += t->ns / 1000000000LL; t->ns = t->ns % 1000000000LL; - - t->s += ngns; } static erts_lcnt_thread_data_t *lcnt_thread_data_alloc(void) { erts_lcnt_thread_data_t *eltd; - + eltd = (erts_lcnt_thread_data_t*)malloc(sizeof(erts_lcnt_thread_data_t)); + if (!eltd) { + ERTS_INTERNAL_ERROR("Lock counter failed to allocate memory!"); + } eltd->timer_set = 0; eltd->lock_in_conflict = 0; @@ -147,7 +165,6 @@ static erts_lcnt_thread_data_t *lcnt_get_thread_data(void) { return (erts_lcnt_thread_data_t *)ethr_tsd_get(lcnt_thr_data_key); } - /* debug */ #if 0 @@ -158,96 +175,100 @@ static char* lock_opt(Uint16 flag) { return "--"; } -static void print_lock_x(erts_lcnt_lock_t *lock, Uint16 flag, char *action, char *extra) { - erts_aint_t colls, tries, w_state, r_state; - erts_lcnt_lock_stats_t *stats = NULL; - +static void print_lock_x(erts_lcnt_lock_t *lock, Uint16 flag, char *action) { + erts_aint_t w_state, r_state; char *type; - int i; - + + if (strcmp(lock->name, "run_queue") != 0) return; type = lcnt_lock_type(lock->flag); r_state = ethr_atomic_read(&lock->r_state); w_state = ethr_atomic_read(&lock->w_state); - if (lock->flag & flag) { - erts_printf("%20s [%30s] [r/w state %4ld/%4ld] id %T %s\r\n", - action, - lock->name, - r_state, - w_state, - lock->id, - extra); - } -} - -static void print_lock(erts_lcnt_lock_t *lock, char *action) { - if (strcmp(lock->name, "proc_main") == 0) { - print_lock_x(lock, ERTS_LCNT_LT_ALL, action, ""); + erts_fprintf(stderr,"%10s [%24s] [r/w state %4ld/%4ld] %2s id %T\r\n", + action, + lock->name, + r_state, + w_state, + type, + lock->id); } } - #endif static erts_lcnt_lock_stats_t *lcnt_get_lock_stats(erts_lcnt_lock_t *lock, char *file, unsigned int line) { unsigned int i; erts_lcnt_lock_stats_t *stats = NULL; - - for (i = 0; i < lock->n_stats; i++) { - if ((lock->stats[i].file == file) && (lock->stats[i].line == line)) { - return &(lock->stats[i]); - } - } - if (lock->n_stats < ERTS_LCNT_MAX_LOCK_LOCATIONS) { - stats = &lock->stats[lock->n_stats]; - lock->n_stats++; - stats->file = file; - stats->line = line; - return stats; + if (erts_lcnt_rt_options & ERTS_LCNT_OPT_LOCATION) { + for (i = 0; i < lock->n_stats; i++) { + if ((lock->stats[i].file == file) && (lock->stats[i].line == line)) { + return &(lock->stats[i]); + } + } + if (lock->n_stats < ERTS_LCNT_MAX_LOCK_LOCATIONS) { + stats = &lock->stats[lock->n_stats]; + lock->n_stats++; + stats->file = file; + stats->line = line; + return stats; + } } return &lock->stats[0]; +} +static void lcnt_update_stats_hist(erts_lcnt_hist_t *hist, erts_lcnt_time_t *time_wait) { + int idx; + unsigned long r; + + if (time_wait->s > 0 || time_wait->ns > ERTS_LCNT_HISTOGRAM_MAX_NS) { + idx = ERTS_LCNT_HISTOGRAM_SLOT_SIZE - 1; + } else { + r = time_wait->ns >> ERTS_LCNT_HISTOGRAM_RSHIFT; + if (r) idx = lcnt_log2(r); + else idx = 0; + } + hist->ns[idx]++; } -static void lcnt_update_stats(erts_lcnt_lock_stats_t *stats, int lock_in_conflict, erts_lcnt_time_t *time_wait) { - +static void lcnt_update_stats(erts_lcnt_lock_stats_t *stats, int lock_in_conflict, + erts_lcnt_time_t *time_wait) { + ethr_atomic_inc(&stats->tries); if (lock_in_conflict) - ethr_atomic_inc(&stats->colls); + ethr_atomic_inc(&stats->colls); if (time_wait) { - lcnt_time_add(&(stats->timer), time_wait); - stats->timer_n++; + lcnt_time_add(&(stats->timer), time_wait); + stats->timer_n++; + lcnt_update_stats_hist(&stats->hist,time_wait); } } -/* - * interface - */ +/* interface */ void erts_lcnt_init() { erts_lcnt_thread_data_t *eltd = NULL; - + /* init lock */ if (ethr_mutex_init(&lcnt_data_lock) != 0) abort(); /* init tsd */ lcnt_n_thr = 0; - - ethr_tsd_key_create(&lcnt_thr_data_key); + ethr_tsd_key_create(&lcnt_thr_data_key,"lcnt_data"); lcnt_lock(); - erts_lcnt_rt_options = ERTS_LCNT_OPT_PROCLOCK | ERTS_LCNT_OPT_LOCATION; - + erts_lcnt_rt_options = ERTS_LCNT_OPT_LOCATION | ERTS_LCNT_OPT_PROCLOCK; eltd = lcnt_thread_data_alloc(); - ethr_tsd_set(lcnt_thr_data_key, eltd); - + /* init lcnt structure */ erts_lcnt_data = (erts_lcnt_data_t*)malloc(sizeof(erts_lcnt_data_t)); + if (!erts_lcnt_data) { + ERTS_INTERNAL_ERROR("Lock counter failed to allocate memory!"); + } erts_lcnt_data->current_locks = erts_lcnt_list_init(); erts_lcnt_data->deleted_locks = erts_lcnt_list_init(); @@ -267,8 +288,11 @@ void erts_lcnt_late_init() { erts_lcnt_lock_list_t *erts_lcnt_list_init(void) { erts_lcnt_lock_list_t *list; - + list = (erts_lcnt_lock_list_t*)malloc(sizeof(erts_lcnt_lock_list_t)); + if (!list) { + ERTS_INTERNAL_ERROR("Lock counter failed to allocate memory!"); + } list->head = NULL; list->tail = NULL; list->n = 0; @@ -278,14 +302,14 @@ erts_lcnt_lock_list_t *erts_lcnt_list_init(void) { /* only do this on the list with the deleted locks! */ void erts_lcnt_list_clear(erts_lcnt_lock_list_t *list) { erts_lcnt_lock_t *lock = NULL, - *next = NULL; + *next = NULL; lock = list->head; - + while(lock != NULL) { - next = lock->next; - free(lock); - lock = next; + next = lock->next; + free(lock); + lock = next; } list->head = NULL; @@ -298,26 +322,25 @@ void erts_lcnt_list_insert(erts_lcnt_lock_list_t *list, erts_lcnt_lock_t *lock) tail = list->tail; if (tail) { - tail->next = lock; - lock->prev = tail; + tail->next = lock; + lock->prev = tail; } else { - list->head = lock; - lock->prev = NULL; - ASSERT(!lock->next); + list->head = lock; + lock->prev = NULL; + ASSERT(!lock->next); } lock->next = NULL; list->tail = lock; - + list->n++; } void erts_lcnt_list_delete(erts_lcnt_lock_list_t *list, erts_lcnt_lock_t *lock) { - if (lock->next) lock->next->prev = lock->prev; if (lock->prev) lock->prev->next = lock->next; if (list->head == lock) list->head = lock->next; if (list->tail == lock) list->tail = lock->prev; - + lock->prev = NULL; lock->next = NULL; list->n--; @@ -330,16 +353,14 @@ void erts_lcnt_list_delete(erts_lcnt_lock_list_t *list, erts_lcnt_lock_t *lock) /* interface to erl_threads.h */ /* only lock on init and destroy, all others should use atomics */ void erts_lcnt_init_lock(erts_lcnt_lock_t *lock, char *name, Uint16 flag ) { - erts_lcnt_init_lock_x(lock, name, flag, am_undefined); + erts_lcnt_init_lock_x(lock, name, flag, NIL); } + void erts_lcnt_init_lock_x(erts_lcnt_lock_t *lock, char *name, Uint16 flag, Eterm id) { int i; - if (!name) { - lock->flag = 0; - return; - } + if (name == NULL) { ERTS_LCNT_CLEAR_FLAG(lock); return; } lcnt_lock(); - + lock->next = NULL; lock->prev = NULL; lock->flag = flag; @@ -348,44 +369,55 @@ void erts_lcnt_init_lock_x(erts_lcnt_lock_t *lock, char *name, Uint16 flag, Eter ethr_atomic_init(&lock->r_state, 0); ethr_atomic_init(&lock->w_state, 0); - #ifdef DEBUG ethr_atomic_init(&lock->flowstate, 0); #endif - + lock->n_stats = 1; for (i = 0; i < ERTS_LCNT_MAX_LOCK_LOCATIONS; i++) { - lcnt_clear_stats(&lock->stats[i]); + lcnt_clear_stats(&lock->stats[i]); } erts_lcnt_list_insert(erts_lcnt_data->current_locks, lock); - lcnt_unlock(); } - +/* init empty, instead of zero struct */ +void erts_lcnt_init_lock_empty(erts_lcnt_lock_t *lock) { + lock->next = NULL; + lock->prev = NULL; + lock->flag = 0; + lock->name = NULL; + lock->id = NIL; + ethr_atomic_init(&lock->r_state, 0); + ethr_atomic_init(&lock->w_state, 0); +#ifdef DEBUG + ethr_atomic_init(&lock->flowstate, 0); +#endif + lock->n_stats = 0; + sys_memzero(lock->stats, sizeof(lock->stats)); +} +/* destroy lock */ void erts_lcnt_destroy_lock(erts_lcnt_lock_t *lock) { - erts_lcnt_lock_t *deleted_lock; - - if (!ERTS_LCNT_LOCK_TYPE(lock)) return; - + if (ERTS_LCNT_IS_LOCK_INVALID(lock)) return; lcnt_lock(); if (erts_lcnt_rt_options & ERTS_LCNT_OPT_COPYSAVE) { - /* copy structure and insert the copy */ - - deleted_lock = (erts_lcnt_lock_t*)malloc(sizeof(erts_lcnt_lock_t)); - memcpy(deleted_lock, lock, sizeof(erts_lcnt_lock_t)); - - deleted_lock->next = NULL; - deleted_lock->prev = NULL; - - erts_lcnt_list_insert(erts_lcnt_data->deleted_locks, deleted_lock); + erts_lcnt_lock_t *deleted_lock; + /* copy structure and insert the copy */ + deleted_lock = (erts_lcnt_lock_t*)malloc(sizeof(erts_lcnt_lock_t)); + if (!deleted_lock) { + ERTS_INTERNAL_ERROR("Lock counter failed to allocate memory!"); + } + memcpy(deleted_lock, lock, sizeof(erts_lcnt_lock_t)); + deleted_lock->next = NULL; + deleted_lock->prev = NULL; + erts_lcnt_list_insert(erts_lcnt_data->deleted_locks, deleted_lock); } /* delete original */ erts_lcnt_list_delete(erts_lcnt_data->current_locks, lock); - lock->flag = 0; - + ERTS_LCNT_CLEAR_FLAG(lock); + lcnt_unlock(); } @@ -394,16 +426,15 @@ void erts_lcnt_destroy_lock(erts_lcnt_lock_t *lock) { void erts_lcnt_lock_opt(erts_lcnt_lock_t *lock, Uint16 option) { erts_aint_t r_state = 0, w_state = 0; erts_lcnt_thread_data_t *eltd; - + if (erts_lcnt_rt_options & ERTS_LCNT_OPT_SUSPEND) return; - if (!ERTS_LCNT_LOCK_TYPE(lock)) return; + if (ERTS_LCNT_IS_LOCK_INVALID(lock)) return; eltd = lcnt_get_thread_data(); - ASSERT(eltd); - + w_state = ethr_atomic_read(&lock->w_state); - + if (option & ERTS_LCNT_LO_WRITE) { r_state = ethr_atomic_read(&lock->r_state); ethr_atomic_inc( &lock->w_state); @@ -411,47 +442,47 @@ void erts_lcnt_lock_opt(erts_lcnt_lock_t *lock, Uint16 option) { if (option & ERTS_LCNT_LO_READ) { ethr_atomic_inc( &lock->r_state); } - + /* we cannot acquire w_lock if either w or r are taken */ /* we cannot acquire r_lock if w_lock is taken */ - + if ((w_state > 0) || (r_state > 0)) { - eltd->lock_in_conflict = 1; - if (eltd->timer_set == 0) - lcnt_time(&eltd->timer); - eltd->timer_set++; + eltd->lock_in_conflict = 1; + if (eltd->timer_set == 0) { + lcnt_time(&eltd->timer); + } + eltd->timer_set++; } else { - eltd->lock_in_conflict = 0; + eltd->lock_in_conflict = 0; } } void erts_lcnt_lock(erts_lcnt_lock_t *lock) { erts_aint_t w_state; erts_lcnt_thread_data_t *eltd; - + if (erts_lcnt_rt_options & ERTS_LCNT_OPT_SUSPEND) return; - if (!ERTS_LCNT_LOCK_TYPE(lock)) return; + if (ERTS_LCNT_IS_LOCK_INVALID(lock)) return; w_state = ethr_atomic_read(&lock->w_state); - ethr_atomic_inc( &lock->w_state); - + ethr_atomic_inc(&lock->w_state); eltd = lcnt_get_thread_data(); ASSERT(eltd); if (w_state > 0) { - eltd->lock_in_conflict = 1; - /* only set the timer if nobody else has it - * This should only happen when proc_locks aquires several locks - * 'atomicly'. All other locks will block the thread if w_state > 0 - * i.e. locked. - */ - if (eltd->timer_set == 0) - lcnt_time(&eltd->timer); - eltd->timer_set++; - + eltd->lock_in_conflict = 1; + /* only set the timer if nobody else has it + * This should only happen when proc_locks aquires several locks + * 'atomicly'. All other locks will block the thread if w_state > 0 + * i.e. locked. + */ + if (eltd->timer_set == 0) { + lcnt_time(&eltd->timer); + } + eltd->timer_set++; } else { - eltd->lock_in_conflict = 0; + eltd->lock_in_conflict = 0; } } @@ -459,18 +490,20 @@ void erts_lcnt_lock(erts_lcnt_lock_t *lock) { void erts_lcnt_lock_unaquire(erts_lcnt_lock_t *lock) { /* should check if this thread was "waiting" */ - if (erts_lcnt_rt_options & ERTS_LCNT_OPT_SUSPEND) return; - if (!ERTS_LCNT_LOCK_TYPE(lock)) return; + if (ERTS_LCNT_IS_LOCK_INVALID(lock)) return; - ethr_atomic_dec( &lock->w_state); + ethr_atomic_dec(&lock->w_state); } -/* erts_lcnt_lock_post - * used when we get a lock (i.e. directly after a lock operation) +/* + * erts_lcnt_lock_post + * + * Used when we get a lock (i.e. directly after a lock operation) * if the timer was set then we had to wait for the lock * lock_post will calculate the wait time. */ + void erts_lcnt_lock_post(erts_lcnt_lock_t *lock) { erts_lcnt_lock_post_x(lock, (char*)str_undefined, 0); } @@ -485,38 +518,31 @@ void erts_lcnt_lock_post_x(erts_lcnt_lock_t *lock, char *file, unsigned int line #endif if (erts_lcnt_rt_options & ERTS_LCNT_OPT_SUSPEND) return; - if (!ERTS_LCNT_LOCK_TYPE(lock)) return; - + if (ERTS_LCNT_IS_LOCK_INVALID(lock)) return; + #ifdef DEBUG if (!(lock->flag & (ERTS_LCNT_LT_RWMUTEX | ERTS_LCNT_LT_RWSPINLOCK))) { - flowstate = ethr_atomic_read(&lock->flowstate); - ASSERT(flowstate == 0); - ethr_atomic_inc( &lock->flowstate); + flowstate = ethr_atomic_read(&lock->flowstate); + ASSERT(flowstate == 0); + ethr_atomic_inc(&lock->flowstate); } #endif - + eltd = lcnt_get_thread_data(); - + ASSERT(eltd); /* if lock was in conflict, time it */ - - if (erts_lcnt_rt_options & ERTS_LCNT_OPT_LOCATION) { - stats = lcnt_get_lock_stats(lock, file, line); - } else { - stats = &lock->stats[0]; - } - + stats = lcnt_get_lock_stats(lock, file, line); if (eltd->timer_set) { - lcnt_time(&timer); - - lcnt_time_diff(&time_wait, &timer, &(eltd->timer)); - lcnt_update_stats(stats, eltd->lock_in_conflict, &time_wait); - - eltd->timer_set--; - ASSERT(eltd->timer_set >= 0); + lcnt_time(&timer); + + lcnt_time_diff(&time_wait, &timer, &(eltd->timer)); + lcnt_update_stats(stats, eltd->lock_in_conflict, &time_wait); + eltd->timer_set--; + ASSERT(eltd->timer_set >= 0); } else { - lcnt_update_stats(stats, eltd->lock_in_conflict, NULL); + lcnt_update_stats(stats, eltd->lock_in_conflict, NULL); } } @@ -525,27 +551,28 @@ void erts_lcnt_lock_post_x(erts_lcnt_lock_t *lock, char *file, unsigned int line void erts_lcnt_unlock_opt(erts_lcnt_lock_t *lock, Uint16 option) { if (erts_lcnt_rt_options & ERTS_LCNT_OPT_SUSPEND) return; - if (!ERTS_LCNT_LOCK_TYPE(lock)) return; + if (ERTS_LCNT_IS_LOCK_INVALID(lock)) return; if (option & ERTS_LCNT_LO_WRITE) ethr_atomic_dec(&lock->w_state); if (option & ERTS_LCNT_LO_READ ) ethr_atomic_dec(&lock->r_state); } void erts_lcnt_unlock(erts_lcnt_lock_t *lock) { -#ifdef DEBUG - erts_aint_t w_state; - erts_aint_t flowstate; -#endif if (erts_lcnt_rt_options & ERTS_LCNT_OPT_SUSPEND) return; - if (!ERTS_LCNT_LOCK_TYPE(lock)) return; + if (ERTS_LCNT_IS_LOCK_INVALID(lock)) return; #ifdef DEBUG - /* flowstate */ - flowstate = ethr_atomic_read(&lock->flowstate); - ASSERT(flowstate == 1); - ethr_atomic_dec( &lock->flowstate); - - /* write state */ - w_state = ethr_atomic_read(&lock->w_state); - ASSERT(w_state > 0) + { + erts_aint_t w_state; + erts_aint_t flowstate; + + /* flowstate */ + flowstate = ethr_atomic_read(&lock->flowstate); + ASSERT(flowstate == 1); + ethr_atomic_dec(&lock->flowstate); + + /* write state */ + w_state = ethr_atomic_read(&lock->w_state); + ASSERT(w_state > 0); + } #endif ethr_atomic_dec(&lock->w_state); } @@ -554,37 +581,34 @@ void erts_lcnt_unlock(erts_lcnt_lock_t *lock) { void erts_lcnt_trylock_opt(erts_lcnt_lock_t *lock, int res, Uint16 option) { if (erts_lcnt_rt_options & ERTS_LCNT_OPT_SUSPEND) return; - if (!ERTS_LCNT_LOCK_TYPE(lock)) return; + if (ERTS_LCNT_IS_LOCK_INVALID(lock)) return; /* Determine lock_state via res instead of state */ if (res != EBUSY) { - if (option & ERTS_LCNT_LO_WRITE) ethr_atomic_inc(&lock->w_state); - if (option & ERTS_LCNT_LO_READ ) ethr_atomic_inc(&lock->r_state); - lcnt_update_stats(&(lock->stats[0]), 0, NULL); + if (option & ERTS_LCNT_LO_WRITE) ethr_atomic_inc(&lock->w_state); + if (option & ERTS_LCNT_LO_READ ) ethr_atomic_inc(&lock->r_state); + lcnt_update_stats(&(lock->stats[0]), 0, NULL); } else { ethr_atomic_inc(&lock->stats[0].tries); ethr_atomic_inc(&lock->stats[0].colls); } } - + void erts_lcnt_trylock(erts_lcnt_lock_t *lock, int res) { /* Determine lock_state via res instead of state */ -#ifdef DEBUG - erts_aint_t flowstate; -#endif if (erts_lcnt_rt_options & ERTS_LCNT_OPT_SUSPEND) return; - if (!ERTS_LCNT_LOCK_TYPE(lock)) return; + if (ERTS_LCNT_IS_LOCK_INVALID(lock)) return; if (res != EBUSY) { - #ifdef DEBUG - flowstate = ethr_atomic_read(&lock->flowstate); - ASSERT(flowstate == 0); - ethr_atomic_inc( &lock->flowstate); + { + erts_aint_t flowstate; + flowstate = ethr_atomic_read(&lock->flowstate); + ASSERT(flowstate == 0); + ethr_atomic_inc( &lock->flowstate); + } #endif - ethr_atomic_inc(&lock->w_state); - - lcnt_update_stats(&(lock->stats[0]), 0, NULL); - + ethr_atomic_inc(&lock->w_state); + lcnt_update_stats(&(lock->stats[0]), 0, NULL); } else { ethr_atomic_inc(&lock->stats[0].tries); ethr_atomic_inc(&lock->stats[0].colls); @@ -639,13 +663,13 @@ void erts_lcnt_clear_counters(void) { lcnt_lock(); list = erts_lcnt_data->current_locks; - + for (lock = list->head; lock != NULL; lock = lock->next) { - for( i = 0; i < ERTS_LCNT_MAX_LOCK_LOCATIONS; i++) { - stats = &lock->stats[i]; - lcnt_clear_stats(stats); - } - lock->n_stats = 1; + for( i = 0; i < ERTS_LCNT_MAX_LOCK_LOCATIONS; i++) { + stats = &lock->stats[i]; + lcnt_clear_stats(stats); + } + lock->n_stats = 1; } /* empty deleted locks in lock list */ @@ -658,14 +682,14 @@ void erts_lcnt_clear_counters(void) { erts_lcnt_data_t *erts_lcnt_get_data(void) { erts_lcnt_time_t timer_stop; - + lcnt_lock(); - + lcnt_time(&timer_stop); lcnt_time_diff(&(erts_lcnt_data->duration), &timer_stop, &timer_start); - + lcnt_unlock(); - + return erts_lcnt_data; } diff --git a/erts/emulator/beam/erl_lock_count.h b/erts/emulator/beam/erl_lock_count.h index a4fc91b510..4cc6a5c695 100644 --- a/erts/emulator/beam/erl_lock_count.h +++ b/erts/emulator/beam/erl_lock_count.h @@ -3,16 +3,17 @@ * * Copyright Ericsson AB 2008-2012. All Rights Reserved. * - * The contents of this file are subject to the Erlang Public License, - * Version 1.1, (the "License"); you may not use this file except in - * compliance with the License. You should have received a copy of the - * Erlang Public License along with this software. If not, it can be - * retrieved online at http://www.erlang.org/. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Software distributed under the License is distributed on an "AS IS" - * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See - * the License for the specific language governing rights and limitations - * under the License. + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. * * %CopyrightEnd% */ @@ -20,7 +21,7 @@ /* * Description: Statistics for locks. * - * Author: Bj�rn-Egil Dahlberg + * Author: Björn-Egil Dahlberg * Date: 2008-07-03 * Abstract: * Locks statistics internal representation. @@ -35,6 +36,10 @@ * | | | - collisions (including trylock busy) * | | | - timer (time spent in waiting for lock) * | | | - n_timer (collisions excluding trylock busy) + * | | | - histogram + * | | | | - # 0 = log2(lock wait_time ns) + * | | | | - ... + * | | | | - # n = log2(lock wait_time ns) * * Each instance of a lock is the unique lock, i.e. set and id in that set. * For each lock there is a set of statistics with where and what impact @@ -61,9 +66,24 @@ #define ERTS_LOCK_COUNT_H__ #ifdef ERTS_ENABLE_LOCK_COUNT +#ifndef ERTS_ENABLE_LOCK_POSITION +/* Enable in order for _x variants of mtx functions to be used. */ +#define ERTS_ENABLE_LOCK_POSITION 1 +#endif + #include "ethread.h" -#define ERTS_LCNT_MAX_LOCK_LOCATIONS (10) +#define ERTS_LCNT_MAX_LOCK_LOCATIONS (10) + +/* histogram */ +#define ERTS_LCNT_HISTOGRAM_MAX_NS (((unsigned long)1LL << 28) - 1) +#if 0 || defined(ERTS_HAVE_OS_MONOTONIC_TIME_SUPPORT) +#define ERTS_LCNT_HISTOGRAM_SLOT_SIZE (30) +#define ERTS_LCNT_HISTOGRAM_RSHIFT (0) +#else +#define ERTS_LCNT_HISTOGRAM_SLOT_SIZE (20) +#define ERTS_LCNT_HISTOGRAM_RSHIFT (10) +#endif #define ERTS_LCNT_LT_SPINLOCK (((Uint16) 1) << 0) #define ERTS_LCNT_LT_RWSPINLOCK (((Uint16) 1) << 1) @@ -76,28 +96,37 @@ #define ERTS_LCNT_LO_WRITE (((Uint16) 1) << 7) #define ERTS_LCNT_LO_READ_WRITE ( ERTS_LCNT_LO_READ \ - | ERTS_LCNT_LO_WRITE ) + | ERTS_LCNT_LO_WRITE ) #define ERTS_LCNT_LT_ALL ( ERTS_LCNT_LT_SPINLOCK \ - | ERTS_LCNT_LT_RWSPINLOCK \ - | ERTS_LCNT_LT_MUTEX \ - | ERTS_LCNT_LT_RWMUTEX \ - | ERTS_LCNT_LT_PROCLOCK ) + | ERTS_LCNT_LT_RWSPINLOCK \ + | ERTS_LCNT_LT_MUTEX \ + | ERTS_LCNT_LT_RWMUTEX \ + | ERTS_LCNT_LT_PROCLOCK ) + +#define ERTS_LCNT_LOCK_TYPE(lock) ((lock)->flag & ERTS_LCNT_LT_ALL) +#define ERTS_LCNT_IS_LOCK_INVALID(lock) (!((lock)->flag & ERTS_LCNT_LT_ALL)) +#define ERTS_LCNT_CLEAR_FLAG(lock) ((lock)->flag = 0) + /* runtime options */ -#define ERTS_LCNT_OPT_SUSPEND (((Uint16) 1) << 0) -#define ERTS_LCNT_OPT_LOCATION (((Uint16) 1) << 1) -#define ERTS_LCNT_OPT_PROCLOCK (((Uint16) 1) << 2) -#define ERTS_LCNT_OPT_COPYSAVE (((Uint16) 1) << 3) -#define ERTS_LCNT_OPT_PORTLOCK (((Uint16) 1) << 4) +#define ERTS_LCNT_OPT_SUSPEND (((Uint16) 1) << 0) +#define ERTS_LCNT_OPT_LOCATION (((Uint16) 1) << 1) +#define ERTS_LCNT_OPT_PROCLOCK (((Uint16) 1) << 2) +#define ERTS_LCNT_OPT_PORTLOCK (((Uint16) 1) << 3) +#define ERTS_LCNT_OPT_COPYSAVE (((Uint16) 1) << 4) typedef struct { unsigned long s; unsigned long ns; } erts_lcnt_time_t; - + extern erts_lcnt_time_t timer_start; +typedef struct { + Uint32 ns[ERTS_LCNT_HISTOGRAM_SLOT_SIZE]; /* log2 array of nano seconds occurences */ +} erts_lcnt_hist_t; + typedef struct erts_lcnt_lock_stats_s { /* "tries" and "colls" needs to be atomic since * trylock busy does not aquire a lock and there @@ -106,12 +135,13 @@ typedef struct erts_lcnt_lock_stats_s { char *file; /* which file the lock was taken */ unsigned int line; /* line number in file */ - + ethr_atomic_t tries; /* n tries to get lock */ ethr_atomic_t colls; /* n collisions of tries to get lock */ - + unsigned long timer_n; /* #times waited for lock */ erts_lcnt_time_t timer; /* total wait time for lock */ + erts_lcnt_hist_t hist; } erts_lcnt_lock_stats_t; /* rw locks uses both states, other locks only uses w_state */ @@ -131,7 +161,7 @@ typedef struct erts_lcnt_lock_s { /* statistics */ unsigned int n_stats; erts_lcnt_lock_stats_t stats[ERTS_LCNT_MAX_LOCK_LOCATIONS]; /* first entry is "undefined"*/ - + /* chains for list handling */ /* data is hold by lcnt_lock */ struct erts_lcnt_lock_s *prev; @@ -143,7 +173,7 @@ typedef struct { erts_lcnt_lock_t *tail; unsigned long n; } erts_lcnt_lock_list_t; - + typedef struct { erts_lcnt_time_t duration; /* time since last clear */ erts_lcnt_lock_list_t *current_locks; @@ -181,6 +211,7 @@ void erts_lcnt_list_delete(erts_lcnt_lock_list_t *list, erts_lcnt_lock_t *lock); /* lock operations (global) */ void erts_lcnt_init_lock(erts_lcnt_lock_t *lock, char *name, Uint16 flag); void erts_lcnt_init_lock_x(erts_lcnt_lock_t *lock, char *name, Uint16 flag, Eterm id); +void erts_lcnt_init_lock_empty(erts_lcnt_lock_t *lock); void erts_lcnt_destroy_lock(erts_lcnt_lock_t *lock); void erts_lcnt_lock(erts_lcnt_lock_t *lock); @@ -202,7 +233,5 @@ void erts_lcnt_clear_counters(void); char *erts_lcnt_lock_type(Uint16 type); erts_lcnt_data_t *erts_lcnt_get_data(void); -#define ERTS_LCNT_LOCK_TYPE(lockp) ((lockp)->flag & ERTS_LCNT_LT_ALL) - #endif /* ifdef ERTS_ENABLE_LOCK_COUNT */ #endif /* ifndef ERTS_LOCK_COUNT_H__ */ diff --git a/erts/emulator/beam/erl_map.c b/erts/emulator/beam/erl_map.c new file mode 100644 index 0000000000..3c066cea7b --- /dev/null +++ b/erts/emulator/beam/erl_map.c @@ -0,0 +1,2918 @@ +/* + * %CopyrightBegin% + * + * Copyright Ericsson AB 2014. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * %CopyrightEnd% + * + * hashmaps are an adaption of Rich Hickeys Persistent HashMaps + * which were an adaption of Phil Bagwells - Hash Array Mapped Tries + * + * Author: Björn-Egil Dahlberg + */ + +#ifdef HAVE_CONFIG_H +# include "config.h" +#endif + +#include "sys.h" +#include "erl_vm.h" +#include "global.h" +#include "erl_process.h" +#include "error.h" +#define ERL_WANT_HIPE_BIF_WRAPPER__ +#include "bif.h" +#undef ERL_WANT_HIPE_BIF_WRAPPER__ +#include "erl_binary.h" + +#include "erl_map.h" + +/* BIFs + * + * DONE: + * - erlang:is_map/1 + * - erlang:map_size/1 + * + * - maps:find/2 + * - maps:from_list/1 + * - maps:get/2 + * - maps:is_key/2 + * - maps:keys/1 + * - maps:merge/2 + * - maps:new/0 + * - maps:put/3 + * - maps:remove/2 + * - maps:to_list/1 + * - maps:update/3 + * - maps:values/1 + * + * TODO: + * - maps:foldl/3 + * - maps:foldr/3 + * - maps:map/3 + * - maps:size/1 + * - maps:without/2 + * + * DEBUG: for sharing calculation + * - erts_internal:map_to_tuple_keys/1 + */ + +#ifndef DECL_AM +#define DECL_AM(S) Eterm AM_ ## S = am_atom_put(#S, sizeof(#S) - 1) +#endif + +/* for hashmap_from_list/1 */ +typedef struct { + Uint32 hx; + Uint32 skip; + Uint i; + Eterm val; +} hxnode_t; + + +static Eterm flatmap_merge(Process *p, Eterm nodeA, Eterm nodeB); +static BIF_RETTYPE map_merge_mixed(Process *p, Eterm flat, Eterm tree, int swap_args); +struct HashmapMergeContext_; +static BIF_RETTYPE hashmap_merge(Process *p, Eterm nodeA, Eterm nodeB, int swap_args, + struct HashmapMergeContext_*); +static Export hashmap_merge_trap_export; +static BIF_RETTYPE maps_merge_trap_1(BIF_ALIST_1); +static Uint hashmap_subtree_size(Eterm node); +static Eterm hashmap_to_list(Process *p, Eterm map); +static Eterm hashmap_keys(Process *p, Eterm map); +static Eterm hashmap_values(Process *p, Eterm map); +static Eterm hashmap_delete(Process *p, Uint32 hx, Eterm key, Eterm node); +static Eterm flatmap_from_validated_list(Process *p, Eterm list, Uint size); +static Eterm hashmap_from_validated_list(Process *p, Eterm list, Uint size); +static Eterm hashmap_from_unsorted_array(ErtsHeapFactory*, hxnode_t *hxns, Uint n, int reject_dupkeys); +static Eterm hashmap_from_sorted_unique_array(ErtsHeapFactory*, hxnode_t *hxns, Uint n, int is_root); +static Eterm hashmap_from_chunked_array(ErtsHeapFactory*, hxnode_t *hxns, Uint n, Uint size, int is_root); +static Eterm hashmap_info(Process *p, Eterm node); +static Eterm hashmap_bld_tuple_uint(Uint **hpp, Uint *szp, Uint n, Uint nums[]); +static int hxnodecmp(hxnode_t* a, hxnode_t* b); +static int hxnodecmpkey(hxnode_t* a, hxnode_t* b); + + +void erts_init_map(void) { + erts_init_trap_export(&hashmap_merge_trap_export, + am_maps, am_merge_trap, 1, + &maps_merge_trap_1); + return; +} + + +/* erlang:map_size/1 + * the corresponding instruction is implemented in: + * beam/erl_bif_guard.c + */ + +BIF_RETTYPE map_size_1(BIF_ALIST_1) { + if (is_flatmap(BIF_ARG_1)) { + flatmap_t *mp = (flatmap_t*)flatmap_val(BIF_ARG_1); + BIF_RET(make_small(flatmap_get_size(mp))); + } else if (is_hashmap(BIF_ARG_1)) { + Eterm *head, *hp, res; + Uint size, hsz=0; + + head = hashmap_val(BIF_ARG_1); + size = head[1]; + (void) erts_bld_uint(NULL, &hsz, size); + hp = HAlloc(BIF_P, hsz); + res = erts_bld_uint(&hp, NULL, size); + BIF_RET(res); + } + + BIF_P->fvalue = BIF_ARG_1; + BIF_ERROR(BIF_P, BADMAP); +} + +/* maps:to_list/1 */ + +BIF_RETTYPE maps_to_list_1(BIF_ALIST_1) { + if (is_flatmap(BIF_ARG_1)) { + Uint n; + Eterm* hp; + Eterm *ks,*vs, res, tup; + flatmap_t *mp = (flatmap_t*)flatmap_val(BIF_ARG_1); + + ks = flatmap_get_keys(mp); + vs = flatmap_get_values(mp); + n = flatmap_get_size(mp); + hp = HAlloc(BIF_P, (2 + 3) * n); + res = NIL; + + while(n--) { + tup = TUPLE2(hp, ks[n], vs[n]); hp += 3; + res = CONS(hp, tup, res); hp += 2; + } + + BIF_RET(res); + } else if (is_hashmap(BIF_ARG_1)) { + return hashmap_to_list(BIF_P, BIF_ARG_1); + } + + BIF_P->fvalue = BIF_ARG_1; + BIF_ERROR(BIF_P, BADMAP); +} + +/* maps:find/2 + * return value if key *matches* a key in the map + */ + +const Eterm * +#if HALFWORD_HEAP +erts_maps_get_rel(Eterm key, Eterm map, Eterm *map_base) +#else +erts_maps_get(Eterm key, Eterm map) +#endif +{ + Uint32 hx; + if (is_flatmap_rel(map, map_base)) { + Eterm *ks, *vs; + flatmap_t *mp; + Uint n, i; + + mp = (flatmap_t *)flatmap_val_rel(map, map_base); + n = flatmap_get_size(mp); + + if (n == 0) { + return NULL; + } + + ks = (Eterm *)tuple_val_rel(mp->keys, map_base) + 1; + vs = flatmap_get_values(mp); + + if (is_immed(key)) { + for (i = 0; i < n; i++) { + if (ks[i] == key) { + return &vs[i]; + } + } + } + + for (i = 0; i < n; i++) { + if (eq_rel(ks[i], map_base, key, NULL)) { + return &vs[i]; + } + } + return NULL; + } + ASSERT(is_hashmap_rel(map, map_base)); + hx = hashmap_make_hash(key); + + return erts_hashmap_get_rel(hx, key, map, map_base); +} + +BIF_RETTYPE maps_find_2(BIF_ALIST_2) { + if (is_map(BIF_ARG_2)) { + Eterm *hp, res; + const Eterm *value; + + value = erts_maps_get(BIF_ARG_1, BIF_ARG_2); + if (value) { + hp = HAlloc(BIF_P, 3); + res = make_tuple(hp); + *hp++ = make_arityval(2); + *hp++ = am_ok; + *hp++ = *value; + BIF_RET(res); + } + BIF_RET(am_error); + } + BIF_P->fvalue = BIF_ARG_2; + BIF_ERROR(BIF_P, BADMAP); +} + +/* maps:get/2 + * return value if key *matches* a key in the map + * exception badkey if none matches + */ + +BIF_RETTYPE maps_get_2(BIF_ALIST_2) { + if (is_map(BIF_ARG_2)) { + const Eterm *value; + + value = erts_maps_get(BIF_ARG_1, BIF_ARG_2); + if (value) { + BIF_RET(*value); + } + + BIF_P->fvalue = BIF_ARG_1; + BIF_ERROR(BIF_P, BADKEY); + } + BIF_P->fvalue = BIF_ARG_2; + BIF_ERROR(BIF_P, BADMAP); +} + +/* maps:from_list/1 + * List may be unsorted [{K,V}] + */ + +BIF_RETTYPE maps_from_list_1(BIF_ALIST_1) { + Eterm item = BIF_ARG_1, res, *kv; + Uint size = 0; + if (is_list(item) || is_nil(item)) { + + /* Calculate size and check validity */ + + while(is_list(item)) { + res = CAR(list_val(item)); + if (is_not_tuple(res)) + goto error; + + kv = tuple_val(res); + if (*kv != make_arityval(2)) + goto error; + + size++; + item = CDR(list_val(item)); + } + + if (is_not_nil(item)) + goto error; + + if (size > MAP_SMALL_MAP_LIMIT) { + BIF_RET(hashmap_from_validated_list(BIF_P, BIF_ARG_1, size)); + } else { + BIF_RET(flatmap_from_validated_list(BIF_P, BIF_ARG_1, size)); + } + } + +error: + + BIF_ERROR(BIF_P, BADARG); +} + +static Eterm flatmap_from_validated_list(Process *p, Eterm list, Uint size) { + Eterm *kv, item = list; + Eterm *hp, *thp,*vs, *ks, keys, res; + flatmap_t *mp; + Uint unused_size = 0; + Sint c = 0; + Sint idx = 0; + + + hp = HAlloc(p, 3 + 1 + (2 * size)); + thp = hp; + keys = make_tuple(hp); + *hp++ = make_arityval(size); + ks = hp; + hp += size; + mp = (flatmap_t*)hp; + res = make_flatmap(mp); + hp += MAP_HEADER_FLATMAP_SZ; + vs = hp; + + mp->thing_word = MAP_HEADER_FLATMAP; + mp->size = size; /* set later, might shrink*/ + mp->keys = keys; + + if (size == 0) + return res; + + /* first entry */ + kv = tuple_val(CAR(list_val(item))); + ks[0] = kv[1]; + vs[0] = kv[2]; + size = 1; + item = CDR(list_val(item)); + + /* insert sort key/value pairs */ + while(is_list(item)) { + + kv = tuple_val(CAR(list_val(item))); + + /* compare ks backwards + * idx represent word index to be written (hole position). + * We cannot copy the elements when searching since we might + * have an equal key. So we search for just the index first =( + * + * It is perhaps faster to move the values in the first pass. + * Check for uniqueness during insert phase and then have a + * second phace compacting the map if duplicates are found + * during insert. .. or do someother sort .. shell-sort perhaps. + */ + + idx = size; + + while(idx > 0 && (c = CMP_TERM(kv[1],ks[idx-1])) < 0) { idx--; } + + if (c == 0) { + /* last compare was equal, + * i.e. we have to release memory + * and overwrite that key/value + */ + ks[idx-1] = kv[1]; + vs[idx-1] = kv[2]; + unused_size++; + } else { + Uint i = size; + while(i > idx) { + ks[i] = ks[i-1]; + vs[i] = vs[i-1]; + i--; + } + ks[idx] = kv[1]; + vs[idx] = kv[2]; + size++; + } + item = CDR(list_val(item)); + } + + if (unused_size) { + /* the key tuple is embedded in the heap + * write a bignum to clear it. + */ + /* release values as normal since they are on the top of the heap */ + + ks[size] = make_pos_bignum_header(unused_size - 1); + HRelease(p, vs + size + unused_size, vs + size); + } + + *thp = make_arityval(size); + mp->size = size; + return res; +} + +#define swizzle32(D,S) \ + do { \ + (D) = ((S) & 0x0000000f) << 28 | ((S) & 0x000000f0) << 20 \ + | ((S) & 0x00000f00) << 12 | ((S) & 0x0000f000) << 4 \ + | ((S) & 0x000f0000) >> 4 | ((S) & 0x00f00000) >> 12 \ + | ((S) & 0x0f000000) >> 20 | ((S) & 0xf0000000) >> 28; \ + } while(0) + +#define maskval(V,L) (((V) >> ((7 - (L))*4)) & 0xf) +#define cdepth(V1,V2) (hashmap_clz((V1) ^ (V2)) >> 2) + +static Eterm hashmap_from_validated_list(Process *p, Eterm list, Uint size) { + Eterm item = list; + Eterm *hp; + Eterm *kv, res; + Uint32 sw, hx; + Uint ix = 0; + hxnode_t *hxns; + ErtsHeapFactory factory; + DeclareTmpHeap(tmp,2,p); + ASSERT(size > 0); + + hp = HAlloc(p, (2 * size)); + + /* create tmp hx values and leaf ptrs */ + hxns = (hxnode_t *)erts_alloc(ERTS_ALC_T_TMP, size * sizeof(hxnode_t)); + + UseTmpHeap(2,p); + while(is_list(item)) { + res = CAR(list_val(item)); + kv = tuple_val(res); + hx = hashmap_restore_hash(tmp,0,kv[1]); + swizzle32(sw,hx); + hxns[ix].hx = sw; + hxns[ix].val = CONS(hp, kv[1], kv[2]); hp += 2; + hxns[ix].skip = 1; /* will be reassigned in from_array */ + hxns[ix].i = ix; + ix++; + item = CDR(list_val(item)); + } + UnUseTmpHeap(2,p); + + erts_factory_proc_init(&factory, p); + res = hashmap_from_unsorted_array(&factory, hxns, size, 0); + erts_factory_close(&factory); + + erts_free(ERTS_ALC_T_TMP, (void *) hxns); + ERTS_VERIFY_UNUSED_TEMP_ALLOC(p); + + if (hashmap_size(res) <= MAP_SMALL_MAP_LIMIT) { + DECLARE_WSTACK(wstack); + Eterm *kv, *ks, *vs; + flatmap_t *mp; + Eterm keys; + Uint n = hashmap_size(res); + + /* build flat structure */ + hp = HAlloc(p, 3 + 1 + (2 * n)); + keys = make_tuple(hp); + *hp++ = make_arityval(n); + ks = hp; + hp += n; + mp = (flatmap_t*)hp; + hp += MAP_HEADER_FLATMAP_SZ; + vs = hp; + + mp->thing_word = MAP_HEADER_FLATMAP; + mp->size = n; + mp->keys = keys; + + hashmap_iterator_init(&wstack, res, 0); + + while ((kv=hashmap_iterator_next(&wstack)) != NULL) { + *ks++ = CAR(kv); + *vs++ = CDR(kv); + } + + /* it cannot have multiple keys */ + erts_validate_and_sort_flatmap(mp); + + DESTROY_WSTACK(wstack); + return make_flatmap(mp); + } + + return res; +} + +Eterm erts_hashmap_from_array(ErtsHeapFactory* factory, Eterm *leafs, Uint n, + int reject_dupkeys) { + Uint32 sw, hx; + Uint ix; + hxnode_t *hxns; + Eterm res; + + /* create tmp hx values and leaf ptrs */ + hxns = (hxnode_t *)erts_alloc(ERTS_ALC_T_TMP, n * sizeof(hxnode_t)); + + for (ix = 0; ix < n; ix++) { + hx = hashmap_make_hash(*leafs); + swizzle32(sw,hx); + hxns[ix].hx = sw; + hxns[ix].val = make_list(leafs); + hxns[ix].skip = 1; + hxns[ix].i = ix; + leafs += 2; + } + + res = hashmap_from_unsorted_array(factory, hxns, n, reject_dupkeys); + + erts_free(ERTS_ALC_T_TMP, (void *) hxns); + + return res; +} + + +Eterm erts_hashmap_from_ks_and_vs_extra(Process *p, Eterm *ks, Eterm *vs, Uint n, + Eterm key, Eterm value) { + Uint32 sw, hx; + Uint i,sz; + hxnode_t *hxns; + ErtsHeapFactory factory; + Eterm *hp, res; + + sz = (key == THE_NON_VALUE) ? n : (n + 1); + ASSERT(sz > MAP_SMALL_MAP_LIMIT); + hp = HAlloc(p, 2 * sz); + + /* create tmp hx values and leaf ptrs */ + hxns = (hxnode_t *)erts_alloc(ERTS_ALC_T_TMP, sz * sizeof(hxnode_t)); + + for(i = 0; i < n; i++) { + hx = hashmap_make_hash(ks[i]); + swizzle32(sw,hx); + hxns[i].hx = sw; + hxns[i].val = CONS(hp, ks[i], vs[i]); hp += 2; + hxns[i].skip = 1; /* will be reassigned in from_array */ + hxns[i].i = i; + } + + if (key != THE_NON_VALUE) { + hx = hashmap_make_hash(key); + swizzle32(sw,hx); + hxns[i].hx = sw; + hxns[i].val = CONS(hp, key, value); hp += 2; + hxns[i].skip = 1; + hxns[i].i = i; + } + + erts_factory_proc_init(&factory, p); + res = hashmap_from_unsorted_array(&factory, hxns, sz, 0); + erts_factory_close(&factory); + + erts_free(ERTS_ALC_T_TMP, (void *) hxns); + ERTS_VERIFY_UNUSED_TEMP_ALLOC(p); + + return res; +} + +static Eterm hashmap_from_unsorted_array(ErtsHeapFactory* factory, + hxnode_t *hxns, Uint n, + int reject_dupkeys) { + Uint jx = 0, ix = 0, lx, cx; + Eterm res; + + if (n == 0) { + Eterm *hp; + hp = erts_produce_heap(factory, 2, 0); + hp[0] = MAP_HEADER_HAMT_HEAD_BITMAP(0); + hp[1] = 0; + + return make_hashmap(hp); + } + + /* sort and compact array (remove non-unique entries) */ + qsort(hxns, n, sizeof(hxnode_t), (int (*)(const void *, const void *)) hxnodecmp); + + ix = 0, cx = 0; + while(ix < n - 1) { + if (hxns[ix].hx == hxns[ix+1].hx) { + + /* find region of equal hash values */ + jx = ix + 1; + while(jx < n && hxns[ix].hx == hxns[jx].hx) { jx++; } + /* find all correct keys from region + * (last in list but now hash sorted so we check highest id instead) */ + + /* resort with keys instead of hash value within region */ + + qsort(&hxns[ix], jx - ix, sizeof(hxnode_t), + (int (*)(const void *, const void *)) hxnodecmpkey); + + while(ix < jx) { + lx = ix; + while(++ix < jx && EQ(CAR(list_val(hxns[ix].val)), + CAR(list_val(hxns[lx].val)))) { + if (reject_dupkeys) + return THE_NON_VALUE; + + if (hxns[ix].i > hxns[lx].i) { + lx = ix; + } + } + hxns[cx].hx = hxns[lx].hx; + hxns[cx].val = hxns[lx].val; + cx++; + } + ix = jx; + continue; + } + if (ix > cx) { + hxns[cx].hx = hxns[ix].hx; + hxns[cx].val = hxns[ix].val; + } + cx++; + ix++; + } + + if (ix < n) { + hxns[cx].hx = hxns[ix].hx; + hxns[cx].val = hxns[ix].val; + cx++; + } + + if (cx > 1) { + /* recursive decompose array */ + res = hashmap_from_sorted_unique_array(factory, hxns, cx, 0); + } else { + Eterm *hp; + + /* we only have one item, either because n was 1 or + * because we hade multiples of the same key. + * + * hash value has been swizzled, need to drag it down to get the + * correct slot. */ + + hp = erts_produce_heap(factory, HAMT_HEAD_BITMAP_SZ(1), 0); + hp[0] = MAP_HEADER_HAMT_HEAD_BITMAP(1 << ((hxns[0].hx >> 0x1c) & 0xf)); + hp[1] = 1; + hp[2] = hxns[0].val; + res = make_hashmap(hp); + } + + return res; +} + +static Eterm hashmap_from_sorted_unique_array(ErtsHeapFactory* factory, + hxnode_t *hxns, Uint n, int lvl) { + Eterm res = NIL; + Uint i,ix,jx,elems; + Uint32 sw, hx; + Eterm val; + hxnode_t *tmp; + DeclareTmpHeapNoproc(th,2); + UseTmpHeapNoproc(2); + ASSERT(lvl < 32); + ix = 0; + elems = 1; + while (ix < n - 1) { + if (hxns[ix].hx == hxns[ix+1].hx) { + jx = ix + 1; + while (jx < n && hxns[ix].hx == hxns[jx].hx) { jx++; } + tmp = (hxnode_t *)erts_alloc(ERTS_ALC_T_TMP, ((jx - ix)) * sizeof(hxnode_t)); + + for(i = 0; i < jx - ix; i++) { + val = hxns[i + ix].val; + hx = hashmap_restore_hash(th, lvl + 8, CAR(list_val(val))); + swizzle32(sw,hx); + tmp[i].hx = sw; + tmp[i].val = val; + tmp[i].i = i; + tmp[i].skip = 1; + } + + qsort(tmp, jx - ix, sizeof(hxnode_t), (int (*)(const void *, const void *)) hxnodecmp); + + hxns[ix].skip = jx - ix; + hxns[ix].val = hashmap_from_sorted_unique_array(factory, tmp, jx - ix, lvl + 8); + erts_free(ERTS_ALC_T_TMP, (void *) tmp); + ix = jx; + if (ix < n) { elems++; } + continue; + } + hxns[ix].skip = 1; + elems++; + ix++; + } + + res = hashmap_from_chunked_array(factory, hxns, elems, n, !lvl); + + ERTS_FACTORY_HOLE_CHECK(factory); + + UnUseTmpHeapNoproc(2); + return res; +} + +#define HALLOC_EXTRA 200 +static Eterm hashmap_from_chunked_array(ErtsHeapFactory *factory, hxnode_t *hxns, Uint n, + Uint size, int is_root) { + Uint ix, d, dn, dc, slot, elems; + Uint32 v, vp, vn, hdr; + Uint bp, sz; + DECLARE_ESTACK(stack); + Eterm res = NIL, *hp = NULL, *nhp; + + + /* if we get here with only one element then + * we have eight levels of collisions + */ + + if (n == 1) { + res = hxns[0].val; + v = hxns[0].hx; + for (d = 7; d > 0; d--) { + slot = maskval(v,d); + hp = erts_produce_heap(factory, HAMT_NODE_BITMAP_SZ(1), HALLOC_EXTRA); + hp[0] = MAP_HEADER_HAMT_NODE_BITMAP(1 << slot); + hp[1] = res; + res = make_hashmap(hp); + } + + slot = maskval(v,0); + hp = erts_produce_heap(factory, (is_root ? 3 : 2), 0); + + if (is_root) { + hp[0] = MAP_HEADER_HAMT_HEAD_BITMAP(1 << slot); + hp[1] = size; + hp[2] = res; + } else { + hp[0] = MAP_HEADER_HAMT_NODE_BITMAP(1 << slot); + hp[1] = res; + } + return make_hashmap(hp); + } + + /* push initial nodes on the stack, + * this is the starting depth */ + + ix = 0; + d = 0; + vp = hxns[ix].hx; + v = hxns[ix + hxns[ix].skip].hx; + + ASSERT(vp > v); + slot = maskval(vp,d); + + while(slot == maskval(v,d)) { + ESTACK_PUSH(stack, 1 << slot); + d++; + slot = maskval(vp,d); + } + + res = hxns[ix].val; + + if (hxns[ix].skip > 1) { + dc = 7; + /* build collision nodes */ + while (dc > d) { + hp = erts_produce_heap(factory, HAMT_NODE_BITMAP_SZ(1), HALLOC_EXTRA); + hp[0] = MAP_HEADER_HAMT_NODE_BITMAP(1 << maskval(vp,dc)); + hp[1] = res; + res = make_hashmap(hp); + dc--; + } + } + + ESTACK_PUSH2(stack,res,1 << slot); + + /* all of the other nodes .. */ + elems = n - 2; /* remove first and last elements */ + while(elems--) { + hdr = ESTACK_POP(stack); + ix = ix + hxns[ix].skip; + + /* determine if node or subtree should be built by looking + * at the next value. */ + + vn = hxns[ix + hxns[ix].skip].hx; + dn = cdepth(v,vn); + ASSERT(v > vn); + + res = hxns[ix].val; + + if (hxns[ix].skip > 1) { + int wat = (d > dn) ? d : dn; + dc = 7; + /* build collision nodes */ + while (dc > wat) { + hp = erts_produce_heap(factory, HAMT_NODE_BITMAP_SZ(1), HALLOC_EXTRA); + hp[0] = MAP_HEADER_HAMT_NODE_BITMAP(1 << maskval(v,dc)); + hp[1] = res; + res = make_hashmap(hp); + dc--; + } + } + + /* next depth is higher (implies collision) */ + if (d < dn) { + /* hdr is the popped one initially */ + while(d < dn) { + slot = maskval(v, d); + bp = 1 << slot; + ESTACK_PUSH(stack, hdr | bp); + d++; + hdr = 0; /* clear hdr for all other collisions */ + } + + slot = maskval(v, d); + bp = 1 << slot; + /* no more collisions */ + ESTACK_PUSH2(stack,res,bp); + } else if (d == dn) { + /* no collisions at all */ + slot = maskval(v, d); + bp = 1 << slot; + ESTACK_PUSH2(stack,res,hdr | bp); + } else { + /* dn < n, we have a drop and we are done + * build nodes and subtree */ + while (dn != d) { + slot = maskval(v, d); + bp = 1 << slot; + /* OR bitposition before sz calculation to handle + * redundant collisions */ + hdr |= bp; + sz = hashmap_bitcount(hdr); + hp = erts_produce_heap(factory, HAMT_NODE_BITMAP_SZ(sz), HALLOC_EXTRA); + nhp = hp; + *hp++ = MAP_HEADER_HAMT_NODE_BITMAP(hdr); + *hp++ = res; sz--; + while (sz--) { *hp++ = ESTACK_POP(stack); } + ASSERT((hp - nhp) < 18); + res = make_hashmap(nhp); + + /* we need to pop the next hdr and push if we don't need it */ + + hdr = ESTACK_POP(stack); + d--; + } + ESTACK_PUSH2(stack,res,hdr); + } + + vp = v; + v = vn; + d = dn; + ERTS_FACTORY_HOLE_CHECK(factory); + } + + /* v and vp are reused from above */ + dn = cdepth(vp,v); + ix = ix + hxns[ix].skip; + res = hxns[ix].val; + + if (hxns[ix].skip > 1) { + dc = 7; + /* build collision nodes */ + while (dc > dn) { + hp = erts_produce_heap(factory, HAMT_NODE_BITMAP_SZ(1), HALLOC_EXTRA); + hp[0] = MAP_HEADER_HAMT_NODE_BITMAP(1 << maskval(v,dc)); + hp[1] = res; + res = make_hashmap(hp); + dc--; + } + } + + hdr = ESTACK_POP(stack); + /* pop remaining subtree if any */ + while (dn) { + slot = maskval(v, dn); + bp = 1 << slot; + /* OR bitposition before sz calculation to handle + * redundant collisions */ + hdr |= bp; + sz = hashmap_bitcount(hdr); + hp = erts_produce_heap(factory, HAMT_NODE_BITMAP_SZ(sz), HALLOC_EXTRA); + nhp = hp; + *hp++ = MAP_HEADER_HAMT_NODE_BITMAP(hdr); + *hp++ = res; sz--; + + while (sz--) { *hp++ = ESTACK_POP(stack); } + res = make_hashmap(nhp); + hdr = ESTACK_POP(stack); + dn--; + } + + /* and finally the root .. */ + + slot = maskval(v, dn); + bp = 1 << slot; + hdr |= bp; + sz = hashmap_bitcount(hdr); + hp = erts_produce_heap(factory, sz + /* hdr + item */ (is_root ? 2 : 1), 0); + nhp = hp; + + if (is_root) { + *hp++ = (hdr == 0xffff) ? MAP_HEADER_HAMT_HEAD_ARRAY : MAP_HEADER_HAMT_HEAD_BITMAP(hdr); + *hp++ = size; + } else { + *hp++ = MAP_HEADER_HAMT_NODE_BITMAP(hdr); + } + + *hp++ = res; sz--; + while (sz--) { *hp++ = ESTACK_POP(stack); } + + res = make_hashmap(nhp); + + ASSERT(ESTACK_COUNT(stack) == 0); + DESTROY_ESTACK(stack); + ERTS_FACTORY_HOLE_CHECK(factory); + return res; +} +#undef HALLOC_EXTRA + +static int hxnodecmpkey(hxnode_t *a, hxnode_t *b) { + Sint c = CMP_TERM(CAR(list_val(a->val)), CAR(list_val(b->val))); +#if ERTS_SIZEOF_ETERM <= SIZEOF_INT + return c; +#else + return c > 0 ? 1 : (c < 0 ? -1 : 0); +#endif +} + +static int hxnodecmp(hxnode_t *a, hxnode_t *b) { + if (a->hx < b->hx) + return 1; + else if (a->hx == b->hx) + return 0; + else + return -1; +} + +/* maps:is_key/2 */ + +BIF_RETTYPE maps_is_key_2(BIF_ALIST_2) { + if (is_map(BIF_ARG_2)) { + BIF_RET(erts_maps_get(BIF_ARG_1, BIF_ARG_2) ? am_true : am_false); + } + BIF_P->fvalue = BIF_ARG_2; + BIF_ERROR(BIF_P, BADMAP); +} + +/* maps:keys/1 */ + +BIF_RETTYPE maps_keys_1(BIF_ALIST_1) { + if (is_flatmap(BIF_ARG_1)) { + Eterm *hp, *ks, res = NIL; + flatmap_t *mp; + Uint n; + + mp = (flatmap_t*)flatmap_val(BIF_ARG_1); + n = flatmap_get_size(mp); + + if (n == 0) + BIF_RET(res); + + hp = HAlloc(BIF_P, (2 * n)); + ks = flatmap_get_keys(mp); + + while(n--) { + res = CONS(hp, ks[n], res); hp += 2; + } + + BIF_RET(res); + } else if (is_hashmap(BIF_ARG_1)) { + BIF_RET(hashmap_keys(BIF_P, BIF_ARG_1)); + } + BIF_P->fvalue = BIF_ARG_1; + BIF_ERROR(BIF_P, BADMAP); +} + +/* maps:merge/2 */ + +HIPE_WRAPPER_BIF_DISABLE_GC(maps_merge, 2) + +BIF_RETTYPE maps_merge_2(BIF_ALIST_2) { + if (is_flatmap(BIF_ARG_1)) { + if (is_flatmap(BIF_ARG_2)) { + BIF_RET(flatmap_merge(BIF_P, BIF_ARG_1, BIF_ARG_2)); + } else if (is_hashmap(BIF_ARG_2)) { + /* Will always become a tree */ + return map_merge_mixed(BIF_P, BIF_ARG_1, BIF_ARG_2, 0); + } + BIF_P->fvalue = BIF_ARG_2; + } else if (is_hashmap(BIF_ARG_1)) { + if (is_hashmap(BIF_ARG_2)) { + return hashmap_merge(BIF_P, BIF_ARG_1, BIF_ARG_2, 0, NULL); + } else if (is_flatmap(BIF_ARG_2)) { + /* Will always become a tree */ + return map_merge_mixed(BIF_P, BIF_ARG_2, BIF_ARG_1, 1); + } + BIF_P->fvalue = BIF_ARG_2; + } else { + BIF_P->fvalue = BIF_ARG_1; + } + BIF_ERROR(BIF_P, BADMAP); +} + +static Eterm flatmap_merge(Process *p, Eterm nodeA, Eterm nodeB) { + Eterm *hp,*thp; + Eterm tup; + Eterm *ks,*vs,*ks1,*vs1,*ks2,*vs2; + flatmap_t *mp1,*mp2,*mp_new; + Uint n,n1,n2,i1,i2,need,unused_size=0; + Sint c = 0; + + mp1 = (flatmap_t*)flatmap_val(nodeA); + mp2 = (flatmap_t*)flatmap_val(nodeB); + n1 = flatmap_get_size(mp1); + n2 = flatmap_get_size(mp2); + + need = MAP_HEADER_FLATMAP_SZ + 1 + 2 * (n1 + n2); + + hp = HAlloc(p, need); + thp = hp; + tup = make_tuple(thp); + ks = hp + 1; hp += 1 + n1 + n2; + mp_new = (flatmap_t*)hp; hp += MAP_HEADER_FLATMAP_SZ; + vs = hp; hp += n1 + n2; + + mp_new->thing_word = MAP_HEADER_FLATMAP; + mp_new->size = 0; + mp_new->keys = tup; + + i1 = 0; i2 = 0; + ks1 = flatmap_get_keys(mp1); + vs1 = flatmap_get_values(mp1); + ks2 = flatmap_get_keys(mp2); + vs2 = flatmap_get_values(mp2); + + while(i1 < n1 && i2 < n2) { + c = CMP_TERM(ks1[i1],ks2[i2]); + if (c == 0) { + /* use righthand side arguments map value, + * but advance both maps */ + *ks++ = ks2[i2]; + *vs++ = vs2[i2]; + i1++, i2++, unused_size++; + } else if (c < 0) { + *ks++ = ks1[i1]; + *vs++ = vs1[i1]; + i1++; + } else { + *ks++ = ks2[i2]; + *vs++ = vs2[i2]; + i2++; + } + } + + /* copy remaining */ + while (i1 < n1) { + *ks++ = ks1[i1]; + *vs++ = vs1[i1]; + i1++; + } + + while (i2 < n2) { + *ks++ = ks2[i2]; + *vs++ = vs2[i2]; + i2++; + } + + if (unused_size) { + /* the key tuple is embedded in the heap, write a bignum to clear it. + * + * release values as normal since they are on the top of the heap + * size = n1 + n1 - unused_size + */ + + *ks = make_pos_bignum_header(unused_size - 1); + HRelease(p, vs + unused_size, vs); + } + + n = n1 + n2 - unused_size; + *thp = make_arityval(n); + mp_new->size = n; + + /* Reshape map to a hashmap if the map exceeds the limit */ + + if (n > MAP_SMALL_MAP_LIMIT) { + Uint32 hx,sw; + Uint i; + Eterm res; + hxnode_t *hxns; + ErtsHeapFactory factory; + + ks = flatmap_get_keys(mp_new); + vs = flatmap_get_values(mp_new); + + hp = HAlloc(p, 2 * n); + + hxns = (hxnode_t *)erts_alloc(ERTS_ALC_T_TMP,n * sizeof(hxnode_t)); + + for (i = 0; i < n; i++) { + hx = hashmap_make_hash(ks[i]); + swizzle32(sw,hx); + hxns[i].hx = sw; + hxns[i].val = CONS(hp, ks[i], vs[i]); hp += 2; + hxns[i].skip = 1; + hxns[i].i = i; + } + + erts_factory_proc_init(&factory, p); + res = hashmap_from_unsorted_array(&factory, hxns, n, 0); + erts_factory_close(&factory); + + erts_free(ERTS_ALC_T_TMP, (void *) hxns); + ERTS_VERIFY_UNUSED_TEMP_ALLOC(p); + + return res; + } + + return make_flatmap(mp_new); +} + +static Eterm map_merge_mixed(Process *p, Eterm flat, Eterm tree, int swap_args) { + Eterm *ks, *vs, *hp, res; + flatmap_t *mp; + Uint n, i; + hxnode_t *hxns; + Uint32 sw, hx; + ErtsHeapFactory factory; + + /* convert flat to tree */ + + ASSERT(is_flatmap(flat)); + ASSERT(is_hashmap(tree)); + + mp = (flatmap_t*)flatmap_val(flat); + n = flatmap_get_size(mp); + + ks = flatmap_get_keys(mp); + vs = flatmap_get_values(mp); + + hp = HAlloc(p, 2 * n); + + hxns = (hxnode_t *)erts_alloc(ERTS_ALC_T_TMP, n * sizeof(hxnode_t)); + + for (i = 0; i < n; i++) { + hx = hashmap_make_hash(ks[i]); + swizzle32(sw,hx); + hxns[i].hx = sw; + hxns[i].val = CONS(hp, ks[i], vs[i]); hp += 2; + hxns[i].skip = 1; + hxns[i].i = i; + } + + erts_factory_proc_init(&factory, p); + res = hashmap_from_unsorted_array(&factory, hxns, n, 0); + erts_factory_close(&factory); + + erts_free(ERTS_ALC_T_TMP, (void *) hxns); + ERTS_VERIFY_UNUSED_TEMP_ALLOC(p); + + return hashmap_merge(p, res, tree, swap_args, NULL); +} + +#define PSTACK_TYPE struct HashmapMergePStackType +struct HashmapMergePStackType { + Eterm nodeA, nodeB; + Eterm *srcA, *srcB; + Uint32 abm, bbm, rbm; /* node bitmaps */ + int mix; /* &1: there are unique A stuff in node + * &2: there are unique B stuff in node */ + int ix; + Eterm array[16]; /* temp node construction area */ +}; + +typedef struct HashmapMergeContext_ { + Uint size; /* total key-value counter */ + unsigned int lvl; + Eterm trap_bin; + ErtsPStack pstack; +#ifdef DEBUG + Eterm dbg_map_A, dbg_map_B; +#endif +} HashmapMergeContext; + +static void hashmap_merge_ctx_destructor(Binary* ctx_bin) +{ + HashmapMergeContext* ctx = (HashmapMergeContext*) ERTS_MAGIC_BIN_DATA(ctx_bin); + ASSERT(ERTS_MAGIC_BIN_DESTRUCTOR(ctx_bin) == hashmap_merge_ctx_destructor); + + PSTACK_DESTROY_SAVED(&ctx->pstack); +} + +BIF_RETTYPE maps_merge_trap_1(BIF_ALIST_1) { + Binary* ctx_bin = ((ProcBin *) binary_val(BIF_ARG_1))->val; + + ASSERT(ERTS_MAGIC_BIN_DESTRUCTOR(ctx_bin) == hashmap_merge_ctx_destructor); + + return hashmap_merge(BIF_P, NIL, NIL, 0, + (HashmapMergeContext*) ERTS_MAGIC_BIN_DATA(ctx_bin)); +} + +#define HALLOC_EXTRA 200 +#define MAP_MERGE_LOOP_FACTOR 8 + +static BIF_RETTYPE hashmap_merge(Process *p, Eterm map_A, Eterm map_B, + int swap_args, HashmapMergeContext* ctx) { +#define PSTACK_TYPE struct HashmapMergePStackType + PSTACK_DECLARE(s, 4); + HashmapMergeContext local_ctx; + struct HashmapMergePStackType* sp; + Uint32 hx; + Eterm res = THE_NON_VALUE; + Eterm hdrA, hdrB; + Eterm *hp, *nhp; + Eterm trap_ret; + Sint initial_reds = (Sint) (ERTS_BIF_REDS_LEFT(p) * MAP_MERGE_LOOP_FACTOR); + Sint reds = initial_reds; + DeclareTmpHeap(th,2,p); + UseTmpHeap(2,p); + + /* + * Strategy: Do depth-first traversal of both trees (at the same time) + * and merge each pair of nodes. + */ + + PSTACK_CHANGE_ALLOCATOR(s, ERTS_ALC_T_SAVED_ESTACK); + + if (ctx == NULL) { /* first call */ + hashmap_head_t* a = (hashmap_head_t*) hashmap_val(map_A); + hashmap_head_t* b = (hashmap_head_t*) hashmap_val(map_B); + + sp = PSTACK_PUSH(s); + sp->srcA = swap_args ? &map_B : &map_A; + sp->srcB = swap_args ? &map_A : &map_B; + sp->mix = 0; + local_ctx.size = a->size + b->size; + local_ctx.lvl = 0; + #ifdef DEBUG + local_ctx.dbg_map_A = map_A; + local_ctx.dbg_map_B = map_B; + local_ctx.trap_bin = THE_NON_VALUE; + #endif + ctx = &local_ctx; + } + else { + PSTACK_RESTORE(s, &ctx->pstack); + sp = PSTACK_TOP(s); + goto resume_from_trap; + } + +recurse: + + sp->nodeA = *sp->srcA; + sp->nodeB = *sp->srcB; + + if (sp->nodeA == sp->nodeB) { + res = sp->nodeA; + ctx->size -= is_list(sp->nodeB) ? 1 : hashmap_subtree_size(sp->nodeB); + } + else { + if (is_list(sp->nodeA)) { /* A is LEAF */ + Eterm keyA = CAR(list_val(sp->nodeA)); + + if (is_list(sp->nodeB)) { /* LEAF + LEAF */ + Eterm keyB = CAR(list_val(sp->nodeB)); + + if (EQ(keyA, keyB)) { + --ctx->size; + res = sp->nodeB; + sp->mix = 2; /* We assume values differ. + + Don't spend time comparing big values. + - Might waste some heap space for internal + nodes that could otherwise be reused. */ + goto merge_nodes; + } + } + hx = hashmap_restore_hash(th, ctx->lvl, keyA); + sp->abm = 1 << hashmap_index(hx); + /* keep srcA pointing at the leaf */ + } + else { /* A is NODE */ + sp->srcA = boxed_val(sp->nodeA); + hdrA = *sp->srcA++; + ASSERT(is_header(hdrA)); + switch (hdrA & _HEADER_MAP_SUBTAG_MASK) { + case HAMT_SUBTAG_HEAD_ARRAY: { + sp->srcA++; + sp->abm = 0xffff; + break; + } + case HAMT_SUBTAG_HEAD_BITMAP: sp->srcA++; + case HAMT_SUBTAG_NODE_BITMAP: { + sp->abm = MAP_HEADER_VAL(hdrA); + break; + } + default: + erts_exit(ERTS_ABORT_EXIT, "bad header %ld\r\n", hdrA); + } + } + + if (is_list(sp->nodeB)) { /* B is LEAF */ + Eterm keyB = CAR(list_val(sp->nodeB)); + + hx = hashmap_restore_hash(th, ctx->lvl, keyB); + sp->bbm = 1 << hashmap_index(hx); + /* keep srcB pointing at the leaf */ + } + else { /* B is NODE */ + sp->srcB = boxed_val(sp->nodeB); + hdrB = *sp->srcB++; + ASSERT(is_header(hdrB)); + switch (hdrB & _HEADER_MAP_SUBTAG_MASK) { + case HAMT_SUBTAG_HEAD_ARRAY: { + sp->srcB++; + sp->bbm = 0xffff; + break; + } + case HAMT_SUBTAG_HEAD_BITMAP: sp->srcB++; + case HAMT_SUBTAG_NODE_BITMAP: { + sp->bbm = MAP_HEADER_VAL(hdrB); + break; + } + default: + erts_exit(ERTS_ABORT_EXIT, "bad header %ld\r\n", hdrB); + } + } + } + +merge_nodes: + + for (;;) { + if (is_value(res)) { /* We have a complete (sub-)tree or leaf */ + int child_mix; + if (ctx->lvl == 0) + break; + + /* Pop from stack and continue build parent node */ + ctx->lvl--; + child_mix = sp->mix; + sp = PSTACK_POP(s); + sp->array[sp->ix++] = res; + sp->mix |= child_mix; + res = THE_NON_VALUE; + if (sp->rbm) { + sp->srcA++; + sp->srcB++; + } + } else { /* Start build a node */ + sp->ix = 0; + sp->rbm = sp->abm | sp->bbm; + ASSERT(!(sp->rbm == 0 && ctx->lvl > 0)); + } + + if (--reds <= 0) { + goto trap; + } +resume_from_trap: + + while (sp->rbm) { + Uint32 next = sp->rbm & (sp->rbm-1); + Uint32 bit = sp->rbm ^ next; + sp->rbm = next; + if (sp->abm & bit) { + if (sp->bbm & bit) { + /* Bit clash. Push and resolve by recursive merge */ + Eterm* srcA = sp->srcA; + Eterm* srcB = sp->srcB; + ctx->lvl++; + sp = PSTACK_PUSH(s); + sp->srcA = srcA; + sp->srcB = srcB; + sp->mix = 0; + goto recurse; + } else { + sp->array[sp->ix++] = *sp->srcA++; + sp->mix |= 1; + } + } else { + ASSERT(sp->bbm & bit); + sp->array[sp->ix++] = *sp->srcB++; + sp->mix |= 2; + } + } + + switch (sp->mix) { + case 0: /* Nodes A and B contain the *EXACT* same sub-trees + => fall through and reuse nodeA */ + + case 1: /* Only unique A stuff => reuse nodeA */ + res = sp->nodeA; + break; + + case 2: /* Only unique B stuff => reuse nodeB */ + res = sp->nodeB; + break; + + case 3: /* We have a mix => must build new node */ + ASSERT(sp->ix == hashmap_bitcount(sp->abm | sp->bbm)); + if (ctx->lvl == 0) { + nhp = HAllocX(p, HAMT_HEAD_BITMAP_SZ(sp->ix), HALLOC_EXTRA); + hp = nhp; + *hp++ = (sp->ix == 16 ? MAP_HEADER_HAMT_HEAD_ARRAY + : MAP_HEADER_HAMT_HEAD_BITMAP(sp->abm | sp->bbm)); + *hp++ = ctx->size; + } else { + nhp = HAllocX(p, HAMT_NODE_BITMAP_SZ(sp->ix), HALLOC_EXTRA); + hp = nhp; + *hp++ = MAP_HEADER_HAMT_NODE_BITMAP(sp->abm | sp->bbm); + } + sys_memcpy(hp, sp->array, sp->ix * sizeof(Eterm)); + res = make_boxed(nhp); + break; + default: + erts_exit(ERTS_ABORT_EXIT, "strange mix %d\r\n", sp->mix); + } + } + + /* Done */ + +#ifdef DEBUG + { + Eterm *head = hashmap_val(res); + Uint size = head[1]; + Uint real_size = hashmap_subtree_size(res); + ASSERT(size == real_size); + } +#endif + + if (ctx != &local_ctx) { + ASSERT(ctx->trap_bin != THE_NON_VALUE); + ASSERT(p->flags & F_DISABLE_GC); + erts_set_gc_state(p, 1); + } + else { + ASSERT(ctx->trap_bin == THE_NON_VALUE); + ASSERT(!(p->flags & F_DISABLE_GC)); + } + PSTACK_DESTROY(s); + UnUseTmpHeap(2,p); + BUMP_REDS(p, (initial_reds - reds) / MAP_MERGE_LOOP_FACTOR); + return res; + +trap: /* Yield */ + + if (ctx == &local_ctx) { + Binary* ctx_b = erts_create_magic_binary(sizeof(HashmapMergeContext), + hashmap_merge_ctx_destructor); + ctx = ERTS_MAGIC_BIN_DATA(ctx_b); + sys_memcpy(ctx, &local_ctx, sizeof(HashmapMergeContext)); + hp = HAlloc(p, PROC_BIN_SIZE); + ASSERT(ctx->trap_bin == THE_NON_VALUE); + ctx->trap_bin = erts_mk_magic_binary_term(&hp, &MSO(p), ctx_b); + + erts_set_gc_state(p, 0); + } + else { + ASSERT(ctx->trap_bin != THE_NON_VALUE); + ASSERT(p->flags & F_DISABLE_GC); + } + + PSTACK_SAVE(s, &ctx->pstack); + + BUMP_ALL_REDS(p); + ERTS_BIF_PREP_TRAP1(trap_ret, &hashmap_merge_trap_export, + p, ctx->trap_bin); + UnUseTmpHeap(2,p); + return trap_ret; +} + +static Uint hashmap_subtree_size(Eterm node) { + DECLARE_WSTACK(stack); + Uint size = 0; + + hashmap_iterator_init(&stack, node, 0); + while (hashmap_iterator_next(&stack)) { + size++; + } + DESTROY_WSTACK(stack); + return size; +} + + +static int hash_cmp(Uint32 ha, Uint32 hb) +{ + int i; + for (i=0; i<8; i++) { + int cmp = (int)(ha & 0xF) - (int)(hb & 0xF); + if (cmp) + return cmp; + ha >>= 4; + hb >>= 4; + } + return 0; +} + +int hashmap_key_hash_cmp(Eterm* ap, Eterm* bp) +{ + unsigned int lvl = 0; + DeclareTmpHeapNoproc(th,2); + UseTmpHeapNoproc(2); + + if (ap && bp) { + ASSERT(CMP_TERM(CAR(ap), CAR(bp)) != 0); + for (;;) { + Uint32 ha = hashmap_restore_hash(th, lvl, CAR(ap)); + Uint32 hb = hashmap_restore_hash(th, lvl, CAR(bp)); + int cmp = hash_cmp(ha, hb); + if (cmp) { + UnUseTmpHeapNoproc(2); + return cmp; + } + lvl += 8; + } + } + UnUseTmpHeapNoproc(2); + return ap ? -1 : 1; +} + +/* maps:new/0 */ + +BIF_RETTYPE maps_new_0(BIF_ALIST_0) { + Eterm* hp; + Eterm tup; + flatmap_t *mp; + + hp = HAlloc(BIF_P, (MAP_HEADER_FLATMAP_SZ + 1)); + tup = make_tuple(hp); + *hp++ = make_arityval(0); + + mp = (flatmap_t*)hp; + mp->thing_word = MAP_HEADER_FLATMAP; + mp->size = 0; + mp->keys = tup; + + BIF_RET(make_flatmap(mp)); +} + +/* maps:put/3 */ + +BIF_RETTYPE maps_put_3(BIF_ALIST_3) { + if (is_map(BIF_ARG_3)) { + BIF_RET(erts_maps_put(BIF_P, BIF_ARG_1, BIF_ARG_2, BIF_ARG_3)); + } + BIF_P->fvalue = BIF_ARG_3; + BIF_ERROR(BIF_P, BADMAP); +} + +/* maps:remove/3 */ + +int erts_maps_remove(Process *p, Eterm key, Eterm map, Eterm *res) { + Uint32 hx; + if (is_flatmap(map)) { + Sint n; + Uint need; + Eterm *hp_start; + Eterm *thp, *mhp; + Eterm *ks, *vs, tup; + flatmap_t *mp = (flatmap_t*)flatmap_val(map); + + n = flatmap_get_size(mp); + + if (n == 0) { + *res = map; + return 1; + } + + ks = flatmap_get_keys(mp); + vs = flatmap_get_values(mp); + + /* Assume key exists. + * Release allocated if it didn't. + * Allocate key tuple first. + */ + + need = n + 1 - 1 + 3 + n - 1; /* tuple - 1 + map - 1 */ + hp_start = HAlloc(p, need); + thp = hp_start; + mhp = thp + n; /* offset with tuple heap size */ + + tup = make_tuple(thp); + *thp++ = make_arityval(n - 1); + + *res = make_flatmap(mhp); + *mhp++ = MAP_HEADER_FLATMAP; + *mhp++ = n - 1; + *mhp++ = tup; + + if (is_immed(key)) { + while (1) { + if (*ks == key) { + goto found_key; + } else if (--n) { + *mhp++ = *vs++; + *thp++ = *ks++; + } else + break; + } + } else { + while(1) { + if (EQ(*ks, key)) { + goto found_key; + } else if (--n) { + *mhp++ = *vs++; + *thp++ = *ks++; + } else + break; + } + } + + /* Not found, remove allocated memory + * and return previous map. + */ + HRelease(p, hp_start + need, hp_start); + + *res = map; + return 1; + +found_key: + /* Copy rest of keys and values */ + if (--n) { + sys_memcpy(mhp, vs+1, n*sizeof(Eterm)); + sys_memcpy(thp, ks+1, n*sizeof(Eterm)); + } + return 1; + } + ASSERT(is_hashmap(map)); + hx = hashmap_make_hash(key); + *res = hashmap_delete(p, hx, key, map); + return 1; +} + +BIF_RETTYPE maps_remove_2(BIF_ALIST_2) { + if (is_map(BIF_ARG_2)) { + Eterm res; + if (erts_maps_remove(BIF_P, BIF_ARG_1, BIF_ARG_2, &res)) { + BIF_RET(res); + } + } + BIF_P->fvalue = BIF_ARG_2; + BIF_ERROR(BIF_P, BADMAP); +} + +int erts_maps_update(Process *p, Eterm key, Eterm value, Eterm map, Eterm *res) { + Uint32 hx; + if (is_flatmap(map)) { + Sint n,i; + Eterm* hp,*shp; + Eterm *ks,*vs; + flatmap_t *mp = (flatmap_t*)flatmap_val(map); + + if ((n = flatmap_get_size(mp)) == 0) { + return 0; + } + + ks = flatmap_get_keys(mp); + vs = flatmap_get_values(mp); + + /* only allocate for values, + * assume key-tuple will be intact + */ + + hp = HAlloc(p, MAP_HEADER_FLATMAP_SZ + n); + shp = hp; + *hp++ = MAP_HEADER_FLATMAP; + *hp++ = n; + *hp++ = mp->keys; + + if (is_immed(key)) { + for( i = 0; i < n; i ++) { + if (ks[i] == key) { + goto found_key; + } else { + *hp++ = *vs++; + } + } + } else { + for( i = 0; i < n; i ++) { + if (EQ(ks[i], key)) { + goto found_key; + } else { + *hp++ = *vs++; + } + } + } + + HRelease(p, shp + MAP_HEADER_FLATMAP_SZ + n, shp); + return 0; + +found_key: + *hp++ = value; + vs++; + if (++i < n) + sys_memcpy(hp, vs, (n - i)*sizeof(Eterm)); + *res = make_flatmap(shp); + return 1; + } + + ASSERT(is_hashmap(map)); + hx = hashmap_make_hash(key); + *res = erts_hashmap_insert(p, hx, key, value, map, 1); + if (is_value(*res)) + return 1; + + return 0; +} + +Eterm erts_maps_put(Process *p, Eterm key, Eterm value, Eterm map) { + Uint32 hx; + Eterm res; + if (is_flatmap(map)) { + Sint n,i; + Sint c = 0; + Eterm* hp, *shp; + Eterm *ks, *vs, tup; + flatmap_t *mp = (flatmap_t*)flatmap_val(map); + + n = flatmap_get_size(mp); + + if (n == 0) { + hp = HAlloc(p, MAP_HEADER_FLATMAP_SZ + 1 + 2); + tup = make_tuple(hp); + *hp++ = make_arityval(1); + *hp++ = key; + res = make_flatmap(hp); + *hp++ = MAP_HEADER_FLATMAP; + *hp++ = 1; + *hp++ = tup; + *hp++ = value; + + return res; + } + + ks = flatmap_get_keys(mp); + vs = flatmap_get_values(mp); + + /* only allocate for values, + * assume key-tuple will be intact + */ + + hp = HAlloc(p, MAP_HEADER_FLATMAP_SZ + n); + shp = hp; /* save hp, used if optimistic update fails */ + res = make_flatmap(hp); + *hp++ = MAP_HEADER_FLATMAP; + *hp++ = n; + *hp++ = mp->keys; + + if (is_immed(key)) { + for( i = 0; i < n; i ++) { + if (ks[i] == key) { + *hp++ = value; + vs++; + c = 1; + } else { + *hp++ = *vs++; + } + } + } else { + for( i = 0; i < n; i ++) { + if (EQ(ks[i], key)) { + *hp++ = value; + vs++; + c = 1; + } else { + *hp++ = *vs++; + } + } + } + + if (c) + return res; + + /* the map will grow */ + + if (n >= MAP_SMALL_MAP_LIMIT) { + HRelease(p, shp + MAP_HEADER_FLATMAP_SZ + n, shp); + ks = flatmap_get_keys(mp); + vs = flatmap_get_values(mp); + + res = erts_hashmap_from_ks_and_vs_extra(p,ks,vs,n,key,value); + + return res; + } + + /* still a small map. need to make a new tuple, + * use old hp since it needs to be recreated anyway. */ + + tup = make_tuple(shp); + *shp++ = make_arityval(n+1); + + hp = HAlloc(p, 3 + n + 1); + res = make_flatmap(hp); + *hp++ = MAP_HEADER_FLATMAP; + *hp++ = n + 1; + *hp++ = tup; + + ks = flatmap_get_keys(mp); + vs = flatmap_get_values(mp); + + ASSERT(n >= 0); + + /* copy map in order */ + while (n && ((c = CMP_TERM(*ks, key)) < 0)) { + *shp++ = *ks++; + *hp++ = *vs++; + n--; + } + + *shp++ = key; + *hp++ = value; + + ASSERT(n >= 0); + + while(n--) { + *shp++ = *ks++; + *hp++ = *vs++; + } + /* we have one word remaining + * this will work out fine once we get the size word + * in the header. + */ + *shp = make_pos_bignum_header(0); + return res; + } + ASSERT(is_hashmap(map)); + + hx = hashmap_make_hash(key); + res = erts_hashmap_insert(p, hx, key, value, map, 0); + ASSERT(is_hashmap(res)); + + return res; +} + +/* maps:update/3 */ + +BIF_RETTYPE maps_update_3(BIF_ALIST_3) { + if (is_not_map(BIF_ARG_3)) { + BIF_P->fvalue = BIF_ARG_3; + BIF_ERROR(BIF_P, BADMAP); + } else { + Eterm res; + if (erts_maps_update(BIF_P, BIF_ARG_1, BIF_ARG_2, BIF_ARG_3, &res)) { + BIF_RET(res); + } + BIF_P->fvalue = BIF_ARG_1; + BIF_ERROR(BIF_P, BADKEY); + } +} + + +/* maps:values/1 */ + +BIF_RETTYPE maps_values_1(BIF_ALIST_1) { + if (is_flatmap(BIF_ARG_1)) { + Eterm *hp, *vs, res = NIL; + flatmap_t *mp; + Uint n; + + mp = (flatmap_t*)flatmap_val(BIF_ARG_1); + n = flatmap_get_size(mp); + + if (n == 0) + BIF_RET(res); + + hp = HAlloc(BIF_P, (2 * n)); + vs = flatmap_get_values(mp); + + while(n--) { + res = CONS(hp, vs[n], res); hp += 2; + } + + BIF_RET(res); + } else if (is_hashmap(BIF_ARG_1)) { + BIF_RET(hashmap_values(BIF_P, BIF_ARG_1)); + } + BIF_P->fvalue = BIF_ARG_1; + BIF_ERROR(BIF_P, BADMAP); +} + +static Eterm hashmap_to_list(Process *p, Eterm node) { + DECLARE_WSTACK(stack); + Eterm *hp, *kv; + Eterm res = NIL; + + hp = HAlloc(p, hashmap_size(node) * (2 + 3)); + hashmap_iterator_init(&stack, node, 0); + while ((kv=hashmap_iterator_next(&stack)) != NULL) { + Eterm tup = TUPLE2(hp, CAR(kv), CDR(kv)); + hp += 3; + res = CONS(hp, tup, res); + hp += 2; + } + DESTROY_WSTACK(stack); + return res; +} + +void hashmap_iterator_init(ErtsWStack* s, Eterm node, int reverse) { + Eterm hdr = *hashmap_val(node); + Uint sz; + + switch(hdr & _HEADER_MAP_SUBTAG_MASK) { + case HAMT_SUBTAG_HEAD_ARRAY: + sz = 16; + break; + case HAMT_SUBTAG_HEAD_BITMAP: + case HAMT_SUBTAG_NODE_BITMAP: + sz = hashmap_bitcount(MAP_HEADER_VAL(hdr)); + break; + default: + erts_exit(ERTS_ABORT_EXIT, "bad header"); + } + + WSTACK_PUSH3((*s), (UWord)THE_NON_VALUE, /* end marker */ + (UWord)(!reverse ? 0 : sz+1), + (UWord)node); +} + +Eterm* hashmap_iterator_next(ErtsWStack* s) { + Eterm node, *ptr, hdr; + Uint32 sz; + Uint idx; + + for (;;) { + ASSERT(!WSTACK_ISEMPTY((*s))); + node = (Eterm) WSTACK_POP((*s)); + if (is_non_value(node)) { + return NULL; + } + idx = (Uint) WSTACK_POP((*s)); + for (;;) { + ASSERT(is_boxed(node)); + ptr = boxed_val(node); + hdr = *ptr; + ASSERT(is_header(hdr)); + switch(hdr & _HEADER_MAP_SUBTAG_MASK) { + case HAMT_SUBTAG_HEAD_ARRAY: + ptr++; + sz = 16; + break; + case HAMT_SUBTAG_HEAD_BITMAP: + ptr++; + case HAMT_SUBTAG_NODE_BITMAP: + sz = hashmap_bitcount(MAP_HEADER_VAL(hdr)); + ASSERT(sz < 17); + break; + default: + erts_exit(ERTS_ABORT_EXIT, "bad header"); + } + + idx++; + + if (idx <= sz) { + WSTACK_PUSH2((*s), (UWord)idx, (UWord)node); + + if (is_list(ptr[idx])) { + return list_val(ptr[idx]); + } + ASSERT(is_boxed(ptr[idx])); + node = ptr[idx]; + idx = 0; + } + else + break; /* and pop parent node */ + } + } +} + +Eterm* hashmap_iterator_prev(ErtsWStack* s) { + Eterm node, *ptr, hdr; + Uint32 sz; + Uint idx; + + for (;;) { + ASSERT(!WSTACK_ISEMPTY((*s))); + node = (Eterm) WSTACK_POP((*s)); + if (is_non_value(node)) { + return NULL; + } + idx = (Uint) WSTACK_POP((*s)); + for (;;) { + ASSERT(is_boxed(node)); + ptr = boxed_val(node); + hdr = *ptr; + ASSERT(is_header(hdr)); + switch(hdr & _HEADER_MAP_SUBTAG_MASK) { + case HAMT_SUBTAG_HEAD_ARRAY: + ptr++; + sz = 16; + break; + case HAMT_SUBTAG_HEAD_BITMAP: + ptr++; + case HAMT_SUBTAG_NODE_BITMAP: + sz = hashmap_bitcount(MAP_HEADER_VAL(hdr)); + ASSERT(sz < 17); + break; + default: + erts_exit(ERTS_ERROR_EXIT, "bad header"); + } + + if (idx > sz) + idx = sz; + else + idx--; + + if (idx >= 1) { + WSTACK_PUSH2((*s), (UWord)idx, (UWord)node); + + if (is_list(ptr[idx])) { + return list_val(ptr[idx]); + } + ASSERT(is_boxed(ptr[idx])); + node = ptr[idx]; + idx = 17; + } + else + break; /* and pop parent node */ + } + } +} + +const Eterm * +#if HALFWORD_HEAP +erts_hashmap_get_rel(Uint32 hx, Eterm key, Eterm node, Eterm *map_base) +#else +erts_hashmap_get(Uint32 hx, Eterm key, Eterm node) +#endif +{ + Eterm *ptr, hdr, *res; + Uint ix, lvl = 0; + Uint32 hval,bp; + DeclareTmpHeapNoproc(th,2); + UseTmpHeapNoproc(2); + + ASSERT(is_boxed(node)); + ptr = boxed_val_rel(node, map_base); + hdr = *ptr; + ASSERT(is_header(hdr)); + ASSERT(is_hashmap_header_head(hdr)); + ptr++; + + for (;;) { + hval = MAP_HEADER_VAL(hdr); + ix = hashmap_index(hx); + if (hval != 0xffff) { + bp = 1 << ix; + if (!(bp & hval)) { + /* not occupied */ + res = NULL; + break; + } + ix = hashmap_bitcount(hval & (bp - 1)); + } + node = ptr[ix+1]; + + if (is_list(node)) { /* LEAF NODE [K|V] */ + ptr = list_val_rel(node,map_base); + res = eq_rel(CAR(ptr), map_base, key, NULL) ? &(CDR(ptr)) : NULL; + break; + } + + hx = hashmap_shift_hash(th,hx,lvl,key); + + ASSERT(is_boxed(node)); + ptr = boxed_val_rel(node, map_base); + hdr = *ptr; + ASSERT(is_header(hdr)); + ASSERT(!is_hashmap_header_head(hdr)); + } + + UnUseTmpHeapNoproc(2); + return res; +} + +Eterm erts_hashmap_insert(Process *p, Uint32 hx, Eterm key, Eterm value, + Eterm map, int is_update) { + Uint size, upsz; + Eterm *hp, res = THE_NON_VALUE; + DECLARE_ESTACK(stack); + if (erts_hashmap_insert_down(hx, key, map, &size, &upsz, &stack, is_update)) { + hp = HAlloc(p, size); + res = erts_hashmap_insert_up(hp, key, value, &upsz, &stack); + } + + DESTROY_ESTACK(stack); + ERTS_VERIFY_UNUSED_TEMP_ALLOC(p); + ERTS_HOLE_CHECK(p); + + return res; +} + + +int erts_hashmap_insert_down(Uint32 hx, Eterm key, Eterm node, Uint *sz, + Uint *update_size, ErtsEStack *sp, int is_update) { + Eterm *ptr; + Eterm hdr, ckey; + Uint32 ix, cix, bp, hval, chx; + Uint slot, lvl = 0, clvl; + Uint size = 0, n = 0; + DeclareTmpHeapNoproc(th,2); + + *update_size = 1; + + UseTmpHeapNoproc(2); + for (;;) { + switch(primary_tag(node)) { + case TAG_PRIMARY_LIST: /* LEAF NODE [K|V] */ + ptr = list_val(node); + ckey = CAR(ptr); + if (EQ(ckey, key)) { + *update_size = 0; + goto unroll; + } + if (is_update) { + UnUseTmpHeapNoproc(2); + return 0; + } + goto insert_subnodes; + case TAG_PRIMARY_BOXED: + ptr = boxed_val(node); + hdr = *ptr; + ASSERT(is_header(hdr)); + + switch(hdr & _HEADER_MAP_SUBTAG_MASK) { + case HAMT_SUBTAG_HEAD_ARRAY: + ix = hashmap_index(hx); + hx = hashmap_shift_hash(th,hx,lvl,key); + size += HAMT_HEAD_ARRAY_SZ; + ESTACK_PUSH2(*sp, ix, node); + node = ptr[ix+2]; + break; + case HAMT_SUBTAG_NODE_BITMAP: + hval = MAP_HEADER_VAL(hdr); + ix = hashmap_index(hx); + bp = 1 << ix; + if (hval == 0xffff) { + slot = ix; + n = 16; + } else { + slot = hashmap_bitcount(hval & (bp - 1)); + n = hashmap_bitcount(hval); + } + + ESTACK_PUSH4(*sp, n, bp, slot, node); + + if (!(bp & hval)) { /* not occupied */ + if (is_update) { + UnUseTmpHeapNoproc(2); + return 0; + } + size += HAMT_NODE_BITMAP_SZ(n+1); + goto unroll; + } + + hx = hashmap_shift_hash(th,hx,lvl,key); + node = ptr[slot+1]; + ASSERT(HAMT_NODE_BITMAP_SZ(n) <= 17); + size += HAMT_NODE_BITMAP_SZ(n); + break; + + case HAMT_SUBTAG_HEAD_BITMAP: + hval = MAP_HEADER_VAL(hdr); + ix = hashmap_index(hx); + bp = 1 << ix; + slot = hashmap_bitcount(hval & (bp - 1)); + n = hashmap_bitcount(hval); + + ESTACK_PUSH4(*sp, n, bp, slot, node); + + /* occupied */ + if (bp & hval) { + hx = hashmap_shift_hash(th,hx,lvl,key); + node = ptr[slot+2]; + ASSERT(HAMT_HEAD_BITMAP_SZ(n) <= 18); + size += HAMT_HEAD_BITMAP_SZ(n); + break; + } + /* not occupied */ + if (is_update) { + UnUseTmpHeapNoproc(2); + return 0; + } + size += HAMT_HEAD_BITMAP_SZ(n+1); + goto unroll; + default: + erts_exit(ERTS_ERROR_EXIT, "bad header tag %ld\r\n", hdr & _HEADER_MAP_SUBTAG_MASK); + break; + } + break; + default: + erts_exit(ERTS_ERROR_EXIT, "bad primary tag %p\r\n", node); + break; + } + } +insert_subnodes: + clvl = lvl; + chx = hashmap_restore_hash(th,clvl,ckey); + size += HAMT_NODE_BITMAP_SZ(2); + ix = hashmap_index(hx); + cix = hashmap_index(chx); + + while (cix == ix) { + ESTACK_PUSH4(*sp, 0, 1 << ix, 0, MAP_HEADER_HAMT_NODE_BITMAP(0)); + size += HAMT_NODE_BITMAP_SZ(1); + hx = hashmap_shift_hash(th,hx,lvl,key); + chx = hashmap_shift_hash(th,chx,clvl,ckey); + ix = hashmap_index(hx); + cix = hashmap_index(chx); + } + ESTACK_PUSH3(*sp, cix, ix, node); + +unroll: + *sz = size + /* res cons */ 2; + UnUseTmpHeapNoproc(2); + return 1; +} + +Eterm erts_hashmap_insert_up(Eterm *hp, Eterm key, Eterm value, + Uint *update_size, ErtsEStack *sp) { + Eterm node, *ptr, hdr; + Eterm res; + Eterm *nhp = NULL; + Uint32 ix, cix, bp, hval; + Uint slot, n; + /* Needed for halfword */ + DeclareTmpHeapNoproc(fake,1); + UseTmpHeapNoproc(1); + + res = CONS(hp, key, value); hp += 2; + + do { + node = ESTACK_POP(*sp); + switch(primary_tag(node)) { + case TAG_PRIMARY_LIST: + ix = (Uint32) ESTACK_POP(*sp); + cix = (Uint32) ESTACK_POP(*sp); + + nhp = hp; + *hp++ = MAP_HEADER_HAMT_NODE_BITMAP((1 << ix) | (1 << cix)); + if (ix < cix) { + *hp++ = res; + *hp++ = node; + } else { + *hp++ = node; + *hp++ = res; + } + res = make_hashmap(nhp); + break; + case TAG_PRIMARY_HEADER: + /* subnodes, fake it */ + *fake = node; + node = make_boxed(fake); + case TAG_PRIMARY_BOXED: + ptr = boxed_val(node); + hdr = *ptr; + ASSERT(is_header(hdr)); + + switch(hdr & _HEADER_MAP_SUBTAG_MASK) { + case HAMT_SUBTAG_HEAD_ARRAY: + slot = (Uint) ESTACK_POP(*sp); + nhp = hp; + n = HAMT_HEAD_ARRAY_SZ - 2; + *hp++ = MAP_HEADER_HAMT_HEAD_ARRAY; ptr++; + *hp++ = (*ptr++) + *update_size; + while(n--) { *hp++ = *ptr++; } + nhp[slot+2] = res; + res = make_hashmap(nhp); + break; + case HAMT_SUBTAG_NODE_BITMAP: + slot = (Uint) ESTACK_POP(*sp); + bp = (Uint32) ESTACK_POP(*sp); + n = (Uint32) ESTACK_POP(*sp); + hval = MAP_HEADER_VAL(hdr); + nhp = hp; + *hp++ = MAP_HEADER_HAMT_NODE_BITMAP(hval | bp); ptr++; + + n -= slot; + while(slot--) { *hp++ = *ptr++; } + *hp++ = res; + if (hval & bp) { ptr++; n--; } + while(n--) { *hp++ = *ptr++; } + + res = make_hashmap(nhp); + break; + case HAMT_SUBTAG_HEAD_BITMAP: + slot = (Uint) ESTACK_POP(*sp); + bp = (Uint32) ESTACK_POP(*sp); + n = (Uint32) ESTACK_POP(*sp); + hval = MAP_HEADER_VAL(hdr); + nhp = hp; + *hp++ = MAP_HEADER_HAMT_HEAD_BITMAP(hval | bp); ptr++; + *hp++ = (*ptr++) + *update_size; + + n -= slot; + while(slot--) { *hp++ = *ptr++; } + *hp++ = res; + if (hval & bp) { ptr++; n--; } + while(n--) { *hp++ = *ptr++; } + + if ((hval | bp) == 0xffff) { + *nhp = MAP_HEADER_HAMT_HEAD_ARRAY; + } + res = make_hashmap(nhp); + break; + default: + erts_exit(ERTS_ERROR_EXIT, "bad header tag %x\r\n", hdr & _HEADER_MAP_SUBTAG_MASK); + break; + } + break; + default: + erts_exit(ERTS_ERROR_EXIT, "bad primary tag %x\r\n", primary_tag(node)); + break; + } + + } while(!ESTACK_ISEMPTY(*sp)); + + UnUseTmpHeapNoproc(1); + return res; +} + +static Eterm hashmap_keys(Process* p, Eterm node) { + DECLARE_WSTACK(stack); + hashmap_head_t* root; + Eterm *hp, *kv; + Eterm res = NIL; + + root = (hashmap_head_t*) boxed_val(node); + hp = HAlloc(p, root->size * 2); + hashmap_iterator_init(&stack, node, 0); + while ((kv=hashmap_iterator_next(&stack)) != NULL) { + res = CONS(hp, CAR(kv), res); + hp += 2; + } + DESTROY_WSTACK(stack); + return res; +} + +static Eterm hashmap_values(Process* p, Eterm node) { + DECLARE_WSTACK(stack); + hashmap_head_t* root; + Eterm *hp, *kv; + Eterm res = NIL; + + root = (hashmap_head_t*) boxed_val(node); + hp = HAlloc(p, root->size * 2); + hashmap_iterator_init(&stack, node, 0); + while ((kv=hashmap_iterator_next(&stack)) != NULL) { + res = CONS(hp, CDR(kv), res); + hp += 2; + } + DESTROY_WSTACK(stack); + return res; +} + +static Eterm hashmap_delete(Process *p, Uint32 hx, Eterm key, Eterm map) { + Eterm *hp = NULL, *nhp = NULL, *hp_end = NULL; + Eterm *ptr; + Eterm hdr, res = map, node = map; + Uint32 ix, bp, hval; + Uint slot, lvl = 0; + Uint size = 0, n = 0; + DECLARE_ESTACK(stack); + DeclareTmpHeapNoproc(th,2); + UseTmpHeapNoproc(2); + + for (;;) { + switch(primary_tag(node)) { + case TAG_PRIMARY_LIST: + if (EQ(CAR(list_val(node)), key)) { + goto unroll; + } + goto not_found; + case TAG_PRIMARY_BOXED: + ptr = boxed_val(node); + hdr = *ptr; + ASSERT(is_header(hdr)); + + switch(hdr & _HEADER_MAP_SUBTAG_MASK) { + case HAMT_SUBTAG_HEAD_ARRAY: + ix = hashmap_index(hx); + hx = hashmap_shift_hash(th,hx,lvl,key); + size += HAMT_HEAD_ARRAY_SZ; + ESTACK_PUSH2(stack, ix, node); + node = ptr[ix+2]; + break; + case HAMT_SUBTAG_NODE_BITMAP: + hval = MAP_HEADER_VAL(hdr); + ix = hashmap_index(hx); + bp = 1 << ix; + if (hval == 0xffff) { + slot = ix; + n = 16; + } else if (bp & hval) { + slot = hashmap_bitcount(hval & (bp - 1)); + n = hashmap_bitcount(hval); + } else { + /* not occupied */ + goto not_found; + } + + ESTACK_PUSH4(stack, n, bp, slot, node); + + hx = hashmap_shift_hash(th,hx,lvl,key); + node = ptr[slot+1]; + ASSERT(HAMT_NODE_BITMAP_SZ(n) <= 17); + size += HAMT_NODE_BITMAP_SZ(n); + break; + + case HAMT_SUBTAG_HEAD_BITMAP: + hval = MAP_HEADER_VAL(hdr); + ix = hashmap_index(hx); + bp = 1 << ix; + slot = hashmap_bitcount(hval & (bp - 1)); + n = hashmap_bitcount(hval); + + ESTACK_PUSH4(stack, n, bp, slot, node); + + /* occupied */ + if (bp & hval) { + hx = hashmap_shift_hash(th,hx,lvl,key); + node = ptr[slot+2]; + ASSERT(HAMT_HEAD_BITMAP_SZ(n) <= 18); + size += HAMT_HEAD_BITMAP_SZ(n); + break; + } + /* not occupied */ + goto not_found; + default: + erts_exit(ERTS_ERROR_EXIT, "bad header tag %ld\r\n", hdr & _HEADER_MAP_SUBTAG_MASK); + break; + } + break; + default: + erts_exit(ERTS_ERROR_EXIT, "bad primary tag %p\r\n", node); + break; + } + } + +unroll: + /* the size is bounded and atleast one less than the previous size */ + size -= 1; + n = hashmap_size(map) - 1; + + if (n <= MAP_SMALL_MAP_LIMIT) { + DECLARE_WSTACK(wstack); + Eterm *kv, *ks, *vs; + flatmap_t *mp; + Eterm keys; + + DESTROY_ESTACK(stack); + + /* build flat structure */ + hp = HAlloc(p, 3 + 1 + (2 * n)); + keys = make_tuple(hp); + *hp++ = make_arityval(n); + ks = hp; + hp += n; + mp = (flatmap_t*)hp; + hp += MAP_HEADER_FLATMAP_SZ; + vs = hp; + + mp->thing_word = MAP_HEADER_FLATMAP; + mp->size = n; + mp->keys = keys; + + hashmap_iterator_init(&wstack, map, 0); + + while ((kv=hashmap_iterator_next(&wstack)) != NULL) { + if (EQ(CAR(kv),key)) + continue; + *ks++ = CAR(kv); + *vs++ = CDR(kv); + } + + /* it cannot have multiple keys */ + erts_validate_and_sort_flatmap(mp); + + DESTROY_WSTACK(wstack); + UnUseTmpHeapNoproc(2); + return make_flatmap(mp); + } + + hp = HAlloc(p, size); + hp_end = hp + size; + res = THE_NON_VALUE; + + do { + node = ESTACK_POP(stack); + + /* all nodes are things */ + ptr = boxed_val(node); + hdr = *ptr; + ASSERT(is_header(hdr)); + + switch(hdr & _HEADER_MAP_SUBTAG_MASK) { + case HAMT_SUBTAG_HEAD_ARRAY: + ix = (Uint) ESTACK_POP(stack); + nhp = hp; + if (res == THE_NON_VALUE) { + n = 16; + n -= ix; + *hp++ = MAP_HEADER_HAMT_HEAD_BITMAP(0xffff ^ (1 << ix)); ptr++; + *hp++ = (*ptr++) - 1; + while(ix--) { *hp++ = *ptr++; } + ptr++; n--; + while(n--) { *hp++ = *ptr++; } + res = make_hashmap(nhp); + } else { + n = 16; + *hp++ = MAP_HEADER_HAMT_HEAD_ARRAY; ptr++; + *hp++ = (*ptr++) - 1; + while(n--) { *hp++ = *ptr++; } + nhp[ix+2] = res; + res = make_hashmap(nhp); + } + break; + case HAMT_SUBTAG_NODE_BITMAP: + slot = (Uint) ESTACK_POP(stack); + bp = (Uint32) ESTACK_POP(stack); + n = (Uint32) ESTACK_POP(stack); + nhp = hp; + + /* bitmap change matrix + * res | none leaf bitmap + * ---------------------------- + * n=1 | remove remove keep + * n=2 | other keep keep + * n>2 | shrink keep keep + * + * other: (remember, n is 2) + * shrink if the other bitmap value is a bitmap node + * remove if the other bitmap value is a leaf + * + * remove: + * this bitmap node is removed, res is moved up in tree (could be none) + * this is a special case of shrink + * + * keep: + * the current path index is still used down in the tree, need to keep it + * copy as usual with the updated res + * + * shrink: + * the current path index is no longer used down in the tree, remove it (shrink) + */ + if (res == THE_NON_VALUE) { + if (n == 1) { + break; + } else if (n == 2) { + if (slot == 0) { + ix = 2; /* off by one 'cause hdr */ + } else { + ix = 1; /* off by one 'cause hdr */ + } + if (primary_tag(ptr[ix]) == TAG_PRIMARY_LIST) { + res = ptr[ix]; + } else { + hval = MAP_HEADER_VAL(hdr); + *hp++ = MAP_HEADER_HAMT_NODE_BITMAP(hval ^ bp); + *hp++ = ptr[ix]; + res = make_hashmap(nhp); + } + } else { + /* n > 2 */ + hval = MAP_HEADER_VAL(hdr); + *hp++ = MAP_HEADER_HAMT_NODE_BITMAP(hval ^ bp); ptr++; + n -= slot; + while(slot--) { *hp++ = *ptr++; } + ptr++; n--; + while(n--) { *hp++ = *ptr++; } + res = make_hashmap(nhp); + } + } else if (primary_tag(res) == TAG_PRIMARY_LIST && n == 1) { + break; + } else { + /* res is bitmap or leaf && n > 1, keep */ + n -= slot; + *hp++ = *ptr++; + while(slot--) { *hp++ = *ptr++; } + *hp++ = res; + ptr++; n--; + while(n--) { *hp++ = *ptr++; } + res = make_hashmap(nhp); + } + break; + case HAMT_SUBTAG_HEAD_BITMAP: + slot = (Uint) ESTACK_POP(stack); + bp = (Uint32) ESTACK_POP(stack); + n = (Uint32) ESTACK_POP(stack); + nhp = hp; + + if (res != THE_NON_VALUE) { + *hp++ = *ptr++; + *hp++ = (*ptr++) - 1; + n -= slot; + while(slot--) { *hp++ = *ptr++; } + *hp++ = res; + ptr++; n--; + while(n--) { *hp++ = *ptr++; } + } else { + hval = MAP_HEADER_VAL(hdr); + *hp++ = MAP_HEADER_HAMT_HEAD_BITMAP(hval ^ bp); ptr++; + *hp++ = (*ptr++) - 1; + n -= slot; + while(slot--) { *hp++ = *ptr++; } + ptr++; n--; + while(n--) { *hp++ = *ptr++; } + } + res = make_hashmap(nhp); + break; + default: + erts_exit(ERTS_ERROR_EXIT, "bad header tag %x\r\n", hdr & _HEADER_MAP_SUBTAG_MASK); + break; + } + } while(!ESTACK_ISEMPTY(stack)); + HRelease(p, hp_end, hp); +not_found: + DESTROY_ESTACK(stack); + ERTS_VERIFY_UNUSED_TEMP_ALLOC(p); + ERTS_HOLE_CHECK(p); + UnUseTmpHeapNoproc(2); + return res; +} + + +int erts_validate_and_sort_flatmap(flatmap_t* mp) +{ + Eterm *ks = flatmap_get_keys(mp); + Eterm *vs = flatmap_get_values(mp); + Uint sz = flatmap_get_size(mp); + Uint ix,jx; + Eterm tmp; + Sint c; + + /* sort */ + + for (ix = 1; ix < sz; ix++) { + jx = ix; + while( jx > 0 && (c = CMP_TERM(ks[jx],ks[jx-1])) <= 0 ) { + /* identical key -> error */ + if (c == 0) return 0; + + tmp = ks[jx]; + ks[jx] = ks[jx - 1]; + ks[jx - 1] = tmp; + + tmp = vs[jx]; + vs[jx] = vs[jx - 1]; + vs[jx - 1] = tmp; + + jx--; + } + } + return 1; +} + +#if 0 /* Can't get myself to remove this beautiful piece of code + for probabilistic overestimation of nr of nodes in a hashmap */ + +/* Really rough estimate of sqrt(x) + * Guaranteed not to be less than sqrt(x) + */ +static int int_sqrt_ceiling(Uint x) +{ + int n; + + if (x <= 2) + return x; + + n = erts_fit_in_bits_uint(x-1); + if (n & 1) { + /* Calc: sqrt(2^n) = 2^(n/2) * sqrt(2) ~= 2^(n/2) * 3 / 2 */ + return (1 << (n/2 - 1)) * 3; + } + else { + /* Calc: sqrt(2^n) = 2^(n/2) */ + return 1 << (n / 2); + } +} + +/* May not be enough if hashing is broken (not uniform) + * or if hell freezes over. + */ +Uint hashmap_overestimated_node_count(Uint k) +{ + /* k is nr of key-value pairs. + N(k) is expected nr of nodes in hamt. + + Observation: + For uniformly distributed hash values, average of N varies between + 0.3*k and 0.4*k (with a beautiful sine curve) + and standard deviation of N is about sqrt(k)/3. + + Assuming normal probability distribution, we overestimate nr of nodes + by 15 std.devs above the average, which gives a probability for overrun + less than 1.0e-49 (same magnitude as a git SHA1 collision). + */ + return 2*k/5 + 1 + (15/3)*int_sqrt_ceiling(k); +} +#endif + +BIF_RETTYPE erts_debug_map_info_1(BIF_ALIST_1) { + if (is_hashmap(BIF_ARG_1)) { + BIF_RET(hashmap_info(BIF_P,BIF_ARG_1)); + } else if (is_flatmap(BIF_ARG_1)) { + BIF_ERROR(BIF_P, BADARG); + } else { + BIF_P->fvalue = BIF_ARG_1; + BIF_ERROR(BIF_P, BADMAP); + } +} + +/* + * erts_internal:map_to_tuple_keys/1 + * + * Used in erts_debug:size/1 + */ + +BIF_RETTYPE erts_internal_map_to_tuple_keys_1(BIF_ALIST_1) { + if (is_flatmap(BIF_ARG_1)) { + flatmap_t *mp = (flatmap_t*)flatmap_val(BIF_ARG_1); + BIF_RET(mp->keys); + } else if (is_hashmap(BIF_ARG_1)) { + BIF_ERROR(BIF_P, BADARG); + } else { + BIF_P->fvalue = BIF_ARG_1; + BIF_ERROR(BIF_P, BADMAP); + } +} + +/* + * erts_internal:map_type/1 + * + * Used in erts_debug:size/1 + */ + +BIF_RETTYPE erts_internal_map_type_1(BIF_ALIST_1) { + DECL_AM(hashmap); + DECL_AM(hashmap_node); + DECL_AM(flatmap); + if (is_map(BIF_ARG_1)) { + Eterm hdr = *(boxed_val(BIF_ARG_1)); + ASSERT(is_header(hdr)); + switch (hdr & _HEADER_MAP_SUBTAG_MASK) { + case HAMT_SUBTAG_HEAD_FLATMAP: + BIF_RET(AM_flatmap); + case HAMT_SUBTAG_HEAD_ARRAY: + case HAMT_SUBTAG_HEAD_BITMAP: + BIF_RET(AM_hashmap); + case HAMT_SUBTAG_NODE_BITMAP: + BIF_RET(AM_hashmap_node); + default: + erts_exit(ERTS_ERROR_EXIT, "bad header"); + } + } + BIF_P->fvalue = BIF_ARG_1; + BIF_ERROR(BIF_P, BADMAP); +} + +/* + * erts_internal:map_hashmap_children/1 + * + * Used in erts_debug:size/1 + */ + +BIF_RETTYPE erts_internal_map_hashmap_children_1(BIF_ALIST_1) { + if (is_map(BIF_ARG_1)) { + Eterm node = BIF_ARG_1; + Eterm *ptr, hdr, *hp, res = NIL; + Uint sz = 0; + ptr = boxed_val(node); + hdr = *ptr; + ASSERT(is_header(hdr)); + + switch(hdr & _HEADER_MAP_SUBTAG_MASK) { + case HAMT_SUBTAG_HEAD_FLATMAP: + BIF_ERROR(BIF_P, BADARG); + case HAMT_SUBTAG_HEAD_BITMAP: + ptr++; + case HAMT_SUBTAG_NODE_BITMAP: + ptr++; + sz = hashmap_bitcount(MAP_HEADER_VAL(hdr)); + break; + case HAMT_SUBTAG_HEAD_ARRAY: + sz = 16; + ptr += 2; + break; + default: + erts_exit(ERTS_ERROR_EXIT, "bad header\r\n"); + break; + } + ASSERT(sz < 17); + hp = HAlloc(BIF_P, 2*sz); + while(sz--) { res = CONS(hp, *ptr++, res); hp += 2; } + BIF_RET(res); + } + BIF_P->fvalue = BIF_ARG_1; + BIF_ERROR(BIF_P, BADMAP); +} + + +static Eterm hashmap_info(Process *p, Eterm node) { + Eterm *hp; + Eterm res = NIL, info = NIL; + Eterm *ptr, tup, hdr; + Uint sz; + DECL_AM(depth); + DECL_AM(leafs); + DECL_AM(bitmaps); + DECL_AM(arrays); + Uint nleaf=0, nbitmap=0, narray=0; + Uint bitmap_usage[16], leaf_usage[16]; + Uint lvl = 0, clvl; + DECLARE_ESTACK(stack); + + for (sz = 0; sz < 16; sz++) { + bitmap_usage[sz] = 0; + leaf_usage[sz] = 0; + } + + ptr = boxed_val(node); + ESTACK_PUSH(stack, 0); + ESTACK_PUSH(stack, node); + do { + node = ESTACK_POP(stack); + clvl = ESTACK_POP(stack); + if (lvl < clvl) + lvl = clvl; + switch(primary_tag(node)) { + case TAG_PRIMARY_LIST: + nleaf++; + leaf_usage[clvl] += 1; + break; + case TAG_PRIMARY_BOXED: + ptr = boxed_val(node); + hdr = *ptr; + ASSERT(is_header(hdr)); + switch(hdr & _HEADER_MAP_SUBTAG_MASK) { + case HAMT_SUBTAG_NODE_BITMAP: + nbitmap++; + sz = hashmap_bitcount(MAP_HEADER_VAL(hdr)); + ASSERT(sz < 17); + bitmap_usage[sz-1] += 1; + while(sz--) { + ESTACK_PUSH(stack, clvl + 1); + ESTACK_PUSH(stack, ptr[sz+1]); + } + break; + case HAMT_SUBTAG_HEAD_BITMAP: + nbitmap++; + sz = hashmap_bitcount(MAP_HEADER_VAL(hdr)); + bitmap_usage[sz-1] += 1; + while(sz--) { + ESTACK_PUSH(stack, clvl + 1); + ESTACK_PUSH(stack, ptr[sz+2]); + } + break; + case HAMT_SUBTAG_HEAD_ARRAY: + narray++; + sz = 16; + while(sz--) { + ESTACK_PUSH(stack, clvl + 1); + ESTACK_PUSH(stack, ptr[sz+2]); + } + break; + default: + erts_exit(ERTS_ERROR_EXIT, "bad header\r\n"); + break; + } + } + } while(!ESTACK_ISEMPTY(stack)); + + + /* size */ + sz = 0; + hashmap_bld_tuple_uint(NULL,&sz,16,leaf_usage); + hashmap_bld_tuple_uint(NULL,&sz,16,bitmap_usage); + + /* alloc */ + hp = HAlloc(p, 2+3 + 3*(2+4) + sz); + + info = hashmap_bld_tuple_uint(&hp,NULL,16,leaf_usage); + tup = TUPLE3(hp, AM_leafs, make_small(nleaf),info); hp += 4; + res = CONS(hp, tup, res); hp += 2; + + info = hashmap_bld_tuple_uint(&hp,NULL,16,bitmap_usage); + tup = TUPLE3(hp, AM_bitmaps, make_small(nbitmap), info); hp += 4; + res = CONS(hp, tup, res); hp += 2; + + tup = TUPLE3(hp, AM_arrays, make_small(narray),NIL); hp += 4; + res = CONS(hp, tup, res); hp += 2; + + tup = TUPLE2(hp, AM_depth, make_small(lvl)); hp += 3; + res = CONS(hp, tup, res); hp += 2; + + DESTROY_ESTACK(stack); + ERTS_HOLE_CHECK(p); + return res; +} + +static Eterm hashmap_bld_tuple_uint(Uint **hpp, Uint *szp, Uint n, Uint nums[]) { + Eterm res = THE_NON_VALUE; + Eterm *ts = (Eterm *)erts_alloc(ERTS_ALC_T_TMP, n * sizeof(Eterm)); + Uint i; + + for (i = 0; i < n; i++) { + ts[i] = erts_bld_uint(hpp, szp, nums[i]); + } + res = erts_bld_tuplev(hpp, szp, n, ts); + erts_free(ERTS_ALC_T_TMP, (void *) ts); + return res; +} + + +/* implementation of builtin emulations */ + +#if !ERTS_AT_LEAST_GCC_VSN__(3, 4, 0) +/* Count leading zeros emulation */ +Uint32 hashmap_clz(Uint32 x) { + Uint32 y; + int n = 32; + y = x >>16; if (y != 0) {n = n -16; x = y;} + y = x >> 8; if (y != 0) {n = n - 8; x = y;} + y = x >> 4; if (y != 0) {n = n - 4; x = y;} + y = x >> 2; if (y != 0) {n = n - 2; x = y;} + y = x >> 1; if (y != 0) return n - 2; + return n - x; +} + +const Uint32 SK5 = 0x55555555, SK3 = 0x33333333; +const Uint32 SKF0 = 0xF0F0F0F, SKFF = 0xFF00FF; + +/* CTPOP emulation */ +Uint32 hashmap_bitcount(Uint32 x) { + x -= ((x >> 1 ) & SK5); + x = (x & SK3 ) + ((x >> 2 ) & SK3 ); + x = (x & SKF0) + ((x >> 4 ) & SKF0); + x += x >> 8; + return (x + (x >> 16)) & 0x3F; +} +#endif diff --git a/erts/emulator/beam/erl_map.h b/erts/emulator/beam/erl_map.h new file mode 100644 index 0000000000..4d9d74bc37 --- /dev/null +++ b/erts/emulator/beam/erl_map.h @@ -0,0 +1,211 @@ +/* + * %CopyrightBegin% + * + * Copyright Ericsson AB 2014. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * %CopyrightEnd% + */ + + +#ifndef __ERL_MAP_H__ +#define __ERL_MAP_H__ + +#include "sys.h" + +/* instrinsic wrappers */ +#if ERTS_AT_LEAST_GCC_VSN__(3, 4, 0) +#define hashmap_clz(x) ((Uint32) __builtin_clz((unsigned int)(x))) +#define hashmap_bitcount(x) ((Uint32) __builtin_popcount((unsigned int) (x))) +#else +Uint32 hashmap_clz(Uint32 x); +Uint32 hashmap_bitcount(Uint32 x); +#endif + +/* MAP */ + +typedef struct flatmap_s { + Eterm thing_word; + Uint size; + Eterm keys; /* tuple */ +} flatmap_t; +/* map node + * + * ----------- + * Eterm THING + * Uint size + * Eterm Keys -> {K1, K2, K3, ..., Kn} where n = size + * ---- + * Eterm V1 + * ... + * Eterm Vn, where n = size + * ----------- + */ + +/* the head-node is a bitmap or array with an untagged size */ + + +#define hashmap_size(x) (((hashmap_head_t*) hashmap_val(x))->size) +#define hashmap_size_rel(RTERM, BASE) hashmap_size(rterm2wterm(RTERM, BASE)) +#define hashmap_make_hash(Key) make_internal_hash(Key) + +#define hashmap_restore_hash(Heap,Lvl,Key) \ + (((Lvl) < 8) ? hashmap_make_hash(Key) >> (4*(Lvl)) : hashmap_make_hash(CONS(Heap, make_small((Lvl)>>3), (Key))) >> (4*((Lvl) & 7))) +#define hashmap_shift_hash(Heap,Hx,Lvl,Key) \ + (((++(Lvl)) & 7) ? (Hx) >> 4 : hashmap_make_hash(CONS(Heap, make_small((Lvl)>>3), Key))) + + +/* erl_term.h stuff */ +#define flatmap_get_values(x) (((Eterm *)(x)) + 3) +#define flatmap_get_keys(x) (((Eterm *)tuple_val(((flatmap_t *)(x))->keys)) + 1) +#define flatmap_get_size(x) (((flatmap_t*)(x))->size) + +#ifdef DEBUG +#define MAP_SMALL_MAP_LIMIT (3) +#else +#define MAP_SMALL_MAP_LIMIT (32) +#endif + +struct ErtsWStack_; +struct ErtsEStack_; + +Eterm erts_maps_put(Process *p, Eterm key, Eterm value, Eterm map); +int erts_maps_update(Process *p, Eterm key, Eterm value, Eterm map, Eterm *res); +int erts_maps_remove(Process *p, Eterm key, Eterm map, Eterm *res); + +Eterm erts_hashmap_insert(Process *p, Uint32 hx, Eterm key, Eterm value, + Eterm node, int is_update); +int erts_hashmap_insert_down(Uint32 hx, Eterm key, Eterm node, Uint *sz, + Uint *upsz, struct ErtsEStack_ *sp, int is_update); +Eterm erts_hashmap_insert_up(Eterm *hp, Eterm key, Eterm value, + Uint *upsz, struct ErtsEStack_ *sp); + +int erts_validate_and_sort_flatmap(flatmap_t* map); +void hashmap_iterator_init(struct ErtsWStack_* s, Eterm node, int reverse); +Eterm* hashmap_iterator_next(struct ErtsWStack_* s); +Eterm* hashmap_iterator_prev(struct ErtsWStack_* s); +int hashmap_key_hash_cmp(Eterm* ap, Eterm* bp); +Eterm erts_hashmap_from_array(ErtsHeapFactory*, Eterm *leafs, Uint n, int reject_dupkeys); + +#define erts_hashmap_from_ks_and_vs(P, KS, VS, N) \ + erts_hashmap_from_ks_and_vs_extra((P), (KS), (VS), (N), THE_NON_VALUE, THE_NON_VALUE); + +Eterm erts_hashmap_from_ks_and_vs_extra(Process *p, Eterm *ks, Eterm *vs, Uint n, + Eterm k, Eterm v); + +const Eterm * +#if HALFWORD_HEAP +erts_maps_get_rel(Eterm key, Eterm map, Eterm *map_base); +# define erts_maps_get(A, B) erts_maps_get_rel(A, B, NULL) +#else +erts_maps_get(Eterm key, Eterm map); +# define erts_maps_get_rel(A, B, B_BASE) erts_maps_get(A, B) +#endif + +const Eterm * +#if HALFWORD_HEAP +erts_hashmap_get_rel(Uint32 hx, Eterm key, Eterm node, Eterm *map_base); +# define erts_hashmap_get(Hx, K, M) erts_hashmap_get_rel(Hx, K, M, NULL) +#else +erts_hashmap_get(Uint32 hx, Eterm key, Eterm map); +# define erts_hashmap_get_rel(Hx, K, M, M_BASE) erts_hashmap_get(Hx, K, M) +#endif + +/* hamt nodes v2.0 + * + * node :: leaf | array | bitmap + * head + */ +typedef struct hashmap_head_s { + Eterm thing_word; + Uint size; + Eterm items[1]; +} hashmap_head_t; + +/* thing_word tagscheme + * Need two bits for map subtags + * + * Original HEADER representation: + * + * aaaaaaaaaaaaaaaa aaaaaaaaaatttt00 arity:26, tag:4 + * + * For maps we have: + * + * vvvvvvvvvvvvvvvv aaaaaaaamm111100 val:16, arity:8, mtype:2 + * + * unsure about trailing zeros + * + * map-tag: + * 00 - flat map tag (non-hamt) -> val:16 = #items + * 01 - map-node bitmap tag -> val:16 = bitmap + * 10 - map-head (array-node) -> val:16 = 0xffff + * 11 - map-head (bitmap-node) -> val:16 = bitmap + */ + +/* erl_map.h stuff */ + +#define is_hashmap_header_head(x) ((MAP_HEADER_TYPE(x) & (0x2))) + +#define MAKE_MAP_HEADER(Type,Arity,Val) \ + (_make_header(((((Uint16)(Val)) << MAP_HEADER_ARITY_SZ) | (Arity)) << MAP_HEADER_TAG_SZ | (Type) , _TAG_HEADER_MAP)) + +#define MAP_HEADER_FLATMAP \ + MAKE_MAP_HEADER(MAP_HEADER_TAG_FLATMAP_HEAD,0x1,0x0) + +#define MAP_HEADER_HAMT_HEAD_ARRAY \ + MAKE_MAP_HEADER(MAP_HEADER_TAG_HAMT_HEAD_ARRAY,0x1,0xffff) + +#define MAP_HEADER_HAMT_HEAD_BITMAP(Bmp) \ + MAKE_MAP_HEADER(MAP_HEADER_TAG_HAMT_HEAD_BITMAP,0x1,Bmp) + +#define MAP_HEADER_HAMT_NODE_BITMAP(Bmp) \ + MAKE_MAP_HEADER(MAP_HEADER_TAG_HAMT_NODE_BITMAP,0x0,Bmp) + +#define MAP_HEADER_FLATMAP_SZ (sizeof(flatmap_t) / sizeof(Eterm)) + +#define HAMT_NODE_ARRAY_SZ (17) +#define HAMT_HEAD_ARRAY_SZ (18) +#define HAMT_NODE_BITMAP_SZ(n) (1 + n) +#define HAMT_HEAD_BITMAP_SZ(n) (2 + n) + +/* 2 bits maps tag + 4 bits subtag + 2 ignore bits */ +#define _HEADER_MAP_SUBTAG_MASK (0xfc) +/* 1 bit map tag + 1 ignore bit + 4 bits subtag + 2 ignore bits */ +#define _HEADER_MAP_HASHMAP_HEAD_MASK (0xbc) + +#define HAMT_SUBTAG_NODE_BITMAP ((MAP_HEADER_TAG_HAMT_NODE_BITMAP << _HEADER_ARITY_OFFS) | MAP_SUBTAG) +#define HAMT_SUBTAG_HEAD_ARRAY ((MAP_HEADER_TAG_HAMT_HEAD_ARRAY << _HEADER_ARITY_OFFS) | MAP_SUBTAG) +#define HAMT_SUBTAG_HEAD_BITMAP ((MAP_HEADER_TAG_HAMT_HEAD_BITMAP << _HEADER_ARITY_OFFS) | MAP_SUBTAG) +#define HAMT_SUBTAG_HEAD_FLATMAP ((MAP_HEADER_TAG_FLATMAP_HEAD << _HEADER_ARITY_OFFS) | MAP_SUBTAG) + +#define hashmap_index(hash) (((Uint32)hash) & 0xf) + +/* hashmap heap size: + [one cons cell + one list term in parent node] per key + [one header + one boxed term in parent node] per inner node + [one header + one size word] for root node + Observed average number of nodes per key is about 0.35. +*/ +#define HASHMAP_WORDS_PER_KEY 3 +#define HASHMAP_WORDS_PER_NODE 2 +#ifdef DEBUG +# define HASHMAP_ESTIMATED_TOT_NODE_SIZE(KEYS) \ + (HASHMAP_WORDS_PER_NODE * (KEYS) * 3/10) /* slightly under estimated */ +#else +# define HASHMAP_ESTIMATED_TOT_NODE_SIZE(KEYS) \ + (HASHMAP_WORDS_PER_NODE * (KEYS) * 4/10) /* slightly over estimated */ +#endif +#define HASHMAP_ESTIMATED_HEAP_SIZE(KEYS) \ + ((KEYS)*HASHMAP_WORDS_PER_KEY + HASHMAP_ESTIMATED_TOT_NODE_SIZE(KEYS)) +#endif diff --git a/erts/emulator/beam/erl_math.c b/erts/emulator/beam/erl_math.c index 16d4fdc09c..b46cc37495 100644 --- a/erts/emulator/beam/erl_math.c +++ b/erts/emulator/beam/erl_math.c @@ -3,16 +3,17 @@ * * Copyright Ericsson AB 1997-2009. All Rights Reserved. * - * The contents of this file are subject to the Erlang Public License, - * Version 1.1, (the "License"); you may not use this file except in - * compliance with the License. You should have received a copy of the - * Erlang Public License along with this software. If not, it can be - * retrieved online at http://www.erlang.org/. - * - * Software distributed under the License is distributed on an "AS IS" - * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See - * the License for the specific language governing rights and limitations - * under the License. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. * * %CopyrightEnd% */ @@ -207,6 +208,24 @@ BIF_RETTYPE math_log_1(BIF_ALIST_1) return math_call_1(BIF_P, log, BIF_ARG_1); } +#ifdef HAVE_LOG2 +static double +log2_wrapper(double x) +{ + return log2(x); +} +#else +static double +log2_wrapper(double x) +{ + return log(x) / 0.6931471805599453; /* log(2.0); */ +} +#endif + +BIF_RETTYPE math_log2_1(BIF_ALIST_1) +{ + return math_call_1(BIF_P, log2_wrapper, BIF_ARG_1); +} BIF_RETTYPE math_log10_1(BIF_ALIST_1) { diff --git a/erts/emulator/beam/erl_message.c b/erts/emulator/beam/erl_message.c index 325d77e911..fa6b2fc613 100644 --- a/erts/emulator/beam/erl_message.c +++ b/erts/emulator/beam/erl_message.c @@ -3,16 +3,17 @@ * * Copyright Ericsson AB 1997-2012. All Rights Reserved. * - * The contents of this file are subject to the Erlang Public License, - * Version 1.1, (the "License"); you may not use this file except in - * compliance with the License. You should have received a copy of the - * Erlang Public License along with this software. If not, it can be - * retrieved online at http://www.erlang.org/. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Software distributed under the License is distributed on an "AS IS" - * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See - * the License for the specific language governing rights and limitations - * under the License. + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. * * %CopyrightEnd% */ @@ -46,10 +47,12 @@ ERTS_SCHED_PREF_QUICK_ALLOC_IMPL(message, +#ifdef DEBUG static ERTS_INLINE int in_heapfrag(const Eterm* ptr, const ErlHeapFragment *bp) { return ((unsigned)(ptr - bp->mem) < bp->used_size); } +#endif void @@ -91,9 +94,6 @@ erts_resize_message_buffer(ErlHeapFragment *bp, Uint size, #endif ErlHeapFragment* nbp; - /* ToDo: Make use of 'used_size' to avoid realloc - when shrinking just a few words */ - #ifdef DEBUG { Uint off_sz = size < bp->used_size ? size : bp->used_size; @@ -108,8 +108,10 @@ erts_resize_message_buffer(ErlHeapFragment *bp, Uint size, } #endif - if (size == bp->used_size) + if (size >= (bp->used_size - bp->used_size / 16)) { + bp->used_size = size; return bp; + } #ifdef HARD_DEBUG dbg_brefs = erts_alloc(ERTS_ALC_T_UNDEF, sizeof(Eterm *)*brefs_size); @@ -235,8 +237,7 @@ erts_msg_distext2heap(Process *pp, Eterm msg; Uint tok_sz = 0; Eterm *hp = NULL; - Eterm *hp_end = NULL; - ErlOffHeap *ohp; + ErtsHeapFactory factory; Sint sz; *bpp = NULL; @@ -248,36 +249,26 @@ erts_msg_distext2heap(Process *pp, tok_sz = heap_frag->used_size; sz += tok_sz; } - if (pp) + if (pp) { + ErlOffHeap *ohp; hp = erts_alloc_message_heap(sz, bpp, &ohp, pp, plcksp); + } else { *bpp = new_message_buffer(sz); hp = (*bpp)->mem; - ohp = &(*bpp)->off_heap; } - hp_end = hp + sz; - msg = erts_decode_dist_ext(&hp, ohp, dist_extp); + erts_factory_message_init(&factory, pp, hp, *bpp); + msg = erts_decode_dist_ext(&factory, dist_extp); if (is_non_value(msg)) goto decode_error; if (is_not_nil(*tokenp)) { ErlHeapFragment *heap_frag = erts_dist_ext_trailer(dist_extp); - *tokenp = copy_struct(*tokenp, tok_sz, &hp, ohp); + hp = erts_produce_heap(&factory, tok_sz, 0); + *tokenp = copy_struct(*tokenp, tok_sz, &hp, factory.off_heap); erts_cleanup_offheap(&heap_frag->off_heap); } erts_free_dist_ext_copy(dist_extp); - if (hp_end != hp) { - if (!(*bpp)) { - HRelease(pp, hp_end, hp); - } - else { - Uint final_size = hp - &(*bpp)->mem[0]; - Eterm brefs[2] = {msg, *tokenp}; - ASSERT(sz - (hp_end - hp) == final_size); - *bpp = erts_resize_message_buffer(*bpp, final_size, &brefs[0], 2); - msg = brefs[0]; - *tokenp = brefs[1]; - } - } + erts_factory_close(&factory); return msg; decode_error: @@ -286,13 +277,7 @@ erts_msg_distext2heap(Process *pp, erts_cleanup_offheap(&heap_frag->off_heap); } erts_free_dist_ext_copy(dist_extp); - if (*bpp) { - free_message_buffer(*bpp); - *bpp = NULL; - } - else if (hp) { - HRelease(pp, hp_end, hp); - } + *bpp = NULL; return THE_NON_VALUE; } @@ -367,11 +352,7 @@ erts_queue_dist_message(Process *rcvr, tok_label, tok_lastcnt, tok_serial); } #endif - erts_queue_message(rcvr, rcvr_locks, mbuf, msg, token -#ifdef USE_VM_PROBES - , NIL -#endif - ); + erts_queue_message(rcvr, rcvr_locks, mbuf, msg, token); } else { /* Enqueue message on external format */ @@ -413,7 +394,13 @@ erts_queue_dist_message(Process *rcvr, if (!(*rcvr_locks & ERTS_PROC_LOCK_MSGQ)) erts_smp_proc_unlock(rcvr, ERTS_PROC_LOCK_MSGQ); - erts_proc_notify_new_message(rcvr); + erts_proc_notify_new_message(rcvr, +#ifdef ERTS_SMP + *rcvr_locks +#else + 0 +#endif + ); } } @@ -540,7 +527,13 @@ queue_message(Process *c_p, if (locked_msgq) erts_smp_proc_unlock(receiver, ERTS_PROC_LOCK_MSGQ); - erts_proc_notify_new_message(receiver); + erts_proc_notify_new_message(receiver, +#ifdef ERTS_SMP + *receiver_locks +#else + 0 +#endif + ); #ifndef ERTS_SMP ERTS_HOLE_CHECK(receiver); @@ -549,15 +542,15 @@ queue_message(Process *c_p, } void -erts_queue_message(Process* receiver, - ErtsProcLocks *receiver_locks, - ErlHeapFragment* bp, - Eterm message, - Eterm seq_trace_token #ifdef USE_VM_PROBES - , Eterm dt_utag +erts_queue_message_probe(Process* receiver, ErtsProcLocks *receiver_locks, + ErlHeapFragment* bp, + Eterm message, Eterm seq_trace_token, Eterm dt_utag) +#else +erts_queue_message(Process* receiver, ErtsProcLocks *receiver_locks, + ErlHeapFragment* bp, + Eterm message, Eterm seq_trace_token) #endif - ) { queue_message(NULL, receiver, @@ -841,10 +834,11 @@ erts_msg_attached_data_size_aux(ErlMessage *msg) } void -erts_move_msg_attached_data_to_heap(Eterm **hpp, ErlOffHeap *ohp, ErlMessage *msg) +erts_move_msg_attached_data_to_heap(ErtsHeapFactory* factory, + ErlMessage *msg) { if (is_value(ERL_MESSAGE_TERM(msg))) - erts_move_msg_mbuf_to_heap(hpp, ohp, msg); + erts_move_msg_mbuf_to_heap(&factory->hp, factory->off_heap, msg); else if (msg->data.dist_ext) { ASSERT(msg->data.dist_ext->heap_size >= 0); if (is_not_nil(ERL_MESSAGE_TOKEN(msg))) { @@ -852,12 +846,11 @@ erts_move_msg_attached_data_to_heap(Eterm **hpp, ErlOffHeap *ohp, ErlMessage *ms heap_frag = erts_dist_ext_trailer(msg->data.dist_ext); ERL_MESSAGE_TOKEN(msg) = copy_struct(ERL_MESSAGE_TOKEN(msg), heap_frag->used_size, - hpp, - ohp); + &factory->hp, + factory->off_heap); erts_cleanup_offheap(&heap_frag->off_heap); } - ERL_MESSAGE_TERM(msg) = erts_decode_dist_ext(hpp, - ohp, + ERL_MESSAGE_TERM(msg) = erts_decode_dist_ext(factory, msg->data.dist_ext); erts_free_dist_ext_copy(msg->data.dist_ext); msg->data.dist_ext = NULL; @@ -894,8 +887,10 @@ erts_send_message(Process* sender, #ifdef USE_VM_PROBES *sender_name = *receiver_name = '\0'; if (DTRACE_ENABLED(message_send)) { - erts_snprintf(sender_name, sizeof(DTRACE_CHARBUF_NAME(sender_name)), "%T", sender->common.id); - erts_snprintf(receiver_name, sizeof(DTRACE_CHARBUF_NAME(receiver_name)), "%T", receiver->common.id); + erts_snprintf(sender_name, sizeof(DTRACE_CHARBUF_NAME(sender_name)), + "%T", sender->common.id); + erts_snprintf(receiver_name, sizeof(DTRACE_CHARBUF_NAME(receiver_name)), + "%T", receiver->common.id); } #endif if (SEQ_TRACE_TOKEN(sender) != NIL && !(flags & ERTS_SND_FLG_NO_SEQ_TRACE)) { @@ -1028,7 +1023,6 @@ erts_send_message(Process* sender, } BM_SWAP_TIMER(send,system); } else { -#ifdef ERTS_SMP ErlOffHeap *ohp; Eterm *hp; erts_aint32_t state; @@ -1060,42 +1054,6 @@ erts_send_message(Process* sender, #endif ); BM_SWAP_TIMER(send,system); -#else - ErlMessage* mp = message_alloc(); - Eterm *hp; - BM_SWAP_TIMER(send,size); - msize = size_object(message); - BM_SWAP_TIMER(size,send); - - if (receiver->stop - receiver->htop <= msize) { - BM_SWAP_TIMER(send,system); - erts_garbage_collect(receiver, msize, receiver->arg_reg, receiver->arity); - BM_SWAP_TIMER(system,send); - } - hp = receiver->htop; - receiver->htop = hp + msize; - BM_SWAP_TIMER(send,copy); - message = copy_struct(message, msize, &hp, &receiver->off_heap); - BM_MESSAGE_COPIED(msize); - BM_SWAP_TIMER(copy,send); - DTRACE6(message_send, sender_name, receiver_name, - (uint32_t)msize, tok_label, tok_lastcnt, tok_serial); - ERL_MESSAGE_TERM(mp) = message; - ERL_MESSAGE_TOKEN(mp) = NIL; -#ifdef USE_VM_PROBES - ERL_MESSAGE_DT_UTAG(mp) = NIL; -#endif - mp->next = NULL; - mp->data.attached = NULL; - LINK_MESSAGE(receiver, mp); - res = receiver->msg.len; - erts_proc_notify_new_message(receiver); - - if (IS_TRACED_FL(receiver, F_TRACE_RECEIVE)) { - trace_receive(receiver, message); - } - BM_SWAP_TIMER(send,system); -#endif /* #ifndef ERTS_SMP */ } return res; } @@ -1138,11 +1096,7 @@ erts_deliver_exit_message(Eterm from, Process *to, ErtsProcLocks *to_locksp, /* the trace token must in this case be updated by the caller */ seq_trace_output(token, save, SEQ_TRACE_SEND, to->common.id, NULL); temptoken = copy_struct(token, sz_token, &hp, &bp->off_heap); - erts_queue_message(to, to_locksp, bp, save, temptoken -#ifdef USE_VM_PROBES - , NIL -#endif - ); + erts_queue_message(to, to_locksp, bp, save, temptoken); } else { ErlOffHeap *ohp; sz_reason = size_object(reason); @@ -1159,11 +1113,313 @@ erts_deliver_exit_message(Eterm from, Process *to, ErtsProcLocks *to_locksp, ? from : copy_struct(from, sz_from, &hp, ohp)); save = TUPLE3(hp, am_EXIT, from_copy, mess); - erts_queue_message(to, to_locksp, bp, save, NIL -#ifdef USE_VM_PROBES - , NIL -#endif - ); + erts_queue_message(to, to_locksp, bp, save, NIL); + } +} + +void erts_factory_proc_init(ErtsHeapFactory* factory, + Process* p) +{ + erts_factory_proc_prealloc_init(factory, p, HEAP_LIMIT(p) - HEAP_TOP(p)); +} + +void erts_factory_proc_prealloc_init(ErtsHeapFactory* factory, + Process* p, + Sint size) +{ + factory->mode = FACTORY_HALLOC; + factory->p = p; + factory->hp_start = HAlloc(p, size); + factory->hp = factory->hp_start; + factory->hp_end = factory->hp_start + size; + factory->off_heap = &p->off_heap; + factory->off_heap_saved.first = p->off_heap.first; + factory->off_heap_saved.overhead = p->off_heap.overhead; + factory->heap_frags_saved = p->mbuf; + factory->heap_frags = NULL; /* not used */ + factory->alloc_type = 0; /* not used */ +} + +void erts_factory_message_init(ErtsHeapFactory* factory, + Process* rp, + Eterm* hp, + ErlHeapFragment* bp) +{ + if (bp) { + factory->mode = FACTORY_HEAP_FRAGS; + factory->p = NULL; + factory->hp_start = bp->mem; + factory->hp = hp ? hp : bp->mem; + factory->hp_end = bp->mem + bp->alloc_size; + factory->off_heap = &bp->off_heap; + factory->heap_frags = bp; + factory->heap_frags_saved = bp; + factory->alloc_type = ERTS_ALC_T_HEAP_FRAG; + ASSERT(!bp->next); } + else { + factory->mode = FACTORY_HALLOC; + factory->p = rp; + factory->hp_start = hp; + factory->hp = hp; + factory->hp_end = HEAP_TOP(rp); + factory->off_heap = &rp->off_heap; + factory->heap_frags_saved = rp->mbuf; + factory->heap_frags = NULL; /* not used */ + factory->alloc_type = 0; /* not used */ + } + factory->off_heap_saved.first = factory->off_heap->first; + factory->off_heap_saved.overhead = factory->off_heap->overhead; + + ASSERT(factory->hp >= factory->hp_start && factory->hp <= factory->hp_end); +} + +/* One static sized heap that must suffice. + No extra heap fragments will be allocated. +*/ +void erts_factory_static_init(ErtsHeapFactory* factory, + Eterm* hp, + Uint size, + ErlOffHeap* off_heap) +{ + factory->mode = FACTORY_STATIC; + factory->hp_start = hp; + factory->hp = hp; + factory->hp_end = hp + size; + factory->off_heap = off_heap; + factory->off_heap_saved.first = factory->off_heap->first; + factory->off_heap_saved.overhead = factory->off_heap->overhead; +} + +/* A temporary heap with default buffer allocated/freed by client. + * factory_close is same as factory_undo + */ +void erts_factory_tmp_init(ErtsHeapFactory* factory, Eterm* hp, Uint size, + Uint32 atype) +{ + factory->mode = FACTORY_TMP; + factory->hp_start = hp; + factory->hp = hp; + factory->hp_end = hp + size; + factory->heap_frags = NULL; + factory->off_heap_saved.first = NULL; + factory->off_heap_saved.overhead = 0; + factory->off_heap = &factory->off_heap_saved; + factory->alloc_type = atype; +} + +/* When we know the term is an immediate and need no heap. +*/ +void erts_factory_dummy_init(ErtsHeapFactory* factory) +{ + factory->mode = FACTORY_CLOSED; +} + +static void reserve_heap(ErtsHeapFactory*, Uint need, Uint xtra); + +Eterm* erts_produce_heap(ErtsHeapFactory* factory, Uint need, Uint xtra) +{ + Eterm* res; + + ASSERT((unsigned int)factory->mode > (unsigned int)FACTORY_CLOSED); + if (factory->hp + need > factory->hp_end) { + reserve_heap(factory, need, xtra); + } + res = factory->hp; + factory->hp += need; + return res; +} + +Eterm* erts_reserve_heap(ErtsHeapFactory* factory, Uint need) +{ + ASSERT((unsigned int)factory->mode > (unsigned int)FACTORY_CLOSED); + if (factory->hp + need > factory->hp_end) { + reserve_heap(factory, need, 200); + } + return factory->hp; +} + +static void reserve_heap(ErtsHeapFactory* factory, Uint need, Uint xtra) +{ + ErlHeapFragment* bp; + + switch (factory->mode) { + case FACTORY_HALLOC: + HRelease(factory->p, factory->hp_end, factory->hp); + factory->hp = HAllocX(factory->p, need, xtra); + factory->hp_end = factory->hp + need; + return; + + case FACTORY_HEAP_FRAGS: + case FACTORY_TMP: + bp = factory->heap_frags; + + if (bp) { + ASSERT(factory->hp > bp->mem); + ASSERT(factory->hp <= factory->hp_end); + ASSERT(factory->hp_end == bp->mem + bp->alloc_size); + + bp->used_size = factory->hp - bp->mem; + } + bp = (ErlHeapFragment*) ERTS_HEAP_ALLOC(factory->alloc_type, + ERTS_HEAP_FRAG_SIZE(need+xtra)); + bp->next = factory->heap_frags; + factory->heap_frags = bp; + bp->alloc_size = need + xtra; + bp->used_size = need; + bp->off_heap.first = NULL; + bp->off_heap.overhead = 0; + + factory->hp = bp->mem; + factory->hp_end = bp->mem + bp->alloc_size; + return; + + case FACTORY_STATIC: + case FACTORY_CLOSED: + default: + ASSERT(!"Invalid factory mode"); + } +} + +void erts_factory_close(ErtsHeapFactory* factory) +{ + ErlHeapFragment* bp; + + switch (factory->mode) { + case FACTORY_HALLOC: + HRelease(factory->p, factory->hp_end, factory->hp); + break; + + case FACTORY_HEAP_FRAGS: + bp = factory->heap_frags; + + if (bp) { + ASSERT(factory->hp >= bp->mem); + ASSERT(factory->hp <= factory->hp_end); + ASSERT(factory->hp_end == bp->mem + bp->alloc_size); + + bp->used_size = factory->hp - bp->mem; + } + break; + case FACTORY_TMP: + erts_factory_undo(factory); + break; + case FACTORY_STATIC: break; + case FACTORY_CLOSED: break; + default: + ASSERT(!"Invalid factory mode"); + } + factory->mode = FACTORY_CLOSED; +} + +void erts_factory_trim_and_close(ErtsHeapFactory* factory, + Eterm *brefs, Uint brefs_size) +{ + if (factory->mode == FACTORY_HEAP_FRAGS) { + ErlHeapFragment* bp = factory->heap_frags; + if (bp->next == NULL) { + Uint used_sz = factory->hp - bp->mem; + ASSERT(used_sz <= bp->alloc_size); + factory->heap_frags = erts_resize_message_buffer(bp, used_sz, + brefs, brefs_size); + factory->mode = FACTORY_CLOSED; + return; + } + /*else we don't trim multi fragmented messages for now */ + } + erts_factory_close(factory); +} + +void erts_factory_undo(ErtsHeapFactory* factory) +{ + ErlHeapFragment* bp; + struct erl_off_heap_header *hdr, **hdr_nextp; + + switch (factory->mode) { + case FACTORY_HALLOC: + case FACTORY_STATIC: + /* Cleanup off-heap + */ + hdr_nextp = NULL; + for (hdr = factory->off_heap->first; + hdr != factory->off_heap_saved.first; + hdr = hdr->next) { + + hdr_nextp = &hdr->next; + } + + if (hdr_nextp != NULL) { + *hdr_nextp = NULL; + erts_cleanup_offheap(factory->off_heap); + factory->off_heap->first = factory->off_heap_saved.first; + factory->off_heap->overhead = factory->off_heap_saved.overhead; + } + + if (factory->mode == FACTORY_HALLOC) { + /* Free heap frags + */ + bp = factory->p->mbuf; + if (bp != factory->heap_frags_saved) { + do { + ErlHeapFragment *next_bp = bp->next; + ASSERT(bp->off_heap.first == NULL); + ERTS_HEAP_FREE(ERTS_ALC_T_HEAP_FRAG, (void *) bp, + ERTS_HEAP_FRAG_SIZE(bp->alloc_size)); + bp = next_bp; + } while (bp != factory->heap_frags_saved); + + factory->p->mbuf = bp; + } + + /* Rollback heap top + */ + if (factory->heap_frags_saved == NULL) { /* No heap frags when we started */ + ASSERT(factory->hp_start >= HEAP_START(factory->p)); + ASSERT(factory->hp_start <= HEAP_LIMIT(factory->p)); + + HEAP_TOP(factory->p) = factory->hp_start; + } + else { + ASSERT(factory->heap_frags_saved == factory->p->mbuf); + if (factory->hp_start == factory->heap_frags_saved->mem) { + factory->p->mbuf = factory->p->mbuf->next; + ERTS_HEAP_FREE(ERTS_ALC_T_HEAP_FRAG, factory->heap_frags_saved, + ERTS_HEAP_FRAG_SIZE(factory->heap_frags_saved->alloc_size)); + } + else if (factory->hp_start != factory->hp_end) { + unsigned remains = factory->hp_start - factory->heap_frags_saved->mem; + ASSERT(remains > 0 && remains < factory->heap_frags_saved->used_size); + factory->heap_frags_saved->used_size = remains; + } + } + } + break; + + case FACTORY_TMP: + case FACTORY_HEAP_FRAGS: + erts_cleanup_offheap(factory->off_heap); + factory->off_heap->first = NULL; + + bp = factory->heap_frags; + while (bp != NULL) { + ErlHeapFragment* next_bp = bp->next; + + ASSERT(bp->off_heap.first == NULL); + ERTS_HEAP_FREE(factory->alloc_type, (void *) bp, + ERTS_HEAP_FRAG_SIZE(bp->alloc_size)); + bp = next_bp; + } + break; + + case FACTORY_CLOSED: break; + default: + ASSERT(!"Invalid factory mode"); + } + factory->mode = FACTORY_CLOSED; +#ifdef DEBUG + factory->p = NULL; + factory->hp = NULL; + factory->heap_frags = NULL; +#endif } diff --git a/erts/emulator/beam/erl_message.h b/erts/emulator/beam/erl_message.h index 771eba431f..92ba3e571c 100644 --- a/erts/emulator/beam/erl_message.h +++ b/erts/emulator/beam/erl_message.h @@ -3,16 +3,17 @@ * * Copyright Ericsson AB 1997-2012. All Rights Reserved. * - * The contents of this file are subject to the Erlang Public License, - * Version 1.1, (the "License"); you may not use this file except in - * compliance with the License. You should have received a copy of the - * Erlang Public License along with this software. If not, it can be - * retrieved online at http://www.erlang.org/. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Software distributed under the License is distributed on an "AS IS" - * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See - * the License for the specific language governing rights and limitations - * under the License. + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. * * %CopyrightEnd% */ @@ -46,6 +47,52 @@ typedef struct erl_off_heap { Uint64 overhead; /* Administrative overhead (used to force GC). */ } ErlOffHeap; +#define ERTS_INIT_OFF_HEAP(OHP) \ + do { \ + (OHP)->first = NULL; \ + (OHP)->overhead = 0; \ + } while (0) + +typedef struct { + enum { + FACTORY_CLOSED = 0, + FACTORY_HALLOC, + FACTORY_HEAP_FRAGS, + FACTORY_STATIC, + FACTORY_TMP + } mode; + Process* p; + Eterm* hp_start; + Eterm* hp; + Eterm* hp_end; + struct erl_heap_fragment* heap_frags; + struct erl_heap_fragment* heap_frags_saved; + ErlOffHeap* off_heap; + ErlOffHeap off_heap_saved; + Uint32 alloc_type; +} ErtsHeapFactory; + +void erts_factory_proc_init(ErtsHeapFactory*, Process*); +void erts_factory_proc_prealloc_init(ErtsHeapFactory*, Process*, Sint size); +void erts_factory_message_init(ErtsHeapFactory*, Process*, Eterm* hp, struct erl_heap_fragment*); +void erts_factory_static_init(ErtsHeapFactory*, Eterm* hp, Uint size, ErlOffHeap*); +void erts_factory_tmp_init(ErtsHeapFactory*, Eterm* hp, Uint size, Uint32 atype); +void erts_factory_dummy_init(ErtsHeapFactory*); + +Eterm* erts_produce_heap(ErtsHeapFactory*, Uint need, Uint xtra); +Eterm* erts_reserve_heap(ErtsHeapFactory*, Uint need); +void erts_factory_close(ErtsHeapFactory*); +void erts_factory_trim_and_close(ErtsHeapFactory*,Eterm *brefs, Uint brefs_size); +void erts_factory_undo(ErtsHeapFactory*); + +#ifdef CHECK_FOR_HOLES +# define ERTS_FACTORY_HOLE_CHECK(f) do { \ + /*if ((f)->p) erts_check_for_holes((f)->p);*/ \ + } while (0) +#else +# define ERTS_FACTORY_HOLE_CHECK(p) +#endif + #include "external.h" #include "erl_process.h" @@ -119,7 +166,7 @@ typedef struct { *(p)->msg.last = (mp); \ (p)->msg.last = &(mp)->next; \ (p)->msg.len++; \ -} while(0) +} while (0) #ifdef ERTS_SMP @@ -192,17 +239,23 @@ do { \ do { \ if ((M)->data.attached) { \ Uint need__ = erts_msg_attached_data_size((M)); \ + { SWPO ; } \ if ((ST) - (HT) >= need__) { \ - Uint *htop__ = (HT); \ - erts_move_msg_attached_data_to_heap(&htop__, &MSO((P)), (M));\ - ASSERT(htop__ - (HT) <= need__); \ - (HT) = htop__; \ + ErtsHeapFactory factory__; \ + erts_factory_proc_prealloc_init(&factory__, (P), need__); \ + erts_move_msg_attached_data_to_heap(&factory__, (M)); \ + erts_factory_close(&factory__); \ + if ((P)->mbuf != NULL) { \ + /* Heap was exhausted by messages. This is a rare case */ \ + /* that can currently (OTP 18) only happen if hamts are */ \ + /* far exceeding the estimated heap size. Do GC. */ \ + (FC) -= erts_garbage_collect((P), 0, NULL, 0); \ + } \ } \ else { \ - { SWPO ; } \ (FC) -= erts_garbage_collect((P), 0, NULL, 0); \ - { SWPI ; } \ } \ + { SWPI ; } \ ASSERT(!(M)->data.attached); \ } \ } while (0) @@ -228,11 +281,17 @@ ErlHeapFragment* erts_resize_message_buffer(ErlHeapFragment *, Uint, Eterm *, Uint); void free_message_buffer(ErlHeapFragment *); void erts_queue_dist_message(Process*, ErtsProcLocks*, ErtsDistExternal *, Eterm); -void erts_queue_message(Process*, ErtsProcLocks*, ErlHeapFragment*, Eterm, Eterm #ifdef USE_VM_PROBES - , Eterm dt_utag +void erts_queue_message_probe(Process*, ErtsProcLocks*, ErlHeapFragment*, + Eterm message, Eterm seq_trace_token, Eterm dt_utag); +#define erts_queue_message(RP,RL,BP,Msg,SEQ) \ + erts_queue_message_probe((RP),(RL),(BP),(Msg),(SEQ),NIL) +#else +void erts_queue_message(Process*, ErtsProcLocks*, ErlHeapFragment*, + Eterm message, Eterm seq_trace_token); +#define erts_queue_message_probe(RP,RL,BP,Msg,SEQ,TAG) \ + erts_queue_message((RP),(RL),(BP),(Msg),(SEQ)) #endif -); void erts_deliver_exit_message(Eterm, Process*, ErtsProcLocks *, Eterm, Eterm); Sint erts_send_message(Process*, Process*, ErtsProcLocks*, Eterm, unsigned); void erts_link_mbuf_to_proc(Process *proc, ErlHeapFragment *bp); @@ -240,23 +299,21 @@ void erts_link_mbuf_to_proc(Process *proc, ErlHeapFragment *bp); void erts_move_msg_mbuf_to_heap(Eterm**, ErlOffHeap*, ErlMessage *); Uint erts_msg_attached_data_size_aux(ErlMessage *msg); -void erts_move_msg_attached_data_to_heap(Eterm **, ErlOffHeap *, ErlMessage *); - +void erts_move_msg_attached_data_to_heap(ErtsHeapFactory*, ErlMessage *); Eterm erts_msg_distext2heap(Process *, ErtsProcLocks *, ErlHeapFragment **, Eterm *, ErtsDistExternal *); void erts_cleanup_offheap(ErlOffHeap *offheap); -ERTS_GLB_INLINE Uint erts_msg_used_frag_sz(const ErlMessage *msg); +ERTS_GLB_INLINE Uint erts_used_frag_sz(const ErlHeapFragment*); ERTS_GLB_INLINE Uint erts_msg_attached_data_size(ErlMessage *msg); #if ERTS_GLB_INLINE_INCL_FUNC_DEF -ERTS_GLB_INLINE Uint erts_msg_used_frag_sz(const ErlMessage *msg) +ERTS_GLB_INLINE Uint erts_used_frag_sz(const ErlHeapFragment* bp) { - const ErlHeapFragment *bp; Uint sz = 0; - for (bp = msg->data.heap_frag; bp!=NULL; bp=bp->next) { + for ( ; bp!=NULL; bp=bp->next) { sz += bp->used_size; } return sz; @@ -266,7 +323,7 @@ ERTS_GLB_INLINE Uint erts_msg_attached_data_size(ErlMessage *msg) { ASSERT(msg->data.attached); if (is_value(ERL_MESSAGE_TERM(msg))) - return erts_msg_used_frag_sz(msg); + return erts_used_frag_sz(msg->data.heap_frag); else if (msg->data.dist_ext->heap_size < 0) return erts_msg_attached_data_size_aux(msg); else { diff --git a/erts/emulator/beam/erl_monitors.c b/erts/emulator/beam/erl_monitors.c index 70e592cc5f..bd899fa2f9 100644 --- a/erts/emulator/beam/erl_monitors.c +++ b/erts/emulator/beam/erl_monitors.c @@ -3,16 +3,17 @@ * * Copyright Ericsson AB 2004-2013. All Rights Reserved. * - * The contents of this file are subject to the Erlang Public License, - * Version 1.1, (the "License"); you may not use this file except in - * compliance with the License. You should have received a copy of the - * Erlang Public License along with this software. If not, it can be - * retrieved online at http://www.erlang.org/. - * - * Software distributed under the License is distributed on an "AS IS" - * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See - * the License for the specific language governing rights and limitations - * under the License. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. * * %CopyrightEnd% */ @@ -355,7 +356,7 @@ void erts_add_monitor(ErtsMonitor **root, Uint type, Eterm ref, Eterm pid, tstack[tpos++] = this; this = &((*this)->right); } else { /* Equal key is an error for monitors */ - erl_exit(1,"Insertion of already present monitor!"); + erts_exit(ERTS_ERROR_EXIT,"Insertion of already present monitor!"); break; } } @@ -1024,3 +1025,23 @@ Eterm erts_debug_dump_links_1(BIF_ALIST_1) } } } + +void erts_one_link_size(ErtsLink *lnk, void *vpu) +{ + Uint *pu = vpu; + *pu += ERTS_LINK_SIZE*sizeof(Uint); + if(!IS_CONST(lnk->pid)) + *pu += NC_HEAP_SIZE(lnk->pid)*sizeof(Uint); + if (lnk->type != LINK_NODE && ERTS_LINK_ROOT(lnk) != NULL) { + erts_doforall_links(ERTS_LINK_ROOT(lnk),&erts_one_link_size,vpu); + } +} +void erts_one_mon_size(ErtsMonitor *mon, void *vpu) +{ + Uint *pu = vpu; + *pu += ERTS_MONITOR_SIZE*sizeof(Uint); + if(!IS_CONST(mon->pid)) + *pu += NC_HEAP_SIZE(mon->pid)*sizeof(Uint); + if(!IS_CONST(mon->ref)) + *pu += NC_HEAP_SIZE(mon->ref)*sizeof(Uint); +} diff --git a/erts/emulator/beam/erl_monitors.h b/erts/emulator/beam/erl_monitors.h index 6a360a2336..901ed8940b 100644 --- a/erts/emulator/beam/erl_monitors.h +++ b/erts/emulator/beam/erl_monitors.h @@ -3,16 +3,17 @@ * * Copyright Ericsson AB 2004-2013. All Rights Reserved. * - * The contents of this file are subject to the Erlang Public License, - * Version 1.1, (the "License"); you may not use this file except in - * compliance with the License. You should have received a copy of the - * Erlang Public License along with this software. If not, it can be - * retrieved online at http://www.erlang.org/. - * - * Software distributed under the License is distributed on an "AS IS" - * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See - * the License for the specific language governing rights and limitations - * under the License. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. * * %CopyrightEnd% */ @@ -82,6 +83,7 @@ /* Type tags for monitors */ #define MON_ORIGIN 1 #define MON_TARGET 3 +#define MON_TIME_OFFSET 7 /* Type tags for links */ #define LINK_PID 1 /* ...Or port */ @@ -103,7 +105,7 @@ typedef struct erts_monitor_or_link { typedef struct erts_monitor { struct erts_monitor *left, *right; Sint16 balance; - Uint16 type; /* MON_ORIGIN | MON_TARGET */ + Uint16 type; /* MON_ORIGIN | MON_TARGET | MON_TIME_OFFSET */ Eterm ref; Eterm pid; /* In case of distributed named monitor, this is the nodename atom in MON_ORIGIN process, otherwise a pid or @@ -170,6 +172,8 @@ ErtsSuspendMonitor *erts_lookup_suspend_monitor(ErtsSuspendMonitor *root, Eterm pid); void erts_delete_suspend_monitor(ErtsSuspendMonitor **root, Eterm pid); void erts_init_monitors(void); +void erts_one_link_size(ErtsLink *lnk, void *vpu); +void erts_one_mon_size(ErtsMonitor *mon, void *vpu); #define erts_doforall_monitors erts_sweep_monitors #define erts_doforall_links erts_sweep_links diff --git a/erts/emulator/beam/erl_mtrace.c b/erts/emulator/beam/erl_mtrace.c index e538ba30c2..fdaf02f37a 100644 --- a/erts/emulator/beam/erl_mtrace.c +++ b/erts/emulator/beam/erl_mtrace.c @@ -3,16 +3,17 @@ * * Copyright Ericsson AB 2003-2013. All Rights Reserved. * - * The contents of this file are subject to the Erlang Public License, - * Version 1.1, (the "License"); you may not use this file except in - * compliance with the License. You should have received a copy of the - * Erlang Public License along with this software. If not, it can be - * retrieved online at http://www.erlang.org/. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Software distributed under the License is distributed on an "AS IS" - * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See - * the License for the specific language governing rights and limitations - * under the License. + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. * * %CopyrightEnd% */ @@ -222,6 +223,8 @@ static byte *tracep; static byte *endp; static SysTimeval last_tv; +static ErtsAllocatorWrapper_t mtrace_wrapper; + #if ERTS_MTRACE_SEGMENT_ID >= ERTS_ALC_A_MIN || ERTS_MTRACE_SEGMENT_ID < 0 #error ERTS_MTRACE_SEGMENT_ID >= ERTS_ALC_A_MIN || ERTS_MTRACE_SEGMENT_ID < 0 #endif @@ -555,6 +558,8 @@ write_trace_header(char *nodename, char *pid, char *hostname) return 1; } +static void mtrace_pre_lock(void); +static void mtrace_pre_unlock(void); static void *mtrace_alloc(ErtsAlcType_t, void *, Uint); static void *mtrace_realloc(ErtsAlcType_t, void *, void *, Uint); static void mtrace_free(ErtsAlcType_t, void *, void *); @@ -623,7 +628,7 @@ erts_mtrace_install_wrapper_functions(void) if (erts_mtrace_enabled) { int i; /* Install trace functions */ - ASSERT(sizeof(erts_allctrs) == sizeof(real_allctrs)); + ERTS_CT_ASSERT(sizeof(erts_allctrs) == sizeof(real_allctrs)); sys_memcpy((void *) real_allctrs, (void *) erts_allctrs, @@ -635,12 +640,16 @@ erts_mtrace_install_wrapper_functions(void) erts_allctrs[i].free = mtrace_free; erts_allctrs[i].extra = (void *) &real_allctrs[i]; } + mtrace_wrapper.lock = mtrace_pre_lock; + mtrace_wrapper.unlock = mtrace_pre_unlock; + erts_allctr_wrapper_prelock_init(&mtrace_wrapper); } } void erts_mtrace_stop(void) { + ASSERT(!erts_is_allctr_wrapper_prelocked()); erts_mtx_lock(&mtrace_op_mutex); erts_mtx_lock(&mtrace_buf_mutex); if (erts_mtrace_enabled) { @@ -677,6 +686,7 @@ erts_mtrace_stop(void) void erts_mtrace_exit(Uint32 exit_value) { + ASSERT(!erts_is_allctr_wrapper_prelocked()); erts_mtx_lock(&mtrace_op_mutex); erts_mtx_lock(&mtrace_buf_mutex); if (erts_mtrace_enabled) { @@ -935,18 +945,33 @@ write_free_entry(byte tag, erts_mtx_unlock(&mtrace_buf_mutex); } +static void mtrace_pre_lock(void) +{ + erts_mtx_lock(&mtrace_op_mutex); +} + +static void mtrace_pre_unlock(void) +{ + erts_mtx_unlock(&mtrace_op_mutex); +} + + static void * mtrace_alloc(ErtsAlcType_t n, void *extra, Uint size) { ErtsAllocatorFunctions_t *real_af = (ErtsAllocatorFunctions_t *) extra; void *res; - erts_mtx_lock(&mtrace_op_mutex); + if (!erts_is_allctr_wrapper_prelocked()) { + erts_mtx_lock(&mtrace_op_mutex); + } res = (*real_af->alloc)(n, real_af->extra, size); write_alloc_entry(ERTS_MT_ALLOC_BDY_TAG, res, n, 0, size); - erts_mtx_unlock(&mtrace_op_mutex); + if (!erts_is_allctr_wrapper_prelocked()) { + erts_mtx_unlock(&mtrace_op_mutex); + } return res; } @@ -957,12 +982,16 @@ mtrace_realloc(ErtsAlcType_t n, void *extra, void *ptr, Uint size) ErtsAllocatorFunctions_t *real_af = (ErtsAllocatorFunctions_t *) extra; void *res; - erts_mtx_lock(&mtrace_op_mutex); + if (!erts_is_allctr_wrapper_prelocked()) { + erts_mtx_lock(&mtrace_op_mutex); + } res = (*real_af->realloc)(n, real_af->extra, ptr, size); write_realloc_entry(ERTS_MT_REALLOC_BDY_TAG, res, n, 0, ptr, size); - erts_mtx_unlock(&mtrace_op_mutex); + if (!erts_is_allctr_wrapper_prelocked()) { + erts_mtx_unlock(&mtrace_op_mutex); + } return res; @@ -973,10 +1002,14 @@ mtrace_free(ErtsAlcType_t n, void *extra, void *ptr) { ErtsAllocatorFunctions_t *real_af = (ErtsAllocatorFunctions_t *) extra; - erts_mtx_lock(&mtrace_op_mutex); + if (!erts_is_allctr_wrapper_prelocked()) { + erts_mtx_lock(&mtrace_op_mutex); + } (*real_af->free)(n, real_af->extra, ptr); - write_free_entry(ERTS_MT_FREE_BDY_TAG, n, 0, ptr); + if (!erts_is_allctr_wrapper_prelocked()) { + write_free_entry(ERTS_MT_FREE_BDY_TAG, n, 0, ptr); + } erts_mtx_unlock(&mtrace_op_mutex); } diff --git a/erts/emulator/beam/erl_mtrace.h b/erts/emulator/beam/erl_mtrace.h index 204543ddb0..b34eab3c4a 100644 --- a/erts/emulator/beam/erl_mtrace.h +++ b/erts/emulator/beam/erl_mtrace.h @@ -3,16 +3,17 @@ * * Copyright Ericsson AB 2003-2009. All Rights Reserved. * - * The contents of this file are subject to the Erlang Public License, - * Version 1.1, (the "License"); you may not use this file except in - * compliance with the License. You should have received a copy of the - * Erlang Public License along with this software. If not, it can be - * retrieved online at http://www.erlang.org/. - * - * Software distributed under the License is distributed on an "AS IS" - * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See - * the License for the specific language governing rights and limitations - * under the License. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. * * %CopyrightEnd% */ diff --git a/erts/emulator/beam/erl_nif.c b/erts/emulator/beam/erl_nif.c index d4c2b5bdcc..1097916c97 100644 --- a/erts/emulator/beam/erl_nif.c +++ b/erts/emulator/beam/erl_nif.c @@ -1,18 +1,19 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2009-2013. All Rights Reserved. + * Copyright Ericsson AB 2009-2014. All Rights Reserved. * - * The contents of this file are subject to the Erlang Public License, - * Version 1.1, (the "License"); you may not use this file except in - * compliance with the License. You should have received a copy of the - * Erlang Public License along with this software. If not, it can be - * retrieved online at http://www.erlang.org/. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Software distributed under the License is distributed on an "AS IS" - * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See - * the License for the specific language governing rights and limitations - * under the License. + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. * * %CopyrightEnd% */ @@ -31,9 +32,12 @@ #include "bif.h" #include "error.h" #include "big.h" +#include "erl_map.h" #include "beam_bp.h" #include "erl_thr_progress.h" #include "dtrace-wrapper.h" +#include "erl_process.h" +#include "erl_bif_unique.h" #if defined(USE_DYNAMIC_TRACE) && (defined(USE_DTRACE) || defined(USE_SYSTEMTAP)) #define HAVE_USE_DTRACE 1 #endif @@ -125,6 +129,7 @@ void erts_pre_nif(ErlNifEnv* env, Process* p, struct erl_module_nif* mod_nif) env->heap_frag = NULL; env->fpe_was_unmasked = erts_block_fpe(); env->tmp_obj_list = NULL; + env->exception_thrown = 0; } static void pre_nif_noproc(ErlNifEnv* env, struct erl_module_nif* mod_nif) @@ -310,9 +315,6 @@ int enif_send(ErlNifEnv* env, const ErlNifPid* to_pid, Process* rp; Process* c_p; ErlHeapFragment* frags; -#if defined(ERTS_ENABLE_LOCK_CHECK) && defined(ERTS_SMP) - ErtsProcLocks rp_had_locks; -#endif Eterm receiver = to_pid->pid; int flush_me = 0; int scheduler = erts_get_scheduler_id() != 0; @@ -328,18 +330,14 @@ int enif_send(ErlNifEnv* env, const ErlNifPid* to_pid, #ifdef ERTS_SMP c_p = NULL; #else - erl_exit(ERTS_ABORT_EXIT,"enif_send: env==NULL on non-SMP VM"); + erts_exit(ERTS_ABORT_EXIT,"enif_send: env==NULL on non-SMP VM"); #endif } -#if defined(ERTS_ENABLE_LOCK_CHECK) && defined(ERTS_SMP) - rp_had_locks = rp_locks; -#endif - rp = (scheduler ? erts_proc_lookup(receiver) : erts_pid2proc_opt(c_p, ERTS_PROC_LOCK_MAIN, - receiver, rp_locks, ERTS_P2P_FLG_SMP_INC_REFC)); + receiver, rp_locks, ERTS_P2P_FLG_INC_REFC)); if (rp == NULL) { ASSERT(env == NULL || receiver != c_p->common.id); return 0; @@ -361,17 +359,13 @@ int enif_send(ErlNifEnv* env, const ErlNifPid* to_pid, if (flush_me) { flush_env(env); /* Needed for ERTS_HOLE_CHECK */ } - erts_queue_message(rp, &rp_locks, frags, msg, am_undefined -#ifdef USE_VM_PROBES - , NIL -#endif - ); + erts_queue_message(rp, &rp_locks, frags, msg, am_undefined); if (c_p == rp) rp_locks &= ~ERTS_PROC_LOCK_MAIN; if (rp_locks) erts_smp_proc_unlock(rp, rp_locks); if (!scheduler) - erts_smp_proc_dec_refc(rp); + erts_proc_dec_refc(rp); if (flush_me) { cache_env(env); } @@ -452,7 +446,7 @@ int enif_is_list(ErlNifEnv* env, ERL_NIF_TERM term) int enif_is_exception(ErlNifEnv* env, ERL_NIF_TERM term) { - return term == THE_NON_VALUE; + return env->exception_thrown && term == THE_NON_VALUE; } int enif_is_number(ErlNifEnv* env, ERL_NIF_TERM term) @@ -477,6 +471,18 @@ int enif_inspect_binary(ErlNifEnv* env, Eterm bin_term, ErlNifBinary* bin) struct enif_tmp_obj_t* tmp; byte* raw_ptr; }u; + + if (is_boxed(bin_term) && *binary_val(bin_term) == HEADER_SUB_BIN) { + ErlSubBin* sb = (ErlSubBin*) binary_val(bin_term); + if (sb->is_writable) { + ProcBin* pb = (ProcBin*) binary_val(sb->orig); + ASSERT(pb->thing_word == HEADER_PROC_BIN); + if (pb->flags) { + erts_emasculate_writable_binary(pb); + sb->is_writable = 0; + } + } + } u.tmp = NULL; bin->data = erts_get_aligned_binary_bytes_extra(bin_term, &u.raw_ptr, allocator, sizeof(struct enif_tmp_obj_t)); @@ -544,9 +550,7 @@ int enif_alloc_binary(size_t size, ErlNifBinary* bin) if (refbin == NULL) { return 0; /* The NIF must take action */ } - refbin->flags = BIN_FLAG_DRV; /* BUGBUG: Flag? */ erts_refc_init(&refbin->refc, 1); - refbin->orig_size = (SWord) size; bin->size = size; bin->data = (unsigned char*) refbin->orig_bytes; @@ -566,7 +570,6 @@ int enif_realloc_binary(ErlNifBinary* bin, size_t size) if (!newbin) { return 0; } - newbin->orig_size = size; bin->ref_bin = newbin; bin->data = (unsigned char*) newbin->orig_bytes; bin->size = size; @@ -735,7 +738,22 @@ Eterm enif_make_sub_binary(ErlNifEnv* env, ERL_NIF_TERM bin_term, Eterm enif_make_badarg(ErlNifEnv* env) { - BIF_ERROR(env->proc, BADARG); + return enif_raise_exception(env, am_badarg); +} + +Eterm enif_raise_exception(ErlNifEnv* env, ERL_NIF_TERM reason) +{ + env->exception_thrown = 1; + env->proc->fvalue = reason; + BIF_ERROR(env->proc, EXC_ERROR); +} + +int enif_has_pending_exception(ErlNifEnv* env, ERL_NIF_TERM* reason) +{ + if (env->exception_thrown && reason != NULL) { + *reason = env->proc->fvalue; + } + return env->exception_thrown; } int enif_get_atom(ErlNifEnv* env, Eterm atom, char* buf, unsigned len, @@ -881,7 +899,7 @@ int enif_get_list_cell(ErlNifEnv* env, Eterm term, Eterm* head, Eterm* tail) int enif_get_list_length(ErlNifEnv* env, Eterm term, unsigned* len) { if (is_not_list(term) && is_not_nil(term)) return 0; - *len = list_length(term); + *len = erts_list_length(term); return 1; } @@ -957,8 +975,12 @@ ERL_NIF_TERM enif_make_uint64(ErlNifEnv* env, ErlNifUInt64 i) ERL_NIF_TERM enif_make_double(ErlNifEnv* env, double d) { - Eterm* hp = alloc_heap(env,FLOAT_SIZE_OBJECT); + Eterm* hp; FloatDef f; + + if (!erts_isfinite(d)) + return enif_make_badarg(env); + hp = alloc_heap(env,FLOAT_SIZE_OBJECT); f.fd = d; PUT_DOUBLE(f, hp); return make_float(hp); @@ -971,6 +993,8 @@ ERL_NIF_TERM enif_make_atom(ErlNifEnv* env, const char* name) ERL_NIF_TERM enif_make_atom_len(ErlNifEnv* env, const char* name, size_t len) { + if (len > MAX_ATOM_CHARACTERS) + return enif_make_badarg(env); return erts_atom_put((byte*)name, len, ERTS_ATOM_ENC_LATIN1, 1); } @@ -984,6 +1008,8 @@ int enif_make_existing_atom_len(ErlNifEnv* env, const char* name, size_t len, ERL_NIF_TERM* atom, ErlNifCharEncoding encoding) { ASSERT(encoding == ERL_NIF_LATIN1); + if (len > MAX_ATOM_CHARACTERS) + return 0; return erts_atom_get(name, len, atom, ERTS_ATOM_ENC_LATIN1); } @@ -1147,6 +1173,27 @@ ErlNifTid enif_thread_self(void) { return erl_drv_thread_self(); } int enif_equal_tids(ErlNifTid tid1, ErlNifTid tid2) { return erl_drv_equal_tids(tid1,tid2); } void enif_thread_exit(void *resp) { erl_drv_thread_exit(resp); } int enif_thread_join(ErlNifTid tid, void **respp) { return erl_drv_thread_join(tid,respp); } +int enif_getenv(const char *key, char *value, size_t *value_size) { return erl_drv_getenv(key, value, value_size); } + +ErlNifTime enif_monotonic_time(ErlNifTimeUnit time_unit) +{ + return (ErlNifTime) erts_napi_monotonic_time((int) time_unit); +} + +ErlNifTime enif_time_offset(ErlNifTimeUnit time_unit) +{ + return (ErlNifTime) erts_napi_time_offset((int) time_unit); +} + +ErlNifTime +enif_convert_time_unit(ErlNifTime val, + ErlNifTimeUnit from, + ErlNifTimeUnit to) +{ + return (ErlNifTime) erts_napi_convert_time_unit((ErtsMonotonicTime) val, + (int) from, + (int) to); +} int enif_fprintf(void* filep, const char* format, ...) { @@ -1183,7 +1230,11 @@ typedef struct enif_resource_t struct enif_resource_type_t* type; #ifdef DEBUG erts_refc_t nif_refc; +# ifdef ARCH_32 + byte align__[4]; +# endif #endif + char data[1]; }ErlNifResource; @@ -1220,7 +1271,8 @@ static void close_lib(struct erl_module_nif* lib) lib->entry->unload(&env, lib->priv_data); post_nif_noproc(&env); } - erts_sys_ddll_close(lib->handle); + if (!erts_is_static_nif(lib->handle)) + erts_sys_ddll_close(lib->handle); lib->handle = NULL; } @@ -1241,6 +1293,19 @@ static void steal_resource_type(ErlNifResourceType* type) } } +/* The opened_rt_list is used by enif_open_resource_type() + * in order to rollback "creates" and "take-overs" in case the load fails. + */ +struct opened_resource_type +{ + struct opened_resource_type* next; + + ErlNifResourceFlags op; + ErlNifResourceType* type; + ErlNifResourceDtor* new_dtor; +}; +static struct opened_resource_type* opened_rt_list = NULL; + ErlNifResourceType* enif_open_resource_type(ErlNifEnv* env, const char* module_str, @@ -1262,22 +1327,21 @@ enif_open_resource_type(ErlNifEnv* env, if (type == NULL) { if (flags & ERL_NIF_RT_CREATE) { type = erts_alloc(ERTS_ALC_T_NIF, - sizeof(struct enif_resource_type_t)); - type->dtor = dtor; + sizeof(struct enif_resource_type_t)); type->module = module_am; type->name = name_am; erts_refc_init(&type->refc, 1); - type->owner = env->mod_nif; - type->prev = &resource_type_list; - type->next = resource_type_list.next; - type->next->prev = type; - type->prev->next = type; op = ERL_NIF_RT_CREATE; + #ifdef DEBUG + type->dtor = (void*)1; + type->owner = (void*)2; + type->prev = (void*)3; + type->next = (void*)4; + #endif } } else { - if (flags & ERL_NIF_RT_TAKEOVER) { - steal_resource_type(type); + if (flags & ERL_NIF_RT_TAKEOVER) { op = ERL_NIF_RT_TAKEOVER; } else { @@ -1285,12 +1349,13 @@ enif_open_resource_type(ErlNifEnv* env, } } if (type != NULL) { - type->owner = env->mod_nif; - type->dtor = dtor; - if (type->dtor != NULL) { - erts_refc_inc(&type->owner->rt_dtor_cnt, 1); - } - erts_refc_inc(&type->owner->rt_cnt, 1); + struct opened_resource_type* ort = erts_alloc(ERTS_ALC_T_TMP, + sizeof(struct opened_resource_type)); + ort->op = op; + ort->type = type; + ort->new_dtor = dtor; + ort->next = opened_rt_list; + opened_rt_list = ort; } if (tried != NULL) { *tried = op; @@ -1298,9 +1363,54 @@ enif_open_resource_type(ErlNifEnv* env, return type; } +static void commit_opened_resource_types(struct erl_module_nif* lib) +{ + while (opened_rt_list) { + struct opened_resource_type* ort = opened_rt_list; + + ErlNifResourceType* type = ort->type; + + if (ort->op == ERL_NIF_RT_CREATE) { + type->prev = &resource_type_list; + type->next = resource_type_list.next; + type->next->prev = type; + type->prev->next = type; + } + else { /* ERL_NIF_RT_TAKEOVER */ + steal_resource_type(type); + } + + type->owner = lib; + type->dtor = ort->new_dtor; + + if (type->dtor != NULL) { + erts_refc_inc(&lib->rt_dtor_cnt, 1); + } + erts_refc_inc(&lib->rt_cnt, 1); + + opened_rt_list = ort->next; + erts_free(ERTS_ALC_T_TMP, ort); + } +} + +static void rollback_opened_resource_types(void) +{ + while (opened_rt_list) { + struct opened_resource_type* ort = opened_rt_list; + + if (ort->op == ERL_NIF_RT_CREATE) { + erts_free(ERTS_ALC_T_NIF, ort->type); + } + + opened_rt_list = ort->next; + erts_free(ERTS_ALC_T_TMP, ort); + } +} + + static void nif_resource_dtor(Binary* bin) { - ErlNifResource* resource = (ErlNifResource*) ERTS_MAGIC_BIN_DATA(bin); + ErlNifResource* resource = (ErlNifResource*) ERTS_MAGIC_BIN_UNALIGNED_DATA(bin); ErlNifResourceType* type = resource->type; ASSERT(ERTS_MAGIC_BIN_DESTRUCTOR(bin) == &nif_resource_dtor); @@ -1321,8 +1431,12 @@ static void nif_resource_dtor(Binary* bin) void* enif_alloc_resource(ErlNifResourceType* type, size_t size) { - Binary* bin = erts_create_magic_binary(SIZEOF_ErlNifResource(size), &nif_resource_dtor); - ErlNifResource* resource = ERTS_MAGIC_BIN_DATA(bin); + Binary* bin = erts_create_magic_binary_x(SIZEOF_ErlNifResource(size), + &nif_resource_dtor, + 1); /* unaligned */ + ErlNifResource* resource = ERTS_MAGIC_BIN_UNALIGNED_DATA(bin); + + ASSERT(type->owner && type->next && type->prev); /* not allowed in load/upgrade */ resource->type = type; erts_refc_inc(&bin->refc, 1); #ifdef DEBUG @@ -1335,7 +1449,7 @@ void* enif_alloc_resource(ErlNifResourceType* type, size_t size) void enif_release_resource(void* obj) { ErlNifResource* resource = DATA_TO_RESOURCE(obj); - ErtsBinary* bin = ERTS_MAGIC_BIN_FROM_DATA(resource); + ErtsBinary* bin = ERTS_MAGIC_BIN_FROM_UNALIGNED_DATA(resource); ASSERT(ERTS_MAGIC_BIN_DESTRUCTOR(bin) == &nif_resource_dtor); #ifdef DEBUG @@ -1349,7 +1463,7 @@ void enif_release_resource(void* obj) void enif_keep_resource(void* obj) { ErlNifResource* resource = DATA_TO_RESOURCE(obj); - ErtsBinary* bin = ERTS_MAGIC_BIN_FROM_DATA(resource); + ErtsBinary* bin = ERTS_MAGIC_BIN_FROM_UNALIGNED_DATA(resource); ASSERT(ERTS_MAGIC_BIN_DESTRUCTOR(bin) == &nif_resource_dtor); #ifdef DEBUG @@ -1361,7 +1475,7 @@ void enif_keep_resource(void* obj) ERL_NIF_TERM enif_make_resource(ErlNifEnv* env, void* obj) { ErlNifResource* resource = DATA_TO_RESOURCE(obj); - ErtsBinary* bin = ERTS_MAGIC_BIN_FROM_DATA(resource); + ErtsBinary* bin = ERTS_MAGIC_BIN_FROM_UNALIGNED_DATA(resource); Eterm* hp = alloc_heap(env,PROC_BIN_SIZE); return erts_mk_magic_binary_term(&hp, &MSO(env->proc), &bin->binary); } @@ -1390,7 +1504,7 @@ int enif_get_resource(ErlNifEnv* env, ERL_NIF_TERM term, ErlNifResourceType* typ return 0; / * Or should we allow "resource binaries" as handles? * / }*/ mbin = pb->val; - resource = (ErlNifResource*) ERTS_MAGIC_BIN_DATA(mbin); + resource = (ErlNifResource*) ERTS_MAGIC_BIN_UNALIGNED_DATA(mbin); if (ERTS_MAGIC_BIN_DESTRUCTOR(mbin) != &nif_resource_dtor || resource->type != type) { return 0; @@ -1402,8 +1516,8 @@ int enif_get_resource(ErlNifEnv* env, ERL_NIF_TERM term, ErlNifResourceType* typ size_t enif_sizeof_resource(void* obj) { ErlNifResource* resource = DATA_TO_RESOURCE(obj); - Binary* bin = &ERTS_MAGIC_BIN_FROM_DATA(resource)->binary; - return ERTS_MAGIC_BIN_DATA_SIZE(bin) - offsetof(ErlNifResource,data); + Binary* bin = &ERTS_MAGIC_BIN_FROM_UNALIGNED_DATA(resource)->binary; + return ERTS_MAGIC_BIN_UNALIGNED_DATA_SIZE(bin) - offsetof(ErlNifResource,data); } @@ -1413,7 +1527,7 @@ void* enif_dlopen(const char* lib, ErtsSysDdllError errdesc = ERTS_SYS_DDLL_ERROR_INIT; void* handle; void* init_func; - if (erts_sys_ddll_open2(lib, &handle, &errdesc) == ERL_DE_NO_ERROR) { + if (erts_sys_ddll_open(lib, &handle, &errdesc) == ERL_DE_NO_ERROR) { if (erts_sys_ddll_load_nif_init(handle, &init_func, &errdesc) == ERL_DE_NO_ERROR) { erts_sys_ddll_call_nif_init(init_func); } @@ -1457,6 +1571,681 @@ int enif_consume_timeslice(ErlNifEnv* env, int percent) return ERTS_BIF_REDS_LEFT(env->proc) == 0; } +/* + * NIF exports need a few more items than the Export struct provides, + * including the erl_module_nif* and a NIF function pointer, so the + * NifExport below adds those. The Export member must be first in the + * struct. The saved_mfa, exception_thrown, saved_argc, rootset_extra, and + * rootset members are used to track the MFA, any pending exception, and + * arguments of the top NIF in case a chain of one or more + * enif_schedule_nif() calls results in an exception, since in that case + * the original MFA and registers have to be restored before returning to + * Erlang to ensure stacktrace information associated with the exception is + * correct. + */ +typedef ERL_NIF_TERM (*NativeFunPtr)(ErlNifEnv*, int, const ERL_NIF_TERM[]); + +typedef struct { + Export exp; + struct erl_module_nif* m; + NativeFunPtr fp; + Eterm saved_mfa[3]; + int exception_thrown; + int saved_argc; + int rootset_extra; + Eterm rootset[1]; +} NifExport; + +/* + * If a process has saved arguments, they need to be part of the GC + * rootset. The function below is called from setup_rootset() in + * erl_gc.c. This function is declared in erl_process.h. Any exception term + * saved in the NifExport is also made part of the GC rootset here; it + * always resides in rootset[0]. + */ +int +erts_setup_nif_gc(Process* proc, Eterm** objv, int* nobj) +{ + NifExport* ep = (NifExport*) ERTS_PROC_GET_NIF_TRAP_EXPORT(proc); + int gc = ep && (ep->saved_argc > 0 || ep->rootset[0] != NIL); + + if (gc) { + *objv = ep->rootset; + *nobj = 1 + ep->saved_argc; + } + return gc; +} + +/* + * Allocate a NifExport and set it in proc specific data + */ +static NifExport* +allocate_nif_sched_data(Process* proc, int argc) +{ + NifExport* ep; + size_t total; + int i; + + total = sizeof(NifExport) + argc*sizeof(Eterm); + ep = erts_alloc(ERTS_ALC_T_NIF_TRAP_EXPORT, total); + sys_memset((void*) ep, 0, total); + ep->rootset_extra = argc; + ep->rootset[0] = NIL; + for (i=0; i<ERTS_NUM_CODE_IX; i++) { + ep->exp.addressv[i] = &ep->exp.code[3]; + } + ep->exp.code[3] = (BeamInstr) em_call_nif; + (void) ERTS_PROC_SET_NIF_TRAP_EXPORT(proc, ERTS_PROC_LOCK_MAIN, ep); + return ep; +} + +static ERTS_INLINE void +destroy_nif_export(NifExport *nif_export) +{ + erts_free(ERTS_ALC_T_NIF_TRAP_EXPORT, (void *) nif_export); +} + +void +erts_destroy_nif_export(void *nif_export) +{ + destroy_nif_export((NifExport *) nif_export); +} + +/* + * Initialize a NifExport struct. Create it if needed and store it in the + * proc. The direct_fp function is what will be invoked by op_call_nif, and + * the indirect_fp function, if not NULL, is what the direct_fp function + * will call. If the allocated NifExport isn't enough to hold all of argv, + * allocate a larger one. Save MFA and registers only if the need_save + * parameter is true. + */ +static ERL_NIF_TERM +init_nif_sched_data(ErlNifEnv* env, NativeFunPtr direct_fp, NativeFunPtr indirect_fp, + int need_save, int argc, const ERL_NIF_TERM argv[]) +{ + Process* proc = env->proc; + Eterm* reg = ERTS_PROC_GET_SCHDATA(proc)->x_reg_array; + NifExport* ep; + int i; + + ep = (NifExport*) ERTS_PROC_GET_NIF_TRAP_EXPORT(proc); + if (!ep) + ep = allocate_nif_sched_data(proc, argc); + else if (need_save && ep->rootset_extra < argc) { + NifExport* new_ep = allocate_nif_sched_data(proc, argc); + destroy_nif_export(ep); + ep = new_ep; + } + if (env->exception_thrown) { + ep->exception_thrown = 1; + ep->rootset[0] = env->proc->fvalue; + } else { + ep->exception_thrown = 0; + ep->rootset[0] = NIL; + } + ERTS_VBUMP_ALL_REDS(proc); + for (i = 0; i < argc; i++) { + if (need_save) + ep->rootset[i+1] = reg[i]; + reg[i] = (Eterm) argv[i]; + } + if (need_save) { + ep->saved_mfa[0] = proc->current[0]; + ep->saved_mfa[1] = proc->current[1]; + ep->saved_mfa[2] = proc->current[2]; + ep->saved_argc = argc; + } + proc->i = (BeamInstr*) ep->exp.addressv[0]; + ep->exp.code[0] = (BeamInstr) proc->current[0]; + ep->exp.code[1] = (BeamInstr) proc->current[1]; + ep->exp.code[2] = argc; + ep->exp.code[4] = (BeamInstr) direct_fp; + ep->m = env->mod_nif; + ep->fp = indirect_fp; + proc->freason = TRAP; + proc->arity = argc; + return THE_NON_VALUE; +} + +/* + * Restore saved MFA and registers. Registers are restored only when the + * exception flag is true. + */ +static void +restore_nif_mfa(Process* proc, NifExport* ep, int exception) +{ + int i; + Eterm* reg = ERTS_PROC_GET_SCHDATA(proc)->x_reg_array; + + proc->current[0] = ep->saved_mfa[0]; + proc->current[1] = ep->saved_mfa[1]; + proc->current[2] = ep->saved_mfa[2]; + if (exception) + for (i = 0; i < ep->saved_argc; i++) + reg[i] = ep->rootset[i+1]; + ep->saved_argc = 0; + ep->saved_mfa[0] = THE_NON_VALUE; +} + +#ifdef ERTS_DIRTY_SCHEDULERS + +/* + * Finalize a dirty NIF call. This function is scheduled to cause the VM to + * switch the process off a dirty scheduler thread and back onto a regular + * scheduler thread, and then return the result from the dirty NIF. It also + * restores the original NIF MFA when necessary based on the value of + * ep->fp set by execute_dirty_nif via init_nif_sched_data -- non-NULL + * means restore, NULL means do not restore. + */ +static ERL_NIF_TERM +dirty_nif_finalizer(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) +{ + Process* proc = env->proc; + NifExport* ep; + + ASSERT(argc == 1); + ASSERT(!ERTS_SCHEDULER_IS_DIRTY(env->proc->scheduler_data)); + ep = (NifExport*) ERTS_PROC_GET_NIF_TRAP_EXPORT(proc); + ASSERT(ep); + ASSERT(!ep->exception_thrown); + if (ep->fp) + restore_nif_mfa(proc, ep, 0); + return argv[0]; +} + +/* Finalize a dirty NIF call that raised an exception. Otherwise same as + * the dirty_nif_finalizer() function. + */ +static ERL_NIF_TERM +dirty_nif_exception(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) +{ + Process* proc = env->proc; + NifExport* ep; + + ASSERT(!ERTS_SCHEDULER_IS_DIRTY(env->proc->scheduler_data)); + ep = (NifExport*) ERTS_PROC_GET_NIF_TRAP_EXPORT(proc); + ASSERT(ep); + ASSERT(ep->exception_thrown); + if (ep->fp) + restore_nif_mfa(proc, ep, 1); + return enif_raise_exception(env, ep->rootset[0]); +} + +/* + * Dirty NIF execution wrapper function. Invoke an application's dirty NIF, + * then check the result and schedule the appropriate finalizer function + * where needed. Also restore the original NIF MFA when appropriate. + */ +static ERL_NIF_TERM +execute_dirty_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) +{ + Process* proc = env->proc; + NativeFunPtr fp = (NativeFunPtr) proc->current[6]; + NifExport* ep; + ERL_NIF_TERM result; + + ASSERT(ERTS_SCHEDULER_IS_DIRTY(env->proc->scheduler_data)); + + /* + * Set ep->fp to NULL before the native call so we know later whether it scheduled another NIF for execution + */ + ep = (NifExport*) ERTS_PROC_GET_NIF_TRAP_EXPORT(proc); + ASSERT(ep); + ep->fp = NULL; + result = (*fp)(env, argc, argv); + erts_smp_atomic32_read_band_mb(&proc->state, + ~(ERTS_PSFLG_DIRTY_CPU_PROC + |ERTS_PSFLG_DIRTY_IO_PROC + |ERTS_PSFLG_DIRTY_CPU_PROC_IN_Q + |ERTS_PSFLG_DIRTY_IO_PROC_IN_Q)); + if (erts_refc_dectest(&env->mod_nif->rt_dtor_cnt, 0) == 0 && env->mod_nif->mod == NULL) + close_lib(env->mod_nif); + /* + * If no more NIFs were scheduled by the native call via + * enif_schedule_nif(), then ep->fp will still be NULL as set above, in + * which case we need to restore the original NIF calling + * context. Reuse fp essentially as a boolean for this, passing it to + * init_nif_sched_data below. Both dirty_nif_exception and + * dirty_nif_finalizer then check ep->fp to decide whether or not to + * restore the original calling context. + */ + ep = (NifExport*) ERTS_PROC_GET_NIF_TRAP_EXPORT(proc); + ASSERT(ep); + if (ep->fp) + fp = NULL; + if (is_non_value(result) || env->exception_thrown) { + if (proc->freason != TRAP) { + return init_nif_sched_data(env, dirty_nif_exception, fp, 0, argc, argv); + } else { + if (ep->fp == NULL) + restore_nif_mfa(proc, ep, 1); + return THE_NON_VALUE; + } + } + else + return init_nif_sched_data(env, dirty_nif_finalizer, fp, 0, 1, &result); +} + +/* + * Dirty NIF scheduling wrapper function. Schedule a dirty NIF to execute + * via the execute_dirty_nif() wrapper function. The dirty scheduler thread + * type (CPU or I/O) is indicated in flags parameter. + */ +static ERTS_INLINE ERL_NIF_TERM +schedule_dirty_nif(ErlNifEnv* env, int flags, int argc, const ERL_NIF_TERM argv[]) +{ + erts_aint32_t state, n, a; + Process* proc = env->proc; + NativeFunPtr fp = (NativeFunPtr) proc->current[6]; + NifExport* ep; + int need_save; + + ASSERT(flags==ERL_NIF_DIRTY_JOB_IO_BOUND || flags==ERL_NIF_DIRTY_JOB_CPU_BOUND); + + a = erts_smp_atomic32_read_acqb(&proc->state); + while (1) { + n = state = a; + /* + * clear any current dirty flags and dirty queue indicators, + * in case the application is shifting a job from one type + * of dirty scheduler to the other + */ + n &= ~(ERTS_PSFLG_DIRTY_CPU_PROC|ERTS_PSFLG_DIRTY_IO_PROC + |ERTS_PSFLG_DIRTY_CPU_PROC_IN_Q|ERTS_PSFLG_DIRTY_IO_PROC_IN_Q); + if (flags == ERL_NIF_DIRTY_JOB_CPU_BOUND) + n |= ERTS_PSFLG_DIRTY_CPU_PROC; + else + n |= ERTS_PSFLG_DIRTY_IO_PROC; + a = erts_smp_atomic32_cmpxchg_mb(&proc->state, n, state); + if (a == state) + break; + } + erts_refc_inc(&env->mod_nif->rt_dtor_cnt, 1); + ep = (NifExport*) ERTS_PROC_GET_NIF_TRAP_EXPORT(proc); + need_save = (ep == NULL || is_non_value(ep->saved_mfa[0])); + return init_nif_sched_data(env, execute_dirty_nif, fp, need_save, argc, argv); +} + +static ERL_NIF_TERM +schedule_dirty_io_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) +{ + return schedule_dirty_nif(env, ERL_NIF_DIRTY_JOB_IO_BOUND, argc, argv); +} + +static ERL_NIF_TERM +schedule_dirty_cpu_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) +{ + return schedule_dirty_nif(env, ERL_NIF_DIRTY_JOB_CPU_BOUND, argc, argv); +} + +#endif /* ERTS_DIRTY_SCHEDULERS */ + +/* + * NIF execution wrapper used by enif_schedule_nif() for regular NIFs. It + * calls the actual NIF, restores original NIF MFA if necessary, and + * then returns the NIF result. + */ +static ERL_NIF_TERM +execute_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) +{ + Process* proc = env->proc; + NativeFunPtr fp = (NativeFunPtr) proc->current[6]; + NifExport* ep; + ERL_NIF_TERM result; + + ASSERT(!env->exception_thrown); + ep = (NifExport*) ERTS_PROC_GET_NIF_TRAP_EXPORT(proc); + ASSERT(ep); + ep->fp = NULL; + result = (*fp)(env, argc, argv); + ep = (NifExport*) ERTS_PROC_GET_NIF_TRAP_EXPORT(proc); + ASSERT(ep); + /* + * If no NIFs were scheduled by the native call via + * enif_schedule_nif(), then ep->fp will still be NULL as set above, in + * which case we need to restore the original NIF MFA. + */ + if (ep->fp == NULL) + restore_nif_mfa(proc, ep, env->exception_thrown); + return result; +} + +ERL_NIF_TERM +enif_schedule_nif(ErlNifEnv* env, const char* fun_name, int flags, + ERL_NIF_TERM (*fp)(ErlNifEnv*, int, const ERL_NIF_TERM[]), + int argc, const ERL_NIF_TERM argv[]) +{ + Process* proc = env->proc; + NifExport* ep; + ERL_NIF_TERM fun_name_atom, result; + int need_save; + + if (argc > MAX_ARG) + return enif_make_badarg(env); + fun_name_atom = enif_make_atom(env, fun_name); + if (enif_is_exception(env, fun_name_atom)) + return fun_name_atom; + + ep = (NifExport*) ERTS_PROC_GET_NIF_TRAP_EXPORT(proc); + need_save = (ep == NULL || is_non_value(ep->saved_mfa[0])); + + if (flags) { +#ifdef ERTS_DIRTY_SCHEDULERS + NativeFunPtr sched_fun; + int chkflgs = (flags & (ERL_NIF_DIRTY_JOB_IO_BOUND|ERL_NIF_DIRTY_JOB_CPU_BOUND)); + if (chkflgs == ERL_NIF_DIRTY_JOB_IO_BOUND) + sched_fun = schedule_dirty_io_nif; + else if (chkflgs == ERL_NIF_DIRTY_JOB_CPU_BOUND) + sched_fun = schedule_dirty_cpu_nif; + else + return enif_make_badarg(env); + result = init_nif_sched_data(env, sched_fun, fp, need_save, argc, argv); +#else + return enif_make_badarg(env); +#endif + } + else + result = init_nif_sched_data(env, execute_nif, fp, need_save, argc, argv); + + ep = (NifExport*) ERTS_PROC_GET_NIF_TRAP_EXPORT(proc); + ASSERT(ep); + ep->exp.code[1] = (BeamInstr) fun_name_atom; + return result; +} + +#ifdef ERL_NIF_DIRTY_SCHEDULER_SUPPORT + +int +enif_is_on_dirty_scheduler(ErlNifEnv* env) +{ + return ERTS_SCHEDULER_IS_DIRTY(env->proc->scheduler_data); +} + +#endif /* ERL_NIF_DIRTY_SCHEDULER_SUPPORT */ + +/* Maps */ + +int enif_is_map(ErlNifEnv* env, ERL_NIF_TERM term) +{ + return is_map(term); +} + +int enif_get_map_size(ErlNifEnv* env, ERL_NIF_TERM term, size_t *size) +{ + if (is_flatmap(term)) { + flatmap_t *mp; + mp = (flatmap_t*)flatmap_val(term); + *size = flatmap_get_size(mp); + return 1; + } + else if (is_hashmap(term)) { + *size = hashmap_size(term); + return 1; + } + return 0; +} + +ERL_NIF_TERM enif_make_new_map(ErlNifEnv* env) +{ + Eterm* hp = alloc_heap(env,MAP_HEADER_FLATMAP_SZ+1); + Eterm tup; + flatmap_t *mp; + + tup = make_tuple(hp); + *hp++ = make_arityval(0); + mp = (flatmap_t*)hp; + mp->thing_word = MAP_HEADER_FLATMAP; + mp->size = 0; + mp->keys = tup; + + return make_flatmap(mp); +} + +int enif_make_map_put(ErlNifEnv* env, + Eterm map_in, + Eterm key, + Eterm value, + Eterm *map_out) +{ + if (!is_map(map_in)) { + return 0; + } + flush_env(env); + *map_out = erts_maps_put(env->proc, key, value, map_in); + cache_env(env); + return 1; +} + +int enif_get_map_value(ErlNifEnv* env, + Eterm map, + Eterm key, + Eterm *value) +{ + const Eterm *ret; + if (!is_map(map)) { + return 0; + } + ret = erts_maps_get(key, map); + if (ret) { + *value = *ret; + return 1; + } + return 0; +} + +int enif_make_map_update(ErlNifEnv* env, + Eterm map_in, + Eterm key, + Eterm value, + Eterm *map_out) +{ + int res; + if (!is_map(map_in)) { + return 0; + } + + flush_env(env); + res = erts_maps_update(env->proc, key, value, map_in, map_out); + cache_env(env); + return res; +} + +int enif_make_map_remove(ErlNifEnv* env, + Eterm map_in, + Eterm key, + Eterm *map_out) +{ + int res; + if (!is_map(map_in)) { + return 0; + } + flush_env(env); + res = erts_maps_remove(env->proc, key, map_in, map_out); + cache_env(env); + return res; +} + +int enif_map_iterator_create(ErlNifEnv *env, + Eterm map, + ErlNifMapIterator *iter, + ErlNifMapIteratorEntry entry) +{ + if (is_flatmap(map)) { + flatmap_t *mp = (flatmap_t*)flatmap_val(map); + size_t offset; + + switch (entry) { + case ERL_NIF_MAP_ITERATOR_FIRST: offset = 0; break; + case ERL_NIF_MAP_ITERATOR_LAST: offset = flatmap_get_size(mp) - 1; break; + default: goto error; + } + + /* empty maps are ok but will leave the iterator + * in bad shape. + */ + + iter->map = map; + iter->u.flat.ks = ((Eterm *)flatmap_get_keys(mp)) + offset; + iter->u.flat.vs = ((Eterm *)flatmap_get_values(mp)) + offset; + iter->size = flatmap_get_size(mp); + iter->idx = offset + 1; + + return 1; + } + else if (is_hashmap(map)) { + iter->map = map; + iter->size = hashmap_size(map); + iter->u.hash.wstack = erts_alloc(ERTS_ALC_T_NIF, sizeof(ErtsDynamicWStack)); + WSTACK_INIT(iter->u.hash.wstack, ERTS_ALC_T_NIF); + + switch (entry) { + case ERL_NIF_MAP_ITERATOR_FIRST: + iter->idx = 1; + hashmap_iterator_init(&iter->u.hash.wstack->ws, map, 0); + iter->u.hash.kv = hashmap_iterator_next(&iter->u.hash.wstack->ws); + break; + case ERL_NIF_MAP_ITERATOR_LAST: + iter->idx = hashmap_size(map); + hashmap_iterator_init(&iter->u.hash.wstack->ws, map, 1); + iter->u.hash.kv = hashmap_iterator_prev(&iter->u.hash.wstack->ws); + break; + default: + goto error; + } + ASSERT(!!iter->u.hash.kv == (iter->idx >= 1 && + iter->idx <= iter->size)); + return 1; + } +error: +#ifdef DEBUG + iter->map = THE_NON_VALUE; +#endif + return 0; +} + +void enif_map_iterator_destroy(ErlNifEnv *env, ErlNifMapIterator *iter) +{ + if (is_hashmap(iter->map)) { + WSTACK_DESTROY(iter->u.hash.wstack->ws); + erts_free(ERTS_ALC_T_NIF, iter->u.hash.wstack); + } + else + ASSERT(is_flatmap(iter->map)); + +#ifdef DEBUG + iter->map = THE_NON_VALUE; +#endif +} + +int enif_map_iterator_is_tail(ErlNifEnv *env, ErlNifMapIterator *iter) +{ + ASSERT(iter); + if (is_flatmap(iter->map)) { + ASSERT(iter->idx >= 0); + ASSERT(iter->idx <= flatmap_get_size(flatmap_val(iter->map)) + 1); + return (iter->size == 0 || iter->idx > iter->size); + } + else { + ASSERT(is_hashmap(iter->map)); + return iter->idx > iter->size; + } +} + +int enif_map_iterator_is_head(ErlNifEnv *env, ErlNifMapIterator *iter) +{ + ASSERT(iter); + if (is_flatmap(iter->map)) { + ASSERT(iter->idx >= 0); + ASSERT(iter->idx <= flatmap_get_size(flatmap_val(iter->map)) + 1); + return (iter->size == 0 || iter->idx == 0); + } + else { + ASSERT(is_hashmap(iter->map)); + return iter->idx == 0; + } +} + + +int enif_map_iterator_next(ErlNifEnv *env, ErlNifMapIterator *iter) +{ + ASSERT(iter); + if (is_flatmap(iter->map)) { + if (iter->idx <= iter->size) { + iter->idx++; + iter->u.flat.ks++; + iter->u.flat.vs++; + } + return (iter->idx <= iter->size); + } + else { + ASSERT(is_hashmap(iter->map)); + + if (iter->idx <= hashmap_size(iter->map)) { + if (iter->idx < 1) { + hashmap_iterator_init(&iter->u.hash.wstack->ws, iter->map, 0); + } + iter->u.hash.kv = hashmap_iterator_next(&iter->u.hash.wstack->ws); + iter->idx++; + ASSERT(!!iter->u.hash.kv == (iter->idx <= iter->size)); + } + return iter->idx <= iter->size; + } +} + +int enif_map_iterator_prev(ErlNifEnv *env, ErlNifMapIterator *iter) +{ + ASSERT(iter); + if (is_flatmap(iter->map)) { + if (iter->idx > 0) { + iter->idx--; + iter->u.flat.ks--; + iter->u.flat.vs--; + } + return iter->idx > 0; + } + else { + ASSERT(is_hashmap(iter->map)); + + if (iter->idx > 0) { + if (iter->idx > iter->size) { + hashmap_iterator_init(&iter->u.hash.wstack->ws, iter->map, 1); + } + iter->u.hash.kv = hashmap_iterator_prev(&iter->u.hash.wstack->ws); + iter->idx--; + ASSERT(!!iter->u.hash.kv == (iter->idx > 0)); + } + return iter->idx > 0; + } +} + +int enif_map_iterator_get_pair(ErlNifEnv *env, + ErlNifMapIterator *iter, + Eterm *key, + Eterm *value) +{ + ASSERT(iter); + if (is_flatmap(iter->map)) { + if (iter->idx > 0 && iter->idx <= iter->size) { + ASSERT(iter->u.flat.ks >= flatmap_get_keys(flatmap_val(iter->map)) && + iter->u.flat.ks < (flatmap_get_keys(flatmap_val(iter->map)) + flatmap_get_size(flatmap_val(iter->map)))); + ASSERT(iter->u.flat.vs >= flatmap_get_values(flatmap_val(iter->map)) && + iter->u.flat.vs < (flatmap_get_values(flatmap_val(iter->map)) + flatmap_get_size(flatmap_val(iter->map)))); + *key = *(iter->u.flat.ks); + *value = *(iter->u.flat.vs); + return 1; + } + } + else { + ASSERT(is_hashmap(iter->map)); + if (iter->idx > 0 && iter->idx <= iter->size) { + *key = CAR(iter->u.hash.kv); + *value = CDR(iter->u.hash.kv); + return 1; + } + } + return 0; +} + /*************************************************************************** ** load_nif/2 ** ***************************************************************************/ @@ -1564,6 +2353,35 @@ static Eterm load_nif_error(Process* p, const char* atom, const char* format, .. return ret; } +/* + * The function below is for looping through ErlNifFunc arrays, helping + * provide backwards compatibility across the version 2.7 change that added + * the "flags" field to ErlNifFunc. + */ +static ErlNifFunc* next_func(ErlNifEntry* entry, int* incrp, ErlNifFunc* func) +{ + ASSERT(incrp); + if (!*incrp) { + if (entry->major > 2 || (entry->major == 2 && entry->minor >= 7)) + *incrp = sizeof(ErlNifFunc); + else { + /* + * ErlNifFuncV1 below is what ErlNifFunc was before the + * addition of the flags field for 2.7, and is needed to handle + * backward compatibility. + */ + typedef struct { + const char* name; + unsigned arity; + ERL_NIF_TERM (*fptr)(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]); + }ErlNifFuncV1; + *incrp = sizeof(ErlNifFuncV1); + } + } + return (ErlNifFunc*) ((char*)func + *incrp); +} + + BIF_RETTYPE load_nif_2(BIF_ALIST_2) { static const char bad_lib[] = "bad_lib"; @@ -1571,12 +2389,13 @@ BIF_RETTYPE load_nif_2(BIF_ALIST_2) static const char upgrade[] = "upgrade"; char* lib_name = NULL; void* handle = NULL; - void* init_func; + void* init_func = NULL; ErlNifEntry* entry = NULL; ErlNifEnv env; - int len, i, err; + int i, err, encoding; Module* mod; Eterm mod_atom; + const Atom* mod_atomp; Eterm f_atom; BeamInstr* caller; ErtsSysDdllError errdesc = ERTS_SYS_DDLL_ERROR_INIT; @@ -1585,18 +2404,18 @@ BIF_RETTYPE load_nif_2(BIF_ALIST_2) struct erl_module_nif* lib = NULL; int reload_warning = 0; - len = list_length(BIF_ARG_1); - if (len < 0) { - BIF_ERROR(BIF_P, BADARG); + encoding = erts_get_native_filename_encoding(); + if (encoding == ERL_FILENAME_WIN_WCHAR) { + /* Do not convert the lib name to utf-16le yet, do that in win32 specific code */ + /* since lib_name is used in error messages */ + encoding = ERL_FILENAME_UTF8; } - - lib_name = (char *) erts_alloc(ERTS_ALC_T_TMP, len + 1); - - if (intlist_to_buf(BIF_ARG_1, lib_name, len) != len) { - erts_free(ERTS_ALC_T_TMP, lib_name); + lib_name = erts_convert_filename_to_encoding(BIF_ARG_1, NULL, 0, + ERTS_ALC_T_TMP, 1, 0, encoding, + NULL, 0); + if (!lib_name) { BIF_ERROR(BIF_P, BADARG); } - lib_name[len] = '\0'; if (!erts_try_seize_code_write_permission(BIF_P)) { erts_free(ERTS_ALC_T_TMP, lib_name); @@ -1620,13 +2439,19 @@ BIF_RETTYPE load_nif_2(BIF_ALIST_2) mod=erts_get_module(mod_atom, erts_active_code_ix()); ASSERT(mod != NULL); + mod_atomp = atom_tab(atom_val(mod_atom)); + init_func = erts_static_nif_get_nif_init((char*)mod_atomp->name, mod_atomp->len); + if (init_func != NULL) + handle = init_func; + if (!in_area(caller, mod->curr.code, mod->curr.code_length)) { ASSERT(in_area(caller, mod->old.code, mod->old.code_length)); ret = load_nif_error(BIF_P, "old_code", "Calling load_nif from old " "module '%T' not allowed", mod_atom); } - else if ((err=erts_sys_ddll_open2(lib_name, &handle, &errdesc)) != ERL_DE_NO_ERROR) { + else if (init_func == NULL && + (err=erts_sys_ddll_open(lib_name, &handle, &errdesc)) != ERL_DE_NO_ERROR) { const char slogan[] = "Failed to load NIF library"; if (strstr(errdesc.str, lib_name) != NULL) { ret = load_nif_error(BIF_P, "load_failed", "%s: '%s'", slogan, errdesc.str); @@ -1635,7 +2460,8 @@ BIF_RETTYPE load_nif_2(BIF_ALIST_2) ret = load_nif_error(BIF_P, "load_failed", "%s %s: '%s'", slogan, lib_name, errdesc.str); } } - else if (erts_sys_ddll_load_nif_init(handle, &init_func, &errdesc) != ERL_DE_NO_ERROR) { + else if (init_func == NULL && + erts_sys_ddll_load_nif_init(handle, &init_func, &errdesc) != ERL_DE_NO_ERROR) { ret = load_nif_error(BIF_P, bad_lib, "Failed to find library init" " function: '%s'", errdesc.str); @@ -1644,8 +2470,11 @@ BIF_RETTYPE load_nif_2(BIF_ALIST_2) (entry = erts_sys_ddll_call_nif_init(init_func)) == NULL)) { ret = load_nif_error(BIF_P, bad_lib, "Library init-call unsuccessful"); } - else if (entry->major != ERL_NIF_MAJOR_VERSION - || entry->minor > ERL_NIF_MINOR_VERSION) { + else if (entry->major < ERL_NIF_MIN_REQUIRED_MAJOR_VERSION_ON_LOAD + || (ERL_NIF_MAJOR_VERSION < entry->major + || (ERL_NIF_MAJOR_VERSION == entry->major + && ERL_NIF_MINOR_VERSION < entry->minor)) + || (entry->major==2 && entry->minor == 5)) { /* experimental maps */ ret = load_nif_error(BIF_P, bad_lib, "Library version (%d.%d) not compatible (with %d.%d).", entry->major, entry->minor, ERL_NIF_MAJOR_VERSION, ERL_NIF_MINOR_VERSION); @@ -1662,22 +2491,48 @@ BIF_RETTYPE load_nif_2(BIF_ALIST_2) } else { /*erts_fprintf(stderr, "Found module %T\r\n", mod_atom);*/ - + + int maybe_dirty_nifs = ((entry->major > 2 || (entry->major == 2 && entry->minor >= 7)) + && (entry->options & ERL_NIF_DIRTY_NIF_OPTION)); + int incr = 0; + ErlNifFunc* f = entry->funcs; for (i=0; i < entry->num_of_funcs && ret==am_ok; i++) { BeamInstr** code_pp; - ErlNifFunc* f = &entry->funcs[i]; if (!erts_atom_get(f->name, sys_strlen(f->name), &f_atom, ERTS_ATOM_ENC_LATIN1) || (code_pp = get_func_pp(mod->curr.code, f_atom, f->arity))==NULL) { ret = load_nif_error(BIF_P,bad_lib,"Function not found %T:%s/%u", mod_atom, f->name, f->arity); - } - else if (code_pp[1] - code_pp[0] < (5+3)) { + } + else if (maybe_dirty_nifs && f->flags) { + /* + * If the flags field is non-zero and this emulator was + * built with dirty scheduler support, check that the flags + * value is legal. But if this emulator was built without + * dirty scheduler support, treat a non-zero flags field as + * a load error. + */ +#ifdef ERTS_DIRTY_SCHEDULERS + if (f->flags != ERL_NIF_DIRTY_JOB_IO_BOUND && f->flags != ERL_NIF_DIRTY_JOB_CPU_BOUND) + ret = load_nif_error(BIF_P, bad_lib, "Illegal flags field value %d for NIF %T:%s/%u", + f->flags, mod_atom, f->name, f->arity); +#else + ret = load_nif_error(BIF_P, bad_lib, "NIF %T:%s/%u requires a runtime with dirty scheduler support.", + mod_atom, f->name, f->arity); +#endif + } +#ifdef ERTS_DIRTY_SCHEDULERS + else if (code_pp[1] - code_pp[0] < (5+4)) +#else + else if (code_pp[1] - code_pp[0] < (5+3)) +#endif + { ret = load_nif_error(BIF_P,bad_lib,"No explicit call to load_nif" - " in module (%T:%s/%u to small)", - mod_atom, entry->funcs[i].name, entry->funcs[i].arity); + " in module (%T:%s/%u too small)", + mod_atom, f->name, f->arity); } /*erts_fprintf(stderr, "Found NIF %T:%s/%u\r\n", - mod_atom, entry->funcs[i].name, entry->funcs[i].arity);*/ + mod_atom, f->name, f->arity);*/ + f = next_func(entry, &incr, f); } } @@ -1694,10 +2549,17 @@ BIF_RETTYPE load_nif_2(BIF_ALIST_2) lib->entry = entry; erts_refc_init(&lib->rt_cnt, 0); erts_refc_init(&lib->rt_dtor_cnt, 0); + ASSERT(opened_rt_list == NULL); lib->mod = mod; env.mod_nif = lib; - if (mod->curr.nif != NULL) { /* Reload */ - int k; + if (mod->curr.nif != NULL) { /*************** Reload ******************/ + /* + * Repeated load_nif calls from same Erlang module instance ("reload") + * is deprecated and was only ment as a development feature not to + * be used in production systems. (See warning below) + */ + int k, old_incr = 0; + ErlNifFunc* old_func; lib->priv_data = mod->curr.nif->priv_data; ASSERT(mod->curr.nif->entry != NULL); @@ -1706,13 +2568,16 @@ BIF_RETTYPE load_nif_2(BIF_ALIST_2) goto error; } /* Check that no NIF is removed */ + old_func = mod->curr.nif->entry->funcs; for (k=0; k < mod->curr.nif->entry->num_of_funcs; k++) { - ErlNifFunc* old_func = &mod->curr.nif->entry->funcs[k]; + int incr = 0; + ErlNifFunc* f = entry->funcs; for (i=0; i < entry->num_of_funcs; i++) { - if (old_func->arity == entry->funcs[i].arity - && sys_strcmp(old_func->name, entry->funcs[i].name) == 0) { + if (old_func->arity == f->arity + && sys_strcmp(old_func->name, f->name) == 0) { break; } + f = next_func(entry, &incr, f); } if (i == entry->num_of_funcs) { ret = load_nif_error(BIF_P,reload,"Reloaded library missing " @@ -1720,7 +2585,8 @@ BIF_RETTYPE load_nif_2(BIF_ALIST_2) old_func->name, old_func->arity); goto error; } - } + old_func = next_func(mod->curr.nif->entry, &old_incr, old_func); + } erts_pre_nif(&env, BIF_P, lib); veto = entry->reload(&env, &lib->priv_data, BIF_ARG_2); erts_post_nif(&env); @@ -1728,6 +2594,7 @@ BIF_RETTYPE load_nif_2(BIF_ALIST_2) ret = load_nif_error(BIF_P, reload, "Library reload-call unsuccessful."); } else { + commit_opened_resource_types(lib); mod->curr.nif->entry = NULL; /* to prevent 'unload' callback */ erts_unload_nif(mod->curr.nif); reload_warning = 1; @@ -1735,7 +2602,7 @@ BIF_RETTYPE load_nif_2(BIF_ALIST_2) } else { lib->priv_data = NULL; - if (mod->old.nif != NULL) { /* Upgrade */ + if (mod->old.nif != NULL) { /**************** Upgrade ***************/ void* prev_old_data = mod->old.nif->priv_data; if (entry->upgrade == NULL) { ret = load_nif_error(BIF_P, upgrade, "Upgrade not supported by this NIF library."); @@ -1748,30 +2615,35 @@ BIF_RETTYPE load_nif_2(BIF_ALIST_2) mod->old.nif->priv_data = prev_old_data; ret = load_nif_error(BIF_P, upgrade, "Library upgrade-call unsuccessful."); } - /*else if (mod->old_nif->priv_data != prev_old_data) { - refresh_cached_nif_data(mod->old_code, mod->old_nif); - }*/ + else + commit_opened_resource_types(lib); } - else if (entry->load != NULL) { /* Initial load */ + else if (entry->load != NULL) { /********* Initial load ***********/ erts_pre_nif(&env, BIF_P, lib); veto = entry->load(&env, &lib->priv_data, BIF_ARG_2); erts_post_nif(&env); if (veto) { ret = load_nif_error(BIF_P, "load", "Library load-call unsuccessful."); } + else + commit_opened_resource_types(lib); } } if (ret == am_ok) { /* ** Everything ok, patch the beam code with op_call_nif */ - mod->curr.nif = lib; + + int incr = 0; + ErlNifFunc* f = entry->funcs; + + mod->curr.nif = lib; for (i=0; i < entry->num_of_funcs; i++) { BeamInstr* code_ptr; - erts_atom_get(entry->funcs[i].name, sys_strlen(entry->funcs[i].name), &f_atom, ERTS_ATOM_ENC_LATIN1); - code_ptr = *get_func_pp(mod->curr.code, f_atom, entry->funcs[i].arity); - + erts_atom_get(f->name, sys_strlen(f->name), &f_atom, ERTS_ATOM_ENC_LATIN1); + code_ptr = *get_func_pp(mod->curr.code, f_atom, f->arity); + if (code_ptr[1] == 0) { code_ptr[5+0] = (BeamInstr) BeamOp(op_call_nif); } @@ -1779,19 +2651,31 @@ BIF_RETTYPE load_nif_2(BIF_ALIST_2) GenericBp* g = (GenericBp *) code_ptr[1]; ASSERT(code_ptr[5+0] == (BeamInstr) BeamOp(op_i_generic_breakpoint)); - g->orig_instr = (BeamInstr) BeamOp(op_call_nif); - } - code_ptr[5+1] = (BeamInstr) entry->funcs[i].fptr; + g->orig_instr = (BeamInstr) BeamOp(op_call_nif); + } + if ((entry->major > 2 || (entry->major == 2 && entry->minor >= 7)) + && (entry->options & ERL_NIF_DIRTY_NIF_OPTION) && f->flags) { +#ifdef ERL_NIF_DIRTY_SCHEDULER_SUPPORT + code_ptr[5+3] = (BeamInstr) f->fptr; + code_ptr[5+1] = (f->flags == ERL_NIF_DIRTY_JOB_IO_BOUND) ? + (BeamInstr) schedule_dirty_io_nif : + (BeamInstr) schedule_dirty_cpu_nif; +#endif + } + else + code_ptr[5+1] = (BeamInstr) f->fptr; code_ptr[5+2] = (BeamInstr) lib; + f = next_func(entry, &incr, f); } } else { error: + rollback_opened_resource_types(); ASSERT(ret != am_ok); if (lib != NULL) { erts_free(ERTS_ALC_T_NIF, lib); } - if (handle != NULL) { + if (handle != NULL && !erts_is_static_nif(handle)) { erts_sys_ddll_close(handle); } erts_sys_ddll_free_error(&errdesc); @@ -1856,6 +2740,8 @@ erts_unload_nif(struct erl_module_nif* lib) void erl_nif_init() { + ERTS_CT_ASSERT((offsetof(ErlNifResource,data) % 8) == ERTS_MAGIC_BIN_BYTES_TO_ALIGN); + resource_type_list.next = &resource_type_list; resource_type_list.prev = &resource_type_list; resource_type_list.dtor = NULL; diff --git a/erts/emulator/beam/erl_nif.h b/erts/emulator/beam/erl_nif.h index 8006741a63..75070ad901 100644 --- a/erts/emulator/beam/erl_nif.h +++ b/erts/emulator/beam/erl_nif.h @@ -1,18 +1,19 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2009-2013. All Rights Reserved. + * Copyright Ericsson AB 2009-2014. All Rights Reserved. * - * The contents of this file are subject to the Erlang Public License, - * Version 1.1, (the "License"); you may not use this file except in - * compliance with the License. You should have received a copy of the - * Erlang Public License along with this software. If not, it can be - * retrieved online at http://www.erlang.org/. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Software distributed under the License is distributed on an "AS IS" - * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See - * the License for the specific language governing rights and limitations - * under the License. + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. * * %CopyrightEnd% */ @@ -23,7 +24,11 @@ #ifndef __ERL_NIF_H__ #define __ERL_NIF_H__ +#ifdef HAVE_CONFIG_H +# include "config.h" +#endif +#include "erl_native_features_config.h" #include "erl_drv_nif.h" /* Version history: @@ -34,67 +39,65 @@ ** 2.2: R14B03 enif_is_exception ** 2.3: R15 enif_make_reverse_list, enif_is_number ** 2.4: R16 enif_consume_timeslice +** 2.5: First experimental maps API additions (libs of this version is not compatible with any other VM) +** 2.5: R17 Maps API additions +** 2.6: R17 with maps +** R17 dirty schedulers +** 2.7: 17.3 add enif_schedule_nif +** remove enif_schedule_dirty_nif, enif_schedule_dirty_nif_finalizer, enif_dirty_nif_finalizer +** add ErlNifEntry options +** add ErlNifFunc flags +** 2.8: 18.0 add enif_has_pending_exception +** 2.9: 18.2 enif_getenv +** 2.10: Time API */ #define ERL_NIF_MAJOR_VERSION 2 -#define ERL_NIF_MINOR_VERSION 4 +#define ERL_NIF_MINOR_VERSION 10 -#include <stdlib.h> +/* + * The emulator will refuse to load a nif-lib with a major version + * lower than ERL_NIF_MIN_REQUIRED_MAJOR_VERSION_ON_LOAD. The load + * may however fail if user have not removed use of deprecated + * symbols. + * + * The ERL_NIF_MIN_REQUIRED_MAJOR_VERSION_ON_LOAD have to allow + * loading of nif-libs built at least two major OTP releases + * ago. + */ +#define ERL_NIF_MIN_REQUIRED_MAJOR_VERSION_ON_LOAD 2 -#ifdef SIZEOF_CHAR -# define SIZEOF_CHAR_SAVED__ SIZEOF_CHAR -# undef SIZEOF_CHAR -#endif -#ifdef SIZEOF_SHORT -# define SIZEOF_SHORT_SAVED__ SIZEOF_SHORT -# undef SIZEOF_SHORT -#endif -#ifdef SIZEOF_INT -# define SIZEOF_INT_SAVED__ SIZEOF_INT -# undef SIZEOF_INT -#endif -#ifdef SIZEOF_LONG -# define SIZEOF_LONG_SAVED__ SIZEOF_LONG -# undef SIZEOF_LONG -#endif -#ifdef SIZEOF_LONG_LONG -# define SIZEOF_LONG_LONG_SAVED__ SIZEOF_LONG_LONG -# undef SIZEOF_LONG_LONG -#endif -#ifdef HALFWORD_HEAP_EMULATOR -# define HALFWORD_HEAP_EMULATOR_SAVED__ HALFWORD_HEAP_EMULATOR -# undef HALFWORD_HEAP_EMULATOR -#endif -#include "erl_int_sizes_config.h" +#include <stdlib.h> #ifdef __cplusplus extern "C" { #endif -#if (defined(__WIN32__) || defined(_WIN32) || defined(_WIN32_)) -typedef unsigned __int64 ErlNifUInt64; -typedef __int64 ErlNifSInt64; -#elif SIZEOF_LONG == 8 -typedef unsigned long ErlNifUInt64; -typedef long ErlNifSInt64; -#elif SIZEOF_LONG_LONG == 8 -typedef unsigned long long ErlNifUInt64; -typedef long long ErlNifSInt64; -#else -#error No 64-bit integer type -#endif +typedef ErlNapiUInt64 ErlNifUInt64; +typedef ErlNapiSInt64 ErlNifSInt64; +typedef ErlNapiUInt ErlNifUInt; +typedef ErlNapiSInt ErlNifSInt; #ifdef HALFWORD_HEAP_EMULATOR # define ERL_NIF_VM_VARIANT "beam.halfword" typedef unsigned int ERL_NIF_TERM; #else # define ERL_NIF_VM_VARIANT "beam.vanilla" -# if SIZEOF_LONG == SIZEOF_VOID_P -typedef unsigned long ERL_NIF_TERM; -# elif SIZEOF_LONG_LONG == SIZEOF_VOID_P -typedef unsigned long long ERL_NIF_TERM; -# endif +typedef ErlNifUInt ERL_NIF_TERM; #endif +typedef ERL_NIF_TERM ERL_NIF_UINT; + +typedef ErlNifSInt64 ErlNifTime; + +#define ERL_NIF_TIME_ERROR ((ErlNifSInt64) ERTS_NAPI_TIME_ERROR__) + +typedef enum { + ERL_NIF_SEC = ERTS_NAPI_SEC__, + ERL_NIF_MSEC = ERTS_NAPI_MSEC__, + ERL_NIF_USEC = ERTS_NAPI_USEC__, + ERL_NIF_NSEC = ERTS_NAPI_NSEC__ +} ErlNifTimeUnit; + struct enif_environment_t; typedef struct enif_environment_t ErlNifEnv; @@ -103,8 +106,10 @@ typedef struct const char* name; unsigned arity; ERL_NIF_TERM (*fptr)(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]); + unsigned flags; }ErlNifFunc; + typedef struct enif_entry_t { int major; @@ -117,8 +122,11 @@ typedef struct enif_entry_t int (*upgrade)(ErlNifEnv*, void** priv_data, void** old_priv_data, ERL_NIF_TERM load_info); void (*unload) (ErlNifEnv*, void* priv_data); const char* vm_variant; + unsigned options; }ErlNifEntry; +/* Field bits for ErlNifEntry options */ +#define ERL_NIF_DIRTY_NIF_OPTION 1 typedef struct @@ -159,16 +167,52 @@ typedef int ErlNifTSDKey; typedef ErlDrvThreadOpts ErlNifThreadOpts; +#ifdef ERL_NIF_DIRTY_SCHEDULER_SUPPORT +typedef enum +{ + ERL_NIF_DIRTY_JOB_CPU_BOUND = ERL_DRV_DIRTY_JOB_CPU_BOUND, + ERL_NIF_DIRTY_JOB_IO_BOUND = ERL_DRV_DIRTY_JOB_IO_BOUND +}ErlNifDirtyTaskFlags; +#endif + +typedef struct /* All fields all internal and may change */ +{ + ERL_NIF_TERM map; + ERL_NIF_UINT size; + ERL_NIF_UINT idx; + union { + struct { + ERL_NIF_TERM *ks; + ERL_NIF_TERM *vs; + }flat; + struct { + struct ErtsDynamicWStack_* wstack; + ERL_NIF_TERM* kv; + }hash; + }u; + void* __spare__[2]; /* for future additions to be ABI compatible (same struct size) */ +} ErlNifMapIterator; + +typedef enum { + ERL_NIF_MAP_ITERATOR_FIRST = 1, + ERL_NIF_MAP_ITERATOR_LAST = 2, + + /* deprecated synonyms (undocumented in 17 and 18-rc) */ + ERL_NIF_MAP_ITERATOR_HEAD = ERL_NIF_MAP_ITERATOR_FIRST, + ERL_NIF_MAP_ITERATOR_TAIL = ERL_NIF_MAP_ITERATOR_LAST +} ErlNifMapIteratorEntry; + #if (defined(__WIN32__) || defined(_WIN32) || defined(_WIN32_)) # define ERL_NIF_API_FUNC_DECL(RET_TYPE, NAME, ARGS) RET_TYPE (*NAME) ARGS typedef struct { # include "erl_nif_api_funcs.h" + void* erts_alc_test; } TWinDynNifCallbacks; extern TWinDynNifCallbacks WinDynNifCallbacks; # undef ERL_NIF_API_FUNC_DECL #endif -#if (defined(__WIN32__) || defined(_WIN32) || defined(_WIN32_)) && !defined(STATIC_ERLANG_DRIVER) +#if (defined(__WIN32__) || defined(_WIN32) || defined(_WIN32_)) && !defined(STATIC_ERLANG_DRIVER) && !defined(STATIC_ERLANG_NIF) # define ERL_NIF_API_FUNC_MACRO(NAME) (WinDynNifCallbacks.NAME) # include "erl_nif_api_funcs.h" /* note that we have to keep ERL_NIF_API_FUNC_MACRO defined */ @@ -180,17 +224,29 @@ extern TWinDynNifCallbacks WinDynNifCallbacks; # undef ERL_NIF_API_FUNC_DECL #endif - #if (defined(__WIN32__) || defined(_WIN32) || defined(_WIN32_)) # define ERL_NIF_INIT_GLOB TWinDynNifCallbacks WinDynNifCallbacks; -# define ERL_NIF_INIT_DECL(MODNAME) __declspec(dllexport) ErlNifEntry* nif_init(TWinDynNifCallbacks* callbacks) +# ifdef STATIC_ERLANG_NIF +# define ERL_NIF_INIT_DECL(MODNAME) __declspec(dllexport) ErlNifEntry* MODNAME ## _nif_init(TWinDynNifCallbacks* callbacks) +# else +# define ERL_NIF_INIT_DECL(MODNAME) __declspec(dllexport) ErlNifEntry* nif_init(TWinDynNifCallbacks* callbacks) +# endif # define ERL_NIF_INIT_BODY memcpy(&WinDynNifCallbacks,callbacks,sizeof(TWinDynNifCallbacks)) #else # define ERL_NIF_INIT_GLOB # define ERL_NIF_INIT_BODY -# define ERL_NIF_INIT_DECL(MODNAME) ErlNifEntry* nif_init(void) +# ifdef STATIC_ERLANG_NIF +# define ERL_NIF_INIT_DECL(MODNAME) ErlNifEntry* MODNAME ## _nif_init(void) +# else +# define ERL_NIF_INIT_DECL(MODNAME) ErlNifEntry* nif_init(void) +# endif #endif +#ifdef ERL_NIF_DIRTY_SCHEDULER_SUPPORT +# define ERL_NIF_ENTRY_OPTIONS ERL_NIF_DIRTY_NIF_OPTION +#else +# define ERL_NIF_ENTRY_OPTIONS 0 +#endif #ifdef __cplusplus } @@ -216,7 +272,8 @@ ERL_NIF_INIT_DECL(NAME) \ sizeof(FUNCS) / sizeof(*FUNCS), \ FUNCS, \ LOAD, RELOAD, UPGRADE, UNLOAD, \ - ERL_NIF_VM_VARIANT \ + ERL_NIF_VM_VARIANT, \ + ERL_NIF_ENTRY_OPTIONS \ }; \ ERL_NIF_INIT_BODY; \ return &entry; \ diff --git a/erts/emulator/beam/erl_nif_api_funcs.h b/erts/emulator/beam/erl_nif_api_funcs.h index 2f841645e1..1448a508a2 100644 --- a/erts/emulator/beam/erl_nif_api_funcs.h +++ b/erts/emulator/beam/erl_nif_api_funcs.h @@ -1,18 +1,19 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2009-2013. All Rights Reserved. + * Copyright Ericsson AB 2009-2014. All Rights Reserved. * - * The contents of this file are subject to the Erlang Public License, - * Version 1.1, (the "License"); you may not use this file except in - * compliance with the License. You should have received a copy of the - * Erlang Public License along with this software. If not, it can be - * retrieved online at http://www.erlang.org/. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Software distributed under the License is distributed on an "AS IS" - * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See - * the License for the specific language governing rights and limitations - * under the License. + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. * * %CopyrightEnd% */ @@ -22,7 +23,7 @@ #endif /* -** WARNING: add new ERL_NIF_API_FUNC_DECL entries at the bottom of the list +** WARNING: Add new ERL_NIF_API_FUNC_DECL entries at the bottom of the list ** to keep compatibility on Windows!!! ** ** And don't forget to increase ERL_NIF_MINOR_VERSION in erl_nif.h @@ -141,11 +142,42 @@ ERL_NIF_API_FUNC_DECL(int,enif_is_number,(ErlNifEnv*, ERL_NIF_TERM term)); ERL_NIF_API_FUNC_DECL(void*,enif_dlopen,(const char* lib, void (*err_handler)(void*,const char*), void* err_arg)); ERL_NIF_API_FUNC_DECL(void*,enif_dlsym,(void* handle, const char* symbol, void (*err_handler)(void*,const char*), void* err_arg)); ERL_NIF_API_FUNC_DECL(int,enif_consume_timeslice,(ErlNifEnv*, int percent)); +ERL_NIF_API_FUNC_DECL(int, enif_is_map, (ErlNifEnv* env, ERL_NIF_TERM term)); +ERL_NIF_API_FUNC_DECL(int, enif_get_map_size, (ErlNifEnv* env, ERL_NIF_TERM term, size_t *size)); +ERL_NIF_API_FUNC_DECL(ERL_NIF_TERM, enif_make_new_map, (ErlNifEnv* env)); +ERL_NIF_API_FUNC_DECL(int, enif_make_map_put, (ErlNifEnv* env, ERL_NIF_TERM map_in, ERL_NIF_TERM key, ERL_NIF_TERM value, ERL_NIF_TERM* map_out)); +ERL_NIF_API_FUNC_DECL(int, enif_get_map_value, (ErlNifEnv* env, ERL_NIF_TERM map, ERL_NIF_TERM key, ERL_NIF_TERM* value)); +ERL_NIF_API_FUNC_DECL(int, enif_make_map_update, (ErlNifEnv* env, ERL_NIF_TERM map_in, ERL_NIF_TERM key, ERL_NIF_TERM value, ERL_NIF_TERM* map_out)); +ERL_NIF_API_FUNC_DECL(int, enif_make_map_remove, (ErlNifEnv* env, ERL_NIF_TERM map_in, ERL_NIF_TERM key, ERL_NIF_TERM* map_out)); +ERL_NIF_API_FUNC_DECL(int, enif_map_iterator_create, (ErlNifEnv *env, ERL_NIF_TERM map, ErlNifMapIterator *iter, ErlNifMapIteratorEntry entry)); +ERL_NIF_API_FUNC_DECL(void, enif_map_iterator_destroy, (ErlNifEnv *env, ErlNifMapIterator *iter)); +ERL_NIF_API_FUNC_DECL(int, enif_map_iterator_is_head, (ErlNifEnv *env, ErlNifMapIterator *iter)); +ERL_NIF_API_FUNC_DECL(int, enif_map_iterator_is_tail, (ErlNifEnv *env, ErlNifMapIterator *iter)); +ERL_NIF_API_FUNC_DECL(int, enif_map_iterator_next, (ErlNifEnv *env, ErlNifMapIterator *iter)); +ERL_NIF_API_FUNC_DECL(int, enif_map_iterator_prev, (ErlNifEnv *env, ErlNifMapIterator *iter)); +ERL_NIF_API_FUNC_DECL(int, enif_map_iterator_get_pair, (ErlNifEnv *env, ErlNifMapIterator *iter, ERL_NIF_TERM *key, ERL_NIF_TERM *value)); +ERL_NIF_API_FUNC_DECL(ERL_NIF_TERM,enif_schedule_nif,(ErlNifEnv*,const char*,int,ERL_NIF_TERM (*)(ErlNifEnv*,int,const ERL_NIF_TERM[]),int,const ERL_NIF_TERM[])); +ERL_NIF_API_FUNC_DECL(int, enif_has_pending_exception, (ErlNifEnv *env, ERL_NIF_TERM* reason)); +ERL_NIF_API_FUNC_DECL(ERL_NIF_TERM, enif_raise_exception, (ErlNifEnv *env, ERL_NIF_TERM reason)); +ERL_NIF_API_FUNC_DECL(int,enif_getenv,(const char* key, char* value, size_t* value_size)); +ERL_NIF_API_FUNC_DECL(ErlNifTime, enif_monotonic_time, (ErlNifTimeUnit)); +ERL_NIF_API_FUNC_DECL(ErlNifTime, enif_time_offset, (ErlNifTimeUnit)); +ERL_NIF_API_FUNC_DECL(ErlNifTime, enif_convert_time_unit, (ErlNifTime, ErlNifTimeUnit, ErlNifTimeUnit)); /* -** Add new entries here to keep compatibility on Windows!!! +** ADD NEW ENTRIES HERE (before this comment) !!! */ + + +/* + * Conditional EXPERIMENTAL stuff always last. + * Must be moved up and made unconditional to support binary backward + * compatibility on Windows. + */ +#ifdef ERL_NIF_DIRTY_SCHEDULER_SUPPORT +ERL_NIF_API_FUNC_DECL(int,enif_is_on_dirty_scheduler,(ErlNifEnv*)); #endif +#endif /* ERL_NIF_API_FUNC_DECL */ /* ** Please keep the ERL_NIF_API_FUNC_MACRO list below in the same order @@ -259,18 +291,47 @@ ERL_NIF_API_FUNC_DECL(int,enif_consume_timeslice,(ErlNifEnv*, int percent)); # define enif_make_int64 ERL_NIF_API_FUNC_MACRO(enif_make_int64) # define enif_make_uint64 ERL_NIF_API_FUNC_MACRO(enif_make_uint64) #endif - # define enif_is_exception ERL_NIF_API_FUNC_MACRO(enif_is_exception) # define enif_make_reverse_list ERL_NIF_API_FUNC_MACRO(enif_make_reverse_list) # define enif_is_number ERL_NIF_API_FUNC_MACRO(enif_is_number) # define enif_dlopen ERL_NIF_API_FUNC_MACRO(enif_dlopen) # define enif_dlsym ERL_NIF_API_FUNC_MACRO(enif_dlsym) # define enif_consume_timeslice ERL_NIF_API_FUNC_MACRO(enif_consume_timeslice) +# define enif_is_map ERL_NIF_API_FUNC_MACRO(enif_is_map) +# define enif_get_map_size ERL_NIF_API_FUNC_MACRO(enif_get_map_size) +# define enif_make_new_map ERL_NIF_API_FUNC_MACRO(enif_make_new_map) +# define enif_make_map_put ERL_NIF_API_FUNC_MACRO(enif_make_map_put) +# define enif_get_map_value ERL_NIF_API_FUNC_MACRO(enif_get_map_value) +# define enif_make_map_update ERL_NIF_API_FUNC_MACRO(enif_make_map_update) +# define enif_make_map_remove ERL_NIF_API_FUNC_MACRO(enif_make_map_remove) +# define enif_map_iterator_create ERL_NIF_API_FUNC_MACRO(enif_map_iterator_create) +# define enif_map_iterator_destroy ERL_NIF_API_FUNC_MACRO(enif_map_iterator_destroy) +# define enif_map_iterator_is_head ERL_NIF_API_FUNC_MACRO(enif_map_iterator_is_head) +# define enif_map_iterator_is_tail ERL_NIF_API_FUNC_MACRO(enif_map_iterator_is_tail) +# define enif_map_iterator_next ERL_NIF_API_FUNC_MACRO(enif_map_iterator_next) +# define enif_map_iterator_prev ERL_NIF_API_FUNC_MACRO(enif_map_iterator_prev) +# define enif_map_iterator_get_pair ERL_NIF_API_FUNC_MACRO(enif_map_iterator_get_pair) +# define enif_schedule_nif ERL_NIF_API_FUNC_MACRO(enif_schedule_nif) +# define enif_has_pending_exception ERL_NIF_API_FUNC_MACRO(enif_has_pending_exception) +# define enif_raise_exception ERL_NIF_API_FUNC_MACRO(enif_raise_exception) +# define enif_getenv ERL_NIF_API_FUNC_MACRO(enif_getenv) +# define enif_monotonic_time ERL_NIF_API_FUNC_MACRO(enif_monotonic_time) +# define enif_time_offset ERL_NIF_API_FUNC_MACRO(enif_time_offset) +# define enif_convert_time_unit ERL_NIF_API_FUNC_MACRO(enif_convert_time_unit) /* -** Add new entries here +** ADD NEW ENTRIES HERE (before this comment) */ + +/* + * Conditional EXPERIMENTAL stuff always last + * Must be moved up and made unconditional to support binary backward + * compatibility on Windows. + */ +#ifdef ERL_NIF_DIRTY_SCHEDULER_SUPPORT +# define enif_is_on_dirty_scheduler ERL_NIF_API_FUNC_MACRO(enif_is_on_dirty_scheduler) #endif +#endif /* ERL_NIF_API_FUNC_MACRO */ #if defined(__GNUC__) && !(defined(__WIN32__) || defined(_WIN32) || defined(_WIN32_)) @@ -490,7 +551,7 @@ static ERL_NIF_INLINE ERL_NIF_TERM enif_make_list9(ErlNifEnv* env, #ifndef enif_make_pid -# define enif_make_pid(ENV, PID) ((const ERL_NIF_TERM)((PID)->pid)) +# define enif_make_pid(ENV, PID) ((void)(ENV),(const ERL_NIF_TERM)((PID)->pid)) #if SIZEOF_LONG == 8 # define enif_get_int64 enif_get_long diff --git a/erts/emulator/beam/erl_node_container_utils.h b/erts/emulator/beam/erl_node_container_utils.h index 0f93a3a9f0..211b1a0090 100644 --- a/erts/emulator/beam/erl_node_container_utils.h +++ b/erts/emulator/beam/erl_node_container_utils.h @@ -3,16 +3,17 @@ * * Copyright Ericsson AB 2001-2013. All Rights Reserved. * - * The contents of this file are subject to the Erlang Public License, - * Version 1.1, (the "License"); you may not use this file except in - * compliance with the License. You should have received a copy of the - * Erlang Public License along with this software. If not, it can be - * retrieved online at http://www.erlang.org/. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Software distributed under the License is distributed on an "AS IS" - * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See - * the License for the specific language governing rights and limitations - * under the License. + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. * * %CopyrightEnd% */ @@ -106,7 +107,7 @@ #define dist_entry_channel_no(x) \ ((x) == erts_this_dist_entry \ ? ((Uint) 0) \ - : (ASSERT_EXPR(is_atom((x)->sysname)), \ + : (ASSERT(is_atom((x)->sysname)), \ (Uint) atom_val((x)->sysname))) #define internal_channel_no(x) ((Uint) ERST_INTERNAL_CHANNEL_NO) #define external_channel_no(x) \ @@ -122,10 +123,10 @@ extern ErtsPTab erts_proc; (D), \ _TAG_IMMED1_PID) -#define internal_pid_index(PID) (ASSERT_EXPR(is_internal_pid((PID))), \ +#define internal_pid_index(PID) (ASSERT(is_internal_pid((PID))), \ erts_ptab_id2pix(&erts_proc, (PID))) -#define internal_pid_data(PID) (ASSERT_EXPR(is_internal_pid((PID))), \ +#define internal_pid_data(PID) (ASSERT(is_internal_pid((PID))), \ erts_ptab_id2data(&erts_proc, (PID))) #define internal_pid_number(x) _GET_PID_NUM(internal_pid_data((x))) @@ -193,10 +194,10 @@ extern ErtsPTab erts_port; (D), \ _TAG_IMMED1_PORT) -#define internal_port_index(PRT) (ASSERT_EXPR(is_internal_port((PRT))), \ +#define internal_port_index(PRT) (ASSERT(is_internal_port((PRT))), \ erts_ptab_id2pix(&erts_port, (PRT))) -#define internal_port_data(PRT) (ASSERT_EXPR(is_internal_port((PRT))), \ +#define internal_port_data(PRT) (ASSERT(is_internal_port((PRT))), \ erts_ptab_id2data(&erts_port, (PRT))) #define internal_port_number(x) _GET_PORT_NUM(internal_port_data((x))) diff --git a/erts/emulator/beam/erl_node_tables.c b/erts/emulator/beam/erl_node_tables.c index ebfba065d1..e69bd49c42 100644 --- a/erts/emulator/beam/erl_node_tables.c +++ b/erts/emulator/beam/erl_node_tables.c @@ -1,18 +1,19 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2001-2012. All Rights Reserved. + * Copyright Ericsson AB 2001-2013. All Rights Reserved. * - * The contents of this file are subject to the Erlang Public License, - * Version 1.1, (the "License"); you may not use this file except in - * compliance with the License. You should have received a copy of the - * Erlang Public License along with this software. If not, it can be - * retrieved online at http://www.erlang.org/. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Software distributed under the License is distributed on an "AS IS" - * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See - * the License for the specific language governing rights and limitations - * under the License. + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. * * %CopyrightEnd% */ @@ -36,21 +37,24 @@ erts_smp_rwmtx_t erts_node_table_rwmtx; DistEntry *erts_hidden_dist_entries; DistEntry *erts_visible_dist_entries; -DistEntry *erts_not_connected_dist_entries; +DistEntry *erts_not_connected_dist_entries; /* including erts_this_dist_entry */ Sint erts_no_of_hidden_dist_entries; Sint erts_no_of_visible_dist_entries; -Sint erts_no_of_not_connected_dist_entries; +Sint erts_no_of_not_connected_dist_entries; /* including erts_this_dist_entry */ DistEntry *erts_this_dist_entry; ErlNode *erts_this_node; char erts_this_node_sysname_BUFFER[256], *erts_this_node_sysname = "uninitialized yet"; -static Uint node_entries; -static Uint dist_entries; +static Uint node_entries = 0; +static Uint dist_entries = 0; static int references_atoms_need_init = 1; +static ErtsMonotonicTime orig_node_tab_delete_delay; +static ErtsMonotonicTime node_tab_delete_delay; + /* -- The distribution table ---------------------------------------------- */ #ifdef DEBUG @@ -87,9 +91,6 @@ dist_table_alloc(void *dep_tmpl) erts_smp_rwmtx_opt_t rwmtx_opt = ERTS_SMP_RWMTX_OPT_DEFAULT_INITER; rwmtx_opt.type = ERTS_SMP_RWMTX_TYPE_FREQUENT_READ; - if(((DistEntry *) dep_tmpl) == erts_this_dist_entry) - return dep_tmpl; - sysname = ((DistEntry *) dep_tmpl)->sysname; chnl_nr = make_small((Uint) atom_val(sysname)); dep = (DistEntry *) erts_alloc(ERTS_ALC_T_DIST_ENTRY, sizeof(DistEntry)); @@ -128,7 +129,9 @@ dist_table_alloc(void *dep_tmpl) /* Link in */ - /* All new dist entries are "not connected" */ + /* All new dist entries are "not connected". + * erts_this_dist_entry is also always included among "not connected" + */ dep->next = erts_not_connected_dist_entries; if(erts_not_connected_dist_entries) { ASSERT(erts_not_connected_dist_entries->prev == NULL); @@ -145,9 +148,6 @@ dist_table_free(void *vdep) { DistEntry *dep = (DistEntry *) vdep; - if(dep == erts_this_dist_entry) - return; - ASSERT(is_nil(dep->cid)); ASSERT(dep->nlinks == NULL); ASSERT(dep->node_links == NULL); @@ -182,7 +182,7 @@ dist_table_free(void *vdep) #endif erts_free(ERTS_ALC_T_DIST_ENTRY, (void *) dep); - ASSERT(dist_entries > 1); + ASSERT(dist_entries > 0); dist_entries--; } @@ -290,21 +290,46 @@ DistEntry *erts_find_dist_entry(Eterm sysname) return res; } -void erts_delete_dist_entry(DistEntry *dep) +static void try_delete_dist_entry(void *vdep) +{ + DistEntry *dep = (DistEntry *) vdep; + erts_aint_t refc; + + erts_smp_rwmtx_rwlock(&erts_dist_table_rwmtx); + /* + * Another thread might have looked up this dist entry after + * we decided to delete it (refc became zero). If so, the other + * thread incremented refc twice. Once for the new reference + * and once for this thread. + * + * If refc reach -1, no one has used the entry since we + * set up the timer. Delete the entry. + * + * If refc reach 0, the entry is currently not in use + * but has been used since we set up the timer. Set up a + * new timer. + * + * If refc > 0, the entry is in use. Keep the entry. + */ + refc = erts_refc_dectest(&dep->refc, -1); + if (refc == -1) + (void) hash_erase(&erts_dist_table, (void *) dep); + erts_smp_rwmtx_rwunlock(&erts_dist_table_rwmtx); + + if (refc == 0) + erts_schedule_delete_dist_entry(dep); +} + +void erts_schedule_delete_dist_entry(DistEntry *dep) { ASSERT(dep != erts_this_dist_entry); - if(dep != erts_this_dist_entry) { - erts_smp_rwmtx_rwlock(&erts_dist_table_rwmtx); - /* - * Another thread might have looked up this dist entry after - * we decided to delete it (refc became zero). If so, the other - * thread incremented refc twice. Once for the new reference - * and once for this thread. Therefore, delete dist entry if - * refc is 0 or -1 after a decrement. - */ - if (erts_refc_dectest(&dep->refc, -1) <= 0) - (void) hash_erase(&erts_dist_table, (void *) dep); - erts_smp_rwmtx_rwunlock(&erts_dist_table_rwmtx); + if (dep != erts_this_dist_entry) { + if (node_tab_delete_delay == 0) + try_delete_dist_entry((void *) dep); + else if (node_tab_delete_delay > 0) + erts_start_timer_callback(node_tab_delete_delay, + try_delete_dist_entry, + (void *) dep); } } @@ -340,8 +365,7 @@ erts_dist_table_size(void) ASSERT(dist_entries == (erts_no_of_visible_dist_entries + erts_no_of_hidden_dist_entries - + erts_no_of_not_connected_dist_entries - + 1 /* erts_this_dist_entry */)); + + erts_no_of_not_connected_dist_entries)); #endif res = (hash_table_sz(&erts_dist_table) @@ -514,9 +538,6 @@ node_table_alloc(void *venp_tmpl) { ErlNode *enp; - if(((ErlNode *) venp_tmpl) == erts_this_node) - return venp_tmpl; - enp = (ErlNode *) erts_alloc(ERTS_ALC_T_NODE_ENTRY, sizeof(ErlNode)); node_entries++; @@ -534,8 +555,7 @@ node_table_free(void *venp) { ErlNode *enp = (ErlNode *) venp; - if(enp == erts_this_node) - return; + ERTS_SMP_LC_ASSERT(enp != erts_this_node || erts_thr_progress_is_blocking()); erts_deref_dist_entry(enp->dist_entry); #ifdef DEBUG @@ -543,7 +563,7 @@ node_table_free(void *venp) #endif erts_free(ERTS_ALC_T_NODE_ENTRY, venp); - ASSERT(node_entries > 1); + ASSERT(node_entries > 0); node_entries--; } @@ -556,14 +576,14 @@ erts_node_table_size(void) #endif int lock = !ERTS_IS_CRASH_DUMPING; if (lock) - erts_smp_rwmtx_rwlock(&erts_node_table_rwmtx); + erts_smp_rwmtx_rlock(&erts_node_table_rwmtx); #ifdef DEBUG hash_get_info(&hi, &erts_node_table); ASSERT(node_entries == hi.objs); #endif res = hash_table_sz(&erts_node_table) + node_entries*sizeof(ErlNode); if (lock) - erts_smp_rwmtx_rwunlock(&erts_node_table_rwmtx); + erts_smp_rwmtx_runlock(&erts_node_table_rwmtx); return res; } @@ -572,10 +592,10 @@ erts_node_table_info(int to, void *to_arg) { int lock = !ERTS_IS_CRASH_DUMPING; if (lock) - erts_smp_rwmtx_rwlock(&erts_node_table_rwmtx); + erts_smp_rwmtx_rlock(&erts_node_table_rwmtx); hash_info(to, to_arg, &erts_node_table); if (lock) - erts_smp_rwmtx_rwunlock(&erts_node_table_rwmtx); + erts_smp_rwmtx_runlock(&erts_node_table_rwmtx); } @@ -609,21 +629,46 @@ ErlNode *erts_find_or_insert_node(Eterm sysname, Uint creation) return res; } -void erts_delete_node(ErlNode *enp) +static void try_delete_node(void *venp) +{ + ErlNode *enp = (ErlNode *) venp; + erts_aint_t refc; + + erts_smp_rwmtx_rwlock(&erts_node_table_rwmtx); + /* + * Another thread might have looked up this node after we + * decided to delete it (refc became zero). If so, the other + * thread incremented refc twice. Once for the new reference + * and once for this thread. + * + * If refc reach -1, no one has used the entry since we + * set up the timer. Delete the entry. + * + * If refc reach 0, the entry is currently not in use + * but has been used since we set up the timer. Set up a + * new timer. + * + * If refc > 0, the entry is in use. Keep the entry. + */ + refc = erts_refc_dectest(&enp->refc, -1); + if (refc == -1) + (void) hash_erase(&erts_node_table, (void *) enp); + erts_smp_rwmtx_rwunlock(&erts_node_table_rwmtx); + + if (refc == 0) + erts_schedule_delete_node(enp); +} + +void erts_schedule_delete_node(ErlNode *enp) { ASSERT(enp != erts_this_node); - if(enp != erts_this_node) { - erts_smp_rwmtx_rwlock(&erts_node_table_rwmtx); - /* - * Another thread might have looked up this node after we - * decided to delete it (refc became zero). If so, the other - * thread incremented refc twice. Once for the new reference - * and once for this thread. Therefore, delete node if refc - * is 0 or -1 after a decrement. - */ - if (erts_refc_dectest(&enp->refc, -1) <= 0) - (void) hash_erase(&erts_node_table, (void *) enp); - erts_smp_rwmtx_rwunlock(&erts_node_table_rwmtx); + if (enp != erts_this_node) { + if (node_tab_delete_delay == 0) + try_delete_node((void *) enp); + else if (node_tab_delete_delay > 0) + erts_start_timer_callback(node_tab_delete_delay, + try_delete_node, + (void *) enp); } } @@ -651,7 +696,7 @@ static void print_node(void *venp, void *vpndp) erts_print(pndp->to, pndp->to_arg, " %d", enp->creation); #ifdef DEBUG erts_print(pndp->to, pndp->to_arg, " (refc=%ld)", - erts_refc_read(&enp->refc, 1)); + erts_refc_read(&enp->refc, 0)); #endif pndp->no_sysname++; } @@ -674,13 +719,13 @@ void erts_print_node_info(int to, pnd.no_total = 0; if (lock) - erts_smp_rwmtx_rwlock(&erts_node_table_rwmtx); + erts_smp_rwmtx_rlock(&erts_node_table_rwmtx); hash_foreach(&erts_node_table, print_node, (void *) &pnd); if (pnd.no_sysname != 0) { erts_print(to, to_arg, "\n"); } if (lock) - erts_smp_rwmtx_rwunlock(&erts_node_table_rwmtx); + erts_smp_rwmtx_runlock(&erts_node_table_rwmtx); if(no_sysname) *no_sysname = pnd.no_sysname; @@ -693,45 +738,67 @@ void erts_print_node_info(int to, void erts_set_this_node(Eterm sysname, Uint creation) { - erts_smp_rwmtx_rwlock(&erts_node_table_rwmtx); - erts_smp_rwmtx_rwlock(&erts_dist_table_rwmtx); + ERTS_SMP_LC_ASSERT(erts_thr_progress_is_blocking()); + ASSERT(erts_refc_read(&erts_this_dist_entry->refc, 2)); - (void) hash_erase(&erts_dist_table, (void *) erts_this_dist_entry); - erts_this_dist_entry->sysname = sysname; - erts_this_dist_entry->creation = creation; - (void) hash_put(&erts_dist_table, (void *) erts_this_dist_entry); + if (erts_refc_dectest(&erts_this_node->refc, 0) == 0) + try_delete_node(erts_this_node); - (void) hash_erase(&erts_node_table, (void *) erts_this_node); - erts_this_node->sysname = sysname; - erts_this_node->creation = creation; - erts_this_node_sysname = erts_this_node_sysname_BUFFER; - erts_snprintf(erts_this_node_sysname, sizeof(erts_this_node_sysname), - "%T", sysname); - (void) hash_put(&erts_node_table, (void *) erts_this_node); + if (erts_refc_dectest(&erts_this_dist_entry->refc, 0) == 0) + try_delete_dist_entry(erts_this_dist_entry); - erts_smp_rwmtx_rwunlock(&erts_dist_table_rwmtx); - erts_smp_rwmtx_rwunlock(&erts_node_table_rwmtx); + erts_this_node = NULL; /* to make sure refc is bumped for this node */ + erts_this_node = erts_find_or_insert_node(sysname, creation); + erts_this_dist_entry = erts_this_node->dist_entry; + + erts_refc_inc(&erts_this_dist_entry->refc, 2); + erts_this_node_sysname = erts_this_node_sysname_BUFFER; + erts_snprintf(erts_this_node_sysname, sizeof(erts_this_node_sysname_BUFFER), + "%T", sysname); +} + +Uint +erts_delayed_node_table_gc(void) +{ + if (node_tab_delete_delay < 0) + return (Uint) ERTS_NODE_TAB_DELAY_GC_INFINITY; + if (node_tab_delete_delay == 0) + return (Uint) 0; + return (Uint) ((node_tab_delete_delay-1)/1000 + 1); } -void erts_init_node_tables(void) +void erts_init_node_tables(int dd_sec) { erts_smp_rwmtx_opt_t rwmtx_opt = ERTS_SMP_RWMTX_OPT_DEFAULT_INITER; HashFunctions f; + ErlNode node_tmpl; + + if (dd_sec == ERTS_NODE_TAB_DELAY_GC_INFINITY) + node_tab_delete_delay = (ErtsMonotonicTime) -1; + else + node_tab_delete_delay = ((ErtsMonotonicTime) dd_sec)*1000; + + orig_node_tab_delete_delay = node_tab_delete_delay; rwmtx_opt.type = ERTS_SMP_RWMTX_TYPE_FREQUENT_READ; rwmtx_opt.lived = ERTS_SMP_RWMTX_LONG_LIVED; + erts_smp_rwmtx_init_opt(&erts_node_table_rwmtx, &rwmtx_opt, "node_table"); + erts_smp_rwmtx_init_opt(&erts_dist_table_rwmtx, &rwmtx_opt, "dist_table"); + f.hash = (H_FUN) dist_table_hash; f.cmp = (HCMP_FUN) dist_table_cmp; f.alloc = (HALLOC_FUN) dist_table_alloc; f.free = (HFREE_FUN) dist_table_free; - - erts_this_dist_entry = erts_alloc(ERTS_ALC_T_DIST_ENTRY, sizeof(DistEntry)); - dist_entries = 1; - hash_init(ERTS_ALC_T_DIST_TABLE, &erts_dist_table, "dist_table", 11, f); + f.hash = (H_FUN) node_table_hash; + f.cmp = (HCMP_FUN) node_table_cmp; + f.alloc = (HALLOC_FUN) node_table_alloc; + f.free = (HFREE_FUN) node_table_free; + hash_init(ERTS_ALC_T_NODE_TABLE, &erts_node_table, "node_table", 11, f); + erts_hidden_dist_entries = NULL; erts_visible_dist_entries = NULL; erts_not_connected_dist_entries = NULL; @@ -739,69 +806,23 @@ void erts_init_node_tables(void) erts_no_of_visible_dist_entries = 0; erts_no_of_not_connected_dist_entries = 0; - erts_this_dist_entry->next = NULL; - erts_this_dist_entry->prev = NULL; - erts_refc_init(&erts_this_dist_entry->refc, 1); /* erts_this_node */ - - erts_smp_rwmtx_init_opt_x(&erts_this_dist_entry->rwmtx, - &rwmtx_opt, - "dist_entry", - make_small(ERST_INTERNAL_CHANNEL_NO)); - erts_this_dist_entry->sysname = am_Noname; - erts_this_dist_entry->cid = NIL; - erts_this_dist_entry->connection_id = 0; - erts_this_dist_entry->status = 0; - erts_this_dist_entry->flags = 0; - erts_this_dist_entry->version = 0; - - erts_smp_mtx_init_x(&erts_this_dist_entry->lnk_mtx, - "dist_entry_links", - make_small(ERST_INTERNAL_CHANNEL_NO)); - erts_this_dist_entry->node_links = NULL; - erts_this_dist_entry->nlinks = NULL; - erts_this_dist_entry->monitors = NULL; - - erts_smp_mtx_init_x(&erts_this_dist_entry->qlock, - "dist_entry_out_queue", - make_small(ERST_INTERNAL_CHANNEL_NO)); - erts_this_dist_entry->qflgs = 0; - erts_this_dist_entry->qsize = 0; - erts_this_dist_entry->out_queue.first = NULL; - erts_this_dist_entry->out_queue.last = NULL; - erts_this_dist_entry->suspended = NULL; - - erts_this_dist_entry->finalized_out_queue.first = NULL; - erts_this_dist_entry->finalized_out_queue.last = NULL; - erts_smp_atomic_init_nob(&erts_this_dist_entry->dist_cmd_scheduled, 0); - erts_port_task_handle_init(&erts_this_dist_entry->dist_cmd); - erts_this_dist_entry->send = NULL; - erts_this_dist_entry->cache = NULL; - - (void) hash_put(&erts_dist_table, (void *) erts_this_dist_entry); - - f.hash = (H_FUN) node_table_hash; - f.cmp = (HCMP_FUN) node_table_cmp; - f.alloc = (HALLOC_FUN) node_table_alloc; - f.free = (HFREE_FUN) node_table_free; + node_tmpl.sysname = am_Noname; + node_tmpl.creation = 0; + erts_this_node = hash_put(&erts_node_table, &node_tmpl); + /* +1 for erts_this_node */ + erts_refc_init(&erts_this_node->refc, 1); - hash_init(ERTS_ALC_T_NODE_TABLE, &erts_node_table, "node_table", 11, f); + ASSERT(erts_this_node->dist_entry != NULL); + erts_this_dist_entry = erts_this_node->dist_entry; + /* +1 for erts_this_dist_entry */ + /* +1 for erts_this_node->dist_entry */ + erts_refc_init(&erts_this_dist_entry->refc, 2); - erts_this_node = erts_alloc(ERTS_ALC_T_NODE_ENTRY, sizeof(ErlNode)); - node_entries = 1; - erts_refc_init(&erts_this_node->refc, 1); /* The system itself */ - erts_this_node->sysname = am_Noname; - erts_this_node->creation = 0; - erts_this_node->dist_entry = erts_this_dist_entry; erts_this_node_sysname = erts_this_node_sysname_BUFFER; - erts_snprintf(erts_this_node_sysname, sizeof(erts_this_node_sysname), + erts_snprintf(erts_this_node_sysname, sizeof(erts_this_node_sysname_BUFFER), "%T", erts_this_node->sysname); - (void) hash_put(&erts_node_table, (void *) erts_this_node); - - erts_smp_rwmtx_init_opt(&erts_node_table_rwmtx, &rwmtx_opt, "node_table"); - erts_smp_rwmtx_init_opt(&erts_dist_table_rwmtx, &rwmtx_opt, "dist_table"); - references_atoms_need_init = 1; } @@ -847,6 +868,7 @@ static Eterm AM_dist_references; static Eterm AM_node_references; static Eterm AM_system; static Eterm AM_timer; +static Eterm AM_delayed_delete_timer; static void setup_reference_table(void); static Eterm reference_table_term(Uint **hpp, Uint *szp); @@ -881,8 +903,10 @@ typedef struct dist_referrer_ { int heap_ref; int node_ref; int ctrl_ref; + int system_ref; Eterm id; Uint creation; + Uint id_heap[ID_HEAP_SIZE]; } DistReferrer; typedef struct { @@ -931,6 +955,7 @@ erts_get_node_and_dist_references(struct process *proc) INIT_AM(node_references); INIT_AM(timer); INIT_AM(system); + INIT_AM(delayed_delete_timer); references_atoms_need_init = 0; } @@ -988,17 +1013,25 @@ insert_dist_referrer(ReferredDist *referred_dist, sizeof(DistReferrer)); drp->next = referred_dist->referrers; referred_dist->referrers = drp; - drp->id = id; + if(IS_CONST(id)) + drp->id = id; + else { + Uint *hp = &drp->id_heap[0]; + ASSERT(is_tuple(id)); + drp->id = copy_struct(id, size_object(id), &hp, NULL); + } drp->creation = creation; drp->heap_ref = 0; drp->node_ref = 0; drp->ctrl_ref = 0; + drp->system_ref = 0; } switch (type) { case NODE_REF: drp->node_ref++; break; case CTRL_REF: drp->ctrl_ref++; break; case HEAP_REF: drp->heap_ref++; break; + case SYSTEM_REF: drp->system_ref++; break; default: ASSERT(0); } } @@ -1017,7 +1050,7 @@ insert_dist_entry(DistEntry *dist, int type, Eterm id, Uint creation) } if(!rdp) - erl_exit(1, + erts_exit(ERTS_ERROR_EXIT, "Reference to non-existing distribution table entry found!\n"); insert_dist_referrer(rdp, type, id, creation); @@ -1078,7 +1111,7 @@ insert_node(ErlNode *node, int type, Eterm id) } if (!rnp) - erl_exit(1, "Reference to non-existing node table entry found!\n"); + erts_exit(ERTS_ERROR_EXIT, "Reference to non-existing node table entry found!\n"); insert_node_referrer(rnp, type, id); } @@ -1261,6 +1294,33 @@ insert_sys_msg(Eterm from, Eterm to, Eterm msg, ErlHeapFragment *bp) #endif static void +insert_delayed_delete_node(void *state, + ErtsMonotonicTime timeout_pos, + void *vnp) +{ + DeclareTmpHeapNoproc(heap,3); + UseTmpHeapNoproc(3); + insert_node((ErlNode *) vnp, + SYSTEM_REF, + TUPLE2(&heap[0], AM_system, AM_delayed_delete_timer)); + UnUseTmpHeapNoproc(3); +} + +static void +insert_delayed_delete_dist_entry(void *state, + ErtsMonotonicTime timeout_pos, + void *vdep) +{ + DeclareTmpHeapNoproc(heap,3); + UseTmpHeapNoproc(3); + insert_dist_entry((DistEntry *) vdep, + SYSTEM_REF, + TUPLE2(&heap[0], AM_system, AM_delayed_delete_timer), + 0); + UnUseTmpHeapNoproc(3); +} + +static void setup_reference_table(void) { ErlHeapFragment *hfp; @@ -1288,11 +1348,22 @@ setup_reference_table(void) /* Go through the hole system, and build a table of all references to ErlNode and DistEntry structures */ + erts_debug_callback_timer_foreach(try_delete_node, + insert_delayed_delete_node, + NULL); + erts_debug_callback_timer_foreach(try_delete_dist_entry, + insert_delayed_delete_dist_entry, + NULL); + UseTmpHeapNoproc(3); insert_node(erts_this_node, SYSTEM_REF, TUPLE2(&heap[0], AM_system, am_undefined)); + insert_dist_entry(erts_this_dist_entry, + SYSTEM_REF, + TUPLE2(&heap[0], AM_system, am_undefined), + erts_this_node->creation); UnUseTmpHeapNoproc(3); max = erts_ptab_max(&erts_proc); @@ -1355,12 +1426,6 @@ setup_reference_table(void) insert_links(ERTS_P_LINKS(proc), proc->common.id); if (ERTS_P_MONITORS(proc)) insert_monitors(ERTS_P_MONITORS(proc), proc->common.id); - /* Insert controller */ - { - DistEntry *dep = ERTS_PROC_GET_DIST_ENTRY(proc); - if (dep) - insert_dist_entry(dep, CTRL_REF, proc->common.id, 0); - } } } @@ -1371,6 +1436,7 @@ setup_reference_table(void) /* Insert all ports */ max = erts_ptab_max(&erts_port); for (i = 0; i < max; i++) { + ErlOffHeap *ohp; erts_aint32_t state; Port *prt; @@ -1389,8 +1455,9 @@ setup_reference_table(void) if (ERTS_P_MONITORS(prt)) insert_monitors(ERTS_P_MONITORS(prt), prt->common.id); /* Insert port data */ - for(hfp = prt->bp; hfp; hfp = hfp->next) - insert_offheap(&(hfp->off_heap), HEAP_REF, prt->common.id); + ohp = erts_port_data_offheap(prt); + if (ohp) + insert_offheap(ohp, HEAP_REF, prt->common.id); /* Insert controller */ if (prt->dist_entry) insert_dist_entry(prt->dist_entry, @@ -1467,7 +1534,7 @@ setup_reference_table(void) erts_db_foreach_table(insert_ets_table, NULL); /* Insert all bif timers */ - erts_bif_timer_foreach(insert_bif_timer, NULL); + erts_debug_bif_timer_foreach(insert_bif_timer, NULL); /* Insert node table (references to dist) */ hash_foreach(&erts_node_table, insert_erl_node, NULL); @@ -1599,7 +1666,7 @@ reference_table_term(Uint **hpp, Uint *szp) tup = MK_2TUP(referred_nodes[i].node->sysname, MK_UINT(referred_nodes[i].node->creation)); - tup = MK_3TUP(tup, MK_UINT(erts_refc_read(&referred_nodes[i].node->refc, 1)), nril); + tup = MK_3TUP(tup, MK_UINT(erts_refc_read(&referred_nodes[i].node->refc, 0)), nril); nl = MK_CONS(tup, nl); } @@ -1622,6 +1689,10 @@ reference_table_term(Uint **hpp, Uint *szp) tup = MK_2TUP(AM_heap, MK_UINT(drp->heap_ref)); drl = MK_CONS(tup, drl); } + if(drp->system_ref) { + tup = MK_2TUP(AM_system, MK_UINT(drp->system_ref)); + drl = MK_CONS(tup, drl); + } if (is_internal_pid(drp->id)) { ASSERT(!drp->node_ref); @@ -1631,6 +1702,14 @@ reference_table_term(Uint **hpp, Uint *szp) ASSERT(drp->ctrl_ref && !drp->node_ref); tup = MK_2TUP(AM_port, drp->id); } + else if (is_tuple(drp->id)) { + Eterm *t; + ASSERT(drp->system_ref && !drp->node_ref + && !drp->ctrl_ref && !drp->heap_ref); + t = tuple_val(drp->id); + ASSERT(2 == arityval(t[0])); + tup = MK_2TUP(t[1], t[2]); + } else { ASSERT(!drp->ctrl_ref && drp->node_ref); ASSERT(is_atom(drp->id)); @@ -1648,7 +1727,7 @@ reference_table_term(Uint **hpp, Uint *szp) /* DistList = [{Dist, Refc, ReferenceIdList}] */ tup = MK_3TUP(referred_dists[i].dist->sysname, - MK_UINT(erts_refc_read(&referred_dists[i].dist->refc, 1)), + MK_UINT(erts_refc_read(&referred_dists[i].dist->refc, 0)), dril); dl = MK_CONS(tup, dl); } @@ -1703,3 +1782,15 @@ delete_reference_table(void) } } +void +erts_debug_test_node_tab_delayed_delete(Sint64 millisecs) +{ + erts_smp_thr_progress_block(); + + if (millisecs < 0) + node_tab_delete_delay = orig_node_tab_delete_delay; + else + node_tab_delete_delay = millisecs; + + erts_smp_thr_progress_unblock(); +} diff --git a/erts/emulator/beam/erl_node_tables.h b/erts/emulator/beam/erl_node_tables.h index af60071ea5..694ac84232 100644 --- a/erts/emulator/beam/erl_node_tables.h +++ b/erts/emulator/beam/erl_node_tables.h @@ -3,16 +3,17 @@ * * Copyright Ericsson AB 2001-2012. All Rights Reserved. * - * The contents of this file are subject to the Erlang Public License, - * Version 1.1, (the "License"); you may not use this file except in - * compliance with the License. You should have received a copy of the - * Erlang Public License along with this software. If not, it can be - * retrieved online at http://www.erlang.org/. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Software distributed under the License is distributed on an "AS IS" - * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See - * the License for the specific language governing rights and limitations - * under the License. + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. * * %CopyrightEnd% */ @@ -46,6 +47,10 @@ #define ERTS_PORT_TASK_ONLY_BASIC_TYPES__ #include "erl_port_task.h" #undef ERTS_PORT_TASK_ONLY_BASIC_TYPES__ + +#define ERTS_NODE_TAB_DELAY_GC_DEFAULT (60) +#define ERTS_NODE_TAB_DELAY_GC_MAX (100*1000*1000) +#define ERTS_NODE_TAB_DELAY_GC_INFINITY (ERTS_NODE_TAB_DELAY_GC_MAX+1) #define ERST_INTERNAL_CHANNEL_NO 0 @@ -166,20 +171,21 @@ extern DistEntry *erts_this_dist_entry; extern ErlNode *erts_this_node; extern char *erts_this_node_sysname; /* must match erl_node_tables.c */ +Uint erts_delayed_node_table_gc(void); DistEntry *erts_channel_no_to_dist_entry(Uint); DistEntry *erts_sysname_to_connected_dist_entry(Eterm); DistEntry *erts_find_or_insert_dist_entry(Eterm); DistEntry *erts_find_dist_entry(Eterm); -void erts_delete_dist_entry(DistEntry *); +void erts_schedule_delete_dist_entry(DistEntry *); Uint erts_dist_table_size(void); void erts_dist_table_info(int, void *); void erts_set_dist_entry_not_connected(DistEntry *); void erts_set_dist_entry_connected(DistEntry *, Eterm, Uint); ErlNode *erts_find_or_insert_node(Eterm, Uint); -void erts_delete_node(ErlNode *); +void erts_schedule_delete_node(ErlNode *); void erts_set_this_node(Eterm, Uint); Uint erts_node_table_size(void); -void erts_init_node_tables(void); +void erts_init_node_tables(int); void erts_node_table_info(int, void *); void erts_print_node_info(int, void *, Eterm, int*, int*); Eterm erts_get_node_and_dist_references(struct process *); @@ -204,7 +210,7 @@ erts_deref_dist_entry(DistEntry *dep) { ASSERT(dep); if (erts_refc_dectest(&dep->refc, 0) == 0) - erts_delete_dist_entry(dep); + erts_schedule_delete_dist_entry(dep); } ERTS_GLB_INLINE void @@ -212,7 +218,7 @@ erts_deref_node_entry(ErlNode *np) { ASSERT(np); if (erts_refc_dectest(&np->refc, 0) == 0) - erts_delete_node(np); + erts_schedule_delete_node(np); } ERTS_GLB_INLINE void @@ -253,5 +259,6 @@ erts_smp_de_links_unlock(DistEntry *dep) #endif /* #if ERTS_GLB_INLINE_INCL_FUNC_DEF */ +void erts_debug_test_node_tab_delayed_delete(Sint64 millisecs); #endif diff --git a/erts/emulator/beam/erl_port.h b/erts/emulator/beam/erl_port.h index 377aa72ed5..acd68ef0ad 100644 --- a/erts/emulator/beam/erl_port.h +++ b/erts/emulator/beam/erl_port.h @@ -3,16 +3,17 @@ * * Copyright Ericsson AB 2012-2013. All Rights Reserved. * - * The contents of this file are subject to the Erlang Public License, - * Version 1.1, (the "License"); you may not use this file except in - * compliance with the License. You should have received a copy of the - * Erlang Public License along with this software. If not, it can be - * retrieved online at http://www.erlang.org/. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Software distributed under the License is distributed on an "AS IS" - * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See - * the License for the specific language governing rights and limitations - * under the License. + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. * * %CopyrightEnd% */ @@ -167,8 +168,7 @@ struct _erl_drv_port { #endif erts_atomic_t connected; /* A connected process */ Eterm caller; /* Current caller. */ - Eterm data; /* Data associated with port. */ - ErlHeapFragment* bp; /* Heap fragment holding data (NULL if imm data). */ + erts_smp_atomic_t data; /* Data associated with port. */ Uint bytes_in; /* Number of bytes read */ Uint bytes_out; /* Number of bytes written */ @@ -189,6 +189,12 @@ struct _erl_drv_port { int reds; /* Only used while executing driver callbacks */ }; + +void erts_init_port_data(Port *); +void erts_cleanup_port_data(Port *); +Uint erts_port_data_size(Port *); +ErlOffHeap *erts_port_data_offheap(Port *); + #define ERTS_PORT_GET_CONNECTED(PRT) \ ((Eterm) erts_atomic_read_nob(&(PRT)->connected)) #define ERTS_PORT_SET_CONNECTED(PRT, PID) \ @@ -261,8 +267,7 @@ erts_prtsd_set(Port *prt, int ix, void *data) #endif -extern erts_smp_atomic_t erts_bytes_out; /* no bytes written out */ -extern erts_smp_atomic_t erts_bytes_in; /* no bytes sent into the system */ +Eterm erts_request_io_bytes(Process *c_p); /* port status flags */ @@ -326,8 +331,6 @@ extern erts_smp_atomic_t erts_bytes_in; /* no bytes sent into the system */ #define ERTS_PORT_REDS_CONTROL (CONTEXT_REDS/100) #define ERTS_PORT_REDS_CALL (CONTEXT_REDS/50) #define ERTS_PORT_REDS_INFO (CONTEXT_REDS/100) -#define ERTS_PORT_REDS_SET_DATA (CONTEXT_REDS/100) -#define ERTS_PORT_REDS_GET_DATA (CONTEXT_REDS/100) #define ERTS_PORT_REDS_TERMINATE (CONTEXT_REDS/50) void print_port_info(Port *, int, void *); @@ -347,6 +350,7 @@ int erts_lc_is_port_locked(Port *); ERTS_GLB_INLINE void erts_port_inc_refc(Port *prt); ERTS_GLB_INLINE void erts_port_dec_refc(Port *prt); ERTS_GLB_INLINE void erts_port_add_refc(Port *prt, Sint32 add_refc); +ERTS_GLB_INLINE Sint erts_port_read_refc(Port *prt); ERTS_GLB_INLINE int erts_smp_port_trylock(Port *prt); ERTS_GLB_INLINE void erts_smp_port_lock(Port *prt); @@ -356,37 +360,26 @@ ERTS_GLB_INLINE void erts_smp_port_unlock(Port *prt); ERTS_GLB_INLINE void erts_port_inc_refc(Port *prt) { -#ifdef ERTS_SMP - erts_ptab_inc_refc(&prt->common); -#else - erts_atomic32_inc_nob(&prt->refc); -#endif + erts_ptab_atmc_inc_refc(&prt->common); } ERTS_GLB_INLINE void erts_port_dec_refc(Port *prt) { -#ifdef ERTS_SMP - int referred = erts_ptab_dec_test_refc(&prt->common); + int referred = erts_ptab_atmc_dec_test_refc(&prt->common); if (!referred) erts_port_free(prt); -#else - int refc = erts_atomic32_dec_read_nob(&prt->refc); - if (refc == 0) - erts_port_free(prt); -#endif } ERTS_GLB_INLINE void erts_port_add_refc(Port *prt, Sint32 add_refc) { -#ifdef ERTS_SMP - int referred = erts_ptab_add_test_refc(&prt->common, add_refc); + int referred = erts_ptab_atmc_add_test_refc(&prt->common, add_refc); if (!referred) erts_port_free(prt); -#else - int refc = erts_atomic32_add_read_nob(&prt->refc, add_refc); - if (refc == 0) - erts_port_free(prt); -#endif +} + +ERTS_GLB_INLINE Sint erts_port_read_refc(Port *prt) +{ + return erts_ptab_atmc_read_refc(&prt->common); } ERTS_GLB_INLINE int @@ -794,8 +787,6 @@ struct binary; #define ERTS_P2P_SIG_TYPE_INFO 7 #define ERTS_P2P_SIG_TYPE_LINK 8 #define ERTS_P2P_SIG_TYPE_UNLINK 9 -#define ERTS_P2P_SIG_TYPE_SET_DATA 10 -#define ERTS_P2P_SIG_TYPE_GET_DATA 11 #define ERTS_P2P_SIG_TYPE_BITS 4 #define ERTS_P2P_SIG_TYPE_MASK \ @@ -810,6 +801,7 @@ struct binary; #define ERTS_P2P_SIG_DATA_FLG_BAD_OUTPUT ERTS_P2P_SIG_DATA_FLG(4) #define ERTS_P2P_SIG_DATA_FLG_BROKEN_LINK ERTS_P2P_SIG_DATA_FLG(5) #define ERTS_P2P_SIG_DATA_FLG_SCHED ERTS_P2P_SIG_DATA_FLG(6) +#define ERTS_P2P_SIG_DATA_FLG_ASYNC ERTS_P2P_SIG_DATA_FLG(7) struct ErtsProc2PortSigData_ { int flags; @@ -856,10 +848,6 @@ struct ErtsProc2PortSigData_ { struct { Eterm from; } unlink; - struct { - ErlHeapFragment *bp; - Eterm data; - } set_data; } u; } ; @@ -919,6 +907,7 @@ erts_schedule_proc2port_signal(Process *, Eterm *, ErtsProc2PortSigData *, int, + ErtsPortTaskHandle *, ErtsProc2PortSigCallback); int erts_deliver_port_exit(Port *, Eterm, Eterm, int); @@ -932,6 +921,7 @@ int erts_deliver_port_exit(Port *, Eterm, Eterm, int); #define ERTS_PORT_SIG_FLG_BROKEN_LINK ERTS_P2P_SIG_DATA_FLG_BROKEN_LINK #define ERTS_PORT_SIG_FLG_BAD_OUTPUT ERTS_P2P_SIG_DATA_FLG_BAD_OUTPUT #define ERTS_PORT_SIG_FLG_FORCE_SCHED ERTS_P2P_SIG_DATA_FLG_SCHED +#define ERTS_PORT_SIG_FLG_ASYNC ERTS_P2P_SIG_DATA_FLG_ASYNC /* ERTS_PORT_SIG_FLG_FORCE_IMM_CALL only when crash dumping... */ #define ERTS_PORT_SIG_FLG_FORCE_IMM_CALL ERTS_P2P_SIG_DATA_FLG_BAD_OUTPUT @@ -953,7 +943,5 @@ ErtsPortOpResult erts_port_unlink(Process *, Port *, Eterm, Eterm *); ErtsPortOpResult erts_port_control(Process *, Port *, unsigned int, Eterm, Eterm *); ErtsPortOpResult erts_port_call(Process *, Port *, unsigned int, Eterm, Eterm *); ErtsPortOpResult erts_port_info(Process *, Port *, Eterm, Eterm *); -ErtsPortOpResult erts_port_set_data(Process *, Port *, Eterm, Eterm *); -ErtsPortOpResult erts_port_get_data(Process *, Port *, Eterm *); #endif diff --git a/erts/emulator/beam/erl_port_task.c b/erts/emulator/beam/erl_port_task.c index ce045ec94e..ddcca06b0a 100644 --- a/erts/emulator/beam/erl_port_task.c +++ b/erts/emulator/beam/erl_port_task.c @@ -3,16 +3,17 @@ * * Copyright Ericsson AB 2006-2013. All Rights Reserved. * - * The contents of this file are subject to the Erlang Public License, - * Version 1.1, (the "License"); you may not use this file except in - * compliance with the License. You should have received a copy of the - * Erlang Public License along with this software. If not, it can be - * retrieved online at http://www.erlang.org/. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Software distributed under the License is distributed on an "AS IS" - * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See - * the License for the specific language governing rights and limitations - * under the License. + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. * * %CopyrightEnd% */ @@ -32,6 +33,7 @@ #include "global.h" #include "erl_port_task.h" #include "dist.h" +#include "erl_check_io.h" #include "dtrace-wrapper.h" #include <stdarg.h> @@ -68,6 +70,13 @@ static void chk_task_queues(Port *pp, ErtsPortTask *execq, int processing_busy_q #define DTRACE_DRIVER(PROBE_NAME, PP) do {} while(0) #endif +#define ERTS_SMP_LC_VERIFY_RQ(RQ, PP) \ + do { \ + ERTS_SMP_LC_ASSERT(erts_smp_lc_runq_is_locked(runq)); \ + ERTS_SMP_LC_ASSERT((RQ) == ((ErtsRunQueue *) \ + erts_smp_atomic_read_nob(&(PP)->run_queue))); \ + } while (0) + erts_smp_atomic_t erts_port_task_outstanding_io_tasks; #define ERTS_PT_STATE_SCHEDULED 0 @@ -151,9 +160,10 @@ static ERTS_INLINE void schedule_port_task_free(ErtsPortTask *ptp) { #ifdef ERTS_SMP - erts_schedule_thr_prgr_later_op(call_port_task_free, - (void *) ptp, - &ptp->u.release); + erts_schedule_thr_prgr_later_cleanup_op(call_port_task_free, + (void *) ptp, + &ptp->u.release, + sizeof(ErtsPortTask)); #else port_task_free(ptp); #endif @@ -542,6 +552,16 @@ reset_handle(ErtsPortTask *ptp) } static ERTS_INLINE void +reset_executed_io_task_handle(ErtsPortTask *ptp) +{ + if (ptp->u.alive.handle) { + ASSERT(ptp == handle2task(ptp->u.alive.handle)); + erts_io_notify_port_task_executed(ptp->u.alive.handle); + reset_port_task_handle(ptp->u.alive.handle); + } +} + +static ERTS_INLINE void set_handle(ErtsPortTask *ptp, ErtsPortTaskHandle *pthp) { ptp->u.alive.handle = pthp; @@ -551,6 +571,19 @@ set_handle(ErtsPortTask *ptp, ErtsPortTaskHandle *pthp) } } +static ERTS_INLINE void +set_tmp_handle(ErtsPortTask *ptp, ErtsPortTaskHandle *pthp) +{ + ptp->u.alive.handle = NULL; + if (pthp) { + /* + * IMPORTANT! Task either need to be aborted, or task handle + * need to be detached before thread progress has been made. + */ + erts_smp_atomic_set_relb(pthp, (erts_aint_t) ptp); + } +} + /* * Busy port queue management @@ -772,9 +805,10 @@ static void schedule_port_task_handle_list_free(ErtsPortTaskHandleList *pthlp) { #ifdef ERTS_SMP - erts_schedule_thr_prgr_later_op(free_port_task_handle_list, - (void *) pthlp, - &pthlp->u.release); + erts_schedule_thr_prgr_later_cleanup_op(free_port_task_handle_list, + (void *) pthlp, + &pthlp->u.release, + sizeof(ErtsPortTaskHandleList)); #else erts_free(ERTS_ALC_T_PT_HNDL_LIST, pthlp); #endif @@ -783,12 +817,13 @@ schedule_port_task_handle_list_free(ErtsPortTaskHandleList *pthlp) static ERTS_INLINE void abort_nosuspend_task(Port *pp, ErtsPortTaskType type, - ErtsPortTaskTypeData *tdp) + ErtsPortTaskTypeData *tdp, + int bpq_data) { ASSERT(type == ERTS_PORT_TASK_PROC_SIG); - if (!pp->sched.taskq.bpq) + if (!bpq_data) tdp->psig.callback(NULL, ERTS_PORT_SFLG_INVALID, ERTS_PROC2PORT_SIG_ABORT_NOSUSPEND, @@ -862,6 +897,11 @@ enqueue_port(ErtsRunQueue *runq, Port *pp) ASSERT(runq->ports.start && runq->ports.end); erts_smp_inc_runq_len(runq, &runq->ports.info, ERTS_PORT_PRIO_LEVEL); + +#ifdef ERTS_SMP + if (runq->halt_in_progress) + erts_non_empty_runq(runq); +#endif } static ERTS_INLINE Port * @@ -971,6 +1011,7 @@ static ERTS_INLINE int finalize_exec(Port *pp, ErtsPortTask **execq, int processing_busy_q) { erts_aint32_t act; + unsigned int prof_runnable_ports; if (!processing_busy_q) pp->sched.taskq.local.first = *execq; @@ -987,6 +1028,10 @@ finalize_exec(Port *pp, ErtsPortTask **execq, int processing_busy_q) if (act & ERTS_PTS_FLG_CHK_UNSET_BUSY_PORT_Q) act = check_unset_busy_port_q(pp, act, pp->sched.taskq.bpq); + prof_runnable_ports = erts_system_profile_flags.runnable_ports; + if (prof_runnable_ports) + erts_port_task_sched_lock(&pp->sched); + while (1) { erts_aint32_t new, exp; @@ -998,12 +1043,24 @@ finalize_exec(Port *pp, ErtsPortTask **execq, int processing_busy_q) act = erts_smp_atomic32_cmpxchg_relb(&pp->sched.flags, new, exp); - ASSERT(!(act & ERTS_PTS_FLG_IN_RUNQ)); + ERTS_LC_ASSERT(!(act & ERTS_PTS_FLG_IN_RUNQ)); + ERTS_LC_ASSERT(!(act & ERTS_PTS_FLG_EXEC_IMM)); if (exp == act) break; } + if (prof_runnable_ports | IS_TRACED_FL(pp, F_TRACE_SCHED_PORTS)) { + /* trace port scheduling, out */ + if (IS_TRACED_FL(pp, F_TRACE_SCHED_PORTS)) + trace_sched_ports(pp, am_out); + if (prof_runnable_ports) { + if (!(act & (ERTS_PTS_FLG_EXEC_IMM|ERTS_PTS_FLG_HAVE_TASKS))) + profile_runnable_port(pp, am_inactive); + erts_port_task_sched_unlock(&pp->sched); + } + } + return (act & ERTS_PTS_FLG_HAVE_TASKS) != 0; } @@ -1180,6 +1237,13 @@ erl_drv_consume_timeslice(ErlDrvPort dprt, int percent) return 1; } +void +erts_port_task_tmp_handle_detach(ErtsPortTaskHandle *pthp) +{ + ERTS_SMP_LC_ASSERT(erts_thr_progress_lc_is_delaying()); + reset_port_task_handle(pthp); +} + /* * Abort a scheduled task. */ @@ -1204,7 +1268,7 @@ erts_port_task_abort(ErtsPortTaskHandle *pthp) ERTS_SMP_READ_MEMORY_BARRIER; old_state = erts_smp_atomic32_read_nob(&ptp->state); if (old_state == ERTS_PT_STATE_SCHEDULED) { - ASSERT(saved_pthp == pthp); + ASSERT(!saved_pthp || saved_pthp == pthp); } #endif @@ -1225,9 +1289,6 @@ erts_port_task_abort(ErtsPortTaskHandle *pthp) &erts_port_task_outstanding_io_tasks) > 0); erts_smp_atomic_dec_relb(&erts_port_task_outstanding_io_tasks); break; - case ERTS_PORT_TASK_PROC_SIG: - ERTS_INTERNAL_ERROR("Aborted process to port signal"); - break; default: break; } @@ -1321,7 +1382,7 @@ erts_port_task_abort_nosuspend_tasks(Port *pp) #endif schedule_port_task_handle_list_free(pthlp); - abort_nosuspend_task(pp, type, &td); + abort_nosuspend_task(pp, type, &td, pp->sched.taskq.bpq != NULL); } } @@ -1345,11 +1406,9 @@ erts_port_task_schedule(Eterm id, Port *pp; ErtsPortTask *ptp = NULL; erts_aint32_t act, add_flags; + unsigned int prof_runnable_ports; - if (pthp && erts_port_task_is_scheduled(pthp)) { - ASSERT(0); - erts_port_task_abort(pthp); - } + ERTS_LC_ASSERT(!pthp || !erts_port_task_is_scheduled(pthp)); ASSERT(is_internal_port(id)); @@ -1402,7 +1461,6 @@ erts_port_task_schedule(Eterm id, } case ERTS_PORT_TASK_PROC_SIG: { va_list argp; - ASSERT(!pthp); va_start(argp, type); sigdp = va_arg(argp, ErtsProc2PortSigData *); ptp = p2p_sig_data_to_task(sigdp); @@ -1410,7 +1468,7 @@ erts_port_task_schedule(Eterm id, ptp->u.alive.flags |= va_arg(argp, int); va_end(argp); if (!(ptp->u.alive.flags & ERTS_PT_FLG_NOSUSPEND)) - set_handle(ptp, pthp); + set_tmp_handle(ptp, pthp); else { ns_pthlp = erts_alloc(ERTS_ALC_T_PT_HNDL_LIST, sizeof(ErtsPortTaskHandleList)); @@ -1434,6 +1492,10 @@ erts_port_task_schedule(Eterm id, if (ns_pthlp) add_flags |= ERTS_PTS_FLG_HAVE_NS_TASKS; + prof_runnable_ports = erts_system_profile_flags.runnable_ports; + if (prof_runnable_ports) + erts_port_task_sched_lock(&pp->sched); + while (1) { erts_aint32_t new, exp; @@ -1458,6 +1520,13 @@ erts_port_task_schedule(Eterm id, goto done; /* Died after our task insert... */ } + if (prof_runnable_ports) { + if (!(act & ERTS_PTS_FLG_EXEC_IMM)) + profile_runnable_port(pp, am_active); + erts_port_task_sched_unlock(&pp->sched); + prof_runnable_ports = 0; + } + /* Enqueue port on run-queue */ runq = erts_port_runq(pp); @@ -1466,8 +1535,10 @@ erts_port_task_schedule(Eterm id, #ifdef ERTS_SMP xrunq = erts_check_emigration_need(runq, ERTS_PORT_PRIO_LEVEL); + ERTS_SMP_LC_ASSERT(runq != xrunq); + ERTS_SMP_LC_VERIFY_RQ(runq, pp); if (xrunq) { - /* Port emigrated ... */ + /* Emigrate port ... */ erts_smp_atomic_set_nob(&pp->run_queue, (erts_aint_t) xrunq); erts_smp_runq_unlock(runq); runq = erts_port_runq(pp); @@ -1477,10 +1548,6 @@ erts_port_task_schedule(Eterm id, #endif enqueue_port(runq, pp); - - if (erts_system_profile_flags.runnable_ports) { - profile_runnable_port(pp, am_active); - } erts_smp_runq_unlock(runq); @@ -1488,6 +1555,9 @@ erts_port_task_schedule(Eterm id, done: + if (prof_runnable_ports) + erts_port_task_sched_unlock(&pp->sched); + #ifdef ERTS_SMP if (dhndl != ERTS_THR_PRGR_DHANDLE_MANAGED) erts_port_dec_refc(pp); @@ -1502,7 +1572,7 @@ abort_nosuspend: erts_port_dec_refc(pp); #endif - abort_nosuspend_task(pp, ptp->type, &ptp->u.alive.td); + abort_nosuspend_task(pp, ptp->type, &ptp->u.alive.td, 0); ASSERT(ns_pthlp); erts_free(ERTS_ALC_T_PT_HNDL_LIST, ns_pthlp); @@ -1576,6 +1646,8 @@ erts_port_task_execute(ErtsRunQueue *runq, Port **curr_port_pp) int fpe_was_unmasked; erts_aint32_t state; int active; + Uint64 start_time = 0; + ErtsSchedulerData *esdp = runq->scheduler; ERTS_SMP_LC_ASSERT(erts_smp_lc_runq_is_locked(runq)); @@ -1585,12 +1657,13 @@ erts_port_task_execute(ErtsRunQueue *runq, Port **curr_port_pp) goto done; } + ERTS_SMP_LC_VERIFY_RQ(runq, pp); + erts_smp_runq_unlock(runq); *curr_port_pp = pp; if (erts_sched_stat.enabled) { - ErtsSchedulerData *esdp = erts_get_scheduler_data(); Uint old = ERTS_PORT_SCHED_ID(pp, esdp->no); int migrated = old && old != esdp->no; @@ -1635,7 +1708,9 @@ erts_port_task_execute(ErtsRunQueue *runq, Port **curr_port_pp) goto aborted_port_task; } - reset_handle(ptp); + if (erts_system_monitor_long_schedule != 0) { + start_time = erts_timestamp_millis(); + } ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(pp)); ERTS_SMP_CHK_NO_PROC_LOCKS; @@ -1643,19 +1718,27 @@ erts_port_task_execute(ErtsRunQueue *runq, Port **curr_port_pp) switch (ptp->type) { case ERTS_PORT_TASK_TIMEOUT: - reds = ERTS_PORT_REDS_TIMEOUT; - if (!(state & ERTS_PORT_SFLGS_DEAD)) { - DTRACE_DRIVER(driver_timeout, pp); - (*pp->drv_ptr->timeout)((ErlDrvData) pp->drv_data); - } + reset_handle(ptp); + if (!ERTS_PTMR_IS_TIMED_OUT(pp)) + reds = 0; + else { + ERTS_PTMR_CLEAR(pp); + reds = ERTS_PORT_REDS_TIMEOUT; + if (!(state & ERTS_PORT_SFLGS_DEAD)) { + DTRACE_DRIVER(driver_timeout, pp); + (*pp->drv_ptr->timeout)((ErlDrvData) pp->drv_data); + } + } break; case ERTS_PORT_TASK_INPUT: reds = ERTS_PORT_REDS_INPUT; ASSERT((state & ERTS_PORT_SFLGS_DEAD) == 0); DTRACE_DRIVER(driver_ready_input, pp); - /* NOTE some windows drivers use ->ready_input for input and output */ + /* NOTE some windows/ose drivers use ->ready_input + for input and output */ (*pp->drv_ptr->ready_input)((ErlDrvData) pp->drv_data, ptp->u.alive.td.io.event); + reset_executed_io_task_handle(ptp); io_tasks_executed++; break; case ERTS_PORT_TASK_OUTPUT: @@ -1664,6 +1747,7 @@ erts_port_task_execute(ErtsRunQueue *runq, Port **curr_port_pp) DTRACE_DRIVER(driver_ready_output, pp); (*pp->drv_ptr->ready_output)((ErlDrvData) pp->drv_data, ptp->u.alive.td.io.event); + reset_executed_io_task_handle(ptp); io_tasks_executed++; break; case ERTS_PORT_TASK_EVENT: @@ -1673,10 +1757,12 @@ erts_port_task_execute(ErtsRunQueue *runq, Port **curr_port_pp) (*pp->drv_ptr->event)((ErlDrvData) pp->drv_data, ptp->u.alive.td.io.event, ptp->u.alive.td.io.event_data); + reset_executed_io_task_handle(ptp); io_tasks_executed++; break; case ERTS_PORT_TASK_PROC_SIG: { ErtsProc2PortSigData *sigdp = &ptp->u.alive.td.psig.data; + reset_handle(ptp); ASSERT((state & ERTS_PORT_SFLGS_DEAD) == 0); if (!pp->sched.taskq.bpq) reds = ptp->u.alive.td.psig.callback(pp, @@ -1694,10 +1780,11 @@ erts_port_task_execute(ErtsRunQueue *runq, Port **curr_port_pp) break; } case ERTS_PORT_TASK_DIST_CMD: + reset_handle(ptp); reds = erts_dist_command(pp, CONTEXT_REDS - pp->reds); break; default: - erl_exit(ERTS_ABORT_EXIT, + erts_exit(ERTS_ABORT_EXIT, "Invalid port task type: %d\n", (int) ptp->type); break; @@ -1705,6 +1792,14 @@ erts_port_task_execute(ErtsRunQueue *runq, Port **curr_port_pp) reds += erts_port_driver_callback_epilogue(pp, &state); + if (start_time != 0) { + Sint64 diff = erts_timestamp_millis() - start_time; + if (diff > 0 && (Uint) diff > erts_system_monitor_long_schedule) { + monitor_long_schedule_port(pp,ptp->type,(Uint) diff); + } + } + start_time = 0; + aborted_port_task: schedule_port_task_free(ptp); @@ -1728,10 +1823,6 @@ erts_port_task_execute(ErtsRunQueue *runq, Port **curr_port_pp) erts_unblock_fpe(fpe_was_unmasked); - /* trace port scheduling, out */ - if (IS_TRACED_FL(pp, F_TRACE_SCHED_PORTS)) { - trace_sched_ports(pp, am_out); - } if (io_tasks_executed) { ASSERT(erts_smp_atomic_read_nob(&erts_port_task_outstanding_io_tasks) @@ -1754,11 +1845,7 @@ erts_port_task_execute(ErtsRunQueue *runq, Port **curr_port_pp) erts_smp_runq_lock(runq); - if (!active) { - if (erts_system_profile_flags.runnable_ports) - profile_runnable_port(pp, am_inactive); - } - else { + if (active) { #ifdef ERTS_SMP ErtsRunQueue *xrunq; #endif @@ -1767,6 +1854,8 @@ erts_port_task_execute(ErtsRunQueue *runq, Port **curr_port_pp) #ifdef ERTS_SMP xrunq = erts_check_emigration_need(runq, ERTS_PORT_PRIO_LEVEL); + ERTS_SMP_LC_ASSERT(runq != xrunq); + ERTS_SMP_LC_VERIFY_RQ(runq, pp); if (!xrunq) { #endif enqueue_port(runq, pp); @@ -1774,7 +1863,7 @@ erts_port_task_execute(ErtsRunQueue *runq, Port **curr_port_pp) #ifdef ERTS_SMP } else { - /* Port emigrated ... */ + /* Emigrate port... */ erts_smp_atomic_set_nob(&pp->run_queue, (erts_aint_t) xrunq); erts_smp_runq_unlock(runq); @@ -1796,7 +1885,7 @@ erts_port_task_execute(ErtsRunQueue *runq, Port **curr_port_pp) runq->scheduler->reductions += reds; ERTS_SMP_LC_ASSERT(erts_smp_lc_runq_is_locked(runq)); - ERTS_PORT_REDUCTIONS_EXECUTED(runq, reds); + ERTS_PORT_REDUCTIONS_EXECUTED(esdp, runq, reds); return res; } @@ -1807,6 +1896,16 @@ release_port(void *vport) { erts_port_dec_refc((Port *) vport); } + +static void +schedule_release_port(void *vport) { + Port *pp = (Port*)vport; + /* This is only used when a port release was ordered from a non-scheduler */ + erts_schedule_thr_prgr_later_op(release_port, + (void *) pp, + &pp->common.u.release); +} + #endif static void @@ -1910,18 +2009,21 @@ begin_port_cleanup(Port *pp, ErtsPortTask **execqp, int *processing_busy_q_p) break; case ERTS_PORT_TASK_INPUT: erts_stale_drv_select(pp->common.id, + ERTS_Port2ErlDrvPort(pp), ptp->u.alive.td.io.event, DO_READ, 1); break; case ERTS_PORT_TASK_OUTPUT: erts_stale_drv_select(pp->common.id, + ERTS_Port2ErlDrvPort(pp), ptp->u.alive.td.io.event, DO_WRITE, 1); break; case ERTS_PORT_TASK_EVENT: erts_stale_drv_select(pp->common.id, + ERTS_Port2ErlDrvPort(pp), ptp->u.alive.td.io.event, 0, 1); @@ -1946,7 +2048,7 @@ begin_port_cleanup(Port *pp, ErtsPortTask **execqp, int *processing_busy_q_p) break; } default: - erl_exit(ERTS_ABORT_EXIT, + erts_exit(ERTS_ABORT_EXIT, "Invalid port task type: %d\n", (int) ptp->type); } @@ -1985,9 +2087,9 @@ begin_port_cleanup(Port *pp, ErtsPortTask **execqp, int *processing_busy_q_p) DTRACE_CHARBUF(pid_str, 16); ErtsProcList* plp2 = plp; - erts_snprintf(port_str, sizeof(port_str), "%T", pp->common.id); + erts_snprintf(port_str, sizeof(DTRACE_CHARBUF_NAME(port_str)), "%T", pp->common.id); while (plp2 != NULL) { - erts_snprintf(pid_str, sizeof(pid_str), "%T", plp2->pid); + erts_snprintf(pid_str, sizeof(DTRACE_CHARBUF_NAME(pid_str)), "%T", plp2->pid); DTRACE2(process_port_unblocked, pid_str, port_str); } } @@ -1999,9 +2101,15 @@ begin_port_cleanup(Port *pp, ErtsPortTask **execqp, int *processing_busy_q_p) * Schedule cleanup of port structure... */ #ifdef ERTS_SMP - erts_schedule_thr_prgr_later_op(release_port, - (void *) pp, - &pp->common.u.release); + /* We might not be a scheduler, eg. traceing to port we are sys_msg_dispatcher */ + if (!erts_get_scheduler_data()) { + erts_schedule_misc_aux_work(1, schedule_release_port, (void*)pp); + } else { + /* Has to be more or less immediate to release any driver */ + erts_schedule_thr_prgr_later_op(release_port, + (void *) pp, + &pp->common.u.release); + } #else pp->cleanup = 1; #endif diff --git a/erts/emulator/beam/erl_port_task.h b/erts/emulator/beam/erl_port_task.h index ae6cd69ae2..335f7a77d5 100644 --- a/erts/emulator/beam/erl_port_task.h +++ b/erts/emulator/beam/erl_port_task.h @@ -1,18 +1,19 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2006-2012. All Rights Reserved. + * Copyright Ericsson AB 2006-2013. All Rights Reserved. * - * The contents of this file are subject to the Erlang Public License, - * Version 1.1, (the "License"); you may not use this file except in - * compliance with the License. You should have received a copy of the - * Erlang Public License along with this software. If not, it can be - * retrieved online at http://www.erlang.org/. - * - * Software distributed under the License is distributed on an "AS IS" - * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See - * the License for the specific language governing rights and limitations - * under the License. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. * * %CopyrightEnd% */ @@ -77,6 +78,8 @@ extern erts_smp_atomic_t erts_port_task_outstanding_io_tasks; #define ERTS_PTS_FLG_HAVE_NS_TASKS (((erts_aint32_t) 1) << 8) #define ERTS_PTS_FLG_PARALLELISM (((erts_aint32_t) 1) << 9) #define ERTS_PTS_FLG_FORCE_SCHED (((erts_aint32_t) 1) << 10) +#define ERTS_PTS_FLG_EXITING (((erts_aint32_t) 1) << 11) +#define ERTS_PTS_FLG_EXEC_IMM (((erts_aint32_t) 1) << 12) #define ERTS_PTS_FLGS_BUSY \ (ERTS_PTS_FLG_BUSY_PORT | ERTS_PTS_FLG_BUSY_PORT_Q) @@ -86,7 +89,9 @@ extern erts_smp_atomic_t erts_port_task_outstanding_io_tasks; | ERTS_PTS_FLG_HAVE_BUSY_TASKS \ | ERTS_PTS_FLG_HAVE_TASKS \ | ERTS_PTS_FLG_EXEC \ - | ERTS_PTS_FLG_FORCE_SCHED) + | ERTS_PTS_FLG_EXEC_IMM \ + | ERTS_PTS_FLG_FORCE_SCHED \ + | ERTS_PTS_FLG_EXITING) #define ERTS_PORT_TASK_DEFAULT_BUSY_PORT_Q_HIGH 8192 #define ERTS_PORT_TASK_DEFAULT_BUSY_PORT_Q_LOW 4096 @@ -135,6 +140,7 @@ ERTS_GLB_INLINE void erts_port_task_fini_sched(ErtsPortTaskSched *ptsp); ERTS_GLB_INLINE void erts_port_task_sched_lock(ErtsPortTaskSched *ptsp); ERTS_GLB_INLINE void erts_port_task_sched_unlock(ErtsPortTaskSched *ptsp); ERTS_GLB_INLINE int erts_port_task_sched_lock_is_locked(ErtsPortTaskSched *ptsp); +ERTS_GLB_INLINE void erts_port_task_sched_enter_exiting_state(ErtsPortTaskSched *ptsp); #ifdef ERTS_INCLUDE_SCHEDULER_INTERNALS ERTS_GLB_INLINE int erts_port_task_have_outstanding_io_tasks(void); @@ -151,7 +157,7 @@ erts_port_task_handle_init(ErtsPortTaskHandle *pthp) ERTS_GLB_INLINE int erts_port_task_is_scheduled(ErtsPortTaskHandle *pthp) { - return ((void *) erts_smp_atomic_read_nob(pthp)) != NULL; + return ((void *) erts_smp_atomic_read_acqb(pthp)) != NULL; } ERTS_GLB_INLINE void erts_port_task_pre_init_sched(ErtsPortTaskSched *ptsp, @@ -182,11 +188,13 @@ erts_port_task_init_sched(ErtsPortTaskSched *ptsp, Eterm instr_id) ptsp->taskq.in.last = NULL; erts_smp_atomic32_init_nob(&ptsp->flags, 0); #ifdef ERTS_SMP + erts_mtx_init_x(&ptsp->mtx, lock_str, instr_id, #ifdef ERTS_ENABLE_LOCK_COUNT - if (!(erts_lcnt_rt_options & ERTS_LCNT_OPT_PORTLOCK)) - lock_str = NULL; + (erts_lcnt_rt_options & ERTS_LCNT_OPT_PORTLOCK) +#else + 1 #endif - erts_mtx_init_x(&ptsp->mtx, lock_str, instr_id); + ); #endif } @@ -225,6 +233,12 @@ erts_port_task_fini_sched(ErtsPortTaskSched *ptsp) #endif } +ERTS_GLB_INLINE void +erts_port_task_sched_enter_exiting_state(ErtsPortTaskSched *ptsp) +{ + erts_smp_atomic32_read_bor_nob(&ptsp->flags, ERTS_PTS_FLG_EXITING); +} + #ifdef ERTS_INCLUDE_SCHEDULER_INTERNALS ERTS_GLB_INLINE int @@ -243,7 +257,9 @@ int erts_port_task_execute(ErtsRunQueue *, Port **); void erts_port_task_init(void); #endif +void erts_port_task_tmp_handle_detach(ErtsPortTaskHandle *); int erts_port_task_abort(ErtsPortTaskHandle *); + void erts_port_task_abort_nosuspend_tasks(Port *); int erts_port_task_schedule(Eterm, diff --git a/erts/emulator/beam/erl_printf_term.c b/erts/emulator/beam/erl_printf_term.c index 436147749e..2917d58932 100644 --- a/erts/emulator/beam/erl_printf_term.c +++ b/erts/emulator/beam/erl_printf_term.c @@ -1,18 +1,19 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2005-2013. All Rights Reserved. + * Copyright Ericsson AB 2005-2014. All Rights Reserved. * - * The contents of this file are subject to the Erlang Public License, - * Version 1.1, (the "License"); you may not use this file except in - * compliance with the License. You should have received a copy of the - * Erlang Public License along with this software. If not, it can be - * retrieved online at http://www.erlang.org/. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Software distributed under the License is distributed on an "AS IS" - * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See - * the License for the specific language governing rights and limitations - * under the License. + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. * * %CopyrightEnd% */ @@ -24,6 +25,8 @@ #include "erl_printf_term.h" #include "sys.h" #include "big.h" +#include "erl_map.h" +#include "erl_binary.h" #define PRINT_CHAR(CNT, FN, ARG, C) \ do { \ @@ -137,6 +140,25 @@ is_printable_string(Eterm list, Eterm* base) return 0; } +static int is_printable_ascii(byte* bytep, Uint bytesize, Uint bitoffs) +{ + if (!bitoffs) { + while (bytesize--) { + if (*bytep < ' ' || *bytep >= 127) + return 0; + bytep++; + } + } else { + while (bytesize--) { + byte octet = (bytep[0] << bitoffs) | (bytep[1] >> (8-bitoffs)); + if (octet < ' ' || octet >= 127) + return 0; + bytep++; + } + } + return 1; +} + /* print a atom doing what quoting is necessary */ static int print_atom_name(fmtfn_t fn, void* arg, Eterm atom, long *dcount) { @@ -216,14 +238,26 @@ static int print_atom_name(fmtfn_t fn, void* arg, Eterm atom, long *dcount) } -#define PRT_BAR ((Eterm) 0) -#define PRT_COMMA ((Eterm) 1) -#define PRT_CLOSE_LIST ((Eterm) 2) -#define PRT_CLOSE_TUPLE ((Eterm) 3) -#define PRT_TERM ((Eterm) 4) -#define PRT_ONE_CONS ((Eterm) 5) -#define PRT_PATCH_FUN_SIZE ((Eterm) 6) -#define PRT_LAST_ARRAY_ELEMENT ((Eterm) 7) /* Note! Must be last... */ +#define PRT_BAR ((Eterm) 0) +#define PRT_COMMA ((Eterm) 1) +#define PRT_CLOSE_LIST ((Eterm) 2) +#define PRT_CLOSE_TUPLE ((Eterm) 3) +#define PRT_ASSOC ((Eterm) 4) +#define PRT_TERM ((Eterm) 5) +#define PRT_ONE_CONS ((Eterm) 6) +#define PRT_PATCH_FUN_SIZE ((Eterm) 7) +#define PRT_LAST_ARRAY_ELEMENT ((Eterm) 8) /* Note! Must be last... */ + +#if 0 +static char *format_binary(Uint16 x, char *b) { + int z; + b[16] = '\0'; + for (z = 0; z < 16; z++) { + b[15-z] = ((x>>z) & 0x1) ? '1' : '0'; + } + return b; +} +#endif static int print_term(fmtfn_t fn, void* arg, Eterm obj, long *dcount, @@ -260,6 +294,9 @@ print_term(fmtfn_t fn, void* arg, Eterm obj, long *dcount, case PRT_CLOSE_TUPLE: PRINT_CHAR(res, fn, arg, '}'); goto L_outer_loop; + case PRT_ASSOC: + PRINT_STRING(res, fn, arg, "=>"); + goto L_outer_loop; default: popped.word = WSTACK_POP(s); @@ -278,13 +315,9 @@ print_term(fmtfn_t fn, void* arg, Eterm obj, long *dcount, tl = CDR(cons); if (is_not_nil(tl)) { if (is_list(tl)) { - WSTACK_PUSH(s, tl); - WSTACK_PUSH(s, PRT_ONE_CONS); - WSTACK_PUSH(s, PRT_COMMA); + WSTACK_PUSH3(s, tl, PRT_ONE_CONS, PRT_COMMA); } else { - WSTACK_PUSH(s, tl); - WSTACK_PUSH(s, PRT_TERM); - WSTACK_PUSH(s, PRT_BAR); + WSTACK_PUSH3(s, tl, PRT_TERM, PRT_BAR); } } } @@ -294,9 +327,7 @@ print_term(fmtfn_t fn, void* arg, Eterm obj, long *dcount, break; default: /* PRT_LAST_ARRAY_ELEMENT+1 and upwards */ obj = *popped.ptr; - WSTACK_PUSH(s, (UWord) (popped.ptr + 1)); - WSTACK_PUSH(s, val-1); - WSTACK_PUSH(s, PRT_COMMA); + WSTACK_PUSH3(s, (UWord) (popped.ptr + 1), val-1, PRT_COMMA); break; } break; @@ -426,8 +457,7 @@ print_term(fmtfn_t fn, void* arg, Eterm obj, long *dcount, WSTACK_PUSH(s,PRT_CLOSE_TUPLE); ++nobj; if (i > 0) { - WSTACK_PUSH(s, (UWord) nobj); - WSTACK_PUSH(s, PRT_LAST_ARRAY_ELEMENT+i-1); + WSTACK_PUSH2(s, (UWord) nobj, PRT_LAST_ARRAY_ELEMENT+i-1); } break; case FLOAT_DEF: { @@ -437,17 +467,66 @@ print_term(fmtfn_t fn, void* arg, Eterm obj, long *dcount, } break; case BINARY_DEF: - if (header_is_bin_matchstate(*boxed_val(wobj))) { - PRINT_STRING(res, fn, arg, "#MatchState"); - } - else { - ProcBin* pb = (ProcBin *) binary_val(wobj); - if (pb->size == 1) - PRINT_STRING(res, fn, arg, "<<1 byte>>"); - else { + { + byte* bytep; + Uint bytesize = binary_size_rel(obj,obj_base); + Uint bitoffs; + Uint bitsize; + byte octet; + ERTS_GET_BINARY_BYTES_REL(obj, bytep, bitoffs, bitsize, obj_base); + + if (bitsize || !bytesize + || !is_printable_ascii(bytep, bytesize, bitoffs)) { + int is_first = 1; PRINT_STRING(res, fn, arg, "<<"); - PRINT_UWORD(res, fn, arg, 'u', 0, 1, (ErlPfUWord) pb->size); - PRINT_STRING(res, fn, arg, " bytes>>"); + while (bytesize) { + if (is_first) + is_first = 0; + else + PRINT_CHAR(res, fn, arg, ','); + if (bitoffs) + octet = (bytep[0] << bitoffs) | (bytep[1] >> (8-bitoffs)); + else + octet = bytep[0]; + PRINT_UWORD(res, fn, arg, 'u', 0, 1, octet); + ++bytep; + --bytesize; + } + if (bitsize) { + Uint bits = bitoffs + bitsize; + octet = bytep[0]; + if (bits < 8) + octet >>= 8 - bits; + else if (bits > 8) { + bits -= 8; /* bits in last byte */ + octet <<= bits; + octet |= bytep[1] >> (8 - bits); + } + octet &= (1 << bitsize) - 1; + if (is_first) + is_first = 0; + else + PRINT_CHAR(res, fn, arg, ','); + PRINT_UWORD(res, fn, arg, 'u', 0, 1, octet); + PRINT_CHAR(res, fn, arg, ':'); + PRINT_UWORD(res, fn, arg, 'u', 0, 1, bitsize); + } + PRINT_STRING(res, fn, arg, ">>"); + } + else { + PRINT_STRING(res, fn, arg, "<<\""); + while (bytesize) { + if (bitoffs) + octet = (bytep[0] << bitoffs) | (bytep[1] >> (8-bitoffs)); + else + octet = bytep[0]; + if (octet == '"') + PRINT_CHAR(res, fn, arg, '\\'); + PRINT_CHAR(res, fn, arg, octet); + ++bytep; + --bytesize; + } + PRINT_STRING(res, fn, arg, "\">>"); } } break; @@ -483,7 +562,78 @@ print_term(fmtfn_t fn, void* arg, Eterm obj, long *dcount, PRINT_CHAR(res, fn, arg, '>'); } break; - default: + case MAP_DEF: + if (is_flatmap(wobj)) { + Uint n; + Eterm *ks, *vs; + flatmap_t *mp = (flatmap_t *)flatmap_val(wobj); + n = flatmap_get_size(mp); + ks = flatmap_get_keys(mp); + vs = flatmap_get_values(mp); + + PRINT_CHAR(res, fn, arg, '#'); + PRINT_CHAR(res, fn, arg, '{'); + WSTACK_PUSH(s, PRT_CLOSE_TUPLE); + if (n > 0) { + n--; + WSTACK_PUSH5(s, vs[n], PRT_TERM, PRT_ASSOC, ks[n], PRT_TERM); + while (n--) { + WSTACK_PUSH6(s, PRT_COMMA, vs[n], PRT_TERM, PRT_ASSOC, + ks[n], PRT_TERM); + } + } + } else { + Uint n, mapval; + Eterm *head; + head = hashmap_val(wobj); + mapval = MAP_HEADER_VAL(*head); + switch (MAP_HEADER_TYPE(*head)) { + case MAP_HEADER_TAG_HAMT_HEAD_ARRAY: + case MAP_HEADER_TAG_HAMT_HEAD_BITMAP: + PRINT_STRING(res, fn, arg, "#<"); + PRINT_UWORD(res, fn, arg, 'x', 0, 1, mapval); + PRINT_STRING(res, fn, arg, ">{"); + WSTACK_PUSH(s,PRT_CLOSE_TUPLE); + n = hashmap_bitcount(mapval); + ASSERT(n < 17); + head += 2; + if (n > 0) { + n--; + WSTACK_PUSH(s, head[n]); + WSTACK_PUSH(s, PRT_TERM); + while (n--) { + WSTACK_PUSH(s, PRT_COMMA); + WSTACK_PUSH(s, head[n]); + WSTACK_PUSH(s, PRT_TERM); + } + } + break; + case MAP_HEADER_TAG_HAMT_NODE_BITMAP: + n = hashmap_bitcount(mapval); + head++; + PRINT_CHAR(res, fn, arg, '<'); + PRINT_UWORD(res, fn, arg, 'x', 0, 1, mapval); + PRINT_STRING(res, fn, arg, ">{"); + WSTACK_PUSH(s,PRT_CLOSE_TUPLE); + ASSERT(n < 17); + if (n > 0) { + n--; + WSTACK_PUSH(s, head[n]); + WSTACK_PUSH(s, PRT_TERM); + while (n--) { + WSTACK_PUSH(s, PRT_COMMA); + WSTACK_PUSH(s, head[n]); + WSTACK_PUSH(s, PRT_TERM); + } + } + break; + } + } + break; + case MATCHSTATE_DEF: + PRINT_STRING(res, fn, arg, "#MatchState"); + break; + default: PRINT_STRING(res, fn, arg, "<unknown:"); PRINT_POINTER(res, fn, arg, wobj); PRINT_CHAR(res, fn, arg, '>'); @@ -492,17 +642,17 @@ print_term(fmtfn_t fn, void* arg, Eterm obj, long *dcount, } L_done: - DESTROY_WSTACK(s); return res; } + int erts_printf_term(fmtfn_t fn, void* arg, ErlPfEterm term, long precision, ErlPfEterm* term_base) { int res; - ASSERT(sizeof(ErlPfEterm) == sizeof(Eterm)); + ERTS_CT_ASSERT(sizeof(ErlPfEterm) == sizeof(Eterm)); res = print_term(fn, arg, (Eterm)term, &precision, (Eterm*)term_base); if (res < 0) diff --git a/erts/emulator/beam/erl_printf_term.h b/erts/emulator/beam/erl_printf_term.h index f92c99d713..87618a36d7 100644 --- a/erts/emulator/beam/erl_printf_term.h +++ b/erts/emulator/beam/erl_printf_term.h @@ -3,16 +3,17 @@ * * Copyright Ericsson AB 2005-2013. All Rights Reserved. * - * The contents of this file are subject to the Erlang Public License, - * Version 1.1, (the "License"); you may not use this file except in - * compliance with the License. You should have received a copy of the - * Erlang Public License along with this software. If not, it can be - * retrieved online at http://www.erlang.org/. - * - * Software distributed under the License is distributed on an "AS IS" - * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See - * the License for the specific language governing rights and limitations - * under the License. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. * * %CopyrightEnd% */ diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c index 00247b387a..c2afcc9c4f 100644 --- a/erts/emulator/beam/erl_process.c +++ b/erts/emulator/beam/erl_process.c @@ -1,18 +1,19 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 1996-2013. All Rights Reserved. + * Copyright Ericsson AB 1996-2014. All Rights Reserved. * - * The contents of this file are subject to the Erlang Public License, - * Version 1.1, (the "License"); you may not use this file except in - * compliance with the License. You should have received a copy of the - * Erlang Public License along with this software. If not, it can be - * retrieved online at http://www.erlang.org/. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Software distributed under the License is distributed on an "AS IS" - * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See - * the License for the specific language governing rights and limitations - * under the License. + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. * * %CopyrightEnd% */ @@ -43,7 +44,11 @@ #include "erl_async.h" #include "dtrace-wrapper.h" #include "erl_ptab.h" +#include "erl_bif_unique.h" +#define ERTS_WANT_TIMER_WHEEL_API +#include "erl_time.h" +#define ERTS_CHECK_TIME_REDS CONTEXT_REDS #define ERTS_DELAYED_WAKEUP_INFINITY (~(Uint64) 0) #define ERTS_DELAYED_WAKEUP_REDUCTIONS ((Uint64) CONTEXT_REDS/2) @@ -53,7 +58,11 @@ #define ERTS_PROC_MIN_CONTEXT_SWITCH_REDS_COST (CONTEXT_REDS/10) +#ifndef ERTS_SCHED_MIN_SPIN #define ERTS_SCHED_SPIN_UNTIL_YIELD 100 +#else +#define ERTS_SCHED_SPIN_UNTIL_YIELD 1 +#endif #define ERTS_SCHED_SYS_SLEEP_SPINCOUNT_VERY_LONG 40 #define ERTS_SCHED_AUX_WORK_SLEEP_SPINCOUNT_FACT_VERY_LONG 1000 @@ -143,8 +152,30 @@ extern BeamInstr beam_apply[]; extern BeamInstr beam_exit[]; extern BeamInstr beam_continue_exit[]; +#ifdef __OSE__ +/* Eager check I/O not supported on OSE yet. */ +int erts_eager_check_io = 0; +#else +int erts_eager_check_io = 1; +#endif int erts_sched_compact_load; +int erts_sched_balance_util = 0; Uint erts_no_schedulers; +#ifdef ERTS_DIRTY_SCHEDULERS +Uint erts_no_dirty_cpu_schedulers; +Uint erts_no_dirty_io_schedulers; +#endif + +static char *erts_aux_work_flag_descr[ERTS_SSI_AUX_WORK_NO_FLAGS] = {0}; +int erts_aux_work_no_flags = ERTS_SSI_AUX_WORK_NO_FLAGS; + +#define ERTS_THR_PRGR_LATER_CLEANUP_OP_THRESHOLD_VERY_LAZY (4*1024*1024) +#define ERTS_THR_PRGR_LATER_CLEANUP_OP_THRESHOLD_LAZY (512*1024) +#define ERTS_THR_PRGR_LATER_CLEANUP_OP_THRESHOLD_MEDIUM (64*1024) +#define ERTS_THR_PRGR_LATER_CLEANUP_OP_THRESHOLD_EAGER (16*1024) +#define ERTS_THR_PRGR_LATER_CLEANUP_OP_THRESHOLD_VERY_EAGER (1024) + +static UWord thr_prgr_later_cleanup_op_threshold = ERTS_THR_PRGR_LATER_CLEANUP_OP_THRESHOLD_MEDIUM; ErtsPTab erts_proc erts_align_attribute(ERTS_CACHE_LINE_SIZE); @@ -174,6 +205,13 @@ static ErtsAuxWorkData *aux_thread_aux_work_data; #define ERTS_SCHDLR_SSPND_CHNG_SET(VAL, OLD_VAL) \ erts_smp_atomic32_set_nob(&schdlr_sspnd.changing, (VAL)) +#ifdef ERTS_DIRTY_SCHEDULERS +#define ERTS_SCHDLR_SSPND_DIRTY_CPU_CHNG_SET(VAL, OLD_VAL) \ + erts_smp_atomic32_set_nob(&schdlr_sspnd.dirty_cpu_changing, (VAL)) +#define ERTS_SCHDLR_SSPND_DIRTY_IO_CHNG_SET(VAL, OLD_VAL) \ + erts_smp_atomic32_set_nob(&schdlr_sspnd.dirty_io_changing, (VAL)) +#endif + #else #define ERTS_SCHDLR_SSPND_CHNG_SET(VAL, OLD_VAL) \ @@ -184,6 +222,23 @@ do { \ ASSERT(old_val__ == (OLD_VAL)); \ } while (0) +#ifdef ERTS_DIRTY_SCHEDULERS +#define ERTS_SCHDLR_SSPND_DIRTY_CPU_CHNG_SET(VAL, OLD_VAL) \ +do { \ + erts_aint32_t old_val__; \ + old_val__ = erts_smp_atomic32_xchg_nob(&schdlr_sspnd.dirty_cpu_changing, \ + (VAL)); \ + ASSERT(old_val__ == (OLD_VAL)); \ +} while (0) +#define ERTS_SCHDLR_SSPND_DIRTY_IO_CHNG_SET(VAL, OLD_VAL) \ +do { \ + erts_aint32_t old_val__; \ + old_val__ = erts_smp_atomic32_xchg_nob(&schdlr_sspnd.dirty_io_changing, \ + (VAL)); \ + ASSERT(old_val__ == (OLD_VAL)); \ +} while (0) +#endif + #endif @@ -193,11 +248,29 @@ static struct { int online; int curr_online; int wait_curr_online; +#ifdef ERTS_DIRTY_SCHEDULERS + int dirty_cpu_online; + int dirty_cpu_curr_online; + int dirty_cpu_wait_curr_online; + int dirty_io_online; + int dirty_io_curr_online; + int dirty_io_wait_curr_online; +#endif erts_smp_atomic32_t changing; erts_smp_atomic32_t active; +#ifdef ERTS_DIRTY_SCHEDULERS + erts_smp_atomic32_t dirty_cpu_changing; + erts_smp_atomic32_t dirty_cpu_active; + erts_smp_atomic32_t dirty_io_changing; + erts_smp_atomic32_t dirty_io_active; +#endif struct { int ongoing; long wait_active; +#ifdef ERTS_DIRTY_SCHEDULERS + long dirty_cpu_wait_active; + long dirty_io_wait_active; +#endif ErtsProcList *procs; } msb; /* Multi Scheduling Block */ } schdlr_sspnd; @@ -238,6 +311,10 @@ static erts_smp_atomic32_t function_calls; #ifdef ERTS_SMP static erts_smp_atomic32_t doing_sys_schedule; static erts_smp_atomic32_t no_empty_run_queues; +long erts_runq_supervision_interval = 0; +static ethr_event runq_supervision_event; +static erts_tid_t runq_supervisor_tid; +static erts_atomic_t runq_supervisor_sleeping; #else /* !ERTS_SMP */ ErtsSchedulerData *erts_scheduler_data; #endif @@ -246,6 +323,10 @@ ErtsAlignedRunQueue *erts_aligned_run_queues; Uint erts_no_run_queues; ErtsAlignedSchedulerData *erts_aligned_scheduler_data; +#ifdef ERTS_DIRTY_SCHEDULERS +ErtsAlignedSchedulerData *erts_aligned_dirty_cpu_scheduler_data; +ErtsAlignedSchedulerData *erts_aligned_dirty_io_scheduler_data; +#endif typedef union { ErtsSchedulerSleepInfo ssi; @@ -253,23 +334,65 @@ typedef union { } ErtsAlignedSchedulerSleepInfo; static ErtsAlignedSchedulerSleepInfo *aligned_sched_sleep_info; +#ifdef ERTS_DIRTY_SCHEDULERS +#ifdef ERTS_SMP +static ErtsAlignedSchedulerSleepInfo *aligned_dirty_cpu_sched_sleep_info; +static ErtsAlignedSchedulerSleepInfo *aligned_dirty_io_sched_sleep_info; +#endif +#endif static Uint last_reductions; static Uint last_exact_reductions; Uint erts_default_process_flags; Eterm erts_system_monitor; Eterm erts_system_monitor_long_gc; +Uint erts_system_monitor_long_schedule; Eterm erts_system_monitor_large_heap; struct erts_system_monitor_flags_t erts_system_monitor_flags; /* system performance monitor */ Eterm erts_system_profile; struct erts_system_profile_flags_t erts_system_profile_flags; +int erts_system_profile_ts_type = ERTS_TRACE_FLG_NOW_TIMESTAMP; #if ERTS_MAX_PROCESSES > 0x7fffffff #error "Need to store process_count in another type" #endif +typedef enum { + ERTS_PSTT_GC, /* Garbage Collect */ + ERTS_PSTT_CPC /* Check Process Code */ +} ErtsProcSysTaskType; + +#define ERTS_MAX_PROC_SYS_TASK_ARGS 2 + +struct ErtsProcSysTask_ { + ErtsProcSysTask *next; + ErtsProcSysTask *prev; + ErtsProcSysTaskType type; + Eterm requester; + Eterm reply_tag; + Eterm req_id; + Uint req_id_sz; + Eterm arg[ERTS_MAX_PROC_SYS_TASK_ARGS]; + ErlOffHeap off_heap; + Eterm heap[1]; +}; + +#define ERTS_PROC_SYS_TASK_SIZE(HSz) \ + (sizeof(ErtsProcSysTask) - sizeof(Eterm) + sizeof(Eterm)*(HSz)) + +struct ErtsProcSysTaskQs_ { + int qmask; + int ncount; + ErtsProcSysTask *q[ERTS_NO_PROC_PRIO_LEVELS]; +}; + +ERTS_SCHED_PREF_QUICK_ALLOC_IMPL(proc_sys_task_queues, + ErtsProcSysTaskQs, + 50, + ERTS_ALC_T_PROC_SYS_TSK_QS) + ERTS_SCHED_PREF_QUICK_ALLOC_IMPL(misc_op_list, ErtsMiscOpList, 10, @@ -281,9 +404,19 @@ ERTS_SCHED_PREF_QUICK_ALLOC_IMPL(proclist, ERTS_ALC_T_PROC_LIST) #define ERTS_SCHED_SLEEP_INFO_IX(IX) \ - (ASSERT_EXPR(-1 <= ((int) (IX)) \ + (ASSERT(-1 <= ((int) (IX)) \ && ((int) (IX)) < ((int) erts_no_schedulers)), \ &aligned_sched_sleep_info[(IX)].ssi) +#ifdef ERTS_DIRTY_SCHEDULERS +#define ERTS_DIRTY_CPU_SCHED_SLEEP_INFO_IX(IX) \ + (ASSERT(0 <= ((int) (IX)) \ + && ((int) (IX)) < ((int) erts_no_dirty_cpu_schedulers)), \ + &aligned_dirty_cpu_sched_sleep_info[(IX)].ssi) +#define ERTS_DIRTY_IO_SCHED_SLEEP_INFO_IX(IX) \ + (ASSERT(0 <= ((int) (IX)) \ + && ((int) (IX)) < ((int) erts_no_dirty_io_schedulers)), \ + &aligned_dirty_io_sched_sleep_info[(IX)].ssi) +#endif #define ERTS_FOREACH_RUNQ(RQVAR, DO) \ do { \ @@ -332,13 +465,20 @@ do { \ static void exec_misc_ops(ErtsRunQueue *); static void print_function_from_pc(int to, void *to_arg, BeamInstr* x); -static int stack_element_dump(int to, void *to_arg, Process* p, Eterm* sp, - int yreg); +static int stack_element_dump(int to, void *to_arg, Eterm* sp, int yreg); static void aux_work_timeout(void *unused); static void aux_work_timeout_early_init(int no_schedulers); static void aux_work_timeout_late_init(void); -static void setup_aux_work_timer(void); +static void setup_aux_work_timer(ErtsSchedulerData *esdp); + +static int execute_sys_tasks(Process *c_p, + erts_aint32_t *statep, + int in_reds); +static int cleanup_sys_tasks(Process *c_p, + erts_aint32_t in_state, + int in_reds); + #if defined(DEBUG) || 0 #define ERTS_DBG_CHK_AUX_WORK_VAL(V) dbg_chk_aux_work_val((V)) @@ -360,6 +500,8 @@ dbg_chk_aux_work_val(erts_aint32_t value) valid |= ERTS_SSI_AUX_WORK_MISC_THR_PRGR; valid |= ERTS_SSI_AUX_WORK_DD; valid |= ERTS_SSI_AUX_WORK_DD_THR_PRGR; + valid |= ERTS_SSI_AUX_WORK_CNCLD_TMRS; + valid |= ERTS_SSI_AUX_WORK_CNCLD_TMRS_THR_PRGR; valid |= ERTS_SSI_AUX_WORK_THR_PRGR_LATER_OP; #endif #if HAVE_ERTS_MSEG @@ -371,9 +513,10 @@ dbg_chk_aux_work_val(erts_aint32_t value) #ifdef ERTS_SSI_AUX_WORK_REAP_PORTS valid |= ERTS_SSI_AUX_WORK_REAP_PORTS; #endif + valid |= ERTS_SSI_AUX_WORK_DEBUG_WAIT_COMPLETED; if (~valid & value) - erl_exit(ERTS_ABORT_EXIT, + erts_exit(ERTS_ABORT_EXIT, "Invalid aux_work value found: 0x%x\n", ~valid & value); } @@ -386,7 +529,7 @@ dbg_chk_aux_work_val(erts_aint32_t value) #ifdef ERTS_SMP static void handle_pending_exiters(ErtsProcList *); - +static void wake_scheduler(ErtsRunQueue *rq); #endif #if defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_CHECK) @@ -426,8 +569,43 @@ void erts_pre_init_process(void) { #ifdef USE_THREADS - erts_tsd_key_create(&sched_data_key); -#endif + erts_tsd_key_create(&sched_data_key, "erts_sched_data_key"); +#endif + + erts_aux_work_flag_descr[ERTS_SSI_AUX_WORK_DELAYED_AW_WAKEUP_IX] + = "DELAYED_AW_WAKEUP"; + erts_aux_work_flag_descr[ERTS_SSI_AUX_WORK_DD_IX] + = "DD"; + erts_aux_work_flag_descr[ERTS_SSI_AUX_WORK_DD_THR_PRGR_IX] + = "DD_THR_PRGR"; + erts_aux_work_flag_descr[ERTS_SSI_AUX_WORK_FIX_ALLOC_DEALLOC_IX] + = "FIX_ALLOC_DEALLOC"; + erts_aux_work_flag_descr[ERTS_SSI_AUX_WORK_FIX_ALLOC_LOWER_LIM_IX] + = "FIX_ALLOC_LOWER_LIM"; + erts_aux_work_flag_descr[ERTS_SSI_AUX_WORK_THR_PRGR_LATER_OP_IX] + = "THR_PRGR_LATER_OP"; + erts_aux_work_flag_descr[ERTS_SSI_AUX_WORK_CNCLD_TMRS_IX] + = "CNCLD_TMRS"; + erts_aux_work_flag_descr[ERTS_SSI_AUX_WORK_CNCLD_TMRS_THR_PRGR_IX] + = "CNCLD_TMRS_THR_PRGR"; + erts_aux_work_flag_descr[ERTS_SSI_AUX_WORK_ASYNC_READY_IX] + = "ASYNC_READY"; + erts_aux_work_flag_descr[ERTS_SSI_AUX_WORK_ASYNC_READY_CLEAN_IX] + = "ASYNC_READY_CLEAN"; + erts_aux_work_flag_descr[ERTS_SSI_AUX_WORK_MISC_THR_PRGR_IX] + = "MISC_THR_PRGR"; + erts_aux_work_flag_descr[ERTS_SSI_AUX_WORK_MISC_IX] + = "MISC"; + erts_aux_work_flag_descr[ERTS_SSI_AUX_WORK_CHECK_CHILDREN_IX] + = "CHECK_CHILDREN"; + erts_aux_work_flag_descr[ERTS_SSI_AUX_WORK_SET_TMO_IX] + = "SET_TMO"; + erts_aux_work_flag_descr[ERTS_SSI_AUX_WORK_MSEG_CACHE_CHECK_IX] + = "MSEG_CACHE_CHECK"; + erts_aux_work_flag_descr[ERTS_SSI_AUX_WORK_REAP_PORTS_IX] + = "REAP_PORTS"; + erts_aux_work_flag_descr[ERTS_SSI_AUX_WORK_DEBUG_WAIT_COMPLETED_IX] + = "DEBUG_WAIT_COMPLETED"; #ifdef ERTS_ENABLE_LOCK_CHECK { @@ -448,16 +626,21 @@ erts_pre_init_process(void) erts_psd_required_locks[ERTS_PSD_SCHED_ID].set_locks = ERTS_PSD_SCHED_ID_SET_LOCKS; - erts_psd_required_locks[ERTS_PSD_DIST_ENTRY].get_locks - = ERTS_PSD_DIST_ENTRY_GET_LOCKS; - erts_psd_required_locks[ERTS_PSD_DIST_ENTRY].set_locks - = ERTS_PSD_DIST_ENTRY_SET_LOCKS; - erts_psd_required_locks[ERTS_PSD_CALL_TIME_BP].get_locks = ERTS_PSD_CALL_TIME_BP_GET_LOCKS; erts_psd_required_locks[ERTS_PSD_CALL_TIME_BP].set_locks = ERTS_PSD_CALL_TIME_BP_SET_LOCKS; + erts_psd_required_locks[ERTS_PSD_DELAYED_GC_TASK_QS].get_locks + = ERTS_PSD_DELAYED_GC_TASK_QS_GET_LOCKS; + erts_psd_required_locks[ERTS_PSD_DELAYED_GC_TASK_QS].set_locks + = ERTS_PSD_DELAYED_GC_TASK_QS_SET_LOCKS; + + erts_psd_required_locks[ERTS_PSD_NIF_TRAP_EXPORT].get_locks + = ERTS_PSD_NIF_TRAP_EXPORT_GET_LOCKS; + erts_psd_required_locks[ERTS_PSD_NIF_TRAP_EXPORT].set_locks + = ERTS_PSD_NIF_TRAP_EXPORT_SET_LOCKS; + /* Check that we have locks for all entries */ for (ix = 0; ix < ERTS_PSD_SIZE; ix++) { ERTS_SMP_LC_ASSERT(erts_psd_required_locks[ix].get_locks); @@ -467,17 +650,15 @@ erts_pre_init_process(void) #endif } -#ifdef ERTS_SMP static void release_process(void *vproc) { - erts_smp_proc_dec_refc((Process *) vproc); + erts_proc_dec_refc((Process *) vproc); } -#endif /* initialize the scheduler */ void -erts_init_process(int ncpu, int proc_tab_size) +erts_init_process(int ncpu, int proc_tab_size, int legacy_proc_tab) { #ifdef ERTS_SMP @@ -489,14 +670,18 @@ erts_init_process(int ncpu, int proc_tab_size) erts_ptab_init_table(&erts_proc, ERTS_ALC_T_PROC_TABLE, -#ifdef ERTS_SMP release_process, -#else - NULL, -#endif (ErtsPTabElementCommon *) &erts_invalid_process.common, proc_tab_size, - "process_table"); + sizeof(Process), + "process_table", + legacy_proc_tab, +#ifdef ERTS_SMP + 1 +#else + 0 +#endif + ); last_reductions = 0; last_exact_reductions = 0; @@ -546,6 +731,7 @@ erts_late_init_process(void) static void init_sched_wall_time(ErtsSchedWallTime *swtp) { + swtp->need = erts_sched_balance_util; swtp->enabled = 0; swtp->start = 0; swtp->working.total = 0; @@ -556,8 +742,8 @@ init_sched_wall_time(ErtsSchedWallTime *swtp) static ERTS_INLINE Uint64 sched_wall_time_ts(void) { -#ifdef HAVE_GETHRTIME - return (Uint64) sys_gethrtime(); +#ifdef ERTS_HAVE_OS_MONOTONIC_TIME_SUPPORT + return (Uint64) erts_os_monotonic_time(); #else Uint64 res; SysTimeval tv; @@ -568,27 +754,205 @@ sched_wall_time_ts(void) #endif } +#if ERTS_HAVE_SCHED_UTIL_BALANCING_SUPPORT + +static ERTS_INLINE Uint64 +aschedtime_read(ErtsAtomicSchedTime *var) +{ + return (Uint64) erts_atomic64_read_nob((erts_atomic64_t *) var); +} + +static ERTS_INLINE void +aschedtime_set(ErtsAtomicSchedTime *var, Uint64 val) +{ + erts_atomic64_set_nob((erts_atomic64_t *) var, (erts_aint64_t) val); +} + +static ERTS_INLINE void +aschedtime_init(ErtsAtomicSchedTime *var) +{ + erts_atomic64_init_nob((erts_atomic64_t *) var, (erts_aint64_t) 0); +} + +#define ERTS_GET_AVG_MAX_UNLOCKED_TRY 50 +#define ERTS_SCHED_AVG_UTIL_WRITE_MARKER (~((Uint64) 0)) + +/* Intervals in nanoseconds */ +#define ERTS_SCHED_UTIL_SHORT_INTERVAL ((Uint64) 1*1000*1000*1000) +#define ERTS_SCHED_UTIL_LONG_INTERVAL ((Uint64) 10*1000*1000*1000) + + +#define ERTS_SCHED_UTIL_IGNORE_IMBALANCE_DIFF 5000 /* ppm */ + +static ERTS_INLINE Uint64 +calc_sched_worktime(int is_working, Uint64 now, Uint64 last, + Uint64 interval, Uint64 old_worktime) +{ + Uint64 worktime; + Uint64 new; + + if (now <= last) + return old_worktime; + + new = now - last; + + if (new >= interval) + return is_working ? interval : (Uint64) 0; + + + /* + * Division by 1000 in order to avoid + * overflow. If changed update assertions + * in init_runq_sched_util(). + */ + worktime = old_worktime; + worktime *= (interval - new)/1000; + worktime /= (interval/1000); + if (is_working) + worktime += new; + + ASSERT(0 <= worktime && worktime <= interval); + + return worktime; +} + +static ERTS_INLINE void +update_avg_sched_util(ErtsSchedulerData *esdp, Uint64 now, int is_working) +{ + ErtsRunQueue *rq; + int worked; + Uint64 swt, lwt, last; + + rq = esdp->run_queue; + last = aschedtime_read(&rq->sched_util.last); + + if (now <= last) { + ASSERT(last == ERTS_SCHED_AVG_UTIL_WRITE_MARKER); + return; + } + + ASSERT(now >= last); + + worked = rq->sched_util.is_working; + + swt = calc_sched_worktime(worked, now, last, ERTS_SCHED_UTIL_SHORT_INTERVAL, + rq->sched_util.worktime.short_interval); + lwt = calc_sched_worktime(worked, now, last, ERTS_SCHED_UTIL_LONG_INTERVAL, + rq->sched_util.worktime.long_interval); + + aschedtime_set(&rq->sched_util.last, ERTS_SCHED_AVG_UTIL_WRITE_MARKER); + ERTS_THR_WRITE_MEMORY_BARRIER; + rq->sched_util.is_working = is_working; + rq->sched_util.worktime.short_interval = swt; + rq->sched_util.worktime.long_interval = lwt; + ERTS_THR_WRITE_MEMORY_BARRIER; + aschedtime_set(&rq->sched_util.last, now); +} + +int +erts_get_sched_util(ErtsRunQueue *rq, int initially_locked, int short_interval) +{ + /* Average scheduler utilization in ppm */ + int util, is_working, try = 0, locked = initially_locked; + Uint64 worktime, old_worktime, now, last, interval, *old_worktimep; + + if (short_interval) { + old_worktimep = &rq->sched_util.worktime.short_interval; + interval = ERTS_SCHED_UTIL_SHORT_INTERVAL; + } + else { + old_worktimep = &rq->sched_util.worktime.long_interval; + interval = ERTS_SCHED_UTIL_LONG_INTERVAL; + } + + while (1) { + Uint64 chk_last; + last = aschedtime_read(&rq->sched_util.last); + ERTS_THR_READ_MEMORY_BARRIER; + is_working = rq->sched_util.is_working; + old_worktime = *old_worktimep; + ERTS_THR_READ_MEMORY_BARRIER; + chk_last = aschedtime_read(&rq->sched_util.last); + if (chk_last == last) + break; + if (!locked) { + if (++try >= ERTS_GET_AVG_MAX_UNLOCKED_TRY) { + /* Writer will eventually block on runq-lock */ + erts_smp_runq_lock(rq); + locked = 1; + } + } + } + + if (!initially_locked && locked) + erts_smp_runq_unlock(rq); + + now = sched_wall_time_ts(); + worktime = calc_sched_worktime(is_working, now, last, interval, old_worktime); + + util = (int) ((worktime * 1000000)/interval); + + ASSERT(0 <= util && util <= 1000000); + + return util; +} + +static void +init_runq_sched_util(ErtsRunQueueSchedUtil *rqsu, int enabled) +{ + aschedtime_init(&rqsu->last); + if (!enabled) + aschedtime_set(&rqsu->last, ERTS_SCHED_AVG_UTIL_WRITE_MARKER); + rqsu->is_working = 0; + rqsu->worktime.short_interval = (Uint64) 0; + rqsu->worktime.long_interval = (Uint64) 0; + +#ifdef DEBUG + { + Uint64 intrvl; + /* + * If one of these asserts fail we may have + * overflow in calc_sched_worktime(). Which + * have to be fixed either by shrinking + * interval size, or fix calculation of + * worktime in calc_sched_worktime(). + */ + intrvl = ERTS_SCHED_UTIL_SHORT_INTERVAL; + ASSERT(intrvl*(intrvl/1000) > intrvl); + intrvl = ERTS_SCHED_UTIL_LONG_INTERVAL; + ASSERT(intrvl*(intrvl/1000) > intrvl); + } +#endif +} + +#endif /* ERTS_HAVE_SCHED_UTIL_BALANCING_SUPPORT */ + static ERTS_INLINE void sched_wall_time_change(ErtsSchedulerData *esdp, int working) { - if (esdp->sched_wall_time.enabled) { + if (esdp->sched_wall_time.need) { Uint64 ts = sched_wall_time_ts(); - if (working) { +#if ERTS_HAVE_SCHED_UTIL_BALANCING_SUPPORT + update_avg_sched_util(esdp, ts, working); +#endif + if (esdp->sched_wall_time.enabled) { + if (working) { #ifdef DEBUG - ASSERT(!esdp->sched_wall_time.working.currently); - esdp->sched_wall_time.working.currently = 1; + ASSERT(!esdp->sched_wall_time.working.currently); + esdp->sched_wall_time.working.currently = 1; #endif - ts -= esdp->sched_wall_time.start; - esdp->sched_wall_time.working.start = ts; - } - else { + ts -= esdp->sched_wall_time.start; + esdp->sched_wall_time.working.start = ts; + } + else { #ifdef DEBUG - ASSERT(esdp->sched_wall_time.working.currently); - esdp->sched_wall_time.working.currently = 0; + ASSERT(esdp->sched_wall_time.working.currently); + esdp->sched_wall_time.working.currently = 0; #endif - ts -= esdp->sched_wall_time.start; - ts -= esdp->sched_wall_time.working.start; - esdp->sched_wall_time.working.total += ts; + ts -= esdp->sched_wall_time.start; + ts -= esdp->sched_wall_time.working.start; + esdp->sched_wall_time.working.total += ts; + } } } } @@ -603,11 +967,25 @@ typedef struct { erts_smp_atomic32_t refc; } ErtsSchedWallTimeReq; +typedef struct { + Process *proc; + Eterm ref; + Eterm ref_heap[REF_THING_SIZE]; + Uint req_sched; + erts_smp_atomic32_t refc; +} ErtsSystemCheckReq; + + #if !HALFWORD_HEAP ERTS_SCHED_PREF_QUICK_ALLOC_IMPL(swtreq, ErtsSchedWallTimeReq, 5, ERTS_ALC_T_SCHED_WTIME_REQ) + +ERTS_SCHED_PREF_QUICK_ALLOC_IMPL(screq, + ErtsSystemCheckReq, + 5, + ERTS_ALC_T_SYS_CHECK_REQ) #else static ERTS_INLINE ErtsSchedWallTimeReq * swtreq_alloc(void) @@ -621,6 +999,19 @@ swtreq_free(ErtsSchedWallTimeReq *ptr) { erts_free(ERTS_ALC_T_SCHED_WTIME_REQ, ptr); } + +static ERTS_INLINE ErtsSystemCheckReq * +screq_alloc(void) +{ + return erts_alloc(ERTS_ALC_T_SYS_CHECK_REQ, + sizeof(ErtsSystemCheckReq)); +} + +static ERTS_INLINE void +screq_free(ErtsSystemCheckReq *ptr) +{ + erts_free(ERTS_ALC_T_SYS_CHECK_REQ, ptr); +} #endif static void @@ -641,12 +1032,17 @@ reply_sched_wall_time(void *vswtrp) ErlHeapFragment *bp = NULL; ASSERT(esdp); - +#ifdef ERTS_DIRTY_SCHEDULERS + ASSERT(!ERTS_SCHEDULER_IS_DIRTY(esdp)); +#endif if (swtrp->set) { - if (!swtrp->enable && esdp->sched_wall_time.enabled) + if (!swtrp->enable && esdp->sched_wall_time.enabled) { + esdp->sched_wall_time.need = erts_sched_balance_util; esdp->sched_wall_time.enabled = 0; + } else if (swtrp->enable && !esdp->sched_wall_time.enabled) { Uint64 ts = sched_wall_time_ts(); + esdp->sched_wall_time.need = 1; esdp->sched_wall_time.enabled = 1; esdp->sched_wall_time.start = ts; esdp->sched_wall_time.working.total = 0; @@ -694,11 +1090,7 @@ reply_sched_wall_time(void *vswtrp) hpp = &hp; } - erts_queue_message(rp, &rp_locks, bp, msg, NIL -#ifdef USE_VM_PROBES - , NIL -#endif - ); + erts_queue_message(rp, &rp_locks, bp, msg, NIL); if (swtrp->req_sched == esdp->no) rp_locks &= ~ERTS_PROC_LOCK_MAIN; @@ -706,7 +1098,7 @@ reply_sched_wall_time(void *vswtrp) if (rp_locks) erts_smp_proc_unlock(rp, rp_locks); - erts_smp_proc_dec_refc(rp); + erts_proc_dec_refc(rp); if (erts_smp_atomic32_dec_read_nob(&swtrp->refc) == 0) swtreq_free(vswtrp); @@ -722,6 +1114,9 @@ erts_sched_wall_time_request(Process *c_p, int set, int enable) if (!set && !esdp->sched_wall_time.enabled) return THE_NON_VALUE; +#ifdef ERTS_DIRTY_SCHEDULERS + ASSERT(!ERTS_SCHEDULER_IS_DIRTY(esdp)); +#endif swtrp = swtreq_alloc(); ref = erts_make_ref(c_p); @@ -735,7 +1130,7 @@ erts_sched_wall_time_request(Process *c_p, int set, int enable) erts_smp_atomic32_init_nob(&swtrp->refc, (erts_aint32_t) erts_no_schedulers); - erts_smp_proc_add_refc(c_p, (Sint32) erts_no_schedulers); + erts_proc_add_refc(c_p, (Sint32) erts_no_schedulers); #ifdef ERTS_SMP if (erts_no_schedulers > 1) @@ -750,6 +1145,75 @@ erts_sched_wall_time_request(Process *c_p, int set, int enable) return ref; } +static void +reply_system_check(void *vscrp) +{ + ErtsSchedulerData *esdp = erts_get_scheduler_data(); + ErtsSystemCheckReq *scrp = (ErtsSystemCheckReq *) vscrp; + ErtsProcLocks rp_locks = (scrp->req_sched == esdp->no ? ERTS_PROC_LOCK_MAIN : 0); + Process *rp = scrp->proc; + Eterm msg; + Eterm *hp = NULL; + Eterm **hpp; + Uint sz; + ErlOffHeap *ohp = NULL; + ErlHeapFragment *bp = NULL; + + ASSERT(esdp); +#ifdef ERTS_DIRTY_SCHEDULERS + ASSERT(!ERTS_SCHEDULER_IS_DIRTY(esdp)); +#endif + + sz = REF_THING_SIZE; + hp = erts_alloc_message_heap(sz, &bp, &ohp, rp, &rp_locks); + hpp = &hp; + msg = STORE_NC(hpp, ohp, scrp->ref); + + erts_queue_message(rp, &rp_locks, bp, msg, NIL); + + if (scrp->req_sched == esdp->no) + rp_locks &= ~ERTS_PROC_LOCK_MAIN; + + if (rp_locks) + erts_smp_proc_unlock(rp, rp_locks); + + erts_proc_dec_refc(rp); + + if (erts_smp_atomic32_dec_read_nob(&scrp->refc) == 0) + screq_free(vscrp); +} + + +Eterm erts_system_check_request(Process *c_p) { + ErtsSchedulerData *esdp = ERTS_PROC_GET_SCHDATA(c_p); + Eterm ref; + ErtsSystemCheckReq *scrp; + Eterm *hp; + + scrp = screq_alloc(); + ref = erts_make_ref(c_p); + hp = &scrp->ref_heap[0]; + + scrp->proc = c_p; + scrp->ref = STORE_NC(&hp, NULL, ref); + scrp->req_sched = esdp->no; + erts_smp_atomic32_init_nob(&scrp->refc, (erts_aint32_t) erts_no_schedulers); + + erts_proc_add_refc(c_p, (Sint) erts_no_schedulers); + +#ifdef ERTS_SMP + if (erts_no_schedulers > 1) + erts_schedule_multi_misc_aux_work(1, + erts_no_schedulers, + reply_system_check, + (void *) scrp); +#endif + + reply_system_check((void *) scrp); + + return ref; +} + static ERTS_INLINE ErtsProcList * proclist_create(Process *p) { @@ -796,7 +1260,7 @@ erts_psd_set_init(Process *p, ErtsProcLocks plocks, int ix, void *data) xplocks &= ~plocks; if (xplocks && erts_smp_proc_trylock(p, xplocks) == EBUSY) { if (xplocks & ERTS_PROC_LOCK_MAIN) { - erts_smp_proc_inc_refc(p); + erts_proc_inc_refc(p); erts_smp_proc_unlock(p, plocks); erts_smp_proc_lock(p, ERTS_PROC_LOCKS_ALL); refc = 1; @@ -812,7 +1276,7 @@ erts_psd_set_init(Process *p, ErtsProcLocks plocks, int ix, void *data) if (xplocks) erts_smp_proc_unlock(p, xplocks); if (refc) - erts_smp_proc_dec_refc(p); + erts_proc_dec_refc(p); ASSERT(p->psd); if (p->psd != psd) erts_free(ERTS_ALC_T_PSD, psd); @@ -844,7 +1308,7 @@ erts_sched_finish_poke(ErtsSchedulerSleepInfo *ssi, erts_aint32_t flags) case 0: break; default: - erl_exit(ERTS_ABORT_EXIT, "%s:%d: Internal error\n", + erts_exit(ERTS_ABORT_EXIT, "%s:%d: Internal error\n", __FILE__, __LINE__); break; } @@ -861,11 +1325,11 @@ set_aux_work_flags_wakeup_nob(ErtsSchedulerSleepInfo *ssi, ERTS_DBG_CHK_SSI_AUX_WORK(ssi); old_flgs = erts_atomic32_read_nob(&ssi->aux_work); - if ((old_flgs & flgs) == 0) { + if ((old_flgs & flgs) != flgs) { old_flgs = erts_atomic32_read_bor_nob(&ssi->aux_work, flgs); - if ((old_flgs & flgs) == 0) { + if ((old_flgs & flgs) != flgs) { #ifdef ERTS_SMP erts_sched_poke(ssi); #else @@ -885,7 +1349,7 @@ set_aux_work_flags_wakeup_relb(ErtsSchedulerSleepInfo *ssi, old_flgs = erts_atomic32_read_bor_relb(&ssi->aux_work, flgs); - if ((old_flgs & flgs) == 0) { + if ((old_flgs & flgs) != flgs) { #ifdef ERTS_SMP erts_sched_poke(ssi); #else @@ -909,6 +1373,53 @@ unset_aux_work_flags(ErtsSchedulerSleepInfo *ssi, erts_aint32_t flgs) #ifdef ERTS_SMP static ERTS_INLINE void +haw_chk_later_cleanup_op_wakeup(ErtsAuxWorkData *awdp, ErtsThrPrgrVal val) +{ + if (awdp->later_op.first + && erts_thr_progress_cmp(val, awdp->later_op.thr_prgr) >= 0) { + awdp->later_op.size = thr_prgr_later_cleanup_op_threshold; + } +} + +static ERTS_INLINE void +haw_thr_prgr_wakeup(ErtsAuxWorkData *awdp, ErtsThrPrgrVal val) +{ + int cmp = erts_thr_progress_cmp(val, awdp->latest_wakeup); + if (cmp != 0) { + if (cmp > 0) { + awdp->latest_wakeup = val; + haw_chk_later_cleanup_op_wakeup(awdp, val); + } + erts_thr_progress_wakeup(awdp->esdp, val); + } +} + +static ERTS_INLINE void +haw_thr_prgr_soft_wakeup(ErtsAuxWorkData *awdp, ErtsThrPrgrVal val) +{ + if (erts_thr_progress_cmp(val, awdp->latest_wakeup) > 0) { + awdp->latest_wakeup = val; + haw_chk_later_cleanup_op_wakeup(awdp, val); + erts_thr_progress_wakeup(awdp->esdp, val); + } +} + +static ERTS_INLINE void +haw_thr_prgr_later_cleanup_op_wakeup(ErtsAuxWorkData *awdp, ErtsThrPrgrVal val, UWord size) +{ + if (erts_thr_progress_cmp(val, awdp->latest_wakeup) > 0) { + awdp->later_op.thr_prgr = val; + if (awdp->later_op.size > size) + awdp->later_op.size -= size; + else { + awdp->latest_wakeup = val; + awdp->later_op.size = thr_prgr_later_cleanup_op_threshold; + erts_thr_progress_wakeup(awdp->esdp, val); + } + } +} + +static ERTS_INLINE void haw_thr_prgr_current_reset(ErtsAuxWorkData *awdp) { awdp->current_thr_prgr = ERTS_THR_PRGR_INVALID; @@ -929,6 +1440,9 @@ static ERTS_INLINE void haw_thr_prgr_current_check_progress(ErtsAuxWorkData *awdp) { ErtsThrPrgrVal current = awdp->current_thr_prgr; +#ifdef ERTS_DIRTY_SCHEDULERS + ASSERT(!awdp->esdp || !ERTS_SCHEDULER_IS_DIRTY(awdp->esdp)); +#endif if (current != ERTS_THR_PRGR_INVALID && !erts_thr_progress_equal(current, erts_thr_progress_current())) { /* @@ -945,6 +1459,10 @@ handle_delayed_aux_work_wakeup(ErtsAuxWorkData *awdp, erts_aint32_t aux_work, in { int jix, max_jix; +#ifdef ERTS_DIRTY_SCHEDULERS + ASSERT(!awdp->esdp || !ERTS_SCHEDULER_IS_DIRTY(awdp->esdp)); +#endif + ASSERT(awdp->delayed_wakeup.next != ERTS_DELAYED_WAKEUP_INFINITY); if (!waiting && awdp->delayed_wakeup.next > awdp->esdp->reductions) @@ -1066,8 +1584,7 @@ misc_aux_work_clean(ErtsThrQ_t *q, case ERTS_THR_Q_NEED_THR_PRGR: #ifdef ERTS_SMP set_aux_work_flags(awdp->ssi, ERTS_SSI_AUX_WORK_MISC_THR_PRGR); - erts_thr_progress_wakeup(awdp->esdp, - erts_thr_q_need_thr_progress(q)); + haw_thr_prgr_soft_wakeup(awdp, erts_thr_q_need_thr_progress(q)); #endif case ERTS_THR_Q_CLEAN: break; @@ -1101,6 +1618,9 @@ handle_misc_aux_work_thr_prgr(ErtsAuxWorkData *awdp, erts_aint32_t aux_work, int waiting) { +#ifdef ERTS_DIRTY_SCHEDULERS + ASSERT(!awdp->esdp || !ERTS_SCHEDULER_IS_DIRTY(awdp->esdp)); +#endif if (!erts_thr_progress_has_reached_this(haw_thr_prgr_current(awdp), awdp->misc.thr_prgr)) return aux_work & ~ERTS_SSI_AUX_WORK_MISC_THR_PRGR; @@ -1153,6 +1673,9 @@ erts_schedule_multi_misc_aux_work(int ignore_self, if (ignore_self) { ErtsSchedulerData *esdp = erts_get_scheduler_data(); +#ifdef ERTS_DIRTY_SCHEDULERS + ASSERT(!ERTS_SCHEDULER_IS_DIRTY(esdp)); +#endif if (esdp) self = (int) esdp->no; } @@ -1182,6 +1705,9 @@ handle_async_ready(ErtsAuxWorkData *awdp, int waiting) { ErtsSchedulerSleepInfo *ssi = awdp->ssi; +#ifdef ERTS_DIRTY_SCHEDULERS + ASSERT(!awdp->esdp || !ERTS_SCHEDULER_IS_DIRTY(awdp->esdp)); +#endif unset_aux_work_flags(ssi, ERTS_SSI_AUX_WORK_ASYNC_READY); if (erts_check_async_ready(awdp->async_ready.queue)) { if (set_aux_work_flags(ssi, ERTS_SSI_AUX_WORK_ASYNC_READY) @@ -1206,6 +1732,9 @@ handle_async_ready_clean(ErtsAuxWorkData *awdp, { void *thr_prgr_p; +#ifdef ERTS_DIRTY_SCHEDULERS + ASSERT(!awdp->esdp || !ERTS_SCHEDULER_IS_DIRTY(awdp->esdp)); +#endif #ifdef ERTS_SMP if (awdp->async_ready.need_thr_prgr && !erts_thr_progress_has_reached_this(haw_thr_prgr_current(awdp), @@ -1225,8 +1754,7 @@ handle_async_ready_clean(ErtsAuxWorkData *awdp, return aux_work & ~ERTS_SSI_AUX_WORK_ASYNC_READY_CLEAN; #ifdef ERTS_SMP case ERTS_ASYNC_READY_NEED_THR_PRGR: - erts_thr_progress_wakeup(awdp->esdp, - awdp->async_ready.thr_prgr); + haw_thr_prgr_soft_wakeup(awdp, awdp->async_ready.thr_prgr); awdp->async_ready.need_thr_prgr = 1; return aux_work & ~ERTS_SSI_AUX_WORK_ASYNC_READY_CLEAN; #endif @@ -1244,6 +1772,9 @@ handle_fix_alloc(ErtsAuxWorkData *awdp, erts_aint32_t aux_work, int waiting) ErtsSchedulerSleepInfo *ssi = awdp->ssi; erts_aint32_t res; +#ifdef ERTS_DIRTY_SCHEDULERS + ASSERT(!awdp->esdp || !ERTS_SCHEDULER_IS_DIRTY(awdp->esdp)); +#endif unset_aux_work_flags(ssi, (ERTS_SSI_AUX_WORK_FIX_ALLOC_LOWER_LIM | ERTS_SSI_AUX_WORK_FIX_ALLOC_DEALLOC)); aux_work &= ~(ERTS_SSI_AUX_WORK_FIX_ALLOC_LOWER_LIM @@ -1263,7 +1794,7 @@ void erts_alloc_notify_delayed_dealloc(int ix) { ErtsSchedulerData *esdp = erts_get_scheduler_data(); - if (esdp) + if (esdp && !ERTS_SCHEDULER_IS_DIRTY(esdp)) schedule_aux_work_wakeup(&esdp->aux_work_data, ix, ERTS_SSI_AUX_WORK_DD); @@ -1272,6 +1803,17 @@ erts_alloc_notify_delayed_dealloc(int ix) ERTS_SSI_AUX_WORK_DD); } +void +erts_alloc_ensure_handle_delayed_dealloc_call(int ix) +{ +#ifdef DEBUG + ErtsSchedulerData *esdp = erts_get_scheduler_data(); + ASSERT(!esdp || (ERTS_SCHEDULER_IS_DIRTY(esdp) || ix == (int) esdp->no)); +#endif + set_aux_work_flags_wakeup_nob(ERTS_SCHED_SLEEP_INFO_IX(ix-1), + ERTS_SSI_AUX_WORK_DD); +} + static ERTS_INLINE erts_aint32_t handle_delayed_dealloc(ErtsAuxWorkData *awdp, erts_aint32_t aux_work, int waiting) { @@ -1280,6 +1822,9 @@ handle_delayed_dealloc(ErtsAuxWorkData *awdp, erts_aint32_t aux_work, int waitin ErtsThrPrgrVal wakeup = ERTS_THR_PRGR_INVALID; int more_work = 0; +#ifdef ERTS_DIRTY_SCHEDULERS + ASSERT(!awdp->esdp || !ERTS_SCHEDULER_IS_DIRTY(awdp->esdp)); +#endif unset_aux_work_flags(ssi, ERTS_SSI_AUX_WORK_DD); erts_alloc_scheduler_handle_delayed_dealloc((void *) awdp->esdp, &need_thr_progress, @@ -1300,12 +1845,7 @@ handle_delayed_dealloc(ErtsAuxWorkData *awdp, erts_aint32_t aux_work, int waitin awdp->dd.thr_prgr = wakeup; set_aux_work_flags(ssi, ERTS_SSI_AUX_WORK_DD_THR_PRGR); awdp->dd.thr_prgr = wakeup; - erts_thr_progress_wakeup(awdp->esdp, wakeup); - } - else if (awdp->dd.completed_callback) { - awdp->dd.completed_callback(awdp->dd.completed_arg); - awdp->dd.completed_callback = NULL; - awdp->dd.completed_arg = NULL; + haw_thr_prgr_soft_wakeup(awdp, wakeup); } return aux_work & ~ERTS_SSI_AUX_WORK_DD; } @@ -1319,6 +1859,9 @@ handle_delayed_dealloc_thr_prgr(ErtsAuxWorkData *awdp, erts_aint32_t aux_work, i ErtsThrPrgrVal wakeup = ERTS_THR_PRGR_INVALID; ErtsThrPrgrVal current = haw_thr_prgr_current(awdp); +#ifdef ERTS_DIRTY_SCHEDULERS + ASSERT(!awdp->esdp || !ERTS_SCHEDULER_IS_DIRTY(awdp->esdp)); +#endif if (!erts_thr_progress_has_reached_this(current, awdp->dd.thr_prgr)) return aux_work & ~ERTS_SSI_AUX_WORK_DD_THR_PRGR; @@ -1341,21 +1884,111 @@ handle_delayed_dealloc_thr_prgr(ErtsAuxWorkData *awdp, erts_aint32_t aux_work, i if (wakeup == ERTS_THR_PRGR_INVALID) wakeup = erts_thr_progress_later(awdp->esdp); awdp->dd.thr_prgr = wakeup; - erts_thr_progress_wakeup(awdp->esdp, wakeup); + haw_thr_prgr_soft_wakeup(awdp, wakeup); } else { unset_aux_work_flags(ssi, ERTS_SSI_AUX_WORK_DD_THR_PRGR); - if (awdp->dd.completed_callback) { - awdp->dd.completed_callback(awdp->dd.completed_arg); - awdp->dd.completed_callback = NULL; - awdp->dd.completed_arg = NULL; - } } return aux_work & ~ERTS_SSI_AUX_WORK_DD_THR_PRGR; } /* + * Canceled timers + */ + +void +erts_notify_canceled_timer(ErtsSchedulerData *esdp, int rsid) +{ + ASSERT(esdp && esdp == erts_get_scheduler_data()); + if (esdp && !ERTS_SCHEDULER_IS_DIRTY(esdp)) + schedule_aux_work_wakeup(&esdp->aux_work_data, + rsid, + ERTS_SSI_AUX_WORK_CNCLD_TMRS); + else + set_aux_work_flags_wakeup_relb(ERTS_SCHED_SLEEP_INFO_IX(rsid-1), + ERTS_SSI_AUX_WORK_CNCLD_TMRS); +} + +static ERTS_INLINE erts_aint32_t +handle_canceled_timers(ErtsAuxWorkData *awdp, erts_aint32_t aux_work, int waiting) +{ + ErtsSchedulerSleepInfo *ssi = awdp->ssi; + int need_thr_progress = 0; + ErtsThrPrgrVal wakeup = ERTS_THR_PRGR_INVALID; + int more_work = 0; + +#ifdef ERTS_DIRTY_SCHEDULERS + ASSERT(!awdp->esdp || !ERTS_SCHEDULER_IS_DIRTY(awdp->esdp)); +#endif + unset_aux_work_flags(ssi, ERTS_SSI_AUX_WORK_CNCLD_TMRS); + erts_handle_canceled_timers((void *) awdp->esdp, + &need_thr_progress, + &wakeup, + &more_work); + if (more_work) { + if (set_aux_work_flags(ssi, ERTS_SSI_AUX_WORK_CNCLD_TMRS) + & ERTS_SSI_AUX_WORK_CNCLD_TMRS_THR_PRGR) { + unset_aux_work_flags(ssi, ERTS_SSI_AUX_WORK_CNCLD_TMRS_THR_PRGR); + aux_work &= ~ERTS_SSI_AUX_WORK_CNCLD_TMRS_THR_PRGR; + } + return aux_work; + } + + if (need_thr_progress) { + if (wakeup == ERTS_THR_PRGR_INVALID) + wakeup = erts_thr_progress_later(awdp->esdp); + awdp->cncld_tmrs.thr_prgr = wakeup; + set_aux_work_flags(ssi, ERTS_SSI_AUX_WORK_CNCLD_TMRS_THR_PRGR); + haw_thr_prgr_soft_wakeup(awdp, wakeup); + } + return aux_work & ~ERTS_SSI_AUX_WORK_CNCLD_TMRS; +} + +static ERTS_INLINE erts_aint32_t +handle_canceled_timers_thr_prgr(ErtsAuxWorkData *awdp, erts_aint32_t aux_work, int waiting) +{ + ErtsSchedulerSleepInfo *ssi; + int need_thr_progress; + int more_work; + ErtsThrPrgrVal wakeup = ERTS_THR_PRGR_INVALID; + ErtsThrPrgrVal current = haw_thr_prgr_current(awdp); + +#ifdef ERTS_DIRTY_SCHEDULERS + ASSERT(!awdp->esdp || !ERTS_SCHEDULER_IS_DIRTY(awdp->esdp)); +#endif + if (!erts_thr_progress_has_reached_this(current, awdp->cncld_tmrs.thr_prgr)) + return aux_work & ~ERTS_SSI_AUX_WORK_CNCLD_TMRS_THR_PRGR; + + ssi = awdp->ssi; + need_thr_progress = 0; + more_work = 0; + + erts_handle_canceled_timers((void *) awdp->esdp, + &need_thr_progress, + &wakeup, + &more_work); + if (more_work) { + set_aux_work_flags(ssi, ERTS_SSI_AUX_WORK_CNCLD_TMRS); + unset_aux_work_flags(ssi, ERTS_SSI_AUX_WORK_CNCLD_TMRS_THR_PRGR); + return ((aux_work & ~ERTS_SSI_AUX_WORK_CNCLD_TMRS_THR_PRGR) + | ERTS_SSI_AUX_WORK_CNCLD_TMRS); + } + + if (need_thr_progress) { + if (wakeup == ERTS_THR_PRGR_INVALID) + wakeup = erts_thr_progress_later(awdp->esdp); + awdp->cncld_tmrs.thr_prgr = wakeup; + haw_thr_prgr_soft_wakeup(awdp, wakeup); + } + else { + unset_aux_work_flags(ssi, ERTS_SSI_AUX_WORK_CNCLD_TMRS_THR_PRGR); + } + + return aux_work & ~ERTS_SSI_AUX_WORK_CNCLD_TMRS_THR_PRGR; +} + +/* * Handle scheduled thread progress later operations. */ #define ERTS_MAX_THR_PRGR_LATER_OPS 50 @@ -1366,6 +1999,9 @@ handle_thr_prgr_later_op(ErtsAuxWorkData *awdp, erts_aint32_t aux_work, int wait int lops; ErtsThrPrgrVal current = haw_thr_prgr_current(awdp); +#ifdef ERTS_DIRTY_SCHEDULERS + ASSERT(!awdp->esdp || !ERTS_SCHEDULER_IS_DIRTY(awdp->esdp)); +#endif for (lops = 0; lops < ERTS_MAX_THR_PRGR_LATER_OPS; lops++) { ErtsThrPrgrLaterOp *lop = awdp->later_op.first; if (!erts_thr_progress_has_reached_this(current, lop->later)) @@ -1376,6 +2012,7 @@ handle_thr_prgr_later_op(ErtsAuxWorkData *awdp, erts_aint32_t aux_work, int wait } lop->func(lop->data); if (!awdp->later_op.first) { + awdp->later_op.size = thr_prgr_later_cleanup_op_threshold; awdp->later_op.last = NULL; unset_aux_work_flags(awdp->ssi, ERTS_SSI_AUX_WORK_THR_PRGR_LATER_OP); @@ -1386,6 +2023,29 @@ handle_thr_prgr_later_op(ErtsAuxWorkData *awdp, erts_aint32_t aux_work, int wait return aux_work; } +static ERTS_INLINE ErtsThrPrgrVal +enqueue_later_op(ErtsSchedulerData *esdp, + void (*later_func)(void *), + void *later_data, + ErtsThrPrgrLaterOp *lop) +{ + ErtsThrPrgrVal later = erts_thr_progress_later(esdp); + ASSERT(esdp); + + lop->func = later_func; + lop->data = later_data; + lop->later = later; + lop->next = NULL; + if (!esdp->aux_work_data.later_op.last) + esdp->aux_work_data.later_op.first = lop; + else + esdp->aux_work_data.later_op.last->next = lop; + esdp->aux_work_data.later_op.last = lop; + set_aux_work_flags_wakeup_nob(esdp->ssi, + ERTS_SSI_AUX_WORK_THR_PRGR_LATER_OP); + return later; +} + #endif /* ERTS_SMP */ void @@ -1396,106 +2056,163 @@ erts_schedule_thr_prgr_later_op(void (*later_func)(void *), #ifndef ERTS_SMP later_func(later_data); #else - ErtsSchedulerData *esdp; - ErtsAuxWorkData *awdp; - int request_wakeup = 1; - - esdp = erts_get_scheduler_data(); - ASSERT(esdp); - awdp = &esdp->aux_work_data; + ErtsSchedulerData *esdp = erts_get_scheduler_data(); + ErtsThrPrgrVal later = enqueue_later_op(esdp, later_func, later_data, lop); + haw_thr_prgr_wakeup(&esdp->aux_work_data, later); +#endif +} - lop->func = later_func; - lop->data = later_data; - lop->later = erts_thr_progress_later(esdp); - lop->next = NULL; - if (!awdp->later_op.last) - awdp->later_op.first = lop; - else { - ErtsThrPrgrLaterOp *last = awdp->later_op.last; - last->next = lop; - if (erts_thr_progress_equal(last->later, lop->later)) - request_wakeup = 0; - } - awdp->later_op.last = lop; - set_aux_work_flags_wakeup_nob(awdp->ssi, - ERTS_SSI_AUX_WORK_THR_PRGR_LATER_OP); - if (request_wakeup) - erts_thr_progress_wakeup(esdp, lop->later); +void +erts_schedule_thr_prgr_later_cleanup_op(void (*later_func)(void *), + void *later_data, + ErtsThrPrgrLaterOp *lop, + UWord size) +{ +#ifndef ERTS_SMP + later_func(later_data); +#else + ErtsSchedulerData *esdp = erts_get_scheduler_data(); + ErtsThrPrgrVal later = enqueue_later_op(esdp, later_func, later_data, lop); + haw_thr_prgr_later_cleanup_op_wakeup(&esdp->aux_work_data, later, size); #endif } -#ifdef ERTS_SMP +static ERTS_INLINE erts_aint32_t +handle_debug_wait_completed(ErtsAuxWorkData *awdp, erts_aint32_t aux_work, int waiting) +{ + ErtsSchedulerSleepInfo *ssi = awdp->ssi; + erts_aint32_t saved_aux_work, flags; + +#ifdef ERTS_DIRTY_SCHEDULERS + ASSERT(!awdp->esdp || !ERTS_SCHEDULER_IS_DIRTY(awdp->esdp)); +#endif + + flags = awdp->debug.wait_completed.flags; + + if (aux_work & flags) + return aux_work; + + saved_aux_work = erts_atomic32_read_acqb(&ssi->aux_work); + + if (saved_aux_work & flags) + return aux_work & ~ERTS_SSI_AUX_WORK_DEBUG_WAIT_COMPLETED; + + awdp->debug.wait_completed.callback(awdp->debug.wait_completed.arg); + + awdp->debug.wait_completed.flags = 0; + awdp->debug.wait_completed.callback = NULL; + awdp->debug.wait_completed.arg = NULL; + + unset_aux_work_flags(ssi, ERTS_SSI_AUX_WORK_DEBUG_WAIT_COMPLETED); + + return aux_work & ~ERTS_SSI_AUX_WORK_DEBUG_WAIT_COMPLETED; +} -static erts_atomic32_t completed_dealloc_count; +static erts_atomic32_t debug_wait_completed_count; +static int debug_wait_completed_flags; static void -completed_dealloc(void *vproc) +thr_debug_wait_completed(void *vproc) { - if (erts_atomic32_dec_read_mb(&completed_dealloc_count) == 0) { + if (erts_atomic32_dec_read_mb(&debug_wait_completed_count) == 0) { erts_resume((Process *) vproc, (ErtsProcLocks) 0); - erts_smp_proc_dec_refc((Process *) vproc); + erts_proc_dec_refc((Process *) vproc); } } static void -setup_completed_dealloc(void *vproc) +setup_thr_debug_wait_completed(void *vproc) { ErtsSchedulerData *esdp = erts_get_scheduler_data(); - ErtsAuxWorkData *awdp = (esdp - ? &esdp->aux_work_data - : aux_thread_aux_work_data); - erts_alloc_fix_alloc_shrink(awdp->sched_id, 0); - set_aux_work_flags_wakeup_nob(awdp->ssi, ERTS_SSI_AUX_WORK_DD); - awdp->dd.completed_callback = completed_dealloc; - awdp->dd.completed_arg = vproc; + ErtsAuxWorkData *awdp; + erts_aint32_t wait_flags, aux_work_flags; +#ifdef ERTS_SMP + awdp = esdp ? &esdp->aux_work_data : aux_thread_aux_work_data; +#else + awdp = &esdp->aux_work_data; +#endif + + wait_flags = 0; + aux_work_flags = ERTS_SSI_AUX_WORK_DEBUG_WAIT_COMPLETED; + + if (debug_wait_completed_flags & ERTS_DEBUG_WAIT_COMPLETED_DEALLOCATIONS) { + erts_alloc_fix_alloc_shrink(awdp->sched_id, 0); + wait_flags |= (ERTS_SSI_AUX_WORK_DD + | ERTS_SSI_AUX_WORK_DD_THR_PRGR + | ERTS_SSI_AUX_WORK_THR_PRGR_LATER_OP); +#ifdef ERTS_SMP + aux_work_flags |= ERTS_SSI_AUX_WORK_DD; +#endif + } + + if (debug_wait_completed_flags & ERTS_DEBUG_WAIT_COMPLETED_TIMER_CANCELLATIONS) { + wait_flags |= (ERTS_SSI_AUX_WORK_CNCLD_TMRS + | ERTS_SSI_AUX_WORK_CNCLD_TMRS_THR_PRGR + | ERTS_SSI_AUX_WORK_THR_PRGR_LATER_OP); +#ifdef ERTS_SMP + if (awdp->esdp && !ERTS_SCHEDULER_IS_DIRTY(awdp->esdp)) + aux_work_flags |= ERTS_SSI_AUX_WORK_CNCLD_TMRS; +#endif + } + + set_aux_work_flags_wakeup_nob(awdp->ssi, aux_work_flags); + + awdp->debug.wait_completed.flags = wait_flags; + awdp->debug.wait_completed.callback = thr_debug_wait_completed; + awdp->debug.wait_completed.arg = vproc; } static void -prep_setup_completed_dealloc(void *vproc) +prep_setup_thr_debug_wait_completed(void *vproc) { - erts_aint32_t count = (erts_aint32_t) (erts_no_schedulers+1); - if (erts_atomic32_dec_read_mb(&completed_dealloc_count) == count) { + erts_aint32_t count = (erts_aint32_t) erts_no_schedulers; +#ifdef ERTS_SMP + count += 1; /* aux thread */ +#endif + if (erts_atomic32_dec_read_mb(&debug_wait_completed_count) == count) { /* scheduler threads */ erts_schedule_multi_misc_aux_work(0, erts_no_schedulers, - setup_completed_dealloc, + setup_thr_debug_wait_completed, vproc); +#ifdef ERTS_SMP /* aux_thread */ erts_schedule_misc_aux_work(0, - setup_completed_dealloc, + setup_thr_debug_wait_completed, vproc); +#endif } } -#endif /* ERTS_SMP */ int -erts_debug_wait_deallocations(Process *c_p) +erts_debug_wait_completed(Process *c_p, int flags) { -#ifndef ERTS_SMP - erts_alloc_fix_alloc_shrink(1, 0); - return 1; -#else /* Only one process at a time can do this */ - erts_aint32_t count = (erts_aint32_t) (2*(erts_no_schedulers+1)); - if (0 == erts_atomic32_cmpxchg_mb(&completed_dealloc_count, + erts_aint32_t count = (erts_aint32_t) (2*erts_no_schedulers); +#ifdef ERTS_SMP + count += 2; /* aux thread */ +#endif + if (0 == erts_atomic32_cmpxchg_mb(&debug_wait_completed_count, count, 0)) { + debug_wait_completed_flags = flags; erts_suspend(c_p, ERTS_PROC_LOCK_MAIN, NULL); - erts_smp_proc_inc_refc(c_p); + erts_proc_inc_refc(c_p); /* scheduler threads */ erts_schedule_multi_misc_aux_work(0, erts_no_schedulers, - prep_setup_completed_dealloc, + prep_setup_thr_debug_wait_completed, (void *) c_p); +#ifdef ERTS_SMP /* aux_thread */ erts_schedule_misc_aux_work(0, - prep_setup_completed_dealloc, + prep_setup_thr_debug_wait_completed, (void *) c_p); +#endif return 1; } return 0; -#endif } @@ -1507,6 +2224,14 @@ erts_smp_notify_check_children_needed(void) for (i = 0; i < erts_no_schedulers; i++) set_aux_work_flags_wakeup_nob(ERTS_SCHED_SLEEP_INFO_IX(i), ERTS_SSI_AUX_WORK_CHECK_CHILDREN); +#ifdef ERTS_DIRTY_SCHEDULERS + for (i = 0; i < erts_no_dirty_cpu_schedulers; i++) + set_aux_work_flags_wakeup_nob(ERTS_DIRTY_CPU_SCHED_SLEEP_INFO_IX(i), + ERTS_SSI_AUX_WORK_CHECK_CHILDREN); + for (i = 0; i < erts_no_dirty_io_schedulers; i++) + set_aux_work_flags_wakeup_nob(ERTS_DIRTY_IO_SCHED_SLEEP_INFO_IX(i), + ERTS_SSI_AUX_WORK_CHECK_CHILDREN); +#endif } static ERTS_INLINE erts_aint32_t @@ -1567,7 +2292,7 @@ handle_reap_ports(ErtsAuxWorkData *awdp, erts_aint32_t aux_work, int waiting) erts_port_release(prt); } if (erts_smp_atomic32_dec_read_nob(&erts_halt_progress) == 0) { - erl_exit_flush_async(erts_halt_code, ""); + erts_flush_async_exit(erts_halt_code, ""); } } return aux_work & ~ERTS_SSI_AUX_WORK_REAP_PORTS; @@ -1589,7 +2314,7 @@ static ERTS_INLINE erts_aint32_t handle_setup_aux_work_timer(ErtsAuxWorkData *awdp, erts_aint32_t aux_work, int waiting) { unset_aux_work_flags(awdp->ssi, ERTS_SSI_AUX_WORK_SET_TMO); - setup_aux_work_timer(); + setup_aux_work_timer(awdp->esdp); return aux_work & ~ERTS_SSI_AUX_WORK_SET_TMO; } @@ -1611,6 +2336,7 @@ handle_aux_work(ErtsAuxWorkData *awdp, erts_aint32_t orig_aux_work, int waiting) erts_aint32_t aux_work = orig_aux_work; erts_aint32_t ignore = 0; + ASSERT(!awdp->esdp || !ERTS_SCHEDULER_IS_DIRTY(awdp->esdp)); #ifdef ERTS_SMP haw_thr_prgr_current_reset(awdp); #endif @@ -1649,6 +2375,11 @@ handle_aux_work(ErtsAuxWorkData *awdp, erts_aint32_t orig_aux_work, int waiting) #ifdef ERTS_SMP HANDLE_AUX_WORK(ERTS_SSI_AUX_WORK_THR_PRGR_LATER_OP, handle_thr_prgr_later_op); + HANDLE_AUX_WORK(ERTS_SSI_AUX_WORK_CNCLD_TMRS, + handle_canceled_timers); + /* CNCLD_TMRS must be before CNCLD_TMRS_THR_PRGR */ + HANDLE_AUX_WORK(ERTS_SSI_AUX_WORK_CNCLD_TMRS_THR_PRGR, + handle_canceled_timers_thr_prgr); #endif #if ERTS_USE_ASYNC_READY_Q @@ -1683,6 +2414,14 @@ handle_aux_work(ErtsAuxWorkData *awdp, erts_aint32_t orig_aux_work, int waiting) HANDLE_AUX_WORK(ERTS_SSI_AUX_WORK_REAP_PORTS, handle_reap_ports); + /* + * ERTS_SSI_AUX_WORK_DEBUG_WAIT_COMPLETED *need* to be + * the last flag checked! + */ + + HANDLE_AUX_WORK(ERTS_SSI_AUX_WORK_DEBUG_WAIT_COMPLETED, + handle_debug_wait_completed); + ERTS_DBG_CHK_AUX_WORK_VAL(aux_work); #ifdef ERTS_SMP @@ -1698,8 +2437,8 @@ handle_aux_work(ErtsAuxWorkData *awdp, erts_aint32_t orig_aux_work, int waiting) typedef struct { union { - ErlTimer data; - char align__[ERTS_ALC_CACHE_LINE_ALIGN_SIZE(sizeof(ErlTimer))]; + ErtsTWheelTimer data; + char align__[ERTS_ALC_CACHE_LINE_ALIGN_SIZE(sizeof(ErtsTWheelTimer))]; } timer; int initialized; @@ -1709,6 +2448,22 @@ typedef struct { static ErtsAuxWorkTmo *aux_work_tmo; +static ERTS_INLINE void +start_aux_work_timer(ErtsSchedulerData *esdp) +{ + ErtsMonotonicTime tmo = erts_get_monotonic_time(esdp); + tmo = ERTS_MONOTONIC_TO_CLKTCKS(tmo-1); + tmo += ERTS_MSEC_TO_CLKTCKS(1000) + 1; + erts_twheel_init_timer(&aux_work_tmo->timer.data); + ASSERT(esdp); + erts_twheel_set_timer(esdp->timer_wheel, + &aux_work_tmo->timer.data, + aux_work_timeout, + NULL, + (void *) esdp, + tmo); +} + static void aux_work_timeout_early_init(int no_schedulers) { @@ -1723,6 +2478,9 @@ aux_work_timeout_early_init(int no_schedulers) p = (UWord) malloc((sizeof(ErtsAuxWorkTmo) + sizeof(erts_atomic32_t)*(no_schedulers+1)) + ERTS_CACHE_LINE_SIZE-1); + if (!p) { + ERTS_INTERNAL_ERROR("malloc failed to allocate memory!"); + } if (p & ERTS_CACHE_LINE_MASK) p = (p & ~ERTS_CACHE_LINE_MASK) + ERTS_CACHE_LINE_SIZE; ASSERT((p & ERTS_CACHE_LINE_MASK) == 0); @@ -1738,18 +2496,12 @@ void aux_work_timeout_late_init(void) { aux_work_tmo->initialized = 1; - if (erts_atomic32_read_nob(&aux_work_tmo->refc)) { - aux_work_tmo->timer.data.active = 0; - erts_set_timer(&aux_work_tmo->timer.data, - aux_work_timeout, - NULL, - NULL, - 1000); - } + if (erts_atomic32_read_nob(&aux_work_tmo->refc)) + start_aux_work_timer(erts_get_scheduler_data()); } static void -aux_work_timeout(void *unused) +aux_work_timeout(void *vesdp) { erts_aint32_t refc; int i; @@ -1772,32 +2524,18 @@ aux_work_timeout(void *unused) if (refc != 1 || 1 != erts_atomic32_cmpxchg_relb(&aux_work_tmo->refc, 0, 1)) { /* Setup next timeout... */ - aux_work_tmo->timer.data.active = 0; - erts_set_timer(&aux_work_tmo->timer.data, - aux_work_timeout, - NULL, - NULL, - 1000); + start_aux_work_timer((ErtsSchedulerData *) vesdp); } } static void -setup_aux_work_timer(void) +setup_aux_work_timer(ErtsSchedulerData *esdp) { -#ifndef ERTS_SMP - if (!erts_get_scheduler_data()) + if (!esdp || !esdp->timer_wheel) set_aux_work_flags_wakeup_nob(ERTS_SCHED_SLEEP_INFO_IX(0), ERTS_SSI_AUX_WORK_SET_TMO); else -#endif - { - aux_work_tmo->timer.data.active = 0; - erts_set_timer(&aux_work_tmo->timer.data, - aux_work_timeout, - NULL, - NULL, - 1000); - } + start_aux_work_timer(esdp); } erts_aint32_t @@ -1811,7 +2549,7 @@ erts_set_aux_work_timeout(int ix, erts_aint32_t type, int enable) ERTS_DBG_CHK_AUX_WORK_VAL(type); ERTS_DBG_CHK_AUX_WORK_VAL(erts_atomic32_read_nob(&aux_work_tmo->type[ix])); -// erts_fprintf(stderr, "t(%d, 0x%x, %d)\n", ix, type, enable); + /* erts_fprintf(stderr, "t(%d, 0x%x, %d)\n", ix, type, enable); */ if (!enable) { old = erts_atomic32_read_band_mb(&aux_work_tmo->type[ix], ~type); @@ -1828,7 +2566,7 @@ erts_set_aux_work_timeout(int ix, erts_aint32_t type, int enable) if (refc == 1) { erts_atomic32_inc_acqb(&aux_work_tmo->refc); if (aux_work_tmo->initialized) - setup_aux_work_timer(); + setup_aux_work_timer(erts_get_scheduler_data()); } } return old; @@ -1854,6 +2592,9 @@ static ERTS_INLINE void sched_active_sys(Uint no, ErtsRunQueue *rq) { ERTS_SMP_LC_ASSERT(erts_smp_lc_runq_is_locked(rq)); +#ifdef ERTS_DIRTY_SCHEDULERS + ASSERT(!ERTS_RUNQ_IX_IS_DIRTY(rq->ix)); +#endif ASSERT(rq->waiting < 0); rq->waiting *= -1; rq->waiting--; @@ -1868,7 +2609,6 @@ erts_active_schedulers(void) ERTS_ATOMIC_FOREACH_RUNQ(rq, as -= abs(rq->waiting)); - ASSERT(as >= 0); return as; } @@ -1889,19 +2629,47 @@ try_set_sys_scheduling(void) #endif static ERTS_INLINE int -prepare_for_sys_schedule(void) +prepare_for_sys_schedule(ErtsSchedulerData *esdp, int non_blocking) { + if (non_blocking && erts_eager_check_io) { #ifdef ERTS_SMP - while (!erts_port_task_have_outstanding_io_tasks() - && try_set_sys_scheduling()) { - if (!erts_port_task_have_outstanding_io_tasks()) - return 1; - clear_sys_scheduling(); +#ifdef ERTS_SCHED_ONLY_POLL_SCHED_1 + if (esdp->no != 1) { + /* If we are not scheduler 1 and ERTS_SCHED_ONLY_POLL_SCHED_1 is used + then we make sure to wake scheduler 1 */ + ErtsRunQueue *rq = ERTS_RUNQ_IX(0); + wake_scheduler(rq); + return 0; + } +#endif + return try_set_sys_scheduling(); +#else + return 1; +#endif } - return 0; + else { +#ifdef ERTS_SMP + while (!erts_port_task_have_outstanding_io_tasks() + && try_set_sys_scheduling()) { +#ifdef ERTS_SCHED_ONLY_POLL_SCHED_1 + if (esdp->no != 1) { + /* If we are not scheduler 1 and ERTS_SCHED_ONLY_POLL_SCHED_1 is used + then we make sure to wake scheduler 1 */ + ErtsRunQueue *rq = ERTS_RUNQ_IX(0); + clear_sys_scheduling(); + wake_scheduler(rq); + return 0; + } +#endif + if (!erts_port_task_have_outstanding_io_tasks()) + return 1; + clear_sys_scheduling(); + } + return 0; #else - return !erts_port_task_have_outstanding_io_tasks(); + return !erts_port_task_have_outstanding_io_tasks(); #endif + } } #ifdef ERTS_SMP @@ -1910,6 +2678,9 @@ static ERTS_INLINE void sched_change_waiting_sys_to_waiting(Uint no, ErtsRunQueue *rq) { ERTS_SMP_LC_ASSERT(erts_smp_lc_runq_is_locked(rq)); +#ifdef ERTS_DIRTY_SCHEDULERS + ASSERT(!ERTS_RUNQ_IX_IS_DIRTY(rq->ix)); +#endif ASSERT(rq->waiting < 0); rq->waiting *= -1; } @@ -1925,7 +2696,7 @@ sched_waiting(Uint no, ErtsRunQueue *rq) else rq->waiting++; rq->woken = 0; - if (erts_system_profile_flags.scheduler) + if (!ERTS_RUNQ_IX_IS_DIRTY(rq->ix) && erts_system_profile_flags.scheduler) profile_scheduler(make_small(no), am_inactive); } @@ -1937,7 +2708,7 @@ sched_active(Uint no, ErtsRunQueue *rq) rq->waiting++; else rq->waiting--; - if (erts_system_profile_flags.scheduler) + if (!ERTS_RUNQ_IX_IS_DIRTY(rq->ix) && erts_system_profile_flags.scheduler) profile_scheduler(make_small(no), am_active); } @@ -1949,10 +2720,9 @@ ongoing_multi_scheduling_block(void) } static ERTS_INLINE void -empty_runq(ErtsRunQueue *rq) +empty_runq_aux(ErtsRunQueue *rq, Uint32 old_flags) { - Uint32 old_flags = ERTS_RUNQ_FLGS_UNSET(rq, ERTS_RUNQ_FLG_NONEMPTY|ERTS_RUNQ_FLG_PROTECTED); - if (old_flags & ERTS_RUNQ_FLG_NONEMPTY) { + if (!ERTS_RUNQ_IX_IS_DIRTY(rq->ix) && old_flags & ERTS_RUNQ_FLG_NONEMPTY) { #ifdef DEBUG erts_aint32_t empty = erts_smp_atomic32_read_nob(&no_empty_run_queues); /* @@ -1961,15 +2731,38 @@ empty_runq(ErtsRunQueue *rq) */ ASSERT(0 <= empty && empty < 2*erts_no_run_queues); #endif - erts_smp_atomic32_inc_relb(&no_empty_run_queues); + if (!erts_runq_supervision_interval) + erts_smp_atomic32_inc_relb(&no_empty_run_queues); + else { + erts_smp_atomic32_inc_mb(&no_empty_run_queues); + if (erts_atomic_read_nob(&runq_supervisor_sleeping)) + ethr_event_set(&runq_supervision_event); + } } } static ERTS_INLINE void +empty_runq(ErtsRunQueue *rq) +{ + Uint32 old_flags = ERTS_RUNQ_FLGS_UNSET(rq, ERTS_RUNQ_FLG_NONEMPTY|ERTS_RUNQ_FLG_PROTECTED); + empty_runq_aux(rq, old_flags); +} + +static ERTS_INLINE Uint32 +empty_protected_runq(ErtsRunQueue *rq) +{ + Uint32 old_flags = ERTS_RUNQ_FLGS_BSET(rq, + ERTS_RUNQ_FLG_NONEMPTY|ERTS_RUNQ_FLG_PROTECTED, + ERTS_RUNQ_FLG_PROTECTED); + empty_runq_aux(rq, old_flags); + return old_flags; +} + +static ERTS_INLINE void non_empty_runq(ErtsRunQueue *rq) { Uint32 old_flags = ERTS_RUNQ_FLGS_SET(rq, ERTS_RUNQ_FLG_NONEMPTY); - if (!(old_flags & ERTS_RUNQ_FLG_NONEMPTY)) { + if (!ERTS_RUNQ_IX_IS_DIRTY(rq->ix) && (!(old_flags & ERTS_RUNQ_FLG_NONEMPTY))) { #ifdef DEBUG erts_aint32_t empty = erts_smp_atomic32_read_nob(&no_empty_run_queues); /* @@ -1978,10 +2771,29 @@ non_empty_runq(ErtsRunQueue *rq) */ ASSERT(0 < empty && empty <= 2*erts_no_run_queues); #endif - erts_smp_atomic32_dec_relb(&no_empty_run_queues); + if (!erts_runq_supervision_interval) + erts_smp_atomic32_dec_relb(&no_empty_run_queues); + else { + erts_aint32_t no; + no = erts_smp_atomic32_dec_read_mb(&no_empty_run_queues); + if (no > 0 && erts_atomic_read_nob(&runq_supervisor_sleeping)) + ethr_event_set(&runq_supervision_event); + } } } +void +erts_empty_runq(ErtsRunQueue *rq) +{ + empty_runq(rq); +} + +void +erts_non_empty_runq(ErtsRunQueue *rq) +{ + non_empty_runq(rq); +} + static erts_aint32_t sched_prep_spin_wait(ErtsSchedulerSleepInfo *ssi) { @@ -2118,6 +2930,13 @@ thr_prgr_fin_wait(void *vssi) static void init_aux_work_data(ErtsAuxWorkData *awdp, ErtsSchedulerData *esdp, char *dawwp); +void +erts_interupt_aux_thread_timed(ErtsMonotonicTime timeout_time) +{ + /* TODO only poke when needed (based on timeout_time) */ + erts_sched_poke(ERTS_SCHED_SLEEP_INFO_IX(-1)); +} + static void * aux_thread(void *unused) { @@ -2127,6 +2946,13 @@ aux_thread(void *unused) ErtsThrPrgrCallbacks callbacks; int thr_prgr_active = 1; +#ifdef ERTS_ENABLE_LOCK_CHECK + { + char buf[] = "aux_thread"; + erts_lc_set_thread_name(buf); + } +#endif + ssi->event = erts_tse_fetch(); callbacks.arg = (void *) ssi; @@ -2158,6 +2984,8 @@ aux_thread(void *unused) erts_thr_progress_active(NULL, thr_prgr_active = 0); erts_thr_progress_prepare_wait(NULL); + ERTS_SCHED_FAIR_YIELD(); + flgs = sched_spin_wait(ssi, 0); if (flgs & ERTS_SSI_FLG_SLEEPING) { @@ -2195,18 +3023,37 @@ scheduler_wait(int *fcalls, ErtsSchedulerData *esdp, ErtsRunQueue *rq) ERTS_SMP_LC_ASSERT(erts_smp_lc_runq_is_locked(rq)); +#ifdef ERTS_DIRTY_SCHEDULERS + if (ERTS_RUNQ_IX_IS_DIRTY(rq->ix)) + erts_smp_spin_lock(&rq->sleepers.lock); +#endif flgs = sched_prep_spin_wait(ssi); if (flgs & ERTS_SSI_FLG_SUSPENDED) { /* Go suspend instead... */ +#ifdef ERTS_DIRTY_SCHEDULERS + if (ERTS_RUNQ_IX_IS_DIRTY(rq->ix)) + erts_smp_spin_unlock(&rq->sleepers.lock); +#endif return; } +#ifdef ERTS_DIRTY_SCHEDULERS + if (ERTS_RUNQ_IX_IS_DIRTY(rq->ix)) { + ssi->prev = NULL; + ssi->next = rq->sleepers.list; + if (rq->sleepers.list) + rq->sleepers.list->prev = ssi; + rq->sleepers.list = ssi; + erts_smp_spin_unlock(&rq->sleepers.lock); + } +#endif + /* * If all schedulers are waiting, one of them *should* * be waiting in erl_sys_schedule() */ - if (!prepare_for_sys_schedule()) { + if (ERTS_SCHEDULER_IS_DIRTY(esdp) || !prepare_for_sys_schedule(esdp, 0)) { sched_waiting(esdp->no, rq); @@ -2216,13 +3063,14 @@ scheduler_wait(int *fcalls, ErtsSchedulerData *esdp, ErtsRunQueue *rq) tse_wait: - if (thr_prgr_active != working) + if (!ERTS_SCHEDULER_IS_DIRTY(esdp) && thr_prgr_active != working) sched_wall_time_change(esdp, thr_prgr_active); while (1) { + ErtsMonotonicTime current_time; aux_work = erts_atomic32_read_acqb(&ssi->aux_work); - if (aux_work) { + if (aux_work && !ERTS_SCHEDULER_IS_DIRTY(esdp)) { if (!thr_prgr_active) { erts_thr_progress_active(esdp, thr_prgr_active = 1); sched_wall_time_change(esdp, 1); @@ -2232,29 +3080,76 @@ scheduler_wait(int *fcalls, ErtsSchedulerData *esdp, ErtsRunQueue *rq) erts_thr_progress_leader_update(esdp); } - if (aux_work) - flgs = erts_smp_atomic32_read_acqb(&ssi->flags); + if (aux_work) { + if (!ERTS_SCHEDULER_IS_DIRTY(esdp)) { + flgs = erts_smp_atomic32_read_acqb(&ssi->flags); + current_time = erts_get_monotonic_time(esdp); + if (current_time >= erts_next_timeout_time(esdp->next_tmo_ref)) { + if (!thr_prgr_active) { + erts_thr_progress_active(esdp, thr_prgr_active = 1); + sched_wall_time_change(esdp, 1); + } + erts_bump_timers(esdp->timer_wheel, current_time); + } + } + } else { - if (thr_prgr_active) { - erts_thr_progress_active(esdp, thr_prgr_active = 0); - sched_wall_time_change(esdp, 0); + ErtsMonotonicTime timeout_time; + int do_timeout = 0; + if (!ERTS_SCHEDULER_IS_DIRTY(esdp)) { + timeout_time = erts_check_next_timeout_time(esdp); + current_time = erts_get_monotonic_time(esdp); + do_timeout = (current_time >= timeout_time); + } else + timeout_time = ERTS_MONOTONIC_TIME_MAX; + if (do_timeout) { + if (!thr_prgr_active) { + erts_thr_progress_active(esdp, thr_prgr_active = 1); + sched_wall_time_change(esdp, 1); + } } - erts_thr_progress_prepare_wait(esdp); + else { + if (!ERTS_SCHEDULER_IS_DIRTY(esdp)) { + if (thr_prgr_active) { + erts_thr_progress_active(esdp, thr_prgr_active = 0); + sched_wall_time_change(esdp, 0); + } + erts_thr_progress_prepare_wait(esdp); + } - flgs = sched_spin_wait(ssi, spincount); - if (flgs & ERTS_SSI_FLG_SLEEPING) { - ASSERT(flgs & ERTS_SSI_FLG_WAITING); - flgs = sched_set_sleeptype(ssi, ERTS_SSI_FLG_TSE_SLEEPING); + ERTS_SCHED_FAIR_YIELD(); + + flgs = sched_spin_wait(ssi, spincount); if (flgs & ERTS_SSI_FLG_SLEEPING) { - int res; - ASSERT(flgs & ERTS_SSI_FLG_TSE_SLEEPING); ASSERT(flgs & ERTS_SSI_FLG_WAITING); - do { - res = erts_tse_wait(ssi->event); - } while (res == EINTR); + flgs = sched_set_sleeptype(ssi, ERTS_SSI_FLG_TSE_SLEEPING); + if (flgs & ERTS_SSI_FLG_SLEEPING) { + int res; + ASSERT(flgs & ERTS_SSI_FLG_TSE_SLEEPING); + ASSERT(flgs & ERTS_SSI_FLG_WAITING); + current_time = ERTS_SCHEDULER_IS_DIRTY(esdp) ? 0 : + erts_get_monotonic_time(esdp); + do { + Sint64 timeout; + if (current_time >= timeout_time) + break; + if (!ERTS_SCHEDULER_IS_DIRTY(esdp)) { + timeout = ERTS_MONOTONIC_TO_NSEC(timeout_time + - current_time + - 1) + 1; + } else + timeout = -1; + res = erts_tse_twait(ssi->event, timeout); + current_time = ERTS_SCHEDULER_IS_DIRTY(esdp) ? 0 : + erts_get_monotonic_time(esdp); + } while (res == EINTR); + } } + if (!ERTS_SCHEDULER_IS_DIRTY(esdp)) + erts_thr_progress_finalize_wait(esdp); } - erts_thr_progress_finalize_wait(esdp); + if (!ERTS_SCHEDULER_IS_DIRTY(esdp) && current_time >= timeout_time) + erts_bump_timers(esdp->timer_wheel, current_time); } if (!(flgs & ERTS_SSI_FLG_WAITING)) { @@ -2275,7 +3170,7 @@ scheduler_wait(int *fcalls, ErtsSchedulerData *esdp, ErtsRunQueue *rq) if (flgs & ~ERTS_SSI_FLG_SUSPENDED) erts_smp_atomic32_read_band_nob(&ssi->flags, ERTS_SSI_FLG_SUSPENDED); - if (!thr_prgr_active) { + if (!ERTS_SCHEDULER_IS_DIRTY(esdp) && !thr_prgr_active) { erts_thr_progress_active(esdp, thr_prgr_active = 1); sched_wall_time_change(esdp, 1); } @@ -2287,11 +3182,17 @@ scheduler_wait(int *fcalls, ErtsSchedulerData *esdp, ErtsRunQueue *rq) else #endif { - erts_aint_t dt; erts_smp_atomic32_set_relb(&function_calls, 0); *fcalls = 0; +#ifdef ERTS_DIRTY_SCHEDULERS + ASSERT(!ERTS_SCHEDULER_IS_DIRTY(esdp)); +#endif + +#ifdef ERTS_SCHED_ONLY_POLL_SCHED_1 + ASSERT(esdp->no == 1); +#endif sched_waiting_sys(esdp->no, rq); @@ -2305,6 +3206,7 @@ scheduler_wait(int *fcalls, ErtsSchedulerData *esdp, ErtsRunQueue *rq) goto sys_aux_work; while (spincount-- > 0) { + ErtsMonotonicTime current_time; sys_poll_aux_work: @@ -2312,11 +3214,13 @@ scheduler_wait(int *fcalls, ErtsSchedulerData *esdp, ErtsRunQueue *rq) sched_wall_time_change(esdp, working = 0); ASSERT(!erts_port_task_have_outstanding_io_tasks()); - erl_sys_schedule(1); /* Might give us something to do */ - dt = erts_do_time_read_and_reset(); - if (dt) erts_bump_timer(dt); + if (!ERTS_SCHEDULER_IS_DIRTY(esdp)) { + current_time = erts_get_monotonic_time(esdp); + if (current_time >= erts_next_timeout_time(esdp->next_tmo_ref)) + erts_bump_timers(esdp->timer_wheel, current_time); + } sys_aux_work: #ifndef ERTS_SMP @@ -2324,7 +3228,7 @@ scheduler_wait(int *fcalls, ErtsSchedulerData *esdp, ErtsRunQueue *rq) #endif aux_work = erts_atomic32_read_acqb(&ssi->aux_work); - if (aux_work) { + if (aux_work && !ERTS_SCHEDULER_IS_DIRTY(esdp)) { if (!working) sched_wall_time_change(esdp, working = 1); #ifdef ERTS_SMP @@ -2339,7 +3243,7 @@ scheduler_wait(int *fcalls, ErtsSchedulerData *esdp, ErtsRunQueue *rq) } #ifndef ERTS_SMP - if (rq->len != 0 || rq->misc.start) + if (erts_smp_atomic32_read_dirty(&rq->len) != 0 || rq->misc.start) goto sys_woken; #else flgs = erts_smp_atomic32_read_acqb(&ssi->flags); @@ -2358,7 +3262,7 @@ scheduler_wait(int *fcalls, ErtsSchedulerData *esdp, ErtsRunQueue *rq) * Got to check that we still got I/O tasks; otherwise * we have to continue checking for I/O... */ - if (!prepare_for_sys_schedule()) { + if (!prepare_for_sys_schedule(esdp, 0)) { spincount *= ERTS_SCHED_TSE_SLEEP_SPINCOUNT_FACT; goto tse_wait; } @@ -2380,7 +3284,7 @@ scheduler_wait(int *fcalls, ErtsSchedulerData *esdp, ErtsRunQueue *rq) * Got to check that we still got I/O tasks; otherwise * we have to wait in erl_sys_schedule() after all... */ - if (!prepare_for_sys_schedule()) { + if (!prepare_for_sys_schedule(esdp, 0)) { /* * Not allowed to wait in erl_sys_schedule; * do tse wait instead... @@ -2431,11 +3335,14 @@ scheduler_wait(int *fcalls, ErtsSchedulerData *esdp, ErtsRunQueue *rq) erl_sys_schedule(0); - dt = erts_do_time_read_and_reset(); - if (dt) erts_bump_timer(dt); + if (!ERTS_SCHEDULER_IS_DIRTY(esdp)) { + ErtsMonotonicTime current_time = erts_get_monotonic_time(esdp); + if (current_time >= erts_next_timeout_time(esdp->next_tmo_ref)) + erts_bump_timers(esdp->timer_wheel, current_time); + } #ifndef ERTS_SMP - if (rq->len == 0 && !rq->misc.start) + if (erts_smp_atomic32_read_dirty(&rq->len) == 0 && !rq->misc.start) goto sys_aux_work; sys_woken: #else @@ -2484,7 +3391,7 @@ ssi_flags_set_wake(ErtsSchedulerSleepInfo *ssi) } static void -wake_scheduler(ErtsRunQueue *rq, int incq) +wake_scheduler(ErtsRunQueue *rq) { ErtsSchedulerSleepInfo *ssi; erts_aint32_t flgs; @@ -2503,10 +3410,53 @@ wake_scheduler(ErtsRunQueue *rq, int incq) flgs = ssi_flags_set_wake(ssi); erts_sched_finish_poke(ssi, flgs); +} + +#ifdef ERTS_DIRTY_SCHEDULERS +static void +wake_dirty_schedulers(ErtsRunQueue *rq, int one) +{ + ErtsSchedulerSleepInfo *ssi; + ErtsSchedulerSleepList *sl; + + ASSERT(ERTS_RUNQ_IX_IS_DIRTY(rq->ix)); + + sl = &rq->sleepers; + erts_smp_spin_lock(&sl->lock); + ssi = sl->list; + if (!ssi) { + erts_smp_spin_unlock(&sl->lock); + if (one) + wake_scheduler(rq); + } else if (one) { + erts_aint32_t flgs; + if (ssi->prev) + ssi->prev->next = ssi->next; + else { + ASSERT(sl->list == ssi); + sl->list = ssi->next; + } + if (ssi->next) + ssi->next->prev = ssi->prev; + + erts_smp_spin_unlock(&sl->lock); + + ERTS_THR_MEMORY_BARRIER; + flgs = ssi_flags_set_wake(ssi); + erts_sched_finish_poke(ssi, flgs); + } else { + sl->list = NULL; + erts_smp_spin_unlock(&sl->lock); - if (incq && (flgs & ERTS_SSI_FLG_WAITING)) - non_empty_runq(rq); + ERTS_THR_MEMORY_BARRIER; + do { + ErtsSchedulerSleepInfo *wake_ssi = ssi; + ssi = ssi->next; + erts_sched_finish_poke(wake_ssi, ssi_flags_set_wake(wake_ssi)); + } while (ssi); + } } +#endif #define ERTS_NO_USED_RUNQS_SHIFT 16 #define ERTS_NO_RUNQS_MASK 0xffff @@ -2554,6 +3504,8 @@ set_no_active_runqs(int active) erts_aint32_t exp = erts_smp_atomic32_read_nob(&balance_info.no_runqs); while (1) { erts_aint32_t act, new; + if ((exp & ERTS_NO_RUNQS_MASK) == active) + break; new = exp & (ERTS_NO_RUNQS_MASK << ERTS_NO_USED_RUNQS_SHIFT); new |= active & ERTS_NO_RUNQS_MASK; act = erts_smp_atomic32_cmpxchg_nob(&balance_info.no_runqs, new, exp); @@ -2580,7 +3532,6 @@ try_inc_no_active_runqs(int active) return 0; } - static ERTS_INLINE int chk_wake_sched(ErtsRunQueue *crq, int ix, int activate) { @@ -2590,12 +3541,12 @@ chk_wake_sched(ErtsRunQueue *crq, int ix, int activate) return 0; wrq = ERTS_RUNQ_IX(ix); flags = ERTS_RUNQ_FLGS_GET(wrq); + if (activate && !(flags & ERTS_RUNQ_FLG_SUSPENDED)) { + if (try_inc_no_active_runqs(ix+1)) + (void) ERTS_RUNQ_FLGS_UNSET(wrq, ERTS_RUNQ_FLG_INACTIVE); + } if (!(flags & (ERTS_RUNQ_FLG_SUSPENDED|ERTS_RUNQ_FLG_NONEMPTY))) { - if (activate) { - if (try_inc_no_active_runqs(ix+1)) - (void) ERTS_RUNQ_FLGS_UNSET(wrq, ERTS_RUNQ_FLG_INACTIVE); - } - wake_scheduler(wrq, 0); + wake_scheduler(wrq); return 1; } return 0; @@ -2642,8 +3593,14 @@ static ERTS_INLINE void smp_notify_inc_runq(ErtsRunQueue *runq) { #ifdef ERTS_SMP - if (runq) - wake_scheduler(runq, 1); + if (runq) { +#ifdef ERTS_DIRTY_SCHEDULERS + if (ERTS_RUNQ_IX_IS_DIRTY(runq->ix)) + wake_dirty_schedulers(runq, 1); + else +#endif + wake_scheduler(runq); + } #endif } @@ -2661,7 +3618,7 @@ erts_sched_notify_check_cpu_bind(void) for (ix = 0; ix < erts_no_run_queues; ix++) { ErtsRunQueue *rq = ERTS_RUNQ_IX(ix); (void) ERTS_RUNQ_FLGS_SET(rq, ERTS_RUNQ_FLG_CHK_CPU_BIND); - wake_scheduler(rq, 0); + wake_scheduler(rq); } #else erts_sched_check_cpu_bind(erts_get_scheduler_data()); @@ -2746,7 +3703,7 @@ dequeue_process(ErtsRunQueue *runq, int prio_q, erts_aint32_t *statep) if (statep) *statep = state; - prio = (int) (ERTS_PSFLG_PRIO_MASK & state); + prio = (int) ERTS_PSFLGS_GET_PRQ_PRIO(state); rqi = &runq->procs.prio_info[prio]; @@ -2789,6 +3746,11 @@ check_immigration_need(ErtsRunQueue *c_rq, ErtsMigrationPath *mp, int prio) if (!f_rq) return NULL; +#if ERTS_HAVE_SCHED_UTIL_BALANCING_SUPPORT + if (mp->sched_util) + return NULL; +#endif + f_rq_flags = ERTS_RUNQ_FLGS_GET(f_rq); if (f_rq_flags & ERTS_RUNQ_FLG_PROTECTED) return NULL; @@ -2849,7 +3811,7 @@ immigrate(ErtsRunQueue *c_rq, ErtsMigrationPath *mp) prio = ERTS_PORT_PRIO_LEVEL; break; default: - erl_exit(ERTS_ABORT_EXIT, + erts_exit(ERTS_ABORT_EXIT, "%s:%d:%s(): Invalid immigrate queue mask", __FILE__, __LINE__, __func__); prio = 0; @@ -2873,7 +3835,7 @@ immigrate(ErtsRunQueue *c_rq, ErtsMigrationPath *mp) rq = erts_port_runq(prt); if (rq) { if (rq != c_rq) - erl_exit(ERTS_ABORT_EXIT, + erts_exit(ERTS_ABORT_EXIT, "%s:%d:%s(): Internal error", __FILE__, __LINE__, __func__); erts_enqueue_port(c_rq, prt); @@ -2895,7 +3857,7 @@ immigrate(ErtsRunQueue *c_rq, ErtsMigrationPath *mp) erts_aint32_t state; state = erts_smp_atomic32_read_acqb(&proc->state); if (!(ERTS_PSFLG_BOUND & state) - && (prio == (int) (ERTS_PSFLG_PRIO_MASK & state))) { + && (prio == (int) ERTS_PSFLGS_GET_PRQ_PRIO(state))) { ErtsRunQueueInfo *rqi = &rq->procs.prio_info[prio]; unqueue_process(rq, rpq, rqi, prio, prev_proc, proc); erts_smp_runq_unlock(rq); @@ -2928,10 +3890,11 @@ suspend_run_queue(ErtsRunQueue *rq) ERTS_SSI_FLG_SUSPENDED); (void) ERTS_RUNQ_FLGS_SET(rq, ERTS_RUNQ_FLG_SUSPENDED); - wake_scheduler(rq, 0); + wake_scheduler(rq); } static void scheduler_ix_resume_wake(Uint ix); +static void scheduler_ssi_resume_wake(ErtsSchedulerSleepInfo *ssi); static ERTS_INLINE void resume_run_queue(ErtsRunQueue *rq) @@ -2958,7 +3921,10 @@ resume_run_queue(ErtsRunQueue *rq) erts_smp_runq_unlock(rq); - scheduler_ix_resume_wake(rq->ix); +#ifdef ERTS_DIRTY_SCHEDULERS + if (!ERTS_RUNQ_IX_IS_DIRTY(rq->ix)) +#endif + scheduler_ix_resume_wake(rq->ix); } typedef struct { @@ -2977,7 +3943,7 @@ schedule_bound_processes(ErtsRunQueue *rq, while (proc) { erts_aint32_t state = erts_smp_atomic32_read_acqb(&proc->state); next = proc->next; - enqueue_process(rq, (int) (ERTS_PSFLG_PRIO_MASK & state), proc); + enqueue_process(rq, (int) ERTS_PSFLGS_GET_PRQ_PRIO(state), proc); proc = next; } } @@ -2989,20 +3955,28 @@ evacuate_run_queue(ErtsRunQueue *rq, int prio_q; ErtsRunQueue *to_rq; ErtsMigrationPaths *mps; - ErtsMigrationPath *mp; + ErtsMigrationPath *mp = NULL; ERTS_SMP_LC_ASSERT(erts_smp_lc_runq_is_locked(rq)); (void) ERTS_RUNQ_FLGS_UNSET(rq, ERTS_RUNQ_FLG_PROTECTED); - mps = erts_get_migration_paths_managed(); - mp = &mps->mpath[rq->ix]; +#ifdef ERTS_DIRTY_SCHEDULERS + if (!ERTS_RUNQ_IX_IS_DIRTY(rq->ix)) +#endif + { + mps = erts_get_migration_paths_managed(); + mp = &mps->mpath[rq->ix]; + } /* Evacuate scheduled misc ops */ if (rq->misc.start) { ErtsMiscOpList *start, *end; +#ifdef ERTS_DIRTY_SCHEDULERS + ASSERT(!ERTS_RUNQ_IX_IS_DIRTY(rq->ix)); +#endif to_rq = mp->misc_evac_runq; if (!to_rq) return; @@ -3020,6 +3994,9 @@ evacuate_run_queue(ErtsRunQueue *rq, to_rq->misc.start = start; to_rq->misc.end = end; + + non_empty_runq(to_rq); + erts_smp_runq_unlock(to_rq); smp_notify_inc_runq(to_rq); erts_smp_runq_lock(to_rq); @@ -3028,6 +4005,9 @@ evacuate_run_queue(ErtsRunQueue *rq, if (rq->ports.start) { Port *prt; +#ifdef ERTS_DIRTY_SCHEDULERS + ASSERT(!ERTS_RUNQ_IX_IS_DIRTY(rq->ix)); +#endif to_rq = mp->prio[ERTS_PORT_PRIO_LEVEL].runq; if (!to_rq) return; @@ -3046,7 +4026,7 @@ evacuate_run_queue(ErtsRunQueue *rq, prt_rq = erts_port_runq(prt); if (prt_rq) { if (prt_rq != to_rq) - erl_exit(ERTS_ABORT_EXIT, + erts_exit(ERTS_ABORT_EXIT, "%s:%d:%s() internal error\n", __FILE__, __LINE__, __func__); erts_enqueue_port(to_rq, prt); @@ -3063,15 +4043,26 @@ evacuate_run_queue(ErtsRunQueue *rq, erts_aint32_t state; Process *proc; int notify = 0; +#ifdef ERTS_DIRTY_SCHEDULERS + int requeue; +#endif to_rq = NULL; - if (!mp->prio[prio_q].runq) - return; - if (prio_q == PRIORITY_NORMAL && !mp->prio[PRIORITY_LOW].runq) - return; +#ifdef ERTS_DIRTY_SCHEDULERS + if (!ERTS_RUNQ_IX_IS_DIRTY(rq->ix)) +#endif + { + if (!mp->prio[prio_q].runq) + return; + if (prio_q == PRIORITY_NORMAL && !mp->prio[PRIORITY_LOW].runq) + return; + } proc = dequeue_process(rq, prio_q, &state); while (proc) { +#ifdef ERTS_DIRTY_SCHEDULERS + requeue = 1; +#endif if (ERTS_PSFLG_BOUND & state) { /* Bound processes get stuck here... */ proc->next = NULL; @@ -3080,12 +4071,51 @@ evacuate_run_queue(ErtsRunQueue *rq, else sbpp->first = proc; sbpp->last = proc; +#ifdef ERTS_DIRTY_SCHEDULERS + requeue = 0; +#endif } +#ifdef ERTS_DIRTY_SCHEDULERS + else if (state & ERTS_PSFLG_DIRTY_CPU_PROC_IN_Q) { +#ifdef DEBUG + erts_aint32_t old = +#else + (void) +#endif + erts_smp_atomic32_read_band_nob(&proc->state, + ~(ERTS_PSFLG_DIRTY_CPU_PROC + | ERTS_PSFLG_DIRTY_CPU_PROC_IN_Q)); + /* assert that no other dirty flags are set */ + ASSERT(!(old & (ERTS_PSFLG_DIRTY_IO_PROC|ERTS_PSFLG_DIRTY_IO_PROC_IN_Q))); + } else if (state & ERTS_PSFLG_DIRTY_IO_PROC_IN_Q) { +#ifdef DEBUG + erts_aint32_t old = +#else + (void) +#endif + erts_smp_atomic32_read_band_nob(&proc->state, + ~(ERTS_PSFLG_DIRTY_IO_PROC + | ERTS_PSFLG_DIRTY_IO_PROC_IN_Q)); + /* assert that no other dirty flags are set */ + ASSERT(!(old & (ERTS_PSFLG_DIRTY_CPU_PROC|ERTS_PSFLG_DIRTY_CPU_PROC_IN_Q))); + } + if (requeue) { +#else else { - int prio = (int) (ERTS_PSFLG_PRIO_MASK & state); +#endif + int prio = (int) ERTS_PSFLGS_GET_PRQ_PRIO(state); erts_smp_runq_unlock(rq); - to_rq = mp->prio[prio].runq; +#ifdef ERTS_DIRTY_SCHEDULERS + if (ERTS_RUNQ_IX_IS_DIRTY(rq->ix)) + /* + * dirty run queues evacuate only to run + * queue 0 during multi-scheduling blocking + */ + to_rq = ERTS_RUNQ_IX(0); + else +#endif + to_rq = mp->prio[prio].runq; RUNQ_SET_RQ(&proc->run_queue, to_rq); erts_smp_runq_lock(to_rq); @@ -3100,9 +4130,6 @@ evacuate_run_queue(ErtsRunQueue *rq, if (notify) smp_notify_inc_runq(to_rq); } - - if (ERTS_EMPTY_RUNQ(rq)) - empty_runq(rq); } static int @@ -3158,7 +4185,7 @@ try_steal_task_from_victim(ErtsRunQueue *rq, int *rq_lockedp, ErtsRunQueue *vrq, erts_aint32_t state = erts_smp_atomic32_read_acqb(&proc->state); if (!(ERTS_PSFLG_BOUND & state)) { /* Steal process */ - int prio = (int) (ERTS_PSFLG_PRIO_MASK & state); + int prio = (int) ERTS_PSFLGS_GET_PRQ_PRIO(state); ErtsRunQueueInfo *rqi = &vrq->procs.prio_info[prio]; unqueue_process(vrq, rpq, rqi, prio, prev_proc, proc); erts_smp_runq_unlock(vrq); @@ -3200,7 +4227,7 @@ no_procs: return 0; else { if (prt_rq != rq) - erl_exit(ERTS_ABORT_EXIT, + erts_exit(ERTS_ABORT_EXIT, "%s:%d:%s() internal error\n", __FILE__, __LINE__, __func__); *rq_lockedp = 1; @@ -3235,7 +4262,7 @@ try_steal_task(ErtsRunQueue *rq) Uint32 flags; /* Protect jobs we steal from getting stolen from us... */ - flags = ERTS_RUNQ_FLGS_SET(rq, ERTS_RUNQ_FLG_PROTECTED); + flags = empty_protected_runq(rq); if (flags & ERTS_RUNQ_FLG_SUSPENDED) return 0; /* go suspend instead... */ @@ -3314,6 +4341,9 @@ typedef struct { int full_reds_history_change; int oowc; int max_len; +#if ERTS_HAVE_SCHED_UTIL_BALANCING_SUPPORT + int sched_util; +#endif } ErtsRunQueueBalance; static ErtsRunQueueBalance *run_queue_info; @@ -3477,6 +4507,9 @@ check_balance(ErtsRunQueue *c_rq) Sint64 scheds_reds, full_scheds_reds; int forced, active, current_active, oowc, half_full_scheds, full_scheds, mmax_len, blnc_no_rqs, qix, pix, freds_hist_ix; +#if ERTS_HAVE_SCHED_UTIL_BALANCING_SUPPORT + int sched_util_balancing; +#endif if (erts_smp_atomic32_xchg_nob(&balance_info.checking_balance, 1)) { c_rq->check_balance_reds = INT_MAX; @@ -3532,6 +4565,10 @@ check_balance(ErtsRunQueue *c_rq) return; } +#if ERTS_HAVE_SCHED_UTIL_BALANCING_SUPPORT + sched_util_balancing = 0; +#endif + freds_hist_ix = balance_info.full_reds_history_index; balance_info.full_reds_history_index++; if (balance_info.full_reds_history_index >= ERTS_FULL_REDS_HISTORY_SIZE) @@ -3562,7 +4599,12 @@ check_balance(ErtsRunQueue *c_rq) run_queue_info[qix].oowc = rq->out_of_work_count; run_queue_info[qix].max_len = rq->max_len; rq->check_balance_reds = INT_MAX; - + +#if ERTS_HAVE_SCHED_UTIL_BALANCING_SUPPORT + if (erts_sched_balance_util) + run_queue_info[qix].sched_util = erts_get_sched_util(rq, 1, 0); +#endif + erts_smp_runq_unlock(rq); } @@ -3632,8 +4674,38 @@ check_balance(ErtsRunQueue *c_rq) mmax_len = run_queue_info[qix].max_len; } - if (!erts_sched_compact_load) + if (!erts_sched_compact_load) { +#if ERTS_HAVE_SCHED_UTIL_BALANCING_SUPPORT + if (erts_sched_balance_util && full_scheds < blnc_no_rqs) { + int avg_util = 0; + + for (qix = 0; qix < blnc_no_rqs; qix++) + avg_util += run_queue_info[qix].sched_util; + + avg_util /= blnc_no_rqs; /* in ppm */ + + sched_util_balancing = 1; + /* + * In order to avoid renaming a large amount of fields + * we write utilization values instead of lenght values + * in the 'max_len' and 'migration_limit' fields... + */ + for (qix = 0; qix < blnc_no_rqs; qix++) { + run_queue_info[qix].flags = 0; /* Reset for later use... */ + for (pix = 0; pix < ERTS_NO_PRIO_LEVELS; pix++) { + run_queue_info[qix].prio[pix].emigrate_to = -1; + run_queue_info[qix].prio[pix].immigrate_from = -1; + run_queue_info[qix].prio[pix].avail = 100; + run_queue_info[qix].prio[pix].max_len = run_queue_info[qix].sched_util; + run_queue_info[qix].prio[pix].migration_limit = avg_util; + } + } + active = blnc_no_rqs; + goto setup_migration_paths; + } +#endif goto all_active; + } if (!forced && half_full_scheds != blnc_no_rqs) { int min = 1; @@ -3750,15 +4822,30 @@ check_balance(ErtsRunQueue *c_rq) } } +#if ERTS_HAVE_SCHED_UTIL_BALANCING_SUPPORT + setup_migration_paths: +#endif + /* Setup migration paths for all priorities */ for (pix = 0; pix < ERTS_NO_PRIO_LEVELS; pix++) { int low = 0, high = 0; for (qix = 0; qix < blnc_no_rqs; qix++) { int len_diff = run_queue_info[qix].prio[pix].max_len; len_diff -= run_queue_info[qix].prio[pix].migration_limit; + #ifdef DBG_PRINT if (pix == 2) erts_fprintf(stderr, "%d ", len_diff); #endif + +#if ERTS_HAVE_SCHED_UTIL_BALANCING_SUPPORT + if (sched_util_balancing + && -ERTS_SCHED_UTIL_IGNORE_IMBALANCE_DIFF <= len_diff + && len_diff <= ERTS_SCHED_UTIL_IGNORE_IMBALANCE_DIFF) { + /* ignore minor imbalance */ + len_diff = 0; + } +#endif + run_queue_compare[qix].qix = qix; run_queue_compare[qix].len = len_diff; if (len_diff != 0) { @@ -3885,6 +4972,9 @@ erts_fprintf(stderr, "--------------------------------\n"); Uint32 flags = run_queue_info[qix].flags; ErtsMigrationPath *mp = &new_mpaths->mpath[qix]; +#if ERTS_HAVE_SCHED_UTIL_BALANCING_SUPPORT + mp->sched_util = sched_util_balancing; +#endif mp->flags = flags; mp->misc_evac_runq = NULL; @@ -3969,7 +5059,7 @@ erts_fprintf(stderr, "--------------------------------\n"); rq->out_of_work_count = 0; (void) ERTS_RUNQ_FLGS_READ_BSET(rq, ERTS_RUNQ_FLGS_MIGRATION_INFO, flags); - rq->max_len = rq->len; + rq->max_len = erts_smp_atomic32_read_dirty(&rq->len); for (pix = 0; pix < ERTS_NO_PRIO_LEVELS; pix++) { ErtsRunQueueInfo *rqi; rqi = (pix == ERTS_PORT_PRIO_LEVEL @@ -3996,9 +5086,8 @@ static void change_no_used_runqs(int used) { ErtsMigrationPaths *new_mpaths, *old_mpaths; - int active, qix; + int qix; erts_smp_mtx_lock(&balance_info.update_mtx); - get_no_runqs(&active, NULL); set_no_used_runqs(used); old_mpaths = erts_get_migration_paths_managed(); @@ -4128,7 +5217,7 @@ wakeup_other_check(ErtsRunQueue *rq, Uint32 flags) { int wo_reds = rq->wakeup_other_reds; if (wo_reds) { - int left_len = rq->len - 1; + int left_len = erts_smp_atomic32_read_dirty(&rq->len) - 1; if (left_len < 1) { int wo_reduce = wo_reds << wakeup_other.dec_shift; wo_reduce &= wakeup_other.dec_mask; @@ -4140,13 +5229,22 @@ wakeup_other_check(ErtsRunQueue *rq, Uint32 flags) rq->wakeup_other += (left_len*wo_reds + ERTS_WAKEUP_OTHER_FIXED_INC); if (rq->wakeup_other > wakeup_other.limit) { - int empty_rqs = - erts_smp_atomic32_read_acqb(&no_empty_run_queues); - if (flags & ERTS_RUNQ_FLG_PROTECTED) - (void) ERTS_RUNQ_FLGS_UNSET(rq, ERTS_RUNQ_FLG_PROTECTED); - if (empty_rqs != 0) - wake_scheduler_on_empty_runq(rq); - rq->wakeup_other = 0; +#ifdef ERTS_DIRTY_SCHEDULERS + if (ERTS_RUNQ_IX_IS_DIRTY(rq->ix)) { + if (rq->waiting) { + wake_dirty_schedulers(rq, 1); + } + } else +#endif + { + int empty_rqs = + erts_smp_atomic32_read_acqb(&no_empty_run_queues); + if (flags & ERTS_RUNQ_FLG_PROTECTED) + (void) ERTS_RUNQ_FLGS_UNSET(rq, ERTS_RUNQ_FLG_PROTECTED); + if (empty_rqs != 0) + wake_scheduler_on_empty_runq(rq); + rq->wakeup_other = 0; + } } } rq->wakeup_other_reds = 0; @@ -4192,7 +5290,7 @@ wakeup_other_check_legacy(ErtsRunQueue *rq, Uint32 flags) { int wo_reds = rq->wakeup_other_reds; if (wo_reds) { - erts_aint32_t len = rq->len; + erts_aint32_t len = erts_smp_atomic32_read_dirty(&rq->len); if (len < 2) { rq->wakeup_other -= ERTS_WAKEUP_OTHER_DEC_LEGACY*wo_reds; if (rq->wakeup_other < 0) @@ -4250,6 +5348,53 @@ set_wakeup_other_data(void) } } +static int +no_runqs_to_supervise(void) +{ + int used; + erts_aint32_t nerq = erts_smp_atomic32_read_acqb(&no_empty_run_queues); + if (nerq <= 0) + return 0; + get_no_runqs(NULL, &used); + if (nerq >= used) + return 0; + return used; +} + +static void * +runq_supervisor(void *unused) +{ + while (1) { + int ix, no_rqs; + + erts_milli_sleep(erts_runq_supervision_interval); + no_rqs = no_runqs_to_supervise(); + if (!no_rqs) { + erts_atomic_set_nob(&runq_supervisor_sleeping, 1); + while (1) { + ethr_event_reset(&runq_supervision_event); + no_rqs = no_runqs_to_supervise(); + if (no_rqs) { + erts_atomic_set_nob(&runq_supervisor_sleeping, 0); + break; + } + ethr_event_wait(&runq_supervision_event); + } + } + + for (ix = 0; ix < no_rqs; ix++) { + ErtsRunQueue *rq = ERTS_RUNQ_IX(ix); + if (ERTS_RUNQ_FLGS_GET(rq) & ERTS_RUNQ_FLG_NONEMPTY) { + erts_smp_runq_lock(rq); + if (erts_smp_atomic32_read_dirty(&rq->len) != 0) + wake_scheduler_on_empty_runq(rq); /* forced wakeup... */ + erts_smp_runq_unlock(rq); + } + } + } + return NULL; +} + #endif void @@ -4260,11 +5405,17 @@ erts_early_init_scheduling(int no_schedulers) wakeup_other.threshold = ERTS_SCHED_WAKEUP_OTHER_THRESHOLD_MEDIUM; wakeup_other.type = ERTS_SCHED_WAKEUP_OTHER_TYPE_DEFAULT; #endif +#ifndef ERTS_SCHED_MIN_SPIN sched_busy_wait.sys_schedule = ERTS_SCHED_SYS_SLEEP_SPINCOUNT_MEDIUM; sched_busy_wait.tse = (ERTS_SCHED_SYS_SLEEP_SPINCOUNT_MEDIUM * ERTS_SCHED_TSE_SLEEP_SPINCOUNT_FACT); sched_busy_wait.aux_work = (ERTS_SCHED_SYS_SLEEP_SPINCOUNT_MEDIUM * ERTS_SCHED_AUX_WORK_SLEEP_SPINCOUNT_FACT_MEDIUM); +#else + sched_busy_wait.sys_schedule = ERTS_SCHED_SYS_SLEEP_SPINCOUNT_NONE; + sched_busy_wait.tse = ERTS_SCHED_SYS_SLEEP_SPINCOUNT_NONE; + sched_busy_wait.aux_work = ERTS_SCHED_SYS_SLEEP_SPINCOUNT_NONE; +#endif } int @@ -4358,17 +5509,45 @@ erts_sched_set_busy_wait_threshold(char *str) return 0; } + +int +erts_sched_set_wake_cleanup_threshold(char *str) +{ + if (sys_strcmp(str, "very_lazy") == 0) + thr_prgr_later_cleanup_op_threshold = ERTS_THR_PRGR_LATER_CLEANUP_OP_THRESHOLD_VERY_LAZY; + else if (sys_strcmp(str, "lazy") == 0) + thr_prgr_later_cleanup_op_threshold = ERTS_THR_PRGR_LATER_CLEANUP_OP_THRESHOLD_LAZY; + else if (sys_strcmp(str, "medium") == 0) + thr_prgr_later_cleanup_op_threshold = ERTS_THR_PRGR_LATER_CLEANUP_OP_THRESHOLD_MEDIUM; + else if (sys_strcmp(str, "eager") == 0) + thr_prgr_later_cleanup_op_threshold = ERTS_THR_PRGR_LATER_CLEANUP_OP_THRESHOLD_EAGER; + else if (sys_strcmp(str, "very_eager") == 0) + thr_prgr_later_cleanup_op_threshold = ERTS_THR_PRGR_LATER_CLEANUP_OP_THRESHOLD_VERY_EAGER; + else + return EINVAL; + return 0; +} + static void init_aux_work_data(ErtsAuxWorkData *awdp, ErtsSchedulerData *esdp, char *dawwp) { - awdp->sched_id = esdp ? (int) esdp->no : 0; + if (!esdp) + awdp->sched_id = 0; +#ifdef ERTS_DIRTY_SCHEDULERS + else if (ERTS_SCHEDULER_IS_DIRTY(esdp)) + awdp->sched_id = (int) ERTS_DIRTY_SCHEDULER_NO(esdp); +#endif + else + awdp->sched_id = (int) esdp->no; awdp->esdp = esdp; awdp->ssi = esdp ? esdp->ssi : NULL; #ifdef ERTS_SMP + awdp->latest_wakeup = ERTS_THR_PRGR_VAL_FIRST; awdp->misc.thr_prgr = ERTS_THR_PRGR_VAL_WAITING; awdp->dd.thr_prgr = ERTS_THR_PRGR_VAL_WAITING; - awdp->dd.completed_callback = NULL; - awdp->dd.completed_arg = NULL; + awdp->cncld_tmrs.thr_prgr = ERTS_THR_PRGR_VAL_WAITING; + awdp->later_op.thr_prgr = ERTS_THR_PRGR_VAL_FIRST; + awdp->later_op.size = 0; awdp->later_op.first = NULL; awdp->later_op.last = NULL; #endif @@ -4396,43 +5575,138 @@ init_aux_work_data(ErtsAuxWorkData *awdp, ErtsSchedulerData *esdp, char *dawwp) awdp->delayed_wakeup.sched2jix[i] = -1; } #endif + awdp->debug.wait_completed.flags = 0; + awdp->debug.wait_completed.callback = NULL; + awdp->debug.wait_completed.arg = NULL; +} + +static void +init_scheduler_data(ErtsSchedulerData* esdp, int num, + ErtsSchedulerSleepInfo* ssi, + ErtsRunQueue* runq, + char** daww_ptr, size_t daww_sz) +{ + esdp->timer_wheel = NULL; +#ifdef ERTS_SMP + erts_bits_init_state(&esdp->erl_bits_state); + esdp->match_pseudo_process = NULL; + esdp->free_process = NULL; +#endif + esdp->x_reg_array = + erts_alloc_permanent_cache_aligned(ERTS_ALC_T_BEAM_REGISTER, + ERTS_X_REGS_ALLOCATED * + sizeof(Eterm)); + esdp->f_reg_array = + erts_alloc_permanent_cache_aligned(ERTS_ALC_T_BEAM_REGISTER, + MAX_REG * sizeof(FloatDef)); +#if !HEAP_ON_C_STACK + esdp->num_tmp_heap_used = 0; +#endif +#ifdef ERTS_DIRTY_SCHEDULERS + if (ERTS_RUNQ_IX_IS_DIRTY(runq->ix)) { + esdp->no = 0; + ERTS_DIRTY_SCHEDULER_NO(esdp) = (Uint) num; + } + else { + esdp->no = (Uint) num; + ERTS_DIRTY_SCHEDULER_NO(esdp) = 0; + } +#else + esdp->no = (Uint) num; +#endif + + esdp->ssi = ssi; + esdp->current_process = NULL; + esdp->current_port = NULL; + + esdp->virtual_reds = 0; + esdp->cpu_id = -1; + + erts_init_atom_cache_map(&esdp->atom_cache_map); + + esdp->run_queue = runq; + esdp->run_queue->scheduler = esdp; + + esdp->last_monotonic_time = 0; + esdp->check_time_reds = 0; + + esdp->thr_id = (Uint32) num; + erts_sched_bif_unique_init(esdp); + + esdp->io.out = (Uint64) 0; + esdp->io.in = (Uint64) 0; + + if (daww_ptr) { + init_aux_work_data(&esdp->aux_work_data, esdp, *daww_ptr); +#ifdef ERTS_SMP + *daww_ptr += daww_sz; +#endif + } + + esdp->reductions = 0; + + init_sched_wall_time(&esdp->sched_wall_time); + erts_port_task_handle_init(&esdp->nosuspend_port_task_handle); } void -erts_init_scheduling(int no_schedulers, int no_schedulers_online) +erts_init_scheduling(int no_schedulers, int no_schedulers_online +#ifdef ERTS_DIRTY_SCHEDULERS + , int no_dirty_cpu_schedulers, int no_dirty_cpu_schedulers_online, + int no_dirty_io_schedulers +#endif + ) { int ix, n, no_ssi; char *daww_ptr; -#ifdef ERTS_SMP size_t daww_sz; -#endif + size_t size_runqs; init_misc_op_list_alloc(); + init_proc_sys_task_queues_alloc(); #ifdef ERTS_SMP set_wakeup_other_data(); #endif +#if ERTS_HAVE_SCHED_UTIL_BALANCING_SUPPORT + if (erts_sched_balance_util) + erts_sched_compact_load = 0; +#endif + ASSERT(no_schedulers_online <= no_schedulers); ASSERT(no_schedulers_online >= 1); ASSERT(no_schedulers >= 1); +#ifdef ERTS_DIRTY_SCHEDULERS + ASSERT(no_dirty_cpu_schedulers <= no_schedulers); + ASSERT(no_dirty_cpu_schedulers >= 1); + ASSERT(no_dirty_cpu_schedulers_online <= no_schedulers_online); + ASSERT(no_dirty_cpu_schedulers_online >= 1); +#endif /* Create and initialize run queues */ n = no_schedulers; - - erts_aligned_run_queues = - erts_alloc_permanent_cache_aligned(ERTS_ALC_T_RUNQS, - sizeof(ErtsAlignedRunQueue) * n); + size_runqs = sizeof(ErtsAlignedRunQueue) * (n + ERTS_NUM_DIRTY_RUNQS); + erts_aligned_run_queues = + erts_alloc_permanent_cache_aligned(ERTS_ALC_T_RUNQS, size_runqs); #ifdef ERTS_SMP +#ifdef ERTS_DIRTY_SCHEDULERS + erts_aligned_run_queues += ERTS_NUM_DIRTY_RUNQS; +#endif erts_smp_atomic32_init_nob(&no_empty_run_queues, 0); #endif erts_no_run_queues = n; - for (ix = 0; ix < n; ix++) { + for (ix = -(ERTS_NUM_DIRTY_RUNQS); ix < n; ix++) { int pix, rix; +#ifdef ERTS_DIRTY_SCHEDULERS + ErtsRunQueue *rq = ERTS_RUNQ_IX_IS_DIRTY(ix) ? + ERTS_DIRTY_RUNQ_IX(ix) : ERTS_RUNQ_IX(ix); +#else ErtsRunQueue *rq = ERTS_RUNQ_IX(ix); +#endif rq->ix = ix; @@ -4443,6 +5717,14 @@ erts_init_scheduling(int no_schedulers, int no_schedulers_online) erts_smp_mtx_init_x(&rq->mtx, "run_queue", make_small(ix + 1)); erts_smp_cnd_init(&rq->cnd); +#ifdef ERTS_DIRTY_SCHEDULERS +#ifdef ERTS_SMP + if (ERTS_RUNQ_IX_IS_DIRTY(ix)) + erts_smp_spinlock_init(&rq->sleepers.lock, "dirty_run_queue_sleep_list"); + rq->sleepers.list = NULL; +#endif +#endif + rq->waiting = 0; rq->woken = 0; ERTS_RUNQ_FLGS_INIT(rq, ERTS_RUNQ_FLG_NONEMPTY); @@ -4454,7 +5736,7 @@ erts_init_scheduling(int no_schedulers, int no_schedulers_online) } rq->out_of_work_count = 0; rq->max_len = 0; - rq->len = 0; + erts_smp_atomic32_set_nob(&rq->len, 0); rq->wakeup_other = 0; rq->wakeup_other_reds = 0; rq->halt_in_progress = 0; @@ -4481,6 +5763,11 @@ erts_init_scheduling(int no_schedulers, int no_schedulers_online) rq->ports.info.reds = 0; rq->ports.start = NULL; rq->ports.end = NULL; + +#if ERTS_HAVE_SCHED_UTIL_BALANCING_SUPPORT + init_runq_sched_util(&rq->sched_util, erts_sched_balance_util); +#endif + } #ifdef ERTS_SMP @@ -4498,6 +5785,10 @@ erts_init_scheduling(int no_schedulers, int no_schedulers_online) n = (int) no_schedulers; erts_no_schedulers = n; +#ifdef ERTS_DIRTY_SCHEDULERS + erts_no_dirty_cpu_schedulers = no_dirty_cpu_schedulers; + erts_no_dirty_io_schedulers = no_dirty_io_schedulers; +#endif /* Create and initialize scheduler sleep info */ #ifdef ERTS_SMP @@ -4524,6 +5815,29 @@ erts_init_scheduling(int no_schedulers, int no_schedulers_online) #ifdef ERTS_SMP aligned_sched_sleep_info++; + +#ifdef ERTS_DIRTY_SCHEDULERS + aligned_dirty_cpu_sched_sleep_info = + erts_alloc_permanent_cache_aligned( + ERTS_ALC_T_SCHDLR_SLP_INFO, + no_dirty_cpu_schedulers*sizeof(ErtsAlignedSchedulerSleepInfo)); + for (ix = 0; ix < no_dirty_cpu_schedulers; ix++) { + ErtsSchedulerSleepInfo *ssi = &aligned_dirty_cpu_sched_sleep_info[ix].ssi; + erts_smp_atomic32_init_nob(&ssi->flags, 0); + ssi->event = NULL; /* initialized in sched_dirty_cpu_thread_func */ + erts_atomic32_init_nob(&ssi->aux_work, 0); + } + aligned_dirty_io_sched_sleep_info = + erts_alloc_permanent_cache_aligned( + ERTS_ALC_T_SCHDLR_SLP_INFO, + no_dirty_io_schedulers*sizeof(ErtsAlignedSchedulerSleepInfo)); + for (ix = 0; ix < no_dirty_io_schedulers; ix++) { + ErtsSchedulerSleepInfo *ssi = &aligned_dirty_io_sched_sleep_info[ix].ssi; + erts_smp_atomic32_init_nob(&ssi->flags, 0); + ssi->event = NULL; /* initialized in sched_dirty_io_thread_func */ + erts_atomic32_init_nob(&ssi->aux_work, 0); + } +#endif #endif /* Create and initialize scheduler specific data */ @@ -4534,6 +5848,7 @@ erts_init_scheduling(int no_schedulers, int no_schedulers_online) daww_ptr = erts_alloc_permanent_cache_aligned(ERTS_ALC_T_SCHDLR_DATA, daww_sz*n); #else + daww_sz = 0; daww_ptr = NULL; #endif @@ -4543,54 +5858,44 @@ erts_init_scheduling(int no_schedulers, int no_schedulers_online) for (ix = 0; ix < n; ix++) { ErtsSchedulerData *esdp = ERTS_SCHEDULER_IX(ix); -#ifdef ERTS_SMP - erts_bits_init_state(&esdp->erl_bits_state); - esdp->match_pseudo_process = NULL; - esdp->free_process = NULL; -#endif - esdp->x_reg_array = - erts_alloc_permanent_cache_aligned(ERTS_ALC_T_BEAM_REGISTER, - ERTS_X_REGS_ALLOCATED * - sizeof(Eterm)); - esdp->f_reg_array = - erts_alloc_permanent_cache_aligned(ERTS_ALC_T_BEAM_REGISTER, - MAX_REG * sizeof(FloatDef)); -#if !HEAP_ON_C_STACK - esdp->num_tmp_heap_used = 0; -#endif - esdp->no = (Uint) ix+1; - esdp->ssi = ERTS_SCHED_SLEEP_INFO_IX(ix); - esdp->current_process = NULL; - esdp->current_port = NULL; - - esdp->virtual_reds = 0; - esdp->cpu_id = -1; - - erts_init_atom_cache_map(&esdp->atom_cache_map); - - esdp->run_queue = ERTS_RUNQ_IX(ix); - esdp->run_queue->scheduler = esdp; + init_scheduler_data(esdp, ix+1, ERTS_SCHED_SLEEP_INFO_IX(ix), + ERTS_RUNQ_IX(ix), &daww_ptr, daww_sz); + } - init_aux_work_data(&esdp->aux_work_data, esdp, daww_ptr); +#ifdef ERTS_DIRTY_SCHEDULERS #ifdef ERTS_SMP - daww_ptr += daww_sz; -#endif - - esdp->reductions = 0; - - init_sched_wall_time(&esdp->sched_wall_time); + erts_aligned_dirty_cpu_scheduler_data = + erts_alloc_permanent_cache_aligned( + ERTS_ALC_T_SCHDLR_DATA, + no_dirty_cpu_schedulers*sizeof(ErtsAlignedSchedulerData)); + for (ix = 0; ix < no_dirty_cpu_schedulers; ix++) { + ErtsSchedulerData *esdp = ERTS_DIRTY_CPU_SCHEDULER_IX(ix); + init_scheduler_data(esdp, ix+1, ERTS_DIRTY_CPU_SCHED_SLEEP_INFO_IX(ix), + ERTS_DIRTY_CPU_RUNQ, NULL, 0); + } + erts_aligned_dirty_io_scheduler_data = + erts_alloc_permanent_cache_aligned( + ERTS_ALC_T_SCHDLR_DATA, + no_dirty_io_schedulers*sizeof(ErtsAlignedSchedulerData)); + for (ix = 0; ix < no_dirty_io_schedulers; ix++) { + ErtsSchedulerData *esdp = ERTS_DIRTY_IO_SCHEDULER_IX(ix); + init_scheduler_data(esdp, ix+1, ERTS_DIRTY_IO_SCHED_SLEEP_INFO_IX(ix), + ERTS_DIRTY_IO_RUNQ, NULL, 0); } +#endif +#endif init_misc_aux_work(); #if !HALFWORD_HEAP init_swtreq_alloc(); + init_screq_alloc(); #endif + erts_atomic32_init_nob(&debug_wait_completed_count, 0); /* debug only */ + debug_wait_completed_flags = 0; #ifdef ERTS_SMP - erts_atomic32_init_nob(&completed_dealloc_count, 0); /* debug only */ - aux_thread_aux_work_data = erts_alloc_permanent_cache_aligned(ERTS_ALC_T_SCHDLR_DATA, sizeof(ErtsAuxWorkData)); @@ -4603,8 +5908,18 @@ erts_init_scheduling(int no_schedulers, int no_schedulers_online) schdlr_sspnd.curr_online = no_schedulers; schdlr_sspnd.msb.ongoing = 0; erts_smp_atomic32_init_nob(&schdlr_sspnd.active, no_schedulers); +#ifdef ERTS_DIRTY_SCHEDULERS + erts_smp_atomic32_init_nob(&schdlr_sspnd.dirty_cpu_changing, 0); + schdlr_sspnd.dirty_cpu_online = no_dirty_cpu_schedulers_online; + schdlr_sspnd.dirty_cpu_curr_online = no_dirty_cpu_schedulers; + erts_smp_atomic32_init_nob(&schdlr_sspnd.dirty_cpu_active, no_dirty_cpu_schedulers); + erts_smp_atomic32_init_nob(&schdlr_sspnd.dirty_io_changing, 0); + schdlr_sspnd.dirty_io_online = no_dirty_io_schedulers; + schdlr_sspnd.dirty_io_curr_online = no_dirty_io_schedulers; + erts_smp_atomic32_init_nob(&schdlr_sspnd.dirty_io_active, no_dirty_io_schedulers); +#endif schdlr_sspnd.msb.procs = NULL; - init_no_runqs(no_schedulers, no_schedulers_online); + init_no_runqs(no_schedulers_online, no_schedulers_online); balance_info.last_active_runqs = no_schedulers; erts_smp_mtx_init(&balance_info.update_mtx, "migration_info_update"); balance_info.forced_check_balance = 0; @@ -4628,6 +5943,21 @@ erts_init_scheduling(int no_schedulers, int no_schedulers_online) schdlr_sspnd.curr_online *= 2; /* Boot strapping... */ ERTS_SCHDLR_SSPND_CHNG_SET((ERTS_SCHDLR_SSPND_CHNG_ONLN | ERTS_SCHDLR_SSPND_CHNG_WAITER), 0); +#ifdef ERTS_DIRTY_SCHEDULERS + schdlr_sspnd.dirty_cpu_wait_curr_online = no_dirty_cpu_schedulers_online; + schdlr_sspnd.dirty_cpu_curr_online *= 2; + ERTS_SCHDLR_SSPND_DIRTY_CPU_CHNG_SET((ERTS_SCHDLR_SSPND_CHNG_ONLN + | ERTS_SCHDLR_SSPND_CHNG_WAITER), 0); + for (ix = no_dirty_cpu_schedulers_online; ix < no_dirty_cpu_schedulers; ix++) { + ErtsSchedulerData* esdp = ERTS_DIRTY_CPU_SCHEDULER_IX(ix); + erts_smp_atomic32_read_bor_nob(&esdp->ssi->flags, ERTS_SSI_FLG_SUSPENDED); + } + + schdlr_sspnd.dirty_io_wait_curr_online = no_dirty_io_schedulers; + schdlr_sspnd.dirty_io_curr_online *= 2; + ERTS_SCHDLR_SSPND_DIRTY_IO_CHNG_SET((ERTS_SCHDLR_SSPND_CHNG_ONLN + | ERTS_SCHDLR_SSPND_CHNG_WAITER), 0); +#endif erts_smp_atomic32_init_nob(&doing_sys_schedule, 0); @@ -4643,6 +5973,10 @@ erts_init_scheduling(int no_schedulers, int no_schedulers_online) #endif } erts_no_schedulers = 1; +#ifdef ERTS_DIRTY_SCHEDULERS + erts_no_dirty_cpu_schedulers = 0; + erts_no_dirty_io_schedulers = 0; +#endif #endif erts_smp_atomic32_init_nob(&function_calls, 0); @@ -4689,76 +6023,297 @@ erts_get_scheduler_data(void) #endif +static Process * +make_proxy_proc(Process *prev_proxy, Process *proc, erts_aint32_t prio) +{ + erts_aint32_t state; + Process *proxy; +#ifdef ERTS_SMP + ErtsRunQueue *rq = RUNQ_READ_RQ(&proc->run_queue); +#endif + + state = (ERTS_PSFLG_PROXY + | ERTS_PSFLG_IN_RUNQ + | (((erts_aint32_t) 1) << (prio + ERTS_PSFLGS_IN_PRQ_MASK_OFFSET)) + | (prio << ERTS_PSFLGS_PRQ_PRIO_OFFSET) + | (prio << ERTS_PSFLGS_USR_PRIO_OFFSET) + | (prio << ERTS_PSFLGS_ACT_PRIO_OFFSET)); + + if (prev_proxy) { + proxy = prev_proxy; + ASSERT(erts_smp_atomic32_read_nob(&proxy->state) & ERTS_PSFLG_PROXY); + erts_smp_atomic32_set_nob(&proxy->state, state); +#ifdef ERTS_SMP + RUNQ_SET_RQ(&proc->run_queue, rq); +#endif + } + else { + proxy = erts_alloc(ERTS_ALC_T_PROC, sizeof(Process)); +#ifdef DEBUG + { + int i; + Uint32 *ui32 = (Uint32 *) (char *) proxy; + for (i = 0; i < sizeof(Process)/sizeof(Uint32); i++) + ui32[i] = (Uint32) 0xdeadbeef; + } +#endif + erts_smp_atomic32_init_nob(&proxy->state, state); +#ifdef ERTS_SMP + erts_smp_atomic_init_nob(&proxy->run_queue, + erts_smp_atomic_read_nob(&proc->run_queue)); +#endif + } + + proxy->common.id = proc->common.id; + + return proxy; +} + +static ERTS_INLINE void +free_proxy_proc(Process *proxy) +{ + ASSERT(erts_smp_atomic32_read_nob(&proxy->state) & ERTS_PSFLG_PROXY); + erts_free(ERTS_ALC_T_PROC, proxy); +} + +#define ERTS_ENQUEUE_NOT 0 +#define ERTS_ENQUEUE_NORMAL_QUEUE 1 +#ifdef ERTS_DIRTY_SCHEDULERS +#define ERTS_ENQUEUE_DIRTY_CPU_QUEUE 2 +#define ERTS_ENQUEUE_DIRTY_IO_QUEUE 3 +#endif + +static ERTS_INLINE int +check_enqueue_in_prio_queue(Process *c_p, + erts_aint32_t *prq_prio_p, + erts_aint32_t *newp, + erts_aint32_t actual) +{ + erts_aint32_t aprio, qbit, max_qbit; + + aprio = (actual >> ERTS_PSFLGS_ACT_PRIO_OFFSET) & ERTS_PSFLGS_PRIO_MASK; + qbit = 1 << aprio; + + *prq_prio_p = aprio; + +#ifdef ERTS_DIRTY_SCHEDULERS + if (actual & (ERTS_PSFLG_DIRTY_CPU_PROC|ERTS_PSFLG_DIRTY_IO_PROC)) { + /* + * If we have system tasks of a priority higher + * or equal to the user priority, we enqueue + * on ordinary run-queue and take care of + * those system tasks first. + */ + if (actual & ERTS_PSFLG_ACTIVE_SYS) { + erts_aint32_t uprio, stprio, qmask; + uprio = (actual >> ERTS_PSFLGS_USR_PRIO_OFFSET) & ERTS_PSFLGS_PRIO_MASK; + if (aprio < uprio) + goto enqueue_normal_runq; /* system tasks with higher prio */ + erts_smp_proc_lock(c_p, ERTS_PROC_LOCK_STATUS); + qmask = c_p->sys_task_qs->qmask; + erts_smp_proc_unlock(c_p, ERTS_PROC_LOCK_STATUS); + switch (qmask & -qmask) { + case MAX_BIT: + stprio = PRIORITY_MAX; + break; + case HIGH_BIT: + stprio = PRIORITY_HIGH; + break; + case NORMAL_BIT: + stprio = PRIORITY_NORMAL; + break; + case LOW_BIT: + stprio = PRIORITY_LOW; + break; + default: + stprio = PRIORITY_LOW+1; + break; + } + if (stprio <= uprio) + goto enqueue_normal_runq; /* system tasks with higher prio */ + } + + /* Enqueue in dirty run queue if not already enqueued */ + if (actual & (ERTS_PSFLG_DIRTY_CPU_PROC_IN_Q|ERTS_PSFLG_DIRTY_IO_PROC_IN_Q)) + return ERTS_ENQUEUE_NOT; /* already in queue */ + if (actual & ERTS_PSFLG_DIRTY_CPU_PROC) { + *newp |= ERTS_PSFLG_DIRTY_CPU_PROC_IN_Q; + if (actual & ERTS_PSFLG_IN_RUNQ) + return -ERTS_ENQUEUE_DIRTY_CPU_QUEUE; /* use proxy */ + *newp |= ERTS_PSFLG_IN_RUNQ; + return ERTS_ENQUEUE_DIRTY_CPU_QUEUE; + } + *newp |= ERTS_PSFLG_DIRTY_IO_PROC_IN_Q; + if (actual & ERTS_PSFLG_IN_RUNQ) + return -ERTS_ENQUEUE_DIRTY_IO_QUEUE; /* use proxy */ + *newp |= ERTS_PSFLG_IN_RUNQ; + return ERTS_ENQUEUE_DIRTY_IO_QUEUE; + } + + enqueue_normal_runq: +#endif + max_qbit = (actual >> ERTS_PSFLGS_IN_PRQ_MASK_OFFSET) & ERTS_PSFLGS_QMASK; + max_qbit |= 1 << ERTS_PSFLGS_QMASK_BITS; + max_qbit &= -max_qbit; + /* + * max_qbit now either contain bit set for highest prio queue or a bit + * out of range (which will have a value larger than valid range). + */ + + if (qbit >= max_qbit) + return ERTS_ENQUEUE_NOT; /* Already queued in higher or equal prio */ + + /* Need to enqueue (if already enqueued, it is in lower prio) */ + *newp |= qbit << ERTS_PSFLGS_IN_PRQ_MASK_OFFSET; + + if ((actual & (ERTS_PSFLG_IN_RUNQ|ERTS_PSFLGS_USR_PRIO_MASK)) + != (aprio << ERTS_PSFLGS_USR_PRIO_OFFSET)) { + /* + * Process struct already enqueued, or actual prio not + * equal to user prio, i.e., enqueue using proxy. + */ + return -ERTS_ENQUEUE_NORMAL_QUEUE; + } + + /* + * Enqueue using process struct. + */ + *newp &= ~ERTS_PSFLGS_PRQ_PRIO_MASK; + *newp |= ERTS_PSFLG_IN_RUNQ | (aprio << ERTS_PSFLGS_PRQ_PRIO_OFFSET); + return ERTS_ENQUEUE_NORMAL_QUEUE; +} + /* - * scheduler_out_process() return with c_rq locked. + * schedule_out_process() return with c_rq locked. */ static ERTS_INLINE int -schedule_out_process(ErtsRunQueue *c_rq, erts_aint32_t state, Process *p) +schedule_out_process(ErtsRunQueue *c_rq, erts_aint32_t state, Process *p, Process *proxy) { - erts_aint32_t a, e, n; - int res = 0; + erts_aint32_t a, e, n, enq_prio = -1; + int enqueue; /* < 0 -> use proxy */ + Process* sched_p; + ErtsRunQueue* runq; +#ifdef ERTS_SMP + int check_emigration_need; +#endif a = state; while (1) { n = e = a; - ASSERT(a & ERTS_PSFLG_RUNNING); - ASSERT(!(a & ERTS_PSFLG_IN_RUNQ)); + ASSERT(a & (ERTS_PSFLG_RUNNING|ERTS_PSFLG_RUNNING_SYS)); + + enqueue = ERTS_ENQUEUE_NOT; - n &= ~ERTS_PSFLG_RUNNING; - if ((a & (ERTS_PSFLG_ACTIVE|ERTS_PSFLG_SUSPENDED)) == ERTS_PSFLG_ACTIVE) - n |= ERTS_PSFLG_IN_RUNQ; + n &= ~(ERTS_PSFLG_RUNNING|ERTS_PSFLG_RUNNING_SYS); + if (a & ERTS_PSFLG_ACTIVE_SYS + || (a & (ERTS_PSFLG_ACTIVE|ERTS_PSFLG_SUSPENDED)) == ERTS_PSFLG_ACTIVE) { + enqueue = check_enqueue_in_prio_queue(p, &enq_prio, &n, a); + } a = erts_smp_atomic32_cmpxchg_mb(&p->state, n, e); if (a == e) break; } - if (!(n & ERTS_PSFLG_IN_RUNQ)) { - if (erts_system_profile_flags.runnable_procs) - profile_runnable_proc(p, am_inactive); - } - else { - int prio = (int) (ERTS_PSFLG_PRIO_MASK & n); - ErtsRunQueue *runq = erts_get_runq_proc(p); + switch (enqueue) { + case ERTS_ENQUEUE_NOT: + if (erts_system_profile_flags.runnable_procs) { - ASSERT(!(n & ERTS_PSFLG_SUSPENDED)); + /* Status lock prevents out of order "runnable proc" trace msgs */ + ERTS_SMP_LC_ASSERT(ERTS_PROC_LOCK_STATUS & erts_proc_lc_my_proc_locks(p)); -#ifdef ERTS_SMP - if (!(ERTS_PSFLG_BOUND & n)) { - ErtsRunQueue *new_runq = erts_check_emigration_need(runq, prio); - if (new_runq) { - RUNQ_SET_RQ(&p->run_queue, new_runq); - runq = new_runq; + if (!(a & ERTS_PSFLG_ACTIVE_SYS) + && (!(a & ERTS_PSFLG_ACTIVE) + || (a & ERTS_PSFLG_SUSPENDED))) { + /* Process inactive */ + profile_runnable_proc(p, am_inactive); } } + + if (proxy) + free_proxy_proc(proxy); + + erts_smp_runq_lock(c_rq); + + return 0; + +#ifdef ERTS_DIRTY_SCHEDULERS +#ifdef ERTS_SMP + case ERTS_ENQUEUE_DIRTY_CPU_QUEUE: + case -ERTS_ENQUEUE_DIRTY_CPU_QUEUE: + runq = ERTS_DIRTY_CPU_RUNQ; + ASSERT(ERTS_SCHEDULER_IS_DIRTY_CPU(runq->scheduler)); +#ifdef ERTS_SMP + check_emigration_need = 0; +#endif + break; + + case ERTS_ENQUEUE_DIRTY_IO_QUEUE: + case -ERTS_ENQUEUE_DIRTY_IO_QUEUE: + runq = ERTS_DIRTY_IO_RUNQ; + ASSERT(ERTS_SCHEDULER_IS_DIRTY_IO(runq->scheduler)); +#ifdef ERTS_SMP + check_emigration_need = 0; +#endif + break; +#endif +#endif + + default: + ASSERT(enqueue == ERTS_ENQUEUE_NORMAL_QUEUE + || enqueue == -ERTS_ENQUEUE_NORMAL_QUEUE); + + runq = erts_get_runq_proc(p); +#ifdef ERTS_SMP + check_emigration_need = !(ERTS_PSFLG_BOUND & n); #endif - ASSERT(runq); - res = 1; + break; + } - erts_smp_runq_lock(runq); + ASSERT(!(n & ERTS_PSFLG_SUSPENDED) || (n & ERTS_PSFLG_ACTIVE_SYS)); - /* Enqueue the process */ - enqueue_process(runq, prio, p); + if (enqueue < 0) + sched_p = make_proxy_proc(proxy, p, enq_prio); + else { + sched_p = p; + if (proxy) + free_proxy_proc(proxy); + } - if (runq == c_rq) - return res; - erts_smp_runq_unlock(runq); - smp_notify_inc_runq(runq); +#ifdef ERTS_SMP + if (check_emigration_need) { + ErtsRunQueue *new_runq = erts_check_emigration_need(runq, enq_prio); + if (new_runq) { + RUNQ_SET_RQ(&sched_p->run_queue, new_runq); + runq = new_runq; + } } +#endif + + ASSERT(runq); + + erts_smp_runq_lock(runq); + + /* Enqueue the process */ + enqueue_process(runq, (int) enq_prio, sched_p); + + if (runq == c_rq) + return 1; + erts_smp_runq_unlock(runq); + smp_notify_inc_runq(runq); erts_smp_runq_lock(c_rq); - return res; + return 1; } static ERTS_INLINE void -add2runq(Process *p, erts_aint32_t state) +add2runq(Process *p, erts_aint32_t state, erts_aint32_t prio) { - int prio = (int) (ERTS_PSFLG_PRIO_MASK & state); ErtsRunQueue *runq = erts_get_runq_proc(p); #ifdef ERTS_SMP if (!(ERTS_PSFLG_BOUND & state)) { - ErtsRunQueue *new_runq = erts_check_emigration_need(runq, prio); + ErtsRunQueue *new_runq = erts_check_emigration_need(runq, (int) prio); if (new_runq) { RUNQ_SET_RQ(&p->run_queue, new_runq); runq = new_runq; @@ -4770,101 +6325,267 @@ add2runq(Process *p, erts_aint32_t state) erts_smp_runq_lock(runq); /* Enqueue the process */ - enqueue_process(runq, prio, p); + enqueue_process(runq, (int) prio, p); erts_smp_runq_unlock(runq); smp_notify_inc_runq(runq); } -static ERTS_INLINE void -schedule_process(Process *p, erts_aint32_t state, int active_enq) +static ERTS_INLINE int +change_proc_schedule_state(Process *p, + erts_aint32_t clear_state_flags, + erts_aint32_t set_state_flags, + erts_aint32_t *statep, + erts_aint32_t *enq_prio_p, + ErtsProcLocks locks) { - erts_aint32_t a = state, n; + /* + * NOTE: ERTS_PSFLG_RUNNING, ERTS_PSFLG_RUNNING_SYS and + * ERTS_PSFLG_ACTIVE_SYS are not allowed to be + * altered by this function! + */ + erts_aint32_t a = *statep, n; + int enqueue; /* < 0 -> use proxy */ + unsigned int prof_runnable_procs = erts_system_profile_flags.runnable_procs; + unsigned int lock_status = (prof_runnable_procs + && !(locks & ERTS_PROC_LOCK_STATUS)); + + ERTS_SMP_LC_ASSERT(locks == erts_proc_lc_my_proc_locks(p)); + + ASSERT(!(a & ERTS_PSFLG_PROXY)); + ASSERT((clear_state_flags & (ERTS_PSFLG_RUNNING + | ERTS_PSFLG_RUNNING_SYS + | ERTS_PSFLG_ACTIVE_SYS)) == 0); + ASSERT((set_state_flags & (ERTS_PSFLG_RUNNING + | ERTS_PSFLG_RUNNING_SYS + | ERTS_PSFLG_ACTIVE_SYS)) == 0); + + if (lock_status) + erts_smp_proc_lock(p, ERTS_PROC_LOCK_STATUS); while (1) { erts_aint32_t e; n = e = a; + enqueue = ERTS_ENQUEUE_NOT; + if (a & ERTS_PSFLG_FREE) - return; /* We don't want to schedule free processes... */ - n |= ERTS_PSFLG_ACTIVE; - if (!(a & (ERTS_PSFLG_SUSPENDED|ERTS_PSFLG_RUNNING))) - n |= ERTS_PSFLG_IN_RUNQ; + break; /* We don't want to schedule free processes... */ + + if (clear_state_flags) + n &= ~clear_state_flags; + + if (set_state_flags) + n |= set_state_flags; + + if ((n & (ERTS_PSFLG_SUSPENDED + | ERTS_PSFLG_RUNNING + | ERTS_PSFLG_RUNNING_SYS + | ERTS_PSFLG_IN_RUNQ + | ERTS_PSFLG_ACTIVE)) == ERTS_PSFLG_ACTIVE) { + /* + * Active and seemingly need to be enqueued, but + * process may be in a run queue via proxy, need + * further inspection... + */ + enqueue = check_enqueue_in_prio_queue(p, enq_prio_p, &n, a); + } + a = erts_smp_atomic32_cmpxchg_mb(&p->state, n, e); if (a == e) break; - if (!active_enq && (a & ERTS_PSFLG_ACTIVE)) - return; /* Someone else activated process ... */ + if (enqueue == ERTS_ENQUEUE_NOT && n == a) + break; } - if (erts_system_profile_flags.runnable_procs - && !(a & (ERTS_PSFLG_ACTIVE|ERTS_PSFLG_SUSPENDED))) { - profile_runnable_proc(p, am_active); + if (prof_runnable_procs) { + + /* Status lock prevents out of order "runnable proc" trace msgs */ + + if (((n & (ERTS_PSFLG_SUSPENDED + | ERTS_PSFLG_ACTIVE)) == ERTS_PSFLG_ACTIVE) + && (!(a & (ERTS_PSFLG_ACTIVE_SYS + | ERTS_PSFLG_RUNNING + | ERTS_PSFLG_RUNNING_SYS) + && (!(a & ERTS_PSFLG_ACTIVE) + || (a & ERTS_PSFLG_SUSPENDED))))) { + /* We activated a prevously inactive process */ + profile_runnable_proc(p, am_active); + } + + if (lock_status) + erts_smp_proc_unlock(p, ERTS_PROC_LOCK_STATUS); } - if ((n & ERTS_PSFLG_IN_RUNQ) && !(a & ERTS_PSFLG_IN_RUNQ)) - add2runq(p, n); + + *statep = a; + + return enqueue; +} + +static ERTS_INLINE void +schedule_process(Process *p, erts_aint32_t in_state, ErtsProcLocks locks) +{ + erts_aint32_t enq_prio = -1; + erts_aint32_t state = in_state; + int enqueue = change_proc_schedule_state(p, + 0, + ERTS_PSFLG_ACTIVE, + &state, + &enq_prio, + locks); + if (enqueue != ERTS_ENQUEUE_NOT) + add2runq(enqueue > 0 ? p : make_proxy_proc(NULL, p, enq_prio), + state, + enq_prio); } void -erts_schedule_process(Process *p, erts_aint32_t state) +erts_schedule_process(Process *p, erts_aint32_t state, ErtsProcLocks locks) { - schedule_process(p, state, 0); + schedule_process(p, state, locks); +} + +static void +schedule_process_sys_task(Process *p, erts_aint32_t state, Process *proxy) +{ + /* + * Expects status lock to be locked when called, and + * returns with status lock unlocked... + */ + erts_aint32_t a = state, n, enq_prio = -1; + int enqueue; /* < 0 -> use proxy */ + unsigned int prof_runnable_procs = erts_system_profile_flags.runnable_procs; + + /* Status lock prevents out of order "runnable proc" trace msgs */ + ERTS_SMP_LC_ASSERT(ERTS_PROC_LOCK_STATUS & erts_proc_lc_my_proc_locks(p)); + + if (!prof_runnable_procs) + erts_smp_proc_unlock(p, ERTS_PROC_LOCK_STATUS); + + ASSERT(!(state & ERTS_PSFLG_PROXY)); + + while (1) { + erts_aint32_t e; + n = e = a; + + if (a & ERTS_PSFLG_FREE) + goto cleanup; /* We don't want to schedule free processes... */ + + enqueue = ERTS_ENQUEUE_NOT; + n |= ERTS_PSFLG_ACTIVE_SYS; + if (!(a & (ERTS_PSFLG_RUNNING|ERTS_PSFLG_RUNNING_SYS))) + enqueue = check_enqueue_in_prio_queue(p, &enq_prio, &n, a); + a = erts_smp_atomic32_cmpxchg_mb(&p->state, n, e); + if (a == e) + break; + if (a == n && enqueue == ERTS_ENQUEUE_NOT) + goto cleanup; + } + + if (prof_runnable_procs) { + + if (!(a & (ERTS_PSFLG_ACTIVE_SYS + | ERTS_PSFLG_RUNNING + | ERTS_PSFLG_RUNNING_SYS)) + && (!(a & ERTS_PSFLG_ACTIVE) || (a & ERTS_PSFLG_SUSPENDED))) { + /* We activated a prevously inactive process */ + profile_runnable_proc(p, am_active); + } + + erts_smp_proc_unlock(p, ERTS_PROC_LOCK_STATUS); + prof_runnable_procs = 0; + } + + if (enqueue != ERTS_ENQUEUE_NOT) { + Process *sched_p; + if (enqueue > 0) + sched_p = p; + else { + sched_p = make_proxy_proc(proxy, p, enq_prio); + proxy = NULL; + } + add2runq(sched_p, n, enq_prio); + } + +cleanup: + + if (prof_runnable_procs) + erts_smp_proc_unlock(p, ERTS_PROC_LOCK_STATUS); + + if (proxy) + free_proxy_proc(proxy); + + ERTS_SMP_LC_ASSERT(!(ERTS_PROC_LOCK_STATUS & erts_proc_lc_my_proc_locks(p))); } static ERTS_INLINE int suspend_process(Process *c_p, Process *p) { - erts_aint32_t state = erts_smp_atomic32_read_acqb(&p->state); + erts_aint32_t state; int suspended = 0; ERTS_SMP_LC_ASSERT(ERTS_PROC_LOCK_STATUS & erts_proc_lc_my_proc_locks(p)); + state = erts_smp_atomic32_read_acqb(&p->state); + if ((state & ERTS_PSFLG_SUSPENDED)) suspended = -1; else { if (c_p == p) { state = erts_smp_atomic32_read_bor_relb(&p->state, ERTS_PSFLG_SUSPENDED); - state |= ERTS_PSFLG_SUSPENDED; - ASSERT(state & ERTS_PSFLG_RUNNING); - suspended = 1; + ASSERT(state & (ERTS_PSFLG_RUNNING|ERTS_PSFLG_RUNNING_SYS)); + suspended = (state & ERTS_PSFLG_SUSPENDED) ? -1: 1; } else { while (!(state & (ERTS_PSFLG_RUNNING|ERTS_PSFLG_EXITING))) { - erts_aint32_t e, n; + erts_aint32_t n, e; + n = e = state; n |= ERTS_PSFLG_SUSPENDED; state = erts_smp_atomic32_cmpxchg_relb(&p->state, n, e); if (state == e) { - state = n; suspended = 1; break; } + if (state & ERTS_PSFLG_SUSPENDED) { + suspended = -1; + break; + } } } } - if (state & ERTS_PSFLG_SUSPENDED) { + if (suspended) { + + if (suspended > 0 && erts_system_profile_flags.runnable_procs) { - ASSERT(!(ERTS_PSFLG_RUNNING & state) - || p == erts_get_current_process()); + /* 'state' is before our change... */ + + if ((state & (ERTS_PSFLG_ACTIVE + | ERTS_PSFLG_ACTIVE_SYS + | ERTS_PSFLG_RUNNING + | ERTS_PSFLG_RUNNING_SYS + | ERTS_PSFLG_SUSPENDED)) == ERTS_PSFLG_ACTIVE) { + /* We made process inactive */ + profile_runnable_proc(p, am_inactive); + } - if (erts_system_profile_flags.runnable_procs - && (p->rcount == 0) - && (state & ERTS_PSFLG_ACTIVE)) { - profile_runnable_proc(p, am_inactive); } p->rcount++; /* count number of suspend */ } + return suspended; } static ERTS_INLINE void -resume_process(Process *p) +resume_process(Process *p, ErtsProcLocks locks) { - erts_aint32_t state; + erts_aint32_t state, enq_prio = -1; + int enqueue; + ERTS_SMP_LC_ASSERT(ERTS_PROC_LOCK_STATUS & erts_proc_lc_my_proc_locks(p)); ASSERT(p->rcount > 0); @@ -4872,14 +6593,17 @@ resume_process(Process *p) if (--p->rcount > 0) /* multiple suspend */ return; - state = erts_smp_atomic32_read_band_mb(&p->state, ~ERTS_PSFLG_SUSPENDED); - state &= ~ERTS_PSFLG_SUSPENDED; - if ((state & (ERTS_PSFLG_EXITING - | ERTS_PSFLG_ACTIVE - | ERTS_PSFLG_IN_RUNQ - | ERTS_PSFLG_RUNNING)) == ERTS_PSFLG_ACTIVE) { - schedule_process(p, state, 1); - } + state = erts_smp_atomic32_read_nob(&p->state); + enqueue = change_proc_schedule_state(p, + ERTS_PSFLG_SUSPENDED, + 0, + &state, + &enq_prio, + locks); + if (enqueue) + add2runq(enqueue > 0 ? p : make_proxy_proc(NULL, p, enq_prio), + state, + enq_prio); } int @@ -4901,6 +6625,12 @@ static void scheduler_ix_resume_wake(Uint ix) { ErtsSchedulerSleepInfo *ssi = ERTS_SCHED_SLEEP_INFO_IX(ix); + scheduler_ssi_resume_wake(ssi); +} + +static void +scheduler_ssi_resume_wake(ErtsSchedulerSleepInfo *ssi) +{ erts_aint32_t xflgs = (ERTS_SSI_FLG_SLEEPING | ERTS_SSI_FLG_TSE_SLEEPING | ERTS_SSI_FLG_WAITING @@ -4991,6 +6721,304 @@ sched_set_suspended_sleeptype(ErtsSchedulerSleepInfo *ssi) } } +#ifdef ERTS_DIRTY_SCHEDULERS + +static void +suspend_scheduler(ErtsSchedulerData *esdp) +{ + erts_aint32_t flgs; + erts_aint32_t changing; +#ifdef ERTS_DIRTY_SCHEDULERS + long no = (long) (ERTS_SCHEDULER_IS_DIRTY(esdp) + ? ERTS_DIRTY_SCHEDULER_NO(esdp) + : esdp->no); +#else + long no = (long) esdp->no; +#endif + ErtsSchedulerSleepInfo *ssi = esdp->ssi; + long active_schedulers; + int curr_online = 1; + int wake = 0; + erts_aint32_t aux_work; + int thr_prgr_active = 1; + ErtsStuckBoundProcesses sbp = {NULL, NULL}; + int* ss_onlinep; + int* ss_curr_onlinep; + int* ss_wait_curr_onlinep; + long* ss_wait_activep; + long ss_wait_active_target; + erts_smp_atomic32_t* ss_changingp; + erts_smp_atomic32_t* ss_activep; + + /* + * Schedulers may be suspended in two different ways: + * - A scheduler may be suspended since it is not online. + * All schedulers with scheduler ids greater than + * schdlr_sspnd.online are suspended; same for dirty + * schedulers and schdlr_sspnd.dirty_cpu_online and + * schdlr_sspnd.dirty_io_online. + * - Multi scheduling is blocked. All schedulers except the + * scheduler with scheduler id 1 are suspended, and all + * dirty CPU and dirty I/O schedulers are suspended. + * + * Regardless of why a scheduler is suspended, it ends up here. + */ + + ASSERT(ERTS_SCHEDULER_IS_DIRTY(esdp) || no != 1); + +#ifdef ERTS_DIRTY_SCHEDULERS + if (ERTS_SCHEDULER_IS_DIRTY(esdp)) { + if (erts_smp_mtx_trylock(&schdlr_sspnd.mtx) == EBUSY) { + erts_smp_runq_unlock(esdp->run_queue); + erts_smp_mtx_lock(&schdlr_sspnd.mtx); + erts_smp_runq_lock(esdp->run_queue); + } + if (ongoing_multi_scheduling_block()) + evacuate_run_queue(esdp->run_queue, &sbp); + } else +#endif + evacuate_run_queue(esdp->run_queue, &sbp); + + erts_smp_runq_unlock(esdp->run_queue); + +#ifdef ERTS_DIRTY_SCHEDULERS + if (!ERTS_SCHEDULER_IS_DIRTY(esdp)) +#endif + { + erts_sched_check_cpu_bind_prep_suspend(esdp); + + if (erts_system_profile_flags.scheduler) + profile_scheduler(make_small(esdp->no), am_inactive); + + sched_wall_time_change(esdp, 0); + + erts_smp_mtx_lock(&schdlr_sspnd.mtx); + } + + flgs = sched_prep_spin_suspended(ssi, ERTS_SSI_FLG_SUSPENDED); + if (flgs & ERTS_SSI_FLG_SUSPENDED) { + +#ifdef ERTS_DIRTY_SCHEDULERS + if (ERTS_SCHEDULER_IS_DIRTY(esdp)) { + if (ERTS_RUNQ_IS_DIRTY_CPU_RUNQ(esdp->run_queue)) { + active_schedulers = erts_smp_atomic32_dec_read_nob(&schdlr_sspnd.dirty_cpu_active); + ASSERT(active_schedulers >= 0); + changing = erts_smp_atomic32_read_nob(&schdlr_sspnd.dirty_cpu_changing); + ss_onlinep = &schdlr_sspnd.dirty_cpu_online; + ss_curr_onlinep = &schdlr_sspnd.dirty_cpu_curr_online; + ss_wait_curr_onlinep = &schdlr_sspnd.dirty_cpu_wait_curr_online; + ss_changingp = &schdlr_sspnd.dirty_cpu_changing; + ss_wait_activep = &schdlr_sspnd.msb.dirty_cpu_wait_active; + ss_activep = &schdlr_sspnd.dirty_cpu_active; + } else { + active_schedulers = erts_smp_atomic32_dec_read_nob(&schdlr_sspnd.dirty_io_active); + ASSERT(active_schedulers >= 0); + changing = erts_smp_atomic32_read_nob(&schdlr_sspnd.dirty_io_changing); + ss_onlinep = &schdlr_sspnd.dirty_io_online; + ss_curr_onlinep = &schdlr_sspnd.dirty_io_curr_online; + ss_wait_curr_onlinep = &schdlr_sspnd.dirty_io_wait_curr_online; + ss_changingp = &schdlr_sspnd.dirty_io_changing; + ss_wait_activep = &schdlr_sspnd.msb.dirty_io_wait_active; + ss_activep = &schdlr_sspnd.dirty_io_active; + } + ss_wait_active_target = 0; + } + else +#endif + { + active_schedulers = erts_smp_atomic32_dec_read_nob(&schdlr_sspnd.active); + ASSERT(active_schedulers >= 1); + changing = erts_smp_atomic32_read_nob(&schdlr_sspnd.changing); + ss_onlinep = &schdlr_sspnd.online; + ss_curr_onlinep = &schdlr_sspnd.curr_online; + ss_wait_curr_onlinep = &schdlr_sspnd.wait_curr_online; + ss_changingp = &schdlr_sspnd.changing; + ss_wait_activep = &schdlr_sspnd.msb.wait_active; + ss_activep = &schdlr_sspnd.active; + ss_wait_active_target = 1; + } + if (changing & ERTS_SCHDLR_SSPND_CHNG_MSB) { + if (active_schedulers == *ss_wait_activep) + wake = 1; + if (active_schedulers == ss_wait_active_target) { + changing = erts_smp_atomic32_read_band_nob(ss_changingp, + ~ERTS_SCHDLR_SSPND_CHNG_MSB); + changing &= ~ERTS_SCHDLR_SSPND_CHNG_MSB; + } + } + + while (1) { + if (changing & ERTS_SCHDLR_SSPND_CHNG_ONLN) { + int changed = 0; + if (no > *ss_onlinep && curr_online) { + (*ss_curr_onlinep)--; + curr_online = 0; + changed = 1; + } + else if (no <= *ss_onlinep && !curr_online) { + (*ss_curr_onlinep)++; + curr_online = 1; + changed = 1; + } + if (changed + && *ss_curr_onlinep == *ss_wait_curr_onlinep) + wake = 1; + if (*ss_onlinep == *ss_curr_onlinep) { + changing = erts_smp_atomic32_read_band_nob(ss_changingp, + ~ERTS_SCHDLR_SSPND_CHNG_ONLN); + changing &= ~ERTS_SCHDLR_SSPND_CHNG_ONLN; + } + } + + if (wake) { + erts_smp_cnd_signal(&schdlr_sspnd.cnd); + wake = 0; + } + + if (curr_online && !ongoing_multi_scheduling_block()) { + flgs = erts_smp_atomic32_read_acqb(&ssi->flags); + if (!(flgs & ERTS_SSI_FLG_SUSPENDED)) + break; + } + erts_smp_mtx_unlock(&schdlr_sspnd.mtx); + + while (1) { + erts_aint32_t qmask; + erts_aint32_t flgs; + + qmask = (ERTS_RUNQ_FLGS_GET(esdp->run_queue) + & ERTS_RUNQ_FLGS_QMASK); + aux_work = erts_atomic32_read_acqb(&ssi->aux_work); + if (aux_work|qmask) { + if (!ERTS_SCHEDULER_IS_DIRTY(esdp)) { + if (!thr_prgr_active) { + erts_thr_progress_active(esdp, thr_prgr_active = 1); + sched_wall_time_change(esdp, 1); + } + if (aux_work) + aux_work = handle_aux_work(&esdp->aux_work_data, + aux_work, + 1); + + if (aux_work && erts_thr_progress_update(esdp)) + erts_thr_progress_leader_update(esdp); + } + if (qmask) { +#ifdef ERTS_DIRTY_SCHEDULERS + if (ERTS_SCHEDULER_IS_DIRTY(esdp)) { + erts_smp_mtx_lock(&schdlr_sspnd.mtx); + erts_smp_runq_lock(esdp->run_queue); + if (ongoing_multi_scheduling_block()) + evacuate_run_queue(esdp->run_queue, &sbp); + erts_smp_runq_unlock(esdp->run_queue); + erts_smp_mtx_unlock(&schdlr_sspnd.mtx); + } else +#endif + { + erts_smp_runq_lock(esdp->run_queue); + evacuate_run_queue(esdp->run_queue, &sbp); + erts_smp_runq_unlock(esdp->run_queue); + } + } + } + + if (!aux_work) { +#ifdef ERTS_DIRTY_SCHEDULERS + if (!ERTS_SCHEDULER_IS_DIRTY(esdp)) +#endif + { + if (thr_prgr_active) { + erts_thr_progress_active(esdp, thr_prgr_active = 0); + sched_wall_time_change(esdp, 0); + } + erts_thr_progress_prepare_wait(esdp); + } + flgs = sched_spin_suspended(ssi, + ERTS_SCHED_SUSPEND_SLEEP_SPINCOUNT); + if (flgs == (ERTS_SSI_FLG_SLEEPING + | ERTS_SSI_FLG_WAITING + | ERTS_SSI_FLG_SUSPENDED)) { + flgs = sched_set_suspended_sleeptype(ssi); + if (flgs == (ERTS_SSI_FLG_SLEEPING + | ERTS_SSI_FLG_TSE_SLEEPING + | ERTS_SSI_FLG_WAITING + | ERTS_SSI_FLG_SUSPENDED)) { + int res; + + do { + res = erts_tse_twait(ssi->event, -1); + } while (res == EINTR); + } + } +#ifdef ERTS_DIRTY_SCHEDULERS + if (!ERTS_SCHEDULER_IS_DIRTY(esdp)) +#endif + erts_thr_progress_finalize_wait(esdp); + } + + flgs = sched_prep_spin_suspended(ssi, (ERTS_SSI_FLG_WAITING + | ERTS_SSI_FLG_SUSPENDED)); + if (!(flgs & ERTS_SSI_FLG_SUSPENDED)) + break; + changing = erts_smp_atomic32_read_nob(ss_changingp); + if (changing & ~ERTS_SCHDLR_SSPND_CHNG_WAITER) + break; + } + + erts_smp_mtx_lock(&schdlr_sspnd.mtx); + changing = erts_smp_atomic32_read_nob(ss_changingp); + } + + active_schedulers = erts_smp_atomic32_inc_read_nob(ss_activep); + changing = erts_smp_atomic32_read_nob(ss_changingp); + if ((changing & ERTS_SCHDLR_SSPND_CHNG_MSB) + && *ss_onlinep == active_schedulers) { + erts_smp_atomic32_read_band_nob(ss_changingp, + ~ERTS_SCHDLR_SSPND_CHNG_MSB); + } + +#ifdef ERTS_DIRTY_SCHEDULERS + if (!ERTS_SCHEDULER_IS_DIRTY(esdp)) +#endif + ASSERT(no <= *ss_onlinep); + ASSERT(!ongoing_multi_scheduling_block()); + + } + + erts_smp_mtx_unlock(&schdlr_sspnd.mtx); + + ASSERT(curr_online); + +#ifdef ERTS_DIRTY_SCHEDULERS + if (!ERTS_SCHEDULER_IS_DIRTY(esdp)) +#endif + { + if (erts_system_profile_flags.scheduler) + profile_scheduler(make_small(esdp->no), am_active); + + if (!thr_prgr_active) { + erts_thr_progress_active(esdp, thr_prgr_active = 1); + sched_wall_time_change(esdp, 1); + } + } + + if (!ERTS_SCHEDULER_IS_DIRTY(esdp)) + (void) erts_get_monotonic_time(esdp); + erts_smp_runq_lock(esdp->run_queue); + non_empty_runq(esdp->run_queue); + +#ifdef ERTS_DIRTY_SCHEDULERS + if (!ERTS_SCHEDULER_IS_DIRTY(esdp)) +#endif + { + schedule_bound_processes(esdp->run_queue, &sbp); + + erts_sched_check_cpu_bind_post_suspend(esdp); + } +} + +#else /* !ERTS_DIRTY_SCHEDULERS */ + static void suspend_scheduler(ErtsSchedulerData *esdp) { @@ -5083,6 +7111,7 @@ suspend_scheduler(ErtsSchedulerData *esdp) erts_smp_mtx_unlock(&schdlr_sspnd.mtx); while (1) { + ErtsMonotonicTime current_time; erts_aint32_t qmask; erts_aint32_t flgs; @@ -5090,16 +7119,18 @@ suspend_scheduler(ErtsSchedulerData *esdp) & ERTS_RUNQ_FLGS_QMASK); aux_work = erts_atomic32_read_acqb(&ssi->aux_work); if (aux_work|qmask) { - if (!thr_prgr_active) { - erts_thr_progress_active(esdp, thr_prgr_active = 1); - sched_wall_time_change(esdp, 1); + if (!ERTS_SCHEDULER_IS_DIRTY(esdp)) { + if (!thr_prgr_active) { + erts_thr_progress_active(esdp, thr_prgr_active = 1); + sched_wall_time_change(esdp, 1); + } + if (aux_work) + aux_work = handle_aux_work(&esdp->aux_work_data, + aux_work, + 1); + if (aux_work && erts_thr_progress_update(esdp)) + erts_thr_progress_leader_update(esdp); } - if (aux_work) - aux_work = handle_aux_work(&esdp->aux_work_data, - aux_work, - 1); - if (aux_work && erts_thr_progress_update(esdp)) - erts_thr_progress_leader_update(esdp); if (qmask) { erts_smp_runq_lock(esdp->run_queue); evacuate_run_queue(esdp->run_queue, &sbp); @@ -5107,30 +7138,77 @@ suspend_scheduler(ErtsSchedulerData *esdp) } } - if (!aux_work) { - if (thr_prgr_active) { - erts_thr_progress_active(esdp, thr_prgr_active = 0); - sched_wall_time_change(esdp, 0); + if (aux_work) { + if (!ERTS_SCHEDULER_IS_DIRTY(esdp)) { + current_time = erts_get_monotonic_time(esdp); + if (current_time >= erts_next_timeout_time(esdp->next_tmo_ref)) { + if (!thr_prgr_active) { + erts_thr_progress_active(esdp, thr_prgr_active = 1); + sched_wall_time_change(esdp, 1); + } + erts_bump_timers(esdp->timer_wheel, current_time); + } } - erts_thr_progress_prepare_wait(esdp); - flgs = sched_spin_suspended(ssi, - ERTS_SCHED_SUSPEND_SLEEP_SPINCOUNT); - if (flgs == (ERTS_SSI_FLG_SLEEPING - | ERTS_SSI_FLG_WAITING - | ERTS_SSI_FLG_SUSPENDED)) { - flgs = sched_set_suspended_sleeptype(ssi); + } + else { + ErtsMonotonicTime timeout_time; + int do_timeout = 0; + if (!ERTS_SCHEDULER_IS_DIRTY(esdp)) { + timeout_time = erts_check_next_timeout_time(esdp); + current_time = erts_get_monotonic_time(esdp); + do_timeout = (current_time >= timeout_time); + } else + timeout_time = ERTS_MONOTONIC_TIME_MAX; + if (do_timeout) { + if (!thr_prgr_active) { + erts_thr_progress_active(esdp, thr_prgr_active = 1); + sched_wall_time_change(esdp, 1); + } + } + else { + if (!ERTS_SCHEDULER_IS_DIRTY(esdp)) { + if (thr_prgr_active) { + erts_thr_progress_active(esdp, thr_prgr_active = 0); + sched_wall_time_change(esdp, 0); + } + erts_thr_progress_prepare_wait(esdp); + } + flgs = sched_spin_suspended(ssi, + ERTS_SCHED_SUSPEND_SLEEP_SPINCOUNT); if (flgs == (ERTS_SSI_FLG_SLEEPING - | ERTS_SSI_FLG_TSE_SLEEPING | ERTS_SSI_FLG_WAITING | ERTS_SSI_FLG_SUSPENDED)) { - int res; - - do { - res = erts_tse_wait(ssi->event); - } while (res == EINTR); + flgs = sched_set_suspended_sleeptype(ssi); + if (flgs == (ERTS_SSI_FLG_SLEEPING + | ERTS_SSI_FLG_TSE_SLEEPING + | ERTS_SSI_FLG_WAITING + | ERTS_SSI_FLG_SUSPENDED)) { + int res; + + current_time = ERTS_SCHEDULER_IS_DIRTY(esdp) ? 0 : + erts_get_monotonic_time(esdp); + do { + Sint64 timeout; + if (current_time >= timeout_time) + break; + if (!ERTS_SCHEDULER_IS_DIRTY(esdp)) { + timeout = ERTS_MONOTONIC_TO_NSEC(timeout_time + - current_time + - 1) + 1; + } else + timeout = -1; + res = erts_tse_twait(ssi->event, timeout); + current_time = ERTS_SCHEDULER_IS_DIRTY(esdp) ? 0 : + erts_get_monotonic_time(esdp); + } while (res == EINTR); + } } + if (!ERTS_SCHEDULER_IS_DIRTY(esdp)) + erts_thr_progress_finalize_wait(esdp); } - erts_thr_progress_finalize_wait(esdp); + + if (!ERTS_SCHEDULER_IS_DIRTY(esdp) && current_time >= timeout_time) + erts_bump_timers(esdp->timer_wheel, current_time); } flgs = sched_prep_spin_suspended(ssi, (ERTS_SSI_FLG_WAITING @@ -5179,29 +7257,328 @@ suspend_scheduler(ErtsSchedulerData *esdp) erts_sched_check_cpu_bind_post_suspend(esdp); } +#endif + ErtsSchedSuspendResult erts_schedulers_state(Uint *total, Uint *online, Uint *active, + Uint *dirty_cpu, + Uint *dirty_cpu_online, + Uint *dirty_io, int yield_allowed) { - int res; + int res = ERTS_SCHDLR_SSPND_EINVAL; erts_aint32_t changing; erts_smp_mtx_lock(&schdlr_sspnd.mtx); changing = erts_smp_atomic32_read_nob(&schdlr_sspnd.changing); +#ifdef ERTS_DIRTY_SCHEDULERS + changing |= (erts_smp_atomic32_read_nob(&schdlr_sspnd.dirty_cpu_changing) + | erts_smp_atomic32_read_nob(&schdlr_sspnd.dirty_io_changing)); +#endif if (yield_allowed && (changing & ~ERTS_SCHDLR_SSPND_CHNG_WAITER)) res = ERTS_SCHDLR_SSPND_YIELD_RESTART; else { - *active = *online = schdlr_sspnd.online; - if (ongoing_multi_scheduling_block()) + if (active) + *active = schdlr_sspnd.online; + if (online) + *online = schdlr_sspnd.online; + if (ongoing_multi_scheduling_block() && active) *active = 1; +#ifdef ERTS_DIRTY_SCHEDULERS + if (dirty_cpu_online) + *dirty_cpu_online = schdlr_sspnd.dirty_cpu_online; +#endif res = ERTS_SCHDLR_SSPND_DONE; } erts_smp_mtx_unlock(&schdlr_sspnd.mtx); - *total = erts_no_schedulers; + if (total) + *total = erts_no_schedulers; +#ifdef ERTS_DIRTY_SCHEDULERS + if (dirty_cpu) + *dirty_cpu = erts_no_dirty_cpu_schedulers; + if (dirty_io) + *dirty_io = erts_no_dirty_io_schedulers; +#endif return res; } +#ifdef ERTS_DIRTY_SCHEDULERS + +ErtsSchedSuspendResult +erts_set_schedulers_online(Process *p, + ErtsProcLocks plocks, + Sint new_no, + Sint *old_no +#ifdef ERTS_DIRTY_SCHEDULERS + , int dirty_only +#endif + ) +{ + ErtsSchedulerData *esdp; + int ix, res = -1, no, have_unlocked_plocks, end_wait; + erts_aint32_t changing = 0; +#ifdef ERTS_DIRTY_SCHEDULERS + ErtsSchedulerSleepInfo* ssi; + int dirty_no, change_dirty; +#endif + + if (new_no < 1) + return ERTS_SCHDLR_SSPND_EINVAL; +#ifdef ERTS_DIRTY_SCHEDULERS + else if (dirty_only && erts_no_dirty_cpu_schedulers < new_no) + return ERTS_SCHDLR_SSPND_EINVAL; +#endif + else if (erts_no_schedulers < new_no) + return ERTS_SCHDLR_SSPND_EINVAL; + + esdp = ERTS_PROC_GET_SCHDATA(p); + end_wait = 0; + + erts_smp_mtx_lock(&schdlr_sspnd.mtx); + + have_unlocked_plocks = 0; + no = (int) new_no; + +#ifdef ERTS_DIRTY_SCHEDULERS + ASSERT(schdlr_sspnd.dirty_cpu_online <= erts_no_dirty_cpu_schedulers); + if (dirty_only) { + if (no > schdlr_sspnd.online) { + erts_smp_mtx_unlock(&schdlr_sspnd.mtx); + return ERTS_SCHDLR_SSPND_EINVAL; + } + dirty_no = no; + } else { + /* + * Adjust the number of dirty CPU schedulers online relative to the + * adjustment made to the number of normal schedulers online. + */ + int total_pct = erts_no_dirty_cpu_schedulers*100/erts_no_schedulers; + int onln_pct = no*total_pct/schdlr_sspnd.online; + dirty_no = schdlr_sspnd.dirty_cpu_online*onln_pct/100; + if (dirty_no == 0) + dirty_no = 1; + ASSERT(dirty_no <= erts_no_dirty_cpu_schedulers); + } +#endif + changing = erts_smp_atomic32_read_nob(&schdlr_sspnd.changing); +#ifdef ERTS_DIRTY_SCHEDULERS + changing |= erts_smp_atomic32_read_nob(&schdlr_sspnd.dirty_cpu_changing); +#endif + if (changing) { + res = ERTS_SCHDLR_SSPND_YIELD_RESTART; + } + else { + int online = *old_no = schdlr_sspnd.online; +#ifdef ERTS_DIRTY_SCHEDULERS + int dirty_online = schdlr_sspnd.dirty_cpu_online; + + if (dirty_only) { + *old_no = schdlr_sspnd.dirty_cpu_online; + if (dirty_no == schdlr_sspnd.dirty_cpu_online) { + res = ERTS_SCHDLR_SSPND_DONE; + } + change_dirty = 1; + } else { +#endif + if (no == schdlr_sspnd.online) { +#ifdef ERTS_DIRTY_SCHEDULERS + dirty_only = 1; + if (dirty_no == schdlr_sspnd.dirty_cpu_online) +#endif + res = ERTS_SCHDLR_SSPND_DONE; +#ifdef ERTS_DIRTY_SCHEDULERS + else + change_dirty = 1; +#endif + } +#ifdef ERTS_DIRTY_SCHEDULERS + else + change_dirty = (dirty_no != schdlr_sspnd.dirty_cpu_online); + } +#endif + if (res == -1) + { + int increase = (no > online); +#ifdef ERTS_DIRTY_SCHEDULERS + if (!dirty_only) { +#endif + ERTS_SCHDLR_SSPND_CHNG_SET((ERTS_SCHDLR_SSPND_CHNG_ONLN + | ERTS_SCHDLR_SSPND_CHNG_WAITER), 0); + schdlr_sspnd.online = no; +#ifdef ERTS_DIRTY_SCHEDULERS + } else + increase = (dirty_no > dirty_online); + if (change_dirty) { + ERTS_SCHDLR_SSPND_DIRTY_CPU_CHNG_SET((ERTS_SCHDLR_SSPND_CHNG_ONLN + | ERTS_SCHDLR_SSPND_CHNG_WAITER), 0); + schdlr_sspnd.dirty_cpu_online = dirty_no; + } +#endif + if (increase) { + int ix; +#ifdef ERTS_DIRTY_SCHEDULERS + if (!dirty_only) { +#endif + schdlr_sspnd.wait_curr_online = no; + if (ongoing_multi_scheduling_block()) { + for (ix = online; ix < no; ix++) + erts_sched_poke(ERTS_SCHED_SLEEP_INFO_IX(ix)); + } + else { + if (plocks) { + have_unlocked_plocks = 1; + erts_smp_proc_unlock(p, plocks); + } + change_no_used_runqs(no); + + for (ix = online; ix < no; ix++) + resume_run_queue(ERTS_RUNQ_IX(ix)); + + for (ix = no; ix < erts_no_run_queues; ix++) + suspend_run_queue(ERTS_RUNQ_IX(ix)); + } +#ifdef ERTS_DIRTY_SCHEDULERS + } + if (change_dirty) { + schdlr_sspnd.dirty_cpu_wait_curr_online = dirty_no; + ASSERT(schdlr_sspnd.dirty_cpu_curr_online != + schdlr_sspnd.dirty_cpu_wait_curr_online); + if (ongoing_multi_scheduling_block()) { + for (ix = dirty_online; ix < dirty_no; ix++) { + ssi = ERTS_DIRTY_CPU_SCHED_SLEEP_INFO_IX(ix); + erts_sched_poke(ssi); + } + } else { + for (ix = dirty_online; ix < dirty_no; ix++) { + ssi = ERTS_DIRTY_CPU_SCHED_SLEEP_INFO_IX(ix); + scheduler_ssi_resume_wake(ssi); + erts_smp_atomic32_read_band_nob(&ssi->flags, + ~ERTS_SSI_FLG_SUSPENDED); + } + wake_dirty_schedulers(ERTS_DIRTY_CPU_RUNQ, 0); + } + } +#endif + res = ERTS_SCHDLR_SSPND_DONE; + } + else /* if (no < online) */ { +#ifdef ERTS_DIRTY_SCHEDULERS + if (change_dirty) { + schdlr_sspnd.dirty_cpu_wait_curr_online = dirty_no; + ASSERT(schdlr_sspnd.dirty_cpu_curr_online != + schdlr_sspnd.dirty_cpu_wait_curr_online); + if (ongoing_multi_scheduling_block()) { + for (ix = dirty_no; ix < dirty_online; ix++) { + ssi = ERTS_DIRTY_CPU_SCHED_SLEEP_INFO_IX(ix); + erts_sched_poke(ssi); + } + } else { + for (ix = dirty_no; ix < dirty_online; ix++) { + ssi = ERTS_DIRTY_CPU_SCHED_SLEEP_INFO_IX(ix); + erts_smp_atomic32_read_bor_nob(&ssi->flags, + ERTS_SSI_FLG_SUSPENDED); + } + wake_dirty_schedulers(ERTS_DIRTY_CPU_RUNQ, 0); + } + } + if (dirty_only) { + res = ERTS_SCHDLR_SSPND_DONE; + } + else +#endif + { + if (p->scheduler_data->no <= no) { + res = ERTS_SCHDLR_SSPND_DONE; + schdlr_sspnd.wait_curr_online = no; + } + else { + /* + * Yield! Current process needs to migrate + * before bif returns. + */ + res = ERTS_SCHDLR_SSPND_YIELD_DONE; + schdlr_sspnd.wait_curr_online = no+1; + } + + if (ongoing_multi_scheduling_block()) { + for (ix = no; ix < online; ix++) + erts_sched_poke(ERTS_SCHED_SLEEP_INFO_IX(ix)); + } + else { + if (plocks) { + have_unlocked_plocks = 1; + erts_smp_proc_unlock(p, plocks); + } + + change_no_used_runqs(no); + for (ix = no; ix < erts_no_run_queues; ix++) + suspend_run_queue(ERTS_RUNQ_IX(ix)); + + for (ix = no; ix < online; ix++) { + ErtsRunQueue *rq = ERTS_RUNQ_IX(ix); + wake_scheduler(rq); + } + } + } + } + +#ifdef ERTS_DIRTY_SCHEDULERS + if (change_dirty) { + while (schdlr_sspnd.dirty_cpu_curr_online != schdlr_sspnd.dirty_cpu_wait_curr_online) + erts_smp_cnd_wait(&schdlr_sspnd.cnd, &schdlr_sspnd.mtx); + ERTS_SCHDLR_SSPND_DIRTY_CPU_CHNG_SET(0, ERTS_SCHDLR_SSPND_CHNG_WAITER); + erts_smp_atomic32_read_band_nob(&schdlr_sspnd.dirty_cpu_changing, + ~ERTS_SCHDLR_SSPND_CHNG_WAITER); + } + if (!dirty_only) +#endif + { + if (schdlr_sspnd.curr_online != schdlr_sspnd.wait_curr_online) { + erts_smp_mtx_unlock(&schdlr_sspnd.mtx); + if (plocks && !have_unlocked_plocks) { + have_unlocked_plocks = 1; + erts_smp_proc_unlock(p, plocks); + } + erts_thr_progress_active(esdp, 0); + erts_thr_progress_prepare_wait(esdp); + end_wait = 1; + erts_smp_mtx_lock(&schdlr_sspnd.mtx); + } + + while (schdlr_sspnd.curr_online != schdlr_sspnd.wait_curr_online) + erts_smp_cnd_wait(&schdlr_sspnd.cnd, &schdlr_sspnd.mtx); + + ASSERT(res != ERTS_SCHDLR_SSPND_DONE + ? (ERTS_SCHDLR_SSPND_CHNG_WAITER + & erts_smp_atomic32_read_nob(&schdlr_sspnd.changing)) + : (ERTS_SCHDLR_SSPND_CHNG_WAITER + == erts_smp_atomic32_read_nob(&schdlr_sspnd.changing))); + erts_smp_atomic32_read_band_nob(&schdlr_sspnd.changing, + ~ERTS_SCHDLR_SSPND_CHNG_WAITER); + } + } + } + + erts_smp_mtx_unlock(&schdlr_sspnd.mtx); +#ifdef ERTS_DIRTY_SCHEDULERS + ASSERT(schdlr_sspnd.dirty_cpu_online <= schdlr_sspnd.online); + if (!dirty_only) +#endif + { + if (end_wait) { + erts_thr_progress_finalize_wait(esdp); + erts_thr_progress_active(esdp, 1); + } + if (have_unlocked_plocks) + erts_smp_proc_lock(p, plocks); + } + + return res; +} + +#else /* !ERTS_DIRTY_SCHEDULERS */ + ErtsSchedSuspendResult erts_set_schedulers_online(Process *p, ErtsProcLocks plocks, @@ -5288,7 +7665,7 @@ erts_set_schedulers_online(Process *p, for (ix = no; ix < online; ix++) { ErtsRunQueue *rq = ERTS_RUNQ_IX(ix); - wake_scheduler(rq, 0); + wake_scheduler(rq); } } } @@ -5330,15 +7707,24 @@ erts_set_schedulers_online(Process *p, return res; } +#endif + ErtsSchedSuspendResult erts_block_multi_scheduling(Process *p, ErtsProcLocks plocks, int on, int all) { - int ix, res, have_unlocked_plocks = 0; + int ix, res, have_unlocked_plocks = 0, online; erts_aint32_t changing; ErtsProcList *plp; +#ifdef ERTS_DIRTY_SCHEDULERS + ErtsSchedulerSleepInfo* ssi; +#endif erts_smp_mtx_lock(&schdlr_sspnd.mtx); changing = erts_smp_atomic32_read_nob(&schdlr_sspnd.changing); +#ifdef ERTS_DIRTY_SCHEDULERS + changing |= (erts_smp_atomic32_read_nob(&schdlr_sspnd.dirty_cpu_changing) + | erts_smp_atomic32_read_nob(&schdlr_sspnd.dirty_io_changing)); +#endif if (changing) { res = ERTS_SCHDLR_SSPND_YIELD_RESTART; /* Yield */ } @@ -5348,10 +7734,13 @@ erts_block_multi_scheduling(Process *p, ErtsProcLocks plocks, int on, int all) erts_proclist_store_last(&schdlr_sspnd.msb.procs, plp); p->flags |= F_HAVE_BLCKD_MSCHED; ASSERT(erts_smp_atomic32_read_nob(&schdlr_sspnd.active) == 1); +#ifdef ERTS_DIRTY_SCHEDULERS + ASSERT(erts_smp_atomic32_read_nob(&schdlr_sspnd.dirty_cpu_active) == 0); + ASSERT(erts_smp_atomic32_read_nob(&schdlr_sspnd.dirty_io_active) == 0); +#endif ASSERT(p->scheduler_data->no == 1); res = ERTS_SCHDLR_SSPND_DONE_MSCHED_BLOCKED; - } - else { + } else { int online = schdlr_sspnd.online; p->flags |= F_HAVE_BLCKD_MSCHED; if (plocks) { @@ -5363,6 +7752,35 @@ erts_block_multi_scheduling(Process *p, ErtsProcLocks plocks, int on, int all) if (online == 1) { res = ERTS_SCHDLR_SSPND_DONE_MSCHED_BLOCKED; ASSERT(erts_smp_atomic32_read_nob(&schdlr_sspnd.active) == 1); +#ifdef ERTS_DIRTY_SCHEDULERS + ASSERT(erts_smp_atomic32_read_nob(&schdlr_sspnd.dirty_cpu_active) == 1); + ASSERT(!(erts_smp_atomic32_read_nob(&ERTS_DIRTY_CPU_SCHED_SLEEP_INFO_IX(0)->flags) + & ERTS_SSI_FLG_SUSPENDED)); + schdlr_sspnd.msb.dirty_cpu_wait_active = 0; + ERTS_SCHDLR_SSPND_DIRTY_CPU_CHNG_SET((ERTS_SCHDLR_SSPND_CHNG_MSB + | ERTS_SCHDLR_SSPND_CHNG_WAITER), 0); + ssi = ERTS_DIRTY_CPU_SCHED_SLEEP_INFO_IX(0); + erts_smp_atomic32_read_bor_nob(&ssi->flags, ERTS_SSI_FLG_SUSPENDED); + wake_dirty_schedulers(ERTS_DIRTY_CPU_RUNQ, 0); + while (erts_smp_atomic32_read_nob(&schdlr_sspnd.dirty_cpu_active) + != schdlr_sspnd.msb.dirty_cpu_wait_active) + erts_smp_cnd_wait(&schdlr_sspnd.cnd, &schdlr_sspnd.mtx); + ERTS_SCHDLR_SSPND_DIRTY_CPU_CHNG_SET(0, ERTS_SCHDLR_SSPND_CHNG_WAITER); + + schdlr_sspnd.msb.dirty_io_wait_active = 0; + ERTS_SCHDLR_SSPND_DIRTY_IO_CHNG_SET((ERTS_SCHDLR_SSPND_CHNG_MSB + | ERTS_SCHDLR_SSPND_CHNG_WAITER), 0); + for (ix = 0; ix < erts_no_dirty_io_schedulers; ix++) { + ssi = ERTS_DIRTY_IO_SCHED_SLEEP_INFO_IX(ix); + erts_smp_atomic32_read_bor_nob(&ssi->flags, + ERTS_SSI_FLG_SUSPENDED); + } + wake_dirty_schedulers(ERTS_DIRTY_IO_RUNQ, 0); + while (erts_smp_atomic32_read_nob(&schdlr_sspnd.dirty_io_active) + != schdlr_sspnd.msb.dirty_io_wait_active) + erts_smp_cnd_wait(&schdlr_sspnd.cnd, &schdlr_sspnd.mtx); + ERTS_SCHDLR_SSPND_DIRTY_IO_CHNG_SET(0, ERTS_SCHDLR_SSPND_CHNG_WAITER); +#endif ASSERT(p->scheduler_data->no == 1); } else { @@ -5381,13 +7799,44 @@ erts_block_multi_scheduling(Process *p, ErtsProcLocks plocks, int on, int all) schdlr_sspnd.msb.wait_active = 2; } +#ifdef ERTS_DIRTY_SCHEDULERS + schdlr_sspnd.msb.dirty_cpu_wait_active = 0; + ERTS_SCHDLR_SSPND_DIRTY_CPU_CHNG_SET((ERTS_SCHDLR_SSPND_CHNG_MSB + | ERTS_SCHDLR_SSPND_CHNG_WAITER), 0); + for (ix = 0; ix < erts_no_dirty_cpu_schedulers; ix++) { + ssi = ERTS_DIRTY_CPU_SCHED_SLEEP_INFO_IX(ix); + erts_smp_atomic32_read_bor_nob(&ssi->flags, + ERTS_SSI_FLG_SUSPENDED); + } + wake_dirty_schedulers(ERTS_DIRTY_CPU_RUNQ, 0); + while (erts_smp_atomic32_read_nob(&schdlr_sspnd.dirty_cpu_active) + != schdlr_sspnd.msb.dirty_cpu_wait_active) + erts_smp_cnd_wait(&schdlr_sspnd.cnd, &schdlr_sspnd.mtx); + ERTS_SCHDLR_SSPND_DIRTY_CPU_CHNG_SET(0, ERTS_SCHDLR_SSPND_CHNG_WAITER); + ASSERT(schdlr_sspnd.dirty_cpu_curr_online == schdlr_sspnd.dirty_cpu_online); + + schdlr_sspnd.msb.dirty_io_wait_active = 0; + ERTS_SCHDLR_SSPND_DIRTY_IO_CHNG_SET((ERTS_SCHDLR_SSPND_CHNG_MSB + | ERTS_SCHDLR_SSPND_CHNG_WAITER), 0); + for (ix = 0; ix < erts_no_dirty_io_schedulers; ix++) { + ssi = ERTS_DIRTY_IO_SCHED_SLEEP_INFO_IX(ix); + erts_smp_atomic32_read_bor_nob(&ssi->flags, + ERTS_SSI_FLG_SUSPENDED); + } + wake_dirty_schedulers(ERTS_DIRTY_IO_RUNQ, 0); + while (erts_smp_atomic32_read_nob(&schdlr_sspnd.dirty_io_active) + != schdlr_sspnd.msb.dirty_io_wait_active) + erts_smp_cnd_wait(&schdlr_sspnd.cnd, &schdlr_sspnd.mtx); + ERTS_SCHDLR_SSPND_DIRTY_IO_CHNG_SET(0, ERTS_SCHDLR_SSPND_CHNG_WAITER); + ASSERT(schdlr_sspnd.dirty_io_curr_online == schdlr_sspnd.dirty_io_online); +#endif change_no_used_runqs(1); for (ix = 1; ix < erts_no_run_queues; ix++) suspend_run_queue(ERTS_RUNQ_IX(ix)); for (ix = 1; ix < online; ix++) { ErtsRunQueue *rq = ERTS_RUNQ_IX(ix); - wake_scheduler(rq, 0); + wake_scheduler(rq); } if (erts_smp_atomic32_read_nob(&schdlr_sspnd.active) @@ -5421,6 +7870,7 @@ erts_block_multi_scheduling(Process *p, ErtsProcLocks plocks, int on, int all) erts_smp_mtx_lock(&schdlr_sspnd.mtx); } + ASSERT(res != ERTS_SCHDLR_SSPND_DONE_MSCHED_BLOCKED ? (ERTS_SCHDLR_SSPND_CHNG_WAITER & erts_smp_atomic32_read_nob(&schdlr_sspnd.changing)) @@ -5461,12 +7911,12 @@ erts_block_multi_scheduling(Process *p, ErtsProcLocks plocks, int on, int all) p->flags &= ~F_HAVE_BLCKD_MSCHED; schdlr_sspnd.msb.ongoing = 0; if (schdlr_sspnd.online == 1) { - /* No schedulers to resume */ + /* No normal schedulers to resume */ ASSERT(erts_smp_atomic32_read_nob(&schdlr_sspnd.active) == 1); ERTS_SCHDLR_SSPND_CHNG_SET(0, ERTS_SCHDLR_SSPND_CHNG_MSB); } else { - int online = schdlr_sspnd.online; + online = schdlr_sspnd.online; if (plocks) { have_unlocked_plocks = 1; erts_smp_proc_unlock(p, plocks); @@ -5481,6 +7931,27 @@ erts_block_multi_scheduling(Process *p, ErtsProcLocks plocks, int on, int all) for (ix = online; ix < erts_no_run_queues; ix++) suspend_run_queue(ERTS_RUNQ_IX(ix)); } +#ifdef ERTS_DIRTY_SCHEDULERS + ERTS_SCHDLR_SSPND_DIRTY_CPU_CHNG_SET(ERTS_SCHDLR_SSPND_CHNG_MSB, 0); + schdlr_sspnd.msb.dirty_cpu_wait_active = schdlr_sspnd.dirty_cpu_online; + for (ix = 0; ix < schdlr_sspnd.dirty_cpu_online; ix++) { + ssi = ERTS_DIRTY_CPU_SCHED_SLEEP_INFO_IX(ix); + scheduler_ssi_resume_wake(ssi); + erts_smp_atomic32_read_band_nob(&ssi->flags, + ~ERTS_SSI_FLG_SUSPENDED); + } + wake_dirty_schedulers(ERTS_DIRTY_CPU_RUNQ, 0); + + ERTS_SCHDLR_SSPND_DIRTY_IO_CHNG_SET(ERTS_SCHDLR_SSPND_CHNG_MSB, 0); + schdlr_sspnd.msb.dirty_io_wait_active = erts_no_dirty_io_schedulers; + for (ix = 0; ix < erts_no_dirty_io_schedulers; ix++) { + ssi = ERTS_DIRTY_IO_SCHED_SLEEP_INFO_IX(ix); + scheduler_ssi_resume_wake(ssi); + erts_smp_atomic32_read_band_nob(&ssi->flags, + ~ERTS_SSI_FLG_SUSPENDED); + } + wake_dirty_schedulers(ERTS_DIRTY_IO_RUNQ, 0); +#endif res = ERTS_SCHDLR_SSPND_DONE; } } @@ -5557,7 +8028,18 @@ sched_thread_func(void *vesdp) ErtsSchedulerData *esdp = vesdp; Uint no = esdp->no; #ifdef ERTS_SMP - ERTS_SCHED_SLEEP_INFO_IX(no - 1)->event = erts_tse_fetch(); + erts_tse_t *tse; +#endif + + erts_sched_init_time_sup(esdp); + + (void) ERTS_RUNQ_FLGS_SET_NOB(esdp->run_queue, + ERTS_RUNQ_FLG_EXEC); + +#ifdef ERTS_SMP + tse = erts_tse_fetch(); + erts_tse_prepare_timed(tse); + ERTS_SCHED_SLEEP_INFO_IX(no - 1)->event = tse; callbacks.arg = (void *) esdp->ssi; callbacks.wakeup = thr_prgr_wakeup; callbacks.prepare_wait = thr_prgr_prep_wait; @@ -5607,7 +8089,11 @@ sched_thread_func(void *vesdp) erts_smp_atomic32_read_band_nob(&schdlr_sspnd.changing, ~ERTS_SCHDLR_SSPND_CHNG_ONLN); if (no != 1) +#ifdef ERTS_DIRTY_SCHEDULERS + erts_smp_cnd_broadcast(&schdlr_sspnd.cnd); +#else erts_smp_cnd_signal(&schdlr_sspnd.cnd); +#endif } if (no == 1) { @@ -5631,24 +8117,174 @@ sched_thread_func(void *vesdp) process_main(); /* No schedulers should *ever* terminate */ - erl_exit(ERTS_ABORT_EXIT, + erts_exit(ERTS_ABORT_EXIT, "Scheduler thread number %beu terminated\n", no); return NULL; } +#ifdef ERTS_DIRTY_SCHEDULERS +#ifdef ERTS_SMP +static void* +sched_dirty_cpu_thread_func(void *vesdp) +{ + ErtsThrPrgrCallbacks callbacks; + ErtsSchedulerData *esdp = vesdp; + Uint no = ERTS_DIRTY_SCHEDULER_NO(esdp); + ERTS_DIRTY_SCHEDULER_TYPE(esdp) = ERTS_DIRTY_CPU_SCHEDULER; + ASSERT(no != 0); + ERTS_DIRTY_CPU_SCHED_SLEEP_INFO_IX(no-1)->event = erts_tse_fetch(); + callbacks.arg = (void *) esdp->ssi; + callbacks.wakeup = thr_prgr_wakeup; + callbacks.prepare_wait = NULL; + callbacks.wait = NULL; + callbacks.finalize_wait = NULL; + + esdp->thr_id += erts_no_schedulers; + + erts_thr_progress_register_unmanaged_thread(&callbacks); +#ifdef ERTS_ENABLE_LOCK_CHECK + { + char buf[31]; + erts_snprintf(&buf[0], 31, "dirty cpu scheduler %beu", no); + erts_lc_set_thread_name(&buf[0]); + } +#endif + erts_tsd_set(sched_data_key, vesdp); +#if ERTS_USE_ASYNC_READY_Q + esdp->aux_work_data.async_ready.queue = NULL; +#endif + + erts_proc_lock_prepare_proc_lock_waiter(); + +#ifdef HIPE + hipe_thread_signal_init(); +#endif + erts_thread_init_float(); + + erts_smp_mtx_lock(&schdlr_sspnd.mtx); + ASSERT(erts_smp_atomic32_read_nob(&schdlr_sspnd.dirty_cpu_changing) + & ERTS_SCHDLR_SSPND_CHNG_ONLN); + + if (--schdlr_sspnd.dirty_cpu_curr_online == schdlr_sspnd.dirty_cpu_wait_curr_online) { + erts_smp_atomic32_read_band_nob(&schdlr_sspnd.dirty_cpu_changing, + ~ERTS_SCHDLR_SSPND_CHNG_ONLN); + if (no != 1) + erts_smp_cnd_broadcast(&schdlr_sspnd.cnd); + } + + if (no == 1) { + while (schdlr_sspnd.dirty_cpu_curr_online != schdlr_sspnd.dirty_cpu_wait_curr_online) + erts_smp_cnd_wait(&schdlr_sspnd.cnd, &schdlr_sspnd.mtx); + ERTS_SCHDLR_SSPND_DIRTY_CPU_CHNG_SET(0, ERTS_SCHDLR_SSPND_CHNG_WAITER); + } + erts_smp_mtx_unlock(&schdlr_sspnd.mtx); + + process_main(); + /* No schedulers should *ever* terminate */ + erts_exit(ERTS_ABORT_EXIT, + "Dirty CPU scheduler thread number %beu terminated\n", + no); + return NULL; +} + +static void* +sched_dirty_io_thread_func(void *vesdp) +{ + ErtsThrPrgrCallbacks callbacks; + ErtsSchedulerData *esdp = vesdp; + Uint no = ERTS_DIRTY_SCHEDULER_NO(esdp); + ERTS_DIRTY_SCHEDULER_TYPE(esdp) = ERTS_DIRTY_IO_SCHEDULER; + ASSERT(no != 0); + ERTS_DIRTY_IO_SCHED_SLEEP_INFO_IX(no-1)->event = erts_tse_fetch(); + callbacks.arg = (void *) esdp->ssi; + callbacks.wakeup = thr_prgr_wakeup; + callbacks.prepare_wait = NULL; + callbacks.wait = NULL; + callbacks.finalize_wait = NULL; + + esdp->thr_id += erts_no_schedulers + erts_no_dirty_cpu_schedulers; + + erts_thr_progress_register_unmanaged_thread(&callbacks); +#ifdef ERTS_ENABLE_LOCK_CHECK + { + char buf[31]; + erts_snprintf(&buf[0], 31, "dirty io scheduler %beu", no); + erts_lc_set_thread_name(&buf[0]); + } +#endif + erts_tsd_set(sched_data_key, vesdp); +#if ERTS_USE_ASYNC_READY_Q + esdp->aux_work_data.async_ready.queue = NULL; +#endif + + erts_proc_lock_prepare_proc_lock_waiter(); + +#ifdef HIPE + hipe_thread_signal_init(); +#endif + erts_thread_init_float(); + + erts_smp_mtx_lock(&schdlr_sspnd.mtx); + ASSERT(erts_smp_atomic32_read_nob(&schdlr_sspnd.dirty_io_changing) + & ERTS_SCHDLR_SSPND_CHNG_ONLN); + + if (--schdlr_sspnd.dirty_io_curr_online == schdlr_sspnd.dirty_io_wait_curr_online) { + erts_smp_atomic32_read_band_nob(&schdlr_sspnd.dirty_io_changing, + ~ERTS_SCHDLR_SSPND_CHNG_ONLN); + if (no != 1) + erts_smp_cnd_broadcast(&schdlr_sspnd.cnd); + } + + if (no == 1) { + while (schdlr_sspnd.dirty_io_curr_online != schdlr_sspnd.dirty_io_wait_curr_online) + erts_smp_cnd_wait(&schdlr_sspnd.cnd, &schdlr_sspnd.mtx); + ERTS_SCHDLR_SSPND_DIRTY_IO_CHNG_SET(0, ERTS_SCHDLR_SSPND_CHNG_WAITER); + } + erts_smp_mtx_unlock(&schdlr_sspnd.mtx); + + process_main(); + /* No schedulers should *ever* terminate */ + erts_exit(ERTS_ABORT_EXIT, + "Dirty I/O scheduler thread number %beu terminated\n", + no); + return NULL; +} +#endif +#endif + static ethr_tid aux_tid; void erts_start_schedulers(void) { int res = 0; - Uint actual = 0; + Uint actual; Uint wanted = erts_no_schedulers; Uint wanted_no_schedulers = erts_no_schedulers; + char name[16]; ethr_thr_opts opts = ETHR_THR_OPTS_DEFAULT_INITER; opts.detached = 1; + + opts.name = name; + +#ifdef ERTS_SMP + if (erts_runq_supervision_interval) { + opts.suggested_stack_size = 16; + erts_snprintf(opts.name, 16, "runq_supervisor"); + erts_atomic_init_nob(&runq_supervisor_sleeping, 0); + if (0 != ethr_event_init(&runq_supervision_event)) + erts_exit(ERTS_ERROR_EXIT, "Failed to create run-queue supervision event\n"); + if (0 != ethr_thr_create(&runq_supervisor_tid, + runq_supervisor, + NULL, + &opts)) + erts_exit(ERTS_ERROR_EXIT, "Failed to create run-queue supervision thread\n"); + + } +#endif + opts.suggested_stack_size = erts_sched_thread_suggested_stack_size; if (wanted < 1) @@ -5658,27 +8294,63 @@ erts_start_schedulers(void) res = ENOTSUP; } - while (actual < wanted) { + for (actual = 0; actual < wanted; actual++) { ErtsSchedulerData *esdp = ERTS_SCHEDULER_IX(actual); - actual++; - ASSERT(actual == esdp->no); - res = ethr_thr_create(&esdp->tid,sched_thread_func,(void*)esdp,&opts); + + ASSERT(actual == esdp->no - 1); + + erts_snprintf(opts.name, 16, "%lu_scheduler", actual + 1); + +#ifdef __OSE__ + /* This should be done in the bind strategy */ + opts.coreNo = (actual+1) % ose_num_cpus(); +#endif + + res = ethr_thr_create(&esdp->tid, sched_thread_func, (void*)esdp, &opts); + if (res != 0) { - actual--; - break; + break; } } - + erts_no_schedulers = actual; +#ifdef ERTS_DIRTY_SCHEDULERS +#ifdef ERTS_SMP + { + int ix; + for (ix = 0; ix < erts_no_dirty_cpu_schedulers; ix++) { + ErtsSchedulerData *esdp = ERTS_DIRTY_CPU_SCHEDULER_IX(ix); + erts_snprintf(opts.name, 16, "%d_dirty_cpu_scheduler", ix + 1); + res = ethr_thr_create(&esdp->tid,sched_dirty_cpu_thread_func,(void*)esdp,&opts); + if (res != 0) + erts_exit(ERTS_ERROR_EXIT, "Failed to create dirty cpu scheduler thread %d\n", ix); + } + for (ix = 0; ix < erts_no_dirty_io_schedulers; ix++) { + ErtsSchedulerData *esdp = ERTS_DIRTY_IO_SCHEDULER_IX(ix); + erts_snprintf(opts.name, 16, "%d_dirty_io_scheduler", ix + 1); + res = ethr_thr_create(&esdp->tid,sched_dirty_io_thread_func,(void*)esdp,&opts); + if (res != 0) + erts_exit(ERTS_ERROR_EXIT, "Failed to create dirty io scheduler thread %d\n", ix); + } + } +#endif +#endif + ERTS_THR_MEMORY_BARRIER; + erts_snprintf(opts.name, 16, "aux"); + +#ifdef __OSE__ + opts.coreNo = 0; +#endif /* __OSE__ */ + res = ethr_thr_create(&aux_tid, aux_thread, NULL, &opts); if (res != 0) - erl_exit(1, "Failed to create aux thread\n"); + erts_exit(ERTS_ERROR_EXIT, "Failed to create aux thread\n"); if (actual < 1) - erl_exit(1, + erts_exit(ERTS_ERROR_EXIT, "Failed to create any scheduler-threads: %s (%d)\n", erl_errno_id(res), res); @@ -5692,6 +8364,7 @@ erts_start_schedulers(void) actual, actual == 1 ? " was" : "s were"); erts_send_error_to_logger_nogl(dsbufp); } + } #endif /* ERTS_SMP */ @@ -5793,7 +8466,8 @@ handle_pend_sync_suspend(Process *suspendee, } /* suspender is suspended waiting for suspendee to suspend; resume suspender */ - resume_process(suspender); + ASSERT(suspendee != suspender); + resume_process(suspender, ERTS_PROC_LOCK_STATUS); erts_smp_proc_unlock(suspender, ERTS_PROC_LOCK_STATUS); } } @@ -5828,12 +8502,11 @@ pid2proc_not_running(Process *c_p, ErtsProcLocks c_p_locks, ASSERT(c_p->flags & F_P2PNR_RESCHED); c_p->flags &= ~F_P2PNR_RESCHED; if (!suspend && rp) - resume_process(rp); + resume_process(rp, rp_locks); } else { - rp = erts_pid2proc(c_p, c_p_locks|ERTS_PROC_LOCK_STATUS, - pid, pid_locks|ERTS_PROC_LOCK_STATUS); + pid, ERTS_PROC_LOCK_STATUS); if (!rp) { c_p->flags &= ~F_P2PNR_RESCHED; @@ -5842,39 +8515,84 @@ pid2proc_not_running(Process *c_p, ErtsProcLocks c_p_locks, ASSERT(!(c_p->flags & F_P2PNR_RESCHED)); - if (suspend) { - if (suspend_process(c_p, rp)) - goto done; - } - else { - if (!(ERTS_PSFLG_RUNNING & erts_smp_atomic32_read_acqb(&rp->state))) - goto done; + /* + * Suspend the other process in order to prevent + * it from being selected for normal execution. + * This will however not prevent it from being + * selected for execution of a system task. If + * it is selected for execution of a system task + * we might be blocked for quite a while if the + * try-lock below fails. That is, there is room + * for improvement here... + */ - } + if (!suspend_process(c_p, rp)) { + /* Other process running */ - /* Other process running */ + ASSERT(ERTS_PSFLG_RUNNING + & erts_smp_atomic32_read_nob(&rp->state)); - /* - * If we got pending suspenders and suspend ourselves waiting - * to suspend another process we might deadlock. - * In this case we have to yield, be suspended by - * someone else and then do it all over again. - */ - if (!c_p->pending_suspenders) { - /* Mark rp pending for suspend by c_p */ - add_pend_suspend(rp, c_p->common.id, handle_pend_sync_suspend); - ASSERT(is_nil(c_p->suspendee)); + running: - /* Suspend c_p; when rp is suspended c_p will be resumed. */ - suspend_process(c_p, c_p); - c_p->flags |= F_P2PNR_RESCHED; + /* + * If we got pending suspenders and suspend ourselves waiting + * to suspend another process we might deadlock. + * In this case we have to yield, be suspended by + * someone else and then do it all over again. + */ + if (!c_p->pending_suspenders) { + /* Mark rp pending for suspend by c_p */ + add_pend_suspend(rp, c_p->common.id, handle_pend_sync_suspend); + ASSERT(is_nil(c_p->suspendee)); + + /* Suspend c_p; when rp is suspended c_p will be resumed. */ + suspend_process(c_p, c_p); + c_p->flags |= F_P2PNR_RESCHED; + } + /* Yield (caller is assumed to yield immediately in bif). */ + erts_smp_proc_unlock(rp, ERTS_PROC_LOCK_STATUS); + rp = ERTS_PROC_LOCK_BUSY; + } + else { + ErtsProcLocks need_locks = pid_locks & ~ERTS_PROC_LOCK_STATUS; + if (need_locks && erts_smp_proc_trylock(rp, need_locks) == EBUSY) { + if (ERTS_PSFLG_RUNNING_SYS + & erts_smp_atomic32_read_nob(&rp->state)) { + /* Executing system task... */ + resume_process(rp, ERTS_PROC_LOCK_STATUS); + goto running; + } + erts_smp_proc_unlock(rp, ERTS_PROC_LOCK_STATUS); + /* + * If we are unlucky, the process just got selected for + * execution of a system task. In this case we may be + * blocked here for quite a while... Execution of system + * tasks are fortunately quite rare events. We try to + * avoid this by checking if it is in a state executing + * system tasks (above), but it will not prevent all + * scenarios for a long block here... + */ + rp = erts_pid2proc(c_p, c_p_locks|ERTS_PROC_LOCK_STATUS, + pid, pid_locks|ERTS_PROC_LOCK_STATUS); + if (!rp) + goto done; + } + + /* + * The previous suspend has prevented the process + * from being selected for normal execution regardless + * of locks held or not held on it... + */ + ASSERT(!(ERTS_PSFLG_RUNNING + & erts_smp_atomic32_read_nob(&rp->state))); + + if (!suspend) + resume_process(rp, pid_locks|ERTS_PROC_LOCK_STATUS); } - /* Yield (caller is assumed to yield immediately in bif). */ - erts_smp_proc_unlock(rp, pid_locks|ERTS_PROC_LOCK_STATUS); - rp = ERTS_PROC_LOCK_BUSY; } done: + if (rp && rp != ERTS_PROC_LOCK_BUSY && !(pid_locks & ERTS_PROC_LOCK_STATUS)) erts_smp_proc_unlock(rp, ERTS_PROC_LOCK_STATUS); if (unlock_c_p_status) @@ -5985,7 +8703,8 @@ handle_pend_bif_sync_suspend(Process *suspendee, } /* suspender is suspended waiting for suspendee to suspend; resume suspender */ - resume_process(suspender); + ASSERT(suspender != suspendee); + resume_process(suspender, ERTS_PROC_LOCK_LINK|ERTS_PROC_LOCK_STATUS); erts_smp_proc_unlock(suspender, ERTS_PROC_LOCK_LINK|ERTS_PROC_LOCK_STATUS); } @@ -6345,7 +9064,8 @@ resume_process_1(BIF_ALIST_1) ASSERT(ERTS_PSFLG_SUSPENDED & erts_smp_atomic32_read_nob(&suspendee->state)); - resume_process(suspendee); + ASSERT(BIF_P != suspendee); + resume_process(suspendee, ERTS_PROC_LOCK_STATUS); erts_smp_proc_unlock(suspendee, ERTS_PROC_LOCK_STATUS); } @@ -6367,24 +9087,39 @@ resume_process_1(BIF_ALIST_1) } Uint -erts_run_queues_len(Uint *qlen) +erts_run_queues_len(Uint *qlen, int atomic_queues_read, int incl_active_sched) { int i = 0; Uint len = 0; - ERTS_ATOMIC_FOREACH_RUNQ(rq, - { - Sint pqlen = 0; - int pix; - for (pix = 0; pix < ERTS_NO_PROC_PRIO_LEVELS; pix++) - pqlen += RUNQ_READ_LEN(&rq->procs.prio_info[pix].len); + if (atomic_queues_read) + ERTS_ATOMIC_FOREACH_RUNQ(rq, + { + Sint rq_len = (Sint) erts_smp_atomic32_read_dirty(&rq->len); + ASSERT(rq_len >= 0); + if (incl_active_sched + && (ERTS_RUNQ_FLGS_GET_NOB(rq) & ERTS_RUNQ_FLG_EXEC)) { + rq_len++; + } + if (qlen) + qlen[i++] = rq_len; + len += (Uint) rq_len; + } + ); + else { + for (i = 0; i < erts_no_run_queues; i++) { + ErtsRunQueue *rq = ERTS_RUNQ_IX(i); + Sint rq_len = (Sint) erts_smp_atomic32_read_nob(&rq->len); + ASSERT(rq_len >= 0); + if (incl_active_sched + && (ERTS_RUNQ_FLGS_GET_NOB(rq) & ERTS_RUNQ_FLG_EXEC)) { + rq_len++; + } + if (qlen) + qlen[i] = rq_len; + len += (Uint) rq_len; + } - if (pqlen < 0) - pqlen = 0; - if (qlen) - qlen[i++] = pqlen; - len += pqlen; } - ); return len; } @@ -6475,7 +9210,7 @@ erts_resume(Process* process, ErtsProcLocks process_locks) ERTS_SMP_LC_ASSERT(process_locks == erts_proc_lc_my_proc_locks(process)); if (!(process_locks & ERTS_PROC_LOCK_STATUS)) erts_smp_proc_lock(process, ERTS_PROC_LOCK_STATUS); - resume_process(process); + resume_process(process, process_locks|ERTS_PROC_LOCK_STATUS); if (!(process_locks & ERTS_PROC_LOCK_STATUS)) erts_smp_proc_unlock(process, ERTS_PROC_LOCK_STATUS); } @@ -6494,7 +9229,7 @@ erts_resume_processes(ErtsProcList *list) proc = erts_pid2proc(NULL, 0, plp->pid, ERTS_PROC_LOCK_STATUS); if (proc) { if (erts_proclist_same(plp, proc)) { - resume_process(proc); + resume_process(proc, ERTS_PROC_LOCK_STATUS); nresumed++; } erts_smp_proc_unlock(proc, ERTS_PROC_LOCK_STATUS); @@ -6510,7 +9245,7 @@ Eterm erts_get_process_priority(Process *p) { erts_aint32_t state = erts_smp_atomic32_read_nob(&p->state); - switch (state & ERTS_PSFLG_PRIO_MASK) { + switch (ERTS_PSFLGS_GET_USR_PRIO(state)) { case PRIORITY_MAX: return am_max; case PRIORITY_HIGH: return am_high; case PRIORITY_NORMAL: return am_normal; @@ -6533,20 +9268,74 @@ erts_set_process_priority(Process *p, Eterm value) } a = erts_smp_atomic32_read_nob(&p->state); - if (nprio == (a & ERTS_PSFLG_PRIO_MASK)) + if (nprio == ERTS_PSFLGS_GET_USR_PRIO(a)) oprio = nprio; else { - erts_aint32_t e, n; + int slocked = 0; + erts_aint32_t e, n, aprio; + + if (a & ERTS_PSFLG_ACTIVE_SYS) { + erts_smp_proc_lock(p, ERTS_PROC_LOCK_STATUS); + slocked = 1; + } + do { - oprio = a & ERTS_PSFLG_PRIO_MASK; + oprio = ERTS_PSFLGS_GET_USR_PRIO(a); n = e = a; - ASSERT(!(a & ERTS_PSFLG_IN_RUNQ)); + if (!(a & (ERTS_PSFLG_ACTIVE_SYS|ERTS_PSFLG_DELAYED_SYS))) + aprio = nprio; + else { + int max_qbit; + + if (!slocked) { + erts_smp_proc_lock(p, ERTS_PROC_LOCK_STATUS); + slocked = 1; + } + + max_qbit = 0; + if (a & ERTS_PSFLG_ACTIVE_SYS) + max_qbit |= p->sys_task_qs->qmask; + if (a & ERTS_PSFLG_DELAYED_SYS) { + ErtsProcSysTaskQs *qs; + qs = ERTS_PROC_GET_DELAYED_GC_TASK_QS(p); + ASSERT(qs); + max_qbit |= qs->qmask; + } + max_qbit &= -max_qbit; + switch (max_qbit) { + case MAX_BIT: + aprio = PRIORITY_MAX; + break; + case HIGH_BIT: + aprio = PRIORITY_HIGH; + break; + case NORMAL_BIT: + aprio = PRIORITY_NORMAL; + break; + case LOW_BIT: + aprio = PRIORITY_LOW; + break; + default: + ERTS_INTERNAL_ERROR("Invalid qmask"); + aprio = -1; + } + + if (aprio > nprio) /* low value -> high prio */ + aprio = nprio; + } + + n &= ~(ERTS_PSFLGS_USR_PRIO_MASK + | ERTS_PSFLGS_ACT_PRIO_MASK); + n |= ((nprio << ERTS_PSFLGS_USR_PRIO_OFFSET) + | (aprio << ERTS_PSFLGS_ACT_PRIO_OFFSET)); - n &= ~ERTS_PSFLG_PRIO_MASK; - n |= nprio; a = erts_smp_atomic32_cmpxchg_mb(&p->state, n, e); } while (a != e); + + if (slocked) + erts_smp_proc_unlock(p, ERTS_PROC_LOCK_STATUS); + } switch (oprio) { @@ -6578,8 +9367,8 @@ erts_set_process_priority(Process *p, Eterm value) Process *schedule(Process *p, int calls) { + Process *proxy_p = NULL; ErtsRunQueue *rq; - erts_aint_t dt; ErtsSchedulerData *esdp; int context_reds; int fcalls; @@ -6607,7 +9396,8 @@ Process *schedule(Process *p, int calls) input_reductions = INPUT_REDUCTIONS; } - ERTS_SMP_LC_ASSERT(!erts_thr_progress_is_blocking()); + ERTS_SMP_LC_ASSERT(ERTS_SCHEDULER_IS_DIRTY(erts_get_scheduler_data()) + || !erts_thr_progress_is_blocking()); /* * Clean up after the process being scheduled out. @@ -6620,6 +9410,8 @@ Process *schedule(Process *p, int calls) actual_reds = reds = 0; erts_smp_runq_lock(rq); } else { + sched_out_proc: + #ifdef ERTS_SMP ERTS_SMP_CHK_HAVE_ONLY_MAIN_PROC_LOCK(p); esdp = p->scheduler_data; @@ -6673,10 +9465,11 @@ Process *schedule(Process *p, int calls) esdp->reductions += reds; - schedule_out_process(rq, state, p); /* Returns with rq locked! */ + schedule_out_process(rq, state, p, proxy_p); /* Returns with rq locked! */ + proxy_p = NULL; - ERTS_PROC_REDUCTIONS_EXECUTED(rq, - (int) (state & ERTS_PSFLG_PRIO_MASK), + ERTS_PROC_REDUCTIONS_EXECUTED(esdp, rq, + (int) ERTS_PSFLGS_GET_USR_PRIO(state), reds, actual_reds); @@ -6685,18 +9478,16 @@ Process *schedule(Process *p, int calls) p->scheduler_data = NULL; #endif + erts_smp_proc_unlock(p, ERTS_PROC_LOCK_MAIN|ERTS_PROC_LOCK_STATUS); if (state & ERTS_PSFLG_FREE) { #ifdef ERTS_SMP ASSERT(esdp->free_process == p); esdp->free_process = NULL; -#else - erts_free_proc(p); +#else + erts_proc_dec_refc(p); #endif } - - erts_smp_proc_unlock(p, ERTS_PROC_LOCK_MAIN|ERTS_PROC_LOCK_STATUS); - #ifdef ERTS_SMP ASSERT(!esdp->free_process); #endif @@ -6704,58 +9495,61 @@ Process *schedule(Process *p, int calls) ERTS_SMP_CHK_NO_PROC_LOCKS; - dt = erts_do_time_read_and_reset(); - if (dt) { - erts_smp_runq_unlock(rq); - erts_bump_timer(dt); - erts_smp_runq_lock(rq); + if (!ERTS_SCHEDULER_IS_DIRTY(esdp)) { + if (esdp->check_time_reds >= ERTS_CHECK_TIME_REDS) + (void) erts_get_monotonic_time(esdp); + + if (esdp->last_monotonic_time >= erts_next_timeout_time(esdp->next_tmo_ref)) { + erts_smp_runq_unlock(rq); + erts_bump_timers(esdp->timer_wheel, esdp->last_monotonic_time); + erts_smp_runq_lock(rq); + } } BM_STOP_TIMER(system); } - ERTS_SMP_LC_ASSERT(!erts_thr_progress_is_blocking()); + ERTS_SMP_LC_ASSERT(ERTS_SCHEDULER_IS_DIRTY(esdp) + || !erts_thr_progress_is_blocking()); check_activities_to_run: { #ifdef ERTS_SMP ErtsMigrationPaths *mps; ErtsMigrationPath *mp; - -#ifdef ERTS_SMP - { - ErtsProcList *pnd_xtrs = rq->procs.pending_exiters; - if (erts_proclist_fetch(&pnd_xtrs, NULL)) { - rq->procs.pending_exiters = NULL; - erts_smp_runq_unlock(rq); - handle_pending_exiters(pnd_xtrs); - erts_smp_runq_lock(rq); - } - + ErtsProcList *pnd_xtrs = rq->procs.pending_exiters; + if (erts_proclist_fetch(&pnd_xtrs, NULL)) { + rq->procs.pending_exiters = NULL; + erts_smp_runq_unlock(rq); + handle_pending_exiters(pnd_xtrs); + erts_smp_runq_lock(rq); } -#endif - if (rq->check_balance_reds <= 0) - check_balance(rq); + if (!ERTS_SCHEDULER_IS_DIRTY(esdp)) { + if (rq->check_balance_reds <= 0) + check_balance(rq); - ERTS_SMP_LC_ASSERT(!erts_thr_progress_is_blocking()); - ERTS_SMP_LC_ASSERT(erts_smp_lc_runq_is_locked(rq)); + ERTS_SMP_LC_ASSERT(!erts_thr_progress_is_blocking()); - mps = erts_get_migration_paths_managed(); - mp = &mps->mpath[rq->ix]; + mps = erts_get_migration_paths_managed(); + mp = &mps->mpath[rq->ix]; - if (mp->flags & ERTS_RUNQ_FLGS_IMMIGRATE_QMASK) - immigrate(rq, mp); + if (mp->flags & ERTS_RUNQ_FLGS_IMMIGRATE_QMASK) + immigrate(rq, mp); + } + ERTS_SMP_LC_ASSERT(erts_smp_lc_runq_is_locked(rq)); continue_check_activities_to_run: flags = ERTS_RUNQ_FLGS_GET_NOB(rq); continue_check_activities_to_run_known_flags: - + ASSERT(ERTS_SCHEDULER_IS_DIRTY(esdp) + || flags & ERTS_RUNQ_FLG_NONEMPTY); if (flags & (ERTS_RUNQ_FLG_CHK_CPU_BIND|ERTS_RUNQ_FLG_SUSPENDED)) { - if (flags & ERTS_RUNQ_FLG_SUSPENDED) { + (void) ERTS_RUNQ_FLGS_UNSET_NOB(rq, ERTS_RUNQ_FLG_EXEC); suspend_scheduler(esdp); - flags = ERTS_RUNQ_FLGS_GET_NOB(rq); + flags = ERTS_RUNQ_FLGS_SET_NOB(rq, ERTS_RUNQ_FLG_EXEC); + flags |= ERTS_RUNQ_FLG_EXEC; } if (flags & ERTS_RUNQ_FLG_CHK_CPU_BIND) { flags = ERTS_RUNQ_FLGS_UNSET(rq, ERTS_RUNQ_FLG_CHK_CPU_BIND); @@ -6763,22 +9557,30 @@ Process *schedule(Process *p, int calls) erts_sched_check_cpu_bind(esdp); } } +#ifdef ERTS_DIRTY_SCHEDULERS + else if (ERTS_SCHEDULER_IS_DIRTY(esdp) + && (erts_smp_atomic32_read_acqb(&esdp->ssi->flags) + & ERTS_SSI_FLG_SUSPENDED)) + suspend_scheduler(esdp); +#endif - { + if (!ERTS_SCHEDULER_IS_DIRTY(esdp)) { erts_aint32_t aux_work; int leader_update = erts_thr_progress_update(esdp); aux_work = erts_atomic32_read_acqb(&esdp->ssi->aux_work); - if (aux_work | leader_update) { + if (aux_work | leader_update | ERTS_SCHED_FAIR) { erts_smp_runq_unlock(rq); if (leader_update) erts_thr_progress_leader_update(esdp); + else if (ERTS_SCHED_FAIR) + ERTS_SCHED_FAIR_YIELD(); if (aux_work) handle_aux_work(&esdp->aux_work_data, aux_work, 0); erts_smp_runq_lock(rq); } - } - ERTS_SMP_LC_ASSERT(!erts_thr_progress_is_blocking()); + ERTS_SMP_LC_ASSERT(!erts_thr_progress_is_blocking()); + } ERTS_SMP_LC_ASSERT(erts_smp_lc_runq_is_locked(rq)); #else /* ERTS_SMP */ @@ -6792,6 +9594,17 @@ Process *schedule(Process *p, int calls) flags = ERTS_RUNQ_FLGS_GET_NOB(rq); +#ifdef ERTS_DIRTY_SCHEDULERS + if (ERTS_RUNQ_IX_IS_DIRTY(rq->ix) && rq->halt_in_progress) { + /* + * TODO: if halt in progress, need to put the dirty scheduler + * to sleep somewhere around here to prevent it from picking up + * new work + */ + } + else +#endif + if ((!(flags & ERTS_RUNQ_FLGS_QMASK) && !rq->misc.start) || (rq->halt_in_progress && ERTS_EMPTY_RUNQ_PORTS(rq))) { /* Prepare for scheduler wait */ @@ -6801,20 +9614,16 @@ Process *schedule(Process *p, int calls) rq->wakeup_other = 0; rq->wakeup_other_reds = 0; - empty_runq(rq); - flags = ERTS_RUNQ_FLGS_GET_NOB(rq); - if (flags & ERTS_RUNQ_FLG_SUSPENDED) { - non_empty_runq(rq); + if (flags & ERTS_RUNQ_FLG_SUSPENDED) goto continue_check_activities_to_run_known_flags; - } - else if (!(flags & ERTS_RUNQ_FLG_INACTIVE)) { - if (try_steal_task(rq)) { - non_empty_runq(rq); + if (flags & ERTS_RUNQ_FLG_INACTIVE) + empty_runq(rq); + else { + if (!ERTS_RUNQ_IX_IS_DIRTY(rq->ix) && try_steal_task(rq)) goto continue_check_activities_to_run; - } - (void) ERTS_RUNQ_FLGS_UNSET(rq, ERTS_RUNQ_FLG_PROTECTED); + empty_runq(rq); /* * Check for ERTS_RUNQ_FLG_SUSPENDED has to be done @@ -6823,13 +9632,16 @@ Process *schedule(Process *p, int calls) flags = ERTS_RUNQ_FLGS_GET_NOB(rq); if (flags & ERTS_RUNQ_FLG_SUSPENDED) { non_empty_runq(rq); + flags |= ERTS_RUNQ_FLG_NONEMPTY; goto continue_check_activities_to_run_known_flags; } } - #endif + (void) ERTS_RUNQ_FLGS_UNSET(rq, ERTS_RUNQ_FLG_EXEC); scheduler_wait(&fcalls, esdp, rq); + flags = ERTS_RUNQ_FLGS_SET_NOB(rq, ERTS_RUNQ_FLG_EXEC); + flags |= ERTS_RUNQ_FLG_EXEC; #ifdef ERTS_SMP non_empty_runq(rq); @@ -6837,7 +9649,10 @@ Process *schedule(Process *p, int calls) goto check_activities_to_run; } - else if (fcalls > input_reductions && prepare_for_sys_schedule()) { + else if (!ERTS_SCHEDULER_IS_DIRTY(esdp) && + (fcalls > input_reductions && + prepare_for_sys_schedule(esdp, !0))) { + ErtsMonotonicTime current_time; /* * Schedule system-level activities. */ @@ -6845,15 +9660,15 @@ Process *schedule(Process *p, int calls) erts_smp_atomic32_set_relb(&function_calls, 0); fcalls = 0; - ASSERT(!erts_port_task_have_outstanding_io_tasks()); - #if 0 /* Not needed since we wont wait in sys schedule */ erts_sys_schedule_interrupt(0); #endif erts_smp_runq_unlock(rq); erl_sys_schedule(1); - dt = erts_do_time_read_and_reset(); - if (dt) erts_bump_timer(dt); + + current_time = erts_get_monotonic_time(esdp); + if (current_time >= erts_next_timeout_time(esdp->next_tmo_ref)) + erts_bump_timers(esdp->timer_wheel, current_time); #ifdef ERTS_SMP erts_smp_runq_lock(rq); @@ -6878,7 +9693,9 @@ Process *schedule(Process *p, int calls) if (RUNQ_READ_LEN(&rq->ports.info.len)) { int have_outstanding_io; have_outstanding_io = erts_port_task_execute(rq, &esdp->current_port); - if ((have_outstanding_io && fcalls > 2*input_reductions) + if ((!erts_eager_check_io + && have_outstanding_io + && fcalls > 2*input_reductions) || rq->halt_in_progress) { /* * If we have performed more than 2*INPUT_REDUCTIONS since @@ -6903,6 +9720,7 @@ Process *schedule(Process *p, int calls) * Find a new process to run. */ pick_next_process: { + erts_aint32_t psflg_band_mask; int prio_q; int qmask; @@ -6936,18 +9754,67 @@ Process *schedule(Process *p, int calls) ASSERT(p); /* Wrong qmask in rq->flags? */ + psflg_band_mask = ~(((erts_aint32_t) 1) << (ERTS_PSFLGS_GET_PRQ_PRIO(state) + + ERTS_PSFLGS_IN_PRQ_MASK_OFFSET)); + +#ifdef ERTS_DIRTY_SCHEDULERS + ASSERT((state & (ERTS_PSFLG_DIRTY_CPU_PROC|ERTS_PSFLG_DIRTY_IO_PROC)) != + (ERTS_PSFLG_DIRTY_CPU_PROC|ERTS_PSFLG_DIRTY_IO_PROC)); + if (state & (ERTS_PSFLG_DIRTY_CPU_PROC|ERTS_PSFLG_DIRTY_IO_PROC)) { + ASSERT((ERTS_SCHEDULER_IS_DIRTY_CPU(esdp) && (state & ERTS_PSFLG_DIRTY_CPU_PROC)) || + (ERTS_SCHEDULER_IS_DIRTY_IO(esdp) && (state & ERTS_PSFLG_DIRTY_IO_PROC))); + if (!ERTS_SCHEDULER_IS_DIRTY(esdp) && !(state & ERTS_PSFLG_ACTIVE_SYS)) + goto pick_next_process; + state &= ~(ERTS_PSFLG_DIRTY_CPU_PROC_IN_Q|ERTS_PSFLG_DIRTY_IO_PROC_IN_Q); + } +#endif + + if (!(state & ERTS_PSFLG_PROXY)) + psflg_band_mask &= ~ERTS_PSFLG_IN_RUNQ; + else { + proxy_p = p; + p = erts_proc_lookup_raw(proxy_p->common.id); + if (!p) { + free_proxy_proc(proxy_p); + proxy_p = NULL; + goto pick_next_process; + } + state = erts_smp_atomic32_read_nob(&p->state); + } + while (1) { erts_aint32_t exp, new, tmp; tmp = new = exp = state; - new &= ~ERTS_PSFLG_IN_RUNQ; - tmp = state & (ERTS_PSFLG_SUSPENDED|ERTS_PSFLG_PENDING_EXIT); - if (tmp != ERTS_PSFLG_SUSPENDED) - new |= ERTS_PSFLG_RUNNING; + new &= psflg_band_mask; + if (!(state & (ERTS_PSFLG_RUNNING + | ERTS_PSFLG_RUNNING_SYS))) { + tmp = state & (ERTS_PSFLG_SUSPENDED + | ERTS_PSFLG_PENDING_EXIT + | ERTS_PSFLG_ACTIVE_SYS); + if (tmp != ERTS_PSFLG_SUSPENDED) { + if (state & ERTS_PSFLG_ACTIVE_SYS) + new |= ERTS_PSFLG_RUNNING_SYS; + else + new |= ERTS_PSFLG_RUNNING; + } + } state = erts_smp_atomic32_cmpxchg_relb(&p->state, new, exp); if (state == exp) { - tmp = state & (ERTS_PSFLG_SUSPENDED|ERTS_PSFLG_PENDING_EXIT); - if (tmp == ERTS_PSFLG_SUSPENDED) + if ((state & (ERTS_PSFLG_RUNNING + | ERTS_PSFLG_RUNNING_SYS + | ERTS_PSFLG_FREE)) + || ((state & (ERTS_PSFLG_SUSPENDED + | ERTS_PSFLG_PENDING_EXIT + | ERTS_PSFLG_ACTIVE_SYS)) + == ERTS_PSFLG_SUSPENDED)) { + if (state & ERTS_PSFLG_FREE) + erts_proc_dec_refc(p); + if (proxy_p) { + free_proxy_proc(proxy_p); + proxy_p = NULL; + } goto pick_next_process; + } state = new; break; } @@ -6969,6 +9836,8 @@ Process *schedule(Process *p, int calls) erts_smp_proc_lock(p, ERTS_PROC_LOCK_MAIN|ERTS_PROC_LOCK_STATUS); + state = erts_smp_atomic32_read_nob(&p->state); + if (erts_sched_stat.enabled) { int prio; UWord old = ERTS_PROC_SCHED_ID(p, @@ -6977,7 +9846,11 @@ Process *schedule(Process *p, int calls) (UWord) esdp->no); int migrated = old && old != esdp->no; - prio = (int) (state & ERTS_PSFLG_PRIO_MASK); +#ifdef ERTS_DIRTY_SCHEDULERS + ASSERT(!ERTS_SCHEDULER_IS_DIRTY(esdp)); +#endif + + prio = (int) ERTS_PSFLGS_GET_USR_PRIO(state); erts_smp_spin_lock(&erts_sched_stat.lock); erts_sched_stat.prio[prio].total_executed++; @@ -6997,9 +9870,6 @@ Process *schedule(Process *p, int calls) ASSERT(!p->scheduler_data); p->scheduler_data = esdp; #endif - /* Never run a suspended process */ - ASSERT(!(ERTS_PSFLG_SUSPENDED & erts_smp_atomic32_read_nob(&p->state))); - reds = context_reds; if (IS_TRACED(p)) { @@ -7025,21 +9895,792 @@ Process *schedule(Process *p, int calls) erts_check_my_tracer_proc(p); #endif + if (state & ERTS_PSFLG_RUNNING_SYS) { + reds -= execute_sys_tasks(p, &state, reds); + if (reds <= 0 +#ifdef ERTS_DIRTY_SCHEDULERS + || (state & (ERTS_PSFLG_DIRTY_CPU_PROC|ERTS_PSFLG_DIRTY_IO_PROC)) +#endif + ) { + p->fcalls = reds; + goto sched_out_proc; + } + + ASSERT(state & ERTS_PSFLG_RUNNING_SYS); + ASSERT(!(state & ERTS_PSFLG_RUNNING)); + + while (1) { + erts_aint32_t n, e; + + if (((state & (ERTS_PSFLG_SUSPENDED + | ERTS_PSFLG_ACTIVE)) != ERTS_PSFLG_ACTIVE) + && !(state & ERTS_PSFLG_EXITING)) + goto sched_out_proc; + + n = e = state; + n &= ~ERTS_PSFLG_RUNNING_SYS; + n |= ERTS_PSFLG_RUNNING; + + state = erts_smp_atomic32_cmpxchg_mb(&p->state, n, e); + if (state == e) { + state = n; + break; + } + + ASSERT(state & ERTS_PSFLG_RUNNING_SYS); + ASSERT(!(state & ERTS_PSFLG_RUNNING)); + } + } + if (!(state & ERTS_PSFLG_EXITING) && ((FLAGS(p) & F_FORCE_GC) || (MSO(p).overhead > BIN_VHEAP_SZ(p)))) { reds -= erts_garbage_collect(p, 0, p->arg_reg, p->arity); - if (reds < 0) { - reds = 1; + if (reds <= 0) { + p->fcalls = reds; + goto sched_out_proc; } } + + if (proxy_p) { + free_proxy_proc(proxy_p); + proxy_p = NULL; + } p->fcalls = reds; ERTS_SMP_CHK_HAVE_ONLY_MAIN_PROC_LOCK(p); + + /* Never run a suspended process */ + ASSERT(!(ERTS_PSFLG_SUSPENDED & erts_smp_atomic32_read_nob(&p->state))); + + ASSERT(erts_proc_read_refc(p) > 0); + + if (!(state & ERTS_PSFLG_EXITING) && ERTS_PTMR_IS_TIMED_OUT(p)) { + BeamInstr** pi; +#ifdef ERTS_SMP + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore); +#endif + pi = (BeamInstr **) p->def_arg_reg; + p->i = *pi; + p->flags &= ~F_INSLPQUEUE; + p->flags |= F_TIMO; + ERTS_PTMR_CLEAR(p); + } + return p; } } +static int +notify_sys_task_executed(Process *c_p, ErtsProcSysTask *st, Eterm st_result) +{ + Process *rp = erts_proc_lookup(st->requester); + if (rp) { + ErtsProcLocks rp_locks; + ErlOffHeap *ohp; + ErlHeapFragment* bp; + Eterm *hp, msg, req_id, result; + Uint st_result_sz, hsz; +#ifdef DEBUG + Eterm *hp_start; +#endif + + rp_locks = (c_p == rp) ? ERTS_PROC_LOCK_MAIN : 0; + + st_result_sz = is_immed(st_result) ? 0 : size_object(st_result); + hsz = st->req_id_sz + st_result_sz + 4 /* 3-tuple */; + + hp = erts_alloc_message_heap(hsz, + &bp, + &ohp, + rp, + &rp_locks); + +#ifdef DEBUG + hp_start = hp; +#endif + + req_id = st->req_id_sz == 0 ? st->req_id : copy_struct(st->req_id, + st->req_id_sz, + &hp, + ohp); + + result = st_result_sz == 0 ? st_result : copy_struct(st_result, + st_result_sz, + &hp, + ohp); + + ASSERT(is_immed(st->reply_tag)); + + msg = TUPLE3(hp, st->reply_tag, req_id, result); + +#ifdef DEBUG + hp += 4; + ASSERT(hp_start + hsz == hp); +#endif + + erts_queue_message(rp, &rp_locks, bp, msg, NIL); + + if (c_p == rp) + rp_locks &= ~ERTS_PROC_LOCK_MAIN; + + if (rp_locks) + erts_smp_proc_unlock(rp, rp_locks); + } + + erts_cleanup_offheap(&st->off_heap); + + erts_free(ERTS_ALC_T_PROC_SYS_TSK, st); + + return rp ? 1 : 0; +} + +static ERTS_INLINE ErtsProcSysTask * +fetch_sys_task(Process *c_p, erts_aint32_t state, int *qmaskp, int *priop) +{ + ErtsProcSysTaskQs *unused_qs = NULL; + int qbit, qmask; + ErtsProcSysTask *st, **qp; + + *priop = -1; /* Shut up annoying erroneous warning */ + + erts_smp_proc_lock(c_p, ERTS_PROC_LOCK_STATUS); + + if (!c_p->sys_task_qs) { + qmask = 0; + st = NULL; + goto update_state; + } + + qmask = c_p->sys_task_qs->qmask; + + if ((state & (ERTS_PSFLG_ACTIVE + | ERTS_PSFLG_EXITING + | ERTS_PSFLG_SUSPENDED)) == ERTS_PSFLG_ACTIVE) { + /* No sys tasks if we got exclusively higher prio user work to do */ + st = NULL; + switch (ERTS_PSFLGS_GET_USR_PRIO(state)) { + case PRIORITY_MAX: + if (!(qmask & MAX_BIT)) + goto done; + break; + case PRIORITY_HIGH: + if (!(qmask & (MAX_BIT|HIGH_BIT))) + goto done; + break; + default: + break; + } + } + + qbit = qmask & -qmask; + switch (qbit) { + case MAX_BIT: + qp = &c_p->sys_task_qs->q[PRIORITY_MAX]; + *priop = PRIORITY_MAX; + break; + case HIGH_BIT: + qp = &c_p->sys_task_qs->q[PRIORITY_HIGH]; + *priop = PRIORITY_HIGH; + break; + case NORMAL_BIT: + if (!(qmask & PRIORITY_LOW) + || ++c_p->sys_task_qs->ncount <= RESCHEDULE_LOW) { + qp = &c_p->sys_task_qs->q[PRIORITY_NORMAL]; + *priop = PRIORITY_NORMAL; + break; + } + c_p->sys_task_qs->ncount = 0; + /* Fall through */ + case LOW_BIT: + qp = &c_p->sys_task_qs->q[PRIORITY_LOW]; + *priop = PRIORITY_LOW; + break; + default: + ERTS_INTERNAL_ERROR("Invalid qmask"); + } + + st = *qp; + ASSERT(st); + if (st->next != st) { + *qp = st->next; + st->next->prev = st->prev; + st->prev->next = st->next; + } + else { + erts_aint32_t a, e, n, st_prio, qmask2; + + *qp = NULL; + qmask &= ~qbit; + c_p->sys_task_qs->qmask = qmask; + + update_state: + + qmask2 = qmask; + + if (state & ERTS_PSFLG_DELAYED_SYS) { + ErtsProcSysTaskQs *qs = ERTS_PROC_GET_DELAYED_GC_TASK_QS(c_p); + ASSERT(qs); + qmask2 |= qs->qmask; + } + + switch (qmask2 & -qmask2) { + case MAX_BIT: + st_prio = PRIORITY_MAX; + break; + case HIGH_BIT: + st_prio = PRIORITY_HIGH; + break; + case NORMAL_BIT: + st_prio = PRIORITY_NORMAL; + break; + case LOW_BIT: + case 0: + st_prio = PRIORITY_LOW; + break; + default: + ERTS_INTERNAL_ERROR("Invalid qmask"); + } + + if (!qmask) { + unused_qs = c_p->sys_task_qs; + c_p->sys_task_qs = NULL; + } + + a = state; + do { + erts_aint32_t prio = ERTS_PSFLGS_GET_USR_PRIO(a); + + if (prio > st_prio) + prio = st_prio; + + n = e = a; + + n &= ~ERTS_PSFLGS_ACT_PRIO_MASK; + n |= (prio << ERTS_PSFLGS_ACT_PRIO_OFFSET); + + if (!qmask) + n &= ~ERTS_PSFLG_ACTIVE_SYS; + + if (a == n) + break; + a = erts_smp_atomic32_cmpxchg_nob(&c_p->state, n, e); + } while (a != e); + } + +done: + + erts_smp_proc_unlock(c_p, ERTS_PROC_LOCK_STATUS); + + if (unused_qs) + proc_sys_task_queues_free(unused_qs); + + *qmaskp = qmask; + + return st; +} + +static void save_gc_task(Process *c_p, ErtsProcSysTask *st, int prio); + +static int +execute_sys_tasks(Process *c_p, erts_aint32_t *statep, int in_reds) +{ + int garbage_collected = 0; + erts_aint32_t state = *statep; + int max_reds = in_reds; + int reds = 0; + int qmask = 0; + + ERTS_SMP_LC_ASSERT(erts_proc_lc_my_proc_locks(c_p) == ERTS_PROC_LOCK_MAIN); + + do { + ErtsProcSysTask *st; + int st_prio; + Eterm st_res; + + if (state & (ERTS_PSFLG_EXITING|ERTS_PSFLG_PENDING_EXIT)) { +#ifdef ERTS_SMP + if (state & ERTS_PSFLG_PENDING_EXIT) + erts_handle_pending_exit(c_p, ERTS_PROC_LOCK_MAIN); +#endif + ASSERT(ERTS_PROC_IS_EXITING(c_p)); + break; + } + + st = fetch_sys_task(c_p, state, &qmask, &st_prio); + if (!st) + break; + + switch (st->type) { + case ERTS_PSTT_GC: + if (c_p->flags & F_DISABLE_GC) { + save_gc_task(c_p, st, st_prio); + st = NULL; + reds++; + } + else { + if (!garbage_collected) { + FLAGS(c_p) |= F_NEED_FULLSWEEP; + reds += erts_garbage_collect(c_p, + 0, + c_p->arg_reg, + c_p->arity); + garbage_collected = 1; + } + st_res = am_true; + } + break; + case ERTS_PSTT_CPC: + st_res = erts_check_process_code(c_p, + st->arg[0], + st->arg[1] == am_true, + &reds); + if (is_non_value(st_res)) { + /* Needed gc, but gc was disabled */ + save_gc_task(c_p, st, st_prio); + st = NULL; + } + break; + default: + ERTS_INTERNAL_ERROR("Invalid process sys task type"); + st_res = am_false; + } + + if (st) + reds += notify_sys_task_executed(c_p, st, st_res); + + state = erts_smp_atomic32_read_acqb(&c_p->state); + } while (qmask && reds < max_reds); + + *statep = state; + + return reds; +} + +static int +cleanup_sys_tasks(Process *c_p, erts_aint32_t in_state, int in_reds) +{ + erts_aint32_t state = in_state; + int max_reds = in_reds; + int reds = 0; + int qmask = 0; + + ERTS_SMP_LC_ASSERT(erts_proc_lc_my_proc_locks(c_p) == ERTS_PROC_LOCK_MAIN); + + do { + ErtsProcSysTask *st; + Eterm st_res; + int st_prio; + + st = fetch_sys_task(c_p, state, &qmask, &st_prio); + if (!st) + break; + + switch (st->type) { + case ERTS_PSTT_GC: + st_res = am_false; + break; + case ERTS_PSTT_CPC: + st_res = am_false; + break; + default: + ERTS_INTERNAL_ERROR("Invalid process sys task type"); + st_res = am_false; + break; + } + + reds += notify_sys_task_executed(c_p, st, st_res); + + state = erts_smp_atomic32_read_acqb(&c_p->state); + } while (qmask && reds < max_reds); + + return reds; +} + +BIF_RETTYPE +erts_internal_request_system_task_3(BIF_ALIST_3) +{ + Process *rp = erts_proc_lookup(BIF_ARG_1); + ErtsProcSysTaskQs *stqs, *free_stqs = NULL; + ErtsProcSysTask *st = NULL; + erts_aint32_t prio, rp_state; + int rp_locked; + Eterm noproc_res, req_type; + + if (!rp && !is_internal_pid(BIF_ARG_1)) { + if (!is_external_pid(BIF_ARG_1)) + goto badarg; + if (external_pid_dist_entry(BIF_ARG_1) != erts_this_dist_entry) + goto badarg; + } + + switch (BIF_ARG_2) { + case am_max: prio = PRIORITY_MAX; break; + case am_high: prio = PRIORITY_HIGH; break; + case am_normal: prio = PRIORITY_NORMAL; break; + case am_low: prio = PRIORITY_LOW; break; + default: goto badarg; + } + + if (is_not_tuple(BIF_ARG_3)) + goto badarg; + else { + int i; + Eterm *tp = tuple_val(BIF_ARG_3); + Uint arity = arityval(*tp); + Eterm req_id; + Uint req_id_sz; + Eterm arg[ERTS_MAX_PROC_SYS_TASK_ARGS]; + Uint arg_sz[ERTS_MAX_PROC_SYS_TASK_ARGS]; + Uint tot_sz; + Eterm *hp; + + if (arity < 2) + goto badarg; + if (arity > 2 + ERTS_MAX_PROC_SYS_TASK_ARGS) + goto badarg; + req_type = tp[1]; + req_id = tp[2]; + req_id_sz = is_immed(req_id) ? req_id : size_object(req_id); + tot_sz = req_id_sz; + for (i = 0; i < ERTS_MAX_PROC_SYS_TASK_ARGS; i++) { + int tix = 3 + i; + if (tix > arity) { + arg[i] = THE_NON_VALUE; + arg_sz[i] = 0; + } + else { + arg[i] = tp[tix]; + if (is_immed(arg[i])) + arg_sz[i] = 0; + else { + arg_sz[i] = size_object(arg[i]); + tot_sz += arg_sz[i]; + } + } + } + st = erts_alloc(ERTS_ALC_T_PROC_SYS_TSK, + ERTS_PROC_SYS_TASK_SIZE(tot_sz)); + st->next = st->prev = st; /* Prep for empty prio queue */ + ERTS_INIT_OFF_HEAP(&st->off_heap); + hp = &st->heap[0]; + + st->requester = BIF_P->common.id; + st->reply_tag = req_type; + st->req_id_sz = req_id_sz; + st->req_id = req_id_sz == 0 ? req_id : copy_struct(req_id, + req_id_sz, + &hp, + &st->off_heap); + + for (i = 0; i < ERTS_MAX_PROC_SYS_TASK_ARGS; i++) + st->arg[i] = arg_sz[i] == 0 ? arg[i] : copy_struct(arg[i], + arg_sz[i], + &hp, + &st->off_heap); + ASSERT(&st->heap[0] + tot_sz == hp); + } + + switch (req_type) { + + case am_garbage_collect: + st->type = ERTS_PSTT_GC; + noproc_res = am_false; + if (!rp) + goto noproc; + break; + + case am_check_process_code: + if (is_not_atom(st->arg[0])) + goto badarg; + if (st->arg[1] != am_true && st->arg[1] != am_false) + goto badarg; + noproc_res = am_false; + st->type = ERTS_PSTT_CPC; + if (!rp) + goto noproc; + break; + + default: + goto badarg; + } + + rp_state = erts_smp_atomic32_read_nob(&rp->state); + + rp_locked = 0; + + free_stqs = NULL; + if (rp_state & ERTS_PSFLG_ACTIVE_SYS) + stqs = NULL; + else { + alloc_qs: + stqs = proc_sys_task_queues_alloc(); + stqs->qmask = 1 << prio; + stqs->ncount = 0; + stqs->q[PRIORITY_MAX] = NULL; + stqs->q[PRIORITY_HIGH] = NULL; + stqs->q[PRIORITY_NORMAL] = NULL; + stqs->q[PRIORITY_LOW] = NULL; + stqs->q[prio] = st; + } + + if (!rp_locked) { + rp_locked = 1; + erts_smp_proc_lock(rp, ERTS_PROC_LOCK_STATUS); + + rp_state = erts_smp_atomic32_read_nob(&rp->state); + if (rp_state & ERTS_PSFLG_EXITING) { + erts_smp_proc_unlock(rp, ERTS_PROC_LOCK_STATUS); + rp = NULL; + free_stqs = stqs; + goto noproc; + } + } + + if (!rp->sys_task_qs) { + if (stqs) + rp->sys_task_qs = stqs; + else + goto alloc_qs; + } + else { + if (stqs) + free_stqs = stqs; + stqs = rp->sys_task_qs; + if (!stqs->q[prio]) { + stqs->q[prio] = st; + stqs->qmask |= 1 << prio; + } + else { + st->next = stqs->q[prio]; + st->prev = stqs->q[prio]->prev; + st->next->prev = st; + st->prev->next = st; + ASSERT(stqs->qmask & (1 << prio)); + } + } + + if (ERTS_PSFLGS_GET_ACT_PRIO(rp_state) > prio) { + erts_aint32_t n, a, e; + /* Need to elevate actual prio */ + + a = rp_state; + do { + if (ERTS_PSFLGS_GET_ACT_PRIO(a) <= prio) { + n = a; + break; + } + n = e = a; + n &= ~ERTS_PSFLGS_ACT_PRIO_MASK; + n |= (prio << ERTS_PSFLGS_ACT_PRIO_OFFSET); + a = erts_smp_atomic32_cmpxchg_nob(&rp->state, n, e); + } while (a != e); + rp_state = n; + } + + /* + * schedule_process_sys_task() unlocks status + * lock on process. + */ + schedule_process_sys_task(rp, rp_state, NULL); + + if (free_stqs) + proc_sys_task_queues_free(free_stqs); + + BIF_RET(am_ok); + +noproc: + + notify_sys_task_executed(BIF_P, st, noproc_res); + if (free_stqs) + proc_sys_task_queues_free(free_stqs); + BIF_RET(am_ok); + +badarg: + + if (st) { + erts_cleanup_offheap(&st->off_heap); + erts_free(ERTS_ALC_T_PROC_SYS_TSK, st); + } + if (free_stqs) + proc_sys_task_queues_free(free_stqs); + BIF_ERROR(BIF_P, BADARG); +} + +static void +save_gc_task(Process *c_p, ErtsProcSysTask *st, int prio) +{ + erts_aint32_t state; + ErtsProcSysTaskQs *qs; + + ERTS_SMP_LC_ASSERT(ERTS_PROC_LOCK_MAIN == erts_proc_lc_my_proc_locks(c_p)); + + qs = ERTS_PROC_GET_DELAYED_GC_TASK_QS(c_p); + if (!qs) { + st->next = st->prev = st; + qs = proc_sys_task_queues_alloc(); + qs->qmask = 1 << prio; + qs->ncount = 0; + qs->q[PRIORITY_MAX] = NULL; + qs->q[PRIORITY_HIGH] = NULL; + qs->q[PRIORITY_NORMAL] = NULL; + qs->q[PRIORITY_LOW] = NULL; + qs->q[prio] = st; + (void) ERTS_PROC_SET_DELAYED_GC_TASK_QS(c_p, ERTS_PROC_LOCK_MAIN, qs); + } + else { + if (!qs->q[prio]) { + st->next = st->prev = st; + qs->q[prio] = st; + qs->qmask |= 1 << prio; + } + else { + st->next = qs->q[prio]; + st->prev = qs->q[prio]->prev; + st->next->prev = st; + st->prev->next = st; + ASSERT(qs->qmask & (1 << prio)); + } + } + + state = erts_smp_atomic32_read_nob(&c_p->state); + ASSERT((ERTS_PSFLG_RUNNING|ERTS_PSFLG_RUNNING_SYS) & state); + + while (!(state & ERTS_PSFLG_DELAYED_SYS) + || prio < ERTS_PSFLGS_GET_ACT_PRIO(state)) { + erts_aint32_t n, e; + + n = e = state; + n |= ERTS_PSFLG_DELAYED_SYS; + if (prio < ERTS_PSFLGS_GET_ACT_PRIO(state)) { + n &= ~ERTS_PSFLGS_ACT_PRIO_MASK; + n |= prio << ERTS_PSFLGS_ACT_PRIO_OFFSET; + } + state = erts_smp_atomic32_cmpxchg_relb(&c_p->state, n, e); + if (state == e) + break; + } +} + +int +erts_set_gc_state(Process *c_p, int enable) +{ + ErtsProcSysTaskQs *dgc_tsk_qs; + ASSERT(c_p == erts_get_current_process()); + ASSERT((ERTS_PSFLG_RUNNING|ERTS_PSFLG_RUNNING_SYS) + & erts_smp_atomic32_read_nob(&c_p->state)); + ERTS_SMP_LC_ASSERT(ERTS_PROC_LOCK_MAIN == erts_proc_lc_my_proc_locks(c_p)); + + if (!enable) { + c_p->flags |= F_DISABLE_GC; + return 0; + } + + c_p->flags &= ~F_DISABLE_GC; + + dgc_tsk_qs = ERTS_PROC_GET_DELAYED_GC_TASK_QS(c_p); + if (!dgc_tsk_qs) + return 0; + + /* Move delayed gc tasks into sys tasks queues. */ + + erts_smp_proc_lock(c_p, ERTS_PROC_LOCK_STATUS); + + if (!c_p->sys_task_qs) { + c_p->sys_task_qs = dgc_tsk_qs; + dgc_tsk_qs = NULL; + } + else { + ErtsProcSysTaskQs *stsk_qs; + int prio; + + /* + * We push delayed tasks to the front of the queue + * since they have already made it to the front + * once and then been delayed after that. + */ + + stsk_qs = c_p->sys_task_qs; + + while (dgc_tsk_qs->qmask) { + int qbit = dgc_tsk_qs->qmask & -dgc_tsk_qs->qmask; + dgc_tsk_qs->qmask &= ~qbit; + switch (qbit) { + case MAX_BIT: + prio = PRIORITY_MAX; + break; + case HIGH_BIT: + prio = PRIORITY_HIGH; + break; + case NORMAL_BIT: + prio = PRIORITY_NORMAL; + break; + case LOW_BIT: + prio = PRIORITY_LOW; + break; + default: + ERTS_INTERNAL_ERROR("Invalid qmask"); + prio = -1; + break; + } + + ASSERT(dgc_tsk_qs->q[prio]); + + if (!stsk_qs->q[prio]) { + stsk_qs->q[prio] = dgc_tsk_qs->q[prio]; + stsk_qs->qmask |= 1 << prio; + } + else { + ErtsProcSysTask *first1, *last1, *first2, *last2; + + ASSERT(stsk_qs->qmask & (1 << prio)); + first1 = dgc_tsk_qs->q[prio]; + last1 = first1->prev; + first2 = stsk_qs->q[prio]; + last2 = first1->prev; + + last1->next = first2; + first2->prev = last1; + + first1->prev = last2; + last2->next = first1; + + stsk_qs->q[prio] = first1; + } + + } + } + +#ifdef DEBUG + { + int qmask; + erts_aint32_t aprio, state = +#endif + + erts_smp_atomic32_read_bset_nob(&c_p->state, + (ERTS_PSFLG_DELAYED_SYS + | ERTS_PSFLG_ACTIVE_SYS), + ERTS_PSFLG_ACTIVE_SYS); + +#ifdef DEBUG + ASSERT(state & ERTS_PSFLG_DELAYED_SYS); + qmask = c_p->sys_task_qs->qmask; + aprio = ERTS_PSFLGS_GET_ACT_PRIO(state); + ASSERT(ERTS_PSFLGS_GET_USR_PRIO(state) >= aprio); + ASSERT((qmask & -qmask) >= (1 << aprio)); + } +#endif + + erts_smp_proc_unlock(c_p, ERTS_PROC_LOCK_STATUS); + + (void) ERTS_PROC_SET_DELAYED_GC_TASK_QS(c_p, ERTS_PROC_LOCK_MAIN, NULL); + + if (dgc_tsk_qs) + proc_sys_task_queues_free(dgc_tsk_qs); + + return 1; +} + void erts_sched_stat_modify(int what) { @@ -7052,7 +10693,7 @@ erts_sched_stat_modify(int what) break; case ERTS_SCHED_STAT_MODIFY_DISABLE: erts_smp_thr_progress_block(); - erts_sched_stat.enabled = 1; + erts_sched_stat.enabled = 0; erts_smp_thr_progress_unblock(); break; case ERTS_SCHED_STAT_MODIFY_CLEAR: @@ -7139,6 +10780,10 @@ erts_schedule_misc_op(void (*func)(void *), void *arg) rq->misc.start = molp; rq->misc.end = molp; +#ifdef ERTS_SMP + non_empty_runq(rq); +#endif + erts_smp_runq_unlock(rq); smp_notify_inc_runq(rq); @@ -7230,6 +10875,8 @@ erts_free_proc(Process *p) #ifdef ERTS_SMP erts_proc_lock_fin(p); #endif + ASSERT(erts_smp_atomic32_read_nob(&p->state) & ERTS_PSFLG_FREE); + ASSERT(0 == erts_proc_read_refc(p)); erts_free(ERTS_ALC_T_PROC, (void *) p); } @@ -7272,7 +10919,7 @@ alloc_process(ErtsRunQueue *rq, erts_aint32_t state) init_arg.run_queue = rq; init_arg.state = state; - ASSERT(((char *) p) == ((char *) &p->common)); + ERTS_CT_ASSERT(offsetof(Process,common) == 0); if (!erts_ptab_new_element(&erts_proc, &p->common, @@ -7282,10 +10929,13 @@ alloc_process(ErtsRunQueue *rq, erts_aint32_t state) return NULL; } + ASSERT(erts_proc_read_refc(p) > 0); + ASSERT(internal_pid_serial(p->common.id) <= ERTS_MAX_PID_SERIAL); p->approx_started = erts_get_approx_time(); p->rcount = 0; + p->heap = NULL; ASSERT(p == (Process *) (erts_ptab_pix2intptr_nob( @@ -7320,7 +10970,7 @@ erl_create_process(Process* parent, /* Parent of process (default group leader). * Check for errors. */ - if (is_not_atom(mod) || is_not_atom(func) || ((arity = list_length(args)) < 0)) { + if (is_not_atom(mod) || is_not_atom(func) || ((arity = erts_list_length(args)) < 0)) { so->error_code = BADARG; goto error; } @@ -7330,12 +10980,14 @@ erl_create_process(Process* parent, /* Parent of process (default group leader). int ix = so->scheduler-1; ASSERT(0 <= ix && ix < erts_no_run_queues); rq = ERTS_RUNQ_IX(ix); + /* Unsupported feature... */ state |= ERTS_PSFLG_BOUND; } prio = (erts_aint32_t) so->priority; } - state |= (prio & ERTS_PSFLG_PRIO_MASK); + state |= (((prio & ERTS_PSFLGS_PRIO_MASK) << ERTS_PSFLGS_ACT_PRIO_OFFSET) + | ((prio & ERTS_PSFLGS_PRIO_MASK) << ERTS_PSFLGS_USR_PRIO_OFFSET)); if (!rq) rq = erts_get_runq_proc(parent); @@ -7361,6 +11013,9 @@ erl_create_process(Process* parent, /* Parent of process (default group leader). p->flags = erts_default_process_flags; + p->static_flags = 0; + if (so->flags & SPO_SYSTEM_PROC) + p->static_flags |= ERTS_STC_FLG_SYSTEM_PROC; if (so->flags & SPO_USE_ARGS) { p->min_heap_size = so->min_heap_size; p->min_vheap_size = so->min_vheap_size; @@ -7373,9 +11028,9 @@ erl_create_process(Process* parent, /* Parent of process (default group leader). p->schedule_count = 0; ASSERT(p->min_heap_size == erts_next_heap_size(p->min_heap_size, 0)); - p->initial[INITIAL_MOD] = mod; - p->initial[INITIAL_FUN] = func; - p->initial[INITIAL_ARI] = (Uint) arity; + p->u.initial[INITIAL_MOD] = mod; + p->u.initial[INITIAL_FUN] = func; + p->u.initial[INITIAL_ARI] = (Uint) arity; /* * Must initialize binary lists here before copying binaries to process. @@ -7398,7 +11053,6 @@ erl_create_process(Process* parent, /* Parent of process (default group leader). hipe_init_process_smp(&p->hipe_smp); #endif #endif - p->heap = (Eterm *) ERTS_HEAP_ALLOC(ERTS_ALC_T_HEAP, sizeof(Eterm)*sz); p->old_hend = p->old_htop = p->old_heap = NULL; p->high_water = p->heap; @@ -7413,9 +11067,11 @@ erl_create_process(Process* parent, /* Parent of process (default group leader). p->bin_old_vheap = 0; p->bin_vheap_mature = 0; + p->sys_task_qs = NULL; + /* No need to initialize p->fcalls. */ - p->current = p->initial+INITIAL_MOD; + p->current = p->u.initial+INITIAL_MOD; p->i = (BeamInstr *) beam_apply; p->cp = (BeamInstr *) beam_apply+1; @@ -7438,11 +11094,7 @@ erl_create_process(Process* parent, /* Parent of process (default group leader). p->ftrace = NIL; p->reds = 0; -#ifdef ERTS_SMP - p->common.u.alive.ptimer = NULL; -#else - sys_memset(&p->common.u.alive.tm, 0, sizeof(ErlTimer)); -#endif + ERTS_PTMR_INIT(p); p->common.u.alive.reg = NULL; ERTS_P_LINKS(p) = NULL; @@ -7473,7 +11125,10 @@ erl_create_process(Process* parent, /* Parent of process (default group leader). p->msg_inq.last = &p->msg_inq.first; p->msg_inq.len = 0; #endif - p->u.bif_timers = NULL; + p->bif_timers = NULL; +#ifdef ERTS_BTM_ACCESSOR_SUPPORT + p->accessor_bif_timers = NULL; +#endif p->mbuf = NULL; p->mbuf_sz = 0; p->psd = NULL; @@ -7629,12 +11284,9 @@ void erts_init_empty_process(Process *p) p->bin_vheap_sz = BIN_VH_MIN_SIZE; p->bin_old_vheap_sz = BIN_VH_MIN_SIZE; p->bin_old_vheap = 0; + p->sys_task_qs = NULL; p->bin_vheap_mature = 0; -#ifdef ERTS_SMP - p->common.u.alive.ptimer = NULL; -#else - memset(&(p->common.u.alive.tm), 0, sizeof(ErlTimer)); -#endif + ERTS_PTMR_INIT(p); p->next = NULL; p->off_heap.first = NULL; p->off_heap.overhead = 0; @@ -7655,14 +11307,17 @@ void erts_init_empty_process(Process *p) p->msg.last = &p->msg.first; p->msg.save = &p->msg.first; p->msg.len = 0; - p->u.bif_timers = NULL; + p->bif_timers = NULL; +#ifdef ERTS_BTM_ACCESSOR_SUPPORT + p->accessor_bif_timers = NULL; +#endif p->dictionary = NULL; p->seq_trace_clock = 0; p->seq_trace_lastcnt = 0; p->seq_trace_token = NIL; - p->initial[0] = 0; - p->initial[1] = 0; - p->initial[2] = 0; + p->u.initial[0] = 0; + p->u.initial[1] = 0; + p->u.initial[2] = 0; p->catches = 0; p->cp = NULL; p->i = NULL; @@ -7683,6 +11338,8 @@ void erts_init_empty_process(Process *p) p->parent = NIL; p->approx_started = 0; + p->static_flags = 0; + p->common.u.alive.started_interval = 0; #ifdef HIPE @@ -7747,7 +11404,10 @@ erts_debug_verify_clean_empty_process(Process* p) ASSERT(p->suspend_monitors == NULL); ASSERT(p->msg.first == NULL); ASSERT(p->msg.len == 0); - ASSERT(p->u.bif_timers == NULL); + ASSERT(p->bif_timers == NULL); +#ifdef ERTS_BTM_ACCESSOR_SUPPORT + ASSERT(p->accessor_bif_timers == NULL); +#endif ASSERT(p->dictionary == NULL); ASSERT(p->catches == 0); ASSERT(p->cp == NULL); @@ -7884,33 +11544,22 @@ delete_process(Process* p) p->fvalue = NIL; } -static ERTS_INLINE erts_aint32_t -set_proc_exiting_state(Process *p, erts_aint32_t state) -{ - erts_aint32_t a, n, e; - a = state; - while (1) { - n = e = a; - n &= ~(ERTS_PSFLG_SUSPENDED|ERTS_PSFLG_PENDING_EXIT); - n |= ERTS_PSFLG_EXITING|ERTS_PSFLG_ACTIVE; - if (!(a & (ERTS_PSFLG_IN_RUNQ|ERTS_PSFLG_RUNNING))) - n |= ERTS_PSFLG_IN_RUNQ; - a = erts_smp_atomic32_cmpxchg_relb(&p->state, n, e); - if (a == e) - break; - } - return a; -} - static ERTS_INLINE void set_proc_exiting(Process *p, - erts_aint32_t state, + erts_aint32_t in_state, Eterm reason, ErlHeapFragment *bp) { + erts_aint32_t state = in_state, enq_prio = -1; + int enqueue; ERTS_SMP_LC_ASSERT(erts_proc_lc_my_proc_locks(p) == ERTS_PROC_LOCKS_ALL); - state = set_proc_exiting_state(p, state); + enqueue = change_proc_schedule_state(p, + ERTS_PSFLG_SUSPENDED|ERTS_PSFLG_PENDING_EXIT, + ERTS_PSFLG_EXITING|ERTS_PSFLG_ACTIVE, + &state, + &enq_prio, + ERTS_PROC_LOCKS_ALL); p->fvalue = reason; if (bp) @@ -7922,18 +11571,56 @@ set_proc_exiting(Process *p, */ p->freason = EXTAG_EXIT; KILL_CATCHES(p); - cancel_timer(p); p->i = (BeamInstr *) beam_exit; - if (erts_system_profile_flags.runnable_procs - && !(state & (ERTS_PSFLG_ACTIVE|ERTS_PSFLG_SUSPENDED))) { - profile_runnable_proc(p, am_active); +#ifndef ERTS_SMP + if (state & (ERTS_PSFLG_RUNNING|ERTS_PSFLG_RUNNING_SYS)) { + /* + * I non smp case: + * + * Currently executing process might be sent an exit + * signal if it is traced by a port that it also is + * linked to, and the port terminates during the + * trace. In this case we want schedule out the + * process as quickly as possible in order to detect + * the event as fast as possible. + */ + ERTS_VBUMP_ALL_REDS(p); } +#endif - if (!(state & (ERTS_PSFLG_IN_RUNQ|ERTS_PSFLG_RUNNING))) - add2runq(p, state); + if (enqueue) + add2runq(enqueue > 0 ? p : make_proxy_proc(NULL, p, enq_prio), + state, + enq_prio); } +static ERTS_INLINE erts_aint32_t +set_proc_self_exiting(Process *c_p) +{ +#ifdef DEBUG + int enqueue; +#endif + erts_aint32_t state, enq_prio = -1; + + ERTS_SMP_LC_ASSERT(erts_proc_lc_my_proc_locks(c_p) == ERTS_PROC_LOCKS_ALL); + + state = erts_smp_atomic32_read_nob(&c_p->state); + ASSERT(state & (ERTS_PSFLG_RUNNING|ERTS_PSFLG_RUNNING_SYS)); + +#ifdef DEBUG + enqueue = +#endif + change_proc_schedule_state(c_p, + ERTS_PSFLG_SUSPENDED|ERTS_PSFLG_PENDING_EXIT, + ERTS_PSFLG_EXITING|ERTS_PSFLG_ACTIVE, + &state, + &enq_prio, + ERTS_PROC_LOCKS_ALL); + + ASSERT(!enqueue); + return state; +} #ifdef ERTS_SMP @@ -7981,7 +11668,7 @@ handle_pending_exiters(ErtsProcList *pnd_xtrs) if (p) { if (erts_proclist_same(plp, p)) { erts_aint32_t state = erts_smp_atomic32_read_acqb(&p->state); - if (!(state & ERTS_PSFLG_RUNNING)) { + if (!(state & (ERTS_PSFLG_RUNNING|ERTS_PSFLG_RUNNING_SYS))) { ASSERT(state & ERTS_PSFLG_PENDING_EXIT); erts_handle_pending_exit(p, ERTS_PROC_LOCKS_ALL); } @@ -7999,10 +11686,14 @@ save_pending_exiter(Process *p) { ErtsProcList *plp; ErtsRunQueue *rq; + ErtsSchedulerData *esdp = erts_get_scheduler_data(); ERTS_SMP_LC_ASSERT(ERTS_PROC_LOCK_STATUS & erts_proc_lc_my_proc_locks(p)); - rq = erts_get_runq_current(NULL); + if (!esdp) + rq = RUNQ_READ_RQ(&p->run_queue); + else + rq = esdp->run_queue; plp = proclist_create(p); @@ -8010,8 +11701,16 @@ save_pending_exiter(Process *p) erts_proclist_store_last(&rq->procs.pending_exiters, plp); + non_empty_runq(rq); + erts_smp_runq_unlock(rq); - wake_scheduler(rq, 1); +#ifdef ERTS_DIRTY_SCHEDULERS + if (ERTS_RUNQ_IX_IS_DIRTY(rq->ix)) + wake_dirty_schedulers(rq, 0); + else +#endif + wake_scheduler(rq); + } #endif @@ -8037,11 +11736,7 @@ send_exit_message(Process *to, ErtsProcLocks *to_locksp, hp = erts_alloc_message_heap(term_size, &bp, &ohp, to, to_locksp); mess = copy_struct(exit_term, term_size, &hp, ohp); - erts_queue_message(to, to_locksp, bp, mess, NIL -#ifdef USE_VM_PROBES - , NIL -#endif - ); + erts_queue_message(to, to_locksp, bp, mess, NIL); } else { ErlHeapFragment* bp; Eterm* hp; @@ -8057,11 +11752,7 @@ send_exit_message(Process *to, ErtsProcLocks *to_locksp, /* the trace token must in this case be updated by the caller */ seq_trace_output(token, mess, SEQ_TRACE_SEND, to->common.id, NULL); temp_token = copy_struct(token, sz_token, &hp, &bp->off_heap); - erts_queue_message(to, to_locksp, bp, mess, temp_token -#ifdef USE_VM_PROBES - , NIL -#endif - ); + erts_queue_message(to, to_locksp, bp, mess, temp_token); } } @@ -8163,7 +11854,7 @@ send_exit_signal(Process *c_p, /* current process if and only dtrace_pid_str(from, sender_str); dtrace_proc_str(rp, receiver_str); - erts_snprintf(reason_buf, sizeof(reason_buf) - 1, "%T", reason); + erts_snprintf(reason_buf, sizeof(DTRACE_CHARBUF_NAME(reason_buf)) - 1, "%T", reason); DTRACE3(process_exit_signal, sender_str, receiver_str, reason_buf); } #endif @@ -8202,29 +11893,27 @@ send_exit_signal(Process *c_p, /* current process if and only } set_proc_exiting(c_p, state, rsn, NULL); } - else if (!(state & ERTS_PSFLG_RUNNING)) { + else if (!(state & (ERTS_PSFLG_RUNNING|ERTS_PSFLG_RUNNING_SYS))) { /* Process not running ... */ ErtsProcLocks need_locks = ~(*rp_locks) & ERTS_PROC_LOCKS_ALL; if (need_locks && erts_smp_proc_trylock(rp, need_locks) == EBUSY) { /* ... but we havn't got all locks on it ... */ - save_pending_exiter(rp); + save_pending_exiter(rp); /* * The pending exit will be discovered when next * process is scheduled in */ - goto set_pending_exit; - } - else { - /* ...and we have all locks on it... */ - *rp_locks = ERTS_PROC_LOCKS_ALL; - set_proc_exiting(rp, - state, - (is_immed(rsn) - ? rsn - : copy_object(rsn, rp)), - NULL); + goto set_pending_exit; } + /* ...and we have all locks on it... */ + *rp_locks = ERTS_PROC_LOCKS_ALL; + set_proc_exiting(rp, + state, + (is_immed(rsn) + ? rsn + : copy_object(rsn, rp)), + NULL); } else { /* Process running... */ @@ -8314,7 +12003,8 @@ static void doit_exit_monitor(ErtsMonitor *mon, void *vpcontext) ErtsMonitor *rmon; Process *rp; - if (mon->type == MON_ORIGIN) { + switch (mon->type) { + case MON_ORIGIN: /* We are monitoring someone else, we need to demonitor that one.. */ if (is_atom(mon->pid)) { /* remote by name */ ASSERT(is_node_name_atom(mon->pid)); @@ -8377,7 +12067,8 @@ static void doit_exit_monitor(ErtsMonitor *mon, void *vpcontext) } } } - } else { /* type == MON_TARGET */ + break; + case MON_TARGET: ASSERT(mon->type == MON_TARGET); ASSERT(is_pid(mon->pid) || is_internal_port(mon->pid)); if (is_internal_port(mon->pid)) { @@ -8436,6 +12127,12 @@ static void doit_exit_monitor(ErtsMonitor *mon, void *vpcontext) } } } + break; + case MON_TIME_OFFSET: + erts_demonitor_time_offset(mon->ref); + break; + default: + ERTS_INTERNAL_ERROR("Invalid monitor type"); } done: /* As the monitors are previously removed from the process, @@ -8555,7 +12252,7 @@ static void doit_exit_link(ErtsLink *lnk, void *vpcontext) break; default: - erl_exit(1, "bad type in link list\n"); + erts_exit(ERTS_ERROR_EXIT, "bad type in link list\n"); break; } erts_destroy_link(lnk); @@ -8567,8 +12264,9 @@ resume_suspend_monitor(ErtsSuspendMonitor *smon, void *vc_p) Process *suspendee = erts_pid2proc((Process *) vc_p, ERTS_PROC_LOCK_MAIN, smon->pid, ERTS_PROC_LOCK_STATUS); if (suspendee) { + ASSERT(suspendee != vc_p); if (smon->active) - resume_process(suspendee); + resume_process(suspendee, ERTS_PROC_LOCK_STATUS); erts_smp_proc_unlock(suspendee, ERTS_PROC_LOCK_STATUS); } erts_destroy_suspend_monitor(smon); @@ -8579,11 +12277,9 @@ resume_suspend_monitor(ErtsSuspendMonitor *smon, void *vc_p) void erts_do_exit_process(Process* p, Eterm reason) { -#ifdef ERTS_SMP - erts_aint32_t state; -#endif p->arity = 0; /* No live registers */ p->fvalue = reason; + #ifdef USE_VM_PROBES if (DTRACE_ENABLED(process_exit)) { @@ -8596,6 +12292,10 @@ erts_do_exit_process(Process* p, Eterm reason) } #endif + if (p->static_flags & ERTS_STC_FLG_SYSTEM_PROC) + erts_exit(ERTS_DUMP_EXIT, "System process %T terminated: %T\n", + p->common.id, reason); + #ifdef ERTS_SMP ERTS_SMP_CHK_HAVE_ONLY_MAIN_PROC_LOCK(p); /* By locking all locks (main lock is already locked) when going @@ -8606,10 +12306,9 @@ erts_do_exit_process(Process* p, Eterm reason) #endif #ifndef ERTS_SMP - set_proc_exiting_state(p, erts_smp_atomic32_read_nob(&p->state)); + set_proc_self_exiting(p); #else - state = set_proc_exiting_state(p, erts_smp_atomic32_read_nob(&p->state)); - if (state & ERTS_PSFLG_PENDING_EXIT) { + if (ERTS_PSFLG_PENDING_EXIT & set_proc_self_exiting(p)) { /* Process exited before pending exit was received... */ p->pending_exit.reason = THE_NON_VALUE; if (p->pending_exit.bp) { @@ -8636,20 +12335,20 @@ erts_do_exit_process(Process* p, Eterm reason) ASSERT((ERTS_TRACE_FLAGS(p) & F_INITIAL_TRACE_FLAGS) == F_INITIAL_TRACE_FLAGS); - cancel_timer(p); /* Always cancel timer just in case */ - - if (p->u.bif_timers) - erts_cancel_bif_timers(p, ERTS_PROC_LOCKS_ALL); + ASSERT(erts_proc_read_refc(p) > 0); + if (ERTS_PTMR_IS_SET(p)) { + erts_cancel_proc_timer(p); + ASSERT(erts_proc_read_refc(p) > 0); + } erts_smp_proc_unlock(p, ERTS_PROC_LOCKS_ALL_MINOR); /* - * The p->u.bif_timers of this process can *not* be used anymore; + * p->u.initial of this process can *not* be used anymore; * will be overwritten by misc termination data. */ p->u.terminate = NULL; - erts_continue_exit_process(p); } @@ -8663,6 +12362,8 @@ erts_continue_exit_process(Process *p) DistEntry *dep; struct saved_calls *scb; process_breakpoint_time_t *pbt; + erts_aint32_t state; + void *nif_export; #ifdef DEBUG int yield_allowed = 1; @@ -8672,6 +12373,29 @@ erts_continue_exit_process(Process *p) ASSERT(ERTS_PROC_IS_EXITING(p)); + ASSERT(erts_proc_read_refc(p) > 0); + if (p->bif_timers) { + if (erts_cancel_bif_timers(p, p->bif_timers, &p->u.terminate)) { + ASSERT(erts_proc_read_refc(p) > 0); + goto yield; + } + ASSERT(erts_proc_read_refc(p) > 0); + p->bif_timers = NULL; + } + +#ifdef ERTS_BTM_ACCESSOR_SUPPORT + if (p->accessor_bif_timers) { + if (erts_detach_accessor_bif_timers(p, + p->accessor_bif_timers, + &p->u.terminate)) { + ASSERT(erts_proc_read_refc(p) > 0); + goto yield; + } + ASSERT(erts_proc_read_refc(p) > 0); + p->accessor_bif_timers = NULL; + } +#endif + #ifdef ERTS_SMP if (p->flags & F_HAVE_BLCKD_MSCHED) { ErtsSchedSuspendResult ssr; @@ -8687,7 +12411,7 @@ erts_continue_exit_process(Process *p) break; case ERTS_SCHDLR_SSPND_EINVAL: default: - erl_exit(ERTS_ABORT_EXIT, "%s:%d: Internal error: %d\n", + erts_exit(ERTS_ABORT_EXIT, "%s:%d: Internal error: %d\n", __FILE__, __LINE__, (int) ssr); } } @@ -8699,6 +12423,13 @@ erts_continue_exit_process(Process *p) p->flags &= ~F_USING_DB; } + erts_set_gc_state(p, 1); + state = erts_smp_atomic32_read_acqb(&p->state); + if (state & ERTS_PSFLG_ACTIVE_SYS) { + if (cleanup_sys_tasks(p, state, CONTEXT_REDS) >= CONTEXT_REDS/2) + goto yield; + } + if (p->flags & F_USING_DDLL) { erts_ddll_proc_dead(p, ERTS_PROC_LOCK_MAIN); p->flags &= ~F_USING_DDLL; @@ -8758,6 +12489,8 @@ erts_continue_exit_process(Process *p) p->scheduler_data->current_process = NULL; p->scheduler_data->free_process = p; +#else + erts_proc_inc_refc(p); /* Decremented in schedule() */ #endif /* Time of death! */ @@ -8776,22 +12509,29 @@ erts_continue_exit_process(Process *p) { /* Inactivate and notify free */ erts_aint32_t n, e, a = erts_smp_atomic32_read_nob(&p->state); + int refc_inced = 0; while (1) { n = e = a; ASSERT(a & ERTS_PSFLG_EXITING); n |= ERTS_PSFLG_FREE; n &= ~ERTS_PSFLG_ACTIVE; + if ((n & ERTS_PSFLG_IN_RUNQ) && !refc_inced) { + erts_proc_inc_refc(p); + refc_inced = 1; + } a = erts_smp_atomic32_cmpxchg_mb(&p->state, n, e); if (a == e) break; } - } - dep = ((p->flags & F_DISTRIBUTION) - ? ERTS_PROC_SET_DIST_ENTRY(p, ERTS_PROC_LOCKS_ALL, NULL) - : NULL); + if (refc_inced && !(n & ERTS_PSFLG_IN_RUNQ)) + erts_proc_dec_refc(p); + } + + dep = (p->flags & F_DISTRIBUTION) ? erts_this_dist_entry : NULL; scb = ERTS_PROC_SET_SAVED_CALLS_BUF(p, ERTS_PROC_LOCKS_ALL, NULL); pbt = ERTS_PROC_SET_CALL_TIME(p, ERTS_PROC_LOCKS_ALL, NULL); + nif_export = ERTS_PROC_SET_NIF_TRAP_EXPORT(p, ERTS_PROC_LOCKS_ALL, NULL); erts_smp_proc_unlock(p, ERTS_PROC_LOCKS_ALL); #ifdef BM_COUNTERS @@ -8800,8 +12540,6 @@ erts_continue_exit_process(Process *p) if (dep) { erts_do_net_exits(dep, reason); - if(dep) - erts_deref_dist_entry(dep); } /* @@ -8839,6 +12577,9 @@ erts_continue_exit_process(Process *p) if (pbt) erts_free(ERTS_ALC_T_BPD, (void *) pbt); + if (nif_export) + erts_destroy_nif_export(nif_export); + delete_process(p); #ifdef ERTS_SMP @@ -8871,64 +12612,6 @@ erts_continue_exit_process(Process *p) } -/* Callback for process timeout */ -static void -timeout_proc(Process* p) -{ - erts_aint32_t state; - BeamInstr** pi = (BeamInstr **) p->def_arg_reg; - p->i = *pi; - p->flags |= F_TIMO; - p->flags &= ~F_INSLPQUEUE; - - state = erts_smp_atomic32_read_acqb(&p->state); - if (!(state & ERTS_PSFLG_ACTIVE)) - schedule_process(p, state, 0); -} - - -void -cancel_timer(Process* p) -{ - ERTS_SMP_LC_ASSERT(ERTS_PROC_LOCK_MAIN & erts_proc_lc_my_proc_locks(p)); - p->flags &= ~(F_INSLPQUEUE|F_TIMO); -#ifdef ERTS_SMP - erts_cancel_smp_ptimer(p->common.u.alive.ptimer); -#else - erts_cancel_timer(&p->common.u.alive.tm); -#endif -} - -/* - * Insert a process into the time queue, with a timeout 'timeout' in ms. - */ -void -set_timer(Process* p, Uint timeout) -{ - ERTS_SMP_LC_ASSERT(ERTS_PROC_LOCK_MAIN & erts_proc_lc_my_proc_locks(p)); - - /* check for special case timeout=0 DONT ADD TO time queue */ - if (timeout == 0) { - p->flags |= F_TIMO; - return; - } - p->flags |= F_INSLPQUEUE; - p->flags &= ~F_TIMO; - -#ifdef ERTS_SMP - erts_create_smp_ptimer(&p->common.u.alive.ptimer, - p->common.id, - (ErlTimeoutProc) timeout_proc, - timeout); -#else - erts_set_timer(&p->common.u.alive.tm, - (ErlTimeoutProc) timeout_proc, - NULL, - (void*) p, - timeout); -#endif -} - /* * Stack dump functions follow. */ @@ -8944,7 +12627,7 @@ erts_stack_dump(int to, void *to_arg, Process *p) } erts_program_counter_info(to, to_arg, p); for (sp = p->stop; sp < STACK_START(p); sp++) { - yreg = stack_element_dump(to, to_arg, p, sp, yreg); + yreg = stack_element_dump(to, to_arg, sp, yreg); } } @@ -8961,7 +12644,9 @@ erts_program_counter_info(int to, void *to_arg, Process *p) print_function_from_pc(to, to_arg, p->cp); erts_print(to, to_arg, ")\n"); state = erts_smp_atomic32_read_acqb(&p->state); - if (!(state & (ERTS_PSFLG_RUNNING|ERTS_PSFLG_GC))) { + if (!(state & (ERTS_PSFLG_RUNNING + | ERTS_PSFLG_RUNNING_SYS + | ERTS_PSFLG_GC))) { erts_print(to, to_arg, "arity = %d\n",p->arity); if (!ERTS_IS_CRASH_DUMPING) { /* @@ -8999,7 +12684,7 @@ print_function_from_pc(int to, void *to_arg, BeamInstr* x) } static int -stack_element_dump(int to, void *to_arg, Process* p, Eterm* sp, int yreg) +stack_element_dump(int to, void *to_arg, Eterm* sp, int yreg) { Eterm x = *sp; @@ -9028,6 +12713,188 @@ stack_element_dump(int to, void *to_arg, Process* p, Eterm* sp, int yreg) } /* + * Print scheduler information + */ +void +erts_print_scheduler_info(int to, void *to_arg, ErtsSchedulerData *esdp) { + int i; + erts_aint32_t flg; + Process *p; + + erts_print(to, to_arg, "=scheduler:%u\n", esdp->no); + +#ifdef ERTS_SMP + flg = erts_smp_atomic32_read_dirty(&esdp->ssi->flags); + erts_print(to, to_arg, "Scheduler Sleep Info Flags: "); + for (i = 0; i < ERTS_SSI_FLGS_MAX && flg; i++) { + erts_aint32_t chk = (1 << i); + if (flg & chk) { + switch (chk) { + case ERTS_SSI_FLG_SLEEPING: + erts_print(to, to_arg, "SLEEPING"); break; + case ERTS_SSI_FLG_POLL_SLEEPING: + erts_print(to, to_arg, "POLL_SLEEPING"); break; + case ERTS_SSI_FLG_TSE_SLEEPING: + erts_print(to, to_arg, "TSE_SLEEPING"); break; + case ERTS_SSI_FLG_WAITING: + erts_print(to, to_arg, "WAITING"); break; + case ERTS_SSI_FLG_SUSPENDED: + erts_print(to, to_arg, "SUSPENDED"); break; + default: + erts_print(to, to_arg, "UNKNOWN(%d)", flg); break; + } + if (flg > chk) + erts_print(to, to_arg, " | "); + flg -= chk; + } + } + erts_print(to, to_arg, "\n"); +#endif + + flg = erts_atomic32_read_dirty(&esdp->ssi->aux_work); + erts_print(to, to_arg, "Scheduler Sleep Info Aux Work: "); + for (i = 0; i < ERTS_SSI_AUX_WORK_NO_FLAGS && flg; i++) { + erts_aint32_t chk = (1 << i); + if (flg & chk) { + if (erts_aux_work_flag_descr[i]) + erts_print(to, to_arg, "%s", erts_aux_work_flag_descr[i]); + else + erts_print(to, to_arg, "1<<%d", i); + if (flg > chk) + erts_print(to, to_arg, " | "); + flg -= chk; + } + } + erts_print(to, to_arg, "\n"); + + erts_print(to, to_arg, "Current Port: "); + if (esdp->current_port) + erts_print(to, to_arg, "%T", esdp->current_port->common.id); + erts_print(to, to_arg, "\n"); + + for (i = 0; i < ERTS_NO_PROC_PRIO_LEVELS; i++) { + erts_print(to, to_arg, "Run Queue "); + switch (i) { + case PRIORITY_MAX: + erts_print(to, to_arg, "Max "); + break; + case PRIORITY_HIGH: + erts_print(to, to_arg, "High "); + break; + case PRIORITY_NORMAL: + erts_print(to, to_arg, "Normal "); + break; + case PRIORITY_LOW: + erts_print(to, to_arg, "Low "); + break; + default: + erts_print(to, to_arg, "Unknown "); + break; + } + erts_print(to, to_arg, "Length: %d\n", + erts_smp_atomic32_read_dirty(&esdp->run_queue->procs.prio_info[i].len)); + } + erts_print(to, to_arg, "Run Queue Port Length: %d\n", + erts_smp_atomic32_read_dirty(&esdp->run_queue->ports.info.len)); + + flg = erts_smp_atomic32_read_dirty(&esdp->run_queue->flags); + erts_print(to, to_arg, "Run Queue Flags: "); + for (i = 0; i < ERTS_RUNQ_FLG_MAX && flg; i++) { + erts_aint32_t chk = (1 << i); + if (flg & chk) { + switch (chk) { + case (1 << PRIORITY_MAX): + erts_print(to, to_arg, "NONEMPTY_MAX"); break; + case (1 << PRIORITY_HIGH): + erts_print(to, to_arg, "NONEMPTY_HIGH"); break; + case (1 << PRIORITY_NORMAL): + erts_print(to, to_arg, "NONEMPTY_NORMAL"); break; + case (1 << PRIORITY_LOW): + erts_print(to, to_arg, "NONEMPTY_LOW"); break; + case (1 << (PRIORITY_MAX + ERTS_RUNQ_FLGS_EMIGRATE_SHFT)): + erts_print(to, to_arg, "EMIGRATE_MAX"); break; + case (1 << (PRIORITY_HIGH + ERTS_RUNQ_FLGS_EMIGRATE_SHFT)): + erts_print(to, to_arg, "EMIGRATE_HIGH"); break; + case (1 << (PRIORITY_NORMAL + ERTS_RUNQ_FLGS_EMIGRATE_SHFT)): + erts_print(to, to_arg, "EMIGRATE_NORMAL"); break; + case (1 << (PRIORITY_LOW + ERTS_RUNQ_FLGS_EMIGRATE_SHFT)): + erts_print(to, to_arg, "EMIGRATE_LOW"); break; + case (1 << (PRIORITY_MAX + ERTS_RUNQ_FLGS_IMMIGRATE_SHFT)): + erts_print(to, to_arg, "IMMIGRATE_MAX"); break; + case (1 << (PRIORITY_HIGH + ERTS_RUNQ_FLGS_IMMIGRATE_SHFT)): + erts_print(to, to_arg, "IMMIGRATE_HIGH"); break; + case (1 << (PRIORITY_NORMAL + ERTS_RUNQ_FLGS_IMMIGRATE_SHFT)): + erts_print(to, to_arg, "IMMIGRATE_NORMAL"); break; + case (1 << (PRIORITY_LOW + ERTS_RUNQ_FLGS_IMMIGRATE_SHFT)): + erts_print(to, to_arg, "IMMIGRATE_LOW"); break; + case (1 << (PRIORITY_MAX + ERTS_RUNQ_FLGS_EVACUATE_SHFT)): + erts_print(to, to_arg, "EVACUATE_MAX"); break; + case (1 << (PRIORITY_HIGH + ERTS_RUNQ_FLGS_EVACUATE_SHFT)): + erts_print(to, to_arg, "EVACUATE_HIGH"); break; + case (1 << (PRIORITY_NORMAL + ERTS_RUNQ_FLGS_EVACUATE_SHFT)): + erts_print(to, to_arg, "EVACUATE_NORMAL"); break; + case (1 << (PRIORITY_LOW + ERTS_RUNQ_FLGS_EVACUATE_SHFT)): + erts_print(to, to_arg, "EVACUATE_LOW"); break; + case ERTS_RUNQ_FLG_OUT_OF_WORK: + erts_print(to, to_arg, "OUT_OF_WORK"); break; + case ERTS_RUNQ_FLG_HALFTIME_OUT_OF_WORK: + erts_print(to, to_arg, "HALFTIME_OUT_OF_WORK"); break; + case ERTS_RUNQ_FLG_SUSPENDED: + erts_print(to, to_arg, "SUSPENDED"); break; + case ERTS_RUNQ_FLG_CHK_CPU_BIND: + erts_print(to, to_arg, "CHK_CPU_BIND"); break; + case ERTS_RUNQ_FLG_INACTIVE: + erts_print(to, to_arg, "INACTIVE"); break; + case ERTS_RUNQ_FLG_NONEMPTY: + erts_print(to, to_arg, "NONEMPTY"); break; + case ERTS_RUNQ_FLG_PROTECTED: + erts_print(to, to_arg, "PROTECTED"); break; + default: + erts_print(to, to_arg, "UNKNOWN(%d)", flg); break; + } + if (flg > chk) + erts_print(to, to_arg, " | "); + flg -= chk; + } + } + erts_print(to, to_arg, "\n"); + + /* This *MUST* to be the last information in scheduler block */ + p = esdp->current_process; + erts_print(to, to_arg, "Current Process: "); + if (esdp->current_process && !(ERTS_TRACE_FLAGS(p) & F_SENSITIVE)) { + flg = erts_smp_atomic32_read_dirty(&p->state); + erts_print(to, to_arg, "%T\n", p->common.id); + + erts_print(to, to_arg, "Current Process State: "); + erts_dump_process_state(to, to_arg, flg); + + erts_print(to, to_arg, "Current Process Internal State: "); + erts_dump_extended_process_state(to, to_arg, flg); + + erts_print(to, to_arg, "Current Process Program counter: %p (", p->i); + print_function_from_pc(to, to_arg, p->i); + erts_print(to, to_arg, ")\n"); + erts_print(to, to_arg, "Current Process CP: %p (", p->cp); + print_function_from_pc(to, to_arg, p->cp); + erts_print(to, to_arg, ")\n"); + + /* Getting this stacktrace can segfault if we are very very + unlucky if called while a process is being garbage collected. + Therefore we only call this on other schedulers if we either + have protection against segfaults, or we know that the process + is not garbage collecting. It *should* always be safe to call + on a process owned by us, even if it is currently being garbage + collected. + */ + erts_print(to, to_arg, "Current Process Limited Stack Trace:\n"); + erts_limited_stack_trace(to, to_arg, p); + } else + erts_print(to, to_arg, "\n"); + +} + +/* * A nice system halt closing all open port goes as follows: * 1) This function schedules the aux work ERTS_SSI_AUX_WORK_REAP_PORTS * on all schedulers, then schedules itself out. @@ -9041,14 +12908,18 @@ stack_element_dump(int to, void *to_arg, Process* p, Eterm* sp, int yreg) * The same global atomic is used as refcount. * * A BIF that calls this should make sure to schedule out to never come back: - * erl_halt((int)(- code)); + * erts_halt(code); * ERTS_BIF_YIELD1(bif_export[BIF_erlang_halt_1], BIF_P, NIL); */ -void erl_halt(int code) +void erts_halt(int code) { if (-1 == erts_smp_atomic32_cmpxchg_acqb(&erts_halt_progress, erts_no_schedulers, -1)) { +#ifdef ERTS_DIRTY_SCHEDULERS + ERTS_DIRTY_CPU_RUNQ->halt_in_progress = 1; + ERTS_DIRTY_IO_RUNQ->halt_in_progress = 1; +#endif erts_halt_code = code; notify_reap_ports_relb(); } diff --git a/erts/emulator/beam/erl_process.h b/erts/emulator/beam/erl_process.h index 6d1032c292..b536426b0f 100644 --- a/erts/emulator/beam/erl_process.h +++ b/erts/emulator/beam/erl_process.h @@ -1,18 +1,19 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 1996-2012. All Rights Reserved. + * Copyright Ericsson AB 1996-2014. All Rights Reserved. * - * The contents of this file are subject to the Erlang Public License, - * Version 1.1, (the "License"); you may not use this file except in - * compliance with the License. You should have received a copy of the - * Erlang Public License along with this software. If not, it can be - * retrieved online at http://www.erlang.org/. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Software distributed under the License is distributed on an "AS IS" - * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See - * the License for the specific language governing rights and limitations - * under the License. + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. * * %CopyrightEnd% */ @@ -52,12 +53,16 @@ typedef struct process Process; #include "erl_node_container_utils.h" #include "erl_node_tables.h" #include "erl_monitors.h" -#include "erl_bif_timer.h" +#include "erl_hl_timer.h" #include "erl_time.h" #include "erl_atom_table.h" #include "external.h" #include "erl_mseg.h" #include "erl_async.h" +#include "erl_gc.h" +#define ERTS_ONLY_INCLUDE_TRACE_FLAGS +#include "erl_trace.h" +#undef ERTS_ONLY_INCLUDE_TRACE_FLAGS #ifdef HIPE #include "hipe_process.h" @@ -70,7 +75,14 @@ typedef struct process Process; struct ErtsNodesMonitor_; +#define ERTS_HAVE_SCHED_UTIL_BALANCING_SUPPORT_OPT 0 +#define ERTS_HAVE_SCHED_UTIL_BALANCING_SUPPORT 0 + #define ERTS_MAX_NO_OF_SCHEDULERS 1024 +#ifdef ERTS_DIRTY_SCHEDULERS +#define ERTS_MAX_NO_OF_DIRTY_CPU_SCHEDULERS ERTS_MAX_NO_OF_SCHEDULERS +#define ERTS_MAX_NO_OF_DIRTY_IO_SCHEDULERS ERTS_MAX_NO_OF_SCHEDULERS +#endif #define ERTS_DEFAULT_MAX_PROCESSES (1 << 18) @@ -97,8 +109,14 @@ struct saved_calls { }; extern Export exp_send, exp_receive, exp_timeout; +extern int erts_eager_check_io; extern int erts_sched_compact_load; +extern int erts_sched_balance_util; extern Uint erts_no_schedulers; +#ifdef ERTS_DIRTY_SCHEDULERS +extern Uint erts_no_dirty_cpu_schedulers; +extern Uint erts_no_dirty_io_schedulers; +#endif extern Uint erts_no_run_queues; extern int erts_sched_thread_suggested_stack_size; #define ERTS_SCHED_THREAD_MIN_STACK_SIZE 4 /* Kilo words */ @@ -155,6 +173,10 @@ extern int erts_sched_thread_suggested_stack_size; (((Uint32) 1) << (ERTS_RUNQ_FLG_BASE2 + 5)) #define ERTS_RUNQ_FLG_PROTECTED \ (((Uint32) 1) << (ERTS_RUNQ_FLG_BASE2 + 6)) +#define ERTS_RUNQ_FLG_EXEC \ + (((Uint32) 1) << (ERTS_RUNQ_FLG_BASE2 + 7)) + +#define ERTS_RUNQ_FLG_MAX (ERTS_RUNQ_FLG_BASE2 + 8) #define ERTS_RUNQ_FLGS_MIGRATION_QMASKS \ (ERTS_RUNQ_FLGS_EMIGRATE_QMASK \ @@ -198,9 +220,19 @@ extern int erts_sched_thread_suggested_stack_size; #define ERTS_RUNQ_FLGS_SET(RQ, FLGS) \ ((Uint32) erts_smp_atomic32_read_bor_relb(&(RQ)->flags, \ (erts_aint32_t) (FLGS))) +#define ERTS_RUNQ_FLGS_SET_NOB(RQ, FLGS) \ + ((Uint32) erts_smp_atomic32_read_bor_nob(&(RQ)->flags, \ + (erts_aint32_t) (FLGS))) +#define ERTS_RUNQ_FLGS_BSET(RQ, MSK, FLGS) \ + ((Uint32) erts_smp_atomic32_read_bset_relb(&(RQ)->flags, \ + (erts_aint32_t) (MSK), \ + (erts_aint32_t) (FLGS))) #define ERTS_RUNQ_FLGS_UNSET(RQ, FLGS) \ ((Uint32) erts_smp_atomic32_read_band_relb(&(RQ)->flags, \ (erts_aint32_t) ~(FLGS))) +#define ERTS_RUNQ_FLGS_UNSET_NOB(RQ, FLGS) \ + ((Uint32) erts_smp_atomic32_read_band_nob(&(RQ)->flags, \ + (erts_aint32_t) ~(FLGS))) #define ERTS_RUNQ_FLGS_GET(RQ) \ ((Uint32) erts_smp_atomic32_read_acqb(&(RQ)->flags)) #define ERTS_RUNQ_FLGS_GET_NOB(RQ) \ @@ -234,6 +266,8 @@ typedef enum { #define ERTS_SSI_FLG_WAITING (((erts_aint32_t) 1) << 3) #define ERTS_SSI_FLG_SUSPENDED (((erts_aint32_t) 1) << 4) +#define ERTS_SSI_FLGS_MAX 5 + #define ERTS_SSI_FLGS_SLEEP_TYPE \ (ERTS_SSI_FLG_TSE_SLEEPING|ERTS_SSI_FLG_POLL_SLEEPING) @@ -246,27 +280,83 @@ typedef enum { | ERTS_SSI_FLG_SUSPENDED) /* - * Keep ERTS_SSI_AUX_WORK flags in expected frequency order relative - * eachother. Most frequent - lowest bit number. + * Keep ERTS_SSI_AUX_WORK flags ordered in expected frequency + * order relative eachother. Most frequent at lowest at lowest + * index. + * + * ERTS_SSI_AUX_WORK_DEBUG_WAIT_COMPLETED_IX *need* to be + * highest index... + * + * Remember to update description in erts_pre_init_process() + * when adding new flags... */ -#define ERTS_SSI_AUX_WORK_DELAYED_AW_WAKEUP (((erts_aint32_t) 1) << 0) -#define ERTS_SSI_AUX_WORK_DD (((erts_aint32_t) 1) << 1) -#define ERTS_SSI_AUX_WORK_DD_THR_PRGR (((erts_aint32_t) 1) << 2) -#define ERTS_SSI_AUX_WORK_FIX_ALLOC_DEALLOC (((erts_aint32_t) 1) << 3) -#define ERTS_SSI_AUX_WORK_FIX_ALLOC_LOWER_LIM (((erts_aint32_t) 1) << 4) -#define ERTS_SSI_AUX_WORK_THR_PRGR_LATER_OP (((erts_aint32_t) 1) << 5) -#define ERTS_SSI_AUX_WORK_ASYNC_READY (((erts_aint32_t) 1) << 6) -#define ERTS_SSI_AUX_WORK_ASYNC_READY_CLEAN (((erts_aint32_t) 1) << 7) -#define ERTS_SSI_AUX_WORK_MISC_THR_PRGR (((erts_aint32_t) 1) << 8) -#define ERTS_SSI_AUX_WORK_MISC (((erts_aint32_t) 1) << 9) -#define ERTS_SSI_AUX_WORK_CHECK_CHILDREN (((erts_aint32_t) 1) << 10) -#define ERTS_SSI_AUX_WORK_SET_TMO (((erts_aint32_t) 1) << 11) -#define ERTS_SSI_AUX_WORK_MSEG_CACHE_CHECK (((erts_aint32_t) 1) << 12) -#define ERTS_SSI_AUX_WORK_REAP_PORTS (((erts_aint32_t) 1) << 13) +typedef enum { + ERTS_SSI_AUX_WORK_DELAYED_AW_WAKEUP_IX, + ERTS_SSI_AUX_WORK_DD_IX, + ERTS_SSI_AUX_WORK_DD_THR_PRGR_IX, + ERTS_SSI_AUX_WORK_FIX_ALLOC_DEALLOC_IX, + ERTS_SSI_AUX_WORK_FIX_ALLOC_LOWER_LIM_IX, + ERTS_SSI_AUX_WORK_THR_PRGR_LATER_OP_IX, + ERTS_SSI_AUX_WORK_CNCLD_TMRS_IX, + ERTS_SSI_AUX_WORK_CNCLD_TMRS_THR_PRGR_IX, + ERTS_SSI_AUX_WORK_ASYNC_READY_IX, + ERTS_SSI_AUX_WORK_ASYNC_READY_CLEAN_IX, + ERTS_SSI_AUX_WORK_MISC_THR_PRGR_IX, + ERTS_SSI_AUX_WORK_MISC_IX, + ERTS_SSI_AUX_WORK_CHECK_CHILDREN_IX, + ERTS_SSI_AUX_WORK_SET_TMO_IX, + ERTS_SSI_AUX_WORK_MSEG_CACHE_CHECK_IX, + ERTS_SSI_AUX_WORK_REAP_PORTS_IX, + ERTS_SSI_AUX_WORK_DEBUG_WAIT_COMPLETED_IX, /* SHOULD be last flag index */ + + ERTS_SSI_AUX_WORK_NO_FLAGS /* Not a flag index... */ +} ErtsSsiAuxWorkFlagIndex; + +#define ERTS_SSI_AUX_WORK_DELAYED_AW_WAKEUP \ + (((erts_aint32_t) 1) << ERTS_SSI_AUX_WORK_DELAYED_AW_WAKEUP_IX) +#define ERTS_SSI_AUX_WORK_DD \ + (((erts_aint32_t) 1) << ERTS_SSI_AUX_WORK_DD_IX) +#define ERTS_SSI_AUX_WORK_DD_THR_PRGR \ + (((erts_aint32_t) 1) << ERTS_SSI_AUX_WORK_DD_THR_PRGR_IX) +#define ERTS_SSI_AUX_WORK_FIX_ALLOC_DEALLOC \ + (((erts_aint32_t) 1) << ERTS_SSI_AUX_WORK_FIX_ALLOC_DEALLOC_IX) +#define ERTS_SSI_AUX_WORK_FIX_ALLOC_LOWER_LIM \ + (((erts_aint32_t) 1) << ERTS_SSI_AUX_WORK_FIX_ALLOC_LOWER_LIM_IX) +#define ERTS_SSI_AUX_WORK_THR_PRGR_LATER_OP \ + (((erts_aint32_t) 1) << ERTS_SSI_AUX_WORK_THR_PRGR_LATER_OP_IX) +#define ERTS_SSI_AUX_WORK_CNCLD_TMRS \ + (((erts_aint32_t) 1) << ERTS_SSI_AUX_WORK_CNCLD_TMRS_IX) +#define ERTS_SSI_AUX_WORK_CNCLD_TMRS_THR_PRGR \ + (((erts_aint32_t) 1) << ERTS_SSI_AUX_WORK_CNCLD_TMRS_THR_PRGR_IX) +#define ERTS_SSI_AUX_WORK_ASYNC_READY \ + (((erts_aint32_t) 1) << ERTS_SSI_AUX_WORK_ASYNC_READY_IX) +#define ERTS_SSI_AUX_WORK_ASYNC_READY_CLEAN \ + (((erts_aint32_t) 1) << ERTS_SSI_AUX_WORK_ASYNC_READY_CLEAN_IX) +#define ERTS_SSI_AUX_WORK_MISC_THR_PRGR \ + (((erts_aint32_t) 1) << ERTS_SSI_AUX_WORK_MISC_THR_PRGR_IX) +#define ERTS_SSI_AUX_WORK_MISC \ + (((erts_aint32_t) 1) << ERTS_SSI_AUX_WORK_MISC_IX) +#define ERTS_SSI_AUX_WORK_CHECK_CHILDREN \ + (((erts_aint32_t) 1) << ERTS_SSI_AUX_WORK_CHECK_CHILDREN_IX) +#define ERTS_SSI_AUX_WORK_SET_TMO \ + (((erts_aint32_t) 1) << ERTS_SSI_AUX_WORK_SET_TMO_IX) +#define ERTS_SSI_AUX_WORK_MSEG_CACHE_CHECK \ + (((erts_aint32_t) 1) << ERTS_SSI_AUX_WORK_MSEG_CACHE_CHECK_IX) +#define ERTS_SSI_AUX_WORK_REAP_PORTS \ + (((erts_aint32_t) 1) << ERTS_SSI_AUX_WORK_REAP_PORTS_IX) +#define ERTS_SSI_AUX_WORK_DEBUG_WAIT_COMPLETED \ + (((erts_aint32_t) 1) << ERTS_SSI_AUX_WORK_DEBUG_WAIT_COMPLETED_IX) typedef struct ErtsSchedulerSleepInfo_ ErtsSchedulerSleepInfo; +#ifdef ERTS_DIRTY_SCHEDULERS +typedef struct { + erts_smp_spinlock_t lock; + ErtsSchedulerSleepInfo *list; +} ErtsSchedulerSleepList; +#endif + struct ErtsSchedulerSleepInfo_ { #ifdef ERTS_SMP ErtsSchedulerSleepInfo *next; @@ -316,9 +406,34 @@ typedef struct { int reds; } ErtsRunQueueInfo; + +#ifdef ERTS_HAVE_OS_MONOTONIC_TIME_SUPPORT +# undef ERTS_HAVE_SCHED_UTIL_BALANCING_SUPPORT_OPT +# define ERTS_HAVE_SCHED_UTIL_BALANCING_SUPPORT_OPT 1 +#endif + #ifdef ERTS_SMP +#undef ERTS_HAVE_SCHED_UTIL_BALANCING_SUPPORT +#define ERTS_HAVE_SCHED_UTIL_BALANCING_SUPPORT ERTS_HAVE_SCHED_UTIL_BALANCING_SUPPORT_OPT + +typedef erts_atomic64_t ErtsAtomicSchedTime; + +#if ERTS_HAVE_SCHED_UTIL_BALANCING_SUPPORT typedef struct { + ErtsAtomicSchedTime last; + struct { + Uint64 short_interval; + Uint64 long_interval; + } worktime; + int is_working; +} ErtsRunQueueSchedUtil; +#endif + +typedef struct { +#if ERTS_HAVE_SCHED_UTIL_BALANCING_SUPPORT + int sched_util; +#endif Uint32 flags; ErtsRunQueue *misc_evac_runq; struct { @@ -348,6 +463,12 @@ struct ErtsRunQueue_ { erts_smp_mtx_t mtx; erts_smp_cnd_t cnd; +#ifdef ERTS_DIRTY_SCHEDULERS +#ifdef ERTS_SMP + ErtsSchedulerSleepList sleepers; +#endif +#endif + ErtsSchedulerData *scheduler; int waiting; /* < 0 in sys schedule; > 0 on cnd variable */ int woken; @@ -357,7 +478,7 @@ struct ErtsRunQueue_ { int full_reds_history[ERTS_FULL_REDS_HISTORY_SIZE]; int out_of_work_count; erts_aint32_t max_len; - erts_aint32_t len; + erts_smp_atomic32_t len; int wakeup_other; int wakeup_other_reds; int halt_in_progress; @@ -385,8 +506,15 @@ struct ErtsRunQueue_ { Port *start; Port *end; } ports; +#if ERTS_HAVE_SCHED_UTIL_BALANCING_SUPPORT + ErtsRunQueueSchedUtil sched_util; +#endif }; +#ifdef ERTS_SMP +extern long erts_runq_supervision_interval; +#endif + typedef union { ErtsRunQueue runq; char align[ERTS_ALC_CACHE_LINE_ALIGN_SIZE(sizeof(ErtsRunQueue))]; @@ -394,22 +522,25 @@ typedef union { extern ErtsAlignedRunQueue *erts_aligned_run_queues; -#define ERTS_PROC_REDUCTIONS_EXECUTED(RQ, PRIO, REDS, AREDS) \ +#define ERTS_PROC_REDUCTIONS_EXECUTED(SD, RQ, PRIO, REDS, AREDS)\ do { \ (RQ)->procs.reductions += (AREDS); \ (RQ)->procs.prio_info[(PRIO)].reds += (REDS); \ (RQ)->check_balance_reds -= (REDS); \ (RQ)->wakeup_other_reds += (AREDS); \ + (SD)->check_time_reds += (AREDS); \ } while (0) -#define ERTS_PORT_REDUCTIONS_EXECUTED(RQ, REDS) \ +#define ERTS_PORT_REDUCTIONS_EXECUTED(SD, RQ, REDS) \ do { \ (RQ)->ports.info.reds += (REDS); \ (RQ)->check_balance_reds -= (REDS); \ (RQ)->wakeup_other_reds += (REDS); \ + (SD)->check_time_reds += (REDS); \ } while (0) typedef struct { + int need; /* "+sbu true" or scheduler_wall_time enabled */ int enabled; Uint64 start; struct { @@ -430,6 +561,7 @@ typedef struct { ErtsSchedulerSleepInfo *ssi; #ifdef ERTS_SMP ErtsThrPrgrVal current_thr_prgr; + ErtsThrPrgrVal latest_wakeup; #endif struct { int ix; @@ -440,10 +572,13 @@ typedef struct { #ifdef ERTS_SMP struct { ErtsThrPrgrVal thr_prgr; - void (*completed_callback)(void *); - void (*completed_arg)(void *); } dd; struct { + ErtsThrPrgrVal thr_prgr; + } cncld_tmrs; + struct { + ErtsThrPrgrVal thr_prgr; + UWord size; ErtsThrPrgrLaterOp *first; ErtsThrPrgrLaterOp *last; } later_op; @@ -465,8 +600,30 @@ typedef struct { ErtsDelayedAuxWorkWakeupJob *job; } delayed_wakeup; #endif + struct { + struct { + erts_aint32_t flags; + void (*callback)(void *); + void *arg; + } wait_completed; + } debug; } ErtsAuxWorkData; +#ifdef ERTS_DIRTY_SCHEDULERS +typedef enum { + ERTS_DIRTY_CPU_SCHEDULER, + ERTS_DIRTY_IO_SCHEDULER +} ErtsDirtySchedulerType; + +typedef union { + struct { + ErtsDirtySchedulerType type: 1; + Uint num: sizeof(Uint)*8 - 1; + } s; + Uint no; +} ErtsDirtySchedId; +#endif + struct ErtsSchedulerData_ { /* * Keep X registers first (so we get as many low @@ -476,6 +633,9 @@ struct ErtsSchedulerData_ { Eterm* x_reg_array; /* X registers */ FloatDef* f_reg_array; /* Floating point registers. */ + ErtsTimerWheel *timer_wheel; + ErtsNextTimeoutRef next_tmo_ref; + ErtsHLTimerService *timer_service; #ifdef ERTS_SMP ethr_tid tid; /* Thread id */ struct erl_bits_state erl_bits_state; /* erl_bits.c state */ @@ -487,12 +647,14 @@ struct ErtsSchedulerData_ { Eterm tmp_heap[TMP_HEAP_SIZE]; int num_tmp_heap_used; Eterm beam_emu_tmp_heap[BEAM_EMU_TMP_HEAP_SIZE]; - Eterm cmp_tmp_heap[CMP_TMP_HEAP_SIZE]; Eterm erl_arith_tmp_heap[ERL_ARITH_TMP_HEAP_SIZE]; #endif ErtsSchedulerSleepInfo *ssi; Process *current_process; - Uint no; /* Scheduler number */ + Uint no; /* Scheduler number for normal schedulers */ +#ifdef ERTS_DIRTY_SCHEDULERS + ErtsDirtySchedId dirty_no; /* Scheduler number for dirty schedulers */ +#endif Port *current_port; ErtsRunQueue *run_queue; int virtual_reds; @@ -500,10 +662,24 @@ struct ErtsSchedulerData_ { ErtsAuxWorkData aux_work_data; ErtsAtomCacheMap atom_cache_map; + ErtsMonotonicTime last_monotonic_time; + int check_time_reds; + + Uint32 thr_id; + Uint64 unique; + Uint64 ref; + ErtsSchedAllocData alloc_data; + struct { + Uint64 out; + Uint64 in; + } io; + Uint64 reductions; ErtsSchedWallTime sched_wall_time; + ErtsGCInfo gc_info; + ErtsPortTaskHandle nosuspend_port_task_handle; #ifdef ERTS_DO_VERIFY_UNUSED_TEMP_ALLOC erts_alloc_verify_func_t verify_unused_temp_alloc; @@ -517,17 +693,34 @@ typedef union { } ErtsAlignedSchedulerData; extern ErtsAlignedSchedulerData *erts_aligned_scheduler_data; +#ifdef ERTS_DIRTY_SCHEDULERS +extern ErtsAlignedSchedulerData *erts_aligned_dirty_cpu_scheduler_data; +extern ErtsAlignedSchedulerData *erts_aligned_dirty_io_scheduler_data; +#endif #ifndef ERTS_SMP extern ErtsSchedulerData *erts_scheduler_data; #endif +#ifdef ERTS_SCHED_FAIR +#define ERTS_SCHED_FAIR_YIELD() ETHR_YIELD() +#else +#define ERTS_SCHED_FAIR 0 +#define ERTS_SCHED_FAIR_YIELD() +#endif + #if defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_CHECK) int erts_smp_lc_runq_is_locked(ErtsRunQueue *); #endif #ifdef ERTS_INCLUDE_SCHEDULER_INTERNALS +#ifdef ERTS_SMP +void erts_empty_runq(ErtsRunQueue *rq); +void erts_non_empty_runq(ErtsRunQueue *rq); +#endif + + /* * Run queue locked during modifications. We use atomic ops since * other threads peek at values without run queue lock. @@ -546,7 +739,19 @@ erts_smp_inc_runq_len(ErtsRunQueue *rq, ErtsRunQueueInfo *rqi, int prio) ERTS_SMP_LC_ASSERT(erts_smp_lc_runq_is_locked(rq)); - len = erts_smp_atomic32_read_nob(&rqi->len); + len = erts_smp_atomic32_read_dirty(&rq->len); + +#ifdef ERTS_SMP + if (len == 0) + erts_non_empty_runq(rq); +#endif + len++; + if (rq->max_len < len) + rq->max_len = len; + ASSERT(len > 0); + erts_smp_atomic32_set_nob(&rq->len, len); + + len = erts_smp_atomic32_read_dirty(&rqi->len); ASSERT(len >= 0); if (len == 0) { ASSERT((erts_smp_atomic32_read_nob(&rq->flags) @@ -559,11 +764,6 @@ erts_smp_inc_runq_len(ErtsRunQueue *rq, ErtsRunQueueInfo *rqi, int prio) rqi->max_len = len; erts_smp_atomic32_set_relb(&rqi->len, len); - - rq->len++; - if (rq->max_len < rq->len) - rq->max_len = len; - ASSERT(rq->len > 0); } ERTS_GLB_INLINE void @@ -573,7 +773,12 @@ erts_smp_dec_runq_len(ErtsRunQueue *rq, ErtsRunQueueInfo *rqi, int prio) ERTS_SMP_LC_ASSERT(erts_smp_lc_runq_is_locked(rq)); - len = erts_smp_atomic32_read_nob(&rqi->len); + len = erts_smp_atomic32_read_dirty(&rq->len); + len--; + ASSERT(len >= 0); + erts_smp_atomic32_set_nob(&rq->len, len); + + len = erts_smp_atomic32_read_dirty(&rqi->len); len--; ASSERT(len >= 0); if (len == 0) { @@ -584,8 +789,6 @@ erts_smp_dec_runq_len(ErtsRunQueue *rq, ErtsRunQueueInfo *rqi, int prio) } erts_smp_atomic32_set_relb(&rqi->len, len); - rq->len--; - ASSERT(rq->len >= 0); } ERTS_GLB_INLINE void @@ -595,7 +798,7 @@ erts_smp_reset_max_len(ErtsRunQueue *rq, ErtsRunQueueInfo *rqi) ERTS_SMP_LC_ASSERT(erts_smp_lc_runq_is_locked(rq)); - len = erts_smp_atomic32_read_nob(&rqi->len); + len = erts_smp_atomic32_read_dirty(&rqi->len); ASSERT(rqi->max_len >= len); rqi->max_len = len; } @@ -615,10 +818,11 @@ erts_smp_reset_max_len(ErtsRunQueue *rq, ErtsRunQueueInfo *rqi) #define ERTS_PSD_ERROR_HANDLER 0 #define ERTS_PSD_SAVED_CALLS_BUF 1 #define ERTS_PSD_SCHED_ID 2 -#define ERTS_PSD_DIST_ENTRY 3 -#define ERTS_PSD_CALL_TIME_BP 4 +#define ERTS_PSD_CALL_TIME_BP 3 +#define ERTS_PSD_DELAYED_GC_TASK_QS 4 +#define ERTS_PSD_NIF_TRAP_EXPORT 5 -#define ERTS_PSD_SIZE 5 +#define ERTS_PSD_SIZE 6 typedef struct { void *data[ERTS_PSD_SIZE]; @@ -636,12 +840,15 @@ typedef struct { #define ERTS_PSD_SCHED_ID_GET_LOCKS ERTS_PROC_LOCK_STATUS #define ERTS_PSD_SCHED_ID_SET_LOCKS ERTS_PROC_LOCK_STATUS -#define ERTS_PSD_DIST_ENTRY_GET_LOCKS ERTS_PROC_LOCK_MAIN -#define ERTS_PSD_DIST_ENTRY_SET_LOCKS ERTS_PROC_LOCK_MAIN - #define ERTS_PSD_CALL_TIME_BP_GET_LOCKS ERTS_PROC_LOCK_MAIN #define ERTS_PSD_CALL_TIME_BP_SET_LOCKS ERTS_PROC_LOCK_MAIN +#define ERTS_PSD_DELAYED_GC_TASK_QS_GET_LOCKS ERTS_PROC_LOCK_MAIN +#define ERTS_PSD_DELAYED_GC_TASK_QS_SET_LOCKS ERTS_PROC_LOCK_MAIN + +#define ERTS_PSD_NIF_TRAP_EXPORT_GET_LOCKS ERTS_PROC_LOCK_MAIN +#define ERTS_PSD_NIF_TRAP_EXPORT_SET_LOCKS ERTS_PROC_LOCK_MAIN + typedef struct { ErtsProcLocks get_locks; ErtsProcLocks set_locks; @@ -674,6 +881,9 @@ typedef struct { ErlHeapFragment *bp; } ErtsPendExit; +typedef struct ErtsProcSysTask_ ErtsProcSysTask; +typedef struct ErtsProcSysTaskQs_ ErtsProcSysTaskQs; + #ifdef ERTS_SMP typedef struct ErtsPendingSuspend_ ErtsPendingSuspend; @@ -689,6 +899,7 @@ struct ErtsPendingSuspend_ { #endif + /* Defines to ease the change of memory architecture */ # define HEAP_START(p) (p)->heap # define HEAP_TOP(p) (p)->htop @@ -782,10 +993,10 @@ struct process { ErlMessageQueue msg; /* Message queue */ - union { - ErtsBifTimer *bif_timers; /* Bif timers aiming at this process */ - void *terminate; - } u; + ErtsBifTimers *bif_timers; /* Bif timers aiming at this process */ +#ifdef ERTS_BTM_ACCESSOR_SUPPORT + ErtsBifTimers *accessor_bif_timers; /* Accessor bif timers */ +#endif ProcDict *dictionary; /* Process dictionary, may be NULL */ @@ -796,9 +1007,12 @@ struct process { #ifdef USE_VM_PROBES Eterm dt_utag; /* Place to store the dynamc trace user tag */ Uint dt_utag_flags; /* flag field for the dt_utag */ -#endif - BeamInstr initial[3]; /* Initial module(0), function(1), arity(2), often used instead +#endif + union { + void *terminate; + BeamInstr initial[3]; /* Initial module(0), function(1), arity(2), often used instead of pointer to funcinfo instruction, hence the BeamInstr datatype */ + } u; BeamInstr* current; /* Current Erlang function, part of the funcinfo: * module(0), function(1), arity(2) * (module and functions are tagged atoms; @@ -811,6 +1025,8 @@ struct process { Eterm parent; /* Pid of process that created this process. */ erts_approx_time_t approx_started; /* Time when started. */ + Uint32 static_flags; /* Flags that do *not* change */ + /* This is the place, where all fields that differs between memory * architectures, have gone to. */ @@ -831,6 +1047,8 @@ struct process { Uint64 bin_old_vheap_sz; /* Virtual old heap block size for binaries */ Uint64 bin_old_vheap; /* Virtual old heap size for binaries */ + ErtsProcSysTaskQs *sys_task_qs; + erts_smp_atomic32_t state; /* Process state flags (see ERTS_PSFLG_*) */ #ifdef ERTS_SMP @@ -900,24 +1118,81 @@ void erts_check_for_holes(Process* p); # error "Need to increase ERTS_PSFLG_PRIO_SHIFT" #endif -#define ERTS_PSFLG_PRIO_SHIFT 2 +#define ERTS_PSFLGS_PRIO_BITS 2 +#define ERTS_PSFLGS_PRIO_MASK \ + ((((erts_aint32_t) 1) << ERTS_PSFLGS_PRIO_BITS) - 1) + +#define ERTS_PSFLGS_ACT_PRIO_OFFSET (0*ERTS_PSFLGS_PRIO_BITS) +#define ERTS_PSFLGS_USR_PRIO_OFFSET (1*ERTS_PSFLGS_PRIO_BITS) +#define ERTS_PSFLGS_PRQ_PRIO_OFFSET (2*ERTS_PSFLGS_PRIO_BITS) +#define ERTS_PSFLGS_ZERO_BIT_OFFSET (3*ERTS_PSFLGS_PRIO_BITS) + +#define ERTS_PSFLGS_QMASK_BITS 4 +#define ERTS_PSFLGS_QMASK \ + ((((erts_aint32_t) 1) << ERTS_PSFLGS_QMASK_BITS) - 1) +#define ERTS_PSFLGS_IN_PRQ_MASK_OFFSET \ + ERTS_PSFLGS_ZERO_BIT_OFFSET #define ERTS_PSFLG_BIT(N) \ - (((erts_aint32_t) 1) << (ERTS_PSFLG_PRIO_SHIFT + (N))) + (((erts_aint32_t) 1) << (ERTS_PSFLGS_ZERO_BIT_OFFSET + (N))) -#define ERTS_PSFLG_PRIO_MASK (ERTS_PSFLG_BIT(0) - 1) +/* + * ACT_PRIO -> Active prio, i.e., currently active prio. This + * prio may be higher than user prio. + * USR_PRIO -> User prio. i.e., prio the user has set. + * PRQ_PRIO -> Prio queue prio, i.e., prio queue currently + * enqueued in. + */ +#define ERTS_PSFLGS_ACT_PRIO_MASK \ + (ERTS_PSFLGS_PRIO_MASK << ERTS_PSFLGS_ACT_PRIO_OFFSET) +#define ERTS_PSFLGS_USR_PRIO_MASK \ + (ERTS_PSFLGS_PRIO_MASK << ERTS_PSFLGS_USR_PRIO_OFFSET) +#define ERTS_PSFLGS_PRQ_PRIO_MASK \ + (ERTS_PSFLGS_PRIO_MASK << ERTS_PSFLGS_PRQ_PRIO_OFFSET) +#define ERTS_PSFLG_IN_PRQ_MAX ERTS_PSFLG_BIT(0) +#define ERTS_PSFLG_IN_PRQ_HIGH ERTS_PSFLG_BIT(1) +#define ERTS_PSFLG_IN_PRQ_NORMAL ERTS_PSFLG_BIT(2) +#define ERTS_PSFLG_IN_PRQ_LOW ERTS_PSFLG_BIT(3) +#define ERTS_PSFLG_FREE ERTS_PSFLG_BIT(4) +#define ERTS_PSFLG_EXITING ERTS_PSFLG_BIT(5) +#define ERTS_PSFLG_PENDING_EXIT ERTS_PSFLG_BIT(6) +#define ERTS_PSFLG_ACTIVE ERTS_PSFLG_BIT(7) +#define ERTS_PSFLG_IN_RUNQ ERTS_PSFLG_BIT(8) +#define ERTS_PSFLG_RUNNING ERTS_PSFLG_BIT(9) +#define ERTS_PSFLG_SUSPENDED ERTS_PSFLG_BIT(10) +#define ERTS_PSFLG_GC ERTS_PSFLG_BIT(11) +#define ERTS_PSFLG_BOUND ERTS_PSFLG_BIT(12) +#define ERTS_PSFLG_TRAP_EXIT ERTS_PSFLG_BIT(13) +#define ERTS_PSFLG_ACTIVE_SYS ERTS_PSFLG_BIT(14) +#define ERTS_PSFLG_RUNNING_SYS ERTS_PSFLG_BIT(15) +#define ERTS_PSFLG_PROXY ERTS_PSFLG_BIT(16) +#define ERTS_PSFLG_DELAYED_SYS ERTS_PSFLG_BIT(17) +#ifdef ERTS_DIRTY_SCHEDULERS +#define ERTS_PSFLG_DIRTY_CPU_PROC ERTS_PSFLG_BIT(18) +#define ERTS_PSFLG_DIRTY_IO_PROC ERTS_PSFLG_BIT(19) +#define ERTS_PSFLG_DIRTY_CPU_PROC_IN_Q ERTS_PSFLG_BIT(20) +#define ERTS_PSFLG_DIRTY_IO_PROC_IN_Q ERTS_PSFLG_BIT(21) +#define ERTS_PSFLG_MAX (ERTS_PSFLGS_ZERO_BIT_OFFSET + 22) +#else +#define ERTS_PSFLG_MAX (ERTS_PSFLGS_ZERO_BIT_OFFSET + 18) +#endif -#define ERTS_PSFLG_FREE ERTS_PSFLG_BIT(0) -#define ERTS_PSFLG_EXITING ERTS_PSFLG_BIT(1) -#define ERTS_PSFLG_PENDING_EXIT ERTS_PSFLG_BIT(2) -#define ERTS_PSFLG_ACTIVE ERTS_PSFLG_BIT(3) -#define ERTS_PSFLG_IN_RUNQ ERTS_PSFLG_BIT(4) -#define ERTS_PSFLG_RUNNING ERTS_PSFLG_BIT(5) -#define ERTS_PSFLG_SUSPENDED ERTS_PSFLG_BIT(6) -#define ERTS_PSFLG_GC ERTS_PSFLG_BIT(7) -#define ERTS_PSFLG_BOUND ERTS_PSFLG_BIT(8) -#define ERTS_PSFLG_TRAP_EXIT ERTS_PSFLG_BIT(9) +#define ERTS_PSFLGS_IN_PRQ_MASK (ERTS_PSFLG_IN_PRQ_MAX \ + | ERTS_PSFLG_IN_PRQ_HIGH \ + | ERTS_PSFLG_IN_PRQ_NORMAL \ + | ERTS_PSFLG_IN_PRQ_LOW) +#define ERTS_PSFLGS_GET_ACT_PRIO(PSFLGS) \ + (((PSFLGS) >> ERTS_PSFLGS_ACT_PRIO_OFFSET) & ERTS_PSFLGS_PRIO_MASK) +#define ERTS_PSFLGS_GET_USR_PRIO(PSFLGS) \ + (((PSFLGS) >> ERTS_PSFLGS_USR_PRIO_OFFSET) & ERTS_PSFLGS_PRIO_MASK) +#define ERTS_PSFLGS_GET_PRQ_PRIO(PSFLGS) \ + (((PSFLGS) >> ERTS_PSFLGS_PRQ_PRIO_OFFSET) & ERTS_PSFLGS_PRIO_MASK) + +/* + * Static flags that do not change after process creation. + */ +#define ERTS_STC_FLG_SYSTEM_PROC (((Uint32) 1) << 0) /* The sequential tracing token is a tuple of size 5: * @@ -946,6 +1221,7 @@ void erts_check_for_holes(Process* p); #define SPO_LINK 1 #define SPO_USE_ARGS 2 #define SPO_MONITOR 4 +#define SPO_SYSTEM_PROC 8 /* * The following struct contains options for a process to be spawned. @@ -999,6 +1275,7 @@ extern erts_smp_rwmtx_t erts_cpu_bind_rwmtx; */ extern Eterm erts_system_monitor; extern Uint erts_system_monitor_long_gc; +extern Uint erts_system_monitor_long_schedule; extern Uint erts_system_monitor_large_heap; struct erts_system_monitor_flags_t { unsigned int busy_port : 1; @@ -1018,6 +1295,7 @@ struct erts_system_profile_flags_t { unsigned int exclusive : 1; }; extern struct erts_system_profile_flags_t erts_system_profile_flags; +extern int erts_system_profile_ts_type; /* process flags */ #define F_HIBERNATE_SCHED (1 << 0) /* Schedule out after hibernate op */ @@ -1031,64 +1309,92 @@ extern struct erts_system_profile_flags_t erts_system_profile_flags; #define F_HAVE_BLCKD_MSCHED (1 << 8) /* Process has blocked multi-scheduling */ #define F_P2PNR_RESCHED (1 << 9) /* Process has been rescheduled via erts_pid2proc_not_running() */ #define F_FORCE_GC (1 << 10) /* Force gc at process in-scheduling */ +#define F_DISABLE_GC (1 << 11) /* Disable GC */ + +#define ERTS_TRACE_FLAGS_TS_TYPE_SHIFT 0 + +#define F_TRACE_FLAG(N) (1 << (ERTS_TRACE_TS_TYPE_BITS + (N))) /* process trace_flags */ -#define F_SENSITIVE (1 << 0) -#define F_TRACE_SEND (1 << 1) -#define F_TRACE_RECEIVE (1 << 2) -#define F_TRACE_SOS (1 << 3) /* Set on spawn */ -#define F_TRACE_SOS1 (1 << 4) /* Set on first spawn */ -#define F_TRACE_SOL (1 << 5) /* Set on link */ -#define F_TRACE_SOL1 (1 << 6) /* Set on first link */ -#define F_TRACE_CALLS (1 << 7) -#define F_TIMESTAMP (1 << 8) -#define F_TRACE_PROCS (1 << 9) -#define F_TRACE_FIRST_CHILD (1 << 10) -#define F_TRACE_SCHED (1 << 11) -#define F_TRACE_GC (1 << 12) -#define F_TRACE_ARITY_ONLY (1 << 13) -#define F_TRACE_RETURN_TO (1 << 14) /* Return_to trace when breakpoint tracing */ -#define F_TRACE_SILENT (1 << 15) /* No call trace msg suppress */ -#define F_TRACER (1 << 16) /* May be (has been) tracer */ -#define F_EXCEPTION_TRACE (1 << 17) /* May have exception trace on stack */ + +#define F_NOW_TS (ERTS_TRACE_FLG_NOW_TIMESTAMP \ + << ERTS_TRACE_FLAGS_TS_TYPE_SHIFT) +#define F_STRICT_MON_TS (ERTS_TRACE_FLG_STRICT_MONOTONIC_TIMESTAMP \ + << ERTS_TRACE_FLAGS_TS_TYPE_SHIFT) +#define F_MON_TS (ERTS_TRACE_FLG_MONOTONIC_TIMESTAMP \ + << ERTS_TRACE_FLAGS_TS_TYPE_SHIFT) +#define F_SENSITIVE F_TRACE_FLAG(0) +#define F_TRACE_SEND F_TRACE_FLAG(1) +#define F_TRACE_RECEIVE F_TRACE_FLAG(2) +#define F_TRACE_SOS F_TRACE_FLAG(3) /* Set on spawn */ +#define F_TRACE_SOS1 F_TRACE_FLAG(4) /* Set on first spawn */ +#define F_TRACE_SOL F_TRACE_FLAG(5) /* Set on link */ +#define F_TRACE_SOL1 F_TRACE_FLAG(6) /* Set on first link */ +#define F_TRACE_CALLS F_TRACE_FLAG(7) +#define F_TRACE_PROCS F_TRACE_FLAG(8) +#define F_TRACE_FIRST_CHILD F_TRACE_FLAG(9) +#define F_TRACE_SCHED F_TRACE_FLAG(10) +#define F_TRACE_GC F_TRACE_FLAG(11) +#define F_TRACE_ARITY_ONLY F_TRACE_FLAG(12) +#define F_TRACE_RETURN_TO F_TRACE_FLAG(13) /* Return_to trace when breakpoint tracing */ +#define F_TRACE_SILENT F_TRACE_FLAG(14) /* No call trace msg suppress */ +#define F_TRACER F_TRACE_FLAG(15) /* May be (has been) tracer */ +#define F_EXCEPTION_TRACE F_TRACE_FLAG(16) /* May have exception trace on stack */ /* port trace flags, currently the same as process trace flags */ -#define F_TRACE_SCHED_PORTS (1 << 18) /* Trace of port scheduling */ -#define F_TRACE_SCHED_PROCS (1 << 19) /* With virtual scheduling */ -#define F_TRACE_PORTS (1 << 20) /* Ports equivalent to F_TRACE_PROCS */ -#define F_TRACE_SCHED_NO (1 << 21) /* Trace with scheduler id */ -#define F_TRACE_SCHED_EXIT (1 << 22) +#define F_TRACE_SCHED_PORTS F_TRACE_FLAG(17) /* Trace of port scheduling */ +#define F_TRACE_SCHED_PROCS F_TRACE_FLAG(18) /* With virtual scheduling */ +#define F_TRACE_PORTS F_TRACE_FLAG(19) /* Ports equivalent to F_TRACE_PROCS */ +#define F_TRACE_SCHED_NO F_TRACE_FLAG(20) /* Trace with scheduler id */ +#define F_TRACE_SCHED_EXIT F_TRACE_FLAG(21) -#define F_NUM_FLAGS 23 +#define F_NUM_FLAGS (ERTS_TRACE_TS_TYPE_BITS + 22) #ifdef DEBUG # define F_INITIAL_TRACE_FLAGS (5 << F_NUM_FLAGS) #else # define F_INITIAL_TRACE_FLAGS 0 #endif - +/* F_TIMESTAMP_MASK is a bit-field of all all timestamp types */ +#define F_TIMESTAMP_MASK \ + (ERTS_TRACE_TS_TYPE_MASK << ERTS_TRACE_FLAGS_TS_TYPE_SHIFT) #define TRACEE_FLAGS ( F_TRACE_PROCS | F_TRACE_CALLS \ | F_TRACE_SOS | F_TRACE_SOS1| F_TRACE_RECEIVE \ | F_TRACE_SOL | F_TRACE_SOL1 | F_TRACE_SEND \ - | F_TRACE_SCHED | F_TIMESTAMP | F_TRACE_GC \ + | F_TRACE_SCHED | F_TIMESTAMP_MASK | F_TRACE_GC \ | F_TRACE_ARITY_ONLY | F_TRACE_RETURN_TO \ | F_TRACE_SILENT | F_TRACE_SCHED_PROCS | F_TRACE_PORTS \ | F_TRACE_SCHED_PORTS | F_TRACE_SCHED_NO \ | F_TRACE_SCHED_EXIT) #define ERTS_TRACEE_MODIFIER_FLAGS \ - (F_TRACE_SILENT | F_TIMESTAMP | F_TRACE_SCHED_NO) + (F_TRACE_SILENT | F_TIMESTAMP_MASK | F_TRACE_SCHED_NO) #define ERTS_PORT_TRACEE_FLAGS \ (ERTS_TRACEE_MODIFIER_FLAGS | F_TRACE_PORTS | F_TRACE_SCHED_PORTS) #define ERTS_PROC_TRACEE_FLAGS \ ((TRACEE_FLAGS & ~ERTS_PORT_TRACEE_FLAGS) | ERTS_TRACEE_MODIFIER_FLAGS) +#define SEQ_TRACE_FLAG(N) (1 << (ERTS_TRACE_TS_TYPE_BITS + (N))) + /* Sequential trace flags */ + +/* SEQ_TRACE_TIMESTAMP_MASK is a bit-field */ +#define SEQ_TRACE_TIMESTAMP_MASK \ + (ERTS_TRACE_TS_TYPE_MASK << ERTS_SEQ_TRACE_FLAGS_TS_TYPE_SHIFT) + #define SEQ_TRACE_SEND (1 << 0) #define SEQ_TRACE_RECEIVE (1 << 1) #define SEQ_TRACE_PRINT (1 << 2) -#define SEQ_TRACE_TIMESTAMP (1 << 3) + +#define ERTS_SEQ_TRACE_FLAGS_TS_TYPE_SHIFT 3 + +#define SEQ_TRACE_NOW_TS (ERTS_TRACE_FLG_NOW_TIMESTAMP \ + << ERTS_SEQ_TRACE_FLAGS_TS_TYPE_SHIFT) +#define SEQ_TRACE_STRICT_MON_TS (ERTS_TRACE_FLG_STRICT_MONOTONIC_TIMESTAMP \ + << ERTS_SEQ_TRACE_FLAGS_TS_TYPE_SHIFT) +#define SEQ_TRACE_MON_TS (ERTS_TRACE_FLG_MONOTONIC_TIMESTAMP \ + << ERTS_SEQ_TRACE_FLAGS_TS_TYPE_SHIFT) #ifdef USE_VM_PROBES #define DT_UTAG_PERMANENT (1 << 0) @@ -1101,31 +1407,91 @@ extern struct erts_system_profile_flags_t erts_system_profile_flags; #define ERTS_XSIG_FLG_IGN_KILL (((Uint32) 1) << 0) #define ERTS_XSIG_FLG_NO_IGN_NORMAL (((Uint32) 1) << 1) -#define CANCEL_TIMER(p) \ - do { \ - if ((p)->flags & (F_INSLPQUEUE)) \ - cancel_timer(p); \ - else \ - (p)->flags &= ~F_TIMO; \ +#define CANCEL_TIMER(P) \ + do { \ + if ((P)->flags & (F_INSLPQUEUE|F_TIMO)) { \ + if ((P)->flags & F_INSLPQUEUE) \ + erts_cancel_proc_timer((P)); \ + else \ + (P)->flags &= ~F_TIMO; \ + } \ } while (0) +#if defined(ERTS_DIRTY_SCHEDULERS) && defined(ERTS_SMP) +#define ERTS_NUM_DIRTY_RUNQS 2 +#else +#define ERTS_NUM_DIRTY_RUNQS 0 +#endif + #define ERTS_RUNQ_IX(IX) \ - (ASSERT_EXPR(0 <= (IX) && (IX) < erts_no_run_queues), \ + (ASSERT(0 <= (IX) && (IX) < erts_no_run_queues), \ &erts_aligned_run_queues[(IX)].runq) +#ifdef ERTS_DIRTY_SCHEDULERS +#define ERTS_RUNQ_IX_IS_DIRTY(IX) \ + (-(ERTS_NUM_DIRTY_RUNQS) <= (IX) && (IX) < 0) +#define ERTS_DIRTY_RUNQ_IX(IX) \ + (ASSERT(ERTS_RUNQ_IX_IS_DIRTY(IX)), \ + &erts_aligned_run_queues[(IX)].runq) +#define ERTS_DIRTY_CPU_RUNQ (&erts_aligned_run_queues[-1].runq) +#define ERTS_DIRTY_IO_RUNQ (&erts_aligned_run_queues[-2].runq) +#define ERTS_RUNQ_IS_DIRTY_CPU_RUNQ(RQ) ((RQ)->ix == -1) +#define ERTS_RUNQ_IS_DIRTY_IO_RUNQ(RQ) ((RQ)->ix == -2) +#else +#define ERTS_RUNQ_IX_IS_DIRTY(IX) 0 +#endif #define ERTS_SCHEDULER_IX(IX) \ - (ASSERT_EXPR(0 <= (IX) && (IX) < erts_no_schedulers), \ + (ASSERT(0 <= (IX) && (IX) < erts_no_schedulers), \ &erts_aligned_scheduler_data[(IX)].esd) +#ifdef ERTS_DIRTY_SCHEDULERS +#define ERTS_DIRTY_CPU_SCHEDULER_IX(IX) \ + (ASSERT(0 <= (IX) && (IX) < erts_no_dirty_cpu_schedulers), \ + &erts_aligned_dirty_cpu_scheduler_data[(IX)].esd) +#define ERTS_DIRTY_IO_SCHEDULER_IX(IX) \ + (ASSERT(0 <= (IX) && (IX) < erts_no_dirty_io_schedulers), \ + &erts_aligned_dirty_io_scheduler_data[(IX)].esd) +#define ERTS_DIRTY_SCHEDULER_NO(ESDP) \ + ((ESDP)->dirty_no.s.num) +#define ERTS_DIRTY_SCHEDULER_TYPE(ESDP) \ + ((ESDP)->dirty_no.s.type) +#ifdef ERTS_SMP +#define ERTS_SCHEDULER_IS_DIRTY(ESDP) \ + ((ESDP)->dirty_no.s.num != 0) +#define ERTS_SCHEDULER_IS_DIRTY_CPU(ESDP) \ + ((ESDP)->dirty_no.s.type == 0) +#define ERTS_SCHEDULER_IS_DIRTY_IO(ESDP) \ + ((ESDP)->dirty_no.s.type == 1) +#else +#define ERTS_SCHEDULER_IS_DIRTY(ESDP) 0 +#define ERTS_SCHEDULER_IS_DIRTY_CPU(ESDP) 0 +#define ERTS_SCHEDULER_IS_DIRTY_IO(ESDP) 0 +#endif +#else +#define ERTS_RUNQ_IX_IS_DIRTY(IX) 0 +#define ERTS_SCHEDULER_IS_DIRTY(ESDP) 0 +#define ERTS_SCHEDULER_IS_DIRTY_CPU(ESDP) 0 +#define ERTS_SCHEDULER_IS_DIRTY_IO(ESDP) 0 +#endif void erts_pre_init_process(void); void erts_late_init_process(void); void erts_early_init_scheduling(int); -void erts_init_scheduling(int, int); +void erts_init_scheduling(int, int +#ifdef ERTS_DIRTY_SCHEDULERS + , int, int, int +#endif + ); +int erts_set_gc_state(Process *c_p, int enable); Eterm erts_sched_wall_time_request(Process *c_p, int set, int enable); +Eterm erts_system_check_request(Process *c_p); +Eterm erts_gc_info_request(Process *c_p); Uint64 erts_get_proc_interval(void); Uint64 erts_ensure_later_proc_interval(Uint64); Uint64 erts_step_proc_interval(void); +int erts_setup_nif_gc(Process* proc, Eterm** objv, int* nobj); /* see erl_nif.c */ +void erts_destroy_nif_export(void *); /* see erl_nif.c */ + ErtsProcList *erts_proclist_create(Process *); void erts_proclist_destroy(ErtsProcList *); @@ -1298,10 +1664,15 @@ ERTS_GLB_INLINE int erts_proclist_is_last(ErtsProcList *list, int erts_sched_set_wakeup_other_thresold(char *str); int erts_sched_set_wakeup_other_type(char *str); int erts_sched_set_busy_wait_threshold(char *str); +int erts_sched_set_wake_cleanup_threshold(char *); void erts_schedule_thr_prgr_later_op(void (*)(void *), void *, ErtsThrPrgrLaterOp *); +void erts_schedule_thr_prgr_later_cleanup_op(void (*)(void *), + void *, + ErtsThrPrgrLaterOp *, + UWord); #if defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_CHECK) int erts_dbg_check_halloc_lock(Process *p); @@ -1310,20 +1681,30 @@ int erts_dbg_check_halloc_lock(Process *p); void erts_dbg_multi_scheduling_return_trap(Process *, Eterm); #endif int erts_get_max_no_executing_schedulers(void); -#ifdef ERTS_SMP +#if defined(ERTS_SMP) || defined(ERTS_DIRTY_SCHEDULERS) ErtsSchedSuspendResult -erts_schedulers_state(Uint *, Uint *, Uint *, int); +erts_schedulers_state(Uint *, Uint *, Uint *, Uint *, Uint *, Uint *, int); +#endif +#ifdef ERTS_SMP ErtsSchedSuspendResult erts_set_schedulers_online(Process *p, ErtsProcLocks plocks, Sint new_no, - Sint *old_no); + Sint *old_no +#ifdef ERTS_DIRTY_SCHEDULERS + , int dirty_only +#endif + ); ErtsSchedSuspendResult erts_block_multi_scheduling(Process *, ErtsProcLocks, int, int); int erts_is_multi_scheduling_blocked(void); Eterm erts_multi_scheduling_blockers(Process *); void erts_start_schedulers(void); void erts_alloc_notify_delayed_dealloc(int); +void erts_alloc_ensure_handle_delayed_dealloc_call(int); +#ifdef ERTS_SMP +void erts_notify_canceled_timer(ErtsSchedulerData *, int); +#endif void erts_smp_notify_check_children_needed(void); #endif #if ERTS_USE_ASYNC_READY_Q @@ -1343,9 +1724,9 @@ void erts_schedule_multi_misc_aux_work(int ignore_self, erts_aint32_t erts_set_aux_work_timeout(int, erts_aint32_t, int); void erts_sched_notify_check_cpu_bind(void); Uint erts_active_schedulers(void); -void erts_init_process(int, int); +void erts_init_process(int, int, int); Eterm erts_process_status(Process *, ErtsProcLocks, Process *, Eterm); -Uint erts_run_queues_len(Uint *); +Uint erts_run_queues_len(Uint *, int, int); void erts_add_to_runq(Process *); Eterm erts_bound_schedulers_term(Process *c_p); Eterm erts_get_cpu_topology_term(Process *c_p, Eterm which); @@ -1358,8 +1739,6 @@ void erts_schedule_misc_op(void (*)(void *), void *); Eterm erl_create_process(Process*, Eterm, Eterm, Eterm, ErlSpawnOpts*); void erts_do_exit_process(Process*, Eterm); void erts_continue_exit_process(Process *); -void set_timer(Process*, Uint); -void cancel_timer(Process*); /* Begin System profile */ Uint erts_runnable_process_count(void); /* End System profile */ @@ -1369,7 +1748,11 @@ void erts_cleanup_empty_process(Process* p); void erts_debug_verify_clean_empty_process(Process* p); #endif void erts_stack_dump(int to, void *to_arg, Process *); +void erts_limited_stack_trace(int to, void *to_arg, Process *); void erts_program_counter_info(int to, void *to_arg, Process *); +void erts_print_scheduler_info(int to, void *to_arg, ErtsSchedulerData *esdp); +void erts_dump_extended_process_state(int to, void *to_arg, erts_aint32_t psflg); +void erts_dump_process_state(int to, void *to_arg, erts_aint32_t psflg); Eterm erts_get_process_priority(Process *p); Eterm erts_set_process_priority(Process *p, Eterm prio); @@ -1411,7 +1794,13 @@ Eterm erts_get_reader_groups_map(Process *c_p); Eterm erts_debug_reader_groups_map(Process *c_p, int groups); Uint erts_debug_nbalance(void); -int erts_debug_wait_deallocations(Process *c_p); + +#define ERTS_DEBUG_WAIT_COMPLETED_DEALLOCATIONS (1 << 0) +#define ERTS_DEBUG_WAIT_COMPLETED_TIMER_CANCELLATIONS (1 << 1) + +int erts_debug_wait_completed(Process *c_p, int flags); + +Uint erts_process_memory(Process *c_p); #ifdef ERTS_SMP # define ERTS_GET_SCHEDULER_DATA_FROM_PROC(PROC) ((PROC)->scheduler_data) @@ -1427,7 +1816,7 @@ do { \ ErtsSchedulerData *esdp__ = ((P) \ ? ERTS_PROC_GET_SCHDATA((Process *) (P)) \ : erts_get_scheduler_data()); \ - if (esdp__) \ + if (esdp__ && !ERTS_SCHEDULER_IS_DIRTY(esdp__)) \ esdp__->verify_unused_temp_alloc( \ esdp__->verify_unused_temp_alloc_data); \ } while (0) @@ -1449,17 +1838,17 @@ ErtsSchedulerData *erts_get_scheduler_data(void) #endif #endif -void erts_schedule_process(Process *, erts_aint32_t); +void erts_schedule_process(Process *, erts_aint32_t, ErtsProcLocks); -ERTS_GLB_INLINE void erts_proc_notify_new_message(Process *p); +ERTS_GLB_INLINE void erts_proc_notify_new_message(Process *p, ErtsProcLocks locks); #if ERTS_GLB_INLINE_INCL_FUNC_DEF ERTS_GLB_INLINE void -erts_proc_notify_new_message(Process *p) +erts_proc_notify_new_message(Process *p, ErtsProcLocks locks) { /* No barrier needed, due to msg lock */ erts_aint32_t state = erts_smp_atomic32_read_nob(&p->state); if (!(state & ERTS_PSFLG_ACTIVE)) - erts_schedule_process(p, state); + erts_schedule_process(p, state, locks); } #endif @@ -1542,11 +1931,6 @@ erts_psd_set(Process *p, ErtsProcLocks plocks, int ix, void *data) #define ERTS_PROC_SCHED_ID(P, L, ID) \ ((UWord) erts_psd_set((P), (L), ERTS_PSD_SCHED_ID, (void *) (ID))) -#define ERTS_PROC_GET_DIST_ENTRY(P) \ - ((DistEntry *) erts_psd_get((P), ERTS_PSD_DIST_ENTRY)) -#define ERTS_PROC_SET_DIST_ENTRY(P, L, D) \ - ((DistEntry *) erts_psd_set((P), (L), ERTS_PSD_DIST_ENTRY, (void *) (D))) - #define ERTS_PROC_GET_SAVED_CALLS_BUF(P) \ ((struct saved_calls *) erts_psd_get((P), ERTS_PSD_SAVED_CALLS_BUF)) #define ERTS_PROC_SET_SAVED_CALLS_BUF(P, L, SCB) \ @@ -1557,6 +1941,16 @@ erts_psd_set(Process *p, ErtsProcLocks plocks, int ix, void *data) #define ERTS_PROC_SET_CALL_TIME(P, L, PBT) \ ((process_breakpoint_time_t *) erts_psd_set((P), (L), ERTS_PSD_CALL_TIME_BP, (void *) (PBT))) +#define ERTS_PROC_GET_DELAYED_GC_TASK_QS(P) \ + ((ErtsProcSysTaskQs *) erts_psd_get((P), ERTS_PSD_DELAYED_GC_TASK_QS)) +#define ERTS_PROC_SET_DELAYED_GC_TASK_QS(P, L, PBT) \ + ((ErtsProcSysTaskQs *) erts_psd_set((P), (L), ERTS_PSD_DELAYED_GC_TASK_QS, (void *) (PBT))) + +#define ERTS_PROC_GET_NIF_TRAP_EXPORT(P) \ + erts_psd_get((P), ERTS_PSD_NIF_TRAP_EXPORT) +#define ERTS_PROC_SET_NIF_TRAP_EXPORT(P, L, NTE) \ + erts_psd_set((P), (L), ERTS_PSD_NIF_TRAP_EXPORT, (void *) (NTE)) + ERTS_GLB_INLINE Eterm erts_proc_get_error_handler(Process *p); ERTS_GLB_INLINE Eterm erts_proc_set_error_handler(Process *p, @@ -1602,6 +1996,13 @@ erts_proc_set_error_handler(Process *p, ErtsProcLocks plocks, Eterm handler) extern erts_atomic_t erts_migration_paths; +#if ERTS_HAVE_SCHED_UTIL_BALANCING_SUPPORT +int erts_get_sched_util(ErtsRunQueue *rq, + int initially_locked, + int short_interval); +#endif + + ERTS_GLB_INLINE ErtsMigrationPaths *erts_get_migration_paths_managed(void); ERTS_GLB_INLINE ErtsMigrationPaths *erts_get_migration_paths(void); ERTS_GLB_INLINE ErtsRunQueue *erts_check_emigration_need(ErtsRunQueue *c_rq, @@ -1653,22 +2054,36 @@ erts_check_emigration_need(ErtsRunQueue *c_rq, int prio) return mp->prio[prio].runq; } - - if (prio == ERTS_PORT_PRIO_LEVEL) - len = RUNQ_READ_LEN(&c_rq->ports.info.len); +#if ERTS_HAVE_SCHED_UTIL_BALANCING_SUPPORT + if (mp->sched_util) { + ErtsRunQueue *rq = mp->prio[prio].runq; + /* No migration if other is non-empty */ + if (!(ERTS_RUNQ_FLGS_GET(rq) & ERTS_RUNQ_FLG_NONEMPTY) + && erts_get_sched_util(rq, 0, 1) < mp->prio[prio].limit.other + && erts_get_sched_util(c_rq, 0, 1) > mp->prio[prio].limit.this) { + return rq; + } + } else - len = RUNQ_READ_LEN(&c_rq->procs.prio_info[prio].len); - - if (len > mp->prio[prio].limit.this) { - ErtsRunQueue *n_rq = mp->prio[prio].runq; - if (n_rq) { - if (prio == ERTS_PORT_PRIO_LEVEL) - len = RUNQ_READ_LEN(&n_rq->ports.info.len); - else - len = RUNQ_READ_LEN(&n_rq->procs.prio_info[prio].len); - - if (len < mp->prio[prio].limit.other) - return n_rq; +#endif + { + + if (prio == ERTS_PORT_PRIO_LEVEL) + len = RUNQ_READ_LEN(&c_rq->ports.info.len); + else + len = RUNQ_READ_LEN(&c_rq->procs.prio_info[prio].len); + + if (len > mp->prio[prio].limit.this) { + ErtsRunQueue *n_rq = mp->prio[prio].runq; + if (n_rq) { + if (prio == ERTS_PORT_PRIO_LEVEL) + len = RUNQ_READ_LEN(&n_rq->ports.info.len); + else + len = RUNQ_READ_LEN(&n_rq->procs.prio_info[prio].len); + + if (len < mp->prio[prio].limit.other) + return n_rq; + } } } } @@ -1687,14 +2102,10 @@ ERTS_GLB_INLINE Eterm erts_get_current_pid(void); ERTS_GLB_INLINE Uint erts_get_scheduler_id(void); ERTS_GLB_INLINE ErtsRunQueue *erts_get_runq_proc(Process *p); ERTS_GLB_INLINE ErtsRunQueue *erts_get_runq_current(ErtsSchedulerData *esdp); -#ifndef ERTS_ENABLE_LOCK_COUNT ERTS_GLB_INLINE void erts_smp_runq_lock(ErtsRunQueue *rq); -#endif ERTS_GLB_INLINE int erts_smp_runq_trylock(ErtsRunQueue *rq); ERTS_GLB_INLINE void erts_smp_runq_unlock(ErtsRunQueue *rq); -#ifndef ERTS_ENABLE_LOCK_COUNT ERTS_GLB_INLINE void erts_smp_xrunq_lock(ErtsRunQueue *rq, ErtsRunQueue *xrq); -#endif ERTS_GLB_INLINE void erts_smp_xrunq_unlock(ErtsRunQueue *rq, ErtsRunQueue *xrq); ERTS_GLB_INLINE void erts_smp_runqs_lock(ErtsRunQueue *rq1, ErtsRunQueue *rq2); ERTS_GLB_INLINE void erts_smp_runqs_unlock(ErtsRunQueue *rq1, ErtsRunQueue *rq2); @@ -1729,7 +2140,12 @@ Uint erts_get_scheduler_id(void) { #ifdef ERTS_SMP ErtsSchedulerData *esdp = erts_get_scheduler_data(); - return esdp ? esdp->no : (Uint) 0; +#ifdef ERTS_DIRTY_SCHEDULERS + if (esdp && ERTS_SCHEDULER_IS_DIRTY(esdp)) + return 0; + else +#endif + return esdp ? esdp->no : (Uint) 0; #else return erts_get_scheduler_data() ? (Uint) 1 : (Uint) 0; #endif @@ -1759,12 +2175,6 @@ erts_get_runq_current(ErtsSchedulerData *esdp) #endif } -#ifdef ERTS_ENABLE_LOCK_COUNT - -#define erts_smp_runq_lock(rq) erts_smp_mtx_lock_x(&(rq)->mtx, __FILE__, __LINE__) - -#else - ERTS_GLB_INLINE void erts_smp_runq_lock(ErtsRunQueue *rq) { @@ -1773,8 +2183,6 @@ erts_smp_runq_lock(ErtsRunQueue *rq) #endif } -#endif - ERTS_GLB_INLINE int erts_smp_runq_trylock(ErtsRunQueue *rq) { @@ -1793,31 +2201,6 @@ erts_smp_runq_unlock(ErtsRunQueue *rq) #endif } -#ifdef ERTS_ENABLE_LOCK_COUNT - -#define erts_smp_xrunq_lock(rq, xrq) erts_smp_xrunq_lock_x((rq), (xrq), __FILE__, __LINE__) - -ERTS_GLB_INLINE void -erts_smp_xrunq_lock_x(ErtsRunQueue *rq, ErtsRunQueue *xrq, char* file, int line) -{ -#ifdef ERTS_SMP - ERTS_SMP_LC_ASSERT(erts_smp_lc_mtx_is_locked(&rq->mtx)); - if (xrq != rq) { - if (erts_smp_mtx_trylock(&xrq->mtx) == EBUSY) { - if (rq < xrq) - erts_smp_mtx_lock_x(&xrq->mtx, file, line); - else { - erts_smp_mtx_unlock(&rq->mtx); - erts_smp_mtx_lock_x(&xrq->mtx, file, line); - erts_smp_mtx_lock_x(&rq->mtx, file, line); - } - } - } -#endif -} - -#else - ERTS_GLB_INLINE void erts_smp_xrunq_lock(ErtsRunQueue *rq, ErtsRunQueue *xrq) { @@ -1837,8 +2220,6 @@ erts_smp_xrunq_lock(ErtsRunQueue *rq, ErtsRunQueue *xrq) #endif } -#endif - ERTS_GLB_INLINE void erts_smp_xrunq_unlock(ErtsRunQueue *rq, ErtsRunQueue *xrq) { @@ -1938,6 +2319,8 @@ extern int erts_disable_proc_not_running_opt; void erts_smp_notify_inc_runq(ErtsRunQueue *runq); +void erts_interupt_aux_thread_timed(ErtsMonotonicTime timeout_time); + #ifdef ERTS_SMP void erts_sched_finish_poke(ErtsSchedulerSleepInfo *, erts_aint32_t); ERTS_GLB_INLINE void erts_sched_poke(ErtsSchedulerSleepInfo *ssi); @@ -1956,6 +2339,7 @@ erts_sched_poke(ErtsSchedulerSleepInfo *ssi) } } + #endif /* #if ERTS_GLB_INLINE_INCL_FUNC_DEF */ #endif /* #ifdef ERTS_SMP */ @@ -1967,6 +2351,6 @@ erts_sched_poke(ErtsSchedulerSleepInfo *ssi) #endif -void erl_halt(int code); +void erts_halt(int code); extern erts_smp_atomic32_t erts_halt_progress; extern int erts_halt_code; diff --git a/erts/emulator/beam/erl_process_dict.c b/erts/emulator/beam/erl_process_dict.c index a611b52af2..3253ee6b70 100644 --- a/erts/emulator/beam/erl_process_dict.c +++ b/erts/emulator/beam/erl_process_dict.c @@ -3,16 +3,17 @@ * * Copyright Ericsson AB 1999-2013. All Rights Reserved. * - * The contents of this file are subject to the Erlang Public License, - * Version 1.1, (the "License"); you may not use this file except in - * compliance with the License. You should have received a copy of the - * Erlang Public License along with this software. If not, it can be - * retrieved online at http://www.erlang.org/. - * - * Software distributed under the License is distributed on an "AS IS" - * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See - * the License for the specific language governing rights and limitations - * under the License. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. * * %CopyrightEnd% */ @@ -47,7 +48,7 @@ /* Hash constant macros */ #define MAX_HASH 1342177280UL -#define INITIAL_SIZE 10 +#define INITIAL_SIZE (erts_pd_initial_size) /* Hash utility macros */ #define HASH_RANGE(PDict) ((PDict)->homeSize + (PDict)->splitPosition) @@ -82,6 +83,7 @@ static void pd_hash_erase(Process *p, Eterm id, Eterm *ret); static void pd_hash_erase_all(Process *p); static Eterm pd_hash_get_keys(Process *p, Eterm value); +static Eterm pd_hash_get_all_keys(Process *p, ProcDict *pd); static Eterm pd_hash_get_all(Process *p, ProcDict *pd); static Eterm pd_hash_put(Process *p, Eterm id, Eterm value); @@ -275,6 +277,16 @@ BIF_RETTYPE get_1(BIF_ALIST_1) BIF_RET(ret); } +BIF_RETTYPE get_keys_0(BIF_ALIST_0) +{ + Eterm ret; + + PD_CHECK(BIF_P->dictionary); + ret = pd_hash_get_all_keys(BIF_P,BIF_P->dictionary); + PD_CHECK(BIF_P->dictionary); + BIF_RET(ret); +} + BIF_RETTYPE get_keys_1(BIF_ALIST_1) { Eterm ret; @@ -362,7 +374,7 @@ static void pd_hash_erase(Process *p, Eterm id, Eterm *ret) "display term found in line %d:\n" "%T\n", p->common.id, __LINE__, old); #endif - erl_exit(1, "Damaged process dictionary found during erase/1."); + erts_exit(ERTS_ERROR_EXIT, "Damaged process dictionary found during erase/1."); } if ((range = HASH_RANGE(p->dictionary)) > INITIAL_SIZE && range / 2 > (p->dictionary->numElements)) { @@ -407,11 +419,52 @@ Eterm erts_pd_hash_get(Process *p, Eterm id) "display term found in line %d:\n" "%T\n", p->common.id, __LINE__, tmp); #endif - erl_exit(1, "Damaged process dictionary found during get/1."); + erts_exit(ERTS_ERROR_EXIT, "Damaged process dictionary found during get/1."); } return am_undefined; } +#define PD_GET_TKEY(Dst,Src) \ +do { \ + ASSERT(is_tuple((Src))); \ + ASSERT(arityval(*((Eterm*)tuple_val((Src)))) == 2); \ + (Dst) = ((Eterm*)tuple_val((Src)))[1]; \ +} while(0) + +static Eterm pd_hash_get_all_keys(Process *p, ProcDict *pd) { + Eterm* hp; + Eterm res = NIL; + Eterm tmp, tmp2; + unsigned int i; + unsigned int num; + + if (pd == NULL) { + return res; + } + + num = HASH_RANGE(pd); + hp = HAlloc(p, pd->numElements * 2); + + for (i = 0; i < num; ++i) { + tmp = ARRAY_GET(pd, i); + if (is_boxed(tmp)) { + PD_GET_TKEY(tmp,tmp); + res = CONS(hp, tmp, res); + hp += 2; + } else if (is_list(tmp)) { + while (tmp != NIL) { + tmp2 = TCAR(tmp); + PD_GET_TKEY(tmp2,tmp2); + res = CONS(hp, tmp2, res); + hp += 2; + tmp = TCDR(tmp); + } + } + } + return res; +} +#undef PD_GET_TKEY + static Eterm pd_hash_get_keys(Process *p, Eterm value) { Eterm *hp; @@ -617,7 +670,7 @@ static Eterm pd_hash_put(Process *p, Eterm id, Eterm value) "%T\n", p->common.id, __LINE__, old); #endif - erl_exit(1, "Damaged process dictionary found during put/2."); + erts_exit(ERTS_ERROR_EXIT, "Damaged process dictionary found during put/2."); } if (HASH_RANGE(p->dictionary) <= p->dictionary->numElements) { grow(p); @@ -659,7 +712,7 @@ static void shrink(Process *p, Eterm* ret) } else { int needed = 4; if (is_list(hi) && is_list(lo)) { - needed = 2*list_length(hi); + needed = 2*erts_list_length(hi); } if (HeapWordsLeft(p) < needed) { BUMP_REDS(p, erts_garbage_collect(p, needed, ret, 1)); @@ -971,7 +1024,7 @@ static void pd_check(ProcDict *pd) } continue; } else { - erl_exit(1, + erts_exit(ERTS_ERROR_EXIT, "Found tag 0x%08x in process dictionary at position %d", (unsigned long) t, (int) i); } diff --git a/erts/emulator/beam/erl_process_dict.h b/erts/emulator/beam/erl_process_dict.h index 8fad2a67ab..cc53800eb5 100644 --- a/erts/emulator/beam/erl_process_dict.h +++ b/erts/emulator/beam/erl_process_dict.h @@ -3,16 +3,17 @@ * * Copyright Ericsson AB 1999-2009. All Rights Reserved. * - * The contents of this file are subject to the Erlang Public License, - * Version 1.1, (the "License"); you may not use this file except in - * compliance with the License. You should have received a copy of the - * Erlang Public License along with this software. If not, it can be - * retrieved online at http://www.erlang.org/. - * - * Software distributed under the License is distributed on an "AS IS" - * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See - * the License for the specific language governing rights and limitations - * under the License. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. * * %CopyrightEnd% */ diff --git a/erts/emulator/beam/erl_process_dump.c b/erts/emulator/beam/erl_process_dump.c index a93229c473..25f0b1ed38 100644 --- a/erts/emulator/beam/erl_process_dump.c +++ b/erts/emulator/beam/erl_process_dump.c @@ -3,16 +3,17 @@ * * Copyright Ericsson AB 2003-2013. All Rights Reserved. * - * The contents of this file are subject to the Erlang Public License, - * Version 1.1, (the "License"); you may not use this file except in - * compliance with the License. You should have received a copy of the - * Erlang Public License along with this software. If not, it can be - * retrieved online at http://www.erlang.org/. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Software distributed under the License is distributed on an "AS IS" - * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See - * the License for the specific language governing rights and limitations - * under the License. + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. * * %CopyrightEnd% */ @@ -34,8 +35,8 @@ #define ERTS_WANT_EXTERNAL_TAGS #include "external.h" -#define WORD_FMT "%X" -#define ADDR_FMT "%X" +#define PTR_FMT "%bpX" +#define ETERM_FMT "%beX" #define OUR_NIL _make_header(0,_TAG_HEADER_FLOAT) @@ -43,8 +44,9 @@ static void dump_process_info(int to, void *to_arg, Process *p); static void dump_element(int to, void *to_arg, Eterm x); static void dump_dist_ext(int to, void *to_arg, ErtsDistExternal *edep); static void dump_element_nl(int to, void *to_arg, Eterm x); -static int stack_element_dump(int to, void *to_arg, Process* p, Eterm* sp, +static int stack_element_dump(int to, void *to_arg, Eterm* sp, int yreg); +static void stack_trace_dump(int to, void *to_arg, Eterm* sp); static void print_function_from_pc(int to, void *to_arg, BeamInstr* x); static void heap_dump(int to, void *to_arg, Eterm x); static void dump_binaries(int to, void *to_arg, Binary* root); @@ -76,6 +78,43 @@ erts_deep_process_dump(int to, void *to_arg) dump_binaries(to, to_arg, all_binaries); } +Uint erts_process_memory(Process *p) { + ErlMessage *mp; + Uint size = 0; + struct saved_calls *scb; + size += sizeof(Process); + + ERTS_SMP_MSGQ_MV_INQ2PRIVQ(p); + + erts_doforall_links(ERTS_P_LINKS(p), &erts_one_link_size, &size); + erts_doforall_monitors(ERTS_P_MONITORS(p), &erts_one_mon_size, &size); + size += (p->heap_sz + p->mbuf_sz) * sizeof(Eterm); + if (p->old_hend && p->old_heap) + size += (p->old_hend - p->old_heap) * sizeof(Eterm); + + size += p->msg.len * sizeof(ErlMessage); + + for (mp = p->msg.first; mp; mp = mp->next) + if (mp->data.attached) + size += erts_msg_attached_data_size(mp)*sizeof(Eterm); + + if (p->arg_reg != p->def_arg_reg) { + size += p->arity * sizeof(p->arg_reg[0]); + } + + if (p->psd) + size += sizeof(ErtsPSD); + + scb = ERTS_PROC_GET_SAVED_CALLS_BUF(p); + if (scb) { + size += (sizeof(struct saved_calls) + + (scb->len-1) * sizeof(scb->ct[0])); + } + + size += erts_dicts_mem_size(p); + return size; +} + static void dump_process_info(int to, void *to_arg, Process *p) { @@ -111,7 +150,7 @@ dump_process_info(int to, void *to_arg, Process *p) if ((ERTS_TRACE_FLAGS(p) & F_SENSITIVE) == 0) { erts_print(to, to_arg, "=proc_stack:%T\n", p->common.id); for (sp = p->stop; sp < STACK_START(p); sp++) { - yreg = stack_element_dump(to, to_arg, p, sp, yreg); + yreg = stack_element_dump(to, to_arg, sp, yreg); } erts_print(to, to_arg, "=proc_heap:%T\n", p->common.id); @@ -173,9 +212,9 @@ static void dump_element(int to, void *to_arg, Eterm x) { if (is_list(x)) { - erts_print(to, to_arg, "H" WORD_FMT, list_val(x)); + erts_print(to, to_arg, "H" PTR_FMT, list_val(x)); } else if (is_boxed(x)) { - erts_print(to, to_arg, "H" WORD_FMT, boxed_val(x)); + erts_print(to, to_arg, "H" PTR_FMT, boxed_val(x)); } else if (is_immed(x)) { if (is_atom(x)) { unsigned char* s = atom_tab(atom_val(x))->name; @@ -206,9 +245,65 @@ dump_element_nl(int to, void *to_arg, Eterm x) erts_putc(to, to_arg, '\n'); } +static void +stack_trace_dump(int to, void *to_arg, Eterm *sp) { + Eterm x = *sp; + if (is_CP(x)) { + erts_print(to, to_arg, "%p:", sp); + erts_print(to, to_arg, "SReturn addr 0x%X (", cp_val(x)); + print_function_from_pc(to, to_arg, cp_val(x)); + erts_print(to, to_arg, ")\n"); + } +} + +void +erts_limited_stack_trace(int to, void *to_arg, Process *p) +{ + Eterm* sp; + + + if (ERTS_TRACE_FLAGS(p) & F_SENSITIVE) { + return; + } + + if (STACK_START(p) < STACK_TOP(p)) { + return; + } + + if ((STACK_START(p) - STACK_TOP(p)) < 512) { + if (erts_sys_is_area_readable((char*)STACK_TOP(p), + (char*)STACK_START(p))) + for (sp = STACK_TOP(p); sp < STACK_START(p); sp++) + stack_trace_dump(to, to_arg, sp); + else + erts_print(to, to_arg, "Could not read from stack memory: %p - %p\n", + STACK_TOP(p), STACK_START(p)); + } else { + sp = STACK_TOP(p); + if (erts_sys_is_area_readable((char*)STACK_TOP(p), + (char*)(STACK_TOP(p) + 25))) + for (; sp < (STACK_TOP(p) + 256); sp++) + stack_trace_dump(to, to_arg, sp); + else + erts_print(to, to_arg, "Could not read from stack memory: %p - %p\n", + STACK_TOP(p), STACK_TOP(p) + 256); + + erts_print(to, to_arg, "%p: skipping %d frames\n", + sp, STACK_START(p) - STACK_TOP(p) - 512); + + if (erts_sys_is_area_readable((char*)(STACK_START(p) - 256), + (char*)STACK_START(p))) + for (sp = STACK_START(p) - 256; sp < STACK_START(p); sp++) + stack_trace_dump(to, to_arg, sp); + else + erts_print(to, to_arg, "Could not read from stack memory: %p - %p\n", + STACK_START(p) - 256, STACK_START(p)); + } + +} static int -stack_element_dump(int to, void *to_arg, Process* p, Eterm* sp, int yreg) +stack_element_dump(int to, void *to_arg, Eterm* sp, int yreg) { Eterm x = *sp; @@ -274,7 +369,7 @@ heap_dump(int to, void *to_arg, Eterm x) } else if (is_list(x)) { ptr = list_val(x); if (ptr[0] != OUR_NIL) { - erts_print(to, to_arg, ADDR_FMT ":l", ptr); + erts_print(to, to_arg, PTR_FMT ":l", ptr); dump_element(to, to_arg, ptr[0]); erts_putc(to, to_arg, '|'); dump_element(to, to_arg, ptr[1]); @@ -293,12 +388,12 @@ heap_dump(int to, void *to_arg, Eterm x) ptr = boxed_val(x); hdr = *ptr; if (hdr != OUR_NIL) { /* If not visited */ - erts_print(to, to_arg, ADDR_FMT ":", ptr); + erts_print(to, to_arg, PTR_FMT ":", ptr); if (is_arity_value(hdr)) { Uint i; Uint arity = arityval(hdr); - erts_print(to, to_arg, "t" WORD_FMT ":", arity); + erts_print(to, to_arg, "t" ETERM_FMT ":", arity); for (i = 1; i <= arity; i++) { dump_element(to, to_arg, ptr[i]); if (is_immed(ptr[i])) { @@ -351,21 +446,43 @@ heap_dump(int to, void *to_arg, Eterm x) val->flags = (UWord) all_binaries; all_binaries = val; } - erts_print(to, to_arg, "Yc%X:%X:%X", val, + erts_print(to, to_arg, + "Yc" PTR_FMT ":" PTR_FMT ":" PTR_FMT, + val, pb->bytes - (byte *)val->orig_bytes, size); } else if (tag == SUB_BINARY_SUBTAG) { ErlSubBin* Sb = (ErlSubBin *) binary_val(x); - Eterm* real_bin = binary_val(Sb->orig); + Eterm* real_bin; void* val; + /* + * Must use boxed_val() here, because the original + * binary may have been visited and have had its + * header word changed to OUR_NIL (in which case + * binary_val() will cause an assertion failure in + * the DEBUG emulator). + */ + + real_bin = boxed_val(Sb->orig); + if (thing_subtag(*real_bin) == REFC_BINARY_SUBTAG) { + /* + * Unvisited REFC_BINARY: Point directly to + * the binary. + */ ProcBin* pb = (ProcBin *) real_bin; val = pb->val; - } else { /* Heap binary */ + } else { + /* + * Heap binary or visited REFC binary: Point + * to heap binary or ProcBin on the heap. + */ val = real_bin; } - erts_print(to, to_arg, "Ys%X:%X:%X", val, Sb->offs, size); + erts_print(to, to_arg, + "Ys" PTR_FMT ":" PTR_FMT ":" PTR_FMT, + val, Sb->offs, size); } erts_putc(to, to_arg, '\n'); *ptr = OUR_NIL; @@ -401,7 +518,7 @@ dump_binaries(int to, void *to_arg, Binary* current) long size = current->orig_size; byte* bytes = (byte*) current->orig_bytes; - erts_print(to, to_arg, "=binary:%X\n", current); + erts_print(to, to_arg, "=binary:" PTR_FMT "\n", current); erts_print(to, to_arg, "%X:", size); for (i = 0; i < size; i++) { erts_print(to, to_arg, "%02X", bytes[i]); @@ -449,3 +566,114 @@ dump_externally(int to, void *to_arg, Eterm term) erts_print(to, to_arg, "%02X", *s++); } } + +void erts_dump_process_state(int to, void *to_arg, erts_aint32_t psflg) { + if (psflg & ERTS_PSFLG_FREE) + erts_print(to, to_arg, "Non Existing\n"); /* Should never happen */ + else if (psflg & ERTS_PSFLG_EXITING) + erts_print(to, to_arg, "Exiting\n"); + else if (psflg & ERTS_PSFLG_GC) { + erts_print(to, to_arg, "Garbing\n"); + } + else if (psflg & ERTS_PSFLG_SUSPENDED) + erts_print(to, to_arg, "Suspended\n"); + else if (psflg & ERTS_PSFLG_RUNNING) { + erts_print(to, to_arg, "Running\n"); + } + else if (psflg & ERTS_PSFLG_ACTIVE) + erts_print(to, to_arg, "Scheduled\n"); + else + erts_print(to, to_arg, "Waiting\n"); +} + +void +erts_dump_extended_process_state(int to, void *to_arg, erts_aint32_t psflg) { + + int i; + + switch (ERTS_PSFLGS_GET_ACT_PRIO(psflg)) { + case PRIORITY_MAX: erts_print(to, to_arg, "ACT_PRIO_MAX | "); break; + case PRIORITY_HIGH: erts_print(to, to_arg, "ACT_PRIO_HIGH | "); break; + case PRIORITY_NORMAL: erts_print(to, to_arg, "ACT_PRIO_NORMAL | "); break; + case PRIORITY_LOW: erts_print(to, to_arg, "ACT_PRIO_LOW | "); break; + } + switch (ERTS_PSFLGS_GET_USR_PRIO(psflg)) { + case PRIORITY_MAX: erts_print(to, to_arg, "USR_PRIO_MAX | "); break; + case PRIORITY_HIGH: erts_print(to, to_arg, "USR_PRIO_HIGH | "); break; + case PRIORITY_NORMAL: erts_print(to, to_arg, "USR_PRIO_NORMAL | "); break; + case PRIORITY_LOW: erts_print(to, to_arg, "USR_PRIO_LOW | "); break; + } + switch (ERTS_PSFLGS_GET_PRQ_PRIO(psflg)) { + case PRIORITY_MAX: erts_print(to, to_arg, "PRQ_PRIO_MAX"); break; + case PRIORITY_HIGH: erts_print(to, to_arg, "PRQ_PRIO_HIGH"); break; + case PRIORITY_NORMAL: erts_print(to, to_arg, "PRQ_PRIO_NORMAL"); break; + case PRIORITY_LOW: erts_print(to, to_arg, "PRQ_PRIO_LOW"); break; + } + + psflg &= ~(ERTS_PSFLGS_ACT_PRIO_MASK | + ERTS_PSFLGS_USR_PRIO_MASK | + ERTS_PSFLGS_PRQ_PRIO_MASK); + + if (psflg) + erts_print(to, to_arg, " | "); + + for (i = 0; i < ERTS_PSFLG_MAX && psflg; i++) { + erts_aint32_t chk = (1 << i); + if (psflg & chk) { + switch (chk) { + case ERTS_PSFLG_IN_PRQ_MAX: + erts_print(to, to_arg, "IN_PRQ_MAX"); break; + case ERTS_PSFLG_IN_PRQ_HIGH: + erts_print(to, to_arg, "IN_PRQ_HIGH"); break; + case ERTS_PSFLG_IN_PRQ_NORMAL: + erts_print(to, to_arg, "IN_PRQ_NORMAL"); break; + case ERTS_PSFLG_IN_PRQ_LOW: + erts_print(to, to_arg, "IN_PRQ_LOW"); break; + case ERTS_PSFLG_FREE: + erts_print(to, to_arg, "FREE"); break; + case ERTS_PSFLG_EXITING: + erts_print(to, to_arg, "EXITING"); break; + case ERTS_PSFLG_PENDING_EXIT: + erts_print(to, to_arg, "PENDING_EXIT"); break; + case ERTS_PSFLG_ACTIVE: + erts_print(to, to_arg, "ACTIVE"); break; + case ERTS_PSFLG_IN_RUNQ: + erts_print(to, to_arg, "IN_RUNQ"); break; + case ERTS_PSFLG_RUNNING: + erts_print(to, to_arg, "RUNNING"); break; + case ERTS_PSFLG_SUSPENDED: + erts_print(to, to_arg, "SUSPENDED"); break; + case ERTS_PSFLG_GC: + erts_print(to, to_arg, "GC"); break; + case ERTS_PSFLG_BOUND: + erts_print(to, to_arg, "BOUND"); break; + case ERTS_PSFLG_TRAP_EXIT: + erts_print(to, to_arg, "TRAP_EXIT"); break; + case ERTS_PSFLG_ACTIVE_SYS: + erts_print(to, to_arg, "ACTIVE_SYS"); break; + case ERTS_PSFLG_RUNNING_SYS: + erts_print(to, to_arg, "RUNNING_SYS"); break; + case ERTS_PSFLG_PROXY: + erts_print(to, to_arg, "PROXY"); break; + case ERTS_PSFLG_DELAYED_SYS: + erts_print(to, to_arg, "DELAYED_SYS"); break; +#ifdef ERTS_DIRTY_SCHEDULERS + case ERTS_PSFLG_DIRTY_CPU_PROC: + erts_print(to, to_arg, "DIRTY_CPU_PROC"); break; + case ERTS_PSFLG_DIRTY_IO_PROC: + erts_print(to, to_arg, "DIRTY_IO_PROC"); break; + case ERTS_PSFLG_DIRTY_CPU_PROC_IN_Q: + erts_print(to, to_arg, "DIRTY_CPU_PROC_IN_Q"); break; + case ERTS_PSFLG_DIRTY_IO_PROC_IN_Q: + erts_print(to, to_arg, "DIRTY_IO_PROC_IN_Q"); break; +#endif + default: + erts_print(to, to_arg, "UNKNOWN(%d)", chk); break; + } + if (psflg > chk) + erts_print(to, to_arg, " | "); + psflg -= chk; + } + } + erts_print(to, to_arg, "\n"); +} diff --git a/erts/emulator/beam/erl_process_lock.c b/erts/emulator/beam/erl_process_lock.c index 2db5df06b4..0bee2c848c 100644 --- a/erts/emulator/beam/erl_process_lock.c +++ b/erts/emulator/beam/erl_process_lock.c @@ -3,16 +3,17 @@ * * Copyright Ericsson AB 2007-2012. All Rights Reserved. * - * The contents of this file are subject to the Erlang Public License, - * Version 1.1, (the "License"); you may not use this file except in - * compliance with the License. You should have received a copy of the - * Erlang Public License along with this software. If not, it can be - * retrieved online at http://www.erlang.org/. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Software distributed under the License is distributed on an "AS IS" - * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See - * the License for the specific language governing rights and limitations - * under the License. + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. * * %CopyrightEnd% */ @@ -103,12 +104,16 @@ static struct { Sint16 proc_lock_main; Sint16 proc_lock_link; Sint16 proc_lock_msgq; + Sint16 proc_lock_btm; Sint16 proc_lock_status; } lc_id; #endif erts_pix_lock_t erts_pix_locks[ERTS_NO_OF_PIX_LOCKS]; +#ifdef ERTS_ENABLE_LOCK_COUNT +static void lcnt_enable_proc_lock_count(Process *proc, int enable); +#endif void erts_init_proc_lock(int cpus) @@ -117,7 +122,7 @@ erts_init_proc_lock(int cpus) for (i = 0; i < ERTS_NO_OF_PIX_LOCKS; i++) { #ifdef ERTS_ENABLE_LOCK_COUNT erts_mtx_init_x(&erts_pix_locks[i].u.mtx, - "pix_lock", make_small(i)); + "pix_lock", make_small(i), 1); #else erts_mtx_init(&erts_pix_locks[i].u.mtx, "pix_lock"); #endif @@ -145,6 +150,7 @@ erts_init_proc_lock(int cpus) lc_id.proc_lock_main = erts_lc_get_lock_order_id("proc_main"); lc_id.proc_lock_link = erts_lc_get_lock_order_id("proc_link"); lc_id.proc_lock_msgq = erts_lc_get_lock_order_id("proc_msgq"); + lc_id.proc_lock_btm = erts_lc_get_lock_order_id("proc_btm"); lc_id.proc_lock_status = erts_lc_get_lock_order_id("proc_status"); #endif } @@ -707,7 +713,7 @@ proc_safelock(int is_managed, need_locks1 |= unlock_locks; if (!is_managed && !have_locks1) { refc1 = 1; - erts_smp_proc_inc_refc(p1); + erts_proc_inc_refc(p1); } erts_smp_proc_unlock(p1, unlock_locks); } @@ -717,7 +723,7 @@ proc_safelock(int is_managed, need_locks2 |= unlock_locks; if (!is_managed && !have_locks2) { refc2 = 1; - erts_smp_proc_inc_refc(p2); + erts_proc_inc_refc(p2); } erts_smp_proc_unlock(p2, unlock_locks); } @@ -798,9 +804,9 @@ proc_safelock(int is_managed, if (!is_managed) { if (refc1) - erts_smp_proc_dec_refc(p1); + erts_proc_dec_refc(p1); if (refc2) - erts_smp_proc_dec_refc(p2); + erts_proc_dec_refc(p2); } } @@ -861,8 +867,8 @@ erts_pid2proc_opt(Process *c_p, return NULL; need_locks &= ~c_p_have_locks; if (!need_locks) { - if (flags & ERTS_P2P_FLG_SMP_INC_REFC) - erts_smp_proc_inc_refc(c_p); + if (flags & ERTS_P2P_FLG_INC_REFC) + erts_proc_inc_refc(c_p); return c_p; } } @@ -875,8 +881,8 @@ erts_pid2proc_opt(Process *c_p, if (proc->common.id != pid) proc = NULL; else if (!need_locks) { - if (flags & ERTS_P2P_FLG_SMP_INC_REFC) - erts_smp_proc_inc_refc(proc); + if (flags & ERTS_P2P_FLG_INC_REFC) + erts_proc_inc_refc(proc); } else { int busy; @@ -901,7 +907,7 @@ erts_pid2proc_opt(Process *c_p, busy = (int) erts_smp_proc_raw_trylock__(proc, need_locks); #if ERTS_PROC_LOCK_OWN_IMPL && defined(ERTS_ENABLE_LOCK_CHECK) - erts_proc_lc_trylock(proc, need_locks, !busy); + erts_proc_lc_trylock(proc, need_locks, !busy, __FILE__,__LINE__); #endif #ifdef ERTS_PROC_LOCK_DEBUG if (!busy) @@ -916,8 +922,8 @@ erts_pid2proc_opt(Process *c_p, #endif if (!busy) { - if (flags & ERTS_P2P_FLG_SMP_INC_REFC) - erts_smp_proc_inc_refc(proc); + if (flags & ERTS_P2P_FLG_INC_REFC) + erts_proc_inc_refc(proc); #if ERTS_PROC_LOCK_OWN_IMPL && defined(ERTS_ENABLE_LOCK_COUNT) /* all is great */ @@ -932,8 +938,8 @@ erts_pid2proc_opt(Process *c_p, proc = ERTS_PROC_LOCK_BUSY; else { int managed; - if (flags & ERTS_P2P_FLG_SMP_INC_REFC) - erts_smp_proc_inc_refc(proc); + if (flags & ERTS_P2P_FLG_INC_REFC) + erts_proc_inc_refc(proc); #if ERTS_PROC_LOCK_OWN_IMPL && defined(ERTS_ENABLE_LOCK_COUNT) erts_lcnt_proc_lock_unaquire(&proc->lock, lcnt_locks); @@ -941,7 +947,7 @@ erts_pid2proc_opt(Process *c_p, managed = dhndl == ERTS_THR_PRGR_DHANDLE_MANAGED; if (!managed) { - erts_smp_proc_inc_refc(proc); + erts_proc_inc_refc(proc); erts_thr_progress_unmanaged_continue(dhndl); dec_refc_proc = proc; @@ -978,14 +984,14 @@ erts_pid2proc_opt(Process *c_p, erts_smp_proc_unlock(proc, need_locks); - if (flags & ERTS_P2P_FLG_SMP_INC_REFC) + if (flags & ERTS_P2P_FLG_INC_REFC) dec_refc_proc = proc; proc = NULL; } if (dec_refc_proc) - erts_smp_proc_dec_refc(dec_refc_proc); + erts_proc_dec_refc(dec_refc_proc); #if ERTS_PROC_LOCK_OWN_IMPL && defined(ERTS_PROC_LOCK_DEBUG) ERTS_LC_ASSERT(!proc @@ -1001,8 +1007,10 @@ erts_pid2proc_opt(Process *c_p, void erts_proc_lock_init(Process *p) { -#if ERTS_PROC_LOCK_OWN_IMPL +#if ERTS_PROC_LOCK_OWN_IMPL || defined(ERTS_PROC_LOCK_DEBUG) int i; +#endif +#if ERTS_PROC_LOCK_OWN_IMPL /* We always start with all locks locked */ #if ERTS_PROC_LOCK_ATOMIC_IMPL erts_smp_atomic32_init_nob(&p->lock.flags, @@ -1013,31 +1021,43 @@ erts_proc_lock_init(Process *p) for (i = 0; i <= ERTS_PROC_LOCK_MAX_BIT; i++) p->lock.queue[i] = NULL; #ifdef ERTS_ENABLE_LOCK_CHECK - erts_proc_lc_trylock(p, ERTS_PROC_LOCKS_ALL, 1); + erts_proc_lc_trylock(p, ERTS_PROC_LOCKS_ALL, 1,__FILE__,__LINE__); #endif #elif ERTS_PROC_LOCK_RAW_MUTEX_IMPL - erts_mtx_init_x(&p->lock.main, "proc_main", p->common.id); + +#ifdef ERTS_ENABLE_LOCK_COUNT + int do_lock_count = 1; +#else + int do_lock_count = 0; +#endif + + erts_mtx_init_x(&p->lock.main, "proc_main", p->common.id, do_lock_count); ethr_mutex_lock(&p->lock.main.mtx); #ifdef ERTS_ENABLE_LOCK_CHECK erts_lc_trylock(1, &p->lock.main.lc); #endif - erts_mtx_init_x(&p->lock.link, "proc_link", p->common.id); + erts_mtx_init_x(&p->lock.link, "proc_link", p->common.id, do_lock_count); ethr_mutex_lock(&p->lock.link.mtx); #ifdef ERTS_ENABLE_LOCK_CHECK erts_lc_trylock(1, &p->lock.link.lc); #endif - erts_mtx_init_x(&p->lock.msgq, "proc_msgq", p->common.id); + erts_mtx_init_x(&p->lock.msgq, "proc_msgq", p->common.id, do_lock_count); ethr_mutex_lock(&p->lock.msgq.mtx); #ifdef ERTS_ENABLE_LOCK_CHECK erts_lc_trylock(1, &p->lock.msgq.lc); #endif - erts_mtx_init_x(&p->lock.status, "proc_status", p->common.id); + erts_mtx_init_x(&p->lock.btm, "proc_btm", p->common.id, do_lock_count); + ethr_mutex_lock(&p->lock.btm.mtx); +#ifdef ERTS_ENABLE_LOCK_CHECK + erts_lc_trylock(1, &p->lock.btm.lc); +#endif + erts_mtx_init_x(&p->lock.status, "proc_status", p->common.id, + do_lock_count); ethr_mutex_lock(&p->lock.status.mtx); #ifdef ERTS_ENABLE_LOCK_CHECK erts_lc_trylock(1, &p->lock.status.lc); #endif #endif - erts_atomic32_init_nob(&p->lock.refc, 1); #ifdef ERTS_PROC_LOCK_DEBUG for (i = 0; i <= ERTS_PROC_LOCK_MAX_BIT; i++) erts_smp_atomic32_init_nob(&p->lock.locked[i], (erts_aint32_t) 1); @@ -1056,6 +1076,7 @@ erts_proc_lock_fin(Process *p) erts_mtx_destroy(&p->lock.main); erts_mtx_destroy(&p->lock.link); erts_mtx_destroy(&p->lock.msgq); + erts_mtx_destroy(&p->lock.btm); erts_mtx_destroy(&p->lock.status); #endif #if defined(ERTS_ENABLE_LOCK_COUNT) && defined(ERTS_SMP) @@ -1066,141 +1087,109 @@ erts_proc_lock_fin(Process *p) /* --- Process lock counting ----------------------------------------------- */ #if ERTS_PROC_LOCK_OWN_IMPL && defined(ERTS_ENABLE_LOCK_COUNT) -void erts_lcnt_proc_lock_init(Process *p) { - if (erts_lcnt_rt_options & ERTS_LCNT_OPT_PROCLOCK) { - if (p->common.id != ERTS_INVALID_PID) { - erts_lcnt_init_lock_x(&(p->lock.lcnt_main), "proc_main", ERTS_LCNT_LT_PROCLOCK, p->common.id); - erts_lcnt_init_lock_x(&(p->lock.lcnt_msgq), "proc_msgq", ERTS_LCNT_LT_PROCLOCK, p->common.id); - erts_lcnt_init_lock_x(&(p->lock.lcnt_link), "proc_link", ERTS_LCNT_LT_PROCLOCK, p->common.id); - erts_lcnt_init_lock_x(&(p->lock.lcnt_status), "proc_status", ERTS_LCNT_LT_PROCLOCK, p->common.id); - } else { - erts_lcnt_init_lock(&(p->lock.lcnt_main), "proc_main", ERTS_LCNT_LT_PROCLOCK); - erts_lcnt_init_lock(&(p->lock.lcnt_msgq), "proc_msgq", ERTS_LCNT_LT_PROCLOCK); - erts_lcnt_init_lock(&(p->lock.lcnt_link), "proc_link", ERTS_LCNT_LT_PROCLOCK); - erts_lcnt_init_lock(&(p->lock.lcnt_status), "proc_status", ERTS_LCNT_LT_PROCLOCK); - } - } else { - sys_memzero(&(p->lock.lcnt_main), sizeof(p->lock.lcnt_main)); - sys_memzero(&(p->lock.lcnt_msgq), sizeof(p->lock.lcnt_msgq)); - sys_memzero(&(p->lock.lcnt_link), sizeof(p->lock.lcnt_link)); - sys_memzero(&(p->lock.lcnt_status), sizeof(p->lock.lcnt_status)); - } + +void erts_lcnt_enable_proc_lock_count(int enable) { + int ix, max = erts_ptab_max(&erts_proc); + Process *proc = NULL; + for (ix = 0; ix < max; ++ix) { + if ((proc = erts_pix2proc(ix)) != NULL) + lcnt_enable_proc_lock_count(proc, enable); + } /* for all processes */ } - + +void erts_lcnt_proc_lock_init(Process *p) { + if (!(erts_lcnt_rt_options & ERTS_LCNT_OPT_PROCLOCK)) { + erts_lcnt_init_lock_empty(&(p->lock.lcnt_main)); + erts_lcnt_init_lock_empty(&(p->lock.lcnt_msgq)); + erts_lcnt_init_lock_empty(&(p->lock.lcnt_btm)); + erts_lcnt_init_lock_empty(&(p->lock.lcnt_link)); + erts_lcnt_init_lock_empty(&(p->lock.lcnt_status)); + } else { /* now the common case */ + Eterm pid = (p->common.id != ERTS_INVALID_PID) ? p->common.id : NIL; + erts_lcnt_init_lock_x(&(p->lock.lcnt_main), "proc_main", ERTS_LCNT_LT_PROCLOCK, pid); + erts_lcnt_init_lock_x(&(p->lock.lcnt_msgq), "proc_msgq", ERTS_LCNT_LT_PROCLOCK, pid); + erts_lcnt_init_lock_x(&(p->lock.lcnt_btm), "proc_btm", ERTS_LCNT_LT_PROCLOCK, pid); + erts_lcnt_init_lock_x(&(p->lock.lcnt_link), "proc_link", ERTS_LCNT_LT_PROCLOCK, pid); + erts_lcnt_init_lock_x(&(p->lock.lcnt_status),"proc_status",ERTS_LCNT_LT_PROCLOCK, pid); + } /* the lock names should really be aligned to four characters */ +} /* logic reversed */ void erts_lcnt_proc_lock_destroy(Process *p) { erts_lcnt_destroy_lock(&(p->lock.lcnt_main)); erts_lcnt_destroy_lock(&(p->lock.lcnt_msgq)); + erts_lcnt_destroy_lock(&(p->lock.lcnt_btm)); erts_lcnt_destroy_lock(&(p->lock.lcnt_link)); erts_lcnt_destroy_lock(&(p->lock.lcnt_status)); } -void erts_lcnt_proc_lock(erts_proc_lock_t *lock, ErtsProcLocks locks) { - if (erts_lcnt_rt_options & ERTS_LCNT_OPT_PROCLOCK) { - if (locks & ERTS_PROC_LOCK_MAIN) { - erts_lcnt_lock(&(lock->lcnt_main)); - } - if (locks & ERTS_PROC_LOCK_MSGQ) { - erts_lcnt_lock(&(lock->lcnt_msgq)); - } - if (locks & ERTS_PROC_LOCK_LINK) { - erts_lcnt_lock(&(lock->lcnt_link)); - } - if (locks & ERTS_PROC_LOCK_STATUS) { - erts_lcnt_lock(&(lock->lcnt_status)); +static void lcnt_enable_proc_lock_count(Process *proc, int enable) { + if (enable) { + if (!ERTS_LCNT_LOCK_TYPE(&(proc->lock.lcnt_main))) { + erts_lcnt_proc_lock_init(proc); + } } + else { + if (ERTS_LCNT_LOCK_TYPE(&(proc->lock.lcnt_main))) { + erts_lcnt_proc_lock_destroy(proc); + } } } -void erts_lcnt_proc_lock_post_x(erts_proc_lock_t *lock, ErtsProcLocks locks, char *file, unsigned int line) { - if (erts_lcnt_rt_options & ERTS_LCNT_OPT_PROCLOCK) { +void erts_lcnt_proc_lock(erts_proc_lock_t *lock, ErtsProcLocks locks) { + if (!(erts_lcnt_rt_options & ERTS_LCNT_OPT_PROCLOCK)) return; + if (locks & ERTS_PROC_LOCK_MAIN) { erts_lcnt_lock(&(lock->lcnt_main)); } + if (locks & ERTS_PROC_LOCK_MSGQ) { erts_lcnt_lock(&(lock->lcnt_msgq)); } + if (locks & ERTS_PROC_LOCK_BTM) { erts_lcnt_lock(&(lock->lcnt_btm)); } + if (locks & ERTS_PROC_LOCK_LINK) { erts_lcnt_lock(&(lock->lcnt_link)); } + if (locks & ERTS_PROC_LOCK_STATUS) { erts_lcnt_lock(&(lock->lcnt_status)); } +} + +void erts_lcnt_proc_lock_post_x(erts_proc_lock_t *lock, ErtsProcLocks locks, + char *file, unsigned int line) { + if (!(erts_lcnt_rt_options & ERTS_LCNT_OPT_PROCLOCK)) return; if (locks & ERTS_PROC_LOCK_MAIN) { erts_lcnt_lock_post_x(&(lock->lcnt_main), file, line); } if (locks & ERTS_PROC_LOCK_MSGQ) { erts_lcnt_lock_post_x(&(lock->lcnt_msgq), file, line); } + if (locks & ERTS_PROC_LOCK_BTM) { + erts_lcnt_lock_post_x(&(lock->lcnt_btm), file, line); + } if (locks & ERTS_PROC_LOCK_LINK) { erts_lcnt_lock_post_x(&(lock->lcnt_link), file, line); } if (locks & ERTS_PROC_LOCK_STATUS) { erts_lcnt_lock_post_x(&(lock->lcnt_status), file, line); } - } } void erts_lcnt_proc_lock_unaquire(erts_proc_lock_t *lock, ErtsProcLocks locks) { - if (erts_lcnt_rt_options & ERTS_LCNT_OPT_PROCLOCK) { - if (locks & ERTS_PROC_LOCK_MAIN) { - erts_lcnt_lock_unaquire(&(lock->lcnt_main)); - } - if (locks & ERTS_PROC_LOCK_MSGQ) { - erts_lcnt_lock_unaquire(&(lock->lcnt_msgq)); - } - if (locks & ERTS_PROC_LOCK_LINK) { - erts_lcnt_lock_unaquire(&(lock->lcnt_link)); - } - if (locks & ERTS_PROC_LOCK_STATUS) { - erts_lcnt_lock_unaquire(&(lock->lcnt_status)); - } - } + if (!(erts_lcnt_rt_options & ERTS_LCNT_OPT_PROCLOCK)) return; + if (locks & ERTS_PROC_LOCK_MAIN) { erts_lcnt_lock_unaquire(&(lock->lcnt_main)); } + if (locks & ERTS_PROC_LOCK_MSGQ) { erts_lcnt_lock_unaquire(&(lock->lcnt_msgq)); } + if (locks & ERTS_PROC_LOCK_BTM) { erts_lcnt_lock_unaquire(&(lock->lcnt_btm)); } + if (locks & ERTS_PROC_LOCK_LINK) { erts_lcnt_lock_unaquire(&(lock->lcnt_link)); } + if (locks & ERTS_PROC_LOCK_STATUS) { erts_lcnt_lock_unaquire(&(lock->lcnt_status)); } } void erts_lcnt_proc_unlock(erts_proc_lock_t *lock, ErtsProcLocks locks) { - if (erts_lcnt_rt_options & ERTS_LCNT_OPT_PROCLOCK) { - if (locks & ERTS_PROC_LOCK_MAIN) { - erts_lcnt_unlock(&(lock->lcnt_main)); - } - if (locks & ERTS_PROC_LOCK_MSGQ) { - erts_lcnt_unlock(&(lock->lcnt_msgq)); - } - if (locks & ERTS_PROC_LOCK_LINK) { - erts_lcnt_unlock(&(lock->lcnt_link)); - } - if (locks & ERTS_PROC_LOCK_STATUS) { - erts_lcnt_unlock(&(lock->lcnt_status)); - } - } + if (!(erts_lcnt_rt_options & ERTS_LCNT_OPT_PROCLOCK)) return; + if (locks & ERTS_PROC_LOCK_MAIN) { erts_lcnt_unlock(&(lock->lcnt_main)); } + if (locks & ERTS_PROC_LOCK_MSGQ) { erts_lcnt_unlock(&(lock->lcnt_msgq)); } + if (locks & ERTS_PROC_LOCK_BTM) { erts_lcnt_unlock(&(lock->lcnt_btm)); } + if (locks & ERTS_PROC_LOCK_LINK) { erts_lcnt_unlock(&(lock->lcnt_link)); } + if (locks & ERTS_PROC_LOCK_STATUS) { erts_lcnt_unlock(&(lock->lcnt_status)); } } void erts_lcnt_proc_trylock(erts_proc_lock_t *lock, ErtsProcLocks locks, int res) { - if (erts_lcnt_rt_options & ERTS_LCNT_OPT_PROCLOCK) { - if (locks & ERTS_PROC_LOCK_MAIN) { - erts_lcnt_trylock(&(lock->lcnt_main), res); - } - if (locks & ERTS_PROC_LOCK_MSGQ) { - erts_lcnt_trylock(&(lock->lcnt_msgq), res); - } - if (locks & ERTS_PROC_LOCK_LINK) { - erts_lcnt_trylock(&(lock->lcnt_link), res); - } - if (locks & ERTS_PROC_LOCK_STATUS) { - erts_lcnt_trylock(&(lock->lcnt_status), res); - } - } -} - - -void erts_lcnt_enable_proc_lock_count(int enable) -{ - int i, max = erts_ptab_max(&erts_proc); - - for (i = 0; i < max; ++i) { - Process* p = erts_pix2proc(i); - if (p) { - if (enable) { - if (!ERTS_LCNT_LOCK_TYPE(&(p->lock.lcnt_main))) { - erts_lcnt_proc_lock_init(p); - } - } else { - if (ERTS_LCNT_LOCK_TYPE(&(p->lock.lcnt_main))) { - erts_lcnt_proc_lock_destroy(p); - } - } - } - } -} - -#endif /* ifdef ERTS_ENABLE_LOCK_COUNT */ + if (!(erts_lcnt_rt_options & ERTS_LCNT_OPT_PROCLOCK)) return; + if (locks & ERTS_PROC_LOCK_MAIN) { erts_lcnt_trylock(&(lock->lcnt_main), res); } + if (locks & ERTS_PROC_LOCK_MSGQ) { erts_lcnt_trylock(&(lock->lcnt_msgq), res); } + if (locks & ERTS_PROC_LOCK_BTM) { erts_lcnt_trylock(&(lock->lcnt_btm), res); } + if (locks & ERTS_PROC_LOCK_LINK) { erts_lcnt_trylock(&(lock->lcnt_link), res); } + if (locks & ERTS_PROC_LOCK_STATUS) { erts_lcnt_trylock(&(lock->lcnt_status), res); } +} /* reversed logic */ +#endif /* ERTS_ENABLE_LOCK_COUNT */ /* --- Process lock checking ----------------------------------------------- */ @@ -1210,50 +1199,59 @@ void erts_lcnt_enable_proc_lock_count(int enable) #if ERTS_PROC_LOCK_OWN_IMPL void -erts_proc_lc_lock(Process *p, ErtsProcLocks locks) +erts_proc_lc_lock(Process *p, ErtsProcLocks locks, char *file, unsigned int line) { erts_lc_lock_t lck = ERTS_LC_LOCK_INIT(-1, p->common.id, ERTS_LC_FLG_LT_PROCLOCK); if (locks & ERTS_PROC_LOCK_MAIN) { lck.id = lc_id.proc_lock_main; - erts_lc_lock(&lck); + erts_lc_lock_x(&lck,file,line); } if (locks & ERTS_PROC_LOCK_LINK) { lck.id = lc_id.proc_lock_link; - erts_lc_lock(&lck); + erts_lc_lock_x(&lck,file,line); } if (locks & ERTS_PROC_LOCK_MSGQ) { lck.id = lc_id.proc_lock_msgq; - erts_lc_lock(&lck); + erts_lc_lock_x(&lck,file,line); + } + if (locks & ERTS_PROC_LOCK_BTM) { + lck.id = lc_id.proc_lock_btm; + erts_lc_lock_x(&lck,file,line); } if (locks & ERTS_PROC_LOCK_STATUS) { lck.id = lc_id.proc_lock_status; - erts_lc_lock(&lck); + erts_lc_lock_x(&lck,file,line); } } void -erts_proc_lc_trylock(Process *p, ErtsProcLocks locks, int locked) +erts_proc_lc_trylock(Process *p, ErtsProcLocks locks, int locked, + char* file, unsigned int line) { erts_lc_lock_t lck = ERTS_LC_LOCK_INIT(-1, p->common.id, ERTS_LC_FLG_LT_PROCLOCK); if (locks & ERTS_PROC_LOCK_MAIN) { lck.id = lc_id.proc_lock_main; - erts_lc_trylock(locked, &lck); + erts_lc_trylock_x(locked, &lck, file, line); } if (locks & ERTS_PROC_LOCK_LINK) { lck.id = lc_id.proc_lock_link; - erts_lc_trylock(locked, &lck); + erts_lc_trylock_x(locked, &lck, file, line); } if (locks & ERTS_PROC_LOCK_MSGQ) { lck.id = lc_id.proc_lock_msgq; - erts_lc_trylock(locked, &lck); + erts_lc_trylock_x(locked, &lck, file, line); + } + if (locks & ERTS_PROC_LOCK_BTM) { + lck.id = lc_id.proc_lock_btm; + erts_lc_trylock_x(locked, &lck, file, line); } if (locks & ERTS_PROC_LOCK_STATUS) { lck.id = lc_id.proc_lock_status; - erts_lc_trylock(locked, &lck); + erts_lc_trylock_x(locked, &lck, file, line); } } @@ -1267,6 +1265,10 @@ erts_proc_lc_unlock(Process *p, ErtsProcLocks locks) lck.id = lc_id.proc_lock_status; erts_lc_unlock(&lck); } + if (locks & ERTS_PROC_LOCK_BTM) { + lck.id = lc_id.proc_lock_btm; + erts_lc_unlock(&lck); + } if (locks & ERTS_PROC_LOCK_MSGQ) { lck.id = lc_id.proc_lock_msgq; erts_lc_unlock(&lck); @@ -1294,6 +1296,10 @@ erts_proc_lc_might_unlock(Process *p, ErtsProcLocks locks) lck.id = lc_id.proc_lock_status; erts_lc_might_unlock(&lck); } + if (locks & ERTS_PROC_LOCK_BTM) { + lck.id = lc_id.proc_lock_btm; + erts_lc_might_unlock(&lck); + } if (locks & ERTS_PROC_LOCK_MSGQ) { lck.id = lc_id.proc_lock_msgq; erts_lc_might_unlock(&lck); @@ -1313,13 +1319,16 @@ erts_proc_lc_might_unlock(Process *p, ErtsProcLocks locks) erts_lc_might_unlock(&p->lock.link.lc); if (locks & ERTS_PROC_LOCK_MSGQ) erts_lc_might_unlock(&p->lock.msgq.lc); + if (locks & ERTS_PROC_LOCK_BTM) + erts_lc_might_unlock(&p->lock.btm.lc); if (locks & ERTS_PROC_LOCK_STATUS) erts_lc_might_unlock(&p->lock.status.lc); #endif } void -erts_proc_lc_require_lock(Process *p, ErtsProcLocks locks) +erts_proc_lc_require_lock(Process *p, ErtsProcLocks locks, char *file, + unsigned int line) { #if ERTS_PROC_LOCK_OWN_IMPL erts_lc_lock_t lck = ERTS_LC_LOCK_INIT(-1, @@ -1327,29 +1336,35 @@ erts_proc_lc_require_lock(Process *p, ErtsProcLocks locks) ERTS_LC_FLG_LT_PROCLOCK); if (locks & ERTS_PROC_LOCK_MAIN) { lck.id = lc_id.proc_lock_main; - erts_lc_require_lock(&lck); + erts_lc_require_lock(&lck, file, line); } if (locks & ERTS_PROC_LOCK_LINK) { lck.id = lc_id.proc_lock_link; - erts_lc_require_lock(&lck); + erts_lc_require_lock(&lck, file, line); } if (locks & ERTS_PROC_LOCK_MSGQ) { lck.id = lc_id.proc_lock_msgq; - erts_lc_require_lock(&lck); + erts_lc_require_lock(&lck, file, line); + } + if (locks & ERTS_PROC_LOCK_BTM) { + lck.id = lc_id.proc_lock_btm; + erts_lc_require_lock(&lck, file, line); } if (locks & ERTS_PROC_LOCK_STATUS) { lck.id = lc_id.proc_lock_status; - erts_lc_require_lock(&lck); + erts_lc_require_lock(&lck, file, line); } #elif ERTS_PROC_LOCK_RAW_MUTEX_IMPL if (locks & ERTS_PROC_LOCK_MAIN) - erts_lc_require_lock(&p->lock.main.lc); + erts_lc_require_lock(&p->lock.main.lc, file, line); if (locks & ERTS_PROC_LOCK_LINK) - erts_lc_require_lock(&p->lock.link.lc); + erts_lc_require_lock(&p->lock.link.lc, file, line); if (locks & ERTS_PROC_LOCK_MSGQ) - erts_lc_require_lock(&p->lock.msgq.lc); + erts_lc_require_lock(&p->lock.msgq.lc, file, line); + if (locks & ERTS_PROC_LOCK_BTM) + erts_lc_require_lock(&p->lock.btm.lc, file, line); if (locks & ERTS_PROC_LOCK_STATUS) - erts_lc_require_lock(&p->lock.status.lc); + erts_lc_require_lock(&p->lock.status.lc, file, line); #endif } @@ -1364,6 +1379,10 @@ erts_proc_lc_unrequire_lock(Process *p, ErtsProcLocks locks) lck.id = lc_id.proc_lock_status; erts_lc_unrequire_lock(&lck); } + if (locks & ERTS_PROC_LOCK_BTM) { + lck.id = lc_id.proc_lock_btm; + erts_lc_unrequire_lock(&lck); + } if (locks & ERTS_PROC_LOCK_MSGQ) { lck.id = lc_id.proc_lock_msgq; erts_lc_unrequire_lock(&lck); @@ -1383,6 +1402,8 @@ erts_proc_lc_unrequire_lock(Process *p, ErtsProcLocks locks) erts_lc_unrequire_lock(&p->lock.link.lc); if (locks & ERTS_PROC_LOCK_MSGQ) erts_lc_unrequire_lock(&p->lock.msgq.lc); + if (locks & ERTS_PROC_LOCK_BTM) + erts_lc_unrequire_lock(&p->lock.btm.lc); if (locks & ERTS_PROC_LOCK_STATUS) erts_lc_unrequire_lock(&p->lock.status.lc); #endif @@ -1404,6 +1425,8 @@ erts_proc_lc_trylock_force_busy(Process *p, ErtsProcLocks locks) lck.id = lc_id.proc_lock_link; else if (locks & ERTS_PROC_LOCK_MSGQ) lck.id = lc_id.proc_lock_msgq; + else if (locks & ERTS_PROC_LOCK_BTM) + lck.id = lc_id.proc_lock_btm; else if (locks & ERTS_PROC_LOCK_STATUS) lck.id = lc_id.proc_lock_status; else @@ -1438,7 +1461,8 @@ erts_proc_lc_chk_have_proc_locks(Process *p, ErtsProcLocks locks) { int have_locks_len = 0; #if ERTS_PROC_LOCK_OWN_IMPL - erts_lc_lock_t have_locks[4] = {ERTS_PROC_LC_EMPTY_LOCK_INIT, + erts_lc_lock_t have_locks[5] = {ERTS_PROC_LC_EMPTY_LOCK_INIT, + ERTS_PROC_LC_EMPTY_LOCK_INIT, ERTS_PROC_LC_EMPTY_LOCK_INIT, ERTS_PROC_LC_EMPTY_LOCK_INIT, ERTS_PROC_LC_EMPTY_LOCK_INIT}; @@ -1454,18 +1478,24 @@ erts_proc_lc_chk_have_proc_locks(Process *p, ErtsProcLocks locks) have_locks[have_locks_len].id = lc_id.proc_lock_msgq; have_locks[have_locks_len++].extra = p->common.id; } + if (locks & ERTS_PROC_LOCK_BTM) { + have_locks[have_locks_len].id = lc_id.proc_lock_btm; + have_locks[have_locks_len++].extra = p->common.id; + } if (locks & ERTS_PROC_LOCK_STATUS) { have_locks[have_locks_len].id = lc_id.proc_lock_status; have_locks[have_locks_len++].extra = p->common.id; } #elif ERTS_PROC_LOCK_RAW_MUTEX_IMPL - erts_lc_lock_t have_locks[4]; + erts_lc_lock_t have_locks[5]; if (locks & ERTS_PROC_LOCK_MAIN) have_locks[have_locks_len++] = p->lock.main.lc; if (locks & ERTS_PROC_LOCK_LINK) have_locks[have_locks_len++] = p->lock.link.lc; if (locks & ERTS_PROC_LOCK_MSGQ) have_locks[have_locks_len++] = p->lock.msgq.lc; + if (locks & ERTS_PROC_LOCK_BTM) + have_locks[have_locks_len++] = p->lock.btm.lc; if (locks & ERTS_PROC_LOCK_STATUS) have_locks[have_locks_len++] = p->lock.status.lc; #endif @@ -1478,11 +1508,11 @@ erts_proc_lc_chk_proc_locks(Process *p, ErtsProcLocks locks) int have_locks_len = 0; int have_not_locks_len = 0; #if ERTS_PROC_LOCK_OWN_IMPL - erts_lc_lock_t have_locks[4] = {ERTS_PROC_LC_EMPTY_LOCK_INIT, + erts_lc_lock_t have_locks[5] = {ERTS_PROC_LC_EMPTY_LOCK_INIT, ERTS_PROC_LC_EMPTY_LOCK_INIT, ERTS_PROC_LC_EMPTY_LOCK_INIT, ERTS_PROC_LC_EMPTY_LOCK_INIT}; - erts_lc_lock_t have_not_locks[4] = {ERTS_PROC_LC_EMPTY_LOCK_INIT, + erts_lc_lock_t have_not_locks[5] = {ERTS_PROC_LC_EMPTY_LOCK_INIT, ERTS_PROC_LC_EMPTY_LOCK_INIT, ERTS_PROC_LC_EMPTY_LOCK_INIT, ERTS_PROC_LC_EMPTY_LOCK_INIT}; @@ -1511,6 +1541,14 @@ erts_proc_lc_chk_proc_locks(Process *p, ErtsProcLocks locks) have_not_locks[have_not_locks_len].id = lc_id.proc_lock_msgq; have_not_locks[have_not_locks_len++].extra = p->common.id; } + if (locks & ERTS_PROC_LOCK_BTM) { + have_locks[have_locks_len].id = lc_id.proc_lock_btm; + have_locks[have_locks_len++].extra = p->common.id; + } + else { + have_not_locks[have_not_locks_len].id = lc_id.proc_lock_btm; + have_not_locks[have_not_locks_len++].extra = p->common.id; + } if (locks & ERTS_PROC_LOCK_STATUS) { have_locks[have_locks_len].id = lc_id.proc_lock_status; have_locks[have_locks_len++].extra = p->common.id; @@ -1520,8 +1558,8 @@ erts_proc_lc_chk_proc_locks(Process *p, ErtsProcLocks locks) have_not_locks[have_not_locks_len++].extra = p->common.id; } #elif ERTS_PROC_LOCK_RAW_MUTEX_IMPL - erts_lc_lock_t have_locks[4]; - erts_lc_lock_t have_not_locks[4]; + erts_lc_lock_t have_locks[5]; + erts_lc_lock_t have_not_locks[5]; if (locks & ERTS_PROC_LOCK_MAIN) have_locks[have_locks_len++] = p->lock.main.lc; @@ -1535,6 +1573,10 @@ erts_proc_lc_chk_proc_locks(Process *p, ErtsProcLocks locks) have_locks[have_locks_len++] = p->lock.msgq.lc; else have_not_locks[have_not_locks_len++] = p->lock.msgq.lc; + if (locks & ERTS_PROC_LOCK_BTM) + have_locks[have_locks_len++] = p->lock.btm.lc; + else + have_not_locks[have_not_locks_len++] = p->lock.btm.lc; if (locks & ERTS_PROC_LOCK_STATUS) have_locks[have_locks_len++] = p->lock.status.lc; else @@ -1548,10 +1590,10 @@ erts_proc_lc_chk_proc_locks(Process *p, ErtsProcLocks locks) ErtsProcLocks erts_proc_lc_my_proc_locks(Process *p) { - int resv[4]; + int resv[5]; ErtsProcLocks res = 0; #if ERTS_PROC_LOCK_OWN_IMPL - erts_lc_lock_t locks[4] = {ERTS_LC_LOCK_INIT(lc_id.proc_lock_main, + erts_lc_lock_t locks[5] = {ERTS_LC_LOCK_INIT(lc_id.proc_lock_main, p->common.id, ERTS_LC_FLG_LT_PROCLOCK), ERTS_LC_LOCK_INIT(lc_id.proc_lock_link, @@ -1560,17 +1602,21 @@ erts_proc_lc_my_proc_locks(Process *p) ERTS_LC_LOCK_INIT(lc_id.proc_lock_msgq, p->common.id, ERTS_LC_FLG_LT_PROCLOCK), + ERTS_LC_LOCK_INIT(lc_id.proc_lock_btm, + p->common.id, + ERTS_LC_FLG_LT_PROCLOCK), ERTS_LC_LOCK_INIT(lc_id.proc_lock_status, p->common.id, ERTS_LC_FLG_LT_PROCLOCK)}; #elif ERTS_PROC_LOCK_RAW_MUTEX_IMPL - erts_lc_lock_t locks[4] = {p->lock.main.lc, + erts_lc_lock_t locks[5] = {p->lock.main.lc, p->lock.link.lc, p->lock.msgq.lc, + p->lock.btm.lc, p->lock.status.lc}; #endif - erts_lc_have_locks(resv, locks, 4); + erts_lc_have_locks(resv, locks, 5); if (resv[0]) res |= ERTS_PROC_LOCK_MAIN; if (resv[1]) @@ -1578,6 +1624,8 @@ erts_proc_lc_my_proc_locks(Process *p) if (resv[2]) res |= ERTS_PROC_LOCK_MSGQ; if (resv[3]) + res |= ERTS_PROC_LOCK_BTM; + if (resv[4]) res |= ERTS_PROC_LOCK_STATUS; return res; @@ -1586,13 +1634,14 @@ erts_proc_lc_my_proc_locks(Process *p) void erts_proc_lc_chk_no_proc_locks(char *file, int line) { - int resv[4]; - int ids[4] = {lc_id.proc_lock_main, + int resv[5]; + int ids[5] = {lc_id.proc_lock_main, lc_id.proc_lock_link, lc_id.proc_lock_msgq, + lc_id.proc_lock_btm, lc_id.proc_lock_status}; - erts_lc_have_lock_ids(resv, ids, 4); - if (!ERTS_IS_CRASH_DUMPING && (resv[0] || resv[1] || resv[2] || resv[3])) { + erts_lc_have_lock_ids(resv, ids, 5); + if (!ERTS_IS_CRASH_DUMPING && (resv[0] || resv[1] || resv[2] || resv[3] || resv[4])) { erts_lc_fail("%s:%d: Thread has process locks locked when expected " "not to have any process locks locked", file, line); diff --git a/erts/emulator/beam/erl_process_lock.h b/erts/emulator/beam/erl_process_lock.h index 9dd503f3cb..9c59301086 100644 --- a/erts/emulator/beam/erl_process_lock.h +++ b/erts/emulator/beam/erl_process_lock.h @@ -3,16 +3,17 @@ * * Copyright Ericsson AB 2007-2012. All Rights Reserved. * - * The contents of this file are subject to the Erlang Public License, - * Version 1.1, (the "License"); you may not use this file except in - * compliance with the License. You should have received a copy of the - * Erlang Public License along with this software. If not, it can be - * retrieved online at http://www.erlang.org/. - * - * Software distributed under the License is distributed on an "AS IS" - * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See - * the License for the specific language governing rights and limitations - * under the License. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. * * %CopyrightEnd% */ @@ -65,7 +66,7 @@ #endif -#define ERTS_PROC_LOCK_MAX_BIT 3 +#define ERTS_PROC_LOCK_MAX_BIT 4 typedef erts_aint32_t ErtsProcLocks; @@ -81,17 +82,18 @@ typedef struct erts_proc_lock_t_ { erts_lcnt_lock_t lcnt_main; erts_lcnt_lock_t lcnt_link; erts_lcnt_lock_t lcnt_msgq; + erts_lcnt_lock_t lcnt_btm; erts_lcnt_lock_t lcnt_status; #endif #elif ERTS_PROC_LOCK_RAW_MUTEX_IMPL erts_mtx_t main; erts_mtx_t link; erts_mtx_t msgq; + erts_mtx_t btm; erts_mtx_t status; #else # error "no implementation" #endif - erts_atomic32_t refc; #ifdef ERTS_PROC_LOCK_DEBUG erts_smp_atomic32_t locked[ERTS_PROC_LOCK_MAX_BIT+1]; #endif @@ -120,11 +122,17 @@ typedef struct erts_proc_lock_t_ { * Message queue lock: * Protects the following fields in the process structure: * * msg_inq - * * bif_timers */ #define ERTS_PROC_LOCK_MSGQ (((ErtsProcLocks) 1) << 2) /* + * Bif timer lock: + * Protects the following fields in the process structure: + * * bif_timers + */ +#define ERTS_PROC_LOCK_BTM (((ErtsProcLocks) 1) << 3) + +/* * Status lock: * Protects the following fields in the process structure: * * pending_suspenders @@ -215,7 +223,7 @@ typedef struct erts_proc_lock_t_ { /* Lock counter implemetation */ -#ifdef ERTS_ENABLE_LOCK_COUNT +#ifdef ERTS_ENABLE_LOCK_POSITION #define erts_smp_proc_lock__(P,I,L) erts_smp_proc_lock_x__(P,I,L,__FILE__,__LINE__) #define erts_smp_proc_lock(P,L) erts_smp_proc_lock_x(P,L,__FILE__,__LINE__) #endif @@ -243,8 +251,10 @@ void erts_lcnt_enable_proc_lock_count(int enable); erts_proc_lc_chk_no_proc_locks(__FILE__, __LINE__) #define ERTS_SMP_CHK_HAVE_ONLY_MAIN_PROC_LOCK(P) \ erts_proc_lc_chk_only_proc_main((P)) -void erts_proc_lc_lock(Process *p, ErtsProcLocks locks); -void erts_proc_lc_trylock(Process *p, ErtsProcLocks locks, int locked); +void erts_proc_lc_lock(Process *p, ErtsProcLocks locks, + char *file, unsigned int line); +void erts_proc_lc_trylock(Process *p, ErtsProcLocks locks, int locked, + char *file, unsigned int line); void erts_proc_lc_unlock(Process *p, ErtsProcLocks locks); void erts_proc_lc_might_unlock(Process *p, ErtsProcLocks locks); void erts_proc_lc_chk_have_proc_locks(Process *p, ErtsProcLocks locks); @@ -253,7 +263,8 @@ void erts_proc_lc_chk_only_proc_main(Process *p); void erts_proc_lc_chk_no_proc_locks(char *file, int line); ErtsProcLocks erts_proc_lc_my_proc_locks(Process *p); int erts_proc_lc_trylock_force_busy(Process *p, ErtsProcLocks locks); -void erts_proc_lc_require_lock(Process *p, ErtsProcLocks locks); +void erts_proc_lc_require_lock(Process *p, ErtsProcLocks locks, + char* file, unsigned int line); void erts_proc_lc_unrequire_lock(Process *p, ErtsProcLocks locks); #else #define ERTS_SMP_CHK_NO_PROC_LOCKS @@ -372,7 +383,7 @@ ERTS_GLB_INLINE int erts_lc_pix_lock_is_locked(erts_pix_lock_t *); ERTS_GLB_INLINE ErtsProcLocks erts_smp_proc_raw_trylock__(Process *p, ErtsProcLocks locks); -#ifdef ERTS_ENABLE_LOCK_COUNT +#ifdef ERTS_ENABLE_LOCK_POSITION ERTS_GLB_INLINE void erts_smp_proc_lock_x__(Process *, erts_pix_lock_t *, ErtsProcLocks, @@ -460,6 +471,9 @@ erts_smp_proc_raw_trylock__(Process *p, ErtsProcLocks locks) if (locks & ERTS_PROC_LOCK_MSGQ) if (erts_mtx_trylock(&p->lock.msgq) == EBUSY) goto busy_msgq; + if (locks & ERTS_PROC_LOCK_BTM) + if (erts_mtx_trylock(&p->lock.btm) == EBUSY) + goto busy_btm; if (locks & ERTS_PROC_LOCK_STATUS) if (erts_mtx_trylock(&p->lock.status) == EBUSY) goto busy_status; @@ -467,6 +481,9 @@ erts_smp_proc_raw_trylock__(Process *p, ErtsProcLocks locks) return 0; busy_status: + if (locks & ERTS_PROC_LOCK_BTM) + erts_mtx_unlock(&p->lock.btm); +busy_btm: if (locks & ERTS_PROC_LOCK_MSGQ) erts_mtx_unlock(&p->lock.msgq); busy_msgq: @@ -482,7 +499,7 @@ busy_main: } ERTS_GLB_INLINE void -#ifdef ERTS_ENABLE_LOCK_COUNT +#ifdef ERTS_ENABLE_LOCK_POSITION erts_smp_proc_lock_x__(Process *p, erts_pix_lock_t *pix_lck, ErtsProcLocks locks, @@ -506,6 +523,10 @@ erts_smp_proc_lock__(Process *p, ERTS_LC_ASSERT((locks & ~ERTS_PROC_LOCKS_ALL) == 0); +#ifdef ERTS_ENABLE_LOCK_CHECK + erts_proc_lc_lock(p, locks, file, line); +#endif + old_lflgs = erts_smp_proc_raw_trylock__(p, locks); if (old_lflgs != 0) { @@ -527,9 +548,6 @@ erts_smp_proc_lock__(Process *p, #ifdef ERTS_ENABLE_LOCK_COUNT erts_lcnt_proc_lock_post_x(&(p->lock), locks, file, line); #endif -#ifdef ERTS_ENABLE_LOCK_CHECK - erts_proc_lc_lock(p, locks); -#endif #ifdef ERTS_PROC_LOCK_DEBUG erts_proc_lock_op_debug(p, locks, 1); @@ -546,6 +564,8 @@ erts_smp_proc_lock__(Process *p, erts_mtx_lock(&p->lock.link); if (locks & ERTS_PROC_LOCK_MSGQ) erts_mtx_lock(&p->lock.msgq); + if (locks & ERTS_PROC_LOCK_BTM) + erts_mtx_lock(&p->lock.btm); if (locks & ERTS_PROC_LOCK_STATUS) erts_mtx_lock(&p->lock.status); @@ -635,6 +655,8 @@ erts_smp_proc_unlock__(Process *p, if (locks & ERTS_PROC_LOCK_STATUS) erts_mtx_unlock(&p->lock.status); + if (locks & ERTS_PROC_LOCK_BTM) + erts_mtx_unlock(&p->lock.btm); if (locks & ERTS_PROC_LOCK_MSGQ) erts_mtx_unlock(&p->lock.msgq); if (locks & ERTS_PROC_LOCK_LINK) @@ -695,7 +717,7 @@ erts_smp_proc_trylock__(Process *p, #endif #ifdef ERTS_ENABLE_LOCK_CHECK - erts_proc_lc_trylock(p, locks, res == 0); + erts_proc_lc_trylock(p, locks, res == 0, __FILE__, __LINE__); #endif #if ERTS_PROC_LOCK_ATOMIC_IMPL @@ -741,7 +763,7 @@ erts_proc_lock_op_debug(Process *p, ErtsProcLocks locks, int locked) #endif /* ERTS_SMP */ -#ifdef ERTS_ENABLE_LOCK_COUNT +#ifdef ERTS_ENABLE_LOCK_POSITION ERTS_GLB_INLINE void erts_smp_proc_lock_x(Process *, ErtsProcLocks, char *file, unsigned int line); #else ERTS_GLB_INLINE void erts_smp_proc_lock(Process *, ErtsProcLocks); @@ -749,20 +771,21 @@ ERTS_GLB_INLINE void erts_smp_proc_lock(Process *, ErtsProcLocks); ERTS_GLB_INLINE void erts_smp_proc_unlock(Process *, ErtsProcLocks); ERTS_GLB_INLINE int erts_smp_proc_trylock(Process *, ErtsProcLocks); -ERTS_GLB_INLINE void erts_smp_proc_inc_refc(Process *); -ERTS_GLB_INLINE void erts_smp_proc_dec_refc(Process *); -ERTS_GLB_INLINE void erts_smp_proc_add_refc(Process *, Sint32); +ERTS_GLB_INLINE void erts_proc_inc_refc(Process *); +ERTS_GLB_INLINE void erts_proc_dec_refc(Process *); +ERTS_GLB_INLINE void erts_proc_add_refc(Process *, Sint); +ERTS_GLB_INLINE Sint erts_proc_read_refc(Process *); #if ERTS_GLB_INLINE_INCL_FUNC_DEF ERTS_GLB_INLINE void -#ifdef ERTS_ENABLE_LOCK_COUNT +#ifdef ERTS_ENABLE_LOCK_POSITION erts_smp_proc_lock_x(Process *p, ErtsProcLocks locks, char *file, unsigned int line) #else erts_smp_proc_lock(Process *p, ErtsProcLocks locks) #endif { -#if defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_COUNT) +#if defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_POSITION) erts_smp_proc_lock_x__(p, #if ERTS_PROC_LOCK_ATOMIC_IMPL NULL, @@ -811,28 +834,53 @@ erts_smp_proc_trylock(Process *p, ErtsProcLocks locks) #endif } -ERTS_GLB_INLINE void erts_smp_proc_inc_refc(Process *p) +ERTS_GLB_INLINE void erts_proc_inc_refc(Process *p) { + ASSERT(!(erts_smp_atomic32_read_nob(&p->state) & ERTS_PSFLG_PROXY)); #ifdef ERTS_SMP + erts_ptab_atmc_inc_refc(&p->common); +#else erts_ptab_inc_refc(&p->common); #endif } -ERTS_GLB_INLINE void erts_smp_proc_dec_refc(Process *p) +ERTS_GLB_INLINE void erts_proc_dec_refc(Process *p) { + Sint referred; + ASSERT(!(erts_smp_atomic32_read_nob(&p->state) & ERTS_PSFLG_PROXY)); #ifdef ERTS_SMP - int referred = erts_ptab_dec_test_refc(&p->common); - if (!referred) - erts_free_proc(p); + referred = erts_ptab_atmc_dec_test_refc(&p->common); +#else + referred = erts_ptab_dec_test_refc(&p->common); #endif + if (!referred) { + ASSERT(ERTS_PROC_IS_EXITING(p)); + erts_free_proc(p); + } } -ERTS_GLB_INLINE void erts_smp_proc_add_refc(Process *p, Sint32 add_refc) +ERTS_GLB_INLINE void erts_proc_add_refc(Process *p, Sint add_refc) { + Sint referred; + ASSERT(!(erts_smp_atomic32_read_nob(&p->state) & ERTS_PSFLG_PROXY)); #ifdef ERTS_SMP - int referred = erts_ptab_add_test_refc(&p->common, add_refc); - if (!referred) + referred = erts_ptab_atmc_add_test_refc(&p->common, add_refc); +#else + referred = erts_ptab_add_test_refc(&p->common, add_refc); +#endif + if (!referred) { + ASSERT(ERTS_PROC_IS_EXITING(p)); erts_free_proc(p); + } +} + +ERTS_GLB_INLINE Sint erts_proc_read_refc(Process *p) +{ + ASSERT(!(erts_smp_atomic32_read_nob(&p->state) & ERTS_PSFLG_PROXY)); +#ifdef ERTS_SMP + return erts_ptab_atmc_read_refc(&p->common); +#else + return erts_ptab_read_refc(&p->common); #endif } @@ -865,7 +913,7 @@ void erts_proc_safelock(Process *a_proc, #define ERTS_P2P_FLG_ALLOW_OTHER_X (1 << 0) #define ERTS_P2P_FLG_TRY_LOCK (1 << 1) -#define ERTS_P2P_FLG_SMP_INC_REFC (1 << 2) +#define ERTS_P2P_FLG_INC_REFC (1 << 2) #define ERTS_PROC_LOCK_BUSY ((Process *) &erts_invalid_process) @@ -925,11 +973,14 @@ erts_pid2proc_opt(Process *c_p_unused, int flags) { Process *proc = erts_proc_lookup_raw(pid); - return ((!(flags & ERTS_P2P_FLG_ALLOW_OTHER_X) - && proc - && ERTS_PROC_IS_EXITING(proc)) - ? NULL - : proc); + if (!proc) + return NULL; + if (!(flags & ERTS_P2P_FLG_ALLOW_OTHER_X) + && ERTS_PROC_IS_EXITING(proc)) + return NULL; + if (flags & ERTS_P2P_FLG_INC_REFC) + erts_proc_inc_refc(proc); + return proc; } #endif /* !ERTS_SMP */ diff --git a/erts/emulator/beam/erl_ptab.c b/erts/emulator/beam/erl_ptab.c index 87beeafa1a..9ed175fe01 100644 --- a/erts/emulator/beam/erl_ptab.c +++ b/erts/emulator/beam/erl_ptab.c @@ -1,18 +1,19 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2012. All Rights Reserved. + * Copyright Ericsson AB 2012-2013. All Rights Reserved. * - * The contents of this file are subject to the Erlang Public License, - * Version 1.1, (the "License"); you may not use this file except in - * compliance with the License. You should have received a copy of the - * Erlang Public License along with this software. If not, it can be - * retrieved online at http://www.erlang.org/. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Software distributed under the License is distributed on an "AS IS" - * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See - * the License for the specific language governing rights and limitations - * under the License. + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. * * %CopyrightEnd% */ @@ -34,6 +35,8 @@ typedef struct ErtsPTabListBifData_ ErtsPTabListBifData; +#define ERTS_PTAB_NEW_MAX_RESERVE_FAIL 1000 + #define ERTS_PTAB_LIST_BIF_TAB_INSPECT_INDICES_PER_RED 25 #define ERTS_PTAB_LIST_BIF_TAB_CHUNK_SIZE 1000 #define ERTS_PTAB_LIST_BIF_MIN_START_REDS \ @@ -278,124 +281,38 @@ struct ErtsPTabListBifData_ { }; -#ifdef ARCH_32 - -static ERTS_INLINE Uint64 -dw_aint_to_uint64(erts_dw_aint_t *dw) -{ -#ifdef ETHR_SU_DW_NAINT_T__ - return (Uint64) dw->dw_sint; -#else - Uint64 res; - res = (Uint64) ((Uint32) dw->sint[ERTS_DW_AINT_HIGH_WORD]); - res <<= 32; - res |= (Uint64) ((Uint32) dw->sint[ERTS_DW_AINT_LOW_WORD]); - return res; -#endif -} - -static void -unint64_to_dw_aint(erts_dw_aint_t *dw, Uint64 val) -{ -#ifdef ETHR_SU_DW_NAINT_T__ - dw->dw_sint = (ETHR_SU_DW_NAINT_T__) val; -#else - dw->sint[ERTS_DW_AINT_LOW_WORD] = (erts_aint_t) (val & 0xffffffff); - dw->sint[ERTS_DW_AINT_HIGH_WORD] = (erts_aint_t) ((val >> 32) & 0xffffffff); -#endif -} - -static ERTS_INLINE void -last_data_init_nob(ErtsPTab *ptab, Uint64 val) -{ - erts_dw_aint_t dw; - unint64_to_dw_aint(&dw, val); - erts_smp_dw_atomic_init_nob(&ptab->vola.tile.last_data, &dw); -} - -static ERTS_INLINE void -last_data_set_relb(ErtsPTab *ptab, Uint64 val) -{ - erts_dw_aint_t dw; - unint64_to_dw_aint(&dw, val); - erts_smp_dw_atomic_set_relb(&ptab->vola.tile.last_data, &dw); -} - -static ERTS_INLINE Uint64 -last_data_read_nob(ErtsPTab *ptab) -{ - erts_dw_aint_t dw; - erts_smp_dw_atomic_read_nob(&ptab->vola.tile.last_data, &dw); - return dw_aint_to_uint64(&dw); -} - -static ERTS_INLINE Uint64 -last_data_read_acqb(ErtsPTab *ptab) -{ - erts_dw_aint_t dw; - erts_smp_dw_atomic_read_acqb(&ptab->vola.tile.last_data, &dw); - return dw_aint_to_uint64(&dw); -} - -static ERTS_INLINE Uint64 -last_data_cmpxchg_relb(ErtsPTab *ptab, Uint64 new, Uint64 exp) -{ - erts_dw_aint_t dw_new, dw_xchg; - - unint64_to_dw_aint(&dw_new, new); - unint64_to_dw_aint(&dw_xchg, exp); - - if (erts_smp_dw_atomic_cmpxchg_relb(&ptab->vola.tile.last_data, - &dw_new, - &dw_xchg)) - return exp; - else - return dw_aint_to_uint64(&dw_xchg); -} - -#elif defined(ARCH_64) - -union { - erts_smp_atomic_t pid_data; - char align[ERTS_CACHE_LINE_SIZE]; -} last erts_align_attribute(ERTS_CACHE_LINE_SIZE); - static ERTS_INLINE void last_data_init_nob(ErtsPTab *ptab, Uint64 val) { - erts_smp_atomic_init_nob(&ptab->vola.tile.last_data, (erts_aint_t) val); + erts_smp_atomic64_init_nob(&ptab->vola.tile.last_data, (erts_aint64_t) val); } static ERTS_INLINE void last_data_set_relb(ErtsPTab *ptab, Uint64 val) { - erts_smp_atomic_set_relb(&ptab->vola.tile.last_data, (erts_aint_t) val); + erts_smp_atomic64_set_relb(&ptab->vola.tile.last_data, (erts_aint64_t) val); } static ERTS_INLINE Uint64 last_data_read_nob(ErtsPTab *ptab) { - return (Uint64) erts_smp_atomic_read_nob(&ptab->vola.tile.last_data); + return (Uint64) erts_smp_atomic64_read_nob(&ptab->vola.tile.last_data); } static ERTS_INLINE Uint64 last_data_read_acqb(ErtsPTab *ptab) { - return (Uint64) erts_smp_atomic_read_acqb(&ptab->vola.tile.last_data); + return (Uint64) erts_smp_atomic64_read_acqb(&ptab->vola.tile.last_data); } static ERTS_INLINE Uint64 last_data_cmpxchg_relb(ErtsPTab *ptab, Uint64 new, Uint64 exp) { - return (Uint64) erts_smp_atomic_cmpxchg_relb(&ptab->vola.tile.last_data, - (erts_aint_t) new, - (erts_aint_t) exp); + return (Uint64) erts_smp_atomic64_cmpxchg_relb(&ptab->vola.tile.last_data, + (erts_aint64_t) new, + (erts_aint64_t) exp); } -#else -# error "Not 64-bit, nor 32-bit architecture..." -#endif - static ERTS_INLINE int last_data_cmp(Uint64 ld1, Uint64 ld2) { @@ -415,16 +332,40 @@ last_data_cmp(Uint64 ld1, Uint64 ld2) #define ERTS_PTAB_LastData2EtermData(LD) \ ((Eterm) ((LD) & ~(~((Uint64) 0) << ERTS_PTAB_ID_DATA_SIZE))) +static ERTS_INLINE Uint32 +ix_to_free_id_data_ix(ErtsPTab *ptab, Uint32 ix) +{ + Uint32 dix; + + dix = ((ix & ptab->r.o.dix_cl_mask) << ptab->r.o.dix_cl_shift); + dix += ((ix >> ptab->r.o.dix_cli_shift) & ptab->r.o.dix_cli_mask); + ASSERT(0 <= dix && dix < ptab->r.o.max); + return dix; +} + +UWord +erts_ptab_mem_size(ErtsPTab *ptab) +{ + UWord size = ptab->r.o.max*sizeof(erts_smp_atomic_t); + if (ptab->r.o.free_id_data) + size += ptab->r.o.max*sizeof(erts_smp_atomic32_t); + return size; +} + + void erts_ptab_init_table(ErtsPTab *ptab, ErtsAlcType_t atype, void (*release_element)(void *), ErtsPTabElementCommon *invalid_element, int size, - char *name) + UWord element_size, + char *name, + int legacy, + int atomic_refc) { - size_t tab_sz; - int bits; + size_t tab_sz, alloc_sz; + Uint32 bits, cl, cli, ix, ix_per_cache_line, tab_cache_lines; char *tab_end; erts_smp_atomic_t *tab_entry; erts_smp_rwmtx_opt_t rwmtx_opts = ERTS_SMP_RWMTX_OPT_DEFAULT_INITER; @@ -443,10 +384,14 @@ erts_ptab_init_table(ErtsPTab *ptab, bits = erts_fit_in_bits_int32((Sint32) size - 1); } + ptab->r.o.element_size = element_size; ptab->r.o.max = size; tab_sz = ERTS_ALC_CACHE_LINE_ALIGN_SIZE(size*sizeof(erts_smp_atomic_t)); - ptab->r.o.tab = erts_alloc_permanent_cache_aligned(atype, tab_sz); + alloc_sz = tab_sz; + if (!legacy) + alloc_sz += ERTS_ALC_CACHE_LINE_ALIGN_SIZE(size*sizeof(erts_smp_atomic32_t)); + ptab->r.o.tab = erts_alloc_permanent_cache_aligned(atype, alloc_sz); tab_end = ((char *) ptab->r.o.tab) + tab_sz; tab_entry = ptab->r.o.tab; while (tab_end > ((char *) tab_entry)) { @@ -454,25 +399,17 @@ erts_ptab_init_table(ErtsPTab *ptab, tab_entry++; } - ptab->r.o.tab_cache_lines = tab_sz/ERTS_CACHE_LINE_SIZE; - ptab->r.o.pix_per_cache_line = (ERTS_CACHE_LINE_SIZE - / sizeof(erts_smp_atomic_t)); + tab_cache_lines = tab_sz/ERTS_CACHE_LINE_SIZE; + ix_per_cache_line = (ERTS_CACHE_LINE_SIZE/sizeof(erts_smp_atomic_t)); ASSERT((ptab->r.o.max & (ptab->r.o.max - 1)) == 0); /* power of 2 */ - ASSERT((ptab->r.o.pix_per_cache_line - & (ptab->r.o.pix_per_cache_line - 1)) == 0); /* power of 2 */ - ASSERT((ptab->r.o.tab_cache_lines - & (ptab->r.o.tab_cache_lines - 1)) == 0); /* power of 2 */ - - ptab->r.o.pix_mask - = (1 << bits) - 1; - ptab->r.o.pix_cl_mask - = ptab->r.o.tab_cache_lines-1; - ptab->r.o.pix_cl_shift - = erts_fit_in_bits_int32(ptab->r.o.pix_per_cache_line-1); - ptab->r.o.pix_cli_shift - = erts_fit_in_bits_int32(ptab->r.o.pix_cl_mask); - ptab->r.o.pix_cli_mask - = (1 << (bits - ptab->r.o.pix_cli_shift)) - 1; + ASSERT((ix_per_cache_line & (ix_per_cache_line - 1)) == 0); /* power of 2 */ + ASSERT((tab_cache_lines & (tab_cache_lines - 1)) == 0); /* power of 2 */ + + ptab->r.o.pix_mask = (1 << bits) - 1; + ptab->r.o.pix_cl_mask = tab_cache_lines-1; + ptab->r.o.pix_cl_shift = erts_fit_in_bits_int32(ix_per_cache_line-1); + ptab->r.o.pix_cli_shift = erts_fit_in_bits_int32(ptab->r.o.pix_cl_mask); + ptab->r.o.pix_cli_mask = (1 << (bits - ptab->r.o.pix_cli_shift)) - 1; ASSERT(ptab->r.o.pix_cl_shift + ptab->r.o.pix_cli_shift == bits); @@ -480,6 +417,48 @@ erts_ptab_init_table(ErtsPTab *ptab, ptab->r.o.invalid_data = erts_ptab_id2data(ptab, invalid_element->id); ptab->r.o.release_element = release_element; + ptab->r.o.atomic_refc = atomic_refc; + + if (legacy) { + ptab->r.o.free_id_data = NULL; + ptab->r.o.dix_cl_mask = 0; + ptab->r.o.dix_cl_shift = 0; + ptab->r.o.dix_cli_shift = 0; + ptab->r.o.dix_cli_mask = 0; + } + else { + + tab_sz = ERTS_ALC_CACHE_LINE_ALIGN_SIZE(size*sizeof(erts_smp_atomic32_t)); + ptab->r.o.free_id_data = (erts_smp_atomic32_t *) tab_end; + + tab_cache_lines = tab_sz/ERTS_CACHE_LINE_SIZE; + ix_per_cache_line = (ERTS_CACHE_LINE_SIZE/sizeof(erts_smp_atomic32_t)); + + ptab->r.o.dix_cl_mask = tab_cache_lines-1; + ptab->r.o.dix_cl_shift = erts_fit_in_bits_int32(ix_per_cache_line-1); + ptab->r.o.dix_cli_shift = erts_fit_in_bits_int32(ptab->r.o.dix_cl_mask); + ptab->r.o.dix_cli_mask = (1 << (bits - ptab->r.o.dix_cli_shift)) - 1; + + ASSERT((ix_per_cache_line & (ix_per_cache_line - 1)) == 0); /* power of 2 */ + ASSERT((tab_cache_lines & (tab_cache_lines - 1)) == 0); /* power of 2 */ + + ASSERT(ptab->r.o.dix_cl_shift + ptab->r.o.dix_cli_shift == bits); + + ix = 0; + for (cl = 0; cl < tab_cache_lines; cl++) { + for (cli = 0; cli < ix_per_cache_line; cli++) { + erts_smp_atomic32_init_nob(&ptab->r.o.free_id_data[ix], + cli*tab_cache_lines+cl); + ASSERT(erts_smp_atomic32_read_nob(&ptab->r.o.free_id_data[ix]) != ptab->r.o.invalid_data); + ix++; + } + } + + erts_smp_atomic32_init_nob(&ptab->vola.tile.aid_ix, -1); + erts_smp_atomic32_init_nob(&ptab->vola.tile.fid_ix, -1); + + } + erts_smp_interval_init(&ptab->list.data.interval); ptab->list.data.deleted.start = NULL; ptab->list.data.deleted.end = NULL; @@ -520,9 +499,7 @@ erts_ptab_new_element(ErtsPTab *ptab, void *init_arg, void (*init_ptab_el)(void *, Eterm)) { - int pix; - Uint64 ld, exp_ld; - Eterm data; + Uint32 pix, ix, data; erts_aint32_t count; erts_aint_t invalid = (erts_aint_t) ptab->r.o.invalid_element; @@ -549,62 +526,112 @@ erts_ptab_new_element(ErtsPTab *ptab, ptab_el->u.alive.started_interval = erts_smp_current_interval_nob(erts_ptab_interval(ptab)); - ld = last_data_read_acqb(ptab); + if (ptab->r.o.free_id_data) { + do { + ix = (Uint32) erts_smp_atomic32_inc_read_acqb(&ptab->vola.tile.aid_ix); + ix = ix_to_free_id_data_ix(ptab, ix); - /* Reserve slot */ - while (1) { - ld++; - pix = erts_ptab_data2pix(ptab, ERTS_PTAB_LastData2EtermData(ld)); - if (erts_smp_atomic_read_nob(&ptab->r.o.tab[pix]) == ERTS_AINT_NULL) { - erts_aint_t val; - val = erts_smp_atomic_cmpxchg_relb(&ptab->r.o.tab[pix], - invalid, - ERTS_AINT_NULL); + data = erts_smp_atomic32_xchg_nob(&ptab->r.o.free_id_data[ix], + (erts_aint32_t)ptab->r.o.invalid_data); + }while ((Eterm)data == ptab->r.o.invalid_data); + + init_ptab_el(init_arg, (Eterm) data); + + if (ptab->r.o.atomic_refc) + erts_atomic_init_nob(&ptab_el->refc.atmc, 1); + else + ptab_el->refc.sint = 1; + + pix = erts_ptab_data2pix(ptab, (Eterm) data); + +#ifdef DEBUG + ASSERT(ERTS_AINT_NULL == erts_smp_atomic_xchg_relb(&ptab->r.o.tab[pix], + (erts_aint_t) ptab_el)); +#else + erts_smp_atomic_set_relb(&ptab->r.o.tab[pix], (erts_aint_t) ptab_el); +#endif + + erts_ptab_runlock(ptab); - if (ERTS_AINT_NULL == val) - break; - } } + else { + int rlocked = ERTS_PTAB_NEW_MAX_RESERVE_FAIL; + Uint64 ld, exp_ld; + /* Deprecated legacy algorithm... */ - data = ERTS_PTAB_LastData2EtermData(ld); + restart: + + ptab_el->u.alive.started_interval + = erts_smp_current_interval_nob(erts_ptab_interval(ptab)); + + ld = last_data_read_acqb(ptab); + + /* Reserve slot */ + while (1) { + ld++; + pix = erts_ptab_data2pix(ptab, ERTS_PTAB_LastData2EtermData(ld)); + if (erts_smp_atomic_read_nob(&ptab->r.o.tab[pix]) + == ERTS_AINT_NULL) { + erts_aint_t val; + val = erts_smp_atomic_cmpxchg_relb(&ptab->r.o.tab[pix], + invalid, + ERTS_AINT_NULL); + + if (ERTS_AINT_NULL == val) + break; + } + if (rlocked && --rlocked == 0) { + erts_ptab_runlock(ptab); + erts_ptab_rwlock(ptab); + goto restart; + } + } - if (data == ptab->r.o.invalid_data) { - /* Do not use invalid data; fix it... */ - ld += ptab->r.o.max; - ASSERT(pix == erts_ptab_data2pix(ptab, - ERTS_PTAB_LastData2EtermData(ld))); data = ERTS_PTAB_LastData2EtermData(ld); - ASSERT(data != ptab->r.o.invalid_data); - } - exp_ld = last_data_read_nob(ptab); + if (data == ptab->r.o.invalid_data) { + /* Do not use invalid data; fix it... */ + ld += ptab->r.o.max; + ASSERT(pix == erts_ptab_data2pix(ptab, + ERTS_PTAB_LastData2EtermData(ld))); + data = ERTS_PTAB_LastData2EtermData(ld); + ASSERT(data != ptab->r.o.invalid_data); + } - /* Move last data forward */ - while (1) { - Uint64 act_ld; - if (last_data_cmp(ld, exp_ld) < 0) - break; - act_ld = last_data_cmpxchg_relb(ptab, ld, exp_ld); - if (act_ld == exp_ld) - break; - exp_ld = act_ld; - } + exp_ld = last_data_read_nob(ptab); - init_ptab_el(init_arg, data); + /* Move last data forward */ + while (1) { + Uint64 act_ld; + if (last_data_cmp(ld, exp_ld) < 0) + break; + act_ld = last_data_cmpxchg_relb(ptab, ld, exp_ld); + if (act_ld == exp_ld) + break; + exp_ld = act_ld; + } -#ifdef ERTS_SMP - erts_smp_atomic32_init_nob(&ptab_el->refc, 1); -#endif + init_ptab_el(init_arg, data); - /* Move into slot reserved */ + if (ptab->r.o.atomic_refc) + erts_atomic_init_nob(&ptab_el->refc.atmc, 1); + else + ptab_el->refc.sint = 1; + + /* Move into slot reserved */ #ifdef DEBUG - ASSERT(invalid == erts_smp_atomic_xchg_relb(&ptab->r.o.tab[pix], + ASSERT(invalid == erts_smp_atomic_xchg_relb(&ptab->r.o.tab[pix], (erts_aint_t) ptab_el)); #else - erts_smp_atomic_set_relb(&ptab->r.o.tab[pix], (erts_aint_t) ptab_el); + erts_smp_atomic_set_relb(&ptab->r.o.tab[pix], (erts_aint_t) ptab_el); #endif - erts_ptab_runlock(ptab); + if (rlocked) + erts_ptab_runlock(ptab); + else + erts_ptab_rwunlock(ptab); + + } return 1; } @@ -645,9 +672,12 @@ erts_ptab_delete_element(ErtsPTab *ptab, ErtsPTabElementCommon *ptab_el) { int maybe_save; - int pix = erts_ptab_id2pix(ptab, ptab_el->id); + Uint32 pix, ix, data; - ASSERT(erts_get_scheduler_id()); /* *Need* to be a scheduler */ + pix = erts_ptab_id2pix(ptab, ptab_el->id); + + /* *Need* to be an managed thread */ + ERTS_SMP_LC_ASSERT(erts_thr_progress_is_managed_thread()); erts_ptab_rlock(ptab); maybe_save = ptab->list.data.deleted.end != NULL; @@ -658,6 +688,29 @@ erts_ptab_delete_element(ErtsPTab *ptab, erts_smp_atomic_set_relb(&ptab->r.o.tab[pix], ERTS_AINT_NULL); + if (ptab->r.o.free_id_data) { + Uint32 prev_data; + /* Next data for this slot... */ + data = (Uint32) erts_ptab_id2data(ptab, ptab_el->id); + data += ptab->r.o.max; + data &= ~(~((Uint32) 0) << ERTS_PTAB_ID_DATA_SIZE); + if (data == ptab->r.o.invalid_data) { /* make sure not invalid */ + data += ptab->r.o.max; + data &= ~(~((Uint32) 0) << ERTS_PTAB_ID_DATA_SIZE); + } + ASSERT(data != ptab->r.o.invalid_data); + ASSERT(pix == erts_ptab_data2pix(ptab, data)); + + do { + ix = (Uint32) erts_smp_atomic32_inc_read_relb(&ptab->vola.tile.fid_ix); + ix = ix_to_free_id_data_ix(ptab, ix); + + prev_data = erts_smp_atomic32_cmpxchg_nob(&ptab->r.o.free_id_data[ix], + data, + ptab->r.o.invalid_data); + }while ((Eterm)prev_data != ptab->r.o.invalid_data); + } + ASSERT(erts_smp_atomic32_read_nob(&ptab->vola.tile.count) > 0); erts_smp_atomic32_dec_relb(&ptab->vola.tile.count); @@ -670,9 +723,10 @@ erts_ptab_delete_element(ErtsPTab *ptab, } if (ptab->r.o.release_element) - erts_schedule_thr_prgr_later_op(ptab->r.o.release_element, - (void *) ptab_el, - &ptab_el->u.release); + erts_schedule_thr_prgr_later_cleanup_op(ptab->r.o.release_element, + (void *) ptab_el, + &ptab_el->u.release, + ptab->r.o.element_size); } /* @@ -1201,7 +1255,7 @@ ptab_list_bif_engine(Process *c_p, Eterm *res_accp, Binary *mbp) return 1; default: - erl_exit(ERTS_ABORT_EXIT, + erts_exit(ERTS_ABORT_EXIT, "%s:%d:ptab_list_bif_engine(): Invalid state: %d\n", __FILE__, __LINE__, (int) ptlbdp->state); } @@ -1267,6 +1321,31 @@ erts_ptab_init(void) * Debug stuff */ +static void assert_ptab_consistency(ErtsPTab *ptab) +{ +#ifdef DEBUG + if (ptab->r.o.free_id_data) { + Uint32 ix, pix, data; + int free_pids = 0; + int null_slots = 0; + + for (ix=0; ix < ptab->r.o.max; ix++) { + if (erts_smp_atomic32_read_nob(&ptab->r.o.free_id_data[ix]) != ptab->r.o.invalid_data) { + ++free_pids; + data = erts_smp_atomic32_read_nob(&ptab->r.o.free_id_data[ix]); + pix = erts_ptab_data2pix(ptab, (Eterm) data); + ASSERT(erts_ptab_pix2intptr_nob(ptab, pix) == ERTS_AINT_NULL); + } + if (erts_smp_atomic_read_nob(&ptab->r.o.tab[ix]) == ERTS_AINT_NULL) { + ++null_slots; + } + } + ASSERT(free_pids == null_slots); + ASSERT(free_pids == ptab->r.o.max - erts_smp_atomic32_read_nob(&ptab->vola.tile.count)); + } +#endif +} + Sint erts_ptab_test_next_id(ErtsPTab *ptab, int set, Uint next) { @@ -1277,45 +1356,93 @@ erts_ptab_test_next_id(ErtsPTab *ptab, int set, Uint next) erts_ptab_rwlock(ptab); - if (!set) - ld = last_data_read_nob(ptab); - else { + assert_ptab_consistency(ptab); - ld = (Uint64) next; - data = ERTS_PTAB_LastData2EtermData(ld); - if (ptab->r.o.invalid_data == data) { - ld += ptab->r.o.max; - ASSERT(erts_ptab_data2pix(ptab, data) - == erts_ptab_data2pix(ptab, - ERTS_PTAB_LastData2EtermData(ld))); + if (ptab->r.o.free_id_data) { + Uint32 id_ix, dix; + + if (set) { + Uint32 i, max_ix, num, stop_id_ix; + max_ix = ptab->r.o.max - 1; + num = next; + id_ix = (Uint32) erts_smp_atomic32_read_nob(&ptab->vola.tile.aid_ix); + + for (i=0; i <= max_ix; ++i) { + Uint32 pix; + ++num; + num &= ~(~((Uint32) 0) << ERTS_PTAB_ID_DATA_SIZE); + if (num == ptab->r.o.invalid_data) { + num += ptab->r.o.max; + num &= ~(~((Uint32) 0) << ERTS_PTAB_ID_DATA_SIZE); + } + pix = erts_ptab_data2pix(ptab, num); + if (ERTS_AINT_NULL == erts_ptab_pix2intptr_nob(ptab, pix)) { + ++id_ix; + dix = ix_to_free_id_data_ix(ptab, id_ix); + erts_smp_atomic32_set_nob(&ptab->r.o.free_id_data[dix], num); + ASSERT(pix == erts_ptab_data2pix(ptab, num)); + } + } + erts_smp_atomic32_set_nob(&ptab->vola.tile.fid_ix, id_ix); + + /* Write invalid_data in rest of free_id_data[]: */ + stop_id_ix = (1 + erts_smp_atomic32_read_nob(&ptab->vola.tile.aid_ix)) & max_ix; + while (1) { + id_ix = (id_ix+1) & max_ix; + if (id_ix == stop_id_ix) + break; + dix = ix_to_free_id_data_ix(ptab, id_ix); + erts_smp_atomic32_set_nob(&ptab->r.o.free_id_data[dix], + ptab->r.o.invalid_data); + } } - last_data_set_relb(ptab, ld); + id_ix = (Uint32) erts_smp_atomic32_read_nob(&ptab->vola.tile.aid_ix) + 1; + dix = ix_to_free_id_data_ix(ptab, id_ix); + res = (Sint) erts_smp_atomic32_read_nob(&ptab->r.o.free_id_data[dix]); } + else { + /* Deprecated legacy algorithm... */ + if (!set) + ld = last_data_read_nob(ptab); + else { - while (1) { - int pix; - ld++; - pix = (int) (ld % ptab->r.o.max); - if (first_pix < 0) - first_pix = pix; - else if (pix == first_pix) { - res = -1; - break; - } - if (ERTS_AINT_NULL == erts_ptab_pix2intptr_nob(ptab, pix)) { + ld = (Uint64) next; data = ERTS_PTAB_LastData2EtermData(ld); if (ptab->r.o.invalid_data == data) { ld += ptab->r.o.max; ASSERT(erts_ptab_data2pix(ptab, data) == erts_ptab_data2pix(ptab, ERTS_PTAB_LastData2EtermData(ld))); + } + last_data_set_relb(ptab, ld); + } + + while (1) { + int pix; + ld++; + pix = (int) (ld % ptab->r.o.max); + if (first_pix < 0) + first_pix = pix; + else if (pix == first_pix) { + res = -1; + break; + } + if (ERTS_AINT_NULL == erts_ptab_pix2intptr_nob(ptab, pix)) { data = ERTS_PTAB_LastData2EtermData(ld); + if (ptab->r.o.invalid_data == data) { + ld += ptab->r.o.max; + ASSERT(erts_ptab_data2pix(ptab, data) + == erts_ptab_data2pix(ptab, + ERTS_PTAB_LastData2EtermData(ld))); + data = ERTS_PTAB_LastData2EtermData(ld); + } + res = data; + break; } - res = data; - break; } } + assert_ptab_consistency(ptab); erts_ptab_rwunlock(ptab); return res; diff --git a/erts/emulator/beam/erl_ptab.h b/erts/emulator/beam/erl_ptab.h index 8a130f42a3..8fd961e3ce 100644 --- a/erts/emulator/beam/erl_ptab.h +++ b/erts/emulator/beam/erl_ptab.h @@ -1,18 +1,19 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2012. All Rights Reserved. + * Copyright Ericsson AB 2012-2013. All Rights Reserved. * - * The contents of this file are subject to the Erlang Public License, - * Version 1.1, (the "License"); you may not use this file except in - * compliance with the License. You should have received a copy of the - * Erlang Public License along with this software. If not, it can be - * retrieved online at http://www.erlang.org/. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Software distributed under the License is distributed on an "AS IS" - * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See - * the License for the specific language governing rights and limitations - * under the License. + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. * * %CopyrightEnd% */ @@ -51,11 +52,13 @@ typedef struct { Eterm id; -#ifdef ERTS_SMP - erts_atomic32_t refc; -#endif + union { + erts_atomic_t atmc; + Sint sint; + } refc; Eterm tracer_proc; Uint trace_flags; + erts_smp_atomic_t timer; union { /* --- While being alive --- */ struct { @@ -63,11 +66,6 @@ typedef struct { struct reg_proc *reg; ErtsLink *links; ErtsMonitor *monitors; -#ifdef ERTS_SMP - ErtsSmpPTimer *ptimer; -#else - ErlTimer tm; -#endif } alive; /* --- While being released --- */ @@ -88,27 +86,30 @@ typedef struct { } ErtsPTabListData; typedef struct { -#ifdef ARCH_32 - erts_smp_dw_atomic_t last_data; -#else - erts_smp_atomic_t last_data; -#endif + erts_smp_atomic64_t last_data; erts_smp_atomic32_t count; + erts_smp_atomic32_t aid_ix; + erts_smp_atomic32_t fid_ix; } ErtsPTabVolatileData; typedef struct { erts_smp_atomic_t *tab; + erts_smp_atomic32_t *free_id_data; Uint32 max; - Uint32 tab_cache_lines; - Uint32 pix_per_cache_line; Uint32 pix_mask; Uint32 pix_cl_mask; Uint32 pix_cl_shift; Uint32 pix_cli_mask; Uint32 pix_cli_shift; + Uint32 dix_cl_mask; + Uint32 dix_cl_shift; + Uint32 dix_cli_mask; + Uint32 dix_cli_shift; ErtsPTabElementCommon *invalid_element; Eterm invalid_data; void (*release_element)(void *); + UWord element_size; + int atomic_refc; } ErtsPTabReadOnlyData; typedef struct { @@ -177,7 +178,10 @@ void erts_ptab_init_table(ErtsPTab *ptab, void (*release_element)(void *), ErtsPTabElementCommon *invalid_element, int size, - char *name); + UWord element_size, + char *name, + int legacy, + int atomic_refc); int erts_ptab_new_element(ErtsPTab *ptab, ErtsPTabElementCommon *ptab_el, void *init_arg, @@ -185,6 +189,7 @@ int erts_ptab_new_element(ErtsPTab *ptab, void erts_ptab_delete_element(ErtsPTab *ptab, ErtsPTabElementCommon *ptab_el); int erts_ptab_initialized(ErtsPTab *ptab); +UWord erts_ptab_mem_size(ErtsPTab *ptab); ERTS_GLB_INLINE erts_interval_t *erts_ptab_interval(ErtsPTab *ptab); ERTS_GLB_INLINE int erts_ptab_max(ErtsPTab *ptab); @@ -201,9 +206,15 @@ ERTS_GLB_INLINE erts_aint_t erts_ptab_pix2intptr_ddrb(ErtsPTab *ptab, int ix); ERTS_GLB_INLINE erts_aint_t erts_ptab_pix2intptr_rb(ErtsPTab *ptab, int ix); ERTS_GLB_INLINE erts_aint_t erts_ptab_pix2intptr_acqb(ErtsPTab *ptab, int ix); ERTS_GLB_INLINE void erts_ptab_inc_refc(ErtsPTabElementCommon *ptab_el); -ERTS_GLB_INLINE int erts_ptab_dec_test_refc(ErtsPTabElementCommon *ptab_el); -ERTS_GLB_INLINE int erts_ptab_add_test_refc(ErtsPTabElementCommon *ptab_el, - Sint32 add_refc); +ERTS_GLB_INLINE Sint erts_ptab_dec_test_refc(ErtsPTabElementCommon *ptab_el); +ERTS_GLB_INLINE Sint erts_ptab_add_test_refc(ErtsPTabElementCommon *ptab_el, + Sint add_refc); +ERTS_GLB_INLINE Sint erts_ptab_read_refc(ErtsPTabElementCommon *ptab_el); +ERTS_GLB_INLINE void erts_ptab_atmc_inc_refc(ErtsPTabElementCommon *ptab_el); +ERTS_GLB_INLINE Sint erts_ptab_atmc_dec_test_refc(ErtsPTabElementCommon *ptab_el); +ERTS_GLB_INLINE Sint erts_ptab_atmc_add_test_refc(ErtsPTabElementCommon *ptab_el, + Sint add_refc); +ERTS_GLB_INLINE Sint erts_ptab_atmc_read_refc(ErtsPTabElementCommon *ptab_el); ERTS_GLB_INLINE void erts_ptab_rlock(ErtsPTab *ptab); ERTS_GLB_INLINE int erts_ptab_tryrlock(ErtsPTab *ptab); ERTS_GLB_INLINE void erts_ptab_runlock(ErtsPTab *ptab); @@ -360,50 +371,65 @@ ERTS_GLB_INLINE erts_aint_t erts_ptab_pix2intptr_acqb(ErtsPTab *ptab, int ix) return erts_smp_atomic_read_acqb(&ptab->r.o.tab[ix]); } -ERTS_GLB_INLINE void erts_ptab_inc_refc(ErtsPTabElementCommon *ptab_el) +ERTS_GLB_INLINE void erts_ptab_atmc_inc_refc(ErtsPTabElementCommon *ptab_el) { -#ifdef ERTS_SMP #ifdef ERTS_ENABLE_LOCK_CHECK - erts_aint32_t refc = erts_atomic32_inc_read_nob(&ptab_el->refc); - ERTS_SMP_LC_ASSERT(refc > 1); + erts_aint_t refc = erts_atomic_inc_read_nob(&ptab_el->refc.atmc); + ERTS_LC_ASSERT(refc > 1); #else - erts_atomic32_inc_nob(&ptab_el->refc); -#endif + erts_atomic_inc_nob(&ptab_el->refc.atmc); #endif } -ERTS_GLB_INLINE int erts_ptab_dec_test_refc(ErtsPTabElementCommon *ptab_el) +ERTS_GLB_INLINE Sint erts_ptab_atmc_dec_test_refc(ErtsPTabElementCommon *ptab_el) { -#ifdef ERTS_SMP - erts_aint32_t refc = erts_atomic32_dec_read_nob(&ptab_el->refc); + erts_aint_t refc = erts_atomic_dec_read_relb(&ptab_el->refc.atmc); ERTS_SMP_LC_ASSERT(refc >= 0); - return (int) refc; -#else - return 0; +#ifdef ERTS_SMP + if (refc == 0) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore); #endif + return (Sint) refc; } -ERTS_GLB_INLINE int erts_ptab_add_test_refc(ErtsPTabElementCommon *ptab_el, - Sint32 add_refc) +ERTS_GLB_INLINE Sint erts_ptab_atmc_add_test_refc(ErtsPTabElementCommon *ptab_el, + Sint add_refc) { -#ifdef ERTS_SMP - erts_aint32_t refc; + erts_aint_t refc = erts_atomic_add_read_mb(&ptab_el->refc.atmc, + (erts_aint_t) add_refc); + ERTS_SMP_LC_ASSERT(refc >= 0); + return (Sint) refc; +} -#ifndef ERTS_ENABLE_LOCK_CHECK - if (add_refc >= 0) { - erts_atomic32_add_nob(&ptab_el->refc, - (erts_aint32_t) add_refc); - return 1; - } -#endif +ERTS_GLB_INLINE Sint erts_ptab_atmc_read_refc(ErtsPTabElementCommon *ptab_el) +{ + return (Sint) erts_atomic_read_nob(&ptab_el->refc.atmc); +} - refc = erts_atomic32_add_read_nob(&ptab_el->refc, - (erts_aint32_t) add_refc); +ERTS_GLB_INLINE void erts_ptab_inc_refc(ErtsPTabElementCommon *ptab_el) +{ + ptab_el->refc.sint++; + ASSERT(ptab_el->refc.sint > 1); +} + +ERTS_GLB_INLINE Sint erts_ptab_dec_test_refc(ErtsPTabElementCommon *ptab_el) +{ + Sint refc = --ptab_el->refc.sint; ERTS_SMP_LC_ASSERT(refc >= 0); - return (int) refc; -#else - return 0; -#endif + return refc; +} + +ERTS_GLB_INLINE Sint erts_ptab_add_test_refc(ErtsPTabElementCommon *ptab_el, + Sint add_refc) +{ + ptab_el->refc.sint += add_refc; + ERTS_SMP_LC_ASSERT(ptab_el->refc.sint >= 0); + return (Sint) ptab_el->refc.sint; +} + +ERTS_GLB_INLINE Sint erts_ptab_read_refc(ErtsPTabElementCommon *ptab_el) +{ + return ptab_el->refc.sint; } ERTS_GLB_INLINE void erts_ptab_rlock(ErtsPTab *ptab) diff --git a/erts/emulator/beam/erl_rbtree.h b/erts/emulator/beam/erl_rbtree.h new file mode 100644 index 0000000000..5fefaea978 --- /dev/null +++ b/erts/emulator/beam/erl_rbtree.h @@ -0,0 +1,1741 @@ +/* + * %CopyrightBegin% + * + * Copyright Ericsson AB 2015. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * %CopyrightEnd% + */ + +/* + * Description: A Red-Black (binary search) Tree implementation. The search, + * insert, and delete operations are all O(log n) operations + * on a Red-Black Tree. Red-Black Trees are described in + * "Introduction to Algorithms", by Thomas H. Cormen, Charles + * E. Leiserson, and Ronald L. Riverest. + * + * Use by defining mandatory defines as well as defines for + * API functions wanted, and include this header. + * + * Author: Rickard Green + * + * + * Mandatory defines: + * - ERTS_RBT_PREFIX - Prefix to use on functions. + * - ERTS_RBT_T - Type of a tree node. + * - ERTS_RBT_KEY_T - Type of key for a tree node. + * - ERTS_RBT_FLAGS_T - Type of flags for a tree node. + * - ERTS_RBT_INIT_EMPTY_TNODE(T) -Initialize an empty tree node. + * - ERTS_RBT_IS_RED(T) - Is tree node red? + * - ERTS_RBT_SET_RED(T) - Set tree node red. + * - ERTS_RBT_IS_BLACK(T) - Is tree node back? + * - ERTS_RBT_SET_BLACK(T) - Set tree node black. + * - ERTS_RBT_GET_FLAGS(T) - Get flags of tree node (incl colors). + * - ERTS_RBT_SET_FLAGS(T, F) - Set flags of tree note. + * - ERTS_RBT_GET_PARENT(T) - Get parent node. + * - ERTS_RBT_SET_PARENT(T, P) - Set parent node. + * - ERTS_RBT_GET_RIGHT(T) - Get right child node. + * - ERTS_RBT_SET_RIGHT(T, R) - Set right child node. + * - ERTS_RBT_GET_LEFT(T) - Get left child node. + * - ERTS_RBT_SET_LEFT(T, L) - Set left child node. + * - ERTS_RBT_GET_KEY(T) - Get key of node. + * - ERTS_RBT_IS_LT(KX, KY) - Is key KX less than key KY? + * - ERTS_RBT_IS_EQ(KX, KY) - Is key KX equal to key KY? + * + * Optional defines: + * + * - ERTS_RBT_UNDEF - Undefine all user defined ERTS_RBT_* + * defines after use. + * + * - ERTS_RBT_NO_API_INLINE - Do not inline API functions. + * + * Attached data management: + * - ERTS_RBT_UPDATE_ATTACHED_DATA_ROTATE(L, OP, NP) - Called + * when a rotate operation has been performed. If L (in int) + * is a non zero, a left rotation was performed; otherwise, + * a right rotation was performed. OR points to the old + * parent node and NP points to the new parent node. + * - ERTS_RBT_UPDATE_ATTACHED_DATA_DMOD(F, T) - Called when + * a delete operation modifies a tree node. A delete + * modification is either a removal or replacement of a + * node. F points to the parent of the tree node that was + * modified. T points to the next ancestor that will be + * modified. If T is NULL, no more removal and/or + * replacements will be made. One typically wants to update + * the attached data of each node between F and T. If T is + * NULL all the way up to the root. + * - ERTS_RBT_UPDATE_ATTACHED_DATA_CHGROOT(OR, NR) - Called + * when the root node changes. OR points to the old + * root node and NP points to the new root node. + * + * Request implementation of API functions: + * - ERTS_RBT_WANT_DELETE + * - ERTS_RBT_WANT_INSERT + * - ERTS_RBT_WANT_LOOKUP_INSERT + * - ERTS_RBT_WANT_REPLACE + * - ERTS_RBT_WANT_LOOKUP + * - ERTS_RBT_WANT_SMALLEST + * - ERTS_RBT_WANT_LARGEST + * - ERTS_RBT_WANT_FOREACH + * - ERTS_RBT_WANT_FOREACH_DESTROY + * - ERTS_RBT_WANT_FOREACH_YIELDING + * - ERTS_RBT_WANT_FOREACH_DESTROY_YIELDING + * - ERTS_RBT_WANT_FOREACH_SMALL + * - ERTS_RBT_WANT_FOREACH_LARGE + * - ERTS_RBT_WANT_FOREACH_SMALL_DESTROY + * - ERTS_RBT_WANT_FOREACH_LARGE_DESTROY + * - ERTS_RBT_WANT_FOREACH_SMALL_YIELDING + * - ERTS_RBT_WANT_FOREACH_LARGE_YIELDING + * - ERTS_RBT_WANT_FOREACH_SMALL_DESTROY_YIELDING + * - ERTS_RBT_WANT_FOREACH_LARGE_DESTROY_YIELDING + * - ERTS_RBT_WANT_DEBUG_PRINT + * + * The yield state data type will equal + * <ERTS_RBT_PREFIX>_rbt_yield_state_t. + * + * The yield state should be statically initialized by + * ERTS_RBT_YIELD_STAT_INITER. + * + * + * The following API functions are implemented if corresponding + * ERTS_RBT_WANT_<OPERATION> is defined: + * + * - void <ERTS_RBT_PREFIX>_rbt_delete( + * ERTS_RBT_T **tree, + * ERTS_RBT_T *element); + * Delete element from tree. + * + * - void <ERTS_RBT_PREFIX>_rbt_insert( + * ERTS_RBT_T **tree, + * ERTS_RBT_T *element); + * Insert element into tree. + * + * - ERTS_RBT_T * <ERTS_RBT_PREFIX>_rbt_lookup_insert( + * ERTS_RBT_T **tree, + * ERTS_RBT_T *element); + * Look up an element in the tree that compares as equal to the + * element passed as argument, and return the looked up element. + * If no element compared as equal, insert the element passed as + * argument into the tree, and return NULL. + * + * - void <ERTS_RBT_PREFIX>_rbt_replace( + * ERTS_RBT_T **tree, + * ERTS_RBT_T *old_element, + * ERTS_RBT_T *new_element); + * Replace old_element in the tree with new_element. Both elements + * *should* compare as equal. + * + * - ERTS_RBT_T * <ERTS_RBT_PREFIX>_rbt_lookup( + * ERTS_RBT_T *tree, + * ERTS_RBT_KEY_T key); + * Look up an element with a key that compares as equal to + * the key passed as argument. + * + * - ERTS_RBT_T * <ERTS_RBT_PREFIX>_rbt_smallest( + * ERTS_RBT_T *tree); + * Look up the element with the smallest key. + * + * - ERTS_RBT_T * <ERTS_RBT_PREFIX>_rbt_largest( + * ERTS_RBT_T *tree); + * Look up the element with the largest key. + * + * - void <ERTS_RBT_PREFIX>_rbt_foreach( + * ERTS_RBT_T *tree, + * void (*op)(ERTS_RBT_T *, void *), + * void *arg); + * Operate by calling the operator 'op' on each element. + * Order is undefined. + * + * 'arg' is passed as argument to 'op'. + * + * - void <ERTS_RBT_PREFIX>_rbt_foreach_destroy( + * ERTS_RBT_T *tree, + * void (*op)(ERTS_RBT_T *, void *), + * void *arg); + * Operate by calling the operator 'op' on each element. + * Order is undefined. Each element should be destroyed + * by 'op'. + * + * 'arg' is passed as argument to 'op'. + * + * - int <ERTS_RBT_PREFIX>_rbt_foreach_yielding( + * ERTS_RBT_T *tree, + * void (*op)(ERTS_RBT_T *, void *), + * void *arg, + * <ERTS_RBT_PREFIX>_rbt_yield_state_t *ystate, + * Sint ylimit); + * Operate by calling the operator 'op' on each element. + * Order is undefined. + * + * Yield when 'ylimit' elements has been processed. Zero is + * returned when yielding, and a non-zero value is returned when + * the whole tree has been processed. The tree should not be + * modified until all of it has been processed. + * + * 'arg' is passed as argument to 'op'. + * + * - int <ERTS_RBT_PREFIX>_rbt_foreach_destroy_yielding( + * ERTS_RBT_T *tree, + * void (*op)(ERTS_RBT_T *, void *), + * void *arg, + * <ERTS_RBT_PREFIX>_rbt_yield_state_t *ystate, + * Sint ylimit); + * Operate by calling the operator 'op' on each element. + * Order is undefined. Each element should be destroyed + * by 'op'. + * + * Yield when 'ylimit' elements has been processed. Zero is + * returned when yielding, and a non-zero value is returned when + * the whole tree has been processed. + * + * 'arg' is passed as argument to 'op'. + * + * - void <ERTS_RBT_PREFIX>_rbt_foreach_small( + * ERTS_RBT_T *tree, + * void (*op)(ERTS_RBT_T *, void *), + * void *arg); + * Operate by calling the operator 'op' on each element from + * smallest towards larger elements. + * + * 'arg' is passed as argument to 'op'. + * + * - void <ERTS_RBT_PREFIX>_rbt_foreach_large( + * ERTS_RBT_T *tree, + * void (*op)(ERTS_RBT_T *, void *), + * void *arg); + * Operate by calling the operator 'op' on each element from + * largest towards smaller elements. + * + * 'arg' is passed as argument to 'op'. + * + * - int <ERTS_RBT_PREFIX>_rbt_foreach_small_yielding( + * ERTS_RBT_T *tree, + * void (*op)(ERTS_RBT_T *, void *), + * void *arg, + * <ERTS_RBT_PREFIX>_rbt_yield_state_t *ystate, + * Sint ylimit); + * Operate by calling the operator 'op' on each element from + * smallest towards larger elements. + * + * Yield when 'ylimit' elements has been processed. Zero is + * returned when yielding, and a non-zero value is returned when + * the whole tree has been processed. The tree should not be + * modified until all of it has been processed. + * + * 'arg' is passed as argument to 'op'. + * + * - int <ERTS_RBT_PREFIX>_rbt_foreach_large_yielding( + * ERTS_RBT_T *tree, + * void (*op)(ERTS_RBT_T *, void *), + * void *arg, + * <ERTS_RBT_PREFIX>_rbt_yield_state_t *ystate, + * Sint ylimit); + * Operate by calling the operator 'op' on each element from + * largest towards smaller elements. + * + * Yield when 'ylimit' elements has been processed. Zero is + * returned when yielding, and a non-zero value is returned when + * the whole tree has been processed. The tree should not be + * modified until all of it has been processed. + * + * 'arg' is passed as argument to 'op'. + * + * - void <ERTS_RBT_PREFIX>_rbt_foreach_small_destroy( + * ERTS_RBT_T **tree, + * void (*op)(ERTS_RBT_T *, void *), + * void (*destr)(ERTS_RBT_T *, void *), + * void *arg); + * Operate by calling the operator 'op' on each element from + * smallest towards larger elements. + * + * Destroy elements by calling the destructor 'destr'. Elements + * are destroyed when not needed by the tree structure anymore. + * Note that elements are often *not* destroyed in another order + * than the order that the elements are operated on. + * + * 'arg' is passed as argument to 'op' and 'destroy'. + * + * - void <ERTS_RBT_PREFIX>_rbt_foreach_large_destroy( + * ERTS_RBT_T **tree, + * void (*op)(ERTS_RBT_T *, void *), + * void (*destr)(ERTS_RBT_T *, void *), + * void *arg); + * Operate by calling the operator 'op' on each element from + * largest towards smaller elements. + * + * Destroy elements by calling the destructor 'destr'. Elements + * are destroyed when not needed by the tree structure anymore. + * Note that elements are often destroyed in another order + * than the order that the elements are operated on. + * + * 'arg' is passed as argument to 'op' and 'destroy'. + * + * - int <ERTS_RBT_PREFIX>_rbt_foreach_small_destroy_yielding( + * ERTS_RBT_T **tree, + * void (*op)(ERTS_RBT_T *, void *), + * void (*destr)(ERTS_RBT_T *, void *), + * void *arg, + * <ERTS_RBT_PREFIX>_rbt_yield_state_t *ystate, + * Sint ylimit); + * Operate by calling the operator 'op' on each element from + * smallest towards larger elements. + * + * Destroy elements by calling the destructor 'destr'. Elements + * are destroyed when not needed by the tree structure anymore. + * Note that elements are often destroyed in another order + * than the order that the elements are operated on. + * + * Yield when 'ylimit' elements has been processed. Zero is + * returned when yielding, and a non-zero value is returned when + * the whole tree has been processed. The tree should not be + * modified until all of it has been processed. + * + * 'arg' is passed as argument to 'op' and 'destroy'. + * + * - int <ERTS_RBT_PREFIX>_rbt_foreach_large_destroy_yielding( + * ERTS_RBT_T **tree, + * void (*op)(ERTS_RBT_T *, void *), + * void (*destr)(ERTS_RBT_T *, void *), + * void *arg, + * <ERTS_RBT_PREFIX>_rbt_yield_state_t *ystate, + * Sint ylimit); + * Operate by calling the operator 'op' on each element from + * largest towards smaller elements. + * + * Destroy elements by calling the destructor 'destr'. Elements + * are destroyed when not needed by the tree structure anymore. + * Note that elements are often destroyed in another order + * than the order that the elements are operated on. + * + * Yield when 'ylimit' elements has been processed. Zero is + * returned when yielding, and a non-zero value is returned when + * the whole tree has been processed. The tree should not be + * modified until all of it has been processed. + * + * 'arg' is passed as argument to 'op' and 'destroy'. + * + * - void <ERTS_RBT_PREFIX>_rbt_debug_print( + * FILE *filep, + * ERTS_RBT_T *x, + * int indent, + * (void)(*print_node)(ERTS_RBT_T *)); + * Prints the tree. Note that this function is recursive. + * Should only be used for debuging. + */ + + +/* + * Check that we have all mandatory defines + */ +#ifndef ERTS_RBT_PREFIX +# error Missing definition of ERTS_RBT_PREFIX +#endif +#ifndef ERTS_RBT_T +# error Missing definition of ERTS_RBT_T +#endif +#ifndef ERTS_RBT_KEY_T +# error Missing definition of ERTS_RBT_KEY_T +#endif +#ifndef ERTS_RBT_FLAGS_T +# error Missing definition of ERTS_RBT_FLAGS_T +#endif +#ifndef ERTS_RBT_INIT_EMPTY_TNODE +# error Missing definition of ERTS_RBT_INIT_EMPTY_TNODE +#endif +#ifndef ERTS_RBT_IS_RED +# error Missing definition of ERTS_RBT_IS_RED +#endif +#ifndef ERTS_RBT_SET_RED +# error Missing definition of ERTS_RBT_SET_RED +#endif +#ifndef ERTS_RBT_IS_BLACK +# error Missing definition of ERTS_RBT_IS_BLACK +#endif +#ifndef ERTS_RBT_SET_BLACK +# error Missing definition of ERTS_RBT_SET_BLACK +#endif +#ifndef ERTS_RBT_GET_FLAGS +# error Missing definition of ERTS_RBT_GET_FLAGS +#endif +#ifndef ERTS_RBT_SET_FLAGS +# error Missing definition of ERTS_RBT_SET_FLAGS +#endif +#ifndef ERTS_RBT_GET_PARENT +# error Missing definition of ERTS_RBT_GET_PARENT +#endif +#ifndef ERTS_RBT_SET_PARENT +# error Missing definition of ERTS_RBT_SET_PARENT +#endif +#ifndef ERTS_RBT_GET_RIGHT +# error Missing definition of ERTS_RBT_GET_RIGHT +#endif +#ifndef ERTS_RBT_GET_LEFT +# error Missing definition of ERTS_RBT_GET_LEFT +#endif +#ifndef ERTS_RBT_IS_LT +# error Missing definition of ERTS_RBT_IS_LT +#endif +#ifndef ERTS_RBT_GET_KEY +# error Missing definition of ERTS_RBT_GET_KEY +#endif +#ifndef ERTS_RBT_IS_EQ +# error Missing definition of ERTS_RBT_IS_EQ +#endif + +#if defined(ERTS_RBT_HARD_DEBUG) || defined(DEBUG) +# ifndef ERTS_RBT_DEBUG +# define ERTS_RBT_DEBUG 1 +# endif +#endif + +#if defined(ERTS_RBT_HARD_DEBUG) && defined(__GNUC__) +#warning "* * * * * * * * * * * * * * * * * *" +#warning "* ERTS_RBT_HARD_DEBUG IS ENABLED! *" +#warning "* * * * * * * * * * * * * * * * * *" +#endif + +#undef ERTS_RBT_ASSERT +#if defined(ERTS_RBT_DEBUG) +#define ERTS_RBT_ASSERT(E) ERTS_ASSERT(E) +#else +#define ERTS_RBT_ASSERT(E) ((void) 1) +#endif + +#undef ERTS_RBT_API_INLINE__ +#if defined(ERTS_RBT_NO_API_INLINE) || defined(ERTS_RBT_DEBUG) +# define ERTS_RBT_API_INLINE__ +#else +# define ERTS_RBT_API_INLINE__ ERTS_INLINE +#endif + +#ifndef ERTS_RBT_YIELD_STAT_INITER +# define ERTS_RBT_YIELD_STAT_INITER {NULL, 0} +#endif + +#define ERTS_RBT_CONCAT_MACRO_VALUES___(X, Y) \ + X ## Y +#define ERTS_RBT_CONCAT_MACRO_VALUES__(X, Y) \ + ERTS_RBT_CONCAT_MACRO_VALUES___(X, Y) + +#undef ERTS_RBT_YIELD_STATE_T__ +#define ERTS_RBT_YIELD_STATE_T__ \ + ERTS_RBT_CONCAT_MACRO_VALUES__(ERTS_RBT_PREFIX, _rbt_yield_state_t) + +typedef struct { + ERTS_RBT_T *x; + int up; +} ERTS_RBT_YIELD_STATE_T__; + +#define ERTS_RBT_FUNC__(Name) \ + ERTS_RBT_CONCAT_MACRO_VALUES__(ERTS_RBT_PREFIX, _rbt_ ## Name) + +#undef ERTS_RBT_NEED_REPLACE__ +#undef ERTS_RBT_NEED_INSERT__ +#undef ERTS_RBT_NEED_ROTATE__ +#undef ERTS_RBT_NEED_FOREACH_UNORDERED__ +#undef ERTS_RBT_NEED_FOREACH_ORDERED__ +#undef ERTS_RBT_NEED_HDBG_CHECK_TREE__ +#undef ERTS_RBT_HDBG_CHECK_TREE__ + +#if defined(ERTS_RBT_WANT_REPLACE) || defined(ERTS_RBT_WANT_DELETE) +# define ERTS_RBT_NEED_REPLACE__ +#endif +#if defined(ERTS_RBT_WANT_INSERT) || defined(ERTS_RBT_WANT_LOOKUP_INSERT) +# define ERTS_RBT_NEED_INSERT__ +#endif +#if defined(ERTS_RBT_WANT_DELETE) || defined(ERTS_RBT_NEED_INSERT__) +# define ERTS_RBT_NEED_ROTATE__ +#endif +#if defined(ERTS_RBT_WANT_FOREACH) \ + || defined(ERTS_RBT_WANT_FOREACH_YIELDING) \ + || defined(ERTS_RBT_WANT_FOREACH_DESTROY) \ + || defined(ERTS_RBT_WANT_FOREACH_DESTROY_YIELDING) +# define ERTS_RBT_NEED_FOREACH_UNORDERED__ +#endif +#if defined(ERTS_RBT_WANT_FOREACH_SMALL) \ + || defined(ERTS_RBT_WANT_FOREACH_LARGE) \ + || defined(ERTS_RBT_WANT_FOREACH_SMALL_YIELDING) \ + || defined(ERTS_RBT_WANT_FOREACH_LARGE_YIELDING) \ + || defined(ERTS_RBT_WANT_FOREACH_SMALL_DESTROY) \ + || defined(ERTS_RBT_WANT_FOREACH_LARGE_DESTROY) \ + || defined(ERTS_RBT_WANT_FOREACH_SMALL_DESTROY_YIELDING) \ + || defined(ERTS_RBT_WANT_FOREACH_LARGE_DESTROY_YIELDING) +# define ERTS_RBT_NEED_FOREACH_ORDERED__ +#endif +#if defined(ERTS_RBT_HARD_DEBUG) \ + && (defined(ERTS_RBT_WANT_DELETE) \ + || defined(ERTS_RBT_NEED_INSERT__)) +static void ERTS_RBT_FUNC__(hdbg_check_tree)(ERTS_RBT_T *root); +# define ERTS_RBT_NEED_HDBG_CHECK_TREE__ +# define ERTS_RBT_HDBG_CHECK_TREE__(R) \ + ERTS_RBT_FUNC__(hdbg_check_tree)((R)) +#else +# define ERTS_RBT_HDBG_CHECK_TREE__(R) ((void) 1) +#endif + +#ifdef ERTS_RBT_NEED_ROTATE__ + +static ERTS_INLINE void +ERTS_RBT_FUNC__(left_rotate__)(ERTS_RBT_T **root, ERTS_RBT_T *x) +{ + ERTS_RBT_T *y, *l, *p; + + y = ERTS_RBT_GET_RIGHT(x); + l = ERTS_RBT_GET_LEFT(y); + ERTS_RBT_SET_RIGHT(x, l); + + if (l) + ERTS_RBT_SET_PARENT(l, x); + + p = ERTS_RBT_GET_PARENT(x); + ERTS_RBT_SET_PARENT(y, p); + + if (!p) { + ERTS_RBT_ASSERT(*root == x); + *root = y; +#ifdef ERTS_RBT_UPDATE_ATTACHED_DATA_CHGROOT + ERTS_RBT_UPDATE_ATTACHED_DATA_CHGROOT(x, y); +#endif + } + else if (x == ERTS_RBT_GET_LEFT(p)) + ERTS_RBT_SET_LEFT(p, y); + else { + ERTS_RBT_ASSERT(x == ERTS_RBT_GET_RIGHT(p)); + ERTS_RBT_SET_RIGHT(p, y); + } + ERTS_RBT_SET_LEFT(y, x); + ERTS_RBT_SET_PARENT(x, y); + +#ifdef ERTS_RBT_UPDATE_ATTACHED_DATA_ROTATE + ERTS_RBT_UPDATE_ATTACHED_DATA_ROTATE(!0, x, y); +#endif + +} + +static ERTS_INLINE void +ERTS_RBT_FUNC__(right_rotate__)(ERTS_RBT_T **root, ERTS_RBT_T *x) +{ + ERTS_RBT_T *y, *r, *p; + + y = ERTS_RBT_GET_LEFT(x); + r = ERTS_RBT_GET_RIGHT(y); + ERTS_RBT_SET_LEFT(x, r); + + if (r) + ERTS_RBT_SET_PARENT(r, x); + + p = ERTS_RBT_GET_PARENT(x); + ERTS_RBT_SET_PARENT(y, p); + + if (!p) { + ERTS_RBT_ASSERT(*root == x); + *root = y; +#ifdef ERTS_RBT_UPDATE_ATTACHED_DATA_CHGROOT + ERTS_RBT_UPDATE_ATTACHED_DATA_CHGROOT(x, y); +#endif + } + else if (x == ERTS_RBT_GET_RIGHT(p)) + ERTS_RBT_SET_RIGHT(p, y); + else { + ERTS_RBT_ASSERT(x == ERTS_RBT_GET_LEFT(p)); + ERTS_RBT_SET_LEFT(p, y); + } + + ERTS_RBT_SET_RIGHT(y, x); + ERTS_RBT_SET_PARENT(x, y); + +#ifdef ERTS_RBT_UPDATE_ATTACHED_DATA_ROTATE + ERTS_RBT_UPDATE_ATTACHED_DATA_ROTATE(0, x, y); +#endif + +} + +#endif /* ERTS_RBT_NEED_ROTATE__ */ + +#ifdef ERTS_RBT_NEED_REPLACE__ + +/* + * Replace node x with node y + */ +static ERTS_INLINE void +ERTS_RBT_FUNC__(replace__)(ERTS_RBT_T **root, ERTS_RBT_T *x, ERTS_RBT_T *y) +{ + ERTS_RBT_T *p, *r, *l; + ERTS_RBT_FLAGS_T f; + + p = ERTS_RBT_GET_PARENT(x); + if (!p) { + ERTS_RBT_ASSERT(*root == x); + *root = y; +#ifdef ERTS_RBT_UPDATE_ATTACHED_DATA_CHGROOT + ERTS_RBT_UPDATE_ATTACHED_DATA_CHGROOT(x, y); +#endif + } + else if (x == ERTS_RBT_GET_LEFT(p)) + ERTS_RBT_SET_LEFT(p, y); + else { + ERTS_RBT_ASSERT(x == ERTS_RBT_GET_RIGHT(p)); + ERTS_RBT_SET_RIGHT(p, y); + } + l = ERTS_RBT_GET_LEFT(x); + if (l) { + ERTS_RBT_ASSERT(ERTS_RBT_GET_PARENT(l) == x); + ERTS_RBT_SET_PARENT(l, y); + } + r = ERTS_RBT_GET_RIGHT(x); + if (r) { + ERTS_RBT_ASSERT(ERTS_RBT_GET_PARENT(r) == x); + ERTS_RBT_SET_PARENT(r, y); + } + + f = ERTS_RBT_GET_FLAGS(x); + ERTS_RBT_SET_FLAGS(y, f); + ERTS_RBT_SET_PARENT(y, p); + ERTS_RBT_SET_RIGHT(y, r); + ERTS_RBT_SET_LEFT(y, l); +} + +#endif /* ERTS_RBT_NEED_REPLACE__ */ + +#ifdef ERTS_RBT_WANT_REPLACE + +static ERTS_RBT_API_INLINE__ void +ERTS_RBT_FUNC__(replace)(ERTS_RBT_T **root, ERTS_RBT_T *x, ERTS_RBT_T *y) +{ + ERTS_RBT_ASSERT(ERTS_RBT_IS_EQ(ERTS_RBT_GET_KEY(x), + ERTS_RBT_GET_KEY(y))); + + ERTS_RBT_FUNC__(replace__)(root, x, y); +} + +#endif /* ERTS_RBT_WANT_REPLACE */ + +#ifdef ERTS_RBT_WANT_DELETE + +/* + * Delete a node. + */ +static ERTS_RBT_API_INLINE__ void +ERTS_RBT_FUNC__(delete)(ERTS_RBT_T **root, ERTS_RBT_T *n) +{ + int spliced_is_black; + ERTS_RBT_T *p, *x, *y, *z = n; + ERTS_RBT_T null_x; /* null_x is used to get the fixup started when we + splice out a node without children. */ + + ERTS_RBT_HDBG_CHECK_TREE__(*root); + + ERTS_RBT_INIT_EMPTY_TNODE(&null_x); + + /* Remove node from tree... */ + + /* Find node to splice out */ + if (!ERTS_RBT_GET_LEFT(z) || !ERTS_RBT_GET_RIGHT(z)) + y = z; + else { + /* Set y to z:s successor */ + y = ERTS_RBT_GET_RIGHT(z); + while (1) { + ERTS_RBT_T *t = ERTS_RBT_GET_LEFT(y); + if (!t) + break; + y = t; + } + } + /* splice out y */ + x = ERTS_RBT_GET_LEFT(y); + if (!x) + x = ERTS_RBT_GET_RIGHT(y); + spliced_is_black = ERTS_RBT_IS_BLACK(y); + p = ERTS_RBT_GET_PARENT(y); + if (x) + ERTS_RBT_SET_PARENT(x, p); + else if (spliced_is_black) { + x = &null_x; + ERTS_RBT_SET_BLACK(x); + ERTS_RBT_SET_PARENT(x, p); + ERTS_RBT_SET_LEFT(y, x); + } + + if (!p) { + ERTS_RBT_ASSERT(*root == y); + *root = x; +#ifdef ERTS_RBT_UPDATE_ATTACHED_DATA_CHGROOT + ERTS_RBT_UPDATE_ATTACHED_DATA_CHGROOT(y, x); +#endif + } + else { + if (y == ERTS_RBT_GET_LEFT(p)) + ERTS_RBT_SET_LEFT(p, x); + else { + ERTS_RBT_ASSERT(y == ERTS_RBT_GET_RIGHT(p)); + ERTS_RBT_SET_RIGHT(p, x); + } +#ifdef ERTS_RBT_UPDATE_ATTACHED_DATA_DMOD + if (p != z) + ERTS_RBT_UPDATE_ATTACHED_DATA_DMOD(p, y == z ? NULL : z); +#endif + } + if (y != z) { + /* We spliced out the successor of z; replace z by the successor */ + ERTS_RBT_FUNC__(replace__)(root, z, y); +#ifdef ERTS_RBT_UPDATE_ATTACHED_DATA_DMOD + ERTS_RBT_UPDATE_ATTACHED_DATA_DMOD(y, NULL); +#endif + } + + if (spliced_is_black) { + /* We removed a black node which makes the resulting tree + violate the Red-Black Tree properties. Fixup tree... */ + + p = ERTS_RBT_GET_PARENT(x); + while (ERTS_RBT_IS_BLACK(x) && p) { + ERTS_RBT_T *r, *l; + + /* + * x has an "extra black" which we move up the tree + * until we reach the root or until we can get rid of it. + * + * y is the sibbling of x, and p is their parent + */ + + if (x == ERTS_RBT_GET_LEFT(p)) { + y = ERTS_RBT_GET_RIGHT(p); + + ERTS_RBT_ASSERT(y); + + if (ERTS_RBT_IS_RED(y)) { + ERTS_RBT_ASSERT(ERTS_RBT_GET_RIGHT(y)); + ERTS_RBT_ASSERT(ERTS_RBT_GET_LEFT(y)); + + ERTS_RBT_SET_BLACK(y); + + ERTS_RBT_ASSERT(ERTS_RBT_IS_BLACK(p)); + + ERTS_RBT_SET_RED(p); + ERTS_RBT_FUNC__(left_rotate__)(root, p); + p = ERTS_RBT_GET_PARENT(x); + y = ERTS_RBT_GET_RIGHT(p); + } + + ERTS_RBT_ASSERT(y); + ERTS_RBT_ASSERT(ERTS_RBT_IS_BLACK(y)); + + l = ERTS_RBT_GET_LEFT(y); + r = ERTS_RBT_GET_RIGHT(y); + if ((!l || ERTS_RBT_IS_BLACK(l)) + && (!r || ERTS_RBT_IS_BLACK(r))) { + ERTS_RBT_SET_RED(y); + x = p; + p = ERTS_RBT_GET_PARENT(x); + } + else { + if (!r || ERTS_RBT_IS_BLACK(r)) { + ERTS_RBT_SET_BLACK(l); + ERTS_RBT_SET_RED(y); + ERTS_RBT_FUNC__(right_rotate__)(root, y); + p = ERTS_RBT_GET_PARENT(x); + y = ERTS_RBT_GET_RIGHT(p); + } + + ERTS_RBT_ASSERT(y); + + if (p && ERTS_RBT_IS_RED(p)) { + + ERTS_RBT_SET_BLACK(p); + ERTS_RBT_SET_RED(y); + } + + ERTS_RBT_ASSERT(ERTS_RBT_GET_RIGHT(y)); + + ERTS_RBT_SET_BLACK(ERTS_RBT_GET_RIGHT(y)); + ERTS_RBT_FUNC__(left_rotate__)(root, p); + x = *root; + break; + } + } + else { + ERTS_RBT_ASSERT(x == ERTS_RBT_GET_RIGHT(p)); + + y = ERTS_RBT_GET_LEFT(p); + + ERTS_RBT_ASSERT(y); + + if (ERTS_RBT_IS_RED(y)) { + ERTS_RBT_ASSERT(ERTS_RBT_GET_RIGHT(y)); + ERTS_RBT_ASSERT(ERTS_RBT_GET_LEFT(y)); + + ERTS_RBT_SET_BLACK(y); + ERTS_RBT_ASSERT(ERTS_RBT_IS_BLACK(p)); + ERTS_RBT_SET_RED(p); + ERTS_RBT_FUNC__(right_rotate__)(root, p); + + p = ERTS_RBT_GET_PARENT(x); + y = ERTS_RBT_GET_LEFT(p); + } + + ERTS_RBT_ASSERT(y); + ERTS_RBT_ASSERT(ERTS_RBT_IS_BLACK(y)); + + l = ERTS_RBT_GET_LEFT(y); + r = ERTS_RBT_GET_RIGHT(y); + + if ((!r || ERTS_RBT_IS_BLACK(r)) + && (!l || ERTS_RBT_IS_BLACK(l))) { + ERTS_RBT_SET_RED(y); + x = p; + p = ERTS_RBT_GET_PARENT(x); + } + else { + if (!l || ERTS_RBT_IS_BLACK(l)) { + ERTS_RBT_SET_BLACK(r); + ERTS_RBT_SET_RED(y); + ERTS_RBT_FUNC__(left_rotate__)(root, y); + + p = ERTS_RBT_GET_PARENT(x); + y = ERTS_RBT_GET_LEFT(p); + } + + ERTS_RBT_ASSERT(y); + + if (p && ERTS_RBT_IS_RED(p)) { + ERTS_RBT_SET_BLACK(p); + ERTS_RBT_SET_RED(y); + } + + ERTS_RBT_ASSERT(ERTS_RBT_GET_LEFT(y)); + + ERTS_RBT_SET_BLACK(ERTS_RBT_GET_LEFT(y)); + ERTS_RBT_FUNC__(right_rotate__)(root, p); + x = *root; + break; + } + } + } + + ERTS_RBT_SET_BLACK(x); + + x = &null_x; + p = ERTS_RBT_GET_PARENT(x); + + if (p) { + if (ERTS_RBT_GET_LEFT(p) == x) + ERTS_RBT_SET_LEFT(p, NULL); + else { + ERTS_RBT_ASSERT(ERTS_RBT_GET_RIGHT(p) == x); + ERTS_RBT_SET_RIGHT(p, NULL); + } + + ERTS_RBT_ASSERT(!ERTS_RBT_GET_LEFT(x)); + ERTS_RBT_ASSERT(!ERTS_RBT_GET_RIGHT(x)); + } + else if (*root == x) { + *root = NULL; + +#ifdef ERTS_RBT_UPDATE_ATTACHED_DATA_CHGROOT + ERTS_RBT_UPDATE_ATTACHED_DATA_CHGROOT(x, NULL); +#endif + + ERTS_RBT_ASSERT(!ERTS_RBT_GET_LEFT(x)); + ERTS_RBT_ASSERT(!ERTS_RBT_GET_RIGHT(x)); + } + } + + ERTS_RBT_HDBG_CHECK_TREE__(*root); + +} + +#endif /* ERTS_RBT_WANT_DELETE */ + +#ifdef ERTS_RBT_NEED_INSERT__ + +static void +ERTS_RBT_FUNC__(insert_fixup__)(ERTS_RBT_T **root, ERTS_RBT_T *n) +{ + ERTS_RBT_T *x, *y; + + x = n; + + /* + * Rearrange the tree so that it satisfies the Red-Black Tree properties + */ + + ERTS_RBT_ASSERT(x != *root && ERTS_RBT_IS_RED(ERTS_RBT_GET_PARENT(x))); + do { + ERTS_RBT_T *p, *pp; + + /* + * x and its parent are both red. Move the red pair up the tree + * until we get to the root or until we can separate them. + */ + + p = ERTS_RBT_GET_PARENT(x); + pp = ERTS_RBT_GET_PARENT(p); + + ERTS_RBT_ASSERT(p && pp); + ERTS_RBT_ASSERT(ERTS_RBT_IS_RED(x)); + ERTS_RBT_ASSERT(ERTS_RBT_IS_BLACK(pp)); + + if (p == ERTS_RBT_GET_LEFT(pp)) { + y = ERTS_RBT_GET_RIGHT(pp); + if (y && ERTS_RBT_IS_RED(y)) { + ERTS_RBT_SET_BLACK(y); + ERTS_RBT_SET_BLACK(p); + ERTS_RBT_SET_RED(pp); + x = pp; + } + else { + + if (x == ERTS_RBT_GET_RIGHT(p)) { + x = p; + ERTS_RBT_FUNC__(left_rotate__)(root, x); + p = ERTS_RBT_GET_PARENT(x); + pp = ERTS_RBT_GET_PARENT(p); + + ERTS_RBT_ASSERT(p && pp); + } + + ERTS_RBT_ASSERT(x == ERTS_RBT_GET_LEFT(ERTS_RBT_GET_LEFT(pp))); + ERTS_RBT_ASSERT(ERTS_RBT_IS_RED(x)); + ERTS_RBT_ASSERT(ERTS_RBT_IS_RED(p)); + ERTS_RBT_ASSERT(ERTS_RBT_IS_BLACK(pp)); + ERTS_RBT_ASSERT(!y || ERTS_RBT_IS_BLACK(y)); + + + ERTS_RBT_SET_BLACK(p); + ERTS_RBT_SET_RED(pp); + ERTS_RBT_FUNC__(right_rotate__)(root, pp); + + + ERTS_RBT_ASSERT(ERTS_RBT_GET_LEFT(ERTS_RBT_GET_PARENT(x)) == x); + ERTS_RBT_ASSERT(ERTS_RBT_IS_RED(x)); + ERTS_RBT_ASSERT(ERTS_RBT_IS_RED( + ERTS_RBT_GET_RIGHT( + ERTS_RBT_GET_PARENT(x)))); + ERTS_RBT_ASSERT(!ERTS_RBT_GET_PARENT(x) + || ERTS_RBT_IS_BLACK(ERTS_RBT_GET_PARENT(x))); + break; + } + } + else { + ERTS_RBT_ASSERT(p == ERTS_RBT_GET_RIGHT(pp)); + + y = ERTS_RBT_GET_LEFT(pp); + if (y && ERTS_RBT_IS_RED(y)) { + ERTS_RBT_SET_BLACK(y); + ERTS_RBT_SET_BLACK(p); + ERTS_RBT_SET_RED(pp); + x = pp; + } + else { + + if (x == ERTS_RBT_GET_LEFT(p)) { + x = p; + ERTS_RBT_FUNC__(right_rotate__)(root, x); + p = ERTS_RBT_GET_PARENT(x); + pp = ERTS_RBT_GET_PARENT(p); + + ERTS_RBT_ASSERT(p && pp); + } + + ERTS_RBT_ASSERT(x == ERTS_RBT_GET_RIGHT(ERTS_RBT_GET_RIGHT(pp))); + ERTS_RBT_ASSERT(ERTS_RBT_IS_RED(x)); + ERTS_RBT_ASSERT(ERTS_RBT_IS_RED(p)); + ERTS_RBT_ASSERT(ERTS_RBT_IS_BLACK(pp)); + ERTS_RBT_ASSERT(!y || ERTS_RBT_IS_BLACK(y)); + + + ERTS_RBT_SET_BLACK(p); + ERTS_RBT_SET_RED(pp); + ERTS_RBT_FUNC__(left_rotate__)(root, pp); + + + ERTS_RBT_ASSERT(ERTS_RBT_GET_RIGHT(ERTS_RBT_GET_PARENT(x)) == x); + ERTS_RBT_ASSERT(ERTS_RBT_IS_RED(x)); + ERTS_RBT_ASSERT(ERTS_RBT_IS_RED( + ERTS_RBT_GET_LEFT( + ERTS_RBT_GET_PARENT(x)))); + ERTS_RBT_ASSERT(!ERTS_RBT_GET_PARENT(x) + || ERTS_RBT_IS_BLACK(ERTS_RBT_GET_PARENT(x))); + break; + } + } + } while (x != *root && ERTS_RBT_IS_RED(ERTS_RBT_GET_PARENT(x))); + + ERTS_RBT_SET_BLACK(*root); + +} + +static ERTS_INLINE ERTS_RBT_T * +ERTS_RBT_FUNC__(insert_aux__)(ERTS_RBT_T **root, ERTS_RBT_T *n, int lookup) +{ + ERTS_RBT_KEY_T kn = ERTS_RBT_GET_KEY(n); + + ERTS_RBT_HDBG_CHECK_TREE__(*root); + + ERTS_RBT_INIT_EMPTY_TNODE(n); + + if (!*root) { + ERTS_RBT_SET_BLACK(n); + *root = n; +#ifdef ERTS_RBT_UPDATE_ATTACHED_DATA_CHGROOT + ERTS_RBT_UPDATE_ATTACHED_DATA_CHGROOT(NULL, n); +#endif + } + else { + ERTS_RBT_T *p, *x = *root; + + while (1) { + ERTS_RBT_KEY_T kx; + ERTS_RBT_T *c; + + kx = ERTS_RBT_GET_KEY(x); + + if (lookup && ERTS_RBT_IS_EQ(kn, kx)) { + + ERTS_RBT_HDBG_CHECK_TREE__(*root); + + return x; + } + + if (ERTS_RBT_IS_LT(kn, kx)) { + c = ERTS_RBT_GET_LEFT(x); + if (!c) { + ERTS_RBT_SET_PARENT(n, x); + ERTS_RBT_SET_LEFT(x, n); + p = x; + break; + } + } + else { + c = ERTS_RBT_GET_RIGHT(x); + if (!c) { + ERTS_RBT_SET_PARENT(n, x); + ERTS_RBT_SET_RIGHT(x, n); + p = x; + break; + } + } + + x = c; + } + + ERTS_RBT_ASSERT(p); + + ERTS_RBT_SET_RED(n); + if (ERTS_RBT_IS_RED(p)) + ERTS_RBT_FUNC__(insert_fixup__)(root, n); + } + + ERTS_RBT_HDBG_CHECK_TREE__(*root); + + return NULL; +} + +#endif /* ERTS_RBT_NEED_INSERT__ */ + +#ifdef ERTS_RBT_WANT_LOOKUP_INSERT + +static ERTS_RBT_API_INLINE__ ERTS_RBT_T * +ERTS_RBT_FUNC__(lookup_insert)(ERTS_RBT_T **root, ERTS_RBT_T *n) +{ + return ERTS_RBT_FUNC__(insert_aux__)(root, n, !0); +} + +#endif /* ERTS_RBT_WANT_LOOKUP_INSERT */ + +#ifdef ERTS_RBT_WANT_INSERT + +static ERTS_RBT_API_INLINE__ void +ERTS_RBT_FUNC__(insert)(ERTS_RBT_T **root, ERTS_RBT_T *n) +{ + (void) ERTS_RBT_FUNC__(insert_aux__)(root, n, 0); +} + +#endif /* ERTS_RBT_WANT_INSERT */ + +#ifdef ERTS_RBT_WANT_LOOKUP + +static ERTS_RBT_API_INLINE__ ERTS_RBT_T * +ERTS_RBT_FUNC__(lookup)(ERTS_RBT_T *root, ERTS_RBT_KEY_T key) +{ + ERTS_RBT_T *x = root; + + if (!x) + return NULL; + + while (1) { + ERTS_RBT_KEY_T kx = ERTS_RBT_GET_KEY(x); + ERTS_RBT_T *c; + + if (ERTS_RBT_IS_EQ(key, kx)) + return x; + + if (ERTS_RBT_IS_LT(key, kx)) { + c = ERTS_RBT_GET_LEFT(x); + if (!c) + return NULL; + } + else { + c = ERTS_RBT_GET_RIGHT(x); + if (!c) + return NULL; + } + + x = c; + } +} + +#endif /* ERTS_RBT_WANT_LOOKUP */ + +#ifdef ERTS_RBT_WANT_SMALLEST + +static ERTS_RBT_API_INLINE__ ERTS_RBT_T * +ERTS_RBT_FUNC__(smallest)(ERTS_RBT_T *root) +{ + ERTS_RBT_T *x = root; + + if (!x) + return NULL; + + while (1) { + ERTS_RBT_T *c = ERTS_RBT_GET_LEFT(x); + if (!c) + break; + x = c; + } + + return x; +} + +#endif /* ERTS_RBT_WANT_SMALLEST */ + +#ifdef ERTS_RBT_WANT_LARGEST + +static ERTS_RBT_API_INLINE__ ERTS_RBT_T * +ERTS_RBT_FUNC__(largest)(ERTS_RBT_T *root) +{ + ERTS_RBT_T *x = root; + + if (!x) + return NULL; + + while (1) { + ERTS_RBT_T *c = ERTS_RBT_GET_RIGHT(x); + if (!c) + break; + x = c; + } + + return x; +} + +#endif /* ERTS_RBT_WANT_LARGEST */ + +#ifdef ERTS_RBT_NEED_FOREACH_UNORDERED__ + +static ERTS_INLINE int +ERTS_RBT_FUNC__(foreach_unordered__)(ERTS_RBT_T **root, + int destroying, + void (*op)(ERTS_RBT_T *, void *), + void *arg, + int yielding, + ERTS_RBT_YIELD_STATE_T__ *ystate, + Sint ylimit) +{ + ERTS_RBT_T *c, *p, *x; + + ERTS_RBT_ASSERT(!yielding || ystate); + + if (yielding && ystate->x) { + x = ystate->x; + ERTS_RBT_ASSERT(ystate->up); + goto restart_up; + } + else { + x = *root; + if (!x) + return 0; + if (destroying) + *root = NULL; + } + + while (1) { + + while (1) { + + while (1) { + c = ERTS_RBT_GET_LEFT(x); + if (!c) + break; + x = c; + } + + c = ERTS_RBT_GET_RIGHT(x); + if (!c) + break; + x = c; + } + + while (1) { +#ifdef ERTS_RBT_DEBUG + int cdir; +#endif + if (yielding && ylimit-- <= 0) { + ystate->x = x; + ystate->up = 1; + return 1; + } + + restart_up: + + p = ERTS_RBT_GET_PARENT(x); + +#ifdef ERTS_RBT_DEBUG + ERTS_RBT_ASSERT(!destroying || !ERTS_RBT_GET_LEFT(x)); + ERTS_RBT_ASSERT(!destroying || !ERTS_RBT_GET_RIGHT(x)); + + if (p) { + if (x == ERTS_RBT_GET_LEFT(p)) { + cdir = -1; + if (destroying) + ERTS_RBT_SET_LEFT(p, NULL); + } + else { + ERTS_RBT_ASSERT(x == ERTS_RBT_GET_RIGHT(p)); + cdir = 1; + if (destroying) + ERTS_RBT_SET_RIGHT(p, NULL); + } + } +#endif + + (*op)(x, arg); + + if (!p) { + if (yielding) { + ystate->x = NULL; + ystate->up = 0; + } + return 0; /* Done */ + } + + c = ERTS_RBT_GET_RIGHT(p); + if (c && c != x) { + ERTS_RBT_ASSERT(cdir < 0); + + /* Go down tree of x's sibling... */ + x = c; + break; + } + + x = p; + } + } +} + +#endif /* ERTS_RBT_NEED_FOREACH_UNORDERED__ */ + +#ifdef ERTS_RBT_NEED_FOREACH_ORDERED__ + +static ERTS_INLINE int +ERTS_RBT_FUNC__(foreach_ordered__)(ERTS_RBT_T **root, + int from_small, + int destroying, + void (*op)(ERTS_RBT_T *, void *), + void (*destroy)(ERTS_RBT_T *, void *), + void *arg, + int yielding, + ERTS_RBT_YIELD_STATE_T__ *ystate, + Sint ylimit) +{ + ERTS_RBT_T *c, *p, *x; + + ERTS_RBT_ASSERT(!yielding || ystate); + ERTS_RBT_ASSERT(!destroying || destroy); + + if (yielding && ystate->x) { + x = ystate->x; + if (ystate->up) + goto restart_up; + else + goto restart_down; + } + else { + x = *root; + if (!x) + return 0; + if (destroying) + *root = NULL; + } + + while (1) { + + while (1) { + + while (1) { + c = from_small ? ERTS_RBT_GET_LEFT(x) : ERTS_RBT_GET_RIGHT(x); + if (!c) + break; + x = c; + } + + (*op)(x, arg); + + if (yielding && --ylimit <= 0) { + ystate->x = x; + ystate->up = 0; + return 1; + } + + restart_down: + + c = from_small ? ERTS_RBT_GET_RIGHT(x) : ERTS_RBT_GET_LEFT(x); + if (!c) + break; + x = c; + } + + while (1) { + p = ERTS_RBT_GET_PARENT(x); + + if (p) { + + c = from_small ? ERTS_RBT_GET_RIGHT(p) : ERTS_RBT_GET_LEFT(p); + if (!c || c != x) { + ERTS_RBT_ASSERT((from_small + ? ERTS_RBT_GET_LEFT(p) + : ERTS_RBT_GET_RIGHT(p)) == x); + + (*op)(p, arg); + + if (yielding && --ylimit <= 0) { + ystate->x = x; + ystate->up = 1; + return 1; + restart_up: + p = ERTS_RBT_GET_PARENT(x); + } + } + + if (c && c != x) { + ERTS_RBT_ASSERT((from_small + ? ERTS_RBT_GET_LEFT(p) + : ERTS_RBT_GET_RIGHT(p)) == x); + + /* Go down tree of x's sibling... */ + x = c; + break; + } + } + + if (destroying) { + +#ifdef ERTS_RBT_DEBUG + ERTS_RBT_ASSERT(!ERTS_RBT_GET_LEFT(x) + && !ERTS_RBT_GET_RIGHT(x)); + + if (p) { + if (x == ERTS_RBT_GET_LEFT(p)) + ERTS_RBT_SET_LEFT(p, NULL); + else { + ERTS_RBT_ASSERT(x == ERTS_RBT_GET_RIGHT(p)); + ERTS_RBT_SET_RIGHT(p, NULL); + } + } +#endif + + (*destroy)(x, arg); + } + + if (!p) { + if (yielding) { + ystate->x = NULL; + ystate->up = 0; + } + return 1; /* Done */ + } + x = p; + } + } +} + +#endif /* ERTS_RBT_NEED_FOREACH_ORDERED__ */ + +#ifdef ERTS_RBT_WANT_FOREACH + +static ERTS_RBT_API_INLINE__ void +ERTS_RBT_FUNC__(foreach)(ERTS_RBT_T *root, + void (*op)(ERTS_RBT_T *, void *), + void *arg) +{ + (void) ERTS_RBT_FUNC__(foreach_unordered__)(&root, 0, op, arg, + 0, NULL, 0); +} + +#endif /* ERTS_RBT_WANT_FOREACH */ + +#ifdef ERTS_RBT_WANT_FOREACH_SMALL + +static ERTS_RBT_API_INLINE__ void +ERTS_RBT_FUNC__(foreach_small)(ERTS_RBT_T *root, + void (*op)(ERTS_RBT_T *, void *), + void *arg) +{ + (void) ERTS_RBT_FUNC__(foreach_ordered__)(&root, 1, 0, + op, NULL, arg, + 0, NULL, 0); +} + +#endif /* ERTS_RBT_WANT_FOREACH_SMALL */ + +#ifdef ERTS_RBT_WANT_FOREACH_LARGE + +static ERTS_RBT_API_INLINE__ void +ERTS_RBT_FUNC__(foreach_large)(ERTS_RBT_T *root, + void (*op)(ERTS_RBT_T *, void *), + void *arg) +{ + (void) ERTS_RBT_FUNC__(foreach_ordered__)(&root, 0, 0, + op, NULL, arg, + 0, NULL, 0); +} + +#endif /* ERTS_RBT_WANT_FOREACH_LARGE */ + +#ifdef ERTS_RBT_WANT_FOREACH_YIELDING + +static ERTS_RBT_API_INLINE__ void +ERTS_RBT_FUNC__(foreach_yielding)(ERTS_RBT_T *root, + void (*op)(ERTS_RBT_T *, void *), + void *arg, + ERTS_RBT_YIELD_STATE_T__ *ystate, + Sint ylimit) +{ + (void) ERTS_RBT_FUNC__(foreach_unordered__)(*root, 0, op, arg, + 1, ystate, ylimit); +} + +#endif /* ERTS_RBT_WANT_FOREACH_YIELDING */ + +#ifdef ERTS_RBT_WANT_FOREACH_SMALL_YIELDING + +static ERTS_RBT_API_INLINE__ int +ERTS_RBT_FUNC__(foreach_small_yielding)(ERTS_RBT_T *root, + void (*op)(ERTS_RBT_T *, void *), + void *arg, + ERTS_RBT_YIELD_STATE_T__ *ystate, + Sint ylimit) +{ + return ERTS_RBT_FUNC__(foreach_ordered__)(&root, 1, 0, + op, NULL, arg, + 1, ystate, ylimit); +} + +#endif /* ERTS_RBT_WANT_FOREACH_SMALL_YIELDING */ + +#ifdef ERTS_RBT_WANT_FOREACH_LARGE_YIELDING + +static ERTS_RBT_API_INLINE__ int +ERTS_RBT_FUNC__(foreach_large_yielding)(ERTS_RBT_T *root, + void (*op)(ERTS_RBT_T *, void *), + void *arg, + ERTS_RBT_YIELD_STATE_T__ *ystate, + Sint ylimit) +{ + return ERTS_RBT_FUNC__(foreach_ordered__)(&root, 0, 0, + op, NULL, arg, + 1, ystate, ylimit); +} + +#endif /* ERTS_RBT_WANT_FOREACH_LARGE_YIELDING */ + +#ifdef ERTS_RBT_WANT_FOREACH_DESTROY + +static ERTS_RBT_API_INLINE__ void +ERTS_RBT_FUNC__(foreach_destroy)(ERTS_RBT_T **root, + void (*op)(ERTS_RBT_T *, void *), + void *arg) +{ + (void) ERTS_RBT_FUNC__(foreach_unordered__)(root, 1, op, arg, + 0, NULL, 0); +} + +#endif /* ERTS_RBT_WANT_FOREACH_DESTROY */ + +#ifdef ERTS_RBT_WANT_FOREACH_SMALL_DESTROY + +static ERTS_RBT_API_INLINE__ void +ERTS_RBT_FUNC__(foreach_small_destroy)(ERTS_RBT_T **root, + void (*op)(ERTS_RBT_T *, void *), + void (*destr)(ERTS_RBT_T *, void *), + void *arg) +{ + (void) ERTS_RBT_FUNC__(foreach_ordered__)(root, 1, 1, + op, destr, arg, + 0, NULL, 0); +} + +#endif /* ERTS_RBT_WANT_FOREACH_SMALL_DESTROY */ + +#ifdef ERTS_RBT_WANT_FOREACH_LARGE_DESTROY + +static ERTS_RBT_API_INLINE__ void +ERTS_RBT_FUNC__(foreach_large_destroy)(ERTS_RBT_T **root, + void (*op)(ERTS_RBT_T *, void *), + void (*destr)(ERTS_RBT_T *, void *), + void *arg) +{ + (void) ERTS_RBT_FUNC__(foreach_ordered__)(root, 0, 1, + op, destr, arg, + 0, NULL, 0); +} + +#endif /* ERTS_RBT_WANT_FOREACH_LARGE_DESTROY */ + +#ifdef ERTS_RBT_WANT_FOREACH_DESTROY_YIELDING + +static ERTS_RBT_API_INLINE__ int +ERTS_RBT_FUNC__(foreach_destroy_yielding)(ERTS_RBT_T **root, + void (*op)(ERTS_RBT_T *, void *), + void *arg, + ERTS_RBT_YIELD_STATE_T__ *ystate, + Sint ylimit) +{ + return ERTS_RBT_FUNC__(foreach_unordered__)(root, 1, op, arg, + 1, ystate, ylimit); +} + +#endif /* ERTS_RBT_WANT_FOREACH_DESTROY_YIELDING */ + +#ifdef ERTS_RBT_WANT_FOREACH_SMALL_DESTROY_YIELDING + +static ERTS_RBT_API_INLINE__ int +ERTS_RBT_FUNC__(foreach_small_destroy_yielding)(ERTS_RBT_T **root, + void (*op)(ERTS_RBT_T *, void *), + void (*destr)(ERTS_RBT_T *, void *), + void *arg, + ERTS_RBT_YIELD_STATE_T__ *ystate, + Sint ylimit) +{ + return ERTS_RBT_FUNC__(foreach_ordered__)(root, 1, 1, + op, destr, arg, + 1, ystate, ylimit); +} + +#endif /* ERTS_RBT_WANT_FOREACH_SMALL_DESTROY_YIELDING */ + +#ifdef ERTS_RBT_WANT_FOREACH_LARGE_DESTROY_YIELDING + +static ERTS_RBT_API_INLINE__ int +ERTS_RBT_FUNC__(foreach_large_destroy_yielding)(ERTS_RBT_T **root, + void (*op)(ERTS_RBT_T *, void *), + void (*destr)(ERTS_RBT_T *, void *), + void *arg, + ERTS_RBT_YIELD_STATE_T__ *ystate, + Sint ylimit) +{ + return ERTS_RBT_FUNC__(foreach_ordered__)(root, 0, 1, + op, destr, arg, + 1, ystate, ylimit); +} + +#endif /* ERTS_RBT_WANT_FOREACH_LARGE_DESTROY_YIELDING */ + +#ifdef ERTS_RBT_WANT_DEBUG_PRINT + +static void +ERTS_RBT_FUNC__(debug_print)(FILE *filep, ERTS_RBT_T *x, int indent, + void (*print_node)(ERTS_RBT_T *)) +{ + if (x) { + ERTS_RBT_FUNC__(debug_print)(filep, ERTS_RBT_GET_RIGHT(x), + indent+2, print_node); + erts_fprintf(filep, + "%*s[%s:%p:", + indent, "", + ERTS_RBT_IS_BLACK(x) ? "Black" : "Red", + x); + (*print_node)(x); + erts_fprintf(filep, "]\n"); + ERTS_RBT_FUNC__(debug_print)(filep, ERTS_RBT_GET_LEFT(x), + indent+2, print_node); + } +} + +#endif /* ERTS_RBT_WANT_DEBUG_PRINT */ + +#ifdef ERTS_RBT_NEED_HDBG_CHECK_TREE__ + +static void +ERTS_RBT_FUNC__(hdbg_check_tree)(ERTS_RBT_T *root) +{ + int black_depth = -1, no_black = 0; + ERTS_RBT_T *c, *p, *x = root; + ERTS_RBT_KEY_T kx; + ERTS_RBT_KEY_T kc; + + if (!x) + return; + + ERTS_RBT_ASSERT(!ERTS_RBT_GET_PARENT(x)); + + while (1) { + + while (1) { + + while (1) { + + if (ERTS_RBT_IS_BLACK(x)) + no_black++; + else { + c = ERTS_RBT_GET_RIGHT(x); + ERTS_RBT_ASSERT(!c || ERTS_RBT_IS_BLACK(c)); + c = ERTS_RBT_GET_LEFT(x); + ERTS_RBT_ASSERT(!c || ERTS_RBT_IS_BLACK(c)); + } + + c = ERTS_RBT_GET_LEFT(x); + if (!c) + break; + + ERTS_RBT_ASSERT(x == ERTS_RBT_GET_PARENT(c)); + + kx = ERTS_RBT_GET_KEY(x); + kc = ERTS_RBT_GET_KEY(c); + + ERTS_RBT_ASSERT(ERTS_RBT_IS_LT(kc, kx) + || ERTS_RBT_IS_EQ(kc, kx)); + + x = c; + } + + c = ERTS_RBT_GET_RIGHT(x); + if (!c) { + if (black_depth < 0) + black_depth = no_black; + ERTS_RBT_ASSERT(black_depth == no_black); + break; + } + + ERTS_RBT_ASSERT(x == ERTS_RBT_GET_PARENT(c)); + + kx = ERTS_RBT_GET_KEY(x); + kc = ERTS_RBT_GET_KEY(c); + + ERTS_RBT_ASSERT(ERTS_RBT_IS_LT(kx, kc) + || ERTS_RBT_IS_EQ(kx, kc)); + x = c; + } + + while (1) { + p = ERTS_RBT_GET_PARENT(x); + + if (ERTS_RBT_IS_BLACK(x)) + no_black--; + + if (p) { + + ERTS_RBT_ASSERT(x == ERTS_RBT_GET_LEFT(p) + || x == ERTS_RBT_GET_RIGHT(p)); + + c = ERTS_RBT_GET_RIGHT(p); + if (c && c != x) { + ERTS_RBT_ASSERT(ERTS_RBT_GET_LEFT(p) == x); + + kx = ERTS_RBT_GET_KEY(x); + kc = ERTS_RBT_GET_KEY(c); + + ERTS_RBT_ASSERT(ERTS_RBT_IS_LT(kx, kc) + || ERTS_RBT_IS_EQ(kx, kc)); + /* Go down tree of x's sibling... */ + x = c; + break; + } + } + + if (!p) { + ERTS_RBT_ASSERT(root == x); + ERTS_RBT_ASSERT(no_black == 0); + return; /* Done */ + } + + x = p; + } + } +} + +#undef ERTS_RBT_PRINT_TREE__ + +#endif /* ERTS_RBT_NEED_HDBG_CHECK_TREE__ */ + +#undef ERTS_RBT_ASSERT +#undef ERTS_RBT_DEBUG +#undef ERTS_RBT_API_INLINE__ +#undef ERTS_RBT_YIELD_STATE_T__ +#undef ERTS_RBT_NEED_REPLACE__ +#undef ERTS_RBT_NEED_INSERT__ +#undef ERTS_RBT_NEED_ROTATE__ +#undef ERTS_RBT_NEED_FOREACH_UNORDERED__ +#undef ERTS_RBT_NEED_FOREACH_ORDERED__ +#undef ERTS_RBT_NEED_HDBG_CHECK_TREE__ +#undef ERTS_RBT_HDBG_CHECK_TREE__ + +#ifdef ERTS_RBT_UNDEF +# undef ERTS_RBT_PREFIX +# undef ERTS_RBT_T +# undef ERTS_RBT_KEY_T +# undef ERTS_RBT_FLAGS_T +# undef ERTS_RBT_INIT_EMPTY_TNODE +# undef ERTS_RBT_IS_RED +# undef ERTS_RBT_SET_RED +# undef ERTS_RBT_IS_BLACK +# undef ERTS_RBT_SET_BLACK +# undef ERTS_RBT_GET_FLAGS +# undef ERTS_RBT_SET_FLAGS +# undef ERTS_RBT_GET_PARENT +# undef ERTS_RBT_SET_PARENT +# undef ERTS_RBT_GET_RIGHT +# undef ERTS_RBT_SET_RIGHT +# undef ERTS_RBT_GET_LEFT +# undef ERTS_RBT_SET_LEFT +# undef ERTS_RBT_GET_KEY +# undef ERTS_RBT_IS_LT +# undef ERTS_RBT_IS_EQ +# undef ERTS_RBT_UNDEF +# undef ERTS_RBT_NO_API_INLINE +# undef ERTS_RBT_UPDATE_ATTACHED_DATA_ROTATE +# undef ERTS_RBT_UPDATE_ATTACHED_DATA_DMOD +# undef ERTS_RBT_UPDATE_ATTACHED_DATA_CHGROOT +# undef ERTS_RBT_WANT_DELETE +# undef ERTS_RBT_WANT_INSERT +# undef ERTS_RBT_WANT_LOOKUP_INSERT +# undef ERTS_RBT_WANT_REPLACE +# undef ERTS_RBT_WANT_LOOKUP +# undef ERTS_RBT_WANT_SMALLEST +# undef ERTS_RBT_WANT_LARGEST +# undef ERTS_RBT_WANT_FOREACH +# undef ERTS_RBT_WANT_FOREACH_DESTROY +# undef ERTS_RBT_WANT_FOREACH_YIELDING +# undef ERTS_RBT_WANT_FOREACH_DESTROY_YIELDING +# undef ERTS_RBT_WANT_FOREACH_SMALL +# undef ERTS_RBT_WANT_FOREACH_LARGE +# undef ERTS_RBT_WANT_FOREACH_SMALL_DESTROY +# undef ERTS_RBT_WANT_FOREACH_LARGE_DESTROY +# undef ERTS_RBT_WANT_FOREACH_SMALL_YIELDING +# undef ERTS_RBT_WANT_FOREACH_LARGE_YIELDING +# undef ERTS_RBT_WANT_FOREACH_SMALL_DESTROY_YIELDING +# undef ERTS_RBT_WANT_FOREACH_LARGE_DESTROY_YIELDING +# undef ERTS_RBT_WANT_DEBUG_PRINT +#endif diff --git a/erts/emulator/beam/erl_sched_spec_pre_alloc.c b/erts/emulator/beam/erl_sched_spec_pre_alloc.c index a490aec734..caec24bc03 100644 --- a/erts/emulator/beam/erl_sched_spec_pre_alloc.c +++ b/erts/emulator/beam/erl_sched_spec_pre_alloc.c @@ -3,16 +3,17 @@ * * Copyright Ericsson AB 2011-2012. All Rights Reserved. * - * The contents of this file are subject to the Erlang Public License, - * Version 1.1, (the "License"); you may not use this file except in - * compliance with the License. You should have received a copy of the - * Erlang Public License along with this software. If not, it can be - * retrieved online at http://www.erlang.org/. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Software distributed under the License is distributed on an "AS IS" - * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See - * the License for the specific language governing rights and limitations - * under the License. + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. * * %CopyrightEnd% */ diff --git a/erts/emulator/beam/erl_sched_spec_pre_alloc.h b/erts/emulator/beam/erl_sched_spec_pre_alloc.h index 9144c73acd..4d07b0f674 100644 --- a/erts/emulator/beam/erl_sched_spec_pre_alloc.h +++ b/erts/emulator/beam/erl_sched_spec_pre_alloc.h @@ -3,16 +3,17 @@ * * Copyright Ericsson AB 2011-2012. All Rights Reserved. * - * The contents of this file are subject to the Erlang Public License, - * Version 1.1, (the "License"); you may not use this file except in - * compliance with the License. You should have received a copy of the - * Erlang Public License along with this software. If not, it can be - * retrieved online at http://www.erlang.org/. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Software distributed under the License is distributed on an "AS IS" - * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See - * the License for the specific language governing rights and limitations - * under the License. + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. * * %CopyrightEnd% */ diff --git a/erts/emulator/beam/erl_smp.h b/erts/emulator/beam/erl_smp.h index ecb5525022..5fc5e989a6 100644 --- a/erts/emulator/beam/erl_smp.h +++ b/erts/emulator/beam/erl_smp.h @@ -3,16 +3,17 @@ * * Copyright Ericsson AB 2005-2013. All Rights Reserved. * - * The contents of this file are subject to the Erlang Public License, - * Version 1.1, (the "License"); you may not use this file except in - * compliance with the License. You should have received a copy of the - * Erlang Public License along with this software. If not, it can be - * retrieved online at http://www.erlang.org/. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Software distributed under the License is distributed on an "AS IS" - * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See - * the License for the specific language governing rights and limitations - * under the License. + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. * * %CopyrightEnd% */ @@ -26,10 +27,13 @@ #define ERL_SMP_H #include "erl_threads.h" -#ifdef ERTS_ENABLE_LOCK_COUNT +#ifdef ERTS_ENABLE_LOCK_POSITION #define erts_smp_mtx_lock(L) erts_smp_mtx_lock_x(L, __FILE__, __LINE__) +#define erts_smp_mtx_trylock(L) erts_smp_mtx_trylock_x(L, __FILE__, __LINE__) #define erts_smp_spin_lock(L) erts_smp_spin_lock_x(L, __FILE__, __LINE__) +#define erts_smp_rwmtx_tryrlock(L) erts_smp_rwmtx_tryrlock_x(L, __FILE__, __LINE__) #define erts_smp_rwmtx_rlock(L) erts_smp_rwmtx_rlock_x(L, __FILE__, __LINE__) +#define erts_smp_rwmtx_tryrwlock(L) erts_smp_rwmtx_tryrwlock_x(L, __FILE__, __LINE__) #define erts_smp_rwmtx_rwlock(L) erts_smp_rwmtx_rwlock_x(L, __FILE__, __LINE__) #define erts_smp_read_lock(L) erts_smp_read_lock_x(L, __FILE__, __LINE__) #define erts_smp_write_lock(L) erts_smp_write_lock_x(L, __FILE__, __LINE__) @@ -57,6 +61,7 @@ typedef erts_tsd_key_t erts_smp_tsd_key_t; #define erts_smp_dw_atomic_t erts_dw_atomic_t #define erts_smp_atomic_t erts_atomic_t #define erts_smp_atomic32_t erts_atomic32_t +#define erts_smp_atomic64_t erts_atomic64_t typedef erts_spinlock_t erts_smp_spinlock_t; typedef erts_rwlock_t erts_smp_rwlock_t; void erts_thr_fatal_error(int, char *); /* implemented in erl_init.c */ @@ -92,6 +97,7 @@ typedef int erts_smp_tsd_key_t; #define erts_smp_dw_atomic_t erts_no_dw_atomic_t #define erts_smp_atomic_t erts_no_atomic_t #define erts_smp_atomic32_t erts_no_atomic32_t +#define erts_smp_atomic64_t erts_no_atomic64_t #if __GNUC__ > 2 typedef struct { } erts_smp_spinlock_t; typedef struct { } erts_smp_rwlock_t; @@ -131,10 +137,11 @@ ERTS_GLB_INLINE void erts_smp_mtx_init_locked_x(erts_smp_mtx_t *mtx, ERTS_GLB_INLINE void erts_smp_mtx_init(erts_smp_mtx_t *mtx, char *name); ERTS_GLB_INLINE void erts_smp_mtx_init_locked(erts_smp_mtx_t *mtx, char *name); ERTS_GLB_INLINE void erts_smp_mtx_destroy(erts_smp_mtx_t *mtx); -ERTS_GLB_INLINE int erts_smp_mtx_trylock(erts_smp_mtx_t *mtx); -#ifdef ERTS_ENABLE_LOCK_COUNT -ERTS_GLB_INLINE void erts_smp_mtx_lock_x(erts_smp_mtx_t *mtx, char *file, int line); +#ifdef ERTS_ENABLE_LOCK_POSITION +ERTS_GLB_INLINE int erts_smp_mtx_trylock_x(erts_smp_mtx_t *mtx, char *file, unsigned int line); +ERTS_GLB_INLINE void erts_smp_mtx_lock_x(erts_smp_mtx_t *mtx, char *file, unsigned int line); #else +ERTS_GLB_INLINE int erts_smp_mtx_trylock(erts_smp_mtx_t *mtx); ERTS_GLB_INLINE void erts_smp_mtx_lock(erts_smp_mtx_t *mtx); #endif ERTS_GLB_INLINE void erts_smp_mtx_unlock(erts_smp_mtx_t *mtx); @@ -159,16 +166,18 @@ ERTS_GLB_INLINE void erts_smp_rwmtx_init_opt(erts_smp_rwmtx_t *rwmtx, ERTS_GLB_INLINE void erts_smp_rwmtx_init(erts_smp_rwmtx_t *rwmtx, char *name); ERTS_GLB_INLINE void erts_smp_rwmtx_destroy(erts_smp_rwmtx_t *rwmtx); -ERTS_GLB_INLINE int erts_smp_rwmtx_tryrlock(erts_smp_rwmtx_t *rwmtx); -#ifdef ERTS_ENABLE_LOCK_COUNT +#ifdef ERTS_ENABLE_LOCK_POSITION +ERTS_GLB_INLINE int erts_smp_rwmtx_tryrlock_x(erts_smp_rwmtx_t *rwmtx, char *file, unsigned int line); ERTS_GLB_INLINE void erts_smp_rwmtx_rlock_x(erts_smp_rwmtx_t *rwmtx, char *file, unsigned int line); ERTS_GLB_INLINE void erts_smp_rwmtx_rwlock_x(erts_smp_rwmtx_t *rwmtx, char *file, unsigned int line); +ERTS_GLB_INLINE int erts_smp_rwmtx_tryrwlock_x(erts_smp_rwmtx_t *rwmtx, char *file, unsigned int line); #else +ERTS_GLB_INLINE int erts_smp_rwmtx_tryrlock(erts_smp_rwmtx_t *rwmtx); ERTS_GLB_INLINE void erts_smp_rwmtx_rlock(erts_smp_rwmtx_t *rwmtx); ERTS_GLB_INLINE void erts_smp_rwmtx_rwlock(erts_smp_rwmtx_t *rwmtx); +ERTS_GLB_INLINE int erts_smp_rwmtx_tryrwlock(erts_smp_rwmtx_t *rwmtx); #endif ERTS_GLB_INLINE void erts_smp_rwmtx_runlock(erts_smp_rwmtx_t *rwmtx); -ERTS_GLB_INLINE int erts_smp_rwmtx_tryrwlock(erts_smp_rwmtx_t *rwmtx); ERTS_GLB_INLINE void erts_smp_rwmtx_rwunlock(erts_smp_rwmtx_t *rwmtx); ERTS_GLB_INLINE int erts_smp_lc_rwmtx_is_rlocked(erts_smp_rwmtx_t *mtx); ERTS_GLB_INLINE int erts_smp_lc_rwmtx_is_rwlocked(erts_smp_rwmtx_t *mtx); @@ -179,7 +188,7 @@ ERTS_GLB_INLINE void erts_smp_spinlock_init(erts_smp_spinlock_t *lock, char *name); ERTS_GLB_INLINE void erts_smp_spinlock_destroy(erts_smp_spinlock_t *lock); ERTS_GLB_INLINE void erts_smp_spin_unlock(erts_smp_spinlock_t *lock); -#ifdef ERTS_ENABLE_LOCK_COUNT +#ifdef ERTS_ENABLE_LOCK_POSITION ERTS_GLB_INLINE void erts_smp_spin_lock_x(erts_smp_spinlock_t *lock, char *file, unsigned int line); #else ERTS_GLB_INLINE void erts_smp_spin_lock(erts_smp_spinlock_t *lock); @@ -192,7 +201,7 @@ ERTS_GLB_INLINE void erts_smp_rwlock_init(erts_smp_rwlock_t *lock, char *name); ERTS_GLB_INLINE void erts_smp_rwlock_destroy(erts_smp_rwlock_t *lock); ERTS_GLB_INLINE void erts_smp_read_unlock(erts_smp_rwlock_t *lock); -#ifdef ERTS_ENABLE_LOCK_COUNT +#ifdef ERTS_ENABLE_LOCK_POSITION ERTS_GLB_INLINE void erts_smp_read_lock_x(erts_smp_rwlock_t *lock, char *file, unsigned int line); ERTS_GLB_INLINE void erts_smp_write_lock_x(erts_smp_rwlock_t *lock, char *file, unsigned int line); #else @@ -202,7 +211,8 @@ ERTS_GLB_INLINE void erts_smp_write_lock(erts_smp_rwlock_t *lock); ERTS_GLB_INLINE void erts_smp_write_unlock(erts_smp_rwlock_t *lock); ERTS_GLB_INLINE int erts_smp_lc_rwlock_is_rlocked(erts_smp_rwlock_t *lock); ERTS_GLB_INLINE int erts_smp_lc_rwlock_is_rwlocked(erts_smp_rwlock_t *lock); -ERTS_GLB_INLINE void erts_smp_tsd_key_create(erts_smp_tsd_key_t *keyp); +ERTS_GLB_INLINE void erts_smp_tsd_key_create(erts_smp_tsd_key_t *keyp, + char *keyname); ERTS_GLB_INLINE void erts_smp_tsd_key_delete(erts_smp_tsd_key_t key); ERTS_GLB_INLINE void erts_smp_tsd_set(erts_smp_tsd_key_t key, void *value); ERTS_GLB_INLINE void * erts_smp_tsd_get(erts_smp_tsd_key_t key); @@ -482,6 +492,116 @@ ERTS_GLB_INLINE void erts_smp_thr_sigwait(const sigset_t *set, int *sig); #define erts_smp_atomic32_set_dirty erts_atomic32_set_dirty #define erts_smp_atomic32_read_dirty erts_atomic32_read_dirty +/* 64-bit atomics */ + +#define erts_smp_atomic64_init_nob erts_atomic64_init_nob +#define erts_smp_atomic64_set_nob erts_atomic64_set_nob +#define erts_smp_atomic64_read_nob erts_atomic64_read_nob +#define erts_smp_atomic64_inc_read_nob erts_atomic64_inc_read_nob +#define erts_smp_atomic64_dec_read_nob erts_atomic64_dec_read_nob +#define erts_smp_atomic64_inc_nob erts_atomic64_inc_nob +#define erts_smp_atomic64_dec_nob erts_atomic64_dec_nob +#define erts_smp_atomic64_add_read_nob erts_atomic64_add_read_nob +#define erts_smp_atomic64_add_nob erts_atomic64_add_nob +#define erts_smp_atomic64_read_bor_nob erts_atomic64_read_bor_nob +#define erts_smp_atomic64_read_band_nob erts_atomic64_read_band_nob +#define erts_smp_atomic64_xchg_nob erts_atomic64_xchg_nob +#define erts_smp_atomic64_cmpxchg_nob erts_atomic64_cmpxchg_nob +#define erts_smp_atomic64_read_bset_nob erts_atomic64_read_bset_nob + +#define erts_smp_atomic64_init_mb erts_atomic64_init_mb +#define erts_smp_atomic64_set_mb erts_atomic64_set_mb +#define erts_smp_atomic64_read_mb erts_atomic64_read_mb +#define erts_smp_atomic64_inc_read_mb erts_atomic64_inc_read_mb +#define erts_smp_atomic64_dec_read_mb erts_atomic64_dec_read_mb +#define erts_smp_atomic64_inc_mb erts_atomic64_inc_mb +#define erts_smp_atomic64_dec_mb erts_atomic64_dec_mb +#define erts_smp_atomic64_add_read_mb erts_atomic64_add_read_mb +#define erts_smp_atomic64_add_mb erts_atomic64_add_mb +#define erts_smp_atomic64_read_bor_mb erts_atomic64_read_bor_mb +#define erts_smp_atomic64_read_band_mb erts_atomic64_read_band_mb +#define erts_smp_atomic64_xchg_mb erts_atomic64_xchg_mb +#define erts_smp_atomic64_cmpxchg_mb erts_atomic64_cmpxchg_mb +#define erts_smp_atomic64_read_bset_mb erts_atomic64_read_bset_mb + +#define erts_smp_atomic64_init_acqb erts_atomic64_init_acqb +#define erts_smp_atomic64_set_acqb erts_atomic64_set_acqb +#define erts_smp_atomic64_read_acqb erts_atomic64_read_acqb +#define erts_smp_atomic64_inc_read_acqb erts_atomic64_inc_read_acqb +#define erts_smp_atomic64_dec_read_acqb erts_atomic64_dec_read_acqb +#define erts_smp_atomic64_inc_acqb erts_atomic64_inc_acqb +#define erts_smp_atomic64_dec_acqb erts_atomic64_dec_acqb +#define erts_smp_atomic64_add_read_acqb erts_atomic64_add_read_acqb +#define erts_smp_atomic64_add_acqb erts_atomic64_add_acqb +#define erts_smp_atomic64_read_bor_acqb erts_atomic64_read_bor_acqb +#define erts_smp_atomic64_read_band_acqb erts_atomic64_read_band_acqb +#define erts_smp_atomic64_xchg_acqb erts_atomic64_xchg_acqb +#define erts_smp_atomic64_cmpxchg_acqb erts_atomic64_cmpxchg_acqb +#define erts_smp_atomic64_read_bset_acqb erts_atomic64_read_bset_acqb + +#define erts_smp_atomic64_init_relb erts_atomic64_init_relb +#define erts_smp_atomic64_set_relb erts_atomic64_set_relb +#define erts_smp_atomic64_read_relb erts_atomic64_read_relb +#define erts_smp_atomic64_inc_read_relb erts_atomic64_inc_read_relb +#define erts_smp_atomic64_dec_read_relb erts_atomic64_dec_read_relb +#define erts_smp_atomic64_inc_relb erts_atomic64_inc_relb +#define erts_smp_atomic64_dec_relb erts_atomic64_dec_relb +#define erts_smp_atomic64_add_read_relb erts_atomic64_add_read_relb +#define erts_smp_atomic64_add_relb erts_atomic64_add_relb +#define erts_smp_atomic64_read_bor_relb erts_atomic64_read_bor_relb +#define erts_smp_atomic64_read_band_relb erts_atomic64_read_band_relb +#define erts_smp_atomic64_xchg_relb erts_atomic64_xchg_relb +#define erts_smp_atomic64_cmpxchg_relb erts_atomic64_cmpxchg_relb +#define erts_smp_atomic64_read_bset_relb erts_atomic64_read_bset_relb + +#define erts_smp_atomic64_init_ddrb erts_atomic64_init_ddrb +#define erts_smp_atomic64_set_ddrb erts_atomic64_set_ddrb +#define erts_smp_atomic64_read_ddrb erts_atomic64_read_ddrb +#define erts_smp_atomic64_inc_read_ddrb erts_atomic64_inc_read_ddrb +#define erts_smp_atomic64_dec_read_ddrb erts_atomic64_dec_read_ddrb +#define erts_smp_atomic64_inc_ddrb erts_atomic64_inc_ddrb +#define erts_smp_atomic64_dec_ddrb erts_atomic64_dec_ddrb +#define erts_smp_atomic64_add_read_ddrb erts_atomic64_add_read_ddrb +#define erts_smp_atomic64_add_ddrb erts_atomic64_add_ddrb +#define erts_smp_atomic64_read_bor_ddrb erts_atomic64_read_bor_ddrb +#define erts_smp_atomic64_read_band_ddrb erts_atomic64_read_band_ddrb +#define erts_smp_atomic64_xchg_ddrb erts_atomic64_xchg_ddrb +#define erts_smp_atomic64_cmpxchg_ddrb erts_atomic64_cmpxchg_ddrb +#define erts_smp_atomic64_read_bset_ddrb erts_atomic64_read_bset_ddrb + +#define erts_smp_atomic64_init_rb erts_atomic64_init_rb +#define erts_smp_atomic64_set_rb erts_atomic64_set_rb +#define erts_smp_atomic64_read_rb erts_atomic64_read_rb +#define erts_smp_atomic64_inc_read_rb erts_atomic64_inc_read_rb +#define erts_smp_atomic64_dec_read_rb erts_atomic64_dec_read_rb +#define erts_smp_atomic64_inc_rb erts_atomic64_inc_rb +#define erts_smp_atomic64_dec_rb erts_atomic64_dec_rb +#define erts_smp_atomic64_add_read_rb erts_atomic64_add_read_rb +#define erts_smp_atomic64_add_rb erts_atomic64_add_rb +#define erts_smp_atomic64_read_bor_rb erts_atomic64_read_bor_rb +#define erts_smp_atomic64_read_band_rb erts_atomic64_read_band_rb +#define erts_smp_atomic64_xchg_rb erts_atomic64_xchg_rb +#define erts_smp_atomic64_cmpxchg_rb erts_atomic64_cmpxchg_rb +#define erts_smp_atomic64_read_bset_rb erts_atomic64_read_bset_rb + +#define erts_smp_atomic64_init_wb erts_atomic64_init_wb +#define erts_smp_atomic64_set_wb erts_atomic64_set_wb +#define erts_smp_atomic64_read_wb erts_atomic64_read_wb +#define erts_smp_atomic64_inc_read_wb erts_atomic64_inc_read_wb +#define erts_smp_atomic64_dec_read_wb erts_atomic64_dec_read_wb +#define erts_smp_atomic64_inc_wb erts_atomic64_inc_wb +#define erts_smp_atomic64_dec_wb erts_atomic64_dec_wb +#define erts_smp_atomic64_add_read_wb erts_atomic64_add_read_wb +#define erts_smp_atomic64_add_wb erts_atomic64_add_wb +#define erts_smp_atomic64_read_bor_wb erts_atomic64_read_bor_wb +#define erts_smp_atomic64_read_band_wb erts_atomic64_read_band_wb +#define erts_smp_atomic64_xchg_wb erts_atomic64_xchg_wb +#define erts_smp_atomic64_cmpxchg_wb erts_atomic64_cmpxchg_wb +#define erts_smp_atomic64_read_bset_wb erts_atomic64_read_bset_wb + +#define erts_smp_atomic64_set_dirty erts_atomic64_set_dirty +#define erts_smp_atomic64_read_dirty erts_atomic64_read_dirty + #else /* !ERTS_SMP */ /* Double word size atomics */ @@ -744,6 +864,116 @@ ERTS_GLB_INLINE void erts_smp_thr_sigwait(const sigset_t *set, int *sig); #define erts_smp_atomic32_set_dirty erts_no_atomic32_set #define erts_smp_atomic32_read_dirty erts_no_atomic32_read +/* 64-bit atomics */ + +#define erts_smp_atomic64_init_nob erts_no_atomic64_set +#define erts_smp_atomic64_set_nob erts_no_atomic64_set +#define erts_smp_atomic64_read_nob erts_no_atomic64_read +#define erts_smp_atomic64_inc_read_nob erts_no_atomic64_inc_read +#define erts_smp_atomic64_dec_read_nob erts_no_atomic64_dec_read +#define erts_smp_atomic64_inc_nob erts_no_atomic64_inc +#define erts_smp_atomic64_dec_nob erts_no_atomic64_dec +#define erts_smp_atomic64_add_read_nob erts_no_atomic64_add_read +#define erts_smp_atomic64_add_nob erts_no_atomic64_add +#define erts_smp_atomic64_read_bor_nob erts_no_atomic64_read_bor +#define erts_smp_atomic64_read_band_nob erts_no_atomic64_read_band +#define erts_smp_atomic64_xchg_nob erts_no_atomic64_xchg +#define erts_smp_atomic64_cmpxchg_nob erts_no_atomic64_cmpxchg +#define erts_smp_atomic64_read_bset_nob erts_no_atomic64_read_bset + +#define erts_smp_atomic64_init_mb erts_no_atomic64_set +#define erts_smp_atomic64_set_mb erts_no_atomic64_set +#define erts_smp_atomic64_read_mb erts_no_atomic64_read +#define erts_smp_atomic64_inc_read_mb erts_no_atomic64_inc_read +#define erts_smp_atomic64_dec_read_mb erts_no_atomic64_dec_read +#define erts_smp_atomic64_inc_mb erts_no_atomic64_inc +#define erts_smp_atomic64_dec_mb erts_no_atomic64_dec +#define erts_smp_atomic64_add_read_mb erts_no_atomic64_add_read +#define erts_smp_atomic64_add_mb erts_no_atomic64_add +#define erts_smp_atomic64_read_bor_mb erts_no_atomic64_read_bor +#define erts_smp_atomic64_read_band_mb erts_no_atomic64_read_band +#define erts_smp_atomic64_xchg_mb erts_no_atomic64_xchg +#define erts_smp_atomic64_cmpxchg_mb erts_no_atomic64_cmpxchg +#define erts_smp_atomic64_read_bset_mb erts_no_atomic64_read_bset + +#define erts_smp_atomic64_init_acqb erts_no_atomic64_set +#define erts_smp_atomic64_set_acqb erts_no_atomic64_set +#define erts_smp_atomic64_read_acqb erts_no_atomic64_read +#define erts_smp_atomic64_inc_read_acqb erts_no_atomic64_inc_read +#define erts_smp_atomic64_dec_read_acqb erts_no_atomic64_dec_read +#define erts_smp_atomic64_inc_acqb erts_no_atomic64_inc +#define erts_smp_atomic64_dec_acqb erts_no_atomic64_dec +#define erts_smp_atomic64_add_read_acqb erts_no_atomic64_add_read +#define erts_smp_atomic64_add_acqb erts_no_atomic64_add +#define erts_smp_atomic64_read_bor_acqb erts_no_atomic64_read_bor +#define erts_smp_atomic64_read_band_acqb erts_no_atomic64_read_band +#define erts_smp_atomic64_xchg_acqb erts_no_atomic64_xchg +#define erts_smp_atomic64_cmpxchg_acqb erts_no_atomic64_cmpxchg +#define erts_smp_atomic64_read_bset_acqb erts_no_atomic64_read_bset + +#define erts_smp_atomic64_init_relb erts_no_atomic64_set +#define erts_smp_atomic64_set_relb erts_no_atomic64_set +#define erts_smp_atomic64_read_relb erts_no_atomic64_read +#define erts_smp_atomic64_inc_read_relb erts_no_atomic64_inc_read +#define erts_smp_atomic64_dec_read_relb erts_no_atomic64_dec_read +#define erts_smp_atomic64_inc_relb erts_no_atomic64_inc +#define erts_smp_atomic64_dec_relb erts_no_atomic64_dec +#define erts_smp_atomic64_add_read_relb erts_no_atomic64_add_read +#define erts_smp_atomic64_add_relb erts_no_atomic64_add +#define erts_smp_atomic64_read_bor_relb erts_no_atomic64_read_bor +#define erts_smp_atomic64_read_band_relb erts_no_atomic64_read_band +#define erts_smp_atomic64_xchg_relb erts_no_atomic64_xchg +#define erts_smp_atomic64_cmpxchg_relb erts_no_atomic64_cmpxchg +#define erts_smp_atomic64_read_bset_relb erts_no_atomic64_read_bset + +#define erts_smp_atomic64_init_ddrb erts_no_atomic64_set +#define erts_smp_atomic64_set_ddrb erts_no_atomic64_set +#define erts_smp_atomic64_read_ddrb erts_no_atomic64_read +#define erts_smp_atomic64_inc_read_ddrb erts_no_atomic64_inc_read +#define erts_smp_atomic64_dec_read_ddrb erts_no_atomic64_dec_read +#define erts_smp_atomic64_inc_ddrb erts_no_atomic64_inc +#define erts_smp_atomic64_dec_ddrb erts_no_atomic64_dec +#define erts_smp_atomic64_add_read_ddrb erts_no_atomic64_add_read +#define erts_smp_atomic64_add_ddrb erts_no_atomic64_add +#define erts_smp_atomic64_read_bor_ddrb erts_no_atomic64_read_bor +#define erts_smp_atomic64_read_band_ddrb erts_no_atomic64_read_band +#define erts_smp_atomic64_xchg_ddrb erts_no_atomic64_xchg +#define erts_smp_atomic64_cmpxchg_ddrb erts_no_atomic64_cmpxchg +#define erts_smp_atomic64_read_bset_ddrb erts_no_atomic64_read_bset + +#define erts_smp_atomic64_init_rb erts_no_atomic64_set +#define erts_smp_atomic64_set_rb erts_no_atomic64_set +#define erts_smp_atomic64_read_rb erts_no_atomic64_read +#define erts_smp_atomic64_inc_read_rb erts_no_atomic64_inc_read +#define erts_smp_atomic64_dec_read_rb erts_no_atomic64_dec_read +#define erts_smp_atomic64_inc_rb erts_no_atomic64_inc +#define erts_smp_atomic64_dec_rb erts_no_atomic64_dec +#define erts_smp_atomic64_add_read_rb erts_no_atomic64_add_read +#define erts_smp_atomic64_add_rb erts_no_atomic64_add +#define erts_smp_atomic64_read_bor_rb erts_no_atomic64_read_bor +#define erts_smp_atomic64_read_band_rb erts_no_atomic64_read_band +#define erts_smp_atomic64_xchg_rb erts_no_atomic64_xchg +#define erts_smp_atomic64_cmpxchg_rb erts_no_atomic64_cmpxchg +#define erts_smp_atomic64_read_bset_rb erts_no_atomic64_read_bset + +#define erts_smp_atomic64_init_wb erts_no_atomic64_set +#define erts_smp_atomic64_set_wb erts_no_atomic64_set +#define erts_smp_atomic64_read_wb erts_no_atomic64_read +#define erts_smp_atomic64_inc_read_wb erts_no_atomic64_inc_read +#define erts_smp_atomic64_dec_read_wb erts_no_atomic64_dec_read +#define erts_smp_atomic64_inc_wb erts_no_atomic64_inc +#define erts_smp_atomic64_dec_wb erts_no_atomic64_dec +#define erts_smp_atomic64_add_read_wb erts_no_atomic64_add_read +#define erts_smp_atomic64_add_wb erts_no_atomic64_add +#define erts_smp_atomic64_read_bor_wb erts_no_atomic64_read_bor +#define erts_smp_atomic64_read_band_wb erts_no_atomic64_read_band +#define erts_smp_atomic64_xchg_wb erts_no_atomic64_xchg +#define erts_smp_atomic64_cmpxchg_wb erts_no_atomic64_cmpxchg +#define erts_smp_atomic64_read_bset_wb erts_no_atomic64_read_bset + +#define erts_smp_atomic64_set_dirty erts_no_atomic64_set +#define erts_smp_atomic64_read_dirty erts_no_atomic64_read + #endif /* !ERTS_SMP */ #if ERTS_GLB_INLINE_INCL_FUNC_DEF @@ -835,7 +1065,7 @@ ERTS_GLB_INLINE void erts_smp_mtx_init_x(erts_smp_mtx_t *mtx, char *name, Eterm extra) { #ifdef ERTS_SMP - erts_mtx_init_x(mtx, name, extra); + erts_mtx_init_x(mtx, name, extra, 1); #endif } @@ -843,7 +1073,7 @@ ERTS_GLB_INLINE void erts_smp_mtx_init_locked_x(erts_smp_mtx_t *mtx, char *name, Eterm extra) { #ifdef ERTS_SMP - erts_mtx_init_locked_x(mtx, name, extra); + erts_mtx_init_locked_x(mtx, name, extra, 1); #endif } @@ -872,9 +1102,15 @@ erts_smp_mtx_destroy(erts_smp_mtx_t *mtx) } ERTS_GLB_INLINE int +#ifdef ERTS_ENABLE_LOCK_POSITION +erts_smp_mtx_trylock_x(erts_smp_mtx_t *mtx, char *file, unsigned int line) +#else erts_smp_mtx_trylock(erts_smp_mtx_t *mtx) +#endif { -#ifdef ERTS_SMP +#if defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_POSITION) + return erts_mtx_trylock_x(mtx,file,line); +#elif defined(ERTS_SMP) return erts_mtx_trylock(mtx); #else return 0; @@ -884,13 +1120,13 @@ erts_smp_mtx_trylock(erts_smp_mtx_t *mtx) ERTS_GLB_INLINE void -#ifdef ERTS_ENABLE_LOCK_COUNT -erts_smp_mtx_lock_x(erts_smp_mtx_t *mtx, char *file, int line) +#ifdef ERTS_ENABLE_LOCK_POSITION +erts_smp_mtx_lock_x(erts_smp_mtx_t *mtx, char *file, unsigned int line) #else erts_smp_mtx_lock(erts_smp_mtx_t *mtx) #endif { -#if defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_COUNT) +#if defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_POSITION) erts_mtx_lock_x(mtx, file, line); #elif defined(ERTS_SMP) erts_mtx_lock(mtx); @@ -1020,9 +1256,15 @@ erts_smp_rwmtx_destroy(erts_smp_rwmtx_t *rwmtx) } ERTS_GLB_INLINE int +#ifdef ERTS_ENABLE_LOCK_POSITION +erts_smp_rwmtx_tryrlock_x(erts_smp_rwmtx_t *rwmtx, char *file, unsigned int line) +#else erts_smp_rwmtx_tryrlock(erts_smp_rwmtx_t *rwmtx) +#endif { -#ifdef ERTS_SMP +#if defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_POSITION) + return erts_rwmtx_tryrlock_x(rwmtx, file, line); +#elif defined(ERTS_SMP) return erts_rwmtx_tryrlock(rwmtx); #else return 0; @@ -1030,13 +1272,13 @@ erts_smp_rwmtx_tryrlock(erts_smp_rwmtx_t *rwmtx) } ERTS_GLB_INLINE void -#ifdef ERTS_ENABLE_LOCK_COUNT +#ifdef ERTS_ENABLE_LOCK_POSITION erts_smp_rwmtx_rlock_x(erts_smp_rwmtx_t *rwmtx, char *file, unsigned int line) #else erts_smp_rwmtx_rlock(erts_smp_rwmtx_t *rwmtx) #endif { -#if defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_COUNT) +#if defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_POSITION) erts_rwmtx_rlock_x(rwmtx, file, line); #elif defined(ERTS_SMP) erts_rwmtx_rlock(rwmtx); @@ -1053,9 +1295,15 @@ erts_smp_rwmtx_runlock(erts_smp_rwmtx_t *rwmtx) ERTS_GLB_INLINE int +#ifdef ERTS_ENABLE_LOCK_POSITION +erts_smp_rwmtx_tryrwlock_x(erts_smp_rwmtx_t *rwmtx, char *file, unsigned int line) +#else erts_smp_rwmtx_tryrwlock(erts_smp_rwmtx_t *rwmtx) +#endif { -#ifdef ERTS_SMP +#if defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_POSITION) + return erts_rwmtx_tryrwlock_x(rwmtx, file, line); +#elif defined(ERTS_SMP) return erts_rwmtx_tryrwlock(rwmtx); #else return 0; @@ -1063,13 +1311,13 @@ erts_smp_rwmtx_tryrwlock(erts_smp_rwmtx_t *rwmtx) } ERTS_GLB_INLINE void -#ifdef ERTS_ENABLE_LOCK_COUNT +#ifdef ERTS_ENABLE_LOCK_POSITION erts_smp_rwmtx_rwlock_x(erts_smp_rwmtx_t *rwmtx, char *file, unsigned int line) #else erts_smp_rwmtx_rwlock(erts_smp_rwmtx_t *rwmtx) #endif { -#if defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_COUNT) +#if defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_POSITION) erts_rwmtx_rwlock_x(rwmtx, file, line); #elif defined(ERTS_SMP) erts_rwmtx_rwlock(rwmtx); @@ -1171,13 +1419,13 @@ erts_smp_spin_unlock(erts_smp_spinlock_t *lock) } ERTS_GLB_INLINE void -#ifdef ERTS_ENABLE_LOCK_COUNT +#ifdef ERTS_ENABLE_LOCK_POSITION erts_smp_spin_lock_x(erts_smp_spinlock_t *lock, char *file, unsigned int line) #else erts_smp_spin_lock(erts_smp_spinlock_t *lock) #endif { -#if defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_COUNT) +#if defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_POSITION) erts_spin_lock_x(lock, file, line); #elif defined(ERTS_SMP) erts_spin_lock(lock); @@ -1237,13 +1485,13 @@ erts_smp_read_unlock(erts_smp_rwlock_t *lock) } ERTS_GLB_INLINE void -#ifdef ERTS_ENABLE_LOCK_COUNT +#ifdef ERTS_ENABLE_LOCK_POSITION erts_smp_read_lock_x(erts_smp_rwlock_t *lock, char *file, unsigned int line) #else erts_smp_read_lock(erts_smp_rwlock_t *lock) #endif { -#if defined(ERTS_ENABLE_LOCK_COUNT) && defined(ERTS_SMP) +#if defined(ERTS_ENABLE_LOCK_POSITION) && defined(ERTS_SMP) erts_read_lock_x(lock, file, line); #elif defined(ERTS_SMP) erts_read_lock(lock); @@ -1263,13 +1511,13 @@ erts_smp_write_unlock(erts_smp_rwlock_t *lock) } ERTS_GLB_INLINE void -#ifdef ERTS_ENABLE_LOCK_COUNT +#ifdef ERTS_ENABLE_LOCK_POSITION erts_smp_write_lock_x(erts_smp_rwlock_t *lock, char *file, unsigned int line) #else erts_smp_write_lock(erts_smp_rwlock_t *lock) #endif { -#if defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_COUNT) +#if defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_POSITION) erts_write_lock_x(lock, file, line); #elif defined(ERTS_SMP) erts_write_lock(lock); @@ -1299,10 +1547,10 @@ erts_smp_lc_rwlock_is_rwlocked(erts_smp_rwlock_t *lock) } ERTS_GLB_INLINE void -erts_smp_tsd_key_create(erts_smp_tsd_key_t *keyp) +erts_smp_tsd_key_create(erts_smp_tsd_key_t *keyp, char* keyname) { #ifdef ERTS_SMP - erts_tsd_key_create(keyp); + erts_tsd_key_create(keyp,keyname); #endif } diff --git a/erts/emulator/beam/erl_sock.h b/erts/emulator/beam/erl_sock.h index 7ae6116dc5..7be6062115 100644 --- a/erts/emulator/beam/erl_sock.h +++ b/erts/emulator/beam/erl_sock.h @@ -3,16 +3,17 @@ * * Copyright Ericsson AB 2003-2009. All Rights Reserved. * - * The contents of this file are subject to the Erlang Public License, - * Version 1.1, (the "License"); you may not use this file except in - * compliance with the License. You should have received a copy of the - * Erlang Public License along with this software. If not, it can be - * retrieved online at http://www.erlang.org/. - * - * Software distributed under the License is distributed on an "AS IS" - * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See - * the License for the specific language governing rights and limitations - * under the License. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. * * %CopyrightEnd% */ diff --git a/erts/emulator/beam/erl_sys_driver.h b/erts/emulator/beam/erl_sys_driver.h index dab4a94a9b..4031eef0aa 100644 --- a/erts/emulator/beam/erl_sys_driver.h +++ b/erts/emulator/beam/erl_sys_driver.h @@ -3,16 +3,17 @@ * * Copyright Ericsson AB 2001-2013. All Rights Reserved. * - * The contents of this file are subject to the Erlang Public License, - * Version 1.1, (the "License"); you may not use this file except in - * compliance with the License. You should have received a copy of the - * Erlang Public License along with this software. If not, it can be - * retrieved online at http://www.erlang.org/. - * - * Software distributed under the License is distributed on an "AS IS" - * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See - * the License for the specific language governing rights and limitations - * under the License. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. * * %CopyrightEnd% */ diff --git a/erts/emulator/beam/erl_term.c b/erts/emulator/beam/erl_term.c index 2f206ffbec..bae3385d3f 100644 --- a/erts/emulator/beam/erl_term.c +++ b/erts/emulator/beam/erl_term.c @@ -3,16 +3,17 @@ * * Copyright Ericsson AB 2000-2013. All Rights Reserved. * - * The contents of this file are subject to the Erlang Public License, - * Version 1.1, (the "License"); you may not use this file except in - * compliance with the License. You should have received a copy of the - * Erlang Public License along with this software. If not, it can be - * retrieved online at http://www.erlang.org/. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Software distributed under the License is distributed on an "AS IS" - * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See - * the License for the specific language governing rights and limitations - * under the License. + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. * * %CopyrightEnd% */ @@ -23,6 +24,7 @@ #include "sys.h" #include "erl_vm.h" #include "global.h" +#include "erl_map.h" #include <stdlib.h> #include <stdio.h> @@ -39,7 +41,7 @@ et_abort(const char *expr, const char *file, unsigned line) * Prevent infinite loop. */ have_been_called = 1; - erl_exit(1, "TYPE ASSERTION FAILED, file %s, line %u: %s\n", file, line, expr); + erts_exit(ERTS_ERROR_EXIT, "TYPE ASSERTION FAILED, file %s, line %u: %s\n", file, line, expr); } #else erts_fprintf(stderr, "TYPE ASSERTION FAILED, file %s, line %u: %s\n", file, line, expr); @@ -85,8 +87,13 @@ unsigned tag_val_def(Wterm x) case (_TAG_HEADER_EXTERNAL_PID >> _TAG_PRIMARY_SIZE): return EXTERNAL_PID_DEF; case (_TAG_HEADER_EXTERNAL_PORT >> _TAG_PRIMARY_SIZE): return EXTERNAL_PORT_DEF; case (_TAG_HEADER_EXTERNAL_REF >> _TAG_PRIMARY_SIZE): return EXTERNAL_REF_DEF; - default: return BINARY_DEF; + case (_TAG_HEADER_MAP >> _TAG_PRIMARY_SIZE): return MAP_DEF; + case (_TAG_HEADER_REFC_BIN >> _TAG_PRIMARY_SIZE): return BINARY_DEF; + case (_TAG_HEADER_HEAP_BIN >> _TAG_PRIMARY_SIZE): return BINARY_DEF; + case (_TAG_HEADER_SUB_BIN >> _TAG_PRIMARY_SIZE): return BINARY_DEF; + case (_TAG_HEADER_BIN_MATCHSTATE >> _TAG_PRIMARY_SIZE): return MATCHSTATE_DEF; } + break; } case TAG_PRIMARY_IMMED1: { @@ -123,10 +130,10 @@ FUNTY checked_##FUN(ARGTY x, const char *file, unsigned line) \ return _unchecked_##FUN(x); \ } -ET_DEFINE_CHECKED(Eterm,make_boxed,Eterm*,_is_taggable_pointer); +ET_DEFINE_CHECKED(Eterm,make_boxed,const Eterm*,_is_taggable_pointer); ET_DEFINE_CHECKED(int,is_boxed,Eterm,!is_header); ET_DEFINE_CHECKED(Eterm*,boxed_val,Wterm,_boxed_precond); -ET_DEFINE_CHECKED(Eterm,make_list,Eterm*,_is_taggable_pointer); +ET_DEFINE_CHECKED(Eterm,make_list,const Eterm*,_is_taggable_pointer); ET_DEFINE_CHECKED(int,is_not_list,Eterm,!is_header); ET_DEFINE_CHECKED(Eterm*,list_val,Wterm,_list_precond); ET_DEFINE_CHECKED(Uint,unsigned_val,Eterm,is_small); diff --git a/erts/emulator/beam/erl_term.h b/erts/emulator/beam/erl_term.h index 953edf79ea..d5e91d80d9 100644 --- a/erts/emulator/beam/erl_term.h +++ b/erts/emulator/beam/erl_term.h @@ -1,18 +1,19 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2000-2013. All Rights Reserved. + * Copyright Ericsson AB 2000-2014. All Rights Reserved. * - * The contents of this file are subject to the Erlang Public License, - * Version 1.1, (the "License"); you may not use this file except in - * compliance with the License. You should have received a copy of the - * Erlang Public License along with this software. If not, it can be - * retrieved online at http://www.erlang.org/. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Software distributed under the License is distributed on an "AS IS" - * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See - * the License for the specific language governing rights and limitations - * under the License. + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. * * %CopyrightEnd% */ @@ -112,11 +113,11 @@ struct erl_node_; /* Declared in erl_node_tables.h */ * 1000 REFC_BINARY | | * 1001 HEAP_BINARY | BINARIES | * 1010 SUB_BINARY | | - * 1011 Not used + * 1011 Not used; see comment below * 1100 EXTERNAL_PID | | * 1101 EXTERNAL_PORT | EXTERNAL THINGS | * 1110 EXTERNAL_REF | | - * 1111 Not used + * 1111 MAP * * COMMENTS: * @@ -135,30 +136,33 @@ struct erl_node_; /* Declared in erl_node_tables.h */ #define REF_SUBTAG (0x4 << _TAG_PRIMARY_SIZE) /* REF */ #define FUN_SUBTAG (0x5 << _TAG_PRIMARY_SIZE) /* FUN */ #define FLOAT_SUBTAG (0x6 << _TAG_PRIMARY_SIZE) /* FLOAT */ -#define EXPORT_SUBTAG (0x7 << _TAG_PRIMARY_SIZE) /* FLOAT */ +#define EXPORT_SUBTAG (0x7 << _TAG_PRIMARY_SIZE) /* FLOAT */ #define _BINARY_XXX_MASK (0x3 << _TAG_PRIMARY_SIZE) #define REFC_BINARY_SUBTAG (0x8 << _TAG_PRIMARY_SIZE) /* BINARY */ #define HEAP_BINARY_SUBTAG (0x9 << _TAG_PRIMARY_SIZE) /* BINARY */ #define SUB_BINARY_SUBTAG (0xA << _TAG_PRIMARY_SIZE) /* BINARY */ +/* _BINARY_XXX_MASK depends on 0xB being unused */ #define EXTERNAL_PID_SUBTAG (0xC << _TAG_PRIMARY_SIZE) /* EXTERNAL_PID */ #define EXTERNAL_PORT_SUBTAG (0xD << _TAG_PRIMARY_SIZE) /* EXTERNAL_PORT */ #define EXTERNAL_REF_SUBTAG (0xE << _TAG_PRIMARY_SIZE) /* EXTERNAL_REF */ - - -#define _TAG_HEADER_ARITYVAL (TAG_PRIMARY_HEADER|ARITYVAL_SUBTAG) -#define _TAG_HEADER_FUN (TAG_PRIMARY_HEADER|FUN_SUBTAG) -#define _TAG_HEADER_POS_BIG (TAG_PRIMARY_HEADER|POS_BIG_SUBTAG) -#define _TAG_HEADER_NEG_BIG (TAG_PRIMARY_HEADER|NEG_BIG_SUBTAG) -#define _TAG_HEADER_FLOAT (TAG_PRIMARY_HEADER|FLOAT_SUBTAG) -#define _TAG_HEADER_EXPORT (TAG_PRIMARY_HEADER|EXPORT_SUBTAG) -#define _TAG_HEADER_REF (TAG_PRIMARY_HEADER|REF_SUBTAG) -#define _TAG_HEADER_REFC_BIN (TAG_PRIMARY_HEADER|REFC_BINARY_SUBTAG) -#define _TAG_HEADER_HEAP_BIN (TAG_PRIMARY_HEADER|HEAP_BINARY_SUBTAG) -#define _TAG_HEADER_SUB_BIN (TAG_PRIMARY_HEADER|SUB_BINARY_SUBTAG) -#define _TAG_HEADER_EXTERNAL_PID (TAG_PRIMARY_HEADER|EXTERNAL_PID_SUBTAG) -#define _TAG_HEADER_EXTERNAL_PORT (TAG_PRIMARY_HEADER|EXTERNAL_PORT_SUBTAG) -#define _TAG_HEADER_EXTERNAL_REF (TAG_PRIMARY_HEADER|EXTERNAL_REF_SUBTAG) +#define MAP_SUBTAG (0xF << _TAG_PRIMARY_SIZE) /* MAP */ + + +#define _TAG_HEADER_ARITYVAL (TAG_PRIMARY_HEADER|ARITYVAL_SUBTAG) +#define _TAG_HEADER_FUN (TAG_PRIMARY_HEADER|FUN_SUBTAG) +#define _TAG_HEADER_POS_BIG (TAG_PRIMARY_HEADER|POS_BIG_SUBTAG) +#define _TAG_HEADER_NEG_BIG (TAG_PRIMARY_HEADER|NEG_BIG_SUBTAG) +#define _TAG_HEADER_FLOAT (TAG_PRIMARY_HEADER|FLOAT_SUBTAG) +#define _TAG_HEADER_EXPORT (TAG_PRIMARY_HEADER|EXPORT_SUBTAG) +#define _TAG_HEADER_REF (TAG_PRIMARY_HEADER|REF_SUBTAG) +#define _TAG_HEADER_REFC_BIN (TAG_PRIMARY_HEADER|REFC_BINARY_SUBTAG) +#define _TAG_HEADER_HEAP_BIN (TAG_PRIMARY_HEADER|HEAP_BINARY_SUBTAG) +#define _TAG_HEADER_SUB_BIN (TAG_PRIMARY_HEADER|SUB_BINARY_SUBTAG) +#define _TAG_HEADER_EXTERNAL_PID (TAG_PRIMARY_HEADER|EXTERNAL_PID_SUBTAG) +#define _TAG_HEADER_EXTERNAL_PORT (TAG_PRIMARY_HEADER|EXTERNAL_PORT_SUBTAG) +#define _TAG_HEADER_EXTERNAL_REF (TAG_PRIMARY_HEADER|EXTERNAL_REF_SUBTAG) #define _TAG_HEADER_BIN_MATCHSTATE (TAG_PRIMARY_HEADER|BIN_MATCHSTATE_SUBTAG) +#define _TAG_HEADER_MAP (TAG_PRIMARY_HEADER|MAP_SUBTAG) #define _TAG_HEADER_MASK 0x3F @@ -195,7 +199,7 @@ struct erl_node_; /* Declared in erl_node_tables.h */ #endif #define _is_aligned(x) (((Uint)(x) & 0x3) == 0) #define _unchecked_make_boxed(x) ((Uint) COMPRESS_POINTER(x) + TAG_PRIMARY_BOXED) -_ET_DECLARE_CHECKED(Eterm,make_boxed,Eterm*) +_ET_DECLARE_CHECKED(Eterm,make_boxed,const Eterm*) #define make_boxed(x) _ET_APPLY(make_boxed,(x)) #if 1 #define _is_not_boxed(x) ((x) & (_TAG_PRIMARY_MASK-TAG_PRIMARY_BOXED)) @@ -211,7 +215,7 @@ _ET_DECLARE_CHECKED(Eterm*,boxed_val,Wterm) /* cons cell ("list") access methods */ #define _unchecked_make_list(x) ((Uint) COMPRESS_POINTER(x) + TAG_PRIMARY_LIST) -_ET_DECLARE_CHECKED(Eterm,make_list,Eterm*) +_ET_DECLARE_CHECKED(Eterm,make_list,const Eterm*) #define make_list(x) _ET_APPLY(make_list,(x)) #if 1 #define _unchecked_is_not_list(x) ((x) & (_TAG_PRIMARY_MASK-TAG_PRIMARY_LIST)) @@ -293,9 +297,10 @@ _ET_DECLARE_CHECKED(Uint,atom_val,Eterm) #define atom_val(x) _ET_APPLY(atom_val,(x)) /* header (arityval or thing) access methods */ -#define _make_header(sz,tag) ((Uint)(((sz) << _HEADER_ARITY_OFFS) + (tag))) +#define _make_header(sz,tag) ((Uint)(((Uint)(sz) << _HEADER_ARITY_OFFS) + (tag))) #define is_header(x) (((x) & _TAG_PRIMARY_MASK) == TAG_PRIMARY_HEADER) -#define _unchecked_header_arity(x) ((x) >> _HEADER_ARITY_OFFS) +#define _unchecked_header_arity(x) \ + (is_map_header(x) ? MAP_HEADER_ARITY(x) : ((x) >> _HEADER_ARITY_OFFS)) _ET_DECLARE_CHECKED(Uint,header_arity,Eterm) #define header_arity(x) _ET_APPLY(header_arity,(x)) @@ -354,7 +359,11 @@ _ET_DECLARE_CHECKED(Uint,thing_subtag,Eterm) #define is_value(x) ((x) != THE_NON_VALUE) /* binary object access methods */ -#define is_binary_header(x) (((x) & (_TAG_HEADER_MASK-_BINARY_XXX_MASK)) == _TAG_HEADER_REFC_BIN) +#define is_binary_header(x) \ + ((((x) & (_TAG_HEADER_MASK)) == _TAG_HEADER_REFC_BIN) || \ + (((x) & (_TAG_HEADER_MASK)) == _TAG_HEADER_HEAP_BIN) || \ + (((x) & (_TAG_HEADER_MASK)) == _TAG_HEADER_SUB_BIN)) + #define make_binary(x) make_boxed((Eterm*)(x)) #define is_binary(x) (is_boxed((x)) && is_binary_header(*boxed_val((x)))) #define is_not_binary(x) (!is_binary((x))) @@ -887,7 +896,8 @@ typedef struct external_thing_ { (((x) & _TAG_HEADER_MASK) == _TAG_HEADER_EXTERNAL_REF) #define is_external_header(x) \ - (((x) & (_TAG_HEADER_MASK-_BINARY_XXX_MASK)) == _TAG_HEADER_EXTERNAL_PID) + (((x) & (_TAG_HEADER_MASK-_BINARY_XXX_MASK)) == _TAG_HEADER_EXTERNAL_PID \ + && ((x) & _TAG_HEADER_MASK) != _TAG_HEADER_MAP) #define is_external(x) (is_boxed((x)) && is_external_header(*boxed_val((x)))) @@ -983,6 +993,44 @@ _ET_DECLARE_CHECKED(Uint32*,external_ref_data,Wterm) _ET_DECLARE_CHECKED(struct erl_node_*,external_ref_node,Eterm) #define external_ref_node(x) _ET_APPLY(external_ref_node,(x)) +/* maps */ + +#define MAP_HEADER_TAG_SZ (2) +#define MAP_HEADER_ARITY_SZ (8) +#define MAP_HEADER_VAL_SZ (16) + +#define MAP_HEADER_TAG_FLATMAP_HEAD (0x0) +#define MAP_HEADER_TAG_HAMT_NODE_BITMAP (0x1) +#define MAP_HEADER_TAG_HAMT_HEAD_ARRAY (0x2) +#define MAP_HEADER_TAG_HAMT_HEAD_BITMAP (0x3) + +#define MAP_HEADER_TYPE(Hdr) (((Hdr) >> (_HEADER_ARITY_OFFS)) & (0x3)) +#define MAP_HEADER_ARITY(Hdr) (((Hdr) >> (_HEADER_ARITY_OFFS + MAP_HEADER_TAG_SZ)) & (0xff)) +#define MAP_HEADER_VAL(Hdr) (((Hdr) >> (_HEADER_ARITY_OFFS + MAP_HEADER_TAG_SZ + MAP_HEADER_ARITY_SZ)) & (0xffff)) + +#define make_hashmap(x) make_boxed((Eterm*)(x)) +#define make_hashmap_rel make_boxed_rel +#define is_hashmap(x) (is_boxed((x)) && is_hashmap_header(*boxed_val((x)))) +#define is_not_hashmap(x) (!is_hashmap(x)) +#define is_hashmap_rel(RTERM,BASE) is_hashmap(rterm2wterm(RTERM,BASE)) +#define is_hashmap_header(x) (((x) & (_HEADER_MAP_HASHMAP_HEAD_MASK)) == HAMT_SUBTAG_HEAD_ARRAY) +#define hashmap_val(x) _unchecked_boxed_val((x)) +#define hashmap_val_rel(RTERM, BASE) hashmap_val(rterm2wterm(RTERM, BASE)) + +#define make_flatmap(x) make_boxed((Eterm*)(x)) +#define make_flatmap_rel(x, BASE) make_boxed_rel((Eterm*)(x),(BASE)) +#define is_flatmap(x) (is_boxed((x)) && is_flatmap_header(*boxed_val((x)))) +#define is_flatmap_rel(RTERM,BASE) is_flatmap(rterm2wterm(RTERM,BASE)) +#define is_not_flatmap(x) (!is_flatmap((x))) +#define is_flatmap_header(x) (((x) & (_HEADER_MAP_SUBTAG_MASK)) == HAMT_SUBTAG_HEAD_FLATMAP) +#define flatmap_val(x) (_unchecked_boxed_val((x))) +#define flatmap_val_rel(RTERM, BASE) flatmap_val(rterm2wterm(RTERM, BASE)) + +#define is_map_header(x) (((x) & (_TAG_HEADER_MASK)) == _TAG_HEADER_MAP) +#define is_map(x) (is_boxed((x)) && is_map_header(*boxed_val(x))) +#define is_not_map(x) (!is_map(x)) +#define is_map_rel(RTERM,BASE) is_map(rterm2wterm(RTERM,BASE)) + /* number tests */ #define is_integer(x) (is_small(x) || is_big(x)) @@ -1064,8 +1112,8 @@ _ET_DECLARE_CHECKED(Uint,y_reg_index,Uint) /* * Backwards compatibility definitions: - * - #define virtal *_DEF constants with values that fit term order: - * number < atom < ref < fun < port < pid < tuple < nil < cons < binary + * - #define virtual *_DEF constants with values that fit term order: + * number < atom < ref < fun < port < pid < tuple < map < nil < cons < binary * - tag_val_def() function generates virtual _DEF tag * - not_eq_tags() and NUMBER_CODE() defined in terms * of the tag_val_def() function @@ -1074,19 +1122,23 @@ _ET_DECLARE_CHECKED(Uint,y_reg_index,Uint) #define BINARY_DEF 0x0 #define LIST_DEF 0x1 #define NIL_DEF 0x2 -#define TUPLE_DEF 0x3 -#define PID_DEF 0x4 -#define EXTERNAL_PID_DEF 0x5 -#define PORT_DEF 0x6 -#define EXTERNAL_PORT_DEF 0x7 -#define EXPORT_DEF 0x8 -#define FUN_DEF 0x9 -#define REF_DEF 0xa -#define EXTERNAL_REF_DEF 0xb -#define ATOM_DEF 0xc -#define FLOAT_DEF 0xd -#define BIG_DEF 0xe -#define SMALL_DEF 0xf +#define MAP_DEF 0x3 +#define TUPLE_DEF 0x4 +#define PID_DEF 0x5 +#define EXTERNAL_PID_DEF 0x6 +#define PORT_DEF 0x7 +#define EXTERNAL_PORT_DEF 0x8 +#define EXPORT_DEF 0x9 +#define FUN_DEF 0xa +#define REF_DEF 0xb +#define EXTERNAL_REF_DEF 0xc +#define ATOM_DEF 0xd +#define FLOAT_DEF 0xe +#define BIG_DEF 0xf +#define SMALL_DEF 0x10 +#define MATCHSTATE_DEF 0x11 /* not a "real" term */ + +#define FIRST_VACANT_TAG_DEF 0x12 #if ET_DEBUG extern unsigned tag_val_def_debug(Wterm, const char*, unsigned); @@ -1096,8 +1148,8 @@ extern unsigned tag_val_def(Wterm); #endif #define not_eq_tags(X,Y) (tag_val_def((X)) ^ tag_val_def((Y))) -#define NUMBER_CODE(x,y) ((tag_val_def(x) << 4) | tag_val_def(y)) -#define _NUMBER_CODE(TX,TY) ((TX << 4) | TY) +#define NUMBER_CODE(x,y) ((tag_val_def(x) << 5) | tag_val_def(y)) +#define _NUMBER_CODE(TX,TY) ((TX << 5) | TY) #define SMALL_SMALL _NUMBER_CODE(SMALL_DEF,SMALL_DEF) #define SMALL_BIG _NUMBER_CODE(SMALL_DEF,BIG_DEF) #define SMALL_FLOAT _NUMBER_CODE(SMALL_DEF,FLOAT_DEF) @@ -1126,6 +1178,7 @@ extern unsigned tag_val_def(Wterm); #define make_tuple_rel make_boxed_rel #define make_external_rel make_boxed_rel #define make_internal_ref_rel make_boxed_rel +#define make_big_rel make_boxed_rel #define binary_val_rel(RTERM, BASE) binary_val(rterm2wterm(RTERM, BASE)) #define list_val_rel(RTERM, BASE) list_val(rterm2wterm(RTERM, BASE)) diff --git a/erts/emulator/beam/erl_thr_progress.c b/erts/emulator/beam/erl_thr_progress.c index 1292cb0678..7b06fd840f 100644 --- a/erts/emulator/beam/erl_thr_progress.c +++ b/erts/emulator/beam/erl_thr_progress.c @@ -3,16 +3,17 @@ * * Copyright Ericsson AB 2011-2013. All Rights Reserved. * - * The contents of this file are subject to the Erlang Public License, - * Version 1.1, (the "License"); you may not use this file except in - * compliance with the License. You should have received a copy of the - * Erlang Public License along with this software. If not, it can be - * retrieved online at http://www.erlang.org/. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Software distributed under the License is distributed on an "AS IS" - * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See - * the License for the specific language governing rights and limitations - * under the License. + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. * * %CopyrightEnd% */ @@ -115,70 +116,24 @@ #undef read_nob #define read_nob erts_thr_prgr_read_nob__ -#ifdef ARCH_64 - -static ERTS_INLINE void -set_mb(ERTS_THR_PRGR_ATOMIC *atmc, ErtsThrPrgrVal val) -{ - erts_atomic_set_mb(atmc, val); -} - -static ERTS_INLINE void -set_nob(ERTS_THR_PRGR_ATOMIC *atmc, ErtsThrPrgrVal val) -{ - erts_atomic_set_nob(atmc, val); -} - -static ERTS_INLINE void -init_nob(ERTS_THR_PRGR_ATOMIC *atmc, ErtsThrPrgrVal val) -{ - erts_atomic_init_nob(atmc, val); -} - -#else - -#undef dw_aint_to_val -#define dw_aint_to_val erts_thr_prgr_dw_aint_to_val__ - -static void -val_to_dw_aint(erts_dw_aint_t *dw_aint, ErtsThrPrgrVal val) -{ -#ifdef ETHR_SU_DW_NAINT_T__ - dw_aint->dw_sint = (ETHR_SU_DW_NAINT_T__) val; -#else - dw_aint->sint[ERTS_DW_AINT_LOW_WORD] - = (erts_aint_t) (val & 0xffffffff); - dw_aint->sint[ERTS_DW_AINT_HIGH_WORD] - = (erts_aint_t) ((val >> 32) & 0xffffffff); -#endif -} - static ERTS_INLINE void set_mb(ERTS_THR_PRGR_ATOMIC *atmc, ErtsThrPrgrVal val) { - erts_dw_aint_t dw_aint; - val_to_dw_aint(&dw_aint, val); - erts_dw_atomic_set_mb(atmc, &dw_aint); + erts_atomic64_set_mb(atmc, (erts_aint64_t) val); } static ERTS_INLINE void set_nob(ERTS_THR_PRGR_ATOMIC *atmc, ErtsThrPrgrVal val) { - erts_dw_aint_t dw_aint; - val_to_dw_aint(&dw_aint, val); - erts_dw_atomic_set_nob(atmc, &dw_aint); + erts_atomic64_set_nob(atmc, (erts_aint64_t) val); } static ERTS_INLINE void init_nob(ERTS_THR_PRGR_ATOMIC *atmc, ErtsThrPrgrVal val) { - erts_dw_aint_t dw_aint; - val_to_dw_aint(&dw_aint, val); - erts_dw_atomic_init_nob(atmc, &dw_aint); + erts_atomic64_init_nob(atmc, (erts_aint64_t) val); } -#endif - /* #define ERTS_THR_PROGRESS_STATE_DEBUG */ #ifdef ERTS_THR_PROGRESS_STATE_DEBUG @@ -417,8 +372,9 @@ void erts_thr_progress_pre_init(void) { intrnl = NULL; - erts_tsd_key_create(&erts_thr_prgr_data_key__); - init_nob(&erts_thr_prgr__.current, 0); + erts_tsd_key_create(&erts_thr_prgr_data_key__, + "erts_thr_prgr_data_key"); + init_nob(&erts_thr_prgr__.current, ERTS_THR_PRGR_VAL_FIRST); } void @@ -546,7 +502,7 @@ erts_thr_progress_register_unmanaged_thread(ErtsThrPrgrCallbacks *callbacks) if (tpd) { if (!tpd->is_temporary) - erl_exit(ERTS_ABORT_EXIT, + erts_exit(ERTS_ABORT_EXIT, "%s:%d:%s(): Double register of thread\n", __FILE__, __LINE__, __func__); is_blocking = tpd->is_blocking; @@ -568,7 +524,7 @@ erts_thr_progress_register_unmanaged_thread(ErtsThrPrgrCallbacks *callbacks) #endif ASSERT(tpd->id >= 0); if (tpd->id >= intrnl->unmanaged.no) - erl_exit(ERTS_ABORT_EXIT, + erts_exit(ERTS_ABORT_EXIT, "%s:%d:%s(): Too many unmanaged registered threads\n", __FILE__, __LINE__, __func__); @@ -591,7 +547,7 @@ erts_thr_progress_register_managed_thread(ErtsSchedulerData *esdp, if (tpd) { if (!tpd->is_temporary) - erl_exit(ERTS_ABORT_EXIT, + erts_exit(ERTS_ABORT_EXIT, "%s:%d:%s(): Double register of thread\n", __FILE__, __LINE__, __func__); is_blocking = tpd->is_blocking; @@ -612,7 +568,7 @@ erts_thr_progress_register_managed_thread(ErtsSchedulerData *esdp, tpd->id = erts_atomic32_inc_read_nob(&intrnl->misc.data.managed_id); ASSERT(tpd->id >= 0); if (tpd->id >= intrnl->managed.no) - erl_exit(ERTS_ABORT_EXIT, + erts_exit(ERTS_ABORT_EXIT, "%s:%d:%s(): Too many managed registered threads\n", __FILE__, __LINE__, __func__); @@ -1077,7 +1033,7 @@ has_reached_wakeup(ErtsThrPrgrVal wakeup) limit += 1; if (!erts_thr_progress_has_passed__(limit, wakeup)) - erl_exit(ERTS_ABORT_EXIT, + erts_exit(ERTS_ABORT_EXIT, "Invalid wakeup request value found:" " current=%b64u, wakeup=%b64u, limit=%b64u", current, wakeup, limit); @@ -1146,7 +1102,7 @@ request_wakeup_managed(ErtsThrPrgrData *tpd, ErtsThrPrgrVal value) ix = erts_atomic32_inc_read_nob(&mwd->len) - 1; #if ERTS_THR_PRGR_DBG_CHK_WAKEUP_REQUEST_VALUE if (ix >= intrnl->managed.no) - erl_exit(ERTS_ABORT_EXIT, "Internal error: Too many wakeup requests\n"); + erts_exit(ERTS_ABORT_EXIT, "Internal error: Too many wakeup requests\n"); #endif mwd->id[ix] = tpd->id; @@ -1380,25 +1336,10 @@ erts_thr_progress_block(void) thr_progress_block(tmp_thr_prgr_data(NULL), 1); } -void -erts_thr_progress_fatal_error_block(SWord timeout, - ErtsThrPrgrData *tmp_tpd_bufp) +int +erts_thr_progress_fatal_error_block(ErtsThrPrgrData *tmp_tpd_bufp) { ErtsThrPrgrData *tpd = perhaps_thr_prgr_data(NULL); - erts_aint32_t bc; - SWord time_left = timeout; - SysTimeval to; - - /* - * Counting poll intervals may give us a too long timeout - * if cpu is busy. If we got tolerant time of day we use it - * to prevent this. - */ - if (!erts_disable_tolerant_timeofday) { - erts_get_timeval(&to); - to.tv_sec += timeout / 1000; - to.tv_sec += timeout % 1000; - } if (!tpd) { /* @@ -1411,9 +1352,25 @@ erts_thr_progress_fatal_error_block(SWord timeout, init_tmp_thr_prgr_data(tpd); } - bc = thr_progress_block(tpd, 0); - if (bc == 0) - return; /* Succefully blocked all managed threads */ + /* Returns number of threads that have not yes been blocked */ + return thr_progress_block(tpd, 0); +} + +void +erts_thr_progress_fatal_error_wait(SWord timeout) { + erts_aint32_t bc; + SWord time_left = timeout; + ErtsMonotonicTime timeout_time; + ErtsSchedulerData *esdp = erts_get_scheduler_data(); + + /* + * Counting poll intervals may give us a too long timeout + * if cpu is busy. We use timeout time to try to prevent + * this. In case we havn't got time correction this may + * however fail too... + */ + timeout_time = erts_get_monotonic_time(esdp); + timeout_time += ERTS_MSEC_TO_MONOTONIC((ErtsMonotonicTime) timeout); while (1) { if (erts_milli_sleep(ERTS_THR_PRGR_FTL_ERR_BLCK_POLL_INTERVAL) == 0) @@ -1423,14 +1380,8 @@ erts_thr_progress_fatal_error_block(SWord timeout, break; /* Succefully blocked all managed threads */ if (time_left <= 0) break; /* Timeout */ - if (!erts_disable_tolerant_timeofday) { - SysTimeval now; - erts_get_timeval(&now); - if (now.tv_sec > to.tv_sec) - break; /* Timeout */ - if (now.tv_sec == to.tv_sec && now.tv_usec >= to.tv_usec) - break; /* Timeout */ - } + if (timeout_time <= erts_get_monotonic_time(esdp)) + break; /* Timeout */ } } diff --git a/erts/emulator/beam/erl_thr_progress.h b/erts/emulator/beam/erl_thr_progress.h index 1416aa6166..b89cc4c267 100644 --- a/erts/emulator/beam/erl_thr_progress.h +++ b/erts/emulator/beam/erl_thr_progress.h @@ -1,18 +1,19 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2011-2012. All Rights Reserved. + * Copyright Ericsson AB 2011-2013. All Rights Reserved. * - * The contents of this file are subject to the Erlang Public License, - * Version 1.1, (the "License"); you may not use this file except in - * compliance with the License. You should have received a copy of the - * Erlang Public License along with this software. If not, it can be - * retrieved online at http://www.erlang.org/. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Software distributed under the License is distributed on an "AS IS" - * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See - * the License for the specific language governing rights and limitations - * under the License. + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. * * %CopyrightEnd% */ @@ -83,8 +84,8 @@ typedef struct { ErtsThrPrgrLeaderState leader_state; } ErtsThrPrgrData; -void erts_thr_progress_fatal_error_block(SWord timeout, - ErtsThrPrgrData *tmp_tpd_bufp); +int erts_thr_progress_fatal_error_block(ErtsThrPrgrData *tmp_tpd_bufp); +void erts_thr_progress_fatal_error_wait(SWord timeout); #endif /* ERTS_SMP */ @@ -108,16 +109,14 @@ struct ErtsThrPrgrLaterOp_ { #ifdef ERTS_SMP +/* ERTS_THR_PRGR_VAL_FIRST should only be used when initializing... */ +#define ERTS_THR_PRGR_VAL_FIRST ((ErtsThrPrgrVal) 0) #define ERTS_THR_PRGR_VAL_WAITING (~((ErtsThrPrgrVal) 0)) #define ERTS_THR_PRGR_INVALID (~((ErtsThrPrgrVal) 0)) extern erts_tsd_key_t erts_thr_prgr_data_key__; -#ifdef ARCH_64 -# define ERTS_THR_PRGR_ATOMIC erts_atomic_t -#else /* ARCH_32 */ -# define ERTS_THR_PRGR_ATOMIC erts_dw_atomic_t -#endif +#define ERTS_THR_PRGR_ATOMIC erts_atomic64_t typedef struct { void *arg; @@ -156,10 +155,6 @@ void erts_thr_progress_unmanaged_continue__(int umrefc_ix); void erts_thr_progress_dbg_print_state(void); -#ifdef ARCH_32 -#define ERTS_THR_PRGR_ATOMIC erts_dw_atomic_t -ERTS_GLB_INLINE ErtsThrPrgrVal erts_thr_prgr_dw_aint_to_val__(erts_dw_aint_t *dw_aint); -#endif ERTS_GLB_INLINE ErtsThrPrgrVal erts_thr_prgr_read_nob__(ERTS_THR_PRGR_ATOMIC *atmc); ERTS_GLB_INLINE ErtsThrPrgrVal erts_thr_prgr_read_acqb__(ERTS_THR_PRGR_ATOMIC *atmc); ERTS_GLB_INLINE ErtsThrPrgrVal erts_thr_prgr_read_mb__(ERTS_THR_PRGR_ATOMIC *atmc); @@ -182,68 +177,24 @@ ERTS_GLB_INLINE int erts_thr_progress_has_reached(ErtsThrPrgrVal val); #if ERTS_GLB_INLINE_INCL_FUNC_DEF -#ifdef ARCH_64 - ERTS_GLB_INLINE ErtsThrPrgrVal erts_thr_prgr_read_nob__(ERTS_THR_PRGR_ATOMIC *atmc) { - return (ErtsThrPrgrVal) erts_atomic_read_nob(atmc); + return (ErtsThrPrgrVal) erts_atomic64_read_nob(atmc); } ERTS_GLB_INLINE ErtsThrPrgrVal erts_thr_prgr_read_acqb__(ERTS_THR_PRGR_ATOMIC *atmc) { - return (ErtsThrPrgrVal) erts_atomic_read_acqb(atmc); + return (ErtsThrPrgrVal) erts_atomic64_read_acqb(atmc); } ERTS_GLB_INLINE ErtsThrPrgrVal erts_thr_prgr_read_mb__(ERTS_THR_PRGR_ATOMIC *atmc) { - return (ErtsThrPrgrVal) erts_atomic_read_mb(atmc); -} - -#else /* ARCH_32 */ - -ERTS_GLB_INLINE ErtsThrPrgrVal -erts_thr_prgr_dw_aint_to_val__(erts_dw_aint_t *dw_aint) -{ -#ifdef ETHR_SU_DW_NAINT_T__ - return (ErtsThrPrgrVal) dw_aint->dw_sint; -#else - ErtsThrPrgrVal res; - res = (ErtsThrPrgrVal) ((Uint32) dw_aint->sint[ERTS_DW_AINT_HIGH_WORD]); - res <<= 32; - res |= (ErtsThrPrgrVal) ((Uint32) dw_aint->sint[ERTS_DW_AINT_LOW_WORD]); - return res; -#endif + return (ErtsThrPrgrVal) erts_atomic64_read_mb(atmc); } -ERTS_GLB_INLINE ErtsThrPrgrVal -erts_thr_prgr_read_nob__(ERTS_THR_PRGR_ATOMIC *atmc) -{ - erts_dw_aint_t dw_aint; - erts_dw_atomic_read_nob(atmc, &dw_aint); - return erts_thr_prgr_dw_aint_to_val__(&dw_aint); -} - -ERTS_GLB_INLINE ErtsThrPrgrVal -erts_thr_prgr_read_acqb__(ERTS_THR_PRGR_ATOMIC *atmc) -{ - erts_dw_aint_t dw_aint; - erts_dw_atomic_read_acqb(atmc, &dw_aint); - return erts_thr_prgr_dw_aint_to_val__(&dw_aint); -} - -ERTS_GLB_INLINE ErtsThrPrgrVal -erts_thr_prgr_read_mb__(ERTS_THR_PRGR_ATOMIC *atmc) -{ - erts_dw_aint_t dw_aint; - erts_dw_atomic_read_mb(atmc, &dw_aint); - return erts_thr_prgr_dw_aint_to_val__(&dw_aint); -} - -#endif - ERTS_GLB_INLINE int erts_thr_progress_is_managed_thread(void) { diff --git a/erts/emulator/beam/erl_thr_queue.c b/erts/emulator/beam/erl_thr_queue.c index f8ca87ddcc..7ff456b915 100644 --- a/erts/emulator/beam/erl_thr_queue.c +++ b/erts/emulator/beam/erl_thr_queue.c @@ -3,16 +3,17 @@ * * Copyright Ericsson AB 2011-2013. All Rights Reserved. * - * The contents of this file are subject to the Erlang Public License, - * Version 1.1, (the "License"); you may not use this file except in - * compliance with the License. You should have received a copy of the - * Erlang Public License along with this software. If not, it can be - * retrieved online at http://www.erlang.org/. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Software distributed under the License is distributed on an "AS IS" - * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See - * the License for the specific language governing rights and limitations - * under the License. + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. * * %CopyrightEnd% */ @@ -223,7 +224,7 @@ ErtsThrQCleanState_t erts_thr_q_destroy(ErtsThrQ_t *q) { if (!q->q.blk) - erl_exit(ERTS_ABORT_EXIT, + erts_exit(ERTS_ABORT_EXIT, "Trying to destroy not created thread queue\n"); return erts_thr_q_finalize(q); } @@ -588,7 +589,7 @@ enqueue(ErtsThrQ_t *q, void *data, ErtsThrQElement_t *this) #if ERTS_THR_Q_DBG_CHK_DATA if (!data) - erl_exit(ERTS_ABORT_EXIT, "Missing data in enqueue\n"); + erts_exit(ERTS_ABORT_EXIT, "Missing data in enqueue\n"); #endif ASSERT(!q->q.finalizing); @@ -770,7 +771,7 @@ erts_thr_q_dequeue(ErtsThrQ_t *q) #if ERTS_THR_Q_DBG_CHK_DATA head->data.ptr = NULL; if (!res) - erl_exit(ERTS_ABORT_EXIT, "Missing data in dequeue\n"); + erts_exit(ERTS_ABORT_EXIT, "Missing data in dequeue\n"); #endif clean(q, (q->head.deq_fini.automatic diff --git a/erts/emulator/beam/erl_thr_queue.h b/erts/emulator/beam/erl_thr_queue.h index 13af758b3f..27a6d03224 100644 --- a/erts/emulator/beam/erl_thr_queue.h +++ b/erts/emulator/beam/erl_thr_queue.h @@ -3,16 +3,17 @@ * * Copyright Ericsson AB 2011-2013. All Rights Reserved. * - * The contents of this file are subject to the Erlang Public License, - * Version 1.1, (the "License"); you may not use this file except in - * compliance with the License. You should have received a copy of the - * Erlang Public License along with this software. If not, it can be - * retrieved online at http://www.erlang.org/. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Software distributed under the License is distributed on an "AS IS" - * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See - * the License for the specific language governing rights and limitations - * under the License. + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. * * %CopyrightEnd% */ diff --git a/erts/emulator/beam/erl_threads.h b/erts/emulator/beam/erl_threads.h index 759c8f4c33..34f91e2ec8 100644 --- a/erts/emulator/beam/erl_threads.h +++ b/erts/emulator/beam/erl_threads.h @@ -3,16 +3,17 @@ * * Copyright Ericsson AB 2001-2013. All Rights Reserved. * - * The contents of this file are subject to the Erlang Public License, - * Version 1.1, (the "License"); you may not use this file except in - * compliance with the License. You should have received a copy of the - * Erlang Public License along with this software. If not, it can be - * retrieved online at http://www.erlang.org/. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Software distributed under the License is distributed on an "AS IS" - * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See - * the License for the specific language governing rights and limitations - * under the License. + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. * * %CopyrightEnd% */ @@ -281,10 +282,13 @@ #define ERTS_THR_READ_MEMORY_BARRIER ETHR_READ_MEMORY_BARRIER #define ERTS_THR_DATA_DEPENDENCY_READ_MEMORY_BARRIER ETHR_READ_DEPEND_MEMORY_BARRIER -#ifdef ERTS_ENABLE_LOCK_COUNT +#ifdef ERTS_ENABLE_LOCK_POSITION #define erts_mtx_lock(L) erts_mtx_lock_x(L, __FILE__, __LINE__) +#define erts_mtx_trylock(L) erts_mtx_trylock_x(L, __FILE__, __LINE__) #define erts_spin_lock(L) erts_spin_lock_x(L, __FILE__, __LINE__) +#define erts_rwmtx_tryrlock(L) erts_rwmtx_tryrlock_x(L, __FILE__, __LINE__) #define erts_rwmtx_rlock(L) erts_rwmtx_rlock_x(L, __FILE__, __LINE__) +#define erts_rwmtx_tryrwlock(L) erts_rwmtx_tryrwlock_x(L, __FILE__, __LINE__) #define erts_rwmtx_rwlock(L) erts_rwmtx_rwlock_x(L, __FILE__, __LINE__) #define erts_read_lock(L) erts_read_lock_x(L, __FILE__, __LINE__) #define erts_write_lock(L) erts_write_lock_x(L, __FILE__, __LINE__) @@ -341,6 +345,16 @@ typedef ethr_ts_event erts_tse_t; #define erts_aint32_t ethr_sint32_t #define erts_atomic32_t ethr_atomic32_t +#if defined(ARCH_32) +# define erts_atomic64_t ethr_dw_atomic_t +# define erts_aint64_t ethr_sint64_t +#elif defined(ARCH_64) +# define erts_atomic64_t ethr_atomic_t +# define erts_aint64_t ethr_sint_t +#else +# error "Not supported architecture" +#endif + #define ERTS_DW_AINT_HIGH_WORD ETHR_DW_SINT_HIGH_WORD #define ERTS_DW_AINT_LOW_WORD ETHR_DW_SINT_LOW_WORD @@ -411,10 +425,12 @@ typedef int erts_tse_t; typedef struct { SWord sint[2]; } erts_dw_aint_t; typedef SWord erts_aint_t; typedef Sint32 erts_aint32_t; +typedef Sint64 erts_aint64_t; #define erts_dw_atomic_t erts_dw_aint_t #define erts_atomic_t erts_aint_t #define erts_atomic32_t erts_aint32_t +#define erts_atomic64_t erts_aint64_t #if __GNUC__ > 2 typedef struct { } erts_spinlock_t; @@ -443,6 +459,7 @@ typedef struct { int gcc_is_buggy; } erts_rwlock_t; #define erts_no_dw_atomic_t erts_dw_aint_t #define erts_no_atomic_t erts_aint_t #define erts_no_atomic32_t erts_aint32_t +#define erts_no_atomic64_t erts_aint64_t #define ERTS_AINT_NULL ((erts_aint_t) NULL) @@ -460,19 +477,26 @@ ERTS_GLB_INLINE void erts_thr_detach(erts_tid_t tid); ERTS_GLB_INLINE void erts_thr_exit(void *res); ERTS_GLB_INLINE void erts_thr_install_exit_handler(void (*exit_handler)(void)); ERTS_GLB_INLINE erts_tid_t erts_thr_self(void); +ERTS_GLB_INLINE int erts_thr_getname(erts_tid_t tid, char *buf, size_t len); ERTS_GLB_INLINE int erts_equal_tids(erts_tid_t x, erts_tid_t y); -ERTS_GLB_INLINE void erts_mtx_init_x(erts_mtx_t *mtx, char *name, Eterm extra); -ERTS_GLB_INLINE void erts_mtx_init_x_opt(erts_mtx_t *mtx, char *name, Eterm extra, Uint16 opt); +ERTS_GLB_INLINE void erts_mtx_init_x(erts_mtx_t *mtx, char *name, Eterm extra, + int enable_lcnt); +ERTS_GLB_INLINE void erts_mtx_init_x_opt(erts_mtx_t *mtx, char *name, Eterm extra, + Uint16 opt, int enable_lcnt); ERTS_GLB_INLINE void erts_mtx_init_locked_x(erts_mtx_t *mtx, char *name, - Eterm extra); + Eterm extra, + int enable_lcnt); ERTS_GLB_INLINE void erts_mtx_init(erts_mtx_t *mtx, char *name); ERTS_GLB_INLINE void erts_mtx_init_locked(erts_mtx_t *mtx, char *name); ERTS_GLB_INLINE void erts_mtx_destroy(erts_mtx_t *mtx); -ERTS_GLB_INLINE int erts_mtx_trylock(erts_mtx_t *mtx); -#ifdef ERTS_ENABLE_LOCK_COUNT -ERTS_GLB_INLINE void erts_mtx_lock_x(erts_mtx_t *mtx, char *file, unsigned int line); +#ifdef ERTS_ENABLE_LOCK_POSITION +ERTS_GLB_INLINE int erts_mtx_trylock_x(erts_mtx_t *mtx, char *file, + unsigned int line); +ERTS_GLB_INLINE void erts_mtx_lock_x(erts_mtx_t *mtx, char *file, + unsigned int line); #else +ERTS_GLB_INLINE int erts_mtx_trylock(erts_mtx_t *mtx); ERTS_GLB_INLINE void erts_mtx_lock(erts_mtx_t *mtx); #endif ERTS_GLB_INLINE void erts_mtx_unlock(erts_mtx_t *mtx); @@ -496,16 +520,18 @@ ERTS_GLB_INLINE void erts_rwmtx_init_opt(erts_rwmtx_t *rwmtx, ERTS_GLB_INLINE void erts_rwmtx_init(erts_rwmtx_t *rwmtx, char *name); ERTS_GLB_INLINE void erts_rwmtx_destroy(erts_rwmtx_t *rwmtx); -ERTS_GLB_INLINE int erts_rwmtx_tryrlock(erts_rwmtx_t *rwmtx); -#ifdef ERTS_ENABLE_LOCK_COUNT +#ifdef ERTS_ENABLE_LOCK_POSITION +ERTS_GLB_INLINE int erts_rwmtx_tryrlock_x(erts_rwmtx_t *rwmtx, char *file, unsigned int line); ERTS_GLB_INLINE void erts_rwmtx_rlock_x(erts_rwmtx_t *rwmtx, char *file, unsigned int line); ERTS_GLB_INLINE void erts_rwmtx_rwlock_x(erts_rwmtx_t *rwmtx, char *file, unsigned int line); +ERTS_GLB_INLINE int erts_rwmtx_tryrwlock_x(erts_rwmtx_t *rwmtx, char *file, unsigned int line); #else +ERTS_GLB_INLINE int erts_rwmtx_tryrlock(erts_rwmtx_t *rwmtx); ERTS_GLB_INLINE void erts_rwmtx_rlock(erts_rwmtx_t *rwmtx); ERTS_GLB_INLINE void erts_rwmtx_rwlock(erts_rwmtx_t *rwmtx); +ERTS_GLB_INLINE int erts_rwmtx_tryrwlock(erts_rwmtx_t *rwmtx); #endif ERTS_GLB_INLINE void erts_rwmtx_runlock(erts_rwmtx_t *rwmtx); -ERTS_GLB_INLINE int erts_rwmtx_tryrwlock(erts_rwmtx_t *rwmtx); ERTS_GLB_INLINE void erts_rwmtx_rwunlock(erts_rwmtx_t *rwmtx); ERTS_GLB_INLINE int erts_lc_rwmtx_is_rlocked(erts_rwmtx_t *mtx); ERTS_GLB_INLINE int erts_lc_rwmtx_is_rwlocked(erts_rwmtx_t *mtx); @@ -559,6 +585,29 @@ ERTS_GLB_INLINE erts_aint32_t erts_no_atomic32_cmpxchg(erts_no_atomic32_t *xchgp ERTS_GLB_INLINE erts_aint32_t erts_no_atomic32_read_bset(erts_no_atomic32_t *var, erts_aint32_t mask, erts_aint32_t set); +ERTS_GLB_INLINE void erts_no_atomic64_set(erts_no_atomic64_t *var, + erts_aint64_t i); +ERTS_GLB_INLINE erts_aint64_t erts_no_atomic64_read(erts_no_atomic64_t *var); +ERTS_GLB_INLINE erts_aint64_t erts_no_atomic64_inc_read(erts_no_atomic64_t *incp); +ERTS_GLB_INLINE erts_aint64_t erts_no_atomic64_dec_read(erts_no_atomic64_t *decp); +ERTS_GLB_INLINE void erts_no_atomic64_inc(erts_no_atomic64_t *incp); +ERTS_GLB_INLINE void erts_no_atomic64_dec(erts_no_atomic64_t *decp); +ERTS_GLB_INLINE erts_aint64_t erts_no_atomic64_add_read(erts_no_atomic64_t *addp, + erts_aint64_t i); +ERTS_GLB_INLINE void erts_no_atomic64_add(erts_no_atomic64_t *addp, + erts_aint64_t i); +ERTS_GLB_INLINE erts_aint64_t erts_no_atomic64_read_bor(erts_no_atomic64_t *var, + erts_aint64_t mask); +ERTS_GLB_INLINE erts_aint64_t erts_no_atomic64_read_band(erts_no_atomic64_t *var, + erts_aint64_t mask); +ERTS_GLB_INLINE erts_aint64_t erts_no_atomic64_xchg(erts_no_atomic64_t *xchgp, + erts_aint64_t new); +ERTS_GLB_INLINE erts_aint64_t erts_no_atomic64_cmpxchg(erts_no_atomic64_t *xchgp, + erts_aint64_t new, + erts_aint64_t expected); +ERTS_GLB_INLINE erts_aint64_t erts_no_atomic64_read_bset(erts_no_atomic64_t *var, + erts_aint64_t mask, + erts_aint64_t set); ERTS_GLB_INLINE void erts_spinlock_init_x_opt(erts_spinlock_t *lock, char *name, @@ -571,7 +620,7 @@ ERTS_GLB_INLINE void erts_spinlock_init(erts_spinlock_t *lock, char *name); ERTS_GLB_INLINE void erts_spinlock_destroy(erts_spinlock_t *lock); ERTS_GLB_INLINE void erts_spin_unlock(erts_spinlock_t *lock); -#ifdef ERTS_ENABLE_LOCK_COUNT +#ifdef ERTS_ENABLE_LOCK_POSITION ERTS_GLB_INLINE void erts_spin_lock_x(erts_spinlock_t *lock, char *file, unsigned int line); #else ERTS_GLB_INLINE void erts_spin_lock(erts_spinlock_t *lock); @@ -584,7 +633,7 @@ ERTS_GLB_INLINE void erts_rwlock_init(erts_rwlock_t *lock, char *name); ERTS_GLB_INLINE void erts_rwlock_destroy(erts_rwlock_t *lock); ERTS_GLB_INLINE void erts_read_unlock(erts_rwlock_t *lock); -#ifdef ERTS_ENABLE_LOCK_COUNT +#ifdef ERTS_ENABLE_LOCK_POSITION ERTS_GLB_INLINE void erts_read_lock_x(erts_rwlock_t *lock, char *file, unsigned int line); ERTS_GLB_INLINE void erts_write_lock_x(erts_rwlock_t *lock, char *file, unsigned int line); #else @@ -594,26 +643,35 @@ ERTS_GLB_INLINE void erts_write_lock(erts_rwlock_t *lock); ERTS_GLB_INLINE void erts_write_unlock(erts_rwlock_t *lock); ERTS_GLB_INLINE int erts_lc_rwlock_is_rlocked(erts_rwlock_t *lock); ERTS_GLB_INLINE int erts_lc_rwlock_is_rwlocked(erts_rwlock_t *lock); -ERTS_GLB_INLINE void erts_tsd_key_create(erts_tsd_key_t *keyp); +ERTS_GLB_INLINE void erts_tsd_key_create(erts_tsd_key_t *keyp, char *keyname); ERTS_GLB_INLINE void erts_tsd_key_delete(erts_tsd_key_t key); ERTS_GLB_INLINE void erts_tsd_set(erts_tsd_key_t key, void *value); ERTS_GLB_INLINE void * erts_tsd_get(erts_tsd_key_t key); ERTS_GLB_INLINE erts_tse_t *erts_tse_fetch(void); ERTS_GLB_INLINE void erts_tse_return(erts_tse_t *ep); +ERTS_GLB_INLINE void erts_tse_prepare_timed(erts_tse_t *ep); ERTS_GLB_INLINE void erts_tse_set(erts_tse_t *ep); ERTS_GLB_INLINE void erts_tse_reset(erts_tse_t *ep); ERTS_GLB_INLINE int erts_tse_wait(erts_tse_t *ep); ERTS_GLB_INLINE int erts_tse_swait(erts_tse_t *ep, int spincount); +ERTS_GLB_INLINE int erts_tse_twait(erts_tse_t *ep, Sint64 tmo); +ERTS_GLB_INLINE int erts_tse_stwait(erts_tse_t *ep, int spincount, Sint64 tmo); ERTS_GLB_INLINE int erts_tse_is_tmp(erts_tse_t *ep); ERTS_GLB_INLINE void erts_thr_set_main_status(int, int); ERTS_GLB_INLINE int erts_thr_get_main_status(void); ERTS_GLB_INLINE void erts_thr_yield(void); + #ifdef ETHR_HAVE_ETHR_SIG_FUNCS #define ERTS_THR_HAVE_SIG_FUNCS 1 ERTS_GLB_INLINE void erts_thr_sigmask(int how, const sigset_t *set, sigset_t *oset); ERTS_GLB_INLINE void erts_thr_sigwait(const sigset_t *set, int *sig); + +#ifdef USE_THREADS +ERTS_GLB_INLINE void erts_thr_kill(erts_tid_t tid, int sig); +#endif + #endif /* #ifdef HAVE_ETHR_SIG_FUNCS */ #ifdef USE_THREADS @@ -1189,6 +1247,441 @@ erts_atomic32_read_dirty(erts_atomic32_t *var) #endif +/* 64-bit atomics */ + +#if defined(ARCH_64) + +#define erts_atomic64_init_nob ethr_atomic_init +#define erts_atomic64_set_nob ethr_atomic_set +#define erts_atomic64_read_nob ethr_atomic_read +#define erts_atomic64_inc_read_nob ethr_atomic_inc_read +#define erts_atomic64_dec_read_nob ethr_atomic_dec_read +#define erts_atomic64_inc_nob ethr_atomic_inc +#define erts_atomic64_dec_nob ethr_atomic_dec +#define erts_atomic64_add_read_nob ethr_atomic_add_read +#define erts_atomic64_add_nob ethr_atomic_add +#define erts_atomic64_read_bor_nob ethr_atomic_read_bor +#define erts_atomic64_read_band_nob ethr_atomic_read_band +#define erts_atomic64_xchg_nob ethr_atomic_xchg +#define erts_atomic64_cmpxchg_nob ethr_atomic_cmpxchg +#define erts_atomic64_read_bset_nob erts_atomic_read_bset_nob + +#define erts_atomic64_init_mb ethr_atomic_init_mb +#define erts_atomic64_set_mb ethr_atomic_set_mb +#define erts_atomic64_read_mb ethr_atomic_read_mb +#define erts_atomic64_inc_read_mb ethr_atomic_inc_read_mb +#define erts_atomic64_dec_read_mb ethr_atomic_dec_read_mb +#define erts_atomic64_inc_mb ethr_atomic_inc_mb +#define erts_atomic64_dec_mb ethr_atomic_dec_mb +#define erts_atomic64_add_read_mb ethr_atomic_add_read_mb +#define erts_atomic64_add_mb ethr_atomic_add_mb +#define erts_atomic64_read_bor_mb ethr_atomic_read_bor_mb +#define erts_atomic64_read_band_mb ethr_atomic_read_band_mb +#define erts_atomic64_xchg_mb ethr_atomic_xchg_mb +#define erts_atomic64_cmpxchg_mb ethr_atomic_cmpxchg_mb +#define erts_atomic64_read_bset_mb erts_atomic_read_bset_mb + +#define erts_atomic64_init_acqb ethr_atomic_init_acqb +#define erts_atomic64_set_acqb ethr_atomic_set_acqb +#define erts_atomic64_read_acqb ethr_atomic_read_acqb +#define erts_atomic64_inc_read_acqb ethr_atomic_inc_read_acqb +#define erts_atomic64_dec_read_acqb ethr_atomic_dec_read_acqb +#define erts_atomic64_inc_acqb ethr_atomic_inc_acqb +#define erts_atomic64_dec_acqb ethr_atomic_dec_acqb +#define erts_atomic64_add_read_acqb ethr_atomic_add_read_acqb +#define erts_atomic64_add_acqb ethr_atomic_add_acqb +#define erts_atomic64_read_bor_acqb ethr_atomic_read_bor_acqb +#define erts_atomic64_read_band_acqb ethr_atomic_read_band_acqb +#define erts_atomic64_xchg_acqb ethr_atomic_xchg_acqb +#define erts_atomic64_cmpxchg_acqb ethr_atomic_cmpxchg_acqb +#define erts_atomic64_read_bset_acqb erts_atomic_read_bset_acqb + +#define erts_atomic64_init_relb ethr_atomic_init_relb +#define erts_atomic64_set_relb ethr_atomic_set_relb +#define erts_atomic64_read_relb ethr_atomic_read_relb +#define erts_atomic64_inc_read_relb ethr_atomic_inc_read_relb +#define erts_atomic64_dec_read_relb ethr_atomic_dec_read_relb +#define erts_atomic64_inc_relb ethr_atomic_inc_relb +#define erts_atomic64_dec_relb ethr_atomic_dec_relb +#define erts_atomic64_add_read_relb ethr_atomic_add_read_relb +#define erts_atomic64_add_relb ethr_atomic_add_relb +#define erts_atomic64_read_bor_relb ethr_atomic_read_bor_relb +#define erts_atomic64_read_band_relb ethr_atomic_read_band_relb +#define erts_atomic64_xchg_relb ethr_atomic_xchg_relb +#define erts_atomic64_cmpxchg_relb ethr_atomic_cmpxchg_relb +#define erts_atomic64_read_bset_relb erts_atomic_read_bset_relb + +#define erts_atomic64_init_ddrb ethr_atomic_init_ddrb +#define erts_atomic64_set_ddrb ethr_atomic_set_ddrb +#define erts_atomic64_read_ddrb ethr_atomic_read_ddrb +#define erts_atomic64_inc_read_ddrb ethr_atomic_inc_read_ddrb +#define erts_atomic64_dec_read_ddrb ethr_atomic_dec_read_ddrb +#define erts_atomic64_inc_ddrb ethr_atomic_inc_ddrb +#define erts_atomic64_dec_ddrb ethr_atomic_dec_ddrb +#define erts_atomic64_add_read_ddrb ethr_atomic_add_read_ddrb +#define erts_atomic64_add_ddrb ethr_atomic_add_ddrb +#define erts_atomic64_read_bor_ddrb ethr_atomic_read_bor_ddrb +#define erts_atomic64_read_band_ddrb ethr_atomic_read_band_ddrb +#define erts_atomic64_xchg_ddrb ethr_atomic_xchg_ddrb +#define erts_atomic64_cmpxchg_ddrb ethr_atomic_cmpxchg_ddrb +#define erts_atomic64_read_bset_ddrb erts_atomic_read_bset_ddrb + +#define erts_atomic64_init_rb ethr_atomic_init_rb +#define erts_atomic64_set_rb ethr_atomic_set_rb +#define erts_atomic64_read_rb ethr_atomic_read_rb +#define erts_atomic64_inc_read_rb ethr_atomic_inc_read_rb +#define erts_atomic64_dec_read_rb ethr_atomic_dec_read_rb +#define erts_atomic64_inc_rb ethr_atomic_inc_rb +#define erts_atomic64_dec_rb ethr_atomic_dec_rb +#define erts_atomic64_add_read_rb ethr_atomic_add_read_rb +#define erts_atomic64_add_rb ethr_atomic_add_rb +#define erts_atomic64_read_bor_rb ethr_atomic_read_bor_rb +#define erts_atomic64_read_band_rb ethr_atomic_read_band_rb +#define erts_atomic64_xchg_rb ethr_atomic_xchg_rb +#define erts_atomic64_cmpxchg_rb ethr_atomic_cmpxchg_rb +#define erts_atomic64_read_bset_rb erts_atomic_read_bset_rb + +#define erts_atomic64_init_wb ethr_atomic_init_wb +#define erts_atomic64_set_wb ethr_atomic_set_wb +#define erts_atomic64_read_wb ethr_atomic_read_wb +#define erts_atomic64_inc_read_wb ethr_atomic_inc_read_wb +#define erts_atomic64_dec_read_wb ethr_atomic_dec_read_wb +#define erts_atomic64_inc_wb ethr_atomic_inc_wb +#define erts_atomic64_dec_wb ethr_atomic_dec_wb +#define erts_atomic64_add_read_wb ethr_atomic_add_read_wb +#define erts_atomic64_add_wb ethr_atomic_add_wb +#define erts_atomic64_read_bor_wb ethr_atomic_read_bor_wb +#define erts_atomic64_read_band_wb ethr_atomic_read_band_wb +#define erts_atomic64_xchg_wb ethr_atomic_xchg_wb +#define erts_atomic64_cmpxchg_wb ethr_atomic_cmpxchg_wb +#define erts_atomic64_read_bset_wb erts_atomic_read_bset_wb + +#define erts_atomic64_set_dirty erts_atomic_set_dirty +#define erts_atomic64_read_dirty erts_atomic_read_dirty + +#elif defined(ARCH_32) + +#undef ERTS_ATOMIC64_OPS_DECL__ + +#define ERTS_ATOMIC64_OPS_DECL__(BARRIER) \ +ERTS_GLB_INLINE void \ +erts_atomic64_init_ ## BARRIER(erts_atomic64_t *var, \ + erts_aint64_t val); \ +ERTS_GLB_INLINE void \ +erts_atomic64_set_ ## BARRIER(erts_atomic64_t *var, \ + erts_aint64_t val); \ +ERTS_GLB_INLINE erts_aint64_t \ +erts_atomic64_read_ ## BARRIER(erts_atomic64_t *var); \ +ERTS_GLB_INLINE erts_aint64_t \ +erts_atomic64_inc_read_ ## BARRIER(erts_atomic64_t *var); \ +ERTS_GLB_INLINE erts_aint64_t \ +erts_atomic64_dec_read_ ## BARRIER(erts_atomic64_t *var); \ +ERTS_GLB_INLINE void \ +erts_atomic64_inc_ ## BARRIER(erts_atomic64_t *var); \ +ERTS_GLB_INLINE void \ +erts_atomic64_dec_ ## BARRIER(erts_atomic64_t *var); \ +ERTS_GLB_INLINE erts_aint64_t \ +erts_atomic64_add_read_ ## BARRIER(erts_atomic64_t *var, \ + erts_aint64_t val); \ +ERTS_GLB_INLINE void \ +erts_atomic64_add_ ## BARRIER(erts_atomic64_t *var, \ + erts_aint64_t val); \ +ERTS_GLB_INLINE erts_aint64_t \ +erts_atomic64_read_bor_ ## BARRIER(erts_atomic64_t *var, \ + erts_aint64_t val); \ +ERTS_GLB_INLINE erts_aint64_t \ +erts_atomic64_read_band_ ## BARRIER(erts_atomic64_t *var, \ + erts_aint64_t val); \ +ERTS_GLB_INLINE erts_aint64_t \ +erts_atomic64_xchg_ ## BARRIER(erts_atomic64_t *var, \ + erts_aint64_t val); \ +ERTS_GLB_INLINE erts_aint64_t \ +erts_atomic64_cmpxchg_ ## BARRIER(erts_atomic64_t *var, \ + erts_aint64_t new, \ + erts_aint64_t exp); \ +ERTS_GLB_INLINE erts_aint64_t \ +erts_atomic64_read_bset_ ## BARRIER(erts_atomic64_t *var, \ + erts_aint64_t mask, \ + erts_aint64_t set) + +ERTS_ATOMIC64_OPS_DECL__(nob); +ERTS_ATOMIC64_OPS_DECL__(mb); +ERTS_ATOMIC64_OPS_DECL__(acqb); +ERTS_ATOMIC64_OPS_DECL__(relb); +ERTS_ATOMIC64_OPS_DECL__(ddrb); +ERTS_ATOMIC64_OPS_DECL__(rb); +ERTS_ATOMIC64_OPS_DECL__(wb); + +#undef ERTS_ATOMIC64_OPS_DECL__ + +ERTS_GLB_INLINE void +erts_atomic64_set_dirty(erts_atomic64_t *var, erts_aint64_t val); +ERTS_GLB_INLINE erts_aint64_t +erts_atomic64_read_dirty(erts_atomic64_t *var); + +#if ERTS_GLB_INLINE_INCL_FUNC_DEF + +/* + * The ethr_dw_atomic_*_nob() functions below + * are here to make it possible for the + * ERTS_ATOMIC64_OPS_IMPL__() to map erts + * barriers to ethread barriers... + */ +static ERTS_INLINE void +ethr_dw_atomic_init_nob(ethr_dw_atomic_t *var, + ethr_dw_sint_t *val) +{ + ethr_dw_atomic_init(var, val); +} + +static ERTS_INLINE void +ethr_dw_atomic_set_nob(ethr_dw_atomic_t *var, + ethr_dw_sint_t *val) +{ + ethr_dw_atomic_set(var, val); +} + +static ERTS_INLINE void +ethr_dw_atomic_read_nob(ethr_dw_atomic_t *var, + ethr_dw_sint_t *val) +{ + ethr_dw_atomic_read(var, val); +} + +static ERTS_INLINE int +ethr_dw_atomic_cmpxchg_nob(ethr_dw_atomic_t *var, + ethr_dw_sint_t *new, + ethr_dw_sint_t *xchg) +{ + return ethr_dw_atomic_cmpxchg(var, new, xchg); +} + +#undef ERTS_ATOMIC64_OPS_IMPL__ +#undef ERTS_ATOMIC64_DW_CMPXCHG_IMPL__ +#undef ERTS_DW_SINT_TO_AINT64__ +#undef ERTS_AINT64_TO_DW_SINT__ + +#ifdef ETHR_SU_DW_NAINT_T__ +#define ERTS_DW_SINT_TO_AINT64__(DW) \ + ((erts_aint64_t) DW.dw_sint) +#define ERTS_AINT64_TO_DW_SINT__(DW, AINT64) \ + (DW.dw_sint = (ETHR_SU_DW_NAINT_T__) AINT64) +#else /* !ETHR_SU_DW_NAINT_T__ */ +#define ERTS_DW_SINT_TO_AINT64__(DW) \ + ((((erts_aint64_t) DW.sint[ETHR_DW_SINT_HIGH_WORD]) << 32) \ + | (((erts_aint64_t) DW.sint[ETHR_DW_SINT_LOW_WORD]) \ + & ((erts_aint64_t) 0xffffffff))) +#define ERTS_AINT64_TO_DW_SINT__(DW, AINT64) \ + do { \ + DW.sint[ETHR_DW_SINT_LOW_WORD] = \ + (ethr_sint_t) (AINT64 & 0xffffffff); \ + DW.sint[ETHR_DW_SINT_HIGH_WORD] = \ + (ethr_sint_t) ((AINT64 >> 32) & 0xffffffff); \ + } while (0) +#endif /* !ETHR_SU_DW_NAINT_T__ */ + +#define ERTS_ATOMIC64_DW_CMPXCHG_IMPL__(CmpXchgOp, \ + AVarP, XchgVar, NewVar, \ + ModificationCode) \ +do { \ + ethr_dw_sint_t dw_xchg__, dw_new__; \ + ethr_dw_atomic_read(AVarP, &dw_xchg__); \ + do { \ + XchgVar = ERTS_DW_SINT_TO_AINT64__(dw_xchg__); \ + { \ + ModificationCode; \ + } \ + ERTS_AINT64_TO_DW_SINT__(dw_new__, NewVar); \ + } while (!CmpXchgOp((AVarP), &dw_new__, &dw_xchg__)); \ +} while (0) + +#define ERTS_ATOMIC64_OPS_IMPL__(BARRIER) \ + \ +ERTS_GLB_INLINE void \ +erts_atomic64_init_ ## BARRIER(erts_atomic64_t *var, \ + erts_aint64_t val) \ +{ \ + ethr_dw_sint_t dw; \ + ERTS_AINT64_TO_DW_SINT__(dw, val); \ + ethr_dw_atomic_init_ ## BARRIER(var, &dw); \ +} \ + \ +ERTS_GLB_INLINE void \ +erts_atomic64_set_ ## BARRIER(erts_atomic64_t *var, \ + erts_aint64_t val) \ +{ \ + ethr_dw_sint_t dw; \ + ERTS_AINT64_TO_DW_SINT__(dw, val); \ + ethr_dw_atomic_set_ ## BARRIER(var, &dw); \ +} \ + \ +ERTS_GLB_INLINE erts_aint64_t \ +erts_atomic64_read_ ## BARRIER(erts_atomic64_t *var) \ +{ \ + ethr_dw_sint_t dw; \ + ethr_dw_atomic_read_ ## BARRIER(var, &dw); \ + return ERTS_DW_SINT_TO_AINT64__(dw); \ +} \ + \ +ERTS_GLB_INLINE erts_aint64_t \ +erts_atomic64_inc_read_ ## BARRIER(erts_atomic64_t *var) \ +{ \ + erts_aint64_t xchg, new; \ + ERTS_ATOMIC64_DW_CMPXCHG_IMPL__(ethr_dw_atomic_cmpxchg_ ## BARRIER, \ + var, xchg, new, \ + new = xchg + 1); \ + return new; \ +} \ + \ +ERTS_GLB_INLINE erts_aint64_t \ +erts_atomic64_dec_read_ ## BARRIER(erts_atomic64_t *var) \ +{ \ + erts_aint64_t xchg, new; \ + ERTS_ATOMIC64_DW_CMPXCHG_IMPL__(ethr_dw_atomic_cmpxchg_ ## BARRIER, \ + var, xchg, new, \ + new = xchg - 1); \ + return new; \ +} \ + \ +ERTS_GLB_INLINE void \ +erts_atomic64_inc_ ## BARRIER(erts_atomic64_t *var) \ +{ \ + erts_aint64_t xchg, new; \ + ERTS_ATOMIC64_DW_CMPXCHG_IMPL__(ethr_dw_atomic_cmpxchg_ ## BARRIER, \ + var, xchg, new, \ + new = xchg + 1); \ +} \ + \ +ERTS_GLB_INLINE void \ +erts_atomic64_dec_ ## BARRIER(erts_atomic64_t *var) \ +{ \ + erts_aint64_t xchg, new; \ + ERTS_ATOMIC64_DW_CMPXCHG_IMPL__(ethr_dw_atomic_cmpxchg_ ## BARRIER, \ + var, xchg, new, \ + new = xchg - 1); \ +} \ + \ +ERTS_GLB_INLINE erts_aint64_t \ +erts_atomic64_add_read_ ## BARRIER(erts_atomic64_t *var, \ + erts_aint64_t val) \ +{ \ + erts_aint64_t xchg, new; \ + ERTS_ATOMIC64_DW_CMPXCHG_IMPL__(ethr_dw_atomic_cmpxchg_ ## BARRIER, \ + var, xchg, new, \ + new = xchg + val); \ + return new; \ +} \ + \ +ERTS_GLB_INLINE void \ +erts_atomic64_add_ ## BARRIER(erts_atomic64_t *var, \ + erts_aint64_t val) \ +{ \ + erts_aint64_t xchg, new; \ + ERTS_ATOMIC64_DW_CMPXCHG_IMPL__(ethr_dw_atomic_cmpxchg_ ## BARRIER, \ + var, xchg, new, \ + new = xchg + val); \ +} \ + \ +ERTS_GLB_INLINE erts_aint64_t \ +erts_atomic64_read_bor_ ## BARRIER(erts_atomic64_t *var, \ + erts_aint64_t val) \ +{ \ + erts_aint64_t xchg, new; \ + ERTS_ATOMIC64_DW_CMPXCHG_IMPL__(ethr_dw_atomic_cmpxchg_ ## BARRIER, \ + var, xchg, new, \ + new = xchg | val); \ + return xchg; \ +} \ + \ +ERTS_GLB_INLINE erts_aint64_t \ +erts_atomic64_read_band_ ## BARRIER(erts_atomic64_t *var, \ + erts_aint64_t val) \ +{ \ + erts_aint64_t xchg, new; \ + ERTS_ATOMIC64_DW_CMPXCHG_IMPL__(ethr_dw_atomic_cmpxchg_ ## BARRIER, \ + var, xchg, new, \ + new = xchg & val); \ + return xchg; \ +} \ + \ +ERTS_GLB_INLINE erts_aint64_t \ +erts_atomic64_xchg_ ## BARRIER(erts_atomic64_t *var, \ + erts_aint64_t val) \ +{ \ + erts_aint64_t xchg, new; \ + ERTS_ATOMIC64_DW_CMPXCHG_IMPL__(ethr_dw_atomic_cmpxchg_ ## BARRIER, \ + var, xchg, new, \ + new = val); \ + return xchg; \ +} \ + \ +ERTS_GLB_INLINE erts_aint64_t \ +erts_atomic64_cmpxchg_ ## BARRIER(erts_atomic64_t *var, \ + erts_aint64_t new, \ + erts_aint64_t exp) \ +{ \ + ethr_dw_sint_t dw_xchg, dw_new; \ + ERTS_AINT64_TO_DW_SINT__(dw_xchg, exp); \ + ERTS_AINT64_TO_DW_SINT__(dw_new, new); \ + if (ethr_dw_atomic_cmpxchg_ ## BARRIER(var, &dw_new, &dw_xchg)) \ + return exp; \ + return ERTS_DW_SINT_TO_AINT64__(dw_xchg); \ +} \ + \ +ERTS_GLB_INLINE erts_aint64_t \ +erts_atomic64_read_bset_ ## BARRIER(erts_atomic64_t *var, \ + erts_aint64_t mask, \ + erts_aint64_t set) \ +{ \ + erts_aint64_t xchg, new; \ + ERTS_ATOMIC64_DW_CMPXCHG_IMPL__(ethr_dw_atomic_cmpxchg_ ## BARRIER, \ + var, xchg, new, \ + { \ + new = xchg & ~mask; \ + new |= mask & set; \ + }); \ + return xchg; \ +} + +ERTS_ATOMIC64_OPS_IMPL__(nob) +ERTS_ATOMIC64_OPS_IMPL__(mb) +ERTS_ATOMIC64_OPS_IMPL__(acqb) +ERTS_ATOMIC64_OPS_IMPL__(relb) +ERTS_ATOMIC64_OPS_IMPL__(ddrb) +ERTS_ATOMIC64_OPS_IMPL__(rb) +ERTS_ATOMIC64_OPS_IMPL__(wb) + +#undef ERTS_ATOMIC64_OPS_IMPL__ +#undef ERTS_ATOMIC64_DW_CMPXCHG_IMPL__ + +ERTS_GLB_INLINE void +erts_atomic64_set_dirty(erts_atomic64_t *var, erts_aint64_t val) +{ + ethr_sint_t *sint = ethr_dw_atomic_addr(var); + ethr_dw_sint_t dw; + ERTS_AINT64_TO_DW_SINT__(dw, val); + sint[0] = dw.sint[0]; + sint[1] = dw.sint[1]; +} + +ERTS_GLB_INLINE erts_aint64_t +erts_atomic64_read_dirty(erts_atomic64_t *var) +{ + ethr_sint_t *sint; + ethr_dw_sint_t dw; + sint = ethr_dw_atomic_addr(var); + dw.sint[0] = sint[0]; + dw.sint[1] = sint[1]; + return ERTS_DW_SINT_TO_AINT64__(dw); +} + +#undef ERTS_DW_SINT_TO_AINT64__ +#undef ERTS_AINT64_TO_DW_SINT__ + +#endif /* ERTS_GLB_INLINE_INCL_FUNC_DEF */ + +#endif /* ARCH_32 */ + #else /* !USE_THREADS */ /* Double word size atomics */ @@ -1451,6 +1944,116 @@ erts_atomic32_read_dirty(erts_atomic32_t *var) #define erts_atomic32_set_dirty erts_no_atomic32_set #define erts_atomic32_read_dirty erts_no_atomic32_read +/* 64-bit atomics */ + +#define erts_atomic64_init_nob erts_no_atomic64_set +#define erts_atomic64_set_nob erts_no_atomic64_set +#define erts_atomic64_read_nob erts_no_atomic64_read +#define erts_atomic64_inc_read_nob erts_no_atomic64_inc_read +#define erts_atomic64_dec_read_nob erts_no_atomic64_dec_read +#define erts_atomic64_inc_nob erts_no_atomic64_inc +#define erts_atomic64_dec_nob erts_no_atomic64_dec +#define erts_atomic64_add_read_nob erts_no_atomic64_add_read +#define erts_atomic64_add_nob erts_no_atomic64_add +#define erts_atomic64_read_bor_nob erts_no_atomic64_read_bor +#define erts_atomic64_read_band_nob erts_no_atomic64_read_band +#define erts_atomic64_xchg_nob erts_no_atomic64_xchg +#define erts_atomic64_cmpxchg_nob erts_no_atomic64_cmpxchg +#define erts_atomic64_read_bset_nob erts_no_atomic64_read_bset + +#define erts_atomic64_init_mb erts_no_atomic64_set +#define erts_atomic64_set_mb erts_no_atomic64_set +#define erts_atomic64_read_mb erts_no_atomic64_read +#define erts_atomic64_inc_read_mb erts_no_atomic64_inc_read +#define erts_atomic64_dec_read_mb erts_no_atomic64_dec_read +#define erts_atomic64_inc_mb erts_no_atomic64_inc +#define erts_atomic64_dec_mb erts_no_atomic64_dec +#define erts_atomic64_add_read_mb erts_no_atomic64_add_read +#define erts_atomic64_add_mb erts_no_atomic64_add +#define erts_atomic64_read_bor_mb erts_no_atomic64_read_bor +#define erts_atomic64_read_band_mb erts_no_atomic64_read_band +#define erts_atomic64_xchg_mb erts_no_atomic64_xchg +#define erts_atomic64_cmpxchg_mb erts_no_atomic64_cmpxchg +#define erts_atomic64_read_bset_mb erts_no_atomic64_read_bset + +#define erts_atomic64_init_acqb erts_no_atomic64_set +#define erts_atomic64_set_acqb erts_no_atomic64_set +#define erts_atomic64_read_acqb erts_no_atomic64_read +#define erts_atomic64_inc_read_acqb erts_no_atomic64_inc_read +#define erts_atomic64_dec_read_acqb erts_no_atomic64_dec_read +#define erts_atomic64_inc_acqb erts_no_atomic64_inc +#define erts_atomic64_dec_acqb erts_no_atomic64_dec +#define erts_atomic64_add_read_acqb erts_no_atomic64_add_read +#define erts_atomic64_add_acqb erts_no_atomic64_add +#define erts_atomic64_read_bor_acqb erts_no_atomic64_read_bor +#define erts_atomic64_read_band_acqb erts_no_atomic64_read_band +#define erts_atomic64_xchg_acqb erts_no_atomic64_xchg +#define erts_atomic64_cmpxchg_acqb erts_no_atomic64_cmpxchg +#define erts_atomic64_read_bset_acqb erts_no_atomic64_read_bset + +#define erts_atomic64_init_relb erts_no_atomic64_set +#define erts_atomic64_set_relb erts_no_atomic64_set +#define erts_atomic64_read_relb erts_no_atomic64_read +#define erts_atomic64_inc_read_relb erts_no_atomic64_inc_read +#define erts_atomic64_dec_read_relb erts_no_atomic64_dec_read +#define erts_atomic64_inc_relb erts_no_atomic64_inc +#define erts_atomic64_dec_relb erts_no_atomic64_dec +#define erts_atomic64_add_read_relb erts_no_atomic64_add_read +#define erts_atomic64_add_relb erts_no_atomic64_add +#define erts_atomic64_read_bor_relb erts_no_atomic64_read_bor +#define erts_atomic64_read_band_relb erts_no_atomic64_read_band +#define erts_atomic64_xchg_relb erts_no_atomic64_xchg +#define erts_atomic64_cmpxchg_relb erts_no_atomic64_cmpxchg +#define erts_atomic64_read_bset_relb erts_no_atomic64_read_bset + +#define erts_atomic64_init_ddrb erts_no_atomic64_set +#define erts_atomic64_set_ddrb erts_no_atomic64_set +#define erts_atomic64_read_ddrb erts_no_atomic64_read +#define erts_atomic64_inc_read_ddrb erts_no_atomic64_inc_read +#define erts_atomic64_dec_read_ddrb erts_no_atomic64_dec_read +#define erts_atomic64_inc_ddrb erts_no_atomic64_inc +#define erts_atomic64_dec_ddrb erts_no_atomic64_dec +#define erts_atomic64_add_read_ddrb erts_no_atomic64_add_read +#define erts_atomic64_add_ddrb erts_no_atomic64_add +#define erts_atomic64_read_bor_ddrb erts_no_atomic64_read_bor +#define erts_atomic64_read_band_ddrb erts_no_atomic64_read_band +#define erts_atomic64_xchg_ddrb erts_no_atomic64_xchg +#define erts_atomic64_cmpxchg_ddrb erts_no_atomic64_cmpxchg +#define erts_atomic64_read_bset_ddrb erts_no_atomic64_read_bset + +#define erts_atomic64_init_rb erts_no_atomic64_set +#define erts_atomic64_set_rb erts_no_atomic64_set +#define erts_atomic64_read_rb erts_no_atomic64_read +#define erts_atomic64_inc_read_rb erts_no_atomic64_inc_read +#define erts_atomic64_dec_read_rb erts_no_atomic64_dec_read +#define erts_atomic64_inc_rb erts_no_atomic64_inc +#define erts_atomic64_dec_rb erts_no_atomic64_dec +#define erts_atomic64_add_read_rb erts_no_atomic64_add_read +#define erts_atomic64_add_rb erts_no_atomic64_add +#define erts_atomic64_read_bor_rb erts_no_atomic64_read_bor +#define erts_atomic64_read_band_rb erts_no_atomic64_read_band +#define erts_atomic64_xchg_rb erts_no_atomic64_xchg +#define erts_atomic64_cmpxchg_rb erts_no_atomic64_cmpxchg +#define erts_atomic64_read_bset_rb erts_no_atomic64_read_bset + +#define erts_atomic64_init_wb erts_no_atomic64_set +#define erts_atomic64_set_wb erts_no_atomic64_set +#define erts_atomic64_read_wb erts_no_atomic64_read +#define erts_atomic64_inc_read_wb erts_no_atomic64_inc_read +#define erts_atomic64_dec_read_wb erts_no_atomic64_dec_read +#define erts_atomic64_inc_wb erts_no_atomic64_inc +#define erts_atomic64_dec_wb erts_no_atomic64_dec +#define erts_atomic64_add_read_wb erts_no_atomic64_add_read +#define erts_atomic64_add_wb erts_no_atomic64_add +#define erts_atomic64_read_bor_wb erts_no_atomic64_read_bor +#define erts_atomic64_read_band_wb erts_no_atomic64_read_band +#define erts_atomic64_xchg_wb erts_no_atomic64_xchg +#define erts_atomic64_cmpxchg_wb erts_no_atomic64_cmpxchg +#define erts_atomic64_read_bset_wb erts_no_atomic64_read_bset + +#define erts_atomic64_set_dirty erts_no_atomic64_set +#define erts_atomic64_read_dirty erts_no_atomic64_read + #endif /* !USE_THREADS */ #if ERTS_GLB_INLINE_INCL_FUNC_DEF @@ -1537,6 +2140,16 @@ erts_thr_self(void) #endif } +ERTS_GLB_INLINE int +erts_thr_getname(erts_tid_t tid, char *buf, size_t len) +{ +#ifdef USE_THREADS + return ethr_getname(tid, buf, len); +#else + return -1; +#endif +} + ERTS_GLB_INLINE int erts_equal_tids(erts_tid_t x, erts_tid_t y) @@ -1549,7 +2162,7 @@ erts_equal_tids(erts_tid_t x, erts_tid_t y) } ERTS_GLB_INLINE void -erts_mtx_init_x(erts_mtx_t *mtx, char *name, Eterm extra) +erts_mtx_init_x(erts_mtx_t *mtx, char *name, Eterm extra, int enable_lcnt) { #ifdef USE_THREADS int res = ethr_mutex_init(&mtx->mtx); @@ -1559,13 +2172,17 @@ erts_mtx_init_x(erts_mtx_t *mtx, char *name, Eterm extra) erts_lc_init_lock_x(&mtx->lc, name, ERTS_LC_FLG_LT_MUTEX, extra); #endif #ifdef ERTS_ENABLE_LOCK_COUNT - erts_lcnt_init_lock_x(&mtx->lcnt, name, ERTS_LCNT_LT_MUTEX, extra); + if (enable_lcnt) + erts_lcnt_init_lock_x(&mtx->lcnt, name, ERTS_LCNT_LT_MUTEX, extra); + else + erts_lcnt_init_lock_x(&mtx->lcnt, NULL, ERTS_LCNT_LT_MUTEX, extra); #endif #endif } ERTS_GLB_INLINE void -erts_mtx_init_x_opt(erts_mtx_t *mtx, char *name, Eterm extra, Uint16 opt) +erts_mtx_init_x_opt(erts_mtx_t *mtx, char *name, Eterm extra, Uint16 opt, + int enable_lcnt) { #ifdef USE_THREADS int res = ethr_mutex_init(&mtx->mtx); @@ -1575,14 +2192,17 @@ erts_mtx_init_x_opt(erts_mtx_t *mtx, char *name, Eterm extra, Uint16 opt) erts_lc_init_lock_x(&mtx->lc, name, ERTS_LC_FLG_LT_MUTEX, extra); #endif #ifdef ERTS_ENABLE_LOCK_COUNT - erts_lcnt_init_lock_x(&mtx->lcnt, name, ERTS_LCNT_LT_MUTEX | opt, extra); + if (enable_lcnt) + erts_lcnt_init_lock_x(&mtx->lcnt, name, ERTS_LCNT_LT_MUTEX | opt, extra); + else + erts_lcnt_init_lock_x(&mtx->lcnt, NULL, ERTS_LCNT_LT_MUTEX | opt, extra); #endif #endif } ERTS_GLB_INLINE void -erts_mtx_init_locked_x(erts_mtx_t *mtx, char *name, Eterm extra) +erts_mtx_init_locked_x(erts_mtx_t *mtx, char *name, Eterm extra, int enable_lcnt) { #ifdef USE_THREADS int res = ethr_mutex_init(&mtx->mtx); @@ -1592,7 +2212,10 @@ erts_mtx_init_locked_x(erts_mtx_t *mtx, char *name, Eterm extra) erts_lc_init_lock_x(&mtx->lc, name, ERTS_LC_FLG_LT_MUTEX, extra); #endif #ifdef ERTS_ENABLE_LOCK_COUNT - erts_lcnt_init_lock_x(&mtx->lcnt, name, ERTS_LCNT_LT_MUTEX, extra); + if (enable_lcnt) + erts_lcnt_init_lock_x(&mtx->lcnt, name, ERTS_LCNT_LT_MUTEX, extra); + else + erts_lcnt_init_lock_x(&mtx->lcnt, NULL, ERTS_LCNT_LT_MUTEX, extra); #endif ethr_mutex_lock(&mtx->mtx); #ifdef ERTS_ENABLE_LOCK_CHECK @@ -1670,7 +2293,11 @@ erts_mtx_destroy(erts_mtx_t *mtx) } ERTS_GLB_INLINE int +#ifdef ERTS_ENABLE_LOCK_POSITION +erts_mtx_trylock_x(erts_mtx_t *mtx, char *file, unsigned int line) +#else erts_mtx_trylock(erts_mtx_t *mtx) +#endif { #ifdef USE_THREADS int res; @@ -1684,8 +2311,12 @@ erts_mtx_trylock(erts_mtx_t *mtx) res = ethr_mutex_trylock(&mtx->mtx); #ifdef ERTS_ENABLE_LOCK_CHECK +#ifdef ERTS_ENABLE_LOCK_POSITION + erts_lc_trylock_x(res == 0, &mtx->lc,file,line); +#else erts_lc_trylock(res == 0, &mtx->lc); #endif +#endif #ifdef ERTS_ENABLE_LOCK_COUNT erts_lcnt_trylock(&mtx->lcnt, res); #endif @@ -1697,7 +2328,7 @@ erts_mtx_trylock(erts_mtx_t *mtx) } ERTS_GLB_INLINE void -#ifdef ERTS_ENABLE_LOCK_COUNT +#ifdef ERTS_ENABLE_LOCK_POSITION erts_mtx_lock_x(erts_mtx_t *mtx, char *file, unsigned int line) #else erts_mtx_lock(erts_mtx_t *mtx) @@ -1705,8 +2336,12 @@ erts_mtx_lock(erts_mtx_t *mtx) { #ifdef USE_THREADS #ifdef ERTS_ENABLE_LOCK_CHECK +#ifdef ERTS_ENABLE_LOCK_POSITION + erts_lc_lock_x(&mtx->lc, file, line); +#else erts_lc_lock(&mtx->lc); #endif +#endif #ifdef ERTS_ENABLE_LOCK_COUNT erts_lcnt_lock(&mtx->lcnt); #endif @@ -1857,7 +2492,10 @@ erts_rwmtx_init_opt_x(erts_rwmtx_t *rwmtx, erts_lc_init_lock_x(&rwmtx->lc, name, ERTS_LC_FLG_LT_RWMUTEX, extra); #endif #ifdef ERTS_ENABLE_LOCK_COUNT - erts_lcnt_init_lock_x(&rwmtx->lcnt, name, ERTS_LCNT_LT_RWMUTEX, extra); + if (name && name[0] == '\0') + erts_lcnt_init_lock_x(&rwmtx->lcnt, NULL, ERTS_LCNT_LT_RWMUTEX, extra); + else + erts_lcnt_init_lock_x(&rwmtx->lcnt, name, ERTS_LCNT_LT_RWMUTEX, extra); #endif #endif } @@ -1921,7 +2559,11 @@ erts_rwmtx_destroy(erts_rwmtx_t *rwmtx) } ERTS_GLB_INLINE int +#ifdef ERTS_ENABLE_LOCK_POSITION +erts_rwmtx_tryrlock_x(erts_rwmtx_t *rwmtx, char *file, unsigned int line) +#else erts_rwmtx_tryrlock(erts_rwmtx_t *rwmtx) +#endif { #ifdef USE_THREADS int res; @@ -1935,8 +2577,12 @@ erts_rwmtx_tryrlock(erts_rwmtx_t *rwmtx) res = ethr_rwmutex_tryrlock(&rwmtx->rwmtx); #ifdef ERTS_ENABLE_LOCK_CHECK +#ifdef ERTS_ENABLE_LOCK_POSITION + erts_lc_trylock_flg_x(res == 0, &rwmtx->lc, ERTS_LC_FLG_LO_READ,file,line); +#else erts_lc_trylock_flg(res == 0, &rwmtx->lc, ERTS_LC_FLG_LO_READ); #endif +#endif #ifdef ERTS_ENABLE_LOCK_COUNT erts_lcnt_trylock_opt(&rwmtx->lcnt, res, ERTS_LCNT_LO_READ); #endif @@ -1948,7 +2594,7 @@ erts_rwmtx_tryrlock(erts_rwmtx_t *rwmtx) } ERTS_GLB_INLINE void -#ifdef ERTS_ENABLE_LOCK_COUNT +#ifdef ERTS_ENABLE_LOCK_POSITION erts_rwmtx_rlock_x(erts_rwmtx_t *rwmtx, char *file, unsigned int line) #else erts_rwmtx_rlock(erts_rwmtx_t *rwmtx) @@ -1956,8 +2602,12 @@ erts_rwmtx_rlock(erts_rwmtx_t *rwmtx) { #ifdef USE_THREADS #ifdef ERTS_ENABLE_LOCK_CHECK +#ifdef ERTS_ENABLE_LOCK_POSITION + erts_lc_lock_flg_x(&rwmtx->lc, ERTS_LC_FLG_LO_READ,file,line); +#else erts_lc_lock_flg(&rwmtx->lc, ERTS_LC_FLG_LO_READ); #endif +#endif #ifdef ERTS_ENABLE_LOCK_COUNT erts_lcnt_lock_opt(&rwmtx->lcnt, ERTS_LCNT_LO_READ); #endif @@ -1984,7 +2634,11 @@ erts_rwmtx_runlock(erts_rwmtx_t *rwmtx) ERTS_GLB_INLINE int +#ifdef ERTS_ENABLE_LOCK_POSITION +erts_rwmtx_tryrwlock_x(erts_rwmtx_t *rwmtx, char *file, unsigned int line) +#else erts_rwmtx_tryrwlock(erts_rwmtx_t *rwmtx) +#endif { #ifdef USE_THREADS int res; @@ -1998,8 +2652,12 @@ erts_rwmtx_tryrwlock(erts_rwmtx_t *rwmtx) res = ethr_rwmutex_tryrwlock(&rwmtx->rwmtx); #ifdef ERTS_ENABLE_LOCK_CHECK +#ifdef ERTS_ENABLE_LOCK_POSITION + erts_lc_trylock_flg_x(res == 0, &rwmtx->lc, ERTS_LC_FLG_LO_READ_WRITE,file,line); +#else erts_lc_trylock_flg(res == 0, &rwmtx->lc, ERTS_LC_FLG_LO_READ_WRITE); #endif +#endif #ifdef ERTS_ENABLE_LOCK_COUNT erts_lcnt_trylock_opt(&rwmtx->lcnt, res, ERTS_LCNT_LO_READ_WRITE); #endif @@ -2011,7 +2669,7 @@ erts_rwmtx_tryrwlock(erts_rwmtx_t *rwmtx) } ERTS_GLB_INLINE void -#ifdef ERTS_ENABLE_LOCK_COUNT +#ifdef ERTS_ENABLE_LOCK_POSITION erts_rwmtx_rwlock_x(erts_rwmtx_t *rwmtx, char *file, unsigned int line) #else erts_rwmtx_rwlock(erts_rwmtx_t *rwmtx) @@ -2019,8 +2677,12 @@ erts_rwmtx_rwlock(erts_rwmtx_t *rwmtx) { #ifdef USE_THREADS #ifdef ERTS_ENABLE_LOCK_CHECK +#ifdef ERTS_ENABLE_LOCK_POSITION + erts_lc_lock_flg_x(&rwmtx->lc, ERTS_LC_FLG_LO_READ_WRITE,file,line); +#else erts_lc_lock_flg(&rwmtx->lc, ERTS_LC_FLG_LO_READ_WRITE); #endif +#endif #ifdef ERTS_ENABLE_LOCK_COUNT erts_lcnt_lock_opt(&rwmtx->lcnt, ERTS_LCNT_LO_READ_WRITE); #endif @@ -2323,6 +2985,104 @@ erts_no_atomic32_read_bset(erts_no_atomic32_t *var, return old; } +/* atomic64 */ + +ERTS_GLB_INLINE void +erts_no_atomic64_set(erts_no_atomic64_t *var, erts_aint64_t i) +{ + *var = i; +} + +ERTS_GLB_INLINE erts_aint64_t +erts_no_atomic64_read(erts_no_atomic64_t *var) +{ + return *var; +} + +ERTS_GLB_INLINE erts_aint64_t +erts_no_atomic64_inc_read(erts_no_atomic64_t *incp) +{ + return ++(*incp); +} + +ERTS_GLB_INLINE erts_aint64_t +erts_no_atomic64_dec_read(erts_no_atomic64_t *decp) +{ + return --(*decp); +} + +ERTS_GLB_INLINE void +erts_no_atomic64_inc(erts_no_atomic64_t *incp) +{ + ++(*incp); +} + +ERTS_GLB_INLINE void +erts_no_atomic64_dec(erts_no_atomic64_t *decp) +{ + --(*decp); +} + +ERTS_GLB_INLINE erts_aint64_t +erts_no_atomic64_add_read(erts_no_atomic64_t *addp, erts_aint64_t i) +{ + return *addp += i; +} + +ERTS_GLB_INLINE void +erts_no_atomic64_add(erts_no_atomic64_t *addp, erts_aint64_t i) +{ + *addp += i; +} + +ERTS_GLB_INLINE erts_aint64_t +erts_no_atomic64_read_bor(erts_no_atomic64_t *var, erts_aint64_t mask) +{ + erts_aint64_t old; + old = *var; + *var |= mask; + return old; +} + +ERTS_GLB_INLINE erts_aint64_t +erts_no_atomic64_read_band(erts_no_atomic64_t *var, erts_aint64_t mask) +{ + erts_aint64_t old; + old = *var; + *var &= mask; + return old; +} + +ERTS_GLB_INLINE erts_aint64_t +erts_no_atomic64_xchg(erts_no_atomic64_t *xchgp, erts_aint64_t new) +{ + erts_aint64_t old = *xchgp; + *xchgp = new; + return old; +} + +ERTS_GLB_INLINE erts_aint64_t +erts_no_atomic64_cmpxchg(erts_no_atomic64_t *xchgp, + erts_aint64_t new, + erts_aint64_t expected) +{ + erts_aint64_t old = *xchgp; + if (old == expected) + *xchgp = new; + return old; +} + +ERTS_GLB_INLINE erts_aint64_t +erts_no_atomic64_read_bset(erts_no_atomic64_t *var, + erts_aint64_t mask, + erts_aint64_t set) +{ + erts_aint64_t old = *var; + *var &= ~mask; + *var |= (mask & set); + return old; +} + /* spinlock */ ERTS_GLB_INLINE void @@ -2426,7 +3186,7 @@ erts_spin_unlock(erts_spinlock_t *lock) } ERTS_GLB_INLINE void -#ifdef ERTS_ENABLE_LOCK_COUNT +#ifdef ERTS_ENABLE_LOCK_POSITION erts_spin_lock_x(erts_spinlock_t *lock, char *file, unsigned int line) #else erts_spin_lock(erts_spinlock_t *lock) @@ -2434,8 +3194,12 @@ erts_spin_lock(erts_spinlock_t *lock) { #ifdef USE_THREADS #ifdef ERTS_ENABLE_LOCK_CHECK +#ifdef ERTS_ENABLE_LOCK_POSITION + erts_lc_lock_x(&lock->lc,file,line); +#else erts_lc_lock(&lock->lc); #endif +#endif #ifdef ERTS_ENABLE_LOCK_COUNT erts_lcnt_lock(&lock->lcnt); #endif @@ -2545,7 +3309,7 @@ erts_read_unlock(erts_rwlock_t *lock) } ERTS_GLB_INLINE void -#ifdef ERTS_ENABLE_LOCK_COUNT +#ifdef ERTS_ENABLE_LOCK_POSITION erts_read_lock_x(erts_rwlock_t *lock, char *file, unsigned int line) #else erts_read_lock(erts_rwlock_t *lock) @@ -2553,8 +3317,12 @@ erts_read_lock(erts_rwlock_t *lock) { #ifdef USE_THREADS #ifdef ERTS_ENABLE_LOCK_CHECK +#ifdef ERTS_ENABLE_LOCK_POSITION + erts_lc_lock_flg_x(&lock->lc, ERTS_LC_FLG_LO_READ,file,line); +#else erts_lc_lock_flg(&lock->lc, ERTS_LC_FLG_LO_READ); #endif +#endif #ifdef ERTS_ENABLE_LOCK_COUNT erts_lcnt_lock_opt(&lock->lcnt, ERTS_LCNT_LO_READ); #endif @@ -2584,7 +3352,7 @@ erts_write_unlock(erts_rwlock_t *lock) } ERTS_GLB_INLINE void -#ifdef ERTS_ENABLE_LOCK_COUNT +#ifdef ERTS_ENABLE_LOCK_POSITION erts_write_lock_x(erts_rwlock_t *lock, char *file, unsigned int line) #else erts_write_lock(erts_rwlock_t *lock) @@ -2592,8 +3360,12 @@ erts_write_lock(erts_rwlock_t *lock) { #ifdef USE_THREADS #ifdef ERTS_ENABLE_LOCK_CHECK +#ifdef ERTS_ENABLE_LOCK_POSITION + erts_lc_lock_flg_x(&lock->lc, ERTS_LC_FLG_LO_READ_WRITE,file,line); +#else erts_lc_lock_flg(&lock->lc, ERTS_LC_FLG_LO_READ_WRITE); #endif +#endif #ifdef ERTS_ENABLE_LOCK_COUNT erts_lcnt_lock_opt(&lock->lcnt, ERTS_LCNT_LO_READ_WRITE); #endif @@ -2635,10 +3407,10 @@ erts_lc_rwlock_is_rwlocked(erts_rwlock_t *lock) } ERTS_GLB_INLINE void -erts_tsd_key_create(erts_tsd_key_t *keyp) +erts_tsd_key_create(erts_tsd_key_t *keyp, char *keyname) { #ifdef USE_THREADS - int res = ethr_tsd_key_create(keyp); + int res = ethr_tsd_key_create(keyp, keyname); if (res) erts_thr_fatal_error(res, "create thread specific data key"); #endif @@ -2690,6 +3462,15 @@ ERTS_GLB_INLINE void erts_tse_return(erts_tse_t *ep) #endif } +ERTS_GLB_INLINE void erts_tse_prepare_timed(erts_tse_t *ep) +{ +#ifdef USE_THREADS + int res = ethr_event_prepare_timed(&((ethr_ts_event *) ep)->event); + if (res != 0) + erts_thr_fatal_error(res, "prepare timed"); +#endif +} + ERTS_GLB_INLINE void erts_tse_set(erts_tse_t *ep) { #ifdef USE_THREADS @@ -2722,6 +3503,27 @@ ERTS_GLB_INLINE int erts_tse_swait(erts_tse_t *ep, int spincount) #endif } +ERTS_GLB_INLINE int erts_tse_twait(erts_tse_t *ep, Sint64 tmo) +{ +#ifdef USE_THREADS + return ethr_event_twait(&((ethr_ts_event *) ep)->event, + (ethr_sint64_t) tmo); +#else + return ENOTSUP; +#endif +} + +ERTS_GLB_INLINE int erts_tse_stwait(erts_tse_t *ep, int spincount, Sint64 tmo) +{ +#ifdef USE_THREADS + return ethr_event_stwait(&((ethr_ts_event *) ep)->event, + spincount, + (ethr_sint64_t) tmo); +#else + return ENOTSUP; +#endif +} + ERTS_GLB_INLINE int erts_tse_is_tmp(erts_tse_t *ep) { #ifdef USE_THREADS @@ -2766,6 +3568,15 @@ ERTS_GLB_INLINE void erts_thr_yield(void) #ifdef ETHR_HAVE_ETHR_SIG_FUNCS ERTS_GLB_INLINE void +erts_thr_kill(erts_tid_t tid, int sig) { +#ifdef USE_THREADS + int res = ethr_kill((ethr_tid)tid, sig); + if (res) + erts_thr_fatal_error(res, "killing thread"); +#endif +} + +ERTS_GLB_INLINE void erts_thr_sigmask(int how, const sigset_t *set, sigset_t *oset) { #ifdef USE_THREADS diff --git a/erts/emulator/beam/erl_time.h b/erts/emulator/beam/erl_time.h index 4bbdcaa3e3..93a0d556bf 100644 --- a/erts/emulator/beam/erl_time.h +++ b/erts/emulator/beam/erl_time.h @@ -3,16 +3,17 @@ * * Copyright Ericsson AB 2006-2011. All Rights Reserved. * - * The contents of this file are subject to the Erlang Public License, - * Version 1.1, (the "License"); you may not use this file except in - * compliance with the License. You should have received a copy of the - * Erlang Public License along with this software. If not, it can be - * retrieved online at http://www.erlang.org/. - * - * Software distributed under the License is distributed on an "AS IS" - * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See - * the License for the specific language governing rights and limitations - * under the License. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. * * %CopyrightEnd% */ @@ -20,94 +21,49 @@ #ifndef ERL_TIME_H__ #define ERL_TIME_H__ -#define ERTS_SHORT_TIME_T_MAX ERTS_AINT32_T_MAX -#define ERTS_SHORT_TIME_T_MIN ERTS_AINT32_T_MIN -typedef erts_aint32_t erts_short_time_t; +/* timer wheel size NEED to be a power of 2 */ +#ifdef SMALL_MEMORY +#define ERTS_TIW_SIZE (1 << 13) +#else +#define ERTS_TIW_SIZE (1 << 16) +#endif -extern erts_smp_atomic32_t do_time; /* set at clock interrupt */ -extern SysTimeval erts_first_emu_time; +#if defined(DEBUG) || 0 +#define ERTS_TIME_ASSERT(B) ERTS_ASSERT(B) +#else +#define ERTS_TIME_ASSERT(B) ((void) 1) +#endif -/* -** Timer entry: -*/ -typedef struct erl_timer { - struct erl_timer* next; /* next entry tiw slot or chain */ - struct erl_timer* prev; /* prev entry tiw slot or chain */ - Uint slot; /* slot in timer wheel */ - Uint count; /* number of loops remaining */ - int active; /* 1=activated, 0=deactivated */ - /* called when timeout */ - void (*timeout)(void*); - /* called when cancel (may be NULL) */ - void (*cancel)(void*); - void* arg; /* argument to timeout/cancel procs */ -} ErlTimer; +typedef enum { + ERTS_NO_TIME_WARP_MODE, + ERTS_SINGLE_TIME_WARP_MODE, + ERTS_MULTI_TIME_WARP_MODE +} ErtsTimeWarpMode; -typedef void (*ErlTimeoutProc)(void*); -typedef void (*ErlCancelProc)(void*); +typedef struct ErtsTimerWheel_ ErtsTimerWheel; +typedef ErtsMonotonicTime * ErtsNextTimeoutRef; -#ifdef ERTS_SMP -/* - * Process and port timer - */ -typedef union ErtsSmpPTimer_ ErtsSmpPTimer; -union ErtsSmpPTimer_ { - struct { - ErlTimer tm; - Eterm id; - void (*timeout_func)(void*); - ErtsSmpPTimer **timer_ref; - Uint32 flags; - } timer; - ErtsSmpPTimer *next; -}; +extern SysTimeval erts_first_emu_time; -void erts_create_smp_ptimer(ErtsSmpPTimer **timer_ref, - Eterm id, - ErlTimeoutProc timeout_func, - Uint timeout); -void erts_cancel_smp_ptimer(ErtsSmpPTimer *ptimer); -#endif +void erts_monitor_time_offset(Eterm id, Eterm ref); +int erts_demonitor_time_offset(Eterm ref); -/* timer-wheel api */ +int erts_init_time_sup(int, ErtsTimeWarpMode); +void erts_late_init_time_sup(void); -void erts_init_time(void); -void erts_set_timer(ErlTimer*, ErlTimeoutProc, ErlCancelProc, void*, Uint); -void erts_cancel_timer(ErlTimer*); -void erts_bump_timer(erts_short_time_t); +ErtsNextTimeoutRef erts_get_next_timeout_reference(ErtsTimerWheel *); +void erts_init_time(int time_correction, ErtsTimeWarpMode time_warp_mode); +void erts_bump_timers(ErtsTimerWheel *, ErtsMonotonicTime); Uint erts_timer_wheel_memory_size(void); -Uint erts_time_left(ErlTimer *); -erts_short_time_t erts_next_time(void); #ifdef DEBUG void erts_p_slpq(void); #endif -ERTS_GLB_INLINE erts_short_time_t erts_do_time_read_and_reset(void); -ERTS_GLB_INLINE void erts_do_time_add(erts_short_time_t); - -#if ERTS_GLB_INLINE_INCL_FUNC_DEF - -ERTS_GLB_INLINE erts_short_time_t erts_do_time_read_and_reset(void) -{ - erts_short_time_t time = erts_smp_atomic32_xchg_acqb(&do_time, 0); - if (time < 0) - erl_exit(ERTS_ABORT_EXIT, "Internal time management error\n"); - return time; -} - -ERTS_GLB_INLINE void erts_do_time_add(erts_short_time_t elapsed) -{ - erts_smp_atomic32_add_relb(&do_time, elapsed); -} - -#endif /* #if ERTS_GLB_INLINE_INCL_FUNC_DEF */ - - /* time_sup */ -#if (defined(HAVE_GETHRVTIME) || defined(HAVE_CLOCK_GETTIME)) +#if (defined(HAVE_GETHRVTIME) || defined(HAVE_CLOCK_GETTIME_CPU_TIME)) # ifndef HAVE_ERTS_NOW_CPU # define HAVE_ERTS_NOW_CPU # ifdef HAVE_GETHRVTIME @@ -121,25 +77,395 @@ void erts_get_now_cpu(Uint* megasec, Uint* sec, Uint* microsec); typedef UWord erts_approx_time_t; erts_approx_time_t erts_get_approx_time(void); -void erts_get_timeval(SysTimeval *tv); -erts_time_t erts_get_time(void); +int erts_has_time_correction(void); +int erts_check_time_adj_support(int time_correction, + ErtsTimeWarpMode time_warp_mode); + +ErtsTimeWarpMode erts_time_warp_mode(void); + +typedef enum { + ERTS_TIME_OFFSET_PRELIMINARY, + ERTS_TIME_OFFSET_FINAL, + ERTS_TIME_OFFSET_VOLATILE +} ErtsTimeOffsetState; + +ErtsTimeOffsetState erts_time_offset_state(void); +ErtsTimeOffsetState erts_finalize_time_offset(void); +struct process; +Eterm erts_get_monotonic_start_time(struct process *c_p); +Eterm erts_get_monotonic_end_time(struct process *c_p); +Eterm erts_monotonic_time_source(struct process*c_p); +Eterm erts_system_time_source(struct process*c_p); + +#ifdef SYS_CLOCK_RESOLUTION +#define ERTS_CLKTCK_RESOLUTION ((ErtsMonotonicTime) (SYS_CLOCK_RESOLUTION*1000)) +#else +#define ERTS_CLKTCK_RESOLUTION (erts_time_sup__.r.o.clktck_resolution) +#endif + +#define ERTS_TIMER_WHEEL_MSEC (ERTS_TIW_SIZE/(ERTS_CLKTCK_RESOLUTION/1000)) + +struct erts_time_sup_read_only__ { + ErtsMonotonicTime monotonic_time_unit; +#if !ERTS_COMPILE_TIME_MONOTONIC_TIME_UNIT + ErtsMonotonicTime start; + struct { + ErtsMonotonicTime native; + ErtsMonotonicTime nsec; + ErtsMonotonicTime usec; + ErtsMonotonicTime msec; + ErtsMonotonicTime sec; + } start_offset; +#endif +#ifndef SYS_CLOCK_RESOLUTION + ErtsMonotonicTime clktck_resolution; +#endif +}; + +typedef struct { + union { + struct erts_time_sup_read_only__ o; + char align__[(((sizeof(struct erts_time_sup_read_only__) - 1) + / ASSUMED_CACHE_LINE_SIZE) + 1) + * ASSUMED_CACHE_LINE_SIZE]; + } r; +} ErtsTimeSupData; + +extern ErtsTimeSupData erts_time_sup__; + +ErtsMonotonicTime erts_napi_monotonic_time(int time_unit); +ErtsMonotonicTime erts_napi_time_offset(int time_unit); +ErtsMonotonicTime erts_napi_convert_time_unit(ErtsMonotonicTime val, int from, int to); -ERTS_GLB_INLINE int erts_cmp_timeval(SysTimeval *t1p, SysTimeval *t2p); +ERTS_GLB_INLINE Uint64 +erts_time_unit_conversion(Uint64 value, + Uint32 from_time_unit, + Uint32 to_time_unit); #if ERTS_GLB_INLINE_INCL_FUNC_DEF -ERTS_GLB_INLINE int -erts_cmp_timeval(SysTimeval *t1p, SysTimeval *t2p) +ERTS_GLB_INLINE Uint64 +erts_time_unit_conversion(Uint64 value, + Uint32 from_time_unit, + Uint32 to_time_unit) { - if (t1p->tv_sec == t2p->tv_sec) { - if (t1p->tv_usec < t2p->tv_usec) - return -1; - else if (t1p->tv_usec > t2p->tv_usec) - return 1; - return 0; - } - return t1p->tv_sec < t2p->tv_sec ? -1 : 1; + Uint64 high, low, result; + if (value <= ~((Uint64) 0)/to_time_unit) + return (value*to_time_unit)/from_time_unit; + + low = value & ((Uint64) 0xffffffff); + high = (value >> 32) & ((Uint64) 0xffffffff); + + low *= to_time_unit; + high *= to_time_unit; + + high += (low >> 32) & ((Uint64) 0xffffffff); + low &= ((Uint64) 0xffffffff); + + result = high % from_time_unit; + high /= from_time_unit; + high <<= 32; + + result <<= 32; + result += low; + result /= from_time_unit; + result += high; + + return result; } -#endif /* #if ERTS_GLB_INLINE_INCL_FUNC_DEF */ +#endif /* ERTS_GLB_INLINE_INCL_FUNC_DEF */ + +/* + * Range of monotonic time internally + */ + +#define ERTS_MONOTONIC_BEGIN \ + ERTS_MONOTONIC_TIME_UNIT +#define ERTS_MONOTONIC_END \ + ((ERTS_MONOTONIC_TIME_MAX / ERTS_MONOTONIC_TIME_UNIT) \ + * ERTS_MONOTONIC_TIME_UNIT) + +#if ERTS_COMPILE_TIME_MONOTONIC_TIME_UNIT + +/* + * If the monotonic time unit is a compile time constant, + * it is assumed (and need) to be a power of 10. + */ + +#if ERTS_COMPILE_TIME_MONOTONIC_TIME_UNIT < 1000*1000 +# error Compile time time unit needs to be at least 1000000 +#endif + +#if ERTS_COMPILE_TIME_MONOTONIC_TIME_UNIT == 1000*1000*1000 +/* Nano-second time unit */ + +#define ERTS_MONOTONIC_TO_SEC__(NSEC) ((NSEC) / (1000*1000*1000)) +#define ERTS_MONOTONIC_TO_MSEC__(NSEC) ((NSEC) / (1000*1000)) +#define ERTS_MONOTONIC_TO_USEC__(NSEC) ((NSEC) / 1000) +#define ERTS_MONOTONIC_TO_NSEC__(NSEC) (NSEC) + +#define ERTS_SEC_TO_MONOTONIC__(SEC) (((ErtsMonotonicTime) (SEC))*(1000*1000*1000)) +#define ERTS_MSEC_TO_MONOTONIC__(MSEC) (((ErtsMonotonicTime) (MSEC))*(1000*1000)) +#define ERTS_USEC_TO_MONOTONIC__(USEC) (((ErtsMonotonicTime) (USEC))*1000) +#define ERTS_NSEC_TO_MONOTONIC__(NSEC) ((ErtsMonotonicTime) (NSEC)) + +#elif ERTS_COMPILE_TIME_MONOTONIC_TIME_UNIT == 1000*1000 +/* Micro-second time unit */ + +#define ERTS_MONOTONIC_TO_SEC__(USEC) ((USEC) / (1000*1000)) +#define ERTS_MONOTONIC_TO_MSEC__(USEC) ((USEC) / 1000) +#define ERTS_MONOTONIC_TO_USEC__(USEC) (USEC) +#define ERTS_MONOTONIC_TO_NSEC__(USEC) ((USEC)*1000) + +#define ERTS_SEC_TO_MONOTONIC__(SEC) (((ErtsMonotonicTime) (SEC))*(1000*1000)) +#define ERTS_MSEC_TO_MONOTONIC__(MSEC) (((ErtsMonotonicTime) (MSEC))*1000) +#define ERTS_USEC_TO_MONOTONIC__(USEC) ((ErtsMonotonicTime) (USEC)) +#define ERTS_NSEC_TO_MONOTONIC__(NSEC) (((ErtsMonotonicTime) (NSEC))/1000) + +#else +#error Missing implementation for monotonic time unit +#endif + +#define ERTS_MONOTONIC_TIME_UNIT \ + ((ErtsMonotonicTime) ERTS_COMPILE_TIME_MONOTONIC_TIME_UNIT) + +/* + * NOTE! ERTS_MONOTONIC_TIME_START_EXTERNAL *need* to be a multiple + * of ERTS_MONOTONIC_TIME_UNIT. + */ + +#ifdef ARCH_32 +/* + * Want to use a big-num of arity 2 as long as possible (584 years + * in the nano-second time unit case). + */ +#define ERTS_MONOTONIC_TIME_START_EXTERNAL \ + (((((((ErtsMonotonicTime) 1) << 32)-1) \ + / ERTS_MONOTONIC_TIME_UNIT) \ + * ERTS_MONOTONIC_TIME_UNIT) \ + + ERTS_MONOTONIC_TIME_UNIT) + +#else /* ARCH_64 */ + +#if ERTS_COMPILE_TIME_MONOTONIC_TIME_UNIT <= 10*1000*1000 + +/* + * Using micro second time unit or lower. Start at zero since + * time will remain an immediate for a very long time anyway + * (1827 years in the 10 micro second case)... + */ +#define ERTS_MONOTONIC_TIME_START_EXTERNAL ((ErtsMonotonicTime) 0) + +#else /* ERTS_COMPILE_TIME_MONOTONIC_TIME_UNIT > 10*1000*1000 */ + +/* + * Want to use an immediate as long as possible (36 years in the + * nano-second time unit case). +*/ +#define ERTS_MONOTONIC_TIME_START_EXTERNAL \ + ((((ErtsMonotonicTime) MIN_SMALL) \ + / ERTS_MONOTONIC_TIME_UNIT) \ + * ERTS_MONOTONIC_TIME_UNIT) + +#endif /* ERTS_COMPILE_TIME_MONOTONIC_TIME_UNIT > 1000*1000 */ + +#endif /* ARCH_64 */ + +/* + * Offsets from internal monotonic time to external monotonic time + */ + +#define ERTS_MONOTONIC_OFFSET_NATIVE \ + (ERTS_MONOTONIC_TIME_START_EXTERNAL - ERTS_MONOTONIC_BEGIN) +#define ERTS_MONOTONIC_OFFSET_NSEC \ + ERTS_MONOTONIC_TO_NSEC__(ERTS_MONOTONIC_OFFSET_NATIVE) +#define ERTS_MONOTONIC_OFFSET_USEC \ + ERTS_MONOTONIC_TO_USEC__(ERTS_MONOTONIC_OFFSET_NATIVE) +#define ERTS_MONOTONIC_OFFSET_MSEC \ + ERTS_MONOTONIC_TO_MSEC__(ERTS_MONOTONIC_OFFSET_NATIVE) +#define ERTS_MONOTONIC_OFFSET_SEC \ + ERTS_MONOTONIC_TO_SEC__(ERTS_MONOTONIC_OFFSET_NATIVE) + +#define ERTS_MONOTONIC_TO_CLKTCKS__(MON) \ + ((MON) / (ERTS_MONOTONIC_TIME_UNIT/ERTS_CLKTCK_RESOLUTION)) +#define ERTS_CLKTCKS_TO_MONOTONIC__(TCKS) \ + ((TCKS) * (ERTS_MONOTONIC_TIME_UNIT/ERTS_CLKTCK_RESOLUTION)) + +#else /* !ERTS_COMPILE_TIME_MONOTONIC_TIME_UNIT */ + +/* + * Initialized in erts_init_sys_time_sup() + */ +#define ERTS_MONOTONIC_TIME_UNIT (erts_time_sup__.r.o.monotonic_time_unit) + +/* + * Offsets from internal monotonic time to external monotonic time + * + * Initialized in erts_init_time_sup()... + */ +#define ERTS_MONOTONIC_TIME_START_EXTERNAL (erts_time_sup__.r.o.start) +#define ERTS_MONOTONIC_OFFSET_NATIVE (erts_time_sup__.r.o.start_offset.native) +#define ERTS_MONOTONIC_OFFSET_NSEC (erts_time_sup__.r.o.start_offset.nsec) +#define ERTS_MONOTONIC_OFFSET_USEC (erts_time_sup__.r.o.start_offset.usec) +#define ERTS_MONOTONIC_OFFSET_MSEC (erts_time_sup__.r.o.start_offset.msec) +#define ERTS_MONOTONIC_OFFSET_SEC (erts_time_sup__.r.o.start_offset.sec) + +#define ERTS_CONV_FROM_MON_UNIT___(M, TO) \ + ((ErtsMonotonicTime) \ + erts_time_unit_conversion((Uint64) (M), \ + (Uint32) ERTS_MONOTONIC_TIME_UNIT, \ + (Uint32) (TO))) + +#define ERTS_CONV_TO_MON_UNIT___(M, FROM) \ + ((ErtsMonotonicTime) \ + erts_time_unit_conversion((Uint64) (M), \ + (Uint32) (FROM), \ + (Uint32) ERTS_MONOTONIC_TIME_UNIT)) \ + +#define ERTS_MONOTONIC_TO_SEC__(M) \ + ERTS_CONV_FROM_MON_UNIT___((M), 1) +#define ERTS_MONOTONIC_TO_MSEC__(M) \ + ERTS_CONV_FROM_MON_UNIT___((M), 1000) +#define ERTS_MONOTONIC_TO_USEC__(M) \ + ERTS_CONV_FROM_MON_UNIT___((M), 1000*1000) +#define ERTS_MONOTONIC_TO_NSEC__(M) \ + ERTS_CONV_FROM_MON_UNIT___((M), 1000*1000*1000) + +#define ERTS_SEC_TO_MONOTONIC__(SEC) \ + ERTS_CONV_TO_MON_UNIT___((SEC), 1) +#define ERTS_MSEC_TO_MONOTONIC__(MSEC) \ + ERTS_CONV_TO_MON_UNIT___((MSEC), 1000) +#define ERTS_USEC_TO_MONOTONIC__(USEC) \ + ERTS_CONV_TO_MON_UNIT___((USEC), 1000*1000) +#define ERTS_NSEC_TO_MONOTONIC__(NSEC) \ + ERTS_CONV_TO_MON_UNIT___((NSEC), 1000*1000*1000) + +#define ERTS_MONOTONIC_TO_CLKTCKS__(MON) \ + ERTS_CONV_FROM_MON_UNIT___((MON), ERTS_CLKTCK_RESOLUTION) +#define ERTS_CLKTCKS_TO_MONOTONIC__(TCKS) \ + ERTS_CONV_TO_MON_UNIT___((TCKS), ERTS_CLKTCK_RESOLUTION) + +#endif /* !ERTS_COMPILE_TIME_MONOTONIC_TIME_UNIT */ + +#define ERTS_MONOTONIC_TIME_END_EXTERNAL \ + (ERTS_MONOTONIC_TIME_START_EXTERNAL < 0 \ + ? (ERTS_MONOTONIC_TIME_START_EXTERNAL \ + + (ERTS_MONOTONIC_END - ERTS_MONOTONIC_BEGIN)) \ + : (ERTS_MONOTONIC_END - ERTS_MONOTONIC_TIME_START_EXTERNAL)) + +#define ERTS_MSEC_TO_CLKTCKS__(MON) \ + ((MON) * (ERTS_CLKTCK_RESOLUTION/1000)) +#define ERTS_CLKTCKS_TO_MSEC__(TCKS) \ + ((TCKS) / (ERTS_CLKTCK_RESOLUTION/1000)) + +#define ERTS_MONOTONIC_TO_SEC(X) \ + (ERTS_TIME_ASSERT((X) >= 0), \ + ERTS_MONOTONIC_TO_SEC__((X))) +#define ERTS_MONOTONIC_TO_MSEC(X) \ + (ERTS_TIME_ASSERT((X) >= 0), \ + ERTS_MONOTONIC_TO_MSEC__((X))) +#define ERTS_MONOTONIC_TO_USEC(X) \ + (ERTS_TIME_ASSERT((X) >= 0), \ + ERTS_MONOTONIC_TO_USEC__((X))) +#define ERTS_MONOTONIC_TO_NSEC(X) \ + (ERTS_TIME_ASSERT((X) >= 0), \ + ERTS_MONOTONIC_TO_NSEC__((X))) +#define ERTS_SEC_TO_MONOTONIC(X) \ + (ERTS_TIME_ASSERT((X) >= 0), \ + ERTS_SEC_TO_MONOTONIC__((X))) +#define ERTS_MSEC_TO_MONOTONIC(X) \ + (ERTS_TIME_ASSERT((X) >= 0), \ + ERTS_MSEC_TO_MONOTONIC__((X))) +#define ERTS_USEC_TO_MONOTONIC(X) \ + (ERTS_TIME_ASSERT((X) >= 0), \ + ERTS_USEC_TO_MONOTONIC__((X))) +#define ERTS_NSEC_TO_MONOTONIC(X) \ + (ERTS_TIME_ASSERT((X) >= 0), \ + ERTS_NSEC_TO_MONOTONIC__((X))) + +#define ERTS_MONOTONIC_TO_CLKTCKS(X) \ + (ERTS_TIME_ASSERT((X) >= 0), \ + ERTS_MONOTONIC_TO_CLKTCKS__((X))) +#define ERTS_CLKTCKS_TO_MONOTONIC(X) \ + (ERTS_TIME_ASSERT((X) >= 0), \ + ERTS_CLKTCKS_TO_MONOTONIC__((X))) + +#define ERTS_MSEC_TO_CLKTCKS(X) \ + (ERTS_TIME_ASSERT((X) >= 0), \ + ERTS_MSEC_TO_CLKTCKS__((X))) +#define ERTS_CLKTCKS_TO_MSEC(X) \ + (ERTS_TIME_ASSERT((X) >= 0), \ + ERTS_CLKTCKS_TO_MSEC__((X))) + #endif /* ERL_TIME_H__ */ + +/* timer-wheel api */ +#if defined(ERTS_WANT_TIMER_WHEEL_API) && !defined(ERTS_GOT_TIMER_WHEEL_API) +#define ERTS_GOT_TIMER_WHEEL_API + +#include "erl_thr_progress.h" +#include "erl_process.h" + +void erts_sched_init_time_sup(ErtsSchedulerData *esdp); + + +#define ERTS_TWHEEL_SLOT_AT_ONCE -1 +#define ERTS_TWHEEL_SLOT_INACTIVE -2 + +/* +** Timer entry: +*/ +typedef struct erl_timer { + struct erl_timer* next; /* next entry tiw slot or chain */ + struct erl_timer* prev; /* prev entry tiw slot or chain */ + union { + struct { + void (*timeout)(void*); /* called when timeout */ + void (*cancel)(void*); /* called when cancel (may be NULL) */ + void* arg; /* argument to timeout/cancel procs */ + } func; + ErtsThrPrgrLaterOp cleanup; + } u; + ErtsMonotonicTime timeout_pos; /* Timeout in absolute clock ticks */ + int slot; +} ErtsTWheelTimer; + +typedef void (*ErlTimeoutProc)(void*); +typedef void (*ErlCancelProc)(void*); + +void erts_twheel_set_timer(ErtsTimerWheel *tiw, + ErtsTWheelTimer *p, ErlTimeoutProc timeout, + ErlCancelProc cancel, void *arg, + ErtsMonotonicTime timeout_pos); +void erts_twheel_cancel_timer(ErtsTimerWheel *tiw, ErtsTWheelTimer *p); +ErtsTimerWheel *erts_create_timer_wheel(ErtsSchedulerData *esdp); + +ErtsMonotonicTime erts_check_next_timeout_time(ErtsSchedulerData *); + +ERTS_GLB_INLINE void erts_twheel_init_timer(ErtsTWheelTimer *p); +ERTS_GLB_INLINE ErtsMonotonicTime erts_next_timeout_time(ErtsNextTimeoutRef); + +#if ERTS_GLB_INLINE_INCL_FUNC_DEF + +ERTS_GLB_INLINE void erts_twheel_init_timer(ErtsTWheelTimer *p) +{ + p->slot = ERTS_TWHEEL_SLOT_INACTIVE; +} + +ERTS_GLB_INLINE ErtsMonotonicTime erts_next_timeout_time(ErtsNextTimeoutRef nxt_tmo_ref) +{ + return *((ErtsMonotonicTime *) nxt_tmo_ref); +} + +#endif /* ERTS_GLB_INLINE_INCL_FUNC_DEF */ + +void +erts_twheel_debug_foreach(ErtsTimerWheel *tiw, + void (*tclbk)(void *), + void (*func)(void *, + ErtsMonotonicTime, + void *), + void *arg); + +#endif /* timer wheel api */ diff --git a/erts/emulator/beam/erl_time_sup.c b/erts/emulator/beam/erl_time_sup.c index 3272a5326d..99005356ed 100644 --- a/erts/emulator/beam/erl_time_sup.c +++ b/erts/emulator/beam/erl_time_sup.c @@ -1,77 +1,28 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 1999-2012. All Rights Reserved. + * Copyright Ericsson AB 1999-2015. All Rights Reserved. * - * The contents of this file are subject to the Erlang Public License, - * Version 1.1, (the "License"); you may not use this file except in - * compliance with the License. You should have received a copy of the - * Erlang Public License along with this software. If not, it can be - * retrieved online at http://www.erlang.org/. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Software distributed under the License is distributed on an "AS IS" - * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See - * the License for the specific language governing rights and limitations - * under the License. + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. * * %CopyrightEnd% */ /* -** Support routines for the timer wheel -** -** This code contains two strategies for dealing with -** date/time changes in the system. -** If the system has some kind of high resolution timer (HAVE_GETHRTIME), -** the high resolution timer is used to correct the time-of-day and the -** timeouts, the base source is the hrtimer, but at certain intervals the -** OS time-of-day is checked and if it is not within certain bounds, the -** delivered time gets slowly adjusted for each call until -** it corresponds to the system time (built-in adjtime...). -** The call gethrtime() is detected by autoconf on Unix, but other -** platforms may define it in erl_*_sys.h and implement -** their own high resolution timer. The high resolution timer -** strategy is (probably) best on all systems where the timer have -** a resolution higher or equal to gettimeofday (or what's implemented -** is sys_gettimeofday()). The actual resolution is the interesting thing, -** not the unit's thats used (i.e. on VxWorks, nanoseconds can be -** retrieved in terms of units, but the actual resolution is the same as -** for the clock ticks). -** If the systems best timer routine is kernel ticks returned from -** sys_times(), and the actual resolution of sys_gettimeofday() is -** better (like most unixes that does not have any realtime extensions), -** another strategy is used. The tolerant gettimeofday() corrects -** the value with respect to uptime (sys_times() return value) and checks -** for correction both when delivering timeticks and delivering nowtime. -** this strategy is slower, but accurate on systems without better timer -** routines. The kernel tick resolution is not enough to implement -** a gethrtime routine. On Linux and other non solaris unix-boxes the second -** strategy is used, on all other platforms we use the first. -** -** The following is expected (from sys.[ch] and erl_*_sys.h): -** -** 64 bit integers. So it is, and so it will be. -** -** sys_init_time(), will return the clock resolution in MS and -** that's about it. More could be added of course -** If the clock-rate is constant (i.e. 1 ms) one can define -** SYS_CLOCK_RESOLUTION (to 1), -** which makes erts_deliver_time/erts_time_remaining a bit faster. -** -** if HAVE_GETHRTIME is defined: -** sys_gethrtime() will return a SysHrTime (long long) representing -** nanoseconds, sys_init_hrtime() will do any initialization. -** else -** a long (64bit) integer type called Sint64 should be defined. -** -** sys_times() will return clock_ticks since start and -** fill in a SysTimes structure (struct tms). Instead of CLK_TCK, -** SYS_CLK_TCK is used to determine the resolution of kernel ticks. -** -** sys_gettimeofday() will take a SysTimeval (a struct timeval) as parameter -** and fill it in as gettimeofday(X,NULL). -** -*/ + * Support routines for the time + */ + +/* #define ERTS_TIME_CORRECTION_PRINT */ #ifdef HAVE_CONFIG_H # include "config.h" @@ -80,384 +31,1261 @@ #include "sys.h" #include "erl_vm.h" #include "global.h" - +#define ERTS_WANT_TIMER_WHEEL_API +#include "erl_time.h" +#include "erl_driver.h" +#include "erl_nif.h" + static erts_smp_mtx_t erts_timeofday_mtx; - -static SysTimeval inittv; /* Used everywhere, the initial time-of-day */ +static erts_smp_mtx_t erts_get_time_mtx; static SysTimes t_start; /* Used in elapsed_time_both */ -static SysTimeval gtv; /* Used in wall_clock_elapsed_time_both */ -static SysTimeval then; /* Used in get_now */ -static SysTimeval last_emu_time; /* Used in erts_get_emu_time() */ -SysTimeval erts_first_emu_time; /* Used in erts_get_emu_time() */ +static ErtsMonotonicTime prev_wall_clock_elapsed; /* Used in wall_clock_elapsed_time_both */ +static ErtsMonotonicTime previous_now; /* Used in get_now */ + +static ErtsMonitor *time_offset_monitors = NULL; +static Uint no_time_offset_monitors = 0; + +#ifdef DEBUG +static int time_sup_initialized = 0; +#endif -union { - erts_smp_atomic_t time; - char align[ERTS_CACHE_LINE_SIZE]; -} approx erts_align_attribute(ERTS_CACHE_LINE_SIZE); +#define ERTS_MONOTONIC_TIME_KILO \ + ((ErtsMonotonicTime) 1000) +#define ERTS_MONOTONIC_TIME_MEGA \ + (ERTS_MONOTONIC_TIME_KILO*ERTS_MONOTONIC_TIME_KILO) +#define ERTS_MONOTONIC_TIME_GIGA \ + (ERTS_MONOTONIC_TIME_MEGA*ERTS_MONOTONIC_TIME_KILO) +#define ERTS_MONOTONIC_TIME_TERA \ + (ERTS_MONOTONIC_TIME_GIGA*ERTS_MONOTONIC_TIME_KILO) +static void init_time_napi(void); static void -init_approx_time(void) +schedule_send_time_offset_changed_notifications(ErtsMonotonicTime new_offset); + +struct time_sup_read_only__ { + ErtsMonotonicTime (*get_time)(void); + int correction; + ErtsTimeWarpMode warp_mode; +#ifdef ERTS_HAVE_OS_MONOTONIC_TIME_SUPPORT + ErtsMonotonicTime moffset; + int os_corrected_monotonic_time; + int os_monotonic_time_disable; + char *os_monotonic_time_func; + char *os_monotonic_time_clock_id; + int os_monotonic_time_locked; + Uint64 os_monotonic_time_resolution; + Uint64 os_monotonic_time_extended; +#endif + char *os_system_time_func; + char *os_system_time_clock_id; + int os_system_time_locked; + Uint64 os_system_time_resolution; + Uint64 os_system_time_extended; + struct { + ErtsMonotonicTime large_diff; + ErtsMonotonicTime small_diff; + } adj; + struct { + ErtsMonotonicTime error; + ErtsMonotonicTime resolution; + int intervals; + int use_avg; + } drift_adj; +}; + +typedef struct { + ErtsMonotonicTime drift; /* Correction for os monotonic drift */ + ErtsMonotonicTime error; /* Correction for error between system times */ +} ErtsMonotonicCorrection; + +typedef struct { + ErtsMonotonicTime erl_mtime; + ErtsMonotonicTime os_mtime; + ErtsMonotonicCorrection correction; +} ErtsMonotonicCorrectionInstance; + +#define ERTS_MAX_DRIFT_INTERVALS 50 +typedef struct { + struct { + struct { + ErtsMonotonicTime sys; + ErtsMonotonicTime mon; + } diff; + struct { + ErtsMonotonicTime sys; + ErtsMonotonicTime mon; + } time; + } intervals[ERTS_MAX_DRIFT_INTERVALS]; + struct { + ErtsMonotonicTime sys; + ErtsMonotonicTime mon; + } acc; + int ix; + int dirty_counter; +} ErtsMonotonicDriftData; + +typedef struct { + ErtsMonotonicCorrectionInstance prev; + ErtsMonotonicCorrectionInstance curr; +} ErtsMonotonicCorrectionInstances; + +typedef struct { + ErtsMonotonicCorrectionInstances insts; + ErtsMonotonicDriftData drift; + ErtsMonotonicTime last_check; + int short_check_interval; +} ErtsMonotonicCorrectionData; + +struct time_sup_infrequently_changed__ { +#ifdef ERTS_HAVE_OS_MONOTONIC_TIME_SUPPORT + struct { + erts_smp_rwmtx_t rwmtx; + ErtsTWheelTimer timer; + ErtsMonotonicCorrectionData cdata; + } parmon; + ErtsMonotonicTime minit; +#endif + ErtsSystemTime sinit; + ErtsMonotonicTime not_corrected_moffset; + erts_smp_atomic64_t offset; + ErtsMonotonicTime shadow_offset; + erts_smp_atomic32_t preliminary_offset; +}; + +struct time_sup_frequently_changed__ { + ErtsMonotonicTime last_not_corrected_time; +}; + +static struct { + union { + struct time_sup_read_only__ o; + char align__[ERTS_ALC_CACHE_LINE_ALIGN_SIZE(sizeof(struct time_sup_read_only__))]; + } r; + union { + struct time_sup_infrequently_changed__ c; + char align__[ERTS_ALC_CACHE_LINE_ALIGN_SIZE(sizeof(struct time_sup_infrequently_changed__))]; + } inf; + union { + struct time_sup_frequently_changed__ c; + char align__[ERTS_ALC_CACHE_LINE_ALIGN_SIZE(sizeof(struct time_sup_frequently_changed__))]; + } f; +} time_sup erts_align_attribute(ERTS_CACHE_LINE_SIZE); + +ErtsTimeSupData erts_time_sup__ erts_align_attribute(ERTS_CACHE_LINE_SIZE); + +/* + * erts_get_approx_time() returns an *approximate* time + * in seconds. NOTE that this time may jump backwards!!! + */ +erts_approx_time_t +erts_get_approx_time(void) { - erts_smp_atomic_init_nob(&approx.time, 0); + ErtsSystemTime stime = erts_os_system_time(); + return (erts_approx_time_t) ERTS_MONOTONIC_TO_SEC(stime); } -static ERTS_INLINE erts_approx_time_t -get_approx_time(void) +static ERTS_INLINE void +init_time_offset(ErtsMonotonicTime offset) { - return (erts_approx_time_t) erts_smp_atomic_read_nob(&approx.time); + erts_smp_atomic64_init_nob(&time_sup.inf.c.offset, (erts_aint64_t) offset); } static ERTS_INLINE void -update_approx_time(SysTimeval *tv) +set_time_offset(ErtsMonotonicTime offset) +{ + erts_smp_atomic64_set_relb(&time_sup.inf.c.offset, (erts_aint64_t) offset); +} + +static ERTS_INLINE ErtsMonotonicTime +get_time_offset(void) { - erts_approx_time_t new_secs = (erts_approx_time_t) tv->tv_sec; - erts_approx_time_t old_secs = get_approx_time(); - if (old_secs != new_secs) - erts_smp_atomic_set_nob(&approx.time, new_secs); + return (ErtsMonotonicTime) erts_smp_atomic64_read_acqb(&time_sup.inf.c.offset); } +static ERTS_INLINE void +update_last_mtime(ErtsSchedulerData *esdp, ErtsMonotonicTime mtime) +{ + if (!esdp) + esdp = erts_get_scheduler_data(); + if (esdp) { + ASSERT(mtime >= esdp->last_monotonic_time); + esdp->last_monotonic_time = mtime; + esdp->check_time_reds = 0; + } +} + +#ifdef ERTS_HAVE_OS_MONOTONIC_TIME_SUPPORT + /* - * erts_get_approx_time() returns an *approximate* time - * in seconds. NOTE that this time may jump backwards!!! + * Time correction adjustments made due to + * error between Erlang system time and OS + * system time: + * - Large adjustment ~1% + * - Small adjustment ~0.05% */ -erts_approx_time_t -erts_get_approx_time(void) +#define ERTS_TCORR_ERR_UNIT 2048 +#define ERTS_TCORR_ERR_LARGE_ADJ 20 +#define ERTS_TCORR_ERR_SMALL_ADJ 1 + +#define ERTS_INIT_SHORT_INTERVAL_COUNTER 10 +#define ERTS_LONG_TIME_CORRECTION_CHECK ERTS_SEC_TO_MONOTONIC(60) +#define ERTS_SHORT_TIME_CORRECTION_CHECK ERTS_SEC_TO_MONOTONIC(15) + +#define ERTS_TIME_DRIFT_MAX_ADJ_DIFF ERTS_USEC_TO_MONOTONIC(50) +#define ERTS_TIME_DRIFT_MIN_ADJ_DIFF ERTS_USEC_TO_MONOTONIC(5) + +/* + * Maximum drift of the OS monotonic clock expected. + * + * We use 1 milli second per second. If the monotonic + * clock drifts more than this we will fail to adjust for + * drift, and error correction will kick in instead. + * If it is larger than this, one could argue that the + * primitive is to poor to be used... + */ +#define ERTS_MAX_MONOTONIC_DRIFT ERTS_MSEC_TO_MONOTONIC(1) + +/* + * We assume that precision is 32 times worse than the + * resolution. This is a wild guess, but there are no + * practical way to determine actual precision. + */ +#define ERTS_ASSUMED_PRECISION_DROP 32 + +#define ERTS_MIN_MONOTONIC_DRIFT_MEASUREMENT \ + (ERTS_SHORT_TIME_CORRECTION_CHECK - 2*ERTS_MAX_MONOTONIC_DRIFT) + + +static ERTS_INLINE ErtsMonotonicTime +calc_corrected_erl_mtime(ErtsMonotonicTime os_mtime, + ErtsMonotonicCorrectionInstance *cip, + ErtsMonotonicTime *os_mdiff_p, + int os_drift_corrected) { - return get_approx_time(); + ErtsMonotonicTime erl_mtime, diff = os_mtime - cip->os_mtime; + ERTS_TIME_ASSERT(diff >= 0); + if (!os_drift_corrected) + diff += (cip->correction.drift*diff)/ERTS_MONOTONIC_TIME_UNIT; + erl_mtime = cip->erl_mtime; + erl_mtime += diff; + erl_mtime += cip->correction.error*(diff/ERTS_TCORR_ERR_UNIT); + if (os_mdiff_p) + *os_mdiff_p = diff; + return erl_mtime; } -#ifdef HAVE_GETHRTIME +static ERTS_INLINE ErtsMonotonicTime +read_corrected_time(int os_drift_corrected) +{ + ErtsMonotonicTime os_mtime; + ErtsMonotonicCorrectionInstance ci; -int erts_disable_tolerant_timeofday; + erts_smp_rwmtx_rlock(&time_sup.inf.c.parmon.rwmtx); -static SysHrTime hr_init_time, hr_last_correction_check, - hr_correction, hr_last_time; + os_mtime = erts_os_monotonic_time(); -static void init_tolerant_timeofday(void) -{ - /* Should be in sys.c */ -#if defined(HAVE_SYSCONF) && defined(_SC_NPROCESSORS_CONF) - if (sysconf(_SC_NPROCESSORS_CONF) > 1) { - char b[1024]; - int maj,min,build; - os_flavor(b,1024); - os_version(&maj,&min,&build); - if (!strcmp(b,"sunos") && maj <= 5 && min <= 7) { - erts_disable_tolerant_timeofday = 1; - } + if (os_mtime >= time_sup.inf.c.parmon.cdata.insts.curr.os_mtime) + ci = time_sup.inf.c.parmon.cdata.insts.curr; + else { + if (os_mtime < time_sup.inf.c.parmon.cdata.insts.prev.os_mtime) + erts_exit(ERTS_ABORT_EXIT, + "OS monotonic time stepped backwards\n"); + ci = time_sup.inf.c.parmon.cdata.insts.prev; } + + erts_smp_rwmtx_runlock(&time_sup.inf.c.parmon.rwmtx); + + return calc_corrected_erl_mtime(os_mtime, &ci, NULL, + os_drift_corrected); +} + +static ErtsMonotonicTime get_os_drift_corrected_time(void) +{ + return read_corrected_time(!0); +} + +static ErtsMonotonicTime get_corrected_time(void) +{ + return read_corrected_time(0); +} + +#ifdef ERTS_TIME_CORRECTION_PRINT + +static ERTS_INLINE void +print_correction(int change, + ErtsMonotonicTime sdiff, + ErtsMonotonicTime old_ecorr, + ErtsMonotonicTime old_dcorr, + ErtsMonotonicTime new_ecorr, + ErtsMonotonicTime new_dcorr, + Uint tmo) +{ + ErtsMonotonicTime usec_sdiff; + if (sdiff < 0) + usec_sdiff = -1*ERTS_MONOTONIC_TO_USEC(-1*sdiff); + else + usec_sdiff = ERTS_MONOTONIC_TO_USEC(sdiff); + + if (!change) + erts_fprintf(stderr, + "sdiff = %b64d usec : [ec=%b64d ppm, dc=%b64d ppb] : " + "tmo = %bpu msec\r\n", + usec_sdiff, + (1000000*old_ecorr) / ERTS_TCORR_ERR_UNIT, + (1000000000*old_dcorr) / ERTS_MONOTONIC_TIME_UNIT, + tmo); + else + erts_fprintf(stderr, + "sdiff = %b64d usec : [ec=%b64d ppm, dc=%b64d ppb] " + "-> [ec=%b64d ppm, dc=%b64d ppb] : tmo = %bpu msec\r\n", + usec_sdiff, + (1000000*old_ecorr) / ERTS_TCORR_ERR_UNIT, + (1000000000*old_dcorr) / ERTS_MONOTONIC_TIME_UNIT, + (1000000*new_ecorr) / ERTS_TCORR_ERR_UNIT, + (1000000000*new_dcorr) / ERTS_MONOTONIC_TIME_UNIT, + tmo); +} + #endif - hr_init_time = sys_gethrtime(); - hr_last_correction_check = hr_last_time = hr_init_time; - hr_correction = 0; + +static ERTS_INLINE ErtsMonotonicTime +get_timeout_pos(ErtsMonotonicTime now, ErtsMonotonicTime tmo) +{ + ErtsMonotonicTime tpos; + tpos = ERTS_MONOTONIC_TO_CLKTCKS(now - 1); + tpos += ERTS_MSEC_TO_CLKTCKS(tmo); + tpos += 1; + return tpos; } -static void get_tolerant_timeofday(SysTimeval *tv) +static void +check_time_correction(void *vesdp) { - SysHrTime diff_time, curr; + int init_drift_adj = !vesdp; + ErtsSchedulerData *esdp = (ErtsSchedulerData *) vesdp; + ErtsMonotonicCorrection new_correction; + ErtsMonotonicCorrectionInstance ci; + ErtsMonotonicTime mdiff, sdiff, os_mtime, erl_mtime, os_stime, + erl_stime, time_offset, timeout_pos; + Uint timeout; + int os_drift_corrected = time_sup.r.o.os_corrected_monotonic_time; + int set_new_correction = 0, begin_short_intervals = 0; + + erts_smp_rwmtx_rlock(&time_sup.inf.c.parmon.rwmtx); + + erts_os_times(&os_mtime, &os_stime); + + ci = time_sup.inf.c.parmon.cdata.insts.curr; + + erts_smp_rwmtx_runlock(&time_sup.inf.c.parmon.rwmtx); - if (erts_disable_tolerant_timeofday) { - sys_gettimeofday(tv); - return; + if (os_mtime < ci.os_mtime) + erts_exit(ERTS_ABORT_EXIT, + "OS monotonic time stepped backwards\n"); + + erl_mtime = calc_corrected_erl_mtime(os_mtime, &ci, &mdiff, + os_drift_corrected); + time_offset = get_time_offset(); + erl_stime = erl_mtime + time_offset; + + sdiff = erl_stime - os_stime; + + if (time_sup.inf.c.shadow_offset) { + ERTS_TIME_ASSERT(time_sup.r.o.warp_mode == ERTS_SINGLE_TIME_WARP_MODE); + if (erts_smp_atomic32_read_nob(&time_sup.inf.c.preliminary_offset)) + sdiff += time_sup.inf.c.shadow_offset; + else + time_sup.inf.c.shadow_offset = 0; } - *tv = inittv; - diff_time = ((curr = sys_gethrtime()) + hr_correction - hr_init_time) / 1000; - if (curr < hr_init_time) { - erl_exit(1,"Unexpected behaviour from operating system high " - "resolution timer"); + new_correction = ci.correction; + + if (time_sup.r.o.warp_mode == ERTS_MULTI_TIME_WARP_MODE + && (sdiff < -2*time_sup.r.o.adj.small_diff + || 2*time_sup.r.o.adj.small_diff < sdiff)) { + /* System time diff exeeded limits; change time offset... */ + time_offset -= sdiff; + sdiff = 0; + set_time_offset(time_offset); + schedule_send_time_offset_changed_notifications(time_offset); + begin_short_intervals = 1; + if (ci.correction.error != 0) { + set_new_correction = 1; + new_correction.error = 0; + } + } + else if ((time_sup.r.o.warp_mode == ERTS_SINGLE_TIME_WARP_MODE + && erts_smp_atomic32_read_nob(&time_sup.inf.c.preliminary_offset)) + && (sdiff < -2*time_sup.r.o.adj.small_diff + || 2*time_sup.r.o.adj.small_diff < sdiff)) { + /* + * System time diff exeeded limits; change shadow offset + * and let OS system time leap away from Erlang system + * time. + */ + time_sup.inf.c.shadow_offset -= sdiff; + sdiff = 0; + begin_short_intervals = 1; + if (ci.correction.error != 0) { + set_new_correction = 1; + new_correction.error = 0; + } + } + else if (ci.correction.error == 0) { + if (sdiff < -time_sup.r.o.adj.small_diff) { + set_new_correction = 1; + if (sdiff < -time_sup.r.o.adj.large_diff) + new_correction.error = ERTS_TCORR_ERR_LARGE_ADJ; + else + new_correction.error = ERTS_TCORR_ERR_SMALL_ADJ; + } + else if (sdiff > time_sup.r.o.adj.small_diff) { + set_new_correction = 1; + if (sdiff > time_sup.r.o.adj.large_diff) + new_correction.error = -ERTS_TCORR_ERR_LARGE_ADJ; + else + new_correction.error = -ERTS_TCORR_ERR_SMALL_ADJ; + } + } + else if (ci.correction.error > 0) { + if (sdiff < 0) { + if (ci.correction.error != ERTS_TCORR_ERR_LARGE_ADJ + && sdiff < -time_sup.r.o.adj.large_diff) { + new_correction.error = ERTS_TCORR_ERR_LARGE_ADJ; + set_new_correction = 1; + } + } + else if (sdiff > time_sup.r.o.adj.small_diff) { + set_new_correction = 1; + if (sdiff > time_sup.r.o.adj.large_diff) + new_correction.error = -ERTS_TCORR_ERR_LARGE_ADJ; + else + new_correction.error = -ERTS_TCORR_ERR_SMALL_ADJ; + } + else { + set_new_correction = 1; + new_correction.error = 0; + } + } + else /* if (ci.correction.error < 0) */ { + if (0 < sdiff) { + if (ci.correction.error != -ERTS_TCORR_ERR_LARGE_ADJ + && time_sup.r.o.adj.large_diff < sdiff) { + new_correction.error = -ERTS_TCORR_ERR_LARGE_ADJ; + set_new_correction = 1; + } + } + else if (sdiff < -time_sup.r.o.adj.small_diff) { + set_new_correction = 1; + if (sdiff < -time_sup.r.o.adj.large_diff) + new_correction.error = ERTS_TCORR_ERR_LARGE_ADJ; + else + new_correction.error = ERTS_TCORR_ERR_SMALL_ADJ; + } + else { + set_new_correction = 1; + new_correction.error = 0; + } } - if ((curr - hr_last_correction_check) / 1000 > 1000000) { - /* Check the correction need */ - SysHrTime tv_diff, diffdiff; - SysTimeval tmp; - int done = 0; - - sys_gettimeofday(&tmp); - tv_diff = ((SysHrTime) tmp.tv_sec) * 1000000 + tmp.tv_usec; - tv_diff -= ((SysHrTime) inittv.tv_sec) * 1000000 + inittv.tv_usec; - diffdiff = diff_time - tv_diff; - if (diffdiff > 10000) { - SysHrTime corr = (curr - hr_last_time) / 100; - if (corr / 1000 >= diffdiff) { - ++done; - hr_correction -= ((SysHrTime)diffdiff) * 1000; - } else { - hr_correction -= corr; + if (!os_drift_corrected) { + ErtsMonotonicDriftData *ddp = &time_sup.inf.c.parmon.cdata.drift; + int ix = ddp->ix; + ErtsMonotonicTime mtime_diff, old_os_mtime; + + old_os_mtime = ddp->intervals[ix].time.mon; + mtime_diff = os_mtime - old_os_mtime; + + if ((mtime_diff >= ERTS_MIN_MONOTONIC_DRIFT_MEASUREMENT) + | init_drift_adj) { + ErtsMonotonicTime drift_adj, drift_adj_diff, old_os_stime, + smtime_diff, stime_diff, mtime_acc, stime_acc, + avg_drift_adj, max_drift; + + old_os_stime = ddp->intervals[ix].time.sys; + + mtime_acc = ddp->acc.mon; + stime_acc = ddp->acc.sys; + + avg_drift_adj = (((stime_acc - mtime_acc) + * ERTS_MONOTONIC_TIME_UNIT) + / mtime_acc); + + mtime_diff = os_mtime - old_os_mtime; + stime_diff = os_stime - old_os_stime; + smtime_diff = stime_diff - mtime_diff; + ix++; + if (ix >= time_sup.r.o.drift_adj.intervals) + ix = 0; + mtime_acc -= ddp->intervals[ix].diff.mon; + mtime_acc += mtime_diff; + stime_acc -= ddp->intervals[ix].diff.sys; + stime_acc += stime_diff; + + ddp->intervals[ix].diff.mon = mtime_diff; + ddp->intervals[ix].diff.sys = stime_diff; + ddp->intervals[ix].time.mon = os_mtime; + ddp->intervals[ix].time.sys = os_stime; + + ddp->ix = ix; + ddp->acc.mon = mtime_acc; + ddp->acc.sys = stime_acc; + + max_drift = ERTS_MAX_MONOTONIC_DRIFT; + max_drift *= ERTS_MONOTONIC_TO_SEC(mtime_diff); + + if (smtime_diff > time_sup.r.o.drift_adj.error + max_drift + || smtime_diff < -1*time_sup.r.o.drift_adj.error - max_drift) { + dirty_intervals: + /* + * We had a leap in system time. Mark array as + * dirty to ensure that dirty values are rotated + * out before we use it again... + */ + ddp->dirty_counter = time_sup.r.o.drift_adj.intervals; + begin_short_intervals = 1; } - diff_time = (curr + hr_correction - hr_init_time) / 1000; - } else if (diffdiff < -10000) { - SysHrTime corr = (curr - hr_last_time) / 100; - if (corr / 1000 >= -diffdiff) { - ++done; - hr_correction -= ((SysHrTime)diffdiff) * 1000; - } else { - hr_correction += corr; + else if (ddp->dirty_counter > 0) { + if (init_drift_adj) { + new_correction.drift = ((smtime_diff + * ERTS_MONOTONIC_TIME_UNIT) + / mtime_diff); + set_new_correction = 1; + } + ddp->dirty_counter--; + } + else { + if (ddp->dirty_counter == 0) { + /* Force set new drift correction... */ + set_new_correction = 1; + ddp->dirty_counter--; + } + + if (time_sup.r.o.drift_adj.use_avg) + drift_adj = (((stime_acc - mtime_acc) + * ERTS_MONOTONIC_TIME_UNIT) + / mtime_acc); + else + drift_adj = ((smtime_diff + * ERTS_MONOTONIC_TIME_UNIT) + / mtime_diff); + + drift_adj_diff = avg_drift_adj - drift_adj; + if (drift_adj_diff < -ERTS_TIME_DRIFT_MAX_ADJ_DIFF + || ERTS_TIME_DRIFT_MAX_ADJ_DIFF < drift_adj_diff) + goto dirty_intervals; + + drift_adj_diff = drift_adj - new_correction.drift; + if (drift_adj_diff) { + if (drift_adj_diff > ERTS_TIME_DRIFT_MAX_ADJ_DIFF) + drift_adj_diff = ERTS_TIME_DRIFT_MAX_ADJ_DIFF; + else if (drift_adj_diff < -ERTS_TIME_DRIFT_MAX_ADJ_DIFF) + drift_adj_diff = -ERTS_TIME_DRIFT_MAX_ADJ_DIFF; + new_correction.drift += drift_adj_diff; + if (drift_adj_diff < -ERTS_TIME_DRIFT_MIN_ADJ_DIFF + || ERTS_TIME_DRIFT_MIN_ADJ_DIFF < drift_adj_diff) { + set_new_correction = 1; + } + } } - diff_time = (curr + hr_correction - hr_init_time) / 1000; - } else { - ++done; } - if (done) { - hr_last_correction_check = curr; + } + + begin_short_intervals |= set_new_correction; + + if (begin_short_intervals) { + time_sup.inf.c.parmon.cdata.short_check_interval + = ERTS_INIT_SHORT_INTERVAL_COUNTER; + } + else if ((os_mtime - time_sup.inf.c.parmon.cdata.last_check + >= ERTS_SHORT_TIME_CORRECTION_CHECK - ERTS_MONOTONIC_TIME_UNIT) + && time_sup.inf.c.parmon.cdata.short_check_interval > 0) { + time_sup.inf.c.parmon.cdata.short_check_interval--; + } + time_sup.inf.c.parmon.cdata.last_check = os_mtime; + + if (new_correction.error == 0) + timeout = ERTS_MONOTONIC_TO_MSEC(ERTS_LONG_TIME_CORRECTION_CHECK); + else { + ErtsMonotonicTime ecorr = new_correction.error; + ErtsMonotonicTime abs_sdiff; + abs_sdiff = (sdiff < 0) ? -1*sdiff : sdiff; + if (ecorr < 0) + ecorr = -1*ecorr; + if (abs_sdiff > ecorr*(ERTS_LONG_TIME_CORRECTION_CHECK/ERTS_TCORR_ERR_UNIT)) + timeout = ERTS_MONOTONIC_TO_MSEC(ERTS_LONG_TIME_CORRECTION_CHECK); + else { + timeout = ERTS_MONOTONIC_TO_MSEC((ERTS_TCORR_ERR_UNIT*abs_sdiff)/ecorr); + if (timeout < 10) + timeout = 10; } } - tv->tv_sec += (int) (diff_time / ((SysHrTime) 1000000)); - tv->tv_usec += (int) (diff_time % ((SysHrTime) 1000000)); - if (tv->tv_usec >= 1000000) { - tv->tv_usec -= 1000000; - tv->tv_sec += 1; + + if (timeout > ERTS_MONOTONIC_TO_MSEC(ERTS_SHORT_TIME_CORRECTION_CHECK) + && (time_sup.inf.c.parmon.cdata.short_check_interval + || time_sup.inf.c.parmon.cdata.drift.dirty_counter >= 0)) { + timeout = ERTS_MONOTONIC_TO_MSEC(ERTS_SHORT_TIME_CORRECTION_CHECK); + } + + timeout_pos = get_timeout_pos(erl_mtime, timeout); + +#ifdef ERTS_TIME_CORRECTION_PRINT + print_correction(set_new_correction, + sdiff, + ci.correction.error, + ci.correction.drift, + new_correction.error, + new_correction.drift, + timeout); +#endif + + if (set_new_correction) { + erts_smp_rwmtx_rwlock(&time_sup.inf.c.parmon.rwmtx); + + os_mtime = erts_os_monotonic_time(); + + /* Save previous correction instance */ + time_sup.inf.c.parmon.cdata.insts.prev = ci; + + /* + * Current correction instance begin when + * OS monotonic time has increased two units. + */ + os_mtime += 2; + + /* + * Erlang monotonic time corresponding to + * next OS monotonic time using previous + * correction. + */ + erl_mtime = calc_corrected_erl_mtime(os_mtime, &ci, NULL, + os_drift_corrected); + + /* + * Save new current correction instance. + */ + time_sup.inf.c.parmon.cdata.insts.curr.erl_mtime = erl_mtime; + time_sup.inf.c.parmon.cdata.insts.curr.os_mtime = os_mtime; + time_sup.inf.c.parmon.cdata.insts.curr.correction = new_correction; + + erts_smp_rwmtx_rwunlock(&time_sup.inf.c.parmon.rwmtx); } - hr_last_time = curr; + + if (!esdp) + esdp = erts_get_scheduler_data(); + + erts_twheel_set_timer(esdp->timer_wheel, + &time_sup.inf.c.parmon.timer, + check_time_correction, + NULL, + (void *) esdp, + timeout_pos); } -#define correction (hr_correction/1000000) +static ErtsMonotonicTime get_os_corrected_time(void) +{ + ASSERT(time_sup.r.o.warp_mode == ERTS_MULTI_TIME_WARP_MODE); + return erts_os_monotonic_time() + time_sup.r.o.moffset; +} -#else /* !HAVE_GETHRTIME */ -#if !defined(CORRECT_USING_TIMES) -#define init_tolerant_timeofday() -#define get_tolerant_timeofday(tvp) sys_gettimeofday(tvp) -#else +static void +check_time_offset(void *vesdp) +{ + ErtsSchedulerData *esdp = (ErtsSchedulerData *) vesdp; + ErtsMonotonicTime sdiff, os_mtime, erl_mtime, os_stime, + erl_stime, time_offset, timeout, timeout_pos; -typedef Sint64 Milli; - -static clock_t init_ct; -static Sint64 ct_wrap; -static Milli init_tv_m; -static Milli correction_supress; -static Milli last_ct_diff; -static Milli last_cc; -static clock_t last_ct; - -/* sys_times() might need to be wrapped and the values shifted (right) - a bit to cope with newer linux (2.5.*) kernels, this has to be taken care - of dynamically to start with, a special version that uses - the times() return value as a high resolution timer can be made - to fully utilize the faster ticks, like on windows, but for now, we'll - settle with this silly workaround */ -#ifdef ERTS_WRAP_SYS_TIMES -#define KERNEL_TICKS() (sys_times_wrap() & \ - ((1UL << ((sizeof(clock_t) * 8) - 1)) - 1)) -#else -SysTimes dummy_tms; + ASSERT(time_sup.r.o.warp_mode == ERTS_MULTI_TIME_WARP_MODE); + + erts_os_times(&os_mtime, &os_stime); + + erl_mtime = os_mtime + time_sup.r.o.moffset; + time_offset = get_time_offset(); + erl_stime = erl_mtime + time_offset; -#define KERNEL_TICKS() (sys_times(&dummy_tms) & \ - ((1UL << ((sizeof(clock_t) * 8) - 1)) - 1)) + sdiff = erl_stime - os_stime; + if ((sdiff < -2*time_sup.r.o.adj.small_diff + || 2*time_sup.r.o.adj.small_diff < sdiff)) { + /* System time diff exeeded limits; change time offset... */ +#ifdef ERTS_TIME_CORRECTION_PRINT + erts_fprintf(stderr, "sdiff = %b64d nsec -> 0 nsec\n", + ERTS_MONOTONIC_TO_NSEC(sdiff)); #endif + time_offset -= sdiff; + sdiff = 0; + set_time_offset(time_offset); + schedule_send_time_offset_changed_notifications(time_offset); + } +#ifdef ERTS_TIME_CORRECTION_PRINT + else erts_fprintf(stderr, "sdiff = %b64d nsec\n", + ERTS_MONOTONIC_TO_NSEC(sdiff)); +#endif + + timeout = ERTS_MONOTONIC_TO_MSEC(ERTS_LONG_TIME_CORRECTION_CHECK); + timeout_pos = get_timeout_pos(erl_mtime, timeout); + + erts_twheel_set_timer(esdp->timer_wheel, + &time_sup.inf.c.parmon.timer, + check_time_offset, + NULL, + vesdp, + timeout_pos); +} + +static void +init_check_time_correction(void *vesdp) +{ + ErtsMonotonicDriftData *ddp; + ErtsMonotonicTime old_mtime, old_stime, mtime, stime, mtime_diff, + stime_diff, smtime_diff, max_drift; + int ix; + + ddp = &time_sup.inf.c.parmon.cdata.drift; + ix = ddp->ix; + old_mtime = ddp->intervals[0].time.mon; + old_stime = ddp->intervals[0].time.sys; + + erts_os_times(&mtime, &stime); + + mtime_diff = mtime - old_mtime; + stime_diff = stime - old_stime; + smtime_diff = stime_diff - mtime_diff; + + max_drift = ERTS_MAX_MONOTONIC_DRIFT; + max_drift *= ERTS_MONOTONIC_TO_SEC(mtime_diff); + + if (smtime_diff > time_sup.r.o.drift_adj.error + max_drift + || smtime_diff < -1*time_sup.r.o.drift_adj.error - max_drift) { + /* Had a system time leap... pretend no drift... */ + stime_diff = mtime_diff; + } + + /* + * We use old time values in order to trigger + * a drift adjustment, and repeat this interval + * in all slots... + */ + for (ix = 0; ix < time_sup.r.o.drift_adj.intervals; ix++) { + ddp->intervals[ix].diff.mon = mtime_diff; + ddp->intervals[ix].diff.sys = stime_diff; + ddp->intervals[ix].time.mon = old_mtime; + ddp->intervals[ix].time.sys = old_stime; + } + + ddp->acc.sys = stime_diff*time_sup.r.o.drift_adj.intervals; + ddp->acc.mon = mtime_diff*time_sup.r.o.drift_adj.intervals; + ddp->ix = 0; + ddp->dirty_counter = time_sup.r.o.drift_adj.intervals; + + check_time_correction(vesdp); +} + +static ErtsMonotonicTime +finalize_corrected_time_offset(ErtsSystemTime *stimep) +{ + ErtsMonotonicTime os_mtime; + ErtsMonotonicCorrectionInstance ci; + int os_drift_corrected = time_sup.r.o.os_corrected_monotonic_time; + + erts_smp_rwmtx_rlock(&time_sup.inf.c.parmon.rwmtx); + + erts_os_times(&os_mtime, stimep); + + ci = time_sup.inf.c.parmon.cdata.insts.curr; -static void init_tolerant_timeofday(void) + erts_smp_rwmtx_runlock(&time_sup.inf.c.parmon.rwmtx); + + if (os_mtime < ci.os_mtime) + erts_exit(ERTS_ABORT_EXIT, + "OS monotonic time stepped backwards\n"); + + return calc_corrected_erl_mtime(os_mtime, &ci, NULL, + os_drift_corrected); +} + +static void +late_init_time_correction(ErtsSchedulerData *esdp) { - last_ct = init_ct = KERNEL_TICKS(); - last_cc = 0; - init_tv_m = (((Milli) inittv.tv_sec) * 1000) + - (inittv.tv_usec / 1000); - ct_wrap = 0; - correction_supress = 0; + int quick_init_drift_adj; + void (*check_func)(void *); + ErtsMonotonicTime timeout, timeout_pos; + + quick_init_drift_adj = + ERTS_MONOTONIC_TO_USEC(time_sup.r.o.drift_adj.error) == 0; + + if (quick_init_drift_adj) + timeout = ERTS_MONOTONIC_TO_MSEC(ERTS_SHORT_TIME_CORRECTION_CHECK/10); + else + timeout = ERTS_MONOTONIC_TO_MSEC(ERTS_SHORT_TIME_CORRECTION_CHECK); + + if (!time_sup.r.o.os_corrected_monotonic_time) + check_func = init_check_time_correction; + else if (time_sup.r.o.get_time == get_os_corrected_time) { + quick_init_drift_adj = 0; + check_func = check_time_offset; + } + else + check_func = check_time_correction; + + timeout_pos = get_timeout_pos(erts_get_monotonic_time(esdp), + timeout); + + erts_twheel_init_timer(&time_sup.inf.c.parmon.timer); + erts_twheel_set_timer(esdp->timer_wheel, + &time_sup.inf.c.parmon.timer, + check_func, + NULL, + (quick_init_drift_adj + ? NULL + : esdp), + timeout_pos); } +#endif /* ERTS_HAVE_OS_MONOTONIC_TIME_SUPPORT */ -static void get_tolerant_timeofday(SysTimeval *tvp) +static ErtsMonotonicTime get_not_corrected_time(void) { - clock_t current_ct; - SysTimeval current_tv; - Milli ct_diff; - Milli tv_diff; - Milli current_correction; - Milli act_correction; /* long shown to be too small */ - Milli max_adjust; + ErtsMonotonicTime stime, mtime; + + erts_smp_mtx_lock(&erts_get_time_mtx); + + stime = erts_os_system_time(); + + mtime = stime - time_sup.inf.c.not_corrected_moffset; + + if (mtime >= time_sup.f.c.last_not_corrected_time) + time_sup.f.c.last_not_corrected_time = mtime; + else { + mtime = time_sup.f.c.last_not_corrected_time; + + if (time_sup.r.o.warp_mode == ERTS_MULTI_TIME_WARP_MODE) { + ErtsMonotonicTime new_offset = stime - mtime; + new_offset = ERTS_MONOTONIC_TO_USEC(new_offset); + new_offset = ERTS_USEC_TO_MONOTONIC(new_offset); + if (time_sup.inf.c.not_corrected_moffset != new_offset) { + time_sup.inf.c.not_corrected_moffset = new_offset; + set_time_offset(new_offset); + schedule_send_time_offset_changed_notifications(new_offset); + } + } - if (erts_disable_tolerant_timeofday) { - sys_gettimeofday(tvp); - return; } -#ifdef ERTS_WRAP_SYS_TIMES -#define TICK_MS (1000 / SYS_CLK_TCK_WRAP) + ASSERT(stime == mtime + time_sup.inf.c.not_corrected_moffset); + + erts_smp_mtx_unlock(&erts_get_time_mtx); + + return mtime; +} + +int erts_check_time_adj_support(int time_correction, + ErtsTimeWarpMode time_warp_mode) +{ + if (!time_correction) + return 1; + + /* User wants time correction */ + +#ifdef ERTS_HAVE_OS_MONOTONIC_TIME_SUPPORT + return !time_sup.r.o.os_monotonic_time_disable; #else -#define TICK_MS (1000 / SYS_CLK_TCK) + return 0; #endif - current_ct = KERNEL_TICKS(); - sys_gettimeofday(¤t_tv); - - /* I dont know if uptime can move some units backwards - on some systems, but I allow for small backward - jumps to avoid such problems if they exist...*/ - if (last_ct > 100 && current_ct < (last_ct - 100)) { - ct_wrap += ((Sint64) 1) << ((sizeof(clock_t) * 8) - 1); - } - last_ct = current_ct; - ct_diff = ((ct_wrap + current_ct) - init_ct) * TICK_MS; +} + +int +erts_has_time_correction(void) +{ + return time_sup.r.o.correction; +} + +void erts_init_sys_time_sup(void) +{ + ErtsSysInitTimeResult sys_init_time_res + = ERTS_SYS_INIT_TIME_RESULT_INITER; + + sys_init_time(&sys_init_time_res); + + erts_time_sup__.r.o.monotonic_time_unit + = sys_init_time_res.os_monotonic_time_unit; + +#ifndef SYS_CLOCK_RESOLUTION + erts_time_sup__.r.o.clktck_resolution + = sys_init_time_res.sys_clock_resolution; + erts_time_sup__.r.o.clktck_resolution *= 1000; +#endif + +#ifdef ERTS_HAVE_OS_MONOTONIC_TIME_SUPPORT + time_sup.r.o.os_monotonic_time_disable + = !sys_init_time_res.have_os_monotonic_time; + time_sup.r.o.os_corrected_monotonic_time = + sys_init_time_res.have_corrected_os_monotonic_time; + time_sup.r.o.os_monotonic_time_func + = sys_init_time_res.os_monotonic_time_info.func; + time_sup.r.o.os_monotonic_time_clock_id + = sys_init_time_res.os_monotonic_time_info.clock_id; + time_sup.r.o.os_monotonic_time_locked + = sys_init_time_res.os_monotonic_time_info.locked_use; + time_sup.r.o.os_monotonic_time_resolution + = sys_init_time_res.os_monotonic_time_info.resolution; + time_sup.r.o.os_monotonic_time_extended + = sys_init_time_res.os_monotonic_time_info.extended; +#endif + time_sup.r.o.os_system_time_func + = sys_init_time_res.os_system_time_info.func; + time_sup.r.o.os_system_time_clock_id + = sys_init_time_res.os_system_time_info.clock_id; + time_sup.r.o.os_system_time_locked + = sys_init_time_res.os_system_time_info.locked_use; + time_sup.r.o.os_system_time_resolution + = sys_init_time_res.os_system_time_info.resolution; +} + +int +erts_init_time_sup(int time_correction, ErtsTimeWarpMode time_warp_mode) +{ + ErtsMonotonicTime resolution, ilength, intervals, short_isecs; +#if !ERTS_COMPILE_TIME_MONOTONIC_TIME_UNIT + ErtsMonotonicTime abs_native_offset, native_offset; +#endif + + init_time_napi(); + + erts_hl_timer_init(); + + ASSERT(ERTS_MONOTONIC_TIME_MIN < ERTS_MONOTONIC_TIME_MAX); + + erts_smp_mtx_init(&erts_timeofday_mtx, "timeofday"); + erts_smp_mtx_init(&erts_get_time_mtx, "get_time"); + + time_sup.r.o.correction = time_correction; + time_sup.r.o.warp_mode = time_warp_mode; + + if (time_warp_mode == ERTS_SINGLE_TIME_WARP_MODE) + erts_smp_atomic32_init_nob(&time_sup.inf.c.preliminary_offset, 1); + else + erts_smp_atomic32_init_nob(&time_sup.inf.c.preliminary_offset, 0); + time_sup.inf.c.shadow_offset = 0; + +#if !ERTS_COMPILE_TIME_MONOTONIC_TIME_UNIT /* - * We will adjust the time in milliseconds and we allow for 1% - * adjustments, but if this function is called more often then every 100 - * millisecond (which is obviously possible), we will never adjust, so - * we accumulate small times by setting last_ct_diff iff max_adjust > 0 + * NOTE! erts_time_sup__.r.o.start *need* to be a multiple + * of ERTS_MONOTONIC_TIME_UNIT. */ - if ((max_adjust = (ct_diff - last_ct_diff)/100) > 0) - last_ct_diff = ct_diff; - tv_diff = ((((Milli) current_tv.tv_sec) * 1000) + - (current_tv.tv_usec / 1000)) - init_tv_m; +#ifdef ARCH_32 + erts_time_sup__.r.o.start = ((((ErtsMonotonicTime) 1) << 32)-1); + erts_time_sup__.r.o.start /= ERTS_MONOTONIC_TIME_UNIT; + erts_time_sup__.r.o.start *= ERTS_MONOTONIC_TIME_UNIT; + erts_time_sup__.r.o.start += ERTS_MONOTONIC_TIME_UNIT; + native_offset = erts_time_sup__.r.o.start - ERTS_MONOTONIC_BEGIN; + abs_native_offset = native_offset; +#else /* ARCH_64 */ + if (ERTS_MONOTONIC_TIME_UNIT <= 10*1000*1000) { + erts_time_sup__.r.o.start = 0; + native_offset = -ERTS_MONOTONIC_BEGIN; + abs_native_offset = ERTS_MONOTONIC_BEGIN; + } + else { + erts_time_sup__.r.o.start = ((ErtsMonotonicTime) MIN_SMALL); + erts_time_sup__.r.o.start /= ERTS_MONOTONIC_TIME_UNIT; + erts_time_sup__.r.o.start *= ERTS_MONOTONIC_TIME_UNIT; + native_offset = erts_time_sup__.r.o.start - ERTS_MONOTONIC_BEGIN; + abs_native_offset = -1*native_offset; + } +#endif - current_correction = ((ct_diff - tv_diff) / TICK_MS) * TICK_MS; /* trunc */ + erts_time_sup__.r.o.start_offset.native = native_offset; + erts_time_sup__.r.o.start_offset.nsec = (ErtsMonotonicTime) + erts_time_unit_conversion((Uint64) abs_native_offset, + (Uint32) ERTS_MONOTONIC_TIME_UNIT, + (Uint32) 1000*1000*1000); + erts_time_sup__.r.o.start_offset.usec = (ErtsMonotonicTime) + erts_time_unit_conversion((Uint64) abs_native_offset, + (Uint32) ERTS_MONOTONIC_TIME_UNIT, + (Uint32) 1000*1000); + erts_time_sup__.r.o.start_offset.msec = (ErtsMonotonicTime) + erts_time_unit_conversion((Uint64) abs_native_offset, + (Uint32) ERTS_MONOTONIC_TIME_UNIT, + (Uint32) 1000); + erts_time_sup__.r.o.start_offset.sec = (ErtsMonotonicTime) + erts_time_unit_conversion((Uint64) abs_native_offset, + (Uint32) ERTS_MONOTONIC_TIME_UNIT, + (Uint32) 1); + if (native_offset < 0) { + erts_time_sup__.r.o.start_offset.nsec *= -1; + erts_time_sup__.r.o.start_offset.usec *= -1; + erts_time_sup__.r.o.start_offset.msec *= -1; + erts_time_sup__.r.o.start_offset.sec *= -1; + } - /* - * We allow the current_correction value to wobble a little, as it - * suffers from the low resolution of the kernel ticks. - * if it hasn't changed more than one tick in either direction, - * we will keep the old value. - */ - if ((last_cc > current_correction + TICK_MS) || - (last_cc < current_correction - TICK_MS)) { - last_cc = current_correction; - } else { - current_correction = last_cc; +#endif + + resolution = time_sup.r.o.os_system_time_resolution; +#ifdef ERTS_HAVE_OS_MONOTONIC_TIME_SUPPORT + if (resolution > time_sup.r.o.os_monotonic_time_resolution) + resolution = time_sup.r.o.os_monotonic_time_resolution; +#endif + + time_sup.r.o.adj.large_diff = erts_time_sup__.r.o.monotonic_time_unit; + time_sup.r.o.adj.large_diff *= 50; + time_sup.r.o.adj.large_diff /= resolution; + if (time_sup.r.o.adj.large_diff < ERTS_USEC_TO_MONOTONIC(500)) + time_sup.r.o.adj.large_diff = ERTS_USEC_TO_MONOTONIC(500); + time_sup.r.o.adj.small_diff = time_sup.r.o.adj.large_diff/10; + + time_sup.r.o.drift_adj.resolution = resolution; + + if (time_sup.r.o.os_corrected_monotonic_time) { + time_sup.r.o.drift_adj.use_avg = 0; + time_sup.r.o.drift_adj.intervals = 0; + time_sup.r.o.drift_adj.error = 0; + time_sup.inf.c.parmon.cdata.drift.dirty_counter = -1; } - - /* - * As time goes, we try to get the actual correction to 0, - * that is, make erlangs time correspond to the systems dito. - * The act correction is what we seem to need (current_correction) - * minus the correction suppression. The correction supression - * will change slowly (max 1% of elapsed time) but in millisecond steps. - */ - act_correction = current_correction - correction_supress; - if (max_adjust > 0) { + else { /* - * Here we slowly adjust erlangs time to correspond with the - * system time by changing the correction_supress variable. - * It can change max_adjust milliseconds which is 1% of elapsed time + * Calculate length of the interval in seconds needed + * in order to get an error that is at most 1 micro second. + * If this interval is longer than the short time correction + * check interval we use the average of all values instead + * of the latest value. */ - if (act_correction > 0) { - if (current_correction - correction_supress > max_adjust) { - correction_supress += max_adjust; - } else { - correction_supress = current_correction; - } - act_correction = current_correction - correction_supress; - } else if (act_correction < 0) { - if (correction_supress - current_correction > max_adjust) { - correction_supress -= max_adjust; - } else { - correction_supress = current_correction; - } - act_correction = current_correction - correction_supress; + short_isecs = ERTS_MONOTONIC_TO_SEC(ERTS_SHORT_TIME_CORRECTION_CHECK); + ilength = ERTS_ASSUMED_PRECISION_DROP * ERTS_MONOTONIC_TIME_UNIT; + ilength /= (resolution * ERTS_USEC_TO_MONOTONIC(1)); + time_sup.r.o.drift_adj.use_avg = ilength > short_isecs; + + if (ilength == 0) + intervals = 5; + else { + intervals = ilength / short_isecs; + if (intervals > ERTS_MAX_DRIFT_INTERVALS) + intervals = ERTS_MAX_DRIFT_INTERVALS; + else if (intervals < 5) + intervals = 5; } + time_sup.r.o.drift_adj.intervals = (int) intervals; + + /* + * drift_adj.error equals maximum assumed error + * over a short time interval. We use this value also + * when examining a large interval. In this case the + * error will be smaller, but we do not want to + * recalculate this over and over again. + */ + + time_sup.r.o.drift_adj.error = ERTS_MONOTONIC_TIME_UNIT; + time_sup.r.o.drift_adj.error *= ERTS_ASSUMED_PRECISION_DROP; + time_sup.r.o.drift_adj.error /= resolution * short_isecs; } - /* - * The actual correction will correct the timeval so that system - * time warps gets smothed down. - */ - current_tv.tv_sec += act_correction / 1000; - current_tv.tv_usec += (act_correction % 1000) * 1000; - - if (current_tv.tv_usec >= 1000000) { - ++current_tv.tv_sec ; - current_tv.tv_usec -= 1000000; - } else if (current_tv.tv_usec < 0) { - --current_tv.tv_sec; - current_tv.tv_usec += 1000000; +#ifdef ERTS_TIME_CORRECTION_PRINT + erts_fprintf(stderr, "resolution = %b64d\n", resolution); + erts_fprintf(stderr, "adj large diff = %b64d usec\n", + ERTS_MONOTONIC_TO_USEC(time_sup.r.o.adj.large_diff)); + erts_fprintf(stderr, "adj small diff = %b64d usec\n", + ERTS_MONOTONIC_TO_USEC(time_sup.r.o.adj.small_diff)); + if (!time_sup.r.o.os_corrected_monotonic_time) { + erts_fprintf(stderr, "drift intervals = %d\n", + time_sup.r.o.drift_adj.intervals); + erts_fprintf(stderr, "drift adj error = %b64d usec\n", + ERTS_MONOTONIC_TO_USEC(time_sup.r.o.drift_adj.error)); + erts_fprintf(stderr, "drift adj max diff = %b64d nsec\n", + ERTS_MONOTONIC_TO_NSEC(ERTS_TIME_DRIFT_MAX_ADJ_DIFF)); + erts_fprintf(stderr, "drift adj min diff = %b64d nsec\n", + ERTS_MONOTONIC_TO_NSEC(ERTS_TIME_DRIFT_MIN_ADJ_DIFF)); } - *tvp = current_tv; -#undef TICK_MS -} +#endif -#endif /* CORRECT_USING_TIMES */ -#endif /* !HAVE_GETHRTIME */ + if (ERTS_MONOTONIC_TIME_UNIT < ERTS_CLKTCK_RESOLUTION) + ERTS_INTERNAL_ERROR("Too small monotonic time time unit"); -/* -** Why this? Well, most platforms have a constant clock resolution of 1, -** we dont want the deliver_time/time_remaining routines to waste -** time dividing and multiplying by/with a variable that's always one. -** so the return value of sys_init_time is ignored on those platforms. -*/ - -#ifndef SYS_CLOCK_RESOLUTION -static int clock_resolution; -#define CLOCK_RESOLUTION clock_resolution +#ifndef ERTS_HAVE_OS_MONOTONIC_TIME_SUPPORT + time_sup.r.o.correction = 0; #else -#define CLOCK_RESOLUTION SYS_CLOCK_RESOLUTION + if (time_sup.r.o.os_monotonic_time_disable) + time_sup.r.o.correction = 0; + + if (time_sup.r.o.correction) { + ErtsMonotonicCorrectionData *cdatap; + erts_smp_rwmtx_opt_t rwmtx_opts = ERTS_SMP_RWMTX_OPT_DEFAULT_INITER; + ErtsMonotonicTime offset; + erts_os_times(&time_sup.inf.c.minit, + &time_sup.inf.c.sinit); + time_sup.r.o.moffset = -1*time_sup.inf.c.minit; + time_sup.r.o.moffset += ERTS_MONOTONIC_BEGIN; + offset = time_sup.inf.c.sinit; + offset -= ERTS_MONOTONIC_BEGIN; + init_time_offset(offset); + + rwmtx_opts.type = ERTS_SMP_RWMTX_TYPE_EXTREMELY_FREQUENT_READ; + rwmtx_opts.lived = ERTS_SMP_RWMTX_LONG_LIVED; + + erts_smp_rwmtx_init_opt(&time_sup.inf.c.parmon.rwmtx, + &rwmtx_opts, "get_corrected_time"); + + cdatap = &time_sup.inf.c.parmon.cdata; + + cdatap->drift.intervals[0].time.sys = time_sup.inf.c.sinit; + cdatap->drift.intervals[0].time.mon = time_sup.inf.c.minit; + cdatap->insts.curr.correction.drift = 0; + cdatap->insts.curr.correction.error = 0; + cdatap->insts.curr.erl_mtime = ERTS_MONOTONIC_BEGIN; + cdatap->insts.curr.os_mtime = time_sup.inf.c.minit; + cdatap->last_check = time_sup.inf.c.minit; + cdatap->short_check_interval = ERTS_INIT_SHORT_INTERVAL_COUNTER; + cdatap->insts.prev = cdatap->insts.curr; + + if (!time_sup.r.o.os_corrected_monotonic_time) + time_sup.r.o.get_time = get_corrected_time; + else if (time_sup.r.o.warp_mode == ERTS_MULTI_TIME_WARP_MODE) + time_sup.r.o.get_time = get_os_corrected_time; + else + time_sup.r.o.get_time = get_os_drift_corrected_time; + } + else #endif + { + ErtsMonotonicTime stime, offset; + time_sup.r.o.get_time = get_not_corrected_time; + stime = time_sup.inf.c.sinit = erts_os_system_time(); + offset = stime - ERTS_MONOTONIC_BEGIN; + time_sup.inf.c.not_corrected_moffset = offset; + init_time_offset(offset); + time_sup.f.c.last_not_corrected_time = 0; + } -/* -** The clock resolution should really be the resolution of the -** time function in use, which on most platforms -** is 1. On VxWorks the resolution should be -** the number of ticks per second (or 1, which would work nicely to). -** -** Setting lower resolutions is mostly interesting when timers are used -** instead of something like select. -*/ + prev_wall_clock_elapsed = 0; + + previous_now = ERTS_MONOTONIC_TO_USEC(get_time_offset()); -static SysTimeval last_delivered; +#ifdef DEBUG + time_sup_initialized = 1; +#endif -static void init_erts_deliver_time(const SysTimeval *inittv) + return ERTS_CLKTCK_RESOLUTION/1000; +} + +void +erts_late_init_time_sup(void) { - /* We set the initial values for deliver_time here */ - last_delivered = *inittv; - last_delivered.tv_usec = 1000 * (last_delivered.tv_usec / 1000); - /* ms resolution */ + erts_late_sys_init_time(); } -static void do_erts_deliver_time(const SysTimeval *current) +void +erts_sched_init_time_sup(ErtsSchedulerData *esdp) { - SysTimeval cur_time; - erts_time_t elapsed; - - /* calculate and deliver appropriate number of ticks */ - cur_time = *current; - cur_time.tv_usec = 1000 * (cur_time.tv_usec / 1000); /* ms resolution */ - elapsed = (1000 * (cur_time.tv_sec - last_delivered.tv_sec) + - (cur_time.tv_usec - last_delivered.tv_usec) / 1000) / - CLOCK_RESOLUTION; - - /* Sometimes the time jump backwards, - resulting in a negative elapsed time. We compensate for - this by simply pretend as if the time stood still. :) */ - - if (elapsed > 0) { + esdp->timer_wheel = erts_create_timer_wheel(esdp); + esdp->next_tmo_ref = erts_get_next_timeout_reference(esdp->timer_wheel); + esdp->timer_service = erts_create_timer_service(); +#ifdef ERTS_HAVE_OS_MONOTONIC_TIME_SUPPORT + if (esdp->no == 1) { + /* A timer wheel to use must have beeen initialized */ + if (time_sup.r.o.get_time != get_not_corrected_time) + late_init_time_correction(esdp); + } +#endif +} - ASSERT(elapsed < ((erts_time_t) ERTS_SHORT_TIME_T_MAX)); +ErtsTimeWarpMode erts_time_warp_mode(void) +{ + return time_sup.r.o.warp_mode; +} - erts_do_time_add((erts_short_time_t) elapsed); - last_delivered = cur_time; +ErtsTimeOffsetState erts_time_offset_state(void) +{ + switch (time_sup.r.o.warp_mode) { + case ERTS_NO_TIME_WARP_MODE: + return ERTS_TIME_OFFSET_FINAL; + case ERTS_SINGLE_TIME_WARP_MODE: + if (erts_smp_atomic32_read_nob(&time_sup.inf.c.preliminary_offset)) + return ERTS_TIME_OFFSET_PRELIMINARY; + return ERTS_TIME_OFFSET_FINAL; + case ERTS_MULTI_TIME_WARP_MODE: + return ERTS_TIME_OFFSET_VOLATILE; + default: + ERTS_INTERNAL_ERROR("Invalid time warp mode"); + return ERTS_TIME_OFFSET_VOLATILE; } } -int -erts_init_time_sup(void) +/* + * erts_finalize_time_offset() will only change time offset + * the first time it is called when the emulator has been + * started in "single time warp" mode. Returns previous + * state: + * * ERTS_TIME_OFFSET_PRELIMINARY - Finalization performed + * * ERTS_TIME_OFFSET_FINAL - Already finialized; nothing changed + * * ERTS_TIME_OFFSET_VOLATILE - Not supported, either in + * * no correction mode (or multi time warp mode; not yet implemented). + */ + +ErtsTimeOffsetState +erts_finalize_time_offset(void) { - erts_smp_mtx_init(&erts_timeofday_mtx, "timeofday"); + switch (time_sup.r.o.warp_mode) { + case ERTS_NO_TIME_WARP_MODE: + return ERTS_TIME_OFFSET_FINAL; + case ERTS_MULTI_TIME_WARP_MODE: + return ERTS_TIME_OFFSET_VOLATILE; + case ERTS_SINGLE_TIME_WARP_MODE: { + ErtsTimeOffsetState res = ERTS_TIME_OFFSET_FINAL; - init_approx_time(); + erts_smp_mtx_lock(&erts_get_time_mtx); - last_emu_time.tv_sec = 0; - last_emu_time.tv_usec = 0; + if (erts_smp_atomic32_read_nob(&time_sup.inf.c.preliminary_offset)) { + ErtsMonotonicTime mtime, new_offset; -#ifndef SYS_CLOCK_RESOLUTION - clock_resolution = sys_init_time(); -#else - (void) sys_init_time(); +#ifdef ERTS_HAVE_OS_MONOTONIC_TIME_SUPPORT + if (!time_sup.r.o.correction) #endif - sys_gettimeofday(&inittv); - -#ifdef HAVE_GETHRTIME - sys_init_hrtime(); + { + ErtsMonotonicTime stime = erts_os_system_time(); + + mtime = stime - time_sup.inf.c.not_corrected_moffset; + + if (mtime >= time_sup.f.c.last_not_corrected_time) { + time_sup.f.c.last_not_corrected_time = mtime; + new_offset = time_sup.inf.c.not_corrected_moffset; + } + else { + mtime = time_sup.f.c.last_not_corrected_time; + + ASSERT(time_sup.inf.c.not_corrected_moffset != stime - mtime); + new_offset = stime - mtime; + time_sup.inf.c.not_corrected_moffset = new_offset; + } + + } +#ifdef ERTS_HAVE_OS_MONOTONIC_TIME_SUPPORT + else { + ErtsSystemTime stime; + mtime = finalize_corrected_time_offset(&stime); + new_offset = stime - mtime; + } #endif - init_tolerant_timeofday(); + new_offset = ERTS_MONOTONIC_TO_USEC(new_offset); + new_offset = ERTS_USEC_TO_MONOTONIC(new_offset); - init_erts_deliver_time(&inittv); - gtv = inittv; - then.tv_sec = then.tv_usec = 0; + set_time_offset(new_offset); + schedule_send_time_offset_changed_notifications(new_offset); - erts_deliver_time(); + erts_smp_atomic32_set_nob(&time_sup.inf.c.preliminary_offset, 0); + res = ERTS_TIME_OFFSET_PRELIMINARY; + } + + erts_smp_mtx_unlock(&erts_get_time_mtx); + + return res; + } + default: + ERTS_INTERNAL_ERROR("Invalid time warp mode"); + return ERTS_TIME_OFFSET_VOLATILE; + } +} - return CLOCK_RESOLUTION; -} /* info functions */ void @@ -498,23 +1326,17 @@ elapsed_time_both(UWord *ms_user, UWord *ms_sys, void wall_clock_elapsed_time_both(UWord *ms_total, UWord *ms_diff) { - UWord prev_total; - SysTimeval tv; + ErtsMonotonicTime now, elapsed; erts_smp_mtx_lock(&erts_timeofday_mtx); - get_tolerant_timeofday(&tv); - - *ms_total = 1000 * (tv.tv_sec - inittv.tv_sec) + - (tv.tv_usec - inittv.tv_usec) / 1000; + now = time_sup.r.o.get_time(); + update_last_mtime(NULL, now); - prev_total = 1000 * (gtv.tv_sec - inittv.tv_sec) + - (gtv.tv_usec - inittv.tv_usec) / 1000; - *ms_diff = *ms_total - prev_total; - gtv = tv; - - /* must sync the machine's idea of time here */ - do_erts_deliver_time(&tv); + elapsed = ERTS_MONOTONIC_TO_MSEC(now); + *ms_total = (UWord) elapsed; + *ms_diff = (UWord) (elapsed - prev_wall_clock_elapsed); + prev_wall_clock_elapsed = elapsed; erts_smp_mtx_unlock(&erts_timeofday_mtx); } @@ -890,146 +1712,731 @@ univ_to_local(Sint *year, Sint *month, Sint *day, return 0; } - /* get a timestamp */ void get_now(Uint* megasec, Uint* sec, Uint* microsec) { - SysTimeval now; + ErtsMonotonicTime now_megasec, now_sec, now, mtime, time_offset; + mtime = time_sup.r.o.get_time(); + time_offset = get_time_offset(); + update_last_mtime(NULL, mtime); + now = ERTS_MONOTONIC_TO_USEC(mtime + time_offset); + erts_smp_mtx_lock(&erts_timeofday_mtx); - - get_tolerant_timeofday(&now); - do_erts_deliver_time(&now); - - /* Make sure time is later than last */ - if (then.tv_sec > now.tv_sec || - (then.tv_sec == now.tv_sec && then.tv_usec >= now.tv_usec)) { - now = then; - now.tv_usec++; - } - /* Check for carry from above + general reasonability */ - if (now.tv_usec >= 1000000) { - now.tv_usec = 0; - now.tv_sec++; - } - then = now; + + /* Make sure now time is later than last time */ + if (now <= previous_now) + now = previous_now + 1; + + previous_now = now; erts_smp_mtx_unlock(&erts_timeofday_mtx); - - *megasec = (Uint) (now.tv_sec / 1000000); - *sec = (Uint) (now.tv_sec % 1000000); - *microsec = (Uint) (now.tv_usec); - update_approx_time(&now); + now_megasec = now / ERTS_MONOTONIC_TIME_TERA; + now_sec = now / ERTS_MONOTONIC_TIME_MEGA; + *megasec = (Uint) now_megasec; + *sec = (Uint) (now_sec - now_megasec*ERTS_MONOTONIC_TIME_MEGA); + *microsec = (Uint) (now - now_sec*ERTS_MONOTONIC_TIME_MEGA); + + ASSERT(((ErtsMonotonicTime) *megasec)*ERTS_MONOTONIC_TIME_TERA + + ((ErtsMonotonicTime) *sec)*ERTS_MONOTONIC_TIME_MEGA + + ((ErtsMonotonicTime) *microsec) == now); +} + +ErtsMonotonicTime +erts_get_monotonic_time(ErtsSchedulerData *esdp) +{ + ErtsMonotonicTime mtime = time_sup.r.o.get_time(); + update_last_mtime(esdp, mtime); + return mtime; +} + +ErtsMonotonicTime +erts_get_time_offset(void) +{ + return get_time_offset(); +} + +static ERTS_INLINE void +make_timestamp_value(Uint* megasec, Uint* sec, Uint* microsec, + ErtsMonotonicTime mtime, ErtsMonotonicTime offset) +{ + ErtsMonotonicTime stime, as; + Uint ms; + + stime = ERTS_MONOTONIC_TO_USEC(mtime + offset); + + as = stime / ERTS_MONOTONIC_TIME_MEGA; + *megasec = ms = (Uint) (stime / ERTS_MONOTONIC_TIME_TERA); + *sec = (Uint) (as - (((ErtsMonotonicTime) ms) + * ERTS_MONOTONIC_TIME_MEGA)); + *microsec = (Uint) (stime - as*ERTS_MONOTONIC_TIME_MEGA); + + ASSERT(((ErtsMonotonicTime) ms)*ERTS_MONOTONIC_TIME_TERA + + ((ErtsMonotonicTime) *sec)*ERTS_MONOTONIC_TIME_MEGA + + *microsec == stime); +} + +void +erts_make_timestamp_value(Uint* megasec, Uint* sec, Uint* microsec, + ErtsMonotonicTime mtime, ErtsMonotonicTime offset) +{ + make_timestamp_value(megasec, sec, microsec, mtime, offset); } void get_sys_now(Uint* megasec, Uint* sec, Uint* microsec) { - SysTimeval now; - - sys_gettimeofday(&now); - - *megasec = (Uint) (now.tv_sec / 1000000); - *sec = (Uint) (now.tv_sec % 1000000); - *microsec = (Uint) (now.tv_usec); + ErtsSystemTime stime = erts_os_system_time(); + ErtsSystemTime ms, s, us; + + us = ERTS_MONOTONIC_TO_USEC(stime); + s = us / (1000*1000); + ms = s / (1000*1000); - update_approx_time(&now); + *megasec = (Uint) ms; + *sec = (Uint) (s - ms*(1000*1000)); + *microsec = (Uint) (us - s*(1000*1000)); } +#ifdef HAVE_ERTS_NOW_CPU +void erts_get_now_cpu(Uint* megasec, Uint* sec, Uint* microsec) { + SysCpuTime t; + SysTimespec tp; -/* deliver elapsed *ticks* to the machine - takes a pointer - to a struct timeval representing current time (to save - a gettimeofday() where possible) or NULL */ + sys_get_proc_cputime(t, tp); + *microsec = (Uint)(tp.tv_nsec / 1000); + t = (tp.tv_sec / 1000000); + *megasec = (Uint)(t % 1000000); + *sec = (Uint)(tp.tv_sec % 1000000); +} +#endif -void erts_deliver_time(void) { - SysTimeval now; - - erts_smp_mtx_lock(&erts_timeofday_mtx); - - get_tolerant_timeofday(&now); - do_erts_deliver_time(&now); - - erts_smp_mtx_unlock(&erts_timeofday_mtx); +#include "big.h" + +void +erts_monitor_time_offset(Eterm id, Eterm ref) +{ + erts_smp_mtx_lock(&erts_get_time_mtx); + erts_add_monitor(&time_offset_monitors, MON_TIME_OFFSET, ref, id, NIL); + no_time_offset_monitors++; + erts_smp_mtx_unlock(&erts_get_time_mtx); +} - update_approx_time(&now); +int +erts_demonitor_time_offset(Eterm ref) +{ + int res; + ErtsMonitor *mon; + ASSERT(is_internal_ref(ref)); + erts_smp_mtx_lock(&erts_get_time_mtx); + mon = erts_remove_monitor(&time_offset_monitors, ref); + if (!mon) + res = 0; + else { + ASSERT(no_time_offset_monitors > 0); + no_time_offset_monitors--; + res = 1; + } + erts_smp_mtx_unlock(&erts_get_time_mtx); + if (res) + erts_destroy_monitor(mon); + return res; } -/* get *real* time (not ticks) remaining until next timeout - if there - isn't one, give a "long" time, that is guaranteed - to not cause overflow when we report elapsed time later on */ +typedef struct { + Eterm pid; + Eterm ref; + Eterm heap[REF_THING_SIZE]; +} ErtsTimeOffsetMonitorInfo; -void erts_time_remaining(SysTimeval *rem_time) +typedef struct { + Uint ix; + ErtsTimeOffsetMonitorInfo *to_mon_info; +} ErtsTimeOffsetMonitorContext; + +static void +save_time_offset_monitor(ErtsMonitor *mon, void *vcntxt) { - erts_time_t ticks; - SysTimeval cur_time; - erts_time_t elapsed; + ErtsTimeOffsetMonitorContext *cntxt; + Eterm *from_hp, *to_hp; + Uint mix; + int hix; - /* erts_next_time() returns no of ticks to next timeout or -1 if none */ + cntxt = (ErtsTimeOffsetMonitorContext *) vcntxt; + mix = (cntxt->ix)++; + cntxt->to_mon_info[mix].pid = mon->pid; + to_hp = &cntxt->to_mon_info[mix].heap[0]; - ticks = (erts_time_t) erts_next_time(); - if (ticks == (erts_time_t) -1) { - /* timer queue empty */ - /* this will cause at most 100000000 ticks */ - rem_time->tv_sec = 100000; - rem_time->tv_usec = 0; - } else { - /* next timeout after ticks ticks */ - ticks *= CLOCK_RESOLUTION; - - erts_smp_mtx_lock(&erts_timeofday_mtx); - - get_tolerant_timeofday(&cur_time); - cur_time.tv_usec = 1000 * - (cur_time.tv_usec / 1000);/* ms resolution*/ - elapsed = 1000 * (cur_time.tv_sec - last_delivered.tv_sec) + - (cur_time.tv_usec - last_delivered.tv_usec) / 1000; - - erts_smp_mtx_unlock(&erts_timeofday_mtx); + ASSERT(is_internal_ref(mon->ref)); + from_hp = internal_ref_val(mon->ref); + ASSERT(thing_arityval(*from_hp) + 1 == REF_THING_SIZE); + + for (hix = 0; hix < REF_THING_SIZE; hix++) + to_hp[hix] = from_hp[hix]; + + cntxt->to_mon_info[mix].ref + = make_internal_ref(&cntxt->to_mon_info[mix].heap[0]); + +} + +static void +send_time_offset_changed_notifications(void *new_offsetp) +{ + ErtsMonotonicTime new_offset; + ErtsTimeOffsetMonitorInfo *to_mon_info = NULL; /* Shut up faulty warning */ + Uint no_monitors; + char *tmp = NULL; + +#ifdef ARCH_64 + new_offset = (ErtsMonotonicTime) new_offsetp; +#else + new_offset = *((ErtsMonotonicTime *) new_offsetp); + erts_free(ERTS_ALC_T_NEW_TIME_OFFSET, new_offsetp); +#endif + new_offset -= ERTS_MONOTONIC_OFFSET_NATIVE; + + erts_smp_mtx_lock(&erts_get_time_mtx); + + no_monitors = no_time_offset_monitors; + if (no_monitors) { + ErtsTimeOffsetMonitorContext cntxt; + Uint alloc_sz; - if (ticks <= elapsed) { /* Ooops, better hurry */ - rem_time->tv_sec = rem_time->tv_usec = 0; - return; + /* Monitor info array size */ + alloc_sz = no_monitors*sizeof(ErtsTimeOffsetMonitorInfo); + /* + template max size */ + alloc_sz += 6*sizeof(Eterm); /* 5-tuple */ + alloc_sz += ERTS_MAX_SINT64_HEAP_SIZE*sizeof(Eterm); /* max offset size */ + tmp = erts_alloc(ERTS_ALC_T_TMP, alloc_sz); + + to_mon_info = (ErtsTimeOffsetMonitorInfo *) tmp; + cntxt.ix = 0; + cntxt.to_mon_info = to_mon_info; + + erts_doforall_monitors(time_offset_monitors, + save_time_offset_monitor, + &cntxt); + + ASSERT(cntxt.ix == no_monitors); + } + + erts_smp_mtx_unlock(&erts_get_time_mtx); + + if (no_monitors) { + Eterm *hp, *patch_refp, new_offset_term, message_template; + Uint mix, hsz; + + /* Make message template */ + + hp = (Eterm *) (tmp + no_monitors*sizeof(ErtsTimeOffsetMonitorInfo)); + + hsz = 6; /* 5-tuple */ + hsz += REF_THING_SIZE; + hsz += ERTS_SINT64_HEAP_SIZE(new_offset); + + if (IS_SSMALL(new_offset)) + new_offset_term = make_small(new_offset); + else + new_offset_term = erts_sint64_to_big(new_offset, &hp); + message_template = TUPLE5(hp, + am_CHANGE, + THE_NON_VALUE, /* Patch point for ref */ + am_time_offset, + am_clock_service, + new_offset_term); + patch_refp = &hp[2]; + + ASSERT(*patch_refp == THE_NON_VALUE); + + for (mix = 0; mix < no_monitors; mix++) { + Process *rp = erts_proc_lookup(to_mon_info[mix].pid); + if (rp) { + Eterm ref = to_mon_info[mix].ref; + ErtsProcLocks rp_locks = ERTS_PROC_LOCK_LINK; + erts_smp_proc_lock(rp, ERTS_PROC_LOCK_LINK); + if (erts_lookup_monitor(ERTS_P_MONITORS(rp), ref)) { + ErlHeapFragment *bp; + ErlOffHeap *ohp; + Eterm message; + + hp = erts_alloc_message_heap(hsz, &bp, &ohp, rp, &rp_locks); + *patch_refp = ref; + ASSERT(hsz == size_object(message_template)); + message = copy_struct(message_template, hsz, &hp, ohp); + erts_queue_message(rp, &rp_locks, bp, message, NIL); + } + erts_smp_proc_unlock(rp, rp_locks); + } } - rem_time->tv_sec = (ticks - elapsed) / 1000; - rem_time->tv_usec = 1000 * ((ticks - elapsed) % 1000); + + erts_free(ERTS_ALC_T_TMP, tmp); } } -void erts_get_timeval(SysTimeval *tv) +static void +schedule_send_time_offset_changed_notifications(ErtsMonotonicTime new_offset) { - erts_smp_mtx_lock(&erts_timeofday_mtx); - get_tolerant_timeofday(tv); - erts_smp_mtx_unlock(&erts_timeofday_mtx); - update_approx_time(tv); +#ifdef ARCH_64 + void *new_offsetp = (void *) new_offset; + ASSERT(sizeof(void *) == sizeof(ErtsMonotonicTime)); +#else + void *new_offsetp = erts_alloc(ERTS_ALC_T_NEW_TIME_OFFSET, + sizeof(ErtsMonotonicTime)); + *((ErtsMonotonicTime *) new_offsetp) = new_offset; +#endif + erts_schedule_misc_aux_work(1, + send_time_offset_changed_notifications, + new_offsetp); } -erts_time_t -erts_get_time(void) +static ERTS_INLINE Eterm +make_time_val(Process *c_p, ErtsMonotonicTime time_val) { - SysTimeval sys_tv; - - erts_smp_mtx_lock(&erts_timeofday_mtx); - - get_tolerant_timeofday(&sys_tv); - - erts_smp_mtx_unlock(&erts_timeofday_mtx); + Sint64 val = (Sint64) time_val; + Eterm *hp; + Uint sz; - update_approx_time(&sys_tv); + if (IS_SSMALL(val)) + return make_small(val); - return sys_tv.tv_sec; + sz = ERTS_SINT64_HEAP_SIZE(val); + hp = HAlloc(c_p, sz); + return erts_sint64_to_big(val, &hp); } -#ifdef HAVE_ERTS_NOW_CPU -void erts_get_now_cpu(Uint* megasec, Uint* sec, Uint* microsec) { - SysCpuTime t; - SysTimespec tp; +Eterm +erts_get_monotonic_start_time(struct process *c_p) +{ + return make_time_val(c_p, ERTS_MONOTONIC_TIME_START_EXTERNAL); +} - sys_get_proc_cputime(t, tp); - *microsec = (Uint)(tp.tv_nsec / 1000); - t = (tp.tv_sec / 1000000); - *megasec = (Uint)(t % 1000000); - *sec = (Uint)(tp.tv_sec % 1000000); +Eterm +erts_get_monotonic_end_time(struct process *c_p) +{ + return make_time_val(c_p, ERTS_MONOTONIC_TIME_END_EXTERNAL); } + +static Eterm +bld_monotonic_time_source(Uint **hpp, Uint *szp, Sint64 os_mtime) +{ +#ifndef ERTS_HAVE_OS_MONOTONIC_TIME_SUPPORT + return NIL; +#else + int i = 0; + Eterm k[6]; + Eterm v[6]; + + if (time_sup.r.o.os_monotonic_time_disable) + return NIL; + + k[i] = erts_bld_atom(hpp, szp, "function"); + v[i++] = erts_bld_atom(hpp, szp, + time_sup.r.o.os_monotonic_time_func); + + if (time_sup.r.o.os_monotonic_time_clock_id) { + k[i] = erts_bld_atom(hpp, szp, "clock_id"); + v[i++] = erts_bld_atom(hpp, szp, + time_sup.r.o.os_monotonic_time_clock_id); + } + + k[i] = erts_bld_atom(hpp, szp, "resolution"); + v[i++] = erts_bld_uint64(hpp, szp, + time_sup.r.o.os_monotonic_time_resolution); + + k[i] = erts_bld_atom(hpp, szp, "extended"); + v[i++] = time_sup.r.o.os_monotonic_time_extended ? am_yes : am_no; + + k[i] = erts_bld_atom(hpp, szp, "parallel"); + v[i++] = time_sup.r.o.os_monotonic_time_locked ? am_no : am_yes; + + k[i] = erts_bld_atom(hpp, szp, "time"); + v[i++] = erts_bld_sint64(hpp, szp, os_mtime); + + return erts_bld_2tup_list(hpp, szp, (Sint) i, k, v); +#endif +} + +Eterm +erts_monotonic_time_source(struct process *c_p) +{ + Uint hsz = 0; + Eterm *hp = NULL; + Sint64 os_mtime = 0; +#ifdef ERTS_HAVE_OS_MONOTONIC_TIME_SUPPORT + if (!time_sup.r.o.os_monotonic_time_disable) + os_mtime = (Sint64) erts_os_monotonic_time(); +#endif + + bld_monotonic_time_source(NULL, &hsz, os_mtime); + if (hsz) + hp = HAlloc(c_p, hsz); + return bld_monotonic_time_source(&hp, NULL, os_mtime); +} + +static Eterm +bld_system_time_source(Uint **hpp, Uint *szp, Sint64 os_stime) +{ + int i = 0; + Eterm k[5]; + Eterm v[5]; + + k[i] = erts_bld_atom(hpp, szp, "function"); + v[i++] = erts_bld_atom(hpp, szp, + time_sup.r.o.os_system_time_func); + + if (time_sup.r.o.os_system_time_clock_id) { + k[i] = erts_bld_atom(hpp, szp, "clock_id"); + v[i++] = erts_bld_atom(hpp, szp, + time_sup.r.o.os_system_time_clock_id); + } + + k[i] = erts_bld_atom(hpp, szp, "resolution"); + v[i++] = erts_bld_uint64(hpp, szp, + time_sup.r.o.os_system_time_resolution); + + k[i] = erts_bld_atom(hpp, szp, "parallel"); + v[i++] = am_yes; + + k[i] = erts_bld_atom(hpp, szp, "time"); + v[i++] = erts_bld_sint64(hpp, szp, os_stime); + + return erts_bld_2tup_list(hpp, szp, (Sint) i, k, v); +} + +Eterm +erts_system_time_source(struct process *c_p) +{ + Uint hsz = 0; + Eterm *hp = NULL; + Sint64 os_stime = (Sint64) erts_os_system_time(); + + bld_system_time_source(NULL, &hsz, os_stime); + if (hsz) + hp = HAlloc(c_p, hsz); + return bld_system_time_source(&hp, NULL, os_stime); +} + + +#include "bif.h" + +static ERTS_INLINE Eterm +time_unit_conversion(Process *c_p, Eterm term, ErtsMonotonicTime val, ErtsMonotonicTime muloff) +{ + ErtsMonotonicTime result; + BIF_RETTYPE ret; + + if (val < 0) + goto trap_to_erlang_code; + + /* Convert to common user specified time units */ + switch (term) { + case am_seconds: + case make_small(1): + result = ERTS_MONOTONIC_TO_SEC(val) + muloff*ERTS_MONOTONIC_OFFSET_SEC; + ERTS_BIF_PREP_RET(ret, make_time_val(c_p, result)); + break; + case am_milli_seconds: + case make_small(1000): + result = ERTS_MONOTONIC_TO_MSEC(val) + muloff*ERTS_MONOTONIC_OFFSET_MSEC; + ERTS_BIF_PREP_RET(ret, make_time_val(c_p, result)); + break; + case am_micro_seconds: + case make_small(1000*1000): + result = ERTS_MONOTONIC_TO_USEC(val) + muloff*ERTS_MONOTONIC_OFFSET_USEC; + ERTS_BIF_PREP_RET(ret, make_time_val(c_p, result)); + break; +#ifdef ARCH_64 + case am_nano_seconds: + case make_small(1000*1000*1000): + result = ERTS_MONOTONIC_TO_NSEC(val) + muloff*ERTS_MONOTONIC_OFFSET_NSEC; + ERTS_BIF_PREP_RET(ret, make_time_val(c_p, result)); + break; #endif + default: { + Eterm value, native_res; +#ifndef ARCH_64 + Sint user_res; + if (term == am_nano_seconds) + goto to_nano_seconds; + if (term_to_Sint(term, &user_res)) { + if (user_res == 1000*1000*1000) { + to_nano_seconds: + result = (ERTS_MONOTONIC_TO_NSEC(val) + + muloff*ERTS_MONOTONIC_OFFSET_NSEC); + ERTS_BIF_PREP_RET(ret, make_time_val(c_p, result)); + break; + } + if (user_res <= 0) + goto badarg; + } +#else + if (is_small(term)) { + if (signed_val(term) <= 0) + goto badarg; + } +#endif + else if (is_big(term)) { + if (big_sign(term)) + goto badarg; + } + else { + badarg: + ERTS_BIF_PREP_ERROR(ret, c_p, BADARG); + break; + } + + trap_to_erlang_code: + /* Do it in erlang code instead; pass along values to use... */ + value = make_time_val(c_p, val + muloff*ERTS_MONOTONIC_OFFSET_NATIVE); + native_res = make_time_val(c_p, ERTS_MONOTONIC_TIME_UNIT); + + ERTS_BIF_PREP_TRAP3(ret, erts_convert_time_unit_trap, c_p, + value, native_res, term); + + break; + } + } + + return ret; +} + + +/* + * Time Native API (drivers and NIFs) + */ + +#define ERTS_NAPI_TIME_ERROR ((ErtsMonotonicTime) ERTS_NAPI_TIME_ERROR__) + +static void +init_time_napi(void) +{ + /* Verify that time native api constants are as expected... */ + + ASSERT(sizeof(ErtsMonotonicTime) == sizeof(ErlDrvTime)); + ASSERT(ERL_DRV_TIME_ERROR == (ErlDrvTime) ERTS_NAPI_TIME_ERROR); + ASSERT(ERL_DRV_TIME_ERROR < (ErlDrvTime) 0); + ASSERT(ERTS_NAPI_SEC__ == (int) ERL_DRV_SEC); + ASSERT(ERTS_NAPI_MSEC__ == (int) ERL_DRV_MSEC); + ASSERT(ERTS_NAPI_USEC__ == (int) ERL_DRV_USEC); + ASSERT(ERTS_NAPI_NSEC__ == (int) ERL_DRV_NSEC); + + ASSERT(sizeof(ErtsMonotonicTime) == sizeof(ErlNifTime)); + ASSERT(ERL_NIF_TIME_ERROR == (ErlNifTime) ERTS_NAPI_TIME_ERROR); + ASSERT(ERL_NIF_TIME_ERROR < (ErlNifTime) 0); + ASSERT(ERTS_NAPI_SEC__ == (int) ERL_NIF_SEC); + ASSERT(ERTS_NAPI_MSEC__ == (int) ERL_NIF_MSEC); + ASSERT(ERTS_NAPI_USEC__ == (int) ERL_NIF_USEC); + ASSERT(ERTS_NAPI_NSEC__ == (int) ERL_NIF_NSEC); +} + +ErtsMonotonicTime +erts_napi_monotonic_time(int time_unit) +{ + ErtsSchedulerData *esdp; + ErtsMonotonicTime mtime; + + /* At least for now only allow schedulers to do this... */ + esdp = erts_get_scheduler_data(); + if (!esdp) + return ERTS_NAPI_TIME_ERROR; + + mtime = time_sup.r.o.get_time(); + update_last_mtime(esdp, mtime); + + switch (time_unit) { + case ERTS_NAPI_SEC__: + mtime = ERTS_MONOTONIC_TO_SEC(mtime); + mtime += ERTS_MONOTONIC_OFFSET_SEC; + break; + case ERTS_NAPI_MSEC__: + mtime = ERTS_MONOTONIC_TO_MSEC(mtime); + mtime += ERTS_MONOTONIC_OFFSET_MSEC; + break; + case ERTS_NAPI_USEC__: + mtime = ERTS_MONOTONIC_TO_USEC(mtime); + mtime += ERTS_MONOTONIC_OFFSET_USEC; + break; + case ERTS_NAPI_NSEC__: + mtime = ERTS_MONOTONIC_TO_NSEC(mtime); + mtime += ERTS_MONOTONIC_OFFSET_NSEC; + break; + default: + return ERTS_NAPI_TIME_ERROR; + } + + return mtime; +} + +ErtsMonotonicTime +erts_napi_time_offset(int time_unit) +{ + ErtsSchedulerData *esdp; + ErtsSystemTime offs; + + /* At least for now only allow schedulers to do this... */ + esdp = erts_get_scheduler_data(); + if (!esdp) + return ERTS_NAPI_TIME_ERROR; + + offs = get_time_offset(); + switch (time_unit) { + case ERTS_NAPI_SEC__: + offs = ERTS_MONOTONIC_TO_SEC(offs); + offs -= ERTS_MONOTONIC_OFFSET_SEC; + break; + case ERTS_NAPI_MSEC__: + offs = ERTS_MONOTONIC_TO_MSEC(offs); + offs -= ERTS_MONOTONIC_OFFSET_MSEC; + break; + case ERTS_NAPI_USEC__: + offs = ERTS_MONOTONIC_TO_USEC(offs); + offs -= ERTS_MONOTONIC_OFFSET_USEC; + break; + case ERTS_NAPI_NSEC__: + offs = ERTS_MONOTONIC_TO_NSEC(offs); + offs -= ERTS_MONOTONIC_OFFSET_NSEC; + break; + default: + return ERTS_NAPI_TIME_ERROR; + } + return offs; +} + +ErtsMonotonicTime +erts_napi_convert_time_unit(ErtsMonotonicTime val, int from, int to) +{ + ErtsMonotonicTime ffreq, tfreq, denom; + /* + * Convertion between time units using floor function. + * + * Note that this needs to work also for negative + * values. Ordinary integer division on a negative + * value will give ceiling... + */ + + switch ((int) from) { + case ERTS_NAPI_SEC__: ffreq = 1; break; + case ERTS_NAPI_MSEC__: ffreq = 1000; break; + case ERTS_NAPI_USEC__: ffreq = 1000*1000; break; + case ERTS_NAPI_NSEC__: ffreq = 1000*1000*1000; break; + default: return ERTS_NAPI_TIME_ERROR; + } + + switch ((int) to) { + case ERTS_NAPI_SEC__: tfreq = 1; break; + case ERTS_NAPI_MSEC__: tfreq = 1000; break; + case ERTS_NAPI_USEC__: tfreq = 1000*1000; break; + case ERTS_NAPI_NSEC__: tfreq = 1000*1000*1000; break; + default: return ERTS_NAPI_TIME_ERROR; + } + + if (tfreq >= ffreq) + return val * (tfreq / ffreq); + + denom = ffreq / tfreq; + if (val >= 0) + return val / denom; + + return (val - (denom - 1)) / denom; +} + +/* Built in functions */ + +BIF_RETTYPE monotonic_time_0(BIF_ALIST_0) +{ + ErtsMonotonicTime mtime = time_sup.r.o.get_time(); + update_last_mtime(ERTS_PROC_GET_SCHDATA(BIF_P), mtime); + mtime += ERTS_MONOTONIC_OFFSET_NATIVE; + BIF_RET(make_time_val(BIF_P, mtime)); +} + +BIF_RETTYPE monotonic_time_1(BIF_ALIST_1) +{ + ErtsMonotonicTime mtime = time_sup.r.o.get_time(); + update_last_mtime(ERTS_PROC_GET_SCHDATA(BIF_P), mtime); + BIF_RET(time_unit_conversion(BIF_P, BIF_ARG_1, mtime, 1)); +} + +BIF_RETTYPE system_time_0(BIF_ALIST_0) +{ + ErtsMonotonicTime mtime, offset; + mtime = time_sup.r.o.get_time(); + offset = get_time_offset(); + update_last_mtime(ERTS_PROC_GET_SCHDATA(BIF_P), mtime); + BIF_RET(make_time_val(BIF_P, mtime + offset)); +} + +BIF_RETTYPE system_time_1(BIF_ALIST_0) +{ + ErtsMonotonicTime mtime, offset; + mtime = time_sup.r.o.get_time(); + offset = get_time_offset(); + update_last_mtime(ERTS_PROC_GET_SCHDATA(BIF_P), mtime); + BIF_RET(time_unit_conversion(BIF_P, BIF_ARG_1, mtime + offset, 0)); +} + +BIF_RETTYPE erts_internal_time_unit_0(BIF_ALIST_0) +{ + BIF_RET(make_time_val(BIF_P, ERTS_MONOTONIC_TIME_UNIT)); +} + +BIF_RETTYPE time_offset_0(BIF_ALIST_0) +{ + ErtsMonotonicTime time_offset = get_time_offset(); + time_offset -= ERTS_MONOTONIC_OFFSET_NATIVE; + BIF_RET(make_time_val(BIF_P, time_offset)); +} + +BIF_RETTYPE time_offset_1(BIF_ALIST_1) +{ + BIF_RET(time_unit_conversion(BIF_P, BIF_ARG_1, get_time_offset(), -1)); +} + + +BIF_RETTYPE timestamp_0(BIF_ALIST_0) +{ + Eterm *hp, res; + ErtsMonotonicTime mtime, offset; + Uint mega_sec, sec, micro_sec; + + mtime = time_sup.r.o.get_time(); + offset = get_time_offset(); + update_last_mtime(ERTS_PROC_GET_SCHDATA(BIF_P), mtime); + + make_timestamp_value(&mega_sec, &sec, µ_sec, mtime, offset); + + /* + * Mega seconds is the only value that potentially + * ever could be a bignum. However, that wont happen + * during at least the next 4 million years... + * + * (System time will also have wrapped in the + * 64-bit integer before we get there...) + */ + + ASSERT(IS_USMALL(0, mega_sec)); + ASSERT(IS_USMALL(0, sec)); + ASSERT(IS_USMALL(0, micro_sec)); + + hp = HAlloc(BIF_P, 4); + res = TUPLE3(hp, + make_small(mega_sec), + make_small(sec), + make_small(micro_sec)); + BIF_RET(res); +} + +BIF_RETTYPE os_system_time_0(BIF_ALIST_0) +{ + ErtsSystemTime stime = erts_os_system_time(); + BIF_RET(make_time_val(BIF_P, stime)); +} + +BIF_RETTYPE os_system_time_1(BIF_ALIST_0) +{ + ErtsSystemTime stime = erts_os_system_time(); + BIF_RET(time_unit_conversion(BIF_P, BIF_ARG_1, stime, 0)); +} + diff --git a/erts/emulator/beam/erl_trace.c b/erts/emulator/beam/erl_trace.c index 848877d43e..114853cac4 100644 --- a/erts/emulator/beam/erl_trace.c +++ b/erts/emulator/beam/erl_trace.c @@ -1,18 +1,19 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 1999-2012. All Rights Reserved. + * Copyright Ericsson AB 1999-2014. All Rights Reserved. * - * The contents of this file are subject to the Erlang Public License, - * Version 1.1, (the "License"); you may not use this file except in - * compliance with the License. You should have received a copy of the - * Erlang Public License along with this software. If not, it can be - * retrieved online at http://www.erlang.org/. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Software distributed under the License is distributed on an "AS IS" - * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See - * the License for the specific language governing rights and limitations - * under the License. + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. * * %CopyrightEnd% */ @@ -37,6 +38,7 @@ #include "erl_binary.h" #include "erl_bits.h" #include "erl_thr_progress.h" +#include "erl_bif_unique.h" #if 0 #define DEBUG_PRINTOUTS @@ -76,6 +78,264 @@ enum ErtsSysMsgType { SYS_MSG_TYPE_SYSPROF }; +#define ERTS_TRACE_TS_NOW_MAX_SIZE \ + 4 +#define ERTS_TRACE_TS_MONOTONIC_MAX_SIZE \ + ERTS_MAX_SINT64_HEAP_SIZE +#define ERTS_TRACE_TS_STRICT_MONOTONIC_MAX_SIZE \ + (3 + ERTS_MAX_SINT64_HEAP_SIZE \ + + ERTS_MAX_UINT64_HEAP_SIZE) + +#define ERTS_TRACE_PATCH_TS_MAX_SIZE \ + (1 + ((ERTS_TRACE_TS_NOW_MAX_SIZE \ + > ERTS_TRACE_TS_MONOTONIC_MAX_SIZE) \ + ? ((ERTS_TRACE_TS_NOW_MAX_SIZE \ + > ERTS_TRACE_TS_STRICT_MONOTONIC_MAX_SIZE) \ + ? ERTS_TRACE_TS_NOW_MAX_SIZE \ + : ERTS_TRACE_TS_STRICT_MONOTONIC_MAX_SIZE) \ + : ((ERTS_TRACE_TS_MONOTONIC_MAX_SIZE \ + > ERTS_TRACE_TS_STRICT_MONOTONIC_MAX_SIZE) \ + ? ERTS_TRACE_TS_MONOTONIC_MAX_SIZE \ + : ERTS_TRACE_TS_STRICT_MONOTONIC_MAX_SIZE))) + +#define TFLGS_TS_TYPE(p) ERTS_TFLGS2TSTYPE(ERTS_TRACE_FLAGS((p))) + +/* + * FUTURE CHANGES: + * + * The timestamp functionality has intentionally been + * split in two parts for future use even though it + * is not used like this today. take_timestamp() takes + * the timestamp and calculate heap need for it (which + * is not constant). write_timestamp() writes the + * timestamp to the allocated heap. That is, one typically + * want to take the timestamp before allocating the heap + * and then write it to the heap. + * + * The trace output functionality now use patch_ts_size(), + * write_ts(), and patch_ts(). write_ts() both takes the + * timestamp and writes it. Since we don't know the + * heap need when allocating the heap area we need to + * over allocate (maximum size from patch_ts_size()) and + * then potentially (often) shrink the heap area after the + * timestamp has been written. The only reason it is + * currently done this way is because we do not want to + * make major changes of the trace behavior in a patch. + * This is planned to be changed in next major release. + */ + +typedef struct { + int ts_type_flag; + union { + struct { + Uint ms; + Uint s; + Uint us; + } now; + struct { + ErtsMonotonicTime time; + Sint64 raw_unique; + } monotonic; + } u; +} ErtsTraceTimeStamp; + +static ERTS_INLINE Uint +take_timestamp(ErtsTraceTimeStamp *tsp, int ts_type) +{ + int ts_type_flag = ts_type & -ts_type; /* least significant flag */ + + ASSERT(ts_type_flag == ERTS_TRACE_FLG_NOW_TIMESTAMP + || ts_type_flag == ERTS_TRACE_FLG_MONOTONIC_TIMESTAMP + || ts_type_flag == ERTS_TRACE_FLG_STRICT_MONOTONIC_TIMESTAMP + || ts_type_flag == 0); + + tsp->ts_type_flag = ts_type_flag; + switch (ts_type_flag) { + case 0: + return (Uint) 0; + case ERTS_TRACE_FLG_NOW_TIMESTAMP: +#ifdef HAVE_ERTS_NOW_CPU + if (erts_cpu_timestamp) + erts_get_now_cpu(&tsp->u.now.ms, &tsp->u.now.s, &tsp->u.now.us); + else +#endif + get_now(&tsp->u.now.ms, &tsp->u.now.s, &tsp->u.now.us); + return (Uint) 4; + case ERTS_TRACE_FLG_MONOTONIC_TIMESTAMP: + case ERTS_TRACE_FLG_STRICT_MONOTONIC_TIMESTAMP: { + Uint hsz = 0; + ErtsMonotonicTime mtime = erts_get_monotonic_time(NULL); + mtime = ERTS_MONOTONIC_TO_NSEC(mtime); + mtime += ERTS_MONOTONIC_OFFSET_NSEC; + hsz = (IS_SSMALL(mtime) ? + (Uint) 0 + : ERTS_SINT64_HEAP_SIZE((Sint64) mtime)); + tsp->u.monotonic.time = mtime; + if (ts_type_flag == ERTS_TRACE_FLG_STRICT_MONOTONIC_TIMESTAMP) { + Sint64 raw_unique; + hsz += 3; /* 2-tuple */ + raw_unique = erts_raw_get_unique_monotonic_integer(); + tsp->u.monotonic.raw_unique = raw_unique; + hsz += erts_raw_unique_monotonic_integer_heap_size(raw_unique); + } + return hsz; + } + default: + ERTS_INTERNAL_ERROR("invalid timestamp type"); + return 0; + } +} + +static ERTS_INLINE Eterm +write_timestamp(ErtsTraceTimeStamp *tsp, Eterm **hpp) +{ + int ts_type_flag = tsp->ts_type_flag; + Eterm res; + + switch (ts_type_flag) { + case 0: + return NIL; + case ERTS_TRACE_FLG_NOW_TIMESTAMP: + res = TUPLE3(*hpp, + make_small(tsp->u.now.ms), + make_small(tsp->u.now.s), + make_small(tsp->u.now.us)); + *hpp += 4; + return res; + case ERTS_TRACE_FLG_MONOTONIC_TIMESTAMP: + case ERTS_TRACE_FLG_STRICT_MONOTONIC_TIMESTAMP: { + Sint64 mtime, raw; + Eterm unique, emtime; + + mtime = (Sint64) tsp->u.monotonic.time; + emtime = (IS_SSMALL(mtime) + ? make_small((Sint64) mtime) + : erts_sint64_to_big((Sint64) mtime, hpp)); + + if (ts_type_flag == ERTS_TRACE_FLG_MONOTONIC_TIMESTAMP) + return emtime; + + raw = tsp->u.monotonic.raw_unique; + unique = erts_raw_make_unique_monotonic_integer_value(hpp, + raw); + res = TUPLE2(*hpp, emtime, unique); + *hpp += 3; + return res; + } + default: + ERTS_INTERNAL_ERROR("invalid timestamp type"); + return THE_NON_VALUE; + } +} + +#define PATCH_TS_SIZE(p) patch_ts_size(TFLGS_TS_TYPE(p)) + +static ERTS_INLINE Uint +patch_ts_size(int ts_type) +{ + int ts_type_flag = ts_type & -ts_type; /* least significant flag */ + switch (ts_type_flag) { + case 0: + return 0; + case ERTS_TRACE_FLG_NOW_TIMESTAMP: + return 1 + ERTS_TRACE_TS_NOW_MAX_SIZE; + case ERTS_TRACE_FLG_MONOTONIC_TIMESTAMP: + return 1 + ERTS_TRACE_TS_MONOTONIC_MAX_SIZE; + case ERTS_TRACE_FLG_STRICT_MONOTONIC_TIMESTAMP: + return 1 + ERTS_TRACE_TS_STRICT_MONOTONIC_MAX_SIZE; + default: + ERTS_INTERNAL_ERROR("invalid timestamp type"); + return 0; + } +} + +/* + * Write a timestamp. The timestamp MUST be the last + * thing built on the heap. This since write_ts() might + * adjust the size of the used area. + */ +static Eterm +write_ts(int ts_type, Eterm *hp, ErlHeapFragment *bp, Process *tracer) +{ + ErtsTraceTimeStamp ts; + Sint shrink; + Eterm res, *ts_hp = hp; + Uint hsz; + + ASSERT(ts_type); + + hsz = take_timestamp(&ts, ts_type); + + res = write_timestamp(&ts, &ts_hp); + + ASSERT(ts_hp == hp + hsz); + + switch (ts.ts_type_flag) { + case ERTS_TRACE_FLG_MONOTONIC_TIMESTAMP: + shrink = ERTS_TRACE_TS_MONOTONIC_MAX_SIZE; + break; + case ERTS_TRACE_FLG_STRICT_MONOTONIC_TIMESTAMP: + shrink = ERTS_TRACE_TS_STRICT_MONOTONIC_MAX_SIZE; + break; + default: + return res; + } + + shrink -= hsz; + + ASSERT(shrink >= 0); + + if (shrink) { + if (bp) + bp->used_size -= shrink; +#ifndef ERTS_SMP + else if (tracer) { + Eterm *endp = ts_hp + shrink; + HRelease(tracer, endp, ts_hp); + } +#endif + } + + return res; +} + +/* + * Patch a timestamp into a tuple. The tuple MUST be the last thing + * built on the heap before the call, and the timestamp MUST be + * the last thing after the call. This since patch_ts() might adjust + * the size of the used area. + */ + +#define PATCH_TS__(Type, Tuple, Hp, Bp, Tracer) \ + do { \ + int ts_type__ = (Type); \ + if (ts_type__) \ + patch_ts(ts_type__, (Tuple), (Hp), (Bp), (Tracer)); \ + } while (0) + +#ifdef ERTS_SMP +#define PATCH_TS(Type, Tuple, Hp, Bp, Tracer) \ + PATCH_TS__((Type), (Tuple), (Hp), (Bp), NULL) +#else +#define PATCH_TS(Type, Tuple, Hp, Bp, Tracer) \ + PATCH_TS__((Type), (Tuple), (Hp), (Bp), (Tracer)) +#endif + +static ERTS_INLINE void +patch_ts(int ts_type, Eterm tuple, Eterm* hp, ErlHeapFragment *bp, Process *tracer) +{ + Eterm *tptr = tuple_val(tuple); + int arity = arityval(*tptr); + + ASSERT(ts_type); + ASSERT((tptr+arity+1) == hp); + + tptr[0] = make_arityval(arity+1); + tptr[1] = am_trace_ts; + + *hp = write_ts(ts_type, hp+1, bp, tracer); +} + #ifdef ERTS_SMP static void enqueue_sys_msg_unlocked(enum ErtsSysMsgType type, Eterm from, @@ -130,14 +390,9 @@ do { \ enqueue_sys_msg_unlocked(SYS_MSG_TYPE_TRACE, (FPID), (TPID), (MSG), (BP)); \ } while(0) #else -#ifdef USE_VM_PROBES -#define ERTS_ENQ_TRACE_MSG(FPID, TPROC, MSG, BP) \ - erts_queue_message((TPROC), NULL, (BP), (MSG), NIL, NIL) -#else #define ERTS_ENQ_TRACE_MSG(FPID, TPROC, MSG, BP) \ erts_queue_message((TPROC), NULL, (BP), (MSG), NIL) #endif -#endif /* * NOTE that the ERTS_GET_TRACER_REF() returns from the function (!!!) @@ -151,6 +406,11 @@ do { \ message dispatcher thread takes care of that). */ #define ERTS_GET_TRACER_REF(RES, TPID, TRACEE_FLGS) \ do { (RES) = (TPID); } while(0) +int +erts_is_tracer_proc_valid(Process* p) +{ + return 1; +} #else #define ERTS_NULL_TRACER_REF NULL #define ERTS_TRACER_REF_TYPE Process * @@ -163,6 +423,20 @@ do { \ return; \ } \ } while (0) +int +erts_is_tracer_proc_valid(Process* p) +{ + Process* tracer; + + tracer = erts_proc_lookup(ERTS_TRACER_PROC(p)); + if (tracer && ERTS_TRACE_FLAGS(tracer) & F_TRACER) { + return 1; + } else { + ERTS_TRACER_PROC(p) = NIL; + ERTS_TRACE_FLAGS(p) = ~TRACEE_FLAGS; + return 0; + } +} #endif static Uint active_sched; @@ -350,23 +624,6 @@ erts_get_system_profile(void) { return profile; } - -#ifdef HAVE_ERTS_NOW_CPU -# define GET_NOW(m, s, u) \ -do { \ - if (erts_cpu_timestamp) \ - erts_get_now_cpu(m, s, u); \ - else \ - get_now(m, s, u); \ -} while (0) -#else -# define GET_NOW(m, s, u) do {get_now(m, s, u);} while (0) -#endif - - - -static Eterm* patch_ts(Eterm tuple4, Eterm* hp); - #ifdef ERTS_SMP static void do_send_to_port(Eterm to, @@ -405,7 +662,7 @@ WRITE_SYS_MSG_TO_PORT(Eterm unused_to, erts_encode_ext(message, &ptr); if (!(ptr <= buffer+size)) { - erl_exit(1, "Internal error in do_send_to_port: %d\n", ptr-buffer); + erts_exit(ERTS_ERROR_EXIT, "Internal error in do_send_to_port: %d\n", ptr-buffer); } #ifndef ERTS_SMP @@ -421,11 +678,11 @@ WRITE_SYS_MSG_TO_PORT(Eterm unused_to, /* Send {trace_ts, Pid, out, 0, Timestamp} * followed by {trace_ts, Pid, in, 0, NewTimestamp} * - * 'NewTimestamp' is fetched from GET_NOW() through patch_ts(). + * 'NewTimestamp' through patch_ts(). */ static void -do_send_schedfix_to_port(Port *trace_port, Eterm pid, Eterm timestamp) { -#define LOCAL_HEAP_SIZE (4+5+5) +do_send_schedfix_to_port(Port *trace_port, Eterm pid, Eterm timestamp, int ts_type) { +#define LOCAL_HEAP_SIZE (5+5+ERTS_TRACE_PATCH_TS_MAX_SIZE) DeclareTmpHeapNoproc(local_heap,LOCAL_HEAP_SIZE); Eterm message; Eterm *hp; @@ -447,9 +704,11 @@ do_send_schedfix_to_port(Port *trace_port, Eterm pid, Eterm timestamp) { SYS_MSG_TYPE_UNDEFINED, message); - message = TUPLE4(hp, am_trace_ts, pid, am_in, mfarity); - hp += 5; - hp = patch_ts(message, hp); + + message = TUPLE5(hp, am_trace_ts, pid, am_in, mfarity, + NIL /* Will be overwritten by timestamp */); + hp += 6; + hp[-1] = write_ts(ts_type, hp, NULL, NULL); do_send_to_port(trace_port->common.id, trace_port, @@ -466,7 +725,7 @@ do_send_schedfix_to_port(Port *trace_port, Eterm pid, Eterm timestamp) { * It is assumed that 'message' is not an 'out' message. * * 'c_p' is the currently executing process, "tracee" is the traced process - * which 'message' concerns => if (*tracee_flags & F_TIMESTAMP), + * which 'message' concerns => if (*tracee_flags & F_TIMESTAMP_MASK), * 'message' must contain a timestamp. */ static void @@ -474,8 +733,9 @@ send_to_port(Process *c_p, Eterm message, Eterm *tracer_pid, Uint *tracee_flags) { Port* trace_port; #ifndef ERTS_SMP -#define LOCAL_HEAP_SIZE (4) - Eterm ts, *hp; + int ts_type; +#define LOCAL_HEAP_SIZE ERTS_TRACE_PATCH_TS_MAX_SIZE + Eterm ts; DeclareTmpHeapNoproc(local_heap,LOCAL_HEAP_SIZE); #endif @@ -504,7 +764,7 @@ send_to_port(Process *c_p, Eterm message, */ if ( c_p == NULL || - (! IS_TRACED_FL(c_p, F_TRACE_SCHED | F_TIMESTAMP))) { + (! IS_TRACED_FL(c_p, F_TRACE_SCHED | F_TIMESTAMP_MASK))) { #endif do_send_to_port(*tracer_pid, trace_port, @@ -523,22 +783,12 @@ send_to_port(Process *c_p, Eterm message, */ UseTmpHeapNoproc(LOCAL_HEAP_SIZE); - if (*tracee_flags & F_TIMESTAMP) { - ASSERT(is_tuple(message)); - hp = tuple_val(message); - ts = hp[arityval(hp[0])]; - } else { - /* A fake schedule might be needed, - * but this message does not contain a timestamp. - * Create a dummy trace message with timestamp to be - * passed to do_send_schedfix_to_port(). - */ - Uint ms,s,us; - GET_NOW(&ms, &s, &us); - hp = local_heap; - ts = TUPLE3(hp, make_small(ms), make_small(s), make_small(us)); - hp += 4; - } + /* A fake schedule might be needed. + * Create a dummy trace message with timestamp to be + * passed to do_send_schedfix_to_port(). + */ + ts_type = TFLGS_TS_TYPE(c_p); + ts = write_ts(ts_type, local_heap, NULL, NULL); trace_port->control_flags &= ~PORT_CONTROL_FLAG_HEAVY; do_send_to_port(*tracer_pid, @@ -557,7 +807,7 @@ send_to_port(Process *c_p, Eterm message, * just after writning the real trace message, and now gets scheduled * in again. */ - do_send_schedfix_to_port(trace_port, c_p->common.id, ts); + do_send_schedfix_to_port(trace_port, c_p->common.id, ts, ts_type); } erts_port_release(trace_port); @@ -617,11 +867,7 @@ profile_send(Eterm from, Eterm message) { hp = erts_alloc_message_heap(sz, &bp, &off_heap, profile_p, 0); msg = copy_struct(message, sz, &hp, &bp->off_heap); - erts_queue_message(profile_p, NULL, bp, msg, NIL -#ifdef USE_VM_PROBES - , NIL -#endif - ); + erts_queue_message(profile_p, NULL, bp, msg, NIL); } } @@ -630,20 +876,19 @@ profile_send(Eterm from, Eterm message) { /* A fake schedule out/in message pair will be sent, * if the driver so requests. - * If (timestamp == NIL), one is fetched from GET_NOW(). * * 'c_p' is the currently executing process, may be NULL. */ static void seq_trace_send_to_port(Process *c_p, Eterm seq_tracer, - Eterm message, - Eterm timestamp) + Eterm message) { Port* trace_port; #ifndef ERTS_SMP - Eterm ts, *hp; -#define LOCAL_HEAP_SIZE (4) + int ts_type; + Eterm ts; +#define LOCAL_HEAP_SIZE ERTS_TRACE_PATCH_TS_MAX_SIZE DeclareTmpHeapNoproc(local_heap,LOCAL_HEAP_SIZE); UseTmpHeapNoproc(LOCAL_HEAP_SIZE); #endif @@ -668,7 +913,7 @@ seq_trace_send_to_port(Process *c_p, } if (c_p == NULL - || (! IS_TRACED_FL(c_p, F_TRACE_SCHED | F_TIMESTAMP))) { + || (! IS_TRACED_FL(c_p, F_TRACE_SCHED | F_TIMESTAMP_MASK))) { #endif do_send_to_port(seq_tracer, trace_port, @@ -685,20 +930,12 @@ seq_trace_send_to_port(Process *c_p, * with 'running' and 'timestamp'. */ - if (timestamp != NIL) { - ts = timestamp; - } else { - /* A fake schedule might be needed, - * but this message does not contain a timestamp. - * Create a dummy trace message with timestamp to be - * passed to do_send_schedfix_to_port(). - */ - Uint ms,s,us; - GET_NOW(&ms, &s, &us); - hp = local_heap; - ts = TUPLE3(hp, make_small(ms), make_small(s), make_small(us)); - hp += 4; - } + /* A fake schedule might be needed. + * Create a dummy trace message with timestamp to be + * passed to do_send_schedfix_to_port(). + */ + ts_type = TFLGS_TS_TYPE(c_p); + ts = write_ts(ts_type, local_heap, NULL, NULL); trace_port->control_flags &= ~PORT_CONTROL_FLAG_HEAVY; do_send_to_port(seq_tracer, @@ -717,7 +954,7 @@ seq_trace_send_to_port(Process *c_p, * just after writing the real trace message, and now gets scheduled * in again. */ - do_send_schedfix_to_port(trace_port, c_p->common.id, ts); + do_send_schedfix_to_port(trace_port, c_p->common.id, ts, ts_type); } erts_port_release(trace_port); @@ -727,32 +964,6 @@ seq_trace_send_to_port(Process *c_p, #endif } -#define TS_HEAP_WORDS 5 -#define TS_SIZE(p) ((ERTS_TRACE_FLAGS((p)) & F_TIMESTAMP) \ - ? TS_HEAP_WORDS \ - : 0) - -/* - * Patch a timestamp into a tuple. The tuple must be the last thing - * built on the heap. - * - * Returns the new hp pointer. -*/ -static Eterm* -patch_ts(Eterm tuple, Eterm* hp) -{ - Uint ms, s, us; - Eterm* ptr = tuple_val(tuple); - int arity = arityval(*ptr); - - ASSERT((ptr+arity+1) == hp); - ptr[0] = make_arityval(arity+1); - ptr[1] = am_trace_ts; - GET_NOW(&ms, &s, &us); - *hp = TUPLE3(hp+1, make_small(ms), make_small(s), make_small(us)); - return hp+5; -} - static ERTS_INLINE void send_to_tracer(Process *tracee, ERTS_TRACER_REF_TYPE tracer_ref, @@ -765,13 +976,13 @@ send_to_tracer(Process *tracee, erts_smp_mtx_lock(&smq_mtx); - if (ERTS_TRACE_FLAGS(tracee) & F_TIMESTAMP) - *hpp = patch_ts(msg, *hpp); - - if (is_internal_pid(ERTS_TRACER_PROC(tracee))) + if (is_internal_pid(ERTS_TRACER_PROC(tracee))) { + PATCH_TS(TFLGS_TS_TYPE(tracee), msg, *hpp, bp, tracer_ref); ERTS_ENQ_TRACE_MSG(tracee->common.id, tracer_ref, msg, bp); + } else { ASSERT(is_internal_port(ERTS_TRACER_PROC(tracee))); + PATCH_TS(TFLGS_TS_TYPE(tracee), msg, *hpp, NULL, NULL); send_to_port(no_fake_sched ? NULL : tracee, msg, &ERTS_TRACER_PROC(tracee), @@ -785,7 +996,7 @@ send_to_tracer(Process *tracee, static void trace_sched_aux(Process *p, Eterm what, int never_fake_sched) { -#define LOCAL_HEAP_SIZE (5+4+1+TS_HEAP_WORDS) +#define LOCAL_HEAP_SIZE (5+4+1+ERTS_TRACE_PATCH_TS_MAX_SIZE) DeclareTmpHeap(local_heap,LOCAL_HEAP_SIZE,p); Eterm tmp, mess, *hp; ErlHeapFragment *bp = NULL; @@ -841,7 +1052,7 @@ trace_sched_aux(Process *p, Eterm what, int never_fake_sched) size += 4; if (sched_no) size += 1; - size += TS_SIZE(p); + size += PATCH_TS_SIZE(p); hp = ERTS_ALLOC_SYSMSG_HEAP(size, &bp, &off_heap, tracer_ref); } @@ -915,7 +1126,7 @@ trace_send(Process *p, Eterm to, Eterm msg) } if (is_internal_port(ERTS_TRACER_PROC(p))) { -#define LOCAL_HEAP_SIZE (11) +#define LOCAL_HEAP_SIZE (6 + ERTS_TRACE_PATCH_TS_MAX_SIZE) DeclareTmpHeapNoproc(local_heap,LOCAL_HEAP_SIZE); UseTmpHeapNoproc(LOCAL_HEAP_SIZE); @@ -923,9 +1134,7 @@ trace_send(Process *p, Eterm to, Eterm msg) mess = TUPLE5(hp, am_trace, p->common.id, operation, msg, to); hp += 6; erts_smp_mtx_lock(&smq_mtx); - if (ERTS_TRACE_FLAGS(p) & F_TIMESTAMP) { - hp = patch_ts(mess, hp); - } + PATCH_TS(TFLGS_TS_TYPE(p), mess, hp, NULL, NULL); send_to_port(p, mess, &ERTS_TRACER_PROC(p), &ERTS_TRACE_FLAGS(p)); UnUseTmpHeapNoproc(LOCAL_HEAP_SIZE); #undef LOCAL_HEAP_SIZE @@ -944,7 +1153,7 @@ trace_send(Process *p, Eterm to, Eterm msg) sz_msg = size_object(msg); sz_to = size_object(to); - need = sz_msg + sz_to + 6 + TS_SIZE(p); + need = sz_msg + sz_to + 6 + PATCH_TS_SIZE(p); hp = ERTS_ALLOC_SYSMSG_HEAP(need, &bp, &off_heap, tracer_ref); @@ -961,10 +1170,7 @@ trace_send(Process *p, Eterm to, Eterm msg) erts_smp_mtx_lock(&smq_mtx); - if (ERTS_TRACE_FLAGS(p) & F_TIMESTAMP) { - patch_ts(mess, hp); - } - + PATCH_TS(TFLGS_TS_TYPE(p), mess, hp, bp, tracer_ref); ERTS_ENQ_TRACE_MSG(p->common.id, tracer_ref, mess, bp); erts_smp_mtx_unlock(&smq_mtx); } @@ -981,7 +1187,7 @@ trace_receive(Process *rp, Eterm msg) Eterm* hp; if (is_internal_port(ERTS_TRACER_PROC(rp))) { -#define LOCAL_HEAP_SIZE (10) +#define LOCAL_HEAP_SIZE (5+ERTS_TRACE_PATCH_TS_MAX_SIZE) DeclareTmpHeapNoproc(local_heap,LOCAL_HEAP_SIZE); UseTmpHeapNoproc(LOCAL_HEAP_SIZE); @@ -989,9 +1195,7 @@ trace_receive(Process *rp, Eterm msg) mess = TUPLE4(hp, am_trace, rp->common.id, am_receive, msg); hp += 5; erts_smp_mtx_lock(&smq_mtx); - if (ERTS_TRACE_FLAGS(rp) & F_TIMESTAMP) { - hp = patch_ts(mess, hp); - } + PATCH_TS(TFLGS_TS_TYPE(rp), mess, hp, NULL, NULL); send_to_port(rp, mess, &ERTS_TRACER_PROC(rp), &ERTS_TRACE_FLAGS(rp)); UnUseTmpHeapNoproc(LOCAL_HEAP_SIZE); #undef LOCAL_HEAP_SIZE @@ -1010,7 +1214,7 @@ trace_receive(Process *rp, Eterm msg) sz_msg = size_object(msg); - hsz = sz_msg + 5 + TS_SIZE(rp); + hsz = sz_msg + 5 + PATCH_TS_SIZE(rp); hp = ERTS_ALLOC_SYSMSG_HEAP(hsz, &bp, &off_heap, tracer_ref); @@ -1020,10 +1224,7 @@ trace_receive(Process *rp, Eterm msg) erts_smp_mtx_lock(&smq_mtx); - if (ERTS_TRACE_FLAGS(rp) & F_TIMESTAMP) { - patch_ts(mess, hp); - } - + PATCH_TS(TFLGS_TS_TYPE(rp), mess, hp, bp, tracer_ref); ERTS_ENQ_TRACE_MSG(rp->common.id, tracer_ref, mess, bp); erts_smp_mtx_unlock(&smq_mtx); } @@ -1073,6 +1274,7 @@ seq_trace_output_generic(Eterm token, Eterm msg, Uint type, Eterm type_atom; int sz_exit; Eterm seq_tracer; + int ts_type; seq_tracer = erts_get_system_seq_tracer(); @@ -1087,7 +1289,7 @@ seq_trace_output_generic(Eterm token, Eterm msg, Uint type, case SEQ_TRACE_PRINT: type_atom = am_print; break; case SEQ_TRACE_RECEIVE: type_atom = am_receive; break; default: - erl_exit(1, "invalid type in seq_trace_output_generic: %d:\n", type); + erts_exit(ERTS_ERROR_EXIT, "invalid type in seq_trace_output_generic: %d:\n", type); return; /* To avoid warning */ } @@ -1100,8 +1302,10 @@ seq_trace_output_generic(Eterm token, Eterm msg, Uint type, return; /* no need to send anything */ } + ts_type = ERTS_SEQTFLGS2TSTYPE(unsigned_val(SEQ_TRACE_T_FLAGS(token))); + if (is_internal_port(seq_tracer)) { -#define LOCAL_HEAP_SIZE (64) +#define LOCAL_HEAP_SIZE (60 + ERTS_TRACE_PATCH_TS_MAX_SIZE) DeclareTmpHeapNoproc(local_heap,LOCAL_HEAP_SIZE); UseTmpHeapNoproc(LOCAL_HEAP_SIZE); @@ -1117,17 +1321,17 @@ seq_trace_output_generic(Eterm token, Eterm msg, Uint type, mess = TUPLE5(hp, type_atom, lastcnt_serial, SEQ_TRACE_T_SENDER(token), receiver, msg); hp += 6; + erts_smp_mtx_lock(&smq_mtx); - if ((unsigned_val(SEQ_TRACE_T_FLAGS(token)) & SEQ_TRACE_TIMESTAMP) == 0) { + if (!ts_type) { mess = TUPLE3(hp, am_seq_trace, label, mess); - seq_trace_send_to_port(NULL, seq_tracer, mess, NIL); + seq_trace_send_to_port(NULL, seq_tracer, mess); } else { - Uint ms,s,us,ts; - GET_NOW(&ms, &s, &us); - ts = TUPLE3(hp, make_small(ms),make_small(s), make_small(us)); - hp += 4; - mess = TUPLE4(hp, am_seq_trace, label, mess, ts); - seq_trace_send_to_port(process, seq_tracer, mess, ts); + mess = TUPLE4(hp, am_seq_trace, label, mess, + NIL /* Will be overwritten by timestamp */); + hp += 5; + hp[-1] = write_ts(ts_type, hp, NULL, NULL); + seq_trace_send_to_port(process, seq_tracer, mess); } UnUseTmpHeapNoproc(LOCAL_HEAP_SIZE); #undef LOCAL_HEAP_SIZE @@ -1162,8 +1366,7 @@ seq_trace_output_generic(Eterm token, Eterm msg, Uint type, sz_lastcnt_serial = 3; /* TUPLE2 */ sz_msg = size_object(msg); - sz_ts = ((unsigned_val(SEQ_TRACE_T_FLAGS(token)) & SEQ_TRACE_TIMESTAMP) ? - 5 : 0); + sz_ts = patch_ts_size(ts_type); if (exitfrom != NIL) { sz_exit = 4; /* create {'EXIT',exitfrom,msg} */ sz_exitfrom = size_object(exitfrom); @@ -1207,25 +1410,28 @@ seq_trace_output_generic(Eterm token, Eterm msg, Uint type, erts_smp_mtx_lock(&smq_mtx); - if (sz_ts) {/* timestamp should be included */ - Uint ms,s,us,ts; - GET_NOW(&ms, &s, &us); - ts = TUPLE3(hp, make_small(ms),make_small(s), make_small(us)); - hp += 4; - mess = TUPLE4(hp, am_seq_trace, label, mess, ts); - } else { + if (!ts_type) mess = TUPLE3(hp, am_seq_trace, label, mess); + else { + mess = TUPLE4(hp, am_seq_trace, label, mess, + NIL /* Will be overwritten by timestamp */); + hp += 5; + /* Write timestamp in element 6 of the 'msg' tuple */ + hp[-1] = write_ts(ts_type, hp, bp, +#ifndef ERTS_SMP + tracer +#else + NULL +#endif + ); } #ifdef ERTS_SMP enqueue_sys_msg_unlocked(SYS_MSG_TYPE_SEQTRACE, NIL, NIL, mess, bp); erts_smp_mtx_unlock(&smq_mtx); #else - erts_queue_message(tracer, NULL, bp, mess, NIL -#ifdef USE_VM_PROBES - , NIL -#endif - ); /* trace_token must be NIL here */ + /* trace_token must be NIL here */ + erts_queue_message(tracer, NULL, bp, mess, NIL); #endif } } @@ -1236,7 +1442,7 @@ seq_trace_output_generic(Eterm token, Eterm msg, Uint type, void erts_trace_return_to(Process *p, BeamInstr *pc) { -#define LOCAL_HEAP_SIZE (4+5+5) +#define LOCAL_HEAP_SIZE (4+5+ERTS_TRACE_PATCH_TS_MAX_SIZE) Eterm* hp; Eterm mfa; Eterm mess; @@ -1261,9 +1467,7 @@ erts_trace_return_to(Process *p, BeamInstr *pc) erts_smp_mtx_lock(&smq_mtx); - if (ERTS_TRACE_FLAGS(p) & F_TIMESTAMP) { - hp = patch_ts(mess, hp); - } + PATCH_TS(TFLGS_TS_TYPE(p), mess, hp, NULL, NULL); if (is_internal_port(ERTS_TRACER_PROC(p))) { send_to_port(p, mess, &ERTS_TRACER_PROC(p), &ERTS_TRACE_FLAGS(p)); @@ -1310,6 +1514,7 @@ erts_trace_return(Process* p, BeamInstr* fi, Eterm retval, Eterm *tracer_pid) Eterm mod, name; int arity; Uint meta_flags, *tracee_flags; + int ts_type; #ifdef ERTS_SMP Eterm tracee; #endif @@ -1345,7 +1550,7 @@ erts_trace_return(Process* p, BeamInstr* fi, Eterm retval, Eterm *tracer_pid) * meta trace => * use fixed flag set instead of process flags */ - meta_flags = F_TRACE_CALLS | F_TIMESTAMP; + meta_flags = F_TRACE_CALLS | F_NOW_TS; tracee_flags = &meta_flags; #ifdef ERTS_SMP tracee = NIL; @@ -1359,8 +1564,10 @@ erts_trace_return(Process* p, BeamInstr* fi, Eterm retval, Eterm *tracer_pid) name = fi[1]; arity = fi[2]; + ts_type = ERTS_TFLGS2TSTYPE(*tracee_flags); + if (is_internal_port(*tracer_pid)) { -#define LOCAL_HEAP_SIZE (4+6+5) +#define LOCAL_HEAP_SIZE (4+6+ERTS_TRACE_PATCH_TS_MAX_SIZE) DeclareTmpHeapNoproc(local_heap,LOCAL_HEAP_SIZE); UseTmpHeapNoproc(LOCAL_HEAP_SIZE); hp = local_heap; @@ -1369,9 +1576,7 @@ erts_trace_return(Process* p, BeamInstr* fi, Eterm retval, Eterm *tracer_pid) mess = TUPLE5(hp, am_trace, p->common.id, am_return_from, mfa, retval); hp += 6; erts_smp_mtx_lock(&smq_mtx); - if (*tracee_flags & F_TIMESTAMP) { - hp = patch_ts(mess, hp); - } + PATCH_TS(ts_type, mess, hp, NULL, NULL); send_to_port(p, mess, tracer_pid, tracee_flags); UnUseTmpHeapNoproc(LOCAL_HEAP_SIZE); #undef LOCAL_HEAP_SIZE @@ -1382,24 +1587,15 @@ erts_trace_return(Process* p, BeamInstr* fi, Eterm retval, Eterm *tracer_pid) ERTS_TRACER_REF_TYPE tracer_ref; unsigned size; unsigned retval_size; -#ifdef DEBUG - Eterm* limit; -#endif ASSERT(is_internal_pid(*tracer_pid)); ERTS_GET_TRACER_REF(tracer_ref, *tracer_pid, *tracee_flags); - + retval_size = size_object(retval); - size = 6 + 4 + retval_size; - if (*tracee_flags & F_TIMESTAMP) { - size += 1+4; - } + size = 6 + 4 + retval_size + patch_ts_size(ts_type); hp = ERTS_ALLOC_SYSMSG_HEAP(size, &bp, &off_heap, tracer_ref); -#ifdef DEBUG - limit = hp + size; -#endif /* * Build the trace tuple and put it into receive queue of the tracer process. @@ -1413,11 +1609,7 @@ erts_trace_return(Process* p, BeamInstr* fi, Eterm retval, Eterm *tracer_pid) erts_smp_mtx_lock(&smq_mtx); - if (*tracee_flags & F_TIMESTAMP) { - hp = patch_ts(mess, hp); - } - - ASSERT(hp == limit); + PATCH_TS(ts_type, mess, hp, bp, tracer_ref); ERTS_ENQ_TRACE_MSG(tracee, tracer_ref, mess, bp); erts_smp_mtx_unlock(&smq_mtx); @@ -1440,6 +1632,7 @@ erts_trace_exception(Process* p, BeamInstr mfa[3], Eterm class, Eterm value, Eterm cv; Eterm mess; Uint meta_flags, *tracee_flags; + int ts_type; #ifdef ERTS_SMP Eterm tracee; #endif @@ -1478,15 +1671,17 @@ erts_trace_exception(Process* p, BeamInstr mfa[3], Eterm class, Eterm value, * meta trace => * use fixed flag set instead of process flags */ - meta_flags = F_TRACE_CALLS | F_TIMESTAMP; + meta_flags = F_TRACE_CALLS | F_NOW_TS; tracee_flags = &meta_flags; #ifdef ERTS_SMP tracee = NIL; #endif } + ts_type = ERTS_TFLGS2TSTYPE(*tracee_flags); + if (is_internal_port(*tracer_pid)) { -#define LOCAL_HEAP_SIZE (4+3+6+5) +#define LOCAL_HEAP_SIZE (4+3+6+ERTS_TRACE_PATCH_TS_MAX_SIZE) DeclareTmpHeapNoproc(local_heap,LOCAL_HEAP_SIZE); UseTmpHeapNoproc(LOCAL_HEAP_SIZE); @@ -1499,10 +1694,7 @@ erts_trace_exception(Process* p, BeamInstr mfa[3], Eterm class, Eterm value, hp += 6; ASSERT((hp - local_heap) <= LOCAL_HEAP_SIZE); erts_smp_mtx_lock(&smq_mtx); - if (*tracee_flags & F_TIMESTAMP) { - hp = patch_ts(mess, hp); /* hp += 5 */ - ASSERT((hp - local_heap) == LOCAL_HEAP_SIZE); - } + PATCH_TS(ts_type, mess, hp, NULL, NULL); send_to_port(p, mess, tracer_pid, tracee_flags); UnUseTmpHeapNoproc(LOCAL_HEAP_SIZE); #undef LOCAL_HEAP_SIZE @@ -1513,24 +1705,15 @@ erts_trace_exception(Process* p, BeamInstr mfa[3], Eterm class, Eterm value, ERTS_TRACER_REF_TYPE tracer_ref; unsigned size; unsigned value_size; -#ifdef DEBUG - Eterm* limit; -#endif ASSERT(is_internal_pid(*tracer_pid)); ERTS_GET_TRACER_REF(tracer_ref, *tracer_pid, *tracee_flags); value_size = size_object(value); - size = 6 + 4 + 3 + value_size; - if (*tracee_flags & F_TIMESTAMP) { - size += 1+4; - } + size = 6 + 4 + 3 + value_size + patch_ts_size(ts_type); hp = ERTS_ALLOC_SYSMSG_HEAP(size, &bp, &off_heap, tracer_ref); -#ifdef DEBUG - limit = hp + size; -#endif /* * Build the trace tuple and put it into receive queue of the tracer process. @@ -1547,11 +1730,7 @@ erts_trace_exception(Process* p, BeamInstr mfa[3], Eterm class, Eterm value, erts_smp_mtx_lock(&smq_mtx); - if (*tracee_flags & F_TIMESTAMP) { - hp = patch_ts(mess, hp); - } - - ASSERT(hp == limit); + PATCH_TS(ts_type, mess, hp, bp, tracer_ref); ERTS_ENQ_TRACE_MSG(tracee, tracer_ref, mess, bp); erts_smp_mtx_unlock(&smq_mtx); @@ -1584,6 +1763,7 @@ erts_call_trace(Process* p, BeamInstr mfa[3], Binary *match_spec, Eterm pam_result = am_true; Eterm mess; Uint meta_flags, *tracee_flags; + int ts_type; #ifdef ERTS_SMP Eterm tracee; #endif @@ -1625,7 +1805,7 @@ erts_call_trace(Process* p, BeamInstr mfa[3], Binary *match_spec, /* No trace messages for sensitive processes. */ return 0; } - meta_flags = F_TRACE_CALLS | F_TIMESTAMP; + meta_flags = F_TRACE_CALLS | F_NOW_TS; tracee_flags = &meta_flags; #ifdef ERTS_SMP tracee = NIL; @@ -1668,12 +1848,14 @@ erts_call_trace(Process* p, BeamInstr mfa[3], Binary *match_spec, } args = transformed_args; + ts_type = ERTS_TFLGS2TSTYPE(*tracee_flags); + if (is_internal_port(*tracer_pid)) { #if HEAP_ON_C_STACK - Eterm local_heap[64+MAX_ARG]; + Eterm local_heap[60+ERTS_TRACE_PATCH_TS_MAX_SIZE+MAX_ARG]; #else Eterm *local_heap = erts_alloc(ERTS_ALC_T_TEMP_TERM, - sizeof(Eterm)*(64+MAX_ARG)); + sizeof(Eterm)*(60+ERTS_TRACE_PATCH_TS_MAX_SIZE+MAX_ARG)); #endif hp = local_heap; @@ -1788,9 +1970,7 @@ erts_call_trace(Process* p, BeamInstr mfa[3], Binary *match_spec, *hp++ = pam_result; } erts_smp_mtx_lock(&smq_mtx); - if (*tracee_flags & F_TIMESTAMP) { - hp = patch_ts(mess, hp); - } + PATCH_TS(ts_type, mess, hp, NULL, NULL); send_to_port(p, mess, tracer_pid, tracee_flags); erts_smp_mtx_unlock(&smq_mtx); erts_match_set_release_result(p); @@ -1812,9 +1992,6 @@ erts_call_trace(Process* p, BeamInstr mfa[3], Binary *match_spec, unsigned sizes[MAX_ARG]; unsigned pam_result_size = 0; int invalid_tracer; -#ifdef DEBUG - Eterm* limit; -#endif ASSERT(is_internal_pid(*tracer_pid)); @@ -1907,10 +2084,7 @@ erts_call_trace(Process* p, BeamInstr mfa[3], Binary *match_spec, size += sizes[i]; } } - if (*tracee_flags & F_TIMESTAMP) { - size += 1 + 4; - /* One element in trace tuple + timestamp tuple. */ - } + size += patch_ts_size(ts_type); if (pam_result != am_true) { pam_result_size = size_object(pam_result); size += 1 + pam_result_size; @@ -1918,9 +2092,6 @@ erts_call_trace(Process* p, BeamInstr mfa[3], Binary *match_spec, } hp = ERTS_ALLOC_SYSMSG_HEAP(size, &bp, &off_heap, tracer_ref); -#ifdef DEBUG - limit = hp + size; -#endif /* * Build the the {M,F,A} tuple in the message buffer. @@ -1963,11 +2134,7 @@ erts_call_trace(Process* p, BeamInstr mfa[3], Binary *match_spec, erts_smp_mtx_lock(&smq_mtx); - if (*tracee_flags & F_TIMESTAMP) { - hp = patch_ts(mess, hp); - } - - ASSERT(hp == limit); + PATCH_TS(ts_type, mess, hp, bp, tracer_ref); ERTS_ENQ_TRACE_MSG(tracee, tracer_ref, mess, bp); erts_smp_mtx_unlock(&smq_mtx); UnUseTmpHeap(ERL_SUB_BIN_SIZE,p); @@ -2002,9 +2169,7 @@ trace_proc(Process *c_p, Process *t_p, Eterm what, Eterm data) mess = TUPLE4(hp, am_trace, t_p->common.id, what, data); hp += 5; erts_smp_mtx_lock(&smq_mtx); - if (ERTS_TRACE_FLAGS(t_p) & F_TIMESTAMP) { - hp = patch_ts(mess, hp); - } + PATCH_TS(TFLGS_TS_TYPE(t_p), mess, hp, NULL, NULL); send_to_port( #ifndef ERTS_SMP /* No fake schedule out and in again after an exit */ @@ -2034,7 +2199,7 @@ trace_proc(Process *c_p, Process *t_p, Eterm what, Eterm data) sz_data = size_object(data); - need = sz_data + 5 + TS_SIZE(t_p); + need = sz_data + 5 + PATCH_TS_SIZE(t_p); hp = ERTS_ALLOC_SYSMSG_HEAP(need, &bp, &off_heap, tracer_ref); @@ -2044,9 +2209,7 @@ trace_proc(Process *c_p, Process *t_p, Eterm what, Eterm data) erts_smp_mtx_lock(&smq_mtx); - if (ERTS_TRACE_FLAGS(t_p) & F_TIMESTAMP) { - hp = patch_ts(mess, hp); - } + PATCH_TS(TFLGS_TS_TYPE(t_p), mess, hp, bp, tracer_ref); ERTS_ENQ_TRACE_MSG(t_p->common.id, tracer_ref, mess, bp); erts_smp_mtx_unlock(&smq_mtx); @@ -2080,9 +2243,7 @@ trace_proc_spawn(Process *p, Eterm pid, mess = TUPLE5(hp, am_trace, p->common.id, am_spawn, pid, mfa); hp += 6; erts_smp_mtx_lock(&smq_mtx); - if (ERTS_TRACE_FLAGS(p) & F_TIMESTAMP) { - hp = patch_ts(mess, hp); - } + PATCH_TS(TFLGS_TS_TYPE(p), mess, hp, NULL, NULL); send_to_port(p, mess, &ERTS_TRACER_PROC(p), &ERTS_TRACE_FLAGS(p)); UnUseTmpHeapNoproc(LOCAL_HEAP_SIZE); #undef LOCAL_HEAP_SIZE @@ -2103,7 +2264,7 @@ trace_proc_spawn(Process *p, Eterm pid, sz_args = size_object(args); sz_pid = size_object(pid); - need = sz_args + 4 + 6 + TS_SIZE(p); + need = sz_args + 4 + 6 + PATCH_TS_SIZE(p); hp = ERTS_ALLOC_SYSMSG_HEAP(need, &bp, &off_heap, tracer_ref); @@ -2116,9 +2277,7 @@ trace_proc_spawn(Process *p, Eterm pid, erts_smp_mtx_lock(&smq_mtx); - if (ERTS_TRACE_FLAGS(p) & F_TIMESTAMP) { - hp = patch_ts(mess, hp); - } + PATCH_TS(TFLGS_TS_TYPE(p), mess, hp, bp, tracer_ref); ERTS_ENQ_TRACE_MSG(p->common.id, tracer_ref, mess, bp); erts_smp_mtx_unlock(&smq_mtx); @@ -2184,7 +2343,7 @@ trace_gc(Process *p, Eterm what) AM_bin_old_vheap_block_size }; - Uint values[] = { + UWord values[] = { OLD_HEAP(p) ? OLD_HEND(p) - OLD_HEAP(p) : 0, HEAP_SIZE(p), MBUF_SIZE(p), @@ -2198,15 +2357,12 @@ trace_gc(Process *p, Eterm what) BIN_OLD_VHEAP_SZ(p) }; #define LOCAL_HEAP_SIZE \ - (sizeof(values)/sizeof(Eterm)) * \ + (sizeof(values)/sizeof(*values)) * \ (2/*cons*/ + 3/*2-tuple*/ + BIG_UINT_HEAP_SIZE) + \ - 5/*4-tuple */ + TS_HEAP_WORDS + 5/*4-tuple */ + ERTS_TRACE_PATCH_TS_MAX_SIZE DeclareTmpHeap(local_heap,LOCAL_HEAP_SIZE,p); -#ifdef DEBUG - Eterm* limit; -#endif - ASSERT(sizeof(values)/sizeof(Uint) == sizeof(tags)/sizeof(Eterm)); + ERTS_CT_ASSERT(sizeof(values)/sizeof(*values) == sizeof(tags)/sizeof(Eterm)); UseTmpHeap(LOCAL_HEAP_SIZE,p); @@ -2214,12 +2370,12 @@ trace_gc(Process *p, Eterm what) hp = local_heap; #ifdef DEBUG size = 0; - (void) erts_bld_atom_uint_2tup_list(NULL, + (void) erts_bld_atom_uword_2tup_list(NULL, &size, - sizeof(values)/sizeof(Uint), + sizeof(values)/sizeof(*values), tags, values); - size += 5/*4-tuple*/ + TS_SIZE(p); + size += 5/*4-tuple*/ + PATCH_TS_SIZE(p); #endif } else { ASSERT(is_internal_pid(ERTS_TRACER_PROC(p))); @@ -2229,24 +2385,21 @@ trace_gc(Process *p, Eterm what) ERTS_TRACE_FLAGS(p)); size = 0; - (void) erts_bld_atom_uint_2tup_list(NULL, + (void) erts_bld_atom_uword_2tup_list(NULL, &size, - sizeof(values)/sizeof(Uint), + sizeof(values)/sizeof(*values), tags, values); - size += 5/*4-tuple*/ + TS_SIZE(p); + size += 5/*4-tuple*/ + PATCH_TS_SIZE(p); hp = ERTS_ALLOC_SYSMSG_HEAP(size, &bp, &off_heap, tracer_ref); } -#ifdef DEBUG - limit = hp + size; ASSERT(size <= LOCAL_HEAP_SIZE); -#endif - msg = erts_bld_atom_uint_2tup_list(&hp, + msg = erts_bld_atom_uword_2tup_list(&hp, NULL, - sizeof(values)/sizeof(Uint), + sizeof(values)/sizeof(*values), tags, values); @@ -2255,20 +2408,139 @@ trace_gc(Process *p, Eterm what) erts_smp_mtx_lock(&smq_mtx); - if (ERTS_TRACE_FLAGS(p) & F_TIMESTAMP) { - hp = patch_ts(msg, hp); - } - ASSERT(hp == limit); - if (is_internal_port(ERTS_TRACER_PROC(p))) + if (is_internal_port(ERTS_TRACER_PROC(p))) { + PATCH_TS(TFLGS_TS_TYPE(p), msg, hp, NULL, NULL); send_to_port(p, msg, &ERTS_TRACER_PROC(p), &ERTS_TRACE_FLAGS(p)); - else + } + else { + PATCH_TS(TFLGS_TS_TYPE(p), msg, hp, bp, tracer_ref); ERTS_ENQ_TRACE_MSG(p->common.id, tracer_ref, msg, bp); + } erts_smp_mtx_unlock(&smq_mtx); UnUseTmpHeap(LOCAL_HEAP_SIZE,p); #undef LOCAL_HEAP_SIZE } +void +monitor_long_schedule_proc(Process *p, BeamInstr *in_fp, BeamInstr *out_fp, Uint time) +{ + ErlHeapFragment *bp; + ErlOffHeap *off_heap; +#ifndef ERTS_SMP + Process *monitor_p; +#endif + Uint hsz; + Eterm *hp, list, in_mfa = am_undefined, out_mfa = am_undefined; + Eterm in_tpl, out_tpl, tmo_tpl, tmo, msg; + +#ifndef ERTS_SMP + ASSERT(is_internal_pid(system_monitor)); + monitor_p = erts_proc_lookup(system_monitor); + if (!monitor_p || p == monitor_p) { + return; + } +#endif + /* + * Size: {monitor, pid, long_schedule, [{timeout, T}, {in, {M,F,A}},{out,{M,F,A}}]} -> + * 5 (top tuple of 4), (3 (elements) * 2 (cons)) + 3 (timeout tuple of 2) + size of Timeout + + * (2 * 3 (in/out tuple of 2)) + + * 0 (unknown) or 4 (MFA tuple of 3) + 0 (unknown) or 4 (MFA tuple of 3) + * = 20 + (in_fp != NULL) ? 4 : 0 + (out_fp != NULL) ? 4 : 0 + size of Timeout + */ + hsz = 20 + ((in_fp != NULL) ? 4 : 0) + ((out_fp != NULL) ? 4 : 0); + (void) erts_bld_uint(NULL, &hsz, time); + hp = ERTS_ALLOC_SYSMSG_HEAP(hsz, &bp, &off_heap, monitor_p); + tmo = erts_bld_uint(&hp, NULL, time); + if (in_fp != NULL) { + in_mfa = TUPLE3(hp,(Eterm) in_fp[0], (Eterm) in_fp[1], make_small(in_fp[2])); + hp +=4; + } + if (out_fp != NULL) { + out_mfa = TUPLE3(hp,(Eterm) out_fp[0], (Eterm) out_fp[1], make_small(out_fp[2])); + hp +=4; + } + tmo_tpl = TUPLE2(hp,am_timeout, tmo); + hp += 3; + in_tpl = TUPLE2(hp,am_in,in_mfa); + hp += 3; + out_tpl = TUPLE2(hp,am_out,out_mfa); + hp += 3; + list = CONS(hp,out_tpl,NIL); + hp += 2; + list = CONS(hp,in_tpl,list); + hp += 2; + list = CONS(hp,tmo_tpl,list); + hp += 2; + msg = TUPLE4(hp, am_monitor, p->common.id, am_long_schedule, list); + hp += 5; +#ifdef ERTS_SMP + enqueue_sys_msg(SYS_MSG_TYPE_SYSMON, p->common.id, NIL, msg, bp); +#else + erts_queue_message(monitor_p, NULL, bp, msg, NIL); +#endif +} +void +monitor_long_schedule_port(Port *pp, ErtsPortTaskType type, Uint time) +{ + ErlHeapFragment *bp; + ErlOffHeap *off_heap; +#ifndef ERTS_SMP + Process *monitor_p; +#endif + Uint hsz; + Eterm *hp, list, op; + Eterm op_tpl, tmo_tpl, tmo, msg; + + +#ifndef ERTS_SMP + ASSERT(is_internal_pid(system_monitor)); + monitor_p = erts_proc_lookup(system_monitor); + if (!monitor_p) { + return; + } +#endif + /* + * Size: {monitor, port, long_schedule, [{timeout, T}, {op, Operation}]} -> + * 5 (top tuple of 4), (2 (elements) * 2 (cons)) + 3 (timeout tuple of 2) + * + size of Timeout + 3 (op tuple of 2 atoms) + * = 15 + size of Timeout + */ + hsz = 15; + (void) erts_bld_uint(NULL, &hsz, time); + + hp = ERTS_ALLOC_SYSMSG_HEAP(hsz, &bp, &off_heap, monitor_p); + + switch (type) { + case ERTS_PORT_TASK_PROC_SIG: op = am_proc_sig; break; + case ERTS_PORT_TASK_TIMEOUT: op = am_timeout; break; + case ERTS_PORT_TASK_INPUT: op = am_input; break; + case ERTS_PORT_TASK_OUTPUT: op = am_output; break; + case ERTS_PORT_TASK_EVENT: op = am_event; break; + case ERTS_PORT_TASK_DIST_CMD: op = am_dist_cmd; break; + default: op = am_undefined; break; + } + + tmo = erts_bld_uint(&hp, NULL, time); + + op_tpl = TUPLE2(hp,am_port_op,op); + hp += 3; + + tmo_tpl = TUPLE2(hp,am_timeout, tmo); + hp += 3; + + list = CONS(hp,op_tpl,NIL); + hp += 2; + list = CONS(hp,tmo_tpl,list); + hp += 2; + msg = TUPLE4(hp, am_monitor, pp->common.id, am_long_schedule, list); + hp += 5; +#ifdef ERTS_SMP + enqueue_sys_msg(SYS_MSG_TYPE_SYSMON, pp->common.id, NIL, msg, bp); +#else + erts_queue_message(monitor_p, NULL, bp, msg, NIL); +#endif +} void monitor_long_gc(Process *p, Uint time) { @@ -2288,7 +2560,7 @@ monitor_long_gc(Process *p, Uint time) { am_old_heap_size, am_heap_size }; - Eterm values[] = { + UWord values[] = { time, OLD_HEAP(p) ? OLD_HEND(p) - OLD_HEAP(p) : 0, HEAP_SIZE(p), @@ -2309,9 +2581,9 @@ monitor_long_gc(Process *p, Uint time) { #endif hsz = 0; - (void) erts_bld_atom_uint_2tup_list(NULL, + (void) erts_bld_atom_uword_2tup_list(NULL, &hsz, - sizeof(values)/sizeof(Uint), + sizeof(values)/sizeof(*values), tags, values); hsz += 5 /* 4-tuple */; @@ -2322,9 +2594,9 @@ monitor_long_gc(Process *p, Uint time) { hp_end = hp + hsz; #endif - list = erts_bld_atom_uint_2tup_list(&hp, + list = erts_bld_atom_uword_2tup_list(&hp, NULL, - sizeof(values)/sizeof(Uint), + sizeof(values)/sizeof(*values), tags, values); msg = TUPLE4(hp, am_monitor, p->common.id, am_long_gc, list); @@ -2337,11 +2609,7 @@ monitor_long_gc(Process *p, Uint time) { #ifdef ERTS_SMP enqueue_sys_msg(SYS_MSG_TYPE_SYSMON, p->common.id, NIL, msg, bp); #else - erts_queue_message(monitor_p, NULL, bp, msg, NIL -#ifdef USE_VM_PROBES - , NIL -#endif - ); + erts_queue_message(monitor_p, NULL, bp, msg, NIL); #endif } @@ -2362,7 +2630,7 @@ monitor_large_heap(Process *p) { am_old_heap_size, am_heap_size }; - Uint values[] = { + UWord values[] = { OLD_HEAP(p) ? OLD_HEND(p) - OLD_HEAP(p) : 0, HEAP_SIZE(p), MBUF_SIZE(p), @@ -2378,15 +2646,15 @@ monitor_large_heap(Process *p) { #ifndef ERTS_SMP ASSERT(is_internal_pid(system_monitor)); monitor_p = erts_proc_lookup(system_monitor); - if (monitor_p || p == monitor_p) { + if (!monitor_p || p == monitor_p) { return; } #endif hsz = 0; - (void) erts_bld_atom_uint_2tup_list(NULL, + (void) erts_bld_atom_uword_2tup_list(NULL, &hsz, - sizeof(values)/sizeof(Uint), + sizeof(values)/sizeof(*values), tags, values); hsz += 5 /* 4-tuple */; @@ -2397,9 +2665,9 @@ monitor_large_heap(Process *p) { hp_end = hp + hsz; #endif - list = erts_bld_atom_uint_2tup_list(&hp, + list = erts_bld_atom_uword_2tup_list(&hp, NULL, - sizeof(values)/sizeof(Uint), + sizeof(values)/sizeof(*values), tags, values); msg = TUPLE4(hp, am_monitor, p->common.id, am_large_heap, list); @@ -2412,11 +2680,7 @@ monitor_large_heap(Process *p) { #ifdef ERTS_SMP enqueue_sys_msg(SYS_MSG_TYPE_SYSMON, p->common.id, NIL, msg, bp); #else - erts_queue_message(monitor_p, NULL, bp, msg, NIL -#ifdef USE_VM_PROBES - , NIL -#endif - ); + erts_queue_message(monitor_p, NULL, bp, msg, NIL); #endif } @@ -2444,11 +2708,7 @@ monitor_generic(Process *p, Eterm type, Eterm spec) { #ifdef ERTS_SMP enqueue_sys_msg(SYS_MSG_TYPE_SYSMON, p->common.id, NIL, msg, bp); #else - erts_queue_message(monitor_p, NULL, bp, msg, NIL -#ifdef USE_VM_PROBES - , NIL -#endif - ); + erts_queue_message(monitor_p, NULL, bp, msg, NIL); #endif } @@ -2459,19 +2719,18 @@ monitor_generic(Process *p, Eterm type, Eterm spec) { void profile_scheduler(Eterm scheduler_id, Eterm state) { - Eterm *hp, msg, timestamp; - Uint Ms, s, us; + Eterm *hp, msg; + ErlHeapFragment *bp = NULL; #ifndef ERTS_SMP -#define LOCAL_HEAP_SIZE (4 + 7) +#define LOCAL_HEAP_SIZE (7 + ERTS_TRACE_PATCH_TS_MAX_SIZE) DeclareTmpHeapNoproc(local_heap,LOCAL_HEAP_SIZE); UseTmpHeapNoproc(LOCAL_HEAP_SIZE); hp = local_heap; #else - ErlHeapFragment *bp; Uint hsz; - hsz = 4 + 7; + hsz = 7 + patch_ts_size(erts_system_profile_ts_type)-1; bp = new_message_buffer(hsz); hp = bp->mem; @@ -2491,10 +2750,13 @@ profile_scheduler(Eterm scheduler_id, Eterm state) { break; } - GET_NOW(&Ms, &s, &us); - timestamp = TUPLE3(hp, make_small(Ms), make_small(s), make_small(us)); hp += 4; - msg = TUPLE6(hp, am_profile, am_scheduler, scheduler_id, state, - make_small(active_sched), timestamp); hp += 7; + msg = TUPLE6(hp, am_profile, am_scheduler, scheduler_id, + state, make_small(active_sched), + NIL /* Will be overwritten by timestamp */); + hp += 7; + + /* Write timestamp in element 6 of the 'msg' tuple */ + hp[-1] = write_ts(erts_system_profile_ts_type, hp, bp, NULL); #ifndef ERTS_SMP profile_send(NIL, msg); @@ -2565,7 +2827,7 @@ trace_port_open(Port *p, Eterm calling_pid, Eterm drv_name) { Eterm* hp; if (is_internal_port(ERTS_TRACER_PROC(p))) { -#define LOCAL_HEAP_SIZE (5+6) +#define LOCAL_HEAP_SIZE (6+ERTS_TRACE_PATCH_TS_MAX_SIZE) DeclareTmpHeapNoproc(local_heap,LOCAL_HEAP_SIZE); UseTmpHeapNoproc(LOCAL_HEAP_SIZE); @@ -2574,9 +2836,7 @@ trace_port_open(Port *p, Eterm calling_pid, Eterm drv_name) { mess = TUPLE5(hp, am_trace, calling_pid, am_open, p->common.id, drv_name); hp += 6; erts_smp_mtx_lock(&smq_mtx); - if (ERTS_TRACE_FLAGS(p) & F_TIMESTAMP) { - hp = patch_ts(mess, hp); - } + PATCH_TS(TFLGS_TS_TYPE(p), mess, hp, NULL, NULL); /* No fake schedule */ send_to_port(NULL, mess, &ERTS_TRACER_PROC(p), &ERTS_TRACE_FLAGS(p)); UnUseTmpHeapNoproc(LOCAL_HEAP_SIZE); @@ -2590,7 +2850,7 @@ trace_port_open(Port *p, Eterm calling_pid, Eterm drv_name) { ASSERT(is_internal_pid(ERTS_TRACER_PROC(p))); - sz_data = 6 + TS_SIZE(p); + sz_data = 6 + PATCH_TS_SIZE(p); ERTS_GET_TRACER_REF(tracer_ref, ERTS_TRACER_PROC(p), @@ -2603,9 +2863,7 @@ trace_port_open(Port *p, Eterm calling_pid, Eterm drv_name) { erts_smp_mtx_lock(&smq_mtx); - if (ERTS_TRACE_FLAGS(p) & F_TIMESTAMP) { - hp = patch_ts(mess, hp); - } + PATCH_TS(TFLGS_TS_TYPE(p), mess, hp, bp, tracer_ref); ERTS_ENQ_TRACE_MSG(p->common.id, tracer_ref, mess, bp); erts_smp_mtx_unlock(&smq_mtx); @@ -2629,7 +2887,7 @@ trace_port(Port *t_p, Eterm what, Eterm data) { || erts_thr_progress_is_blocking()); if (is_internal_port(ERTS_TRACER_PROC(t_p))) { -#define LOCAL_HEAP_SIZE (5+5) +#define LOCAL_HEAP_SIZE (5+ERTS_TRACE_PATCH_TS_MAX_SIZE) DeclareTmpHeapNoproc(local_heap,LOCAL_HEAP_SIZE); UseTmpHeapNoproc(LOCAL_HEAP_SIZE); @@ -2637,9 +2895,7 @@ trace_port(Port *t_p, Eterm what, Eterm data) { mess = TUPLE4(hp, am_trace, t_p->common.id, what, data); hp += 5; erts_smp_mtx_lock(&smq_mtx); - if (ERTS_TRACE_FLAGS(t_p) & F_TIMESTAMP) { - hp = patch_ts(mess, hp); - } + PATCH_TS(TFLGS_TS_TYPE(t_p), mess, hp, NULL, NULL); /* No fake schedule */ send_to_port(NULL,mess,&ERTS_TRACER_PROC(t_p),&ERTS_TRACE_FLAGS(t_p)); UnUseTmpHeapNoproc(LOCAL_HEAP_SIZE); @@ -2653,7 +2909,7 @@ trace_port(Port *t_p, Eterm what, Eterm data) { ASSERT(is_internal_pid(ERTS_TRACER_PROC(t_p))); - sz_data = 5 + TS_SIZE(t_p); + sz_data = 5 + PATCH_TS_SIZE(t_p); ERTS_GET_TRACER_REF(tracer_ref, ERTS_TRACER_PROC(t_p), @@ -2666,9 +2922,7 @@ trace_port(Port *t_p, Eterm what, Eterm data) { erts_smp_mtx_lock(&smq_mtx); - if (ERTS_TRACE_FLAGS(t_p) & F_TIMESTAMP) { - hp = patch_ts(mess, hp); - } + PATCH_TS(TFLGS_TS_TYPE(t_p), mess, hp, bp, tracer_ref); ERTS_ENQ_TRACE_MSG(t_p->common.id, tracer_ref, mess, bp); erts_smp_mtx_unlock(&smq_mtx); @@ -2696,7 +2950,7 @@ trace_sched_ports_where(Port *p, Eterm what, Eterm where) { Eterm sched_id = am_undefined; if (is_internal_port(ERTS_TRACER_PROC(p))) { -#define LOCAL_HEAP_SIZE (5+6) +#define LOCAL_HEAP_SIZE (6+ERTS_TRACE_PATCH_TS_MAX_SIZE) DeclareTmpHeapNoproc(local_heap,LOCAL_HEAP_SIZE); UseTmpHeapNoproc(LOCAL_HEAP_SIZE); @@ -2719,9 +2973,8 @@ trace_sched_ports_where(Port *p, Eterm what, Eterm where) { hp += ws; erts_smp_mtx_lock(&smq_mtx); - if (ERTS_TRACE_FLAGS(p) & F_TIMESTAMP) { - hp = patch_ts(mess, hp); - } + + PATCH_TS(TFLGS_TS_TYPE(p), mess, hp, NULL, NULL); /* No fake scheduling */ send_to_port(NULL, mess, &ERTS_TRACER_PROC(p), &ERTS_TRACE_FLAGS(p)); @@ -2741,7 +2994,7 @@ trace_sched_ports_where(Port *p, Eterm what, Eterm where) { ERTS_TRACER_PROC(p), ERTS_TRACE_FLAGS(p)); - hp = ERTS_ALLOC_SYSMSG_HEAP(ws+TS_SIZE(p), &bp, &off_heap, tracer_ref); + hp = ERTS_ALLOC_SYSMSG_HEAP(ws+PATCH_TS_SIZE(p), &bp, &off_heap, tracer_ref); if (IS_TRACED_FL(p, F_TRACE_SCHED_NO)) { #ifdef ERTS_SMP @@ -2759,10 +3012,7 @@ trace_sched_ports_where(Port *p, Eterm what, Eterm where) { erts_smp_mtx_lock(&smq_mtx); - if (ERTS_TRACE_FLAGS(p) & F_TIMESTAMP) { - hp = patch_ts(mess, hp); - } - + PATCH_TS(TFLGS_TS_TYPE(p), mess, hp, bp, tracer_ref); ERTS_ENQ_TRACE_MSG(p->common.id, tracer_ref, mess, bp); erts_smp_mtx_unlock(&smq_mtx); } @@ -2772,13 +3022,12 @@ trace_sched_ports_where(Port *p, Eterm what, Eterm where) { void profile_runnable_port(Port *p, Eterm status) { - Uint Ms, s, us; - Eterm *hp, msg, timestamp; - + Eterm *hp, msg; + ErlHeapFragment *bp = NULL; Eterm count = make_small(0); #ifndef ERTS_SMP -#define LOCAL_HEAP_SIZE (4 + 6) +#define LOCAL_HEAP_SIZE (6 + ERTS_TRACE_PATCH_TS_MAX_SIZE) DeclareTmpHeapNoproc(local_heap,LOCAL_HEAP_SIZE); UseTmpHeapNoproc(LOCAL_HEAP_SIZE); @@ -2786,10 +3035,9 @@ profile_runnable_port(Port *p, Eterm status) { hp = local_heap; #else - ErlHeapFragment *bp; Uint hsz; - hsz = 4 + 6; + hsz = 6 + patch_ts_size(erts_system_profile_ts_type)-1; bp = new_message_buffer(hsz); hp = bp->mem; @@ -2797,9 +3045,12 @@ profile_runnable_port(Port *p, Eterm status) { erts_smp_mtx_lock(&smq_mtx); - GET_NOW(&Ms, &s, &us); - timestamp = TUPLE3(hp, make_small(Ms), make_small(s), make_small(us)); hp += 4; - msg = TUPLE5(hp, am_profile, p->common.id, status, count, timestamp); hp += 6; + msg = TUPLE5(hp, am_profile, p->common.id, status, count, + NIL /* Will be overwritten by timestamp */); + hp += 6; + + /* Write timestamp in element 5 of the 'msg' tuple */ + hp[-1] = write_ts(erts_system_profile_ts_type, hp, bp, NULL); #ifndef ERTS_SMP profile_send(p->common.id, msg); @@ -2814,20 +3065,19 @@ profile_runnable_port(Port *p, Eterm status) { /* Process profiling */ void profile_runnable_proc(Process *p, Eterm status){ - Uint Ms, s, us; - Eterm *hp, msg, timestamp; + Eterm *hp, msg; Eterm where = am_undefined; + ErlHeapFragment *bp = NULL; #ifndef ERTS_SMP -#define LOCAL_HEAP_SIZE (4 + 6 + 4) +#define LOCAL_HEAP_SIZE (4 + 6 + ERTS_TRACE_PATCH_TS_MAX_SIZE) DeclareTmpHeapNoproc(local_heap,LOCAL_HEAP_SIZE); UseTmpHeapNoproc(LOCAL_HEAP_SIZE); hp = local_heap; #else - ErlHeapFragment *bp; - Uint hsz = 4 + 6 + 4; + Uint hsz = 4 + 6 + patch_ts_size(erts_system_profile_ts_type)-1; #endif if (!p->current) { @@ -2836,7 +3086,7 @@ profile_runnable_proc(Process *p, Eterm status){ #ifdef ERTS_SMP if (!p->current) { - hsz = 4 + 6; + hsz -= 4; } bp = new_message_buffer(hsz); @@ -2851,9 +3101,13 @@ profile_runnable_proc(Process *p, Eterm status){ erts_smp_mtx_lock(&smq_mtx); - GET_NOW(&Ms, &s, &us); - timestamp = TUPLE3(hp, make_small(Ms), make_small(s), make_small(us)); hp += 4; - msg = TUPLE5(hp, am_profile, p->common.id, status, where, timestamp); hp += 6; + msg = TUPLE5(hp, am_profile, p->common.id, status, where, + NIL /* Will be overwritten by timestamp */); + hp += 6; + + /* Write timestamp in element 5 of the 'msg' tuple */ + hp[-1] = write_ts(erts_system_profile_ts_type, hp, bp, NULL); + #ifndef ERTS_SMP profile_send(p->common.id, msg); UnUseTmpHeapNoproc(LOCAL_HEAP_SIZE); @@ -3011,6 +3265,7 @@ sys_msg_disp_failure(ErtsSysMsgQ *smqp, Eterm receiver) case SYS_MSG_TYPE_SYSMON: if (receiver == NIL && !erts_system_monitor_long_gc + && !erts_system_monitor_long_schedule && !erts_system_monitor_large_heap && !erts_system_monitor_flags.busy_port && !erts_system_monitor_flags.busy_dist_port) @@ -3181,6 +3436,8 @@ sys_msg_dispatcher_func(void *unused) if (erts_thr_progress_update(NULL)) erts_thr_progress_leader_update(NULL); + ERTS_SCHED_FAIR_YIELD(); + #ifdef DEBUG_PRINTOUTS print_msg_type(smqp); #endif @@ -3240,11 +3497,7 @@ sys_msg_dispatcher_func(void *unused) } else { queue_proc_msg: - erts_queue_message(proc,&proc_locks,smqp->bp,smqp->msg,NIL -#ifdef USE_VM_PROBES - , NIL -#endif - ); + erts_queue_message(proc,&proc_locks,smqp->bp,smqp->msg,NIL); #ifdef DEBUG_PRINTOUTS erts_fprintf(stderr, "delivered\n"); #endif @@ -3339,12 +3592,17 @@ static void init_sys_msg_dispatcher(void) { erts_smp_thr_opts_t thr_opts = ERTS_SMP_THR_OPTS_DEFAULT_INITER; +#ifdef __OSE__ + thr_opts.coreNo = 0; +#endif thr_opts.detached = 1; + thr_opts.name = "sys_msg_dispatcher"; init_smq_element_alloc(); sys_message_queue = NULL; sys_message_queue_end = NULL; erts_smp_cnd_init(&smq_cnd); erts_smp_mtx_init(&smq_mtx, "sys_msg_q"); + erts_smp_thr_create(&sys_msg_dispatcher_tid, sys_msg_dispatcher_func, NULL, diff --git a/erts/emulator/beam/erl_trace.h b/erts/emulator/beam/erl_trace.h index 50fb27aab0..a0058264d7 100644 --- a/erts/emulator/beam/erl_trace.h +++ b/erts/emulator/beam/erl_trace.h @@ -1,24 +1,55 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2012. All Rights Reserved. + * Copyright Ericsson AB 2012-2013. All Rights Reserved. * - * The contents of this file are subject to the Erlang Public License, - * Version 1.1, (the "License"); you may not use this file except in - * compliance with the License. You should have received a copy of the - * Erlang Public License along with this software. If not, it can be - * retrieved online at http://www.erlang.org/. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Software distributed under the License is distributed on an "AS IS" - * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See - * the License for the specific language governing rights and limitations - * under the License. + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. * * %CopyrightEnd% */ +#ifndef ERL_TRACE_H__FLAGS__ +#define ERL_TRACE_H__FLAGS__ +/* + * NOTE! The bits used for these flags matter. The flag with + * the least significant bit will take precedence! + * + * The "now timestamp" has highest precedence due to + * compatibility reasons. + */ +#define ERTS_TRACE_FLG_NOW_TIMESTAMP (1 << 0) +#define ERTS_TRACE_FLG_STRICT_MONOTONIC_TIMESTAMP (1 << 1) +#define ERTS_TRACE_FLG_MONOTONIC_TIMESTAMP (1 << 2) + +/* + * The bits used effects trace flags (of processes and ports) + * as well as sequential trace flags. If changed make sure + * these arn't messed up... + */ +#define ERTS_TRACE_TS_TYPE_BITS 3 +#define ERTS_TRACE_TS_TYPE_MASK \ + ((1 << ERTS_TRACE_TS_TYPE_BITS) - 1) + +#define ERTS_TFLGS2TSTYPE(TFLGS) \ + ((int) (((TFLGS) >> ERTS_TRACE_FLAGS_TS_TYPE_SHIFT) \ + & ERTS_TRACE_TS_TYPE_MASK)) +#define ERTS_SEQTFLGS2TSTYPE(SEQTFLGS) \ + ((int) (((SEQTFLGS) >> ERTS_SEQ_TRACE_FLAGS_TS_TYPE_SHIFT) \ + & ERTS_TRACE_TS_TYPE_MASK)) + +#endif /* ERL_TRACE_H__FLAGS__ */ -#ifndef ERL_TRACE_H__ +#if !defined(ERL_TRACE_H__) && !defined(ERTS_ONLY_INCLUDE_TRACE_FLAGS) #define ERL_TRACE_H__ struct binary; @@ -39,6 +70,7 @@ void erts_change_default_tracing(int setflags, Uint *flagsp, Eterm *tracerp); void erts_get_default_tracing(Uint *flagsp, Eterm *tracerp); void erts_set_system_monitor(Eterm monitor); Eterm erts_get_system_monitor(void); +int erts_is_tracer_proc_valid(Process* p); #ifdef ERTS_SMP void erts_check_my_tracer_proc(Process *); @@ -83,6 +115,8 @@ void erts_system_profile_setup_active_schedulers(void); /* system_monitor */ void monitor_long_gc(Process *p, Uint time); +void monitor_long_schedule_proc(Process *p, BeamInstr *in_i, BeamInstr *out_i, Uint time); +void monitor_long_schedule_port(Port *pp, ErtsPortTaskType type, Uint time); void monitor_large_heap(Process *p); void monitor_generic(Process *p, Eterm type, Eterm spec); Uint erts_trace_flag2bit(Eterm flag); diff --git a/erts/emulator/beam/erl_unicode.c b/erts/emulator/beam/erl_unicode.c index fa53fd0937..36d85b0a22 100644 --- a/erts/emulator/beam/erl_unicode.c +++ b/erts/emulator/beam/erl_unicode.c @@ -3,16 +3,17 @@ * * Copyright Ericsson AB 2008-2013. All Rights Reserved. * - * The contents of this file are subject to the Erlang Public License, - * Version 1.1, (the "License"); you may not use this file except in - * compliance with the License. You should have received a copy of the - * Erlang Public License along with this software. If not, it can be - * retrieved online at http://www.erlang.org/. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Software distributed under the License is distributed on an "AS IS" - * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See - * the License for the specific language governing rights and limitations - * under the License. + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. * * %CopyrightEnd% */ @@ -724,7 +725,7 @@ L_Again: /* Restart with sublist, old listend was pushed on stack */ hp = HAlloc(p, 2); obj = CDR(objp); ioterm = CONS(hp, rest_term, obj); - //(*left) = 0; + /* (*left) = 0; */ goto done; } if (rest_term != NIL) { @@ -1476,6 +1477,9 @@ static Eterm do_utf8_to_list_normalize(Process *p, Uint num, byte *bytes, Uint s Uint16 savepoints[4]; int numpoints = 0; + if (num == 0) + return NIL; + ASSERT(num > 0); hp = HAlloc(p,num * 2); /* May be to much */ @@ -1723,14 +1727,14 @@ static BIF_RETTYPE do_bif_utf8_to_list(Process *p, if (b_sz) { ErlSubBin *sb; Eterm orig; - ERTS_DECLARE_DUMMY(Uint offset); + Uint offset; ASSERT(state != ERTS_UTF8_OK); hp = HAlloc(p, ERL_SUB_BIN_SIZE); sb = (ErlSubBin *) hp; ERTS_GET_REAL_BIN(orig_bin, orig, offset, bitoffs, bitsize); sb->thing_word = HEADER_SUB_BIN; sb->size = b_sz; - sb->offs = num_bytes_to_process + num_processed_bytes; + sb->offs = offset + num_bytes_to_process + num_processed_bytes; sb->orig = orig; sb->bitoffs = bitoffs; sb->bitsize = bitsize; @@ -1981,9 +1985,21 @@ BIF_RETTYPE binary_to_existing_atom_2(BIF_ALIST_2) * string routines, that will certainly fail on some OS. */ -char *erts_convert_filename_to_native(Eterm name, char *statbuf, size_t statbuf_size, ErtsAlcType_t alloc_type, int allow_empty, int allow_atom, Sint *used) +char *erts_convert_filename_to_native(Eterm name, char *statbuf, size_t statbuf_size, + ErtsAlcType_t alloc_type, int allow_empty, + int allow_atom, Sint *used) { int encoding = erts_get_native_filename_encoding(); + return erts_convert_filename_to_encoding(name, statbuf, statbuf_size, alloc_type, + allow_empty, allow_atom, encoding, + used, 0); +} + +char *erts_convert_filename_to_encoding(Eterm name, char *statbuf, size_t statbuf_size, + ErtsAlcType_t alloc_type, int allow_empty, + int allow_atom, int encoding, Sint *used, + Uint extra) +{ char* name_buf = NULL; if ((allow_atom && is_atom(name)) || @@ -1995,13 +2011,14 @@ char *erts_convert_filename_to_native(Eterm name, char *statbuf, size_t statbuf_ } if (encoding == ERL_FILENAME_WIN_WCHAR) { need += 2; + extra *= 2; } else { ++need; } if (used) *used = (Sint) need; - if (need > statbuf_size) { - name_buf = (char *) erts_alloc(alloc_type, need); + if (need+extra > statbuf_size) { + name_buf = (char *) erts_alloc(alloc_type, need+extra); } else { name_buf = statbuf; } @@ -2013,52 +2030,27 @@ char *erts_convert_filename_to_native(Eterm name, char *statbuf, size_t statbuf_ } else if (is_binary(name)) { byte *temp_alloc = NULL; byte *bytes; - byte *err_pos; - Uint size,num_chars; + Uint size; size = binary_size(name); bytes = erts_get_aligned_binary_bytes(name, &temp_alloc); + if (encoding != ERL_FILENAME_WIN_WCHAR) { /*Add 0 termination only*/ if (used) *used = (Sint) size+1; - if (size+1 > statbuf_size) { - name_buf = (char *) erts_alloc(alloc_type, size+1); + if (size+1+extra > statbuf_size) { + name_buf = (char *) erts_alloc(alloc_type, size+1+extra); } else { name_buf = statbuf; } memcpy(name_buf,bytes,size); name_buf[size]=0; - } else if (erts_analyze_utf8(bytes,size,&err_pos,&num_chars,NULL) != ERTS_UTF8_OK || - erts_get_user_requested_filename_encoding() == ERL_FILENAME_LATIN1) { - byte *p; - /* What to do now? Maybe latin1, so just take byte for byte instead */ - if (used) - *used = (Sint) (size+1)*2; - if ((size+1)*2 > statbuf_size) { - name_buf = (char *) erts_alloc(alloc_type, (size+1)*2); - } else { - name_buf = statbuf; - } - p = (byte *) name_buf; - while (size--) { - *p++ = *bytes++; - *p++ = 0; - } - *p++ = 0; - *p++ = 0; - } else { /* WIN_WCHAR and valid UTF8 */ - if (used) - *used = (Sint) (num_chars+1)*2; - if ((num_chars+1)*2 > statbuf_size) { - name_buf = (char *) erts_alloc(alloc_type, (num_chars+1)*2); - } else { - name_buf = statbuf; - } - erts_copy_utf8_to_utf16_little((byte *) name_buf, bytes, num_chars); - name_buf[num_chars*2] = 0; - name_buf[num_chars*2+1] = 0; - } + } else { + name_buf = erts_convert_filename_to_wchar(bytes, size, + statbuf, statbuf_size, + alloc_type, used, extra); + } erts_free_aligned_binary_bytes(temp_alloc); } else { return NULL; @@ -2066,6 +2058,50 @@ char *erts_convert_filename_to_native(Eterm name, char *statbuf, size_t statbuf_ return name_buf; } +char* erts_convert_filename_to_wchar(byte* bytes, Uint size, + char *statbuf, size_t statbuf_size, + ErtsAlcType_t alloc_type, Sint* used, + Uint extra_wchars) +{ + byte *err_pos; + Uint num_chars; + char* name_buf = NULL; + Sint need; + char *p; + + if (erts_analyze_utf8(bytes,size,&err_pos,&num_chars,NULL) != ERTS_UTF8_OK || + erts_get_user_requested_filename_encoding() == ERL_FILENAME_LATIN1) { + + /* What to do now? Maybe latin1, so just take byte for byte instead */ + need = (Sint) (size + extra_wchars + 1) * 2; + if (need > statbuf_size) { + name_buf = (char *) erts_alloc(alloc_type, need); + } else { + name_buf = statbuf; + } + p = name_buf; + while (size--) { + *p++ = *bytes++; + *p++ = 0; + } + } else { /* WIN_WCHAR and valid UTF8 */ + need = (Sint) (num_chars + extra_wchars + 1) * 2; + if (need > statbuf_size) { + name_buf = (char *) erts_alloc(alloc_type, need); + } else { + name_buf = statbuf; + } + erts_copy_utf8_to_utf16_little((byte *) name_buf, bytes, num_chars); + p = name_buf + num_chars*2; + } + *p++ = 0; + *p++ = 0; + if (used) + *used = p - name_buf; + return name_buf; +} + + static int filename_len_16bit(byte *str) { byte *p = str; @@ -2091,6 +2127,8 @@ Eterm erts_convert_native_to_filename(Process *p, byte *bytes) mac = 1; case ERL_FILENAME_UTF8: size = strlen((char *) bytes); + if (size == 0) + return NIL; if (erts_analyze_utf8(bytes,size,&err_pos,&num_chars,NULL) != ERTS_UTF8_OK) { goto noconvert; } @@ -2145,16 +2183,31 @@ Sint erts_native_filename_need(Eterm ioterm, int encoding) ap = atom_tab(atom_val(ioterm)); switch (encoding) { case ERL_FILENAME_LATIN1: - need = ap->len; + need = ap->latin1_chars; /* May be -1 */ break; case ERL_FILENAME_UTF8_MAC: case ERL_FILENAME_UTF8: - for (i = 0; i < ap->len; i++) { - need += (ap->name[i] >= 0x80) ? 2 : 1; - } + need = ap->len; break; case ERL_FILENAME_WIN_WCHAR: - need = 2*(ap->len); + if (ap->latin1_chars >= 0) { + need = 2* ap->latin1_chars; + } + else { + for (i = 0; i < ap->len; ) { + if (ap->name[i] < 0x80) { + i++; + } else if (ap->name[i] < 0xE0) { + i += 2; + } else if (ap->name[i] < 0xF0) { + i += 3; + } else { + need = -1; + break; + } + need += 2; + } + } break; default: need = -1; @@ -2284,26 +2337,36 @@ void erts_native_filename_put(Eterm ioterm, int encoding, byte *p) switch (encoding) { case ERL_FILENAME_LATIN1: for (i = 0; i < ap->len; i++) { - *p++ = ap->name[i]; - } - break; - case ERL_FILENAME_UTF8_MAC: - case ERL_FILENAME_UTF8: - for (i = 0; i < ap->len; i++) { - if(ap->name[i] < 0x80) { + if (ap->name[i] < 0x80) { *p++ = ap->name[i]; } else { - *p++ = (((ap->name[i]) >> 6) | ((byte) 0xC0)); - *p++ = (((ap->name[i]) & 0x3F) | ((byte) 0x80)); + ASSERT(ap->name[i] < 0xC4); + *p++ = ((ap->name[i] & 3) << 6) | (ap->name[i+1] & 0x3F); + i++; } } break; + case ERL_FILENAME_UTF8_MAC: + case ERL_FILENAME_UTF8: + sys_memcpy(p, ap->name, ap->len); + break; case ERL_FILENAME_WIN_WCHAR: for (i = 0; i < ap->len; i++) { /* Little endian */ - *p++ = ap->name[i]; - *p++ = 0; - } + if (ap->name[i] < 0x80) { + *p++ = ap->name[i]; + *p++ = 0; + } else if (ap->name[i] < 0xE0) { + *p++ = ((ap->name[i] & 3) << 6) | (ap->name[i+1] & 0x3F); + *p++ = ((ap->name[i] & 0x1C) >> 2); + i++; + } else { + ASSERT(ap->name[i] < 0xF0); + *p++ = ((ap->name[i+1] & 3) << 6) | (ap->name[i+2] & 0x3C); + *p++ = ((ap->name[i] & 0xF) << 4) | ((ap->name[i+1] & 0x3C) >> 2); + i += 2; + } + } break; default: ASSERT(0); @@ -2444,7 +2507,7 @@ void erts_copy_utf8_to_utf16_little(byte *target, byte *bytes, int num_chars) ((Uint) (bytes[3] & ((byte) 0x3F))); bytes += 4; } else { - erl_exit(1,"Internal unicode error in prim_file:internal_name2native/1"); + erts_exit(ERTS_ERROR_EXIT,"Internal unicode error in prim_file:internal_name2native/1"); } *target++ = (byte) (unipoint & 0xFF); *target++ = (byte) ((unipoint >> 8) & 0xFF); diff --git a/erts/emulator/beam/erl_unicode.h b/erts/emulator/beam/erl_unicode.h index 1b63b797c2..4c25d89b7c 100644 --- a/erts/emulator/beam/erl_unicode.h +++ b/erts/emulator/beam/erl_unicode.h @@ -3,16 +3,17 @@ * * Copyright Ericsson AB 2008-2009. All Rights Reserved. * - * The contents of this file are subject to the Erlang Public License, - * Version 1.1, (the "License"); you may not use this file except in - * compliance with the License. You should have received a copy of the - * Erlang Public License along with this software. If not, it can be - * retrieved online at http://www.erlang.org/. - * - * Software distributed under the License is distributed on an "AS IS" - * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See - * the License for the specific language governing rights and limitations - * under the License. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. * * %CopyrightEnd% */ diff --git a/erts/emulator/beam/erl_unicode_normalize.h b/erts/emulator/beam/erl_unicode_normalize.h index fb0a111ca2..16c62db50e 100644 --- a/erts/emulator/beam/erl_unicode_normalize.h +++ b/erts/emulator/beam/erl_unicode_normalize.h @@ -1,21 +1,22 @@ /* -* %CopyrightBegin% -* -* Copyright Ericsson AB 1999-2010. All Rights Reserved. -* -* The contents of this file are subject to the Erlang Public License, -* Version 1.1, (the "License"); you may not use this file except in -* compliance with the License. You should have received a copy of the -* Erlang Public License along with this software. If not, it can be -* retrieved online at http://www.erlang.org/. -* -* Software distributed under the License is distributed on an "AS IS" -* basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See -* the License for the specific language governing rights and limitations -* under the License. -* -* %CopyrightEnd% -*/ + * %CopyrightBegin% + * + * Copyright Ericsson AB 1999-2010. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * %CopyrightEnd% + */ /* * This file is automatically generated by dec.erl, do not edit manually */ diff --git a/erts/emulator/beam/erl_utils.h b/erts/emulator/beam/erl_utils.h index a2064bd8a3..6dab3bf297 100644 --- a/erts/emulator/beam/erl_utils.h +++ b/erts/emulator/beam/erl_utils.h @@ -1,18 +1,19 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2012. All Rights Reserved. + * Copyright Ericsson AB 2012-2014. All Rights Reserved. * - * The contents of this file are subject to the Erlang Public License, - * Version 1.1, (the "License"); you may not use this file except in - * compliance with the License. You should have received a copy of the - * Erlang Public License along with this software. If not, it can be - * retrieved online at http://www.erlang.org/. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Software distributed under the License is distributed on an "AS IS" - * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See - * the License for the specific language governing rights and limitations - * under the License. + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. * * %CopyrightEnd% */ @@ -24,17 +25,15 @@ #include "erl_smp.h" #include "erl_printf.h" +struct process; + typedef struct { #ifdef DEBUG int smp_api; #endif union { Uint64 not_atomic; -#ifdef ARCH_64 - erts_atomic_t atomic; -#else - erts_dw_atomic_t atomic; -#endif + erts_atomic64_t atomic; } counter; } erts_interval_t; @@ -48,9 +47,6 @@ Uint64 erts_ensure_later_interval_nob(erts_interval_t *, Uint64); Uint64 erts_ensure_later_interval_acqb(erts_interval_t *, Uint64); Uint64 erts_smp_ensure_later_interval_nob(erts_interval_t *, Uint64); Uint64 erts_smp_ensure_later_interval_acqb(erts_interval_t *, Uint64); -#ifdef ARCH_32 -ERTS_GLB_INLINE Uint64 erts_interval_dw_aint_to_val__(erts_dw_aint_t *); -#endif ERTS_GLB_INLINE Uint64 erts_current_interval_nob__(erts_interval_t *); ERTS_GLB_INLINE Uint64 erts_current_interval_acqb__(erts_interval_t *); ERTS_GLB_INLINE Uint64 erts_current_interval_nob(erts_interval_t *); @@ -60,46 +56,16 @@ ERTS_GLB_INLINE Uint64 erts_smp_current_interval_acqb(erts_interval_t *); #if ERTS_GLB_INLINE_INCL_FUNC_DEF -#ifdef ARCH_32 - -ERTS_GLB_INLINE Uint64 -erts_interval_dw_aint_to_val__(erts_dw_aint_t *dw) -{ -#ifdef ETHR_SU_DW_NAINT_T__ - return (Uint64) dw->dw_sint; -#else - Uint64 res; - res = (Uint64) ((Uint32) dw->sint[ERTS_DW_AINT_HIGH_WORD]); - res <<= 32; - res |= (Uint64) ((Uint32) dw->sint[ERTS_DW_AINT_LOW_WORD]); - return res; -#endif -} - -#endif - ERTS_GLB_INLINE Uint64 erts_current_interval_nob__(erts_interval_t *icp) { -#ifdef ARCH_64 - return (Uint64) erts_atomic_read_nob(&icp->counter.atomic); -#else - erts_dw_aint_t dw; - erts_dw_atomic_read_nob(&icp->counter.atomic, &dw); - return erts_interval_dw_aint_to_val__(&dw); -#endif + return (Uint64) erts_atomic64_read_nob(&icp->counter.atomic); } ERTS_GLB_INLINE Uint64 erts_current_interval_acqb__(erts_interval_t *icp) { -#ifdef ARCH_64 - return (Uint64) erts_atomic_read_acqb(&icp->counter.atomic); -#else - erts_dw_aint_t dw; - erts_dw_atomic_read_acqb(&icp->counter.atomic, &dw); - return erts_interval_dw_aint_to_val__(&dw); -#endif + return (Uint64) erts_atomic64_read_acqb(&icp->counter.atomic); } ERTS_GLB_INLINE Uint64 @@ -148,13 +114,18 @@ void erts_silence_warn_unused_result(long unused); int erts_fit_in_bits_int64(Sint64); int erts_fit_in_bits_int32(Sint32); -int list_length(Eterm); +int erts_fit_in_bits_uint(Uint); +int erts_list_length(Eterm); int erts_is_builtin(Eterm, Eterm, int); Uint32 make_broken_hash(Eterm); Uint32 block_hash(byte *, unsigned, Uint32); Uint32 make_hash2(Eterm); Uint32 make_hash(Eterm); +Uint32 make_internal_hash(Eterm); +void erts_save_emu_args(int argc, char **argv); +Eterm erts_get_emu_args(struct process *c_p); +Eterm erts_get_ethread_info(struct process * c_p); Eterm erts_bld_atom(Uint **hpp, Uint *szp, char *str); Eterm erts_bld_uint(Uint **hpp, Uint *szp, Uint ui); @@ -163,6 +134,10 @@ Eterm erts_bld_uint64(Uint **hpp, Uint *szp, Uint64 ui64); Eterm erts_bld_sint64(Uint **hpp, Uint *szp, Sint64 si64); Eterm erts_bld_cons(Uint **hpp, Uint *szp, Eterm car, Eterm cdr); Eterm erts_bld_tuple(Uint **hpp, Uint *szp, Uint arity, ...); +#define erts_bld_tuple2(H,S,E1,E2) erts_bld_tuple(H,S,2,E1,E2) +#define erts_bld_tuple3(H,S,E1,E2,E3) erts_bld_tuple(H,S,3,E1,E2,E3) +#define erts_bld_tuple4(H,S,E1,E2,E3,E4) erts_bld_tuple(H,S,4,E1,E2,E3,E4) +#define erts_bld_tuple5(H,S,E1,E2,E3,E4,E5) erts_bld_tuple(H,S,5,E1,E2,E3,E4,E5) Eterm erts_bld_tuplev(Uint **hpp, Uint *szp, Uint arity, Eterm terms[]); Eterm erts_bld_string_n(Uint **hpp, Uint *szp, const char *str, Sint len); #define erts_bld_string(hpp,szp,str) erts_bld_string_n(hpp,szp,str,strlen(str)) @@ -170,8 +145,8 @@ Eterm erts_bld_list(Uint **hpp, Uint *szp, Sint length, Eterm terms[]); Eterm erts_bld_2tup_list(Uint **hpp, Uint *szp, Sint length, Eterm terms1[], Uint terms2[]); Eterm -erts_bld_atom_uint_2tup_list(Uint **hpp, Uint *szp, - Sint length, Eterm atoms[], Uint uints[]); +erts_bld_atom_uword_2tup_list(Uint **hpp, Uint *szp, + Sint length, Eterm atoms[], UWord uints[]); Eterm erts_bld_atom_2uint_3tup_list(Uint **hpp, Uint *szp, Sint length, Eterm atoms[], Uint uints1[], Uint uints2[]); @@ -193,23 +168,40 @@ int eq(Eterm, Eterm); #define EQ(x,y) (((x) == (y)) || (is_not_both_immed((x),(y)) && eq((x),(y)))) #if HALFWORD_HEAP -Sint cmp_rel(Eterm, Eterm*, Eterm, Eterm*); -#define CMP(A,B) cmp_rel(A,NULL,B,NULL) +Sint erts_cmp_rel_opt(Eterm, Eterm*, Eterm, Eterm*, int, int); +#define cmp_rel(A,A_BASE,B,B_BASE) erts_cmp_rel_opt(A,A_BASE,B,B_BASE,0,0) +#define cmp_rel_term(A,A_BASE,B,B_BASE) erts_cmp_rel_opt(A,A_BASE,B,B_BASE,1,0) +#define CMP(A,B) erts_cmp_rel_opt(A,NULL,B,NULL,0,0) +#define CMP_TERM(A,B) erts_cmp_rel_opt(A,NULL,B,NULL,1,0) +#define CMP_EQ_ONLY(A,B) erts_cmp_rel_opt(A,NULL,B,NULL,0,1) #else -Sint cmp(Eterm, Eterm); -#define cmp_rel(A,A_BASE,B,B_BASE) cmp(A,B) -#define CMP(A,B) cmp(A,B) +Sint erts_cmp(Eterm, Eterm, int, int); +Sint cmp(Eterm a, Eterm b); +#define cmp_rel(A,A_BASE,B,B_BASE) erts_cmp(A,B,0,0) +#define cmp_rel_term(A,A_BASE,B,B_BASE) erts_cmp(A,B,1,0) +#define CMP(A,B) erts_cmp(A,B,0,0) +#define CMP_TERM(A,B) erts_cmp(A,B,1,0) +#define CMP_EQ_ONLY(A,B) erts_cmp(A,B,0,1) #endif -#define cmp_lt(a,b) (CMP((a),(b)) < 0) -#define cmp_le(a,b) (CMP((a),(b)) <= 0) -#define cmp_eq(a,b) (CMP((a),(b)) == 0) -#define cmp_ne(a,b) (CMP((a),(b)) != 0) -#define cmp_ge(a,b) (CMP((a),(b)) >= 0) -#define cmp_gt(a,b) (CMP((a),(b)) > 0) - -#define CMP_LT(a,b) ((a) != (b) && cmp_lt((a),(b))) -#define CMP_GE(a,b) ((a) == (b) || cmp_ge((a),(b))) -#define CMP_EQ(a,b) ((a) == (b) || cmp_eq((a),(b))) -#define CMP_NE(a,b) ((a) != (b) && cmp_ne((a),(b))) + +#define cmp_lt(a,b) (CMP((a),(b)) < 0) +#define cmp_le(a,b) (CMP((a),(b)) <= 0) +#define cmp_eq(a,b) (CMP_EQ_ONLY((a),(b)) == 0) +#define cmp_ne(a,b) (CMP_EQ_ONLY((a),(b)) != 0) +#define cmp_ge(a,b) (CMP((a),(b)) >= 0) +#define cmp_gt(a,b) (CMP((a),(b)) > 0) + +#define cmp_lt_term(a,b) (CMP_TERM((a),(b)) < 0) +#define cmp_le_term(a,b) (CMP_TERM((a),(b)) <= 0) +#define cmp_ge_term(a,b) (CMP_TERM((a),(b)) >= 0) +#define cmp_gt_term(a,b) (CMP_TERM((a),(b)) > 0) + +#define CMP_LT(a,b) ((a) != (b) && cmp_lt((a),(b))) +#define CMP_GE(a,b) ((a) == (b) || cmp_ge((a),(b))) +#define CMP_EQ(a,b) ((a) == (b) || cmp_eq((a),(b))) +#define CMP_NE(a,b) ((a) != (b) && cmp_ne((a),(b))) + +#define CMP_LT_TERM(a,b) ((a) != (b) && cmp_lt_term((a),(b))) +#define CMP_GE_TERM(a,b) ((a) == (b) || cmp_ge_term((a),(b))) #endif diff --git a/erts/emulator/beam/erl_vm.h b/erts/emulator/beam/erl_vm.h index c962955de9..8948ca2ea9 100644 --- a/erts/emulator/beam/erl_vm.h +++ b/erts/emulator/beam/erl_vm.h @@ -1,18 +1,19 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 1996-2012. All Rights Reserved. + * Copyright Ericsson AB 1996-2014. All Rights Reserved. * - * The contents of this file are subject to the Erlang Public License, - * Version 1.1, (the "License"); you may not use this file except in - * compliance with the License. You should have received a copy of the - * Erlang Public License along with this software. If not, it can be - * retrieved online at http://www.erlang.org/. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Software distributed under the License is distributed on an "AS IS" - * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See - * the License for the specific language governing rights and limitations - * under the License. + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. * * %CopyrightEnd% */ @@ -20,8 +21,6 @@ #ifndef __ERL_VM_H__ #define __ERL_VM_H__ -/* #define ERTS_OPCODE_COUNTER_SUPPORT */ - /* FORCE_HEAP_FRAGS: * Debug provocation to make HAlloc always create heap fragments (if allowed) * even if there is room on heap. @@ -46,7 +45,6 @@ heap data on the C stack or if we use the buffers in the scheduler data. */ #define TMP_HEAP_SIZE 128 /* Number of Eterm in the schedulers small heap for transient heap data */ -#define CMP_TMP_HEAP_SIZE 32 /* cmp wants its own tmp-heap... */ #define ERL_ARITH_TMP_HEAP_SIZE 4 /* as does erl_arith... */ #define BEAM_EMU_TMP_HEAP_SIZE 2 /* and beam_emu... */ @@ -80,7 +78,7 @@ # ifdef CHECK_FOR_HOLES # define INIT_HEAP_MEM(p,sz) erts_set_hole_marker(HEAP_TOP(p), (sz)) # else -# define INIT_HEAP_MEM(p,sz) memset(HEAP_TOP(p),DEBUG_BAD_BYTE,(sz)*sizeof(Eterm*)) +# define INIT_HEAP_MEM(p,sz) memset(HEAP_TOP(p),0x01,(sz)*sizeof(Eterm*)) # endif #else # define INIT_HEAP_MEM(p,sz) ((void)0) @@ -98,7 +96,7 @@ * failing that, in a heap fragment. */ #define HAllocX(p, sz, xtra) \ - (ASSERT_EXPR((sz) >= 0), \ + (ASSERT((sz) >= 0), \ ErtsHAllocLockCheck(p), \ (IS_FORCE_HEAP_FRAGS || (((HEAP_LIMIT(p) - HEAP_TOP(p)) < (sz))) \ ? erts_heap_alloc((p),(sz),(xtra)) \ @@ -120,9 +118,9 @@ #if defined(DEBUG) || defined(CHECK_FOR_HOLES) #if HALFWORD_HEAP -# define ERTS_HOLE_MARKER (0xaf5e78ccU) +# define ERTS_HOLE_MARKER (0xdeadbeef) #else -# define ERTS_HOLE_MARKER (((0xaf5e78ccUL << 24) << 8) | 0xaf5e78ccUL) +# define ERTS_HOLE_MARKER (((0xdeadbeef << 24) << 8) | 0xdeadbeef) #endif #endif @@ -135,14 +133,14 @@ */ #ifdef CHECK_FOR_HOLES # define HeapOnlyAlloc(p, sz) \ - (ASSERT_EXPR((sz) >= 0), \ - (ASSERT_EXPR(((HEAP_LIMIT(p) - HEAP_TOP(p)) >= (sz))), \ + (ASSERT((sz) >= 0), \ + (ASSERT(((HEAP_LIMIT(p) - HEAP_TOP(p)) >= (sz))), \ (erts_set_hole_marker(HEAP_TOP(p), (sz)), \ (HEAP_TOP(p) = HEAP_TOP(p) + (sz), HEAP_TOP(p) - (sz))))) #else # define HeapOnlyAlloc(p, sz) \ - (ASSERT_EXPR((sz) >= 0), \ - (ASSERT_EXPR(((HEAP_LIMIT(p) - HEAP_TOP(p)) >= (sz))), \ + (ASSERT((sz) >= 0), \ + (ASSERT(((HEAP_LIMIT(p) - HEAP_TOP(p)) >= (sz))), \ (HEAP_TOP(p) = HEAP_TOP(p) + (sz), HEAP_TOP(p) - (sz)))) #endif @@ -175,6 +173,7 @@ extern int H_MIN_SIZE; /* minimum (heap + stack) */ extern int BIN_VH_MIN_SIZE; /* minimum virtual (bin) heap */ extern int erts_atom_table_size;/* Atom table size */ +extern int erts_pd_initial_size;/* Initial Process dictionary table size */ #define ORIG_CREATION 0 #define INTERNAL_CREATION 255 diff --git a/erts/emulator/beam/erl_zlib.c b/erts/emulator/beam/erl_zlib.c index f73d48b6c2..e44e86c39c 100644 --- a/erts/emulator/beam/erl_zlib.c +++ b/erts/emulator/beam/erl_zlib.c @@ -1,18 +1,19 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2009. All Rights Reserved. + * Copyright Ericsson AB 2009-2013. All Rights Reserved. * - * The contents of this file are subject to the Erlang Public License, - * Version 1.1, (the "License"); you may not use this file except in - * compliance with the License. You should have received a copy of the - * Erlang Public License along with this software. If not, it can be - * retrieved online at http://www.erlang.org/. - * - * Software distributed under the License is distributed on an "AS IS" - * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See - * the License for the specific language governing rights and limitations - * under the License. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. * * %CopyrightEnd% */ @@ -44,6 +45,88 @@ void erl_zlib_zfree_callback (voidpf opaque, voidpf ptr) erts_free(ERTS_ALC_T_ZLIB, ptr); } +/* + * Initialize a z_stream with a source, to later *chunk* data into a destination + * Returns Z_OK or Error. + */ +int ZEXPORT erl_zlib_deflate_start(z_stream *streamp, const Bytef* source, + uLong sourceLen, int level) +{ + streamp->next_in = (Bytef*)source; + streamp->avail_in = (uInt)sourceLen; + streamp->total_out = streamp->avail_out = 0; + streamp->next_out = NULL; + erl_zlib_alloc_init(streamp); + return deflateInit(streamp, level); +} +/* + * Deflate a chunk, The destination length is the limit. + * Returns Z_OK if more to process, Z_STREAM_END if we are done. + */ +int ZEXPORT erl_zlib_deflate_chunk(z_stream *streamp, Bytef* dest, uLongf* destLen) +{ + int err; + uLongf last_tot = streamp->total_out; + + streamp->next_out = dest; + streamp->avail_out = (uInt)*destLen; + + if ((uLong)streamp->avail_out != *destLen) return Z_BUF_ERROR; + + err = deflate(streamp, Z_FINISH); + *destLen = streamp->total_out - last_tot; + return err; +} + + +/* + * When we are done, free up the deflate structure + * Retyurns Z_OK or Error + */ +int ZEXPORT erl_zlib_deflate_finish(z_stream *streamp) +{ + return deflateEnd(streamp); +} + +int ZEXPORT erl_zlib_inflate_start(z_stream *streamp, const Bytef* source, + uLong sourceLen) +{ + streamp->next_in = (Bytef*)source; + streamp->avail_in = (uInt)sourceLen; + streamp->total_out = streamp->avail_out = 0; + streamp->next_out = NULL; + erl_zlib_alloc_init(streamp); + return inflateInit(streamp); +} +/* + * Inflate a chunk, The destination length is the limit. + * Returns Z_OK if more to process, Z_STREAM_END if we are done. + */ +int ZEXPORT erl_zlib_inflate_chunk(z_stream *streamp, Bytef* dest, uLongf* destLen) +{ + int err; + uLongf last_tot = streamp->total_out; + + streamp->next_out = dest; + streamp->avail_out = (uInt)*destLen; + + if ((uLong)streamp->avail_out != *destLen) return Z_BUF_ERROR; + + err = inflate(streamp, Z_NO_FLUSH); + ASSERT(err != Z_STREAM_ERROR); + *destLen = streamp->total_out - last_tot; + return err; +} + +/* + * When we are done, free up the inflate structure + * Retyurns Z_OK or Error + */ +int ZEXPORT erl_zlib_inflate_finish(z_stream *streamp) +{ + return inflateEnd(streamp); +} + int ZEXPORT erl_zlib_compress2 (Bytef* dest, uLongf* destLen, const Bytef* source, uLong sourceLen, diff --git a/erts/emulator/beam/erl_zlib.h b/erts/emulator/beam/erl_zlib.h index 9054a5e428..2ca1e3735e 100644 --- a/erts/emulator/beam/erl_zlib.h +++ b/erts/emulator/beam/erl_zlib.h @@ -1,18 +1,19 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2009. All Rights Reserved. + * Copyright Ericsson AB 2009-2013. All Rights Reserved. * - * The contents of this file are subject to the Erlang Public License, - * Version 1.1, (the "License"); you may not use this file except in - * compliance with the License. You should have received a copy of the - * Erlang Public License along with this software. If not, it can be - * retrieved online at http://www.erlang.org/. - * - * Software distributed under the License is distributed on an "AS IS" - * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See - * the License for the specific language governing rights and limitations - * under the License. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. * * %CopyrightEnd% */ @@ -31,6 +32,20 @@ (s)->zfree = erl_zlib_zfree_callback; \ } while (0) +/* + * Chunked interface, used by term_to_binary among others. + */ +int ZEXPORT erl_zlib_deflate_start(z_stream *streamp, const Bytef* source, + uLong sourceLen, int level); +int ZEXPORT erl_zlib_deflate_chunk(z_stream *streamp, Bytef* dest, uLongf* destLen); +int ZEXPORT erl_zlib_deflate_finish(z_stream *streamp); + +int ZEXPORT erl_zlib_inflate_start(z_stream *streamp, const Bytef* source, + uLong sourceLen); +int ZEXPORT erl_zlib_inflate_chunk(z_stream *streamp, Bytef* dest, uLongf* destLen); +int ZEXPORT erl_zlib_inflate_finish(z_stream *streamp); + + /* Use instead of compress */ #define erl_zlib_compress(dest,destLen,source,sourceLen) \ diff --git a/erts/emulator/beam/erlang_dtrace.d b/erts/emulator/beam/erlang_dtrace.d index e3ebbb84f4..c682f76e3a 100644 --- a/erts/emulator/beam/erlang_dtrace.d +++ b/erts/emulator/beam/erlang_dtrace.d @@ -4,16 +4,17 @@ * Copyright Dustin Sallings, Michal Ptaszek, Scott Lystig Fritchie 2011-2012. * All Rights Reserved. * - * The contents of this file are subject to the Erlang Public License, - * Version 1.1, (the "License"); you may not use this file except in - * compliance with the License. You should have received a copy of the - * Erlang Public License along with this software. If not, it can be - * retrieved online at http://www.erlang.org/. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Software distributed under the License is distributed on an "AS IS" - * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See - * the License for the specific language governing rights and limitations - * under the License. + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. * * %CopyrightEnd% */ diff --git a/erts/emulator/beam/error.h b/erts/emulator/beam/error.h index ddc2c1396d..cba8672c68 100644 --- a/erts/emulator/beam/error.h +++ b/erts/emulator/beam/error.h @@ -3,16 +3,17 @@ * * Copyright Ericsson AB 1996-2010. All Rights Reserved. * - * The contents of this file are subject to the Erlang Public License, - * Version 1.1, (the "License"); you may not use this file except in - * compliance with the License. You should have received a copy of the - * Erlang Public License along with this software. If not, it can be - * retrieved online at http://www.erlang.org/. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Software distributed under the License is distributed on an "AS IS" - * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See - * the License for the specific language governing rights and limitations - * under the License. + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. * * %CopyrightEnd% */ @@ -140,7 +141,13 @@ #define EXC_NOTSUP ((17 << 8) | EXC_ERROR) /* Not supported */ -#define NUMBER_EXIT_CODES 18 /* The number of exit code indices */ +#define EXC_BADMAP ((18 << 8) | EXC_ERROR) + /* Bad map */ + +#define EXC_BADKEY ((19 << 8) | EXC_ERROR) + /* Bad key in map */ + +#define NUMBER_EXIT_CODES 20 /* The number of exit code indices */ /* * Internal pseudo-error codes. @@ -152,6 +159,8 @@ */ #define BADARG EXC_BADARG #define BADARITH EXC_BADARITH +#define BADKEY EXC_BADKEY +#define BADMAP EXC_BADMAP #define BADMATCH EXC_BADMATCH #define SYSTEM_LIMIT EXC_SYSTEM_LIMIT diff --git a/erts/emulator/beam/export.c b/erts/emulator/beam/export.c index b0f08d8245..2420df36b5 100644 --- a/erts/emulator/beam/export.c +++ b/erts/emulator/beam/export.c @@ -3,16 +3,17 @@ * * Copyright Ericsson AB 1996-2013. All Rights Reserved. * - * The contents of this file are subject to the Erlang Public License, - * Version 1.1, (the "License"); you may not use this file except in - * compliance with the License. You should have received a copy of the - * Erlang Public License along with this software. If not, it can be - * retrieved online at http://www.erlang.org/. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Software distributed under the License is distributed on an "AS IS" - * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See - * the License for the specific language governing rights and limitations - * under the License. + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. * * %CopyrightEnd% */ diff --git a/erts/emulator/beam/export.h b/erts/emulator/beam/export.h index 61a54de59f..a8bc9d2f66 100644 --- a/erts/emulator/beam/export.h +++ b/erts/emulator/beam/export.h @@ -3,16 +3,17 @@ * * Copyright Ericsson AB 1996-2013. All Rights Reserved. * - * The contents of this file are subject to the Erlang Public License, - * Version 1.1, (the "License"); you may not use this file except in - * compliance with the License. You should have received a copy of the - * Erlang Public License along with this software. If not, it can be - * retrieved online at http://www.erlang.org/. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Software distributed under the License is distributed on an "AS IS" - * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See - * the License for the specific language governing rights and limitations - * under the License. + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. * * %CopyrightEnd% */ diff --git a/erts/emulator/beam/external.c b/erts/emulator/beam/external.c index 5ce0d97c74..ffe3303796 100644 --- a/erts/emulator/beam/external.c +++ b/erts/emulator/beam/external.c @@ -1,18 +1,19 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 1996-2013. All Rights Reserved. + * Copyright Ericsson AB 1996-2014. All Rights Reserved. * - * The contents of this file are subject to the Erlang Public License, - * Version 1.1, (the "License"); you may not use this file except in - * compliance with the License. You should have received a copy of the - * Erlang Public License along with this software. If not, it can be - * retrieved online at http://www.erlang.org/. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Software distributed under the License is distributed on an "AS IS" - * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See - * the License for the specific language governing rights and limitations - * under the License. + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. * * %CopyrightEnd% */ @@ -36,16 +37,16 @@ #include "erl_process.h" #include "error.h" #include "external.h" +#define ERL_WANT_HIPE_BIF_WRAPPER__ #include "bif.h" +#undef ERL_WANT_HIPE_BIF_WRAPPER__ #include "big.h" #include "dist.h" #include "erl_binary.h" #include "erl_bits.h" #include "erl_zlib.h" +#include "erl_map.h" -#ifdef HIPE -#include "hipe_mode_switch.h" -#endif #define in_area(ptr,start,nbytes) ((UWord)((char*)(ptr) - (char*)(start)) < (nbytes)) #define MAX_STRING_LEN 0xffff @@ -61,6 +62,9 @@ */ # define ERTS_DEBUG_USE_DIST_SEP # endif +# define IF_DEBUG(X) X +#else +# define IF_DEBUG(X) #endif /* Does Sint fit in Sint32? @@ -81,17 +85,45 @@ * */ +static Export term_to_binary_trap_export; + static byte* enc_term(ErtsAtomCacheMap *, Eterm, byte*, Uint32, struct erl_off_heap_header** off_heap); +struct TTBEncodeContext_; +static int enc_term_int(struct TTBEncodeContext_*,ErtsAtomCacheMap *acmp, Eterm obj, byte* ep, Uint32 dflags, + struct erl_off_heap_header** off_heap, Sint *reds, byte **res); static Uint is_external_string(Eterm obj, int* p_is_string); static byte* enc_atom(ErtsAtomCacheMap *, Eterm, byte*, Uint32); static byte* enc_pid(ErtsAtomCacheMap *, Eterm, byte*, Uint32); -static byte* dec_term(ErtsDistExternal *, Eterm**, byte*, ErlOffHeap*, Eterm*); +struct B2TContext_t; +static byte* dec_term(ErtsDistExternal*, ErtsHeapFactory*, byte*, Eterm*, struct B2TContext_t*); static byte* dec_atom(ErtsDistExternal *, byte*, Eterm*); -static byte* dec_pid(ErtsDistExternal *, Eterm**, byte*, ErlOffHeap*, Eterm*); -static Sint decoded_size(byte *ep, byte* endp, int internal_tags); +static byte* dec_pid(ErtsDistExternal *, ErtsHeapFactory*, byte*, Eterm*); +static Sint decoded_size(byte *ep, byte* endp, int internal_tags, struct B2TContext_t*); +static BIF_RETTYPE term_to_binary_trap_1(BIF_ALIST_1); +static Eterm erts_term_to_binary_int(Process* p, Eterm Term, int level, Uint flags, + Binary *context_b); static Uint encode_size_struct2(ErtsAtomCacheMap *, Eterm, unsigned); +struct TTBSizeContext_; +static int encode_size_struct_int(struct TTBSizeContext_*, ErtsAtomCacheMap *acmp, Eterm obj, + unsigned dflags, Sint *reds, Uint *res); + +static Export binary_to_term_trap_export; +static BIF_RETTYPE binary_to_term_trap_1(BIF_ALIST_1); +static BIF_RETTYPE binary_to_term_int(Process* p, Uint32 flags, Eterm bin, Binary* context_b, + Export *bif, Eterm arg0, Eterm arg1); + +void erts_init_external(void) { + erts_init_trap_export(&term_to_binary_trap_export, + am_erts_internal, am_term_to_binary_trap, 1, + &term_to_binary_trap_1); + + erts_init_trap_export(&binary_to_term_trap_export, + am_erts_internal, am_binary_to_term_trap, 1, + &binary_to_term_trap_1); + return; +} #define ERTS_MAX_INTERNAL_ATOM_CACHE_ENTRIES 255 @@ -469,15 +501,37 @@ byte *erts_encode_ext_dist_header_finalize(byte *ext, ErtsAtomCache *cache, Uint return ep; } -Uint erts_encode_dist_ext_size(Eterm term, Uint32 flags, ErtsAtomCacheMap *acmp) +int erts_encode_dist_ext_size(Eterm term, Uint32 flags, ErtsAtomCacheMap *acmp, + Uint* szp) { - Uint sz = 0; + Uint sz; + if (encode_size_struct_int(NULL, acmp, term, flags, NULL, &sz)) { + return -1; + } else { #ifndef ERTS_DEBUG_USE_DIST_SEP - if (!(flags & DFLAG_DIST_HDR_ATOM_CACHE)) + if (!(flags & DFLAG_DIST_HDR_ATOM_CACHE)) #endif - sz++ /* VERSION_MAGIC */; - sz += encode_size_struct2(acmp, term, flags); - return sz; + sz++ /* VERSION_MAGIC */; + + *szp += sz; + return 0; + } +} + +int erts_encode_dist_ext_size_int(Eterm term, struct erts_dsig_send_context* ctx, Uint* szp) +{ + Uint sz; + if (encode_size_struct_int(&ctx->u.sc, ctx->acmp, term, ctx->flags, &ctx->reds, &sz)) { + return -1; + } else { +#ifndef ERTS_DEBUG_USE_DIST_SEP + if (!(ctx->flags & DFLAG_DIST_HDR_ATOM_CACHE)) +#endif + sz++ /* VERSION_MAGIC */; + + *szp += sz; + return 0; + } } Uint erts_encode_ext_size(Eterm term) @@ -488,7 +542,7 @@ Uint erts_encode_ext_size(Eterm term) Uint erts_encode_ext_size_2(Eterm term, unsigned dflags) { - return encode_size_struct2(NULL, term, TERM_TO_BINARY_DFLAGS|dflags) + return encode_size_struct2(NULL, term, dflags) + 1 /* VERSION_MAGIC */; } @@ -498,19 +552,16 @@ Uint erts_encode_ext_size_ets(Eterm term) } -void erts_encode_dist_ext(Eterm term, byte **ext, Uint32 flags, ErtsAtomCacheMap *acmp) +int erts_encode_dist_ext(Eterm term, byte **ext, Uint32 flags, ErtsAtomCacheMap *acmp, + TTBEncodeContext* ctx, Sint* reds) { - byte *ep = *ext; -#ifndef ERTS_DEBUG_USE_DIST_SEP - if (!(flags & DFLAG_DIST_HDR_ATOM_CACHE)) -#endif - *ep++ = VERSION_MAGIC; - ep = enc_term(acmp, term, ep, flags, NULL); - if (!ep) - erl_exit(ERTS_ABORT_EXIT, - "%s:%d:erts_encode_dist_ext(): Internal data structure error\n", - __FILE__, __LINE__); - *ext = ep; + if (!ctx || !ctx->wstack.wstart) { + #ifndef ERTS_DEBUG_USE_DIST_SEP + if (!(flags & DFLAG_DIST_HDR_ATOM_CACHE)) + #endif + *(*ext)++ = VERSION_MAGIC; + } + return enc_term_int(ctx, acmp, term, *ext, flags, NULL, reds, ext); } void erts_encode_ext(Eterm term, byte **ext) @@ -519,7 +570,7 @@ void erts_encode_ext(Eterm term, byte **ext) *ep++ = VERSION_MAGIC; ep = enc_term(NULL, term, ep, TERM_TO_BINARY_DFLAGS, NULL); if (!ep) - erl_exit(ERTS_ABORT_EXIT, + erts_exit(ERTS_ABORT_EXIT, "%s:%d:erts_encode_ext(): Internal data structure error\n", __FILE__, __LINE__); *ext = ep; @@ -851,7 +902,7 @@ erts_decode_dist_ext_size(ErtsDistExternal *edep) goto fail; ep = edep->extp+1; } - res = decoded_size(ep, edep->ext_endp, 0); + res = decoded_size(ep, edep->ext_endp, 0, NULL); if (res >= 0) return res; fail: @@ -863,12 +914,12 @@ Sint erts_decode_ext_size(byte *ext, Uint size) { if (size == 0 || *ext != VERSION_MAGIC) return -1; - return decoded_size(ext+1, ext+size, 0); + return decoded_size(ext+1, ext+size, 0, NULL); } Sint erts_decode_ext_size_ets(byte *ext, Uint size) { - Sint sz = decoded_size(ext, ext+size, 1); + Sint sz = decoded_size(ext, ext+size, 1, NULL); ASSERT(sz >= 0); return sz; } @@ -880,8 +931,7 @@ Sint erts_decode_ext_size_ets(byte *ext, Uint size) ** on return hpp is updated to point after allocated data */ Eterm -erts_decode_dist_ext(Eterm** hpp, - ErlOffHeap* off_heap, +erts_decode_dist_ext(ErtsHeapFactory* factory, ErtsDistExternal *edep) { Eterm obj; @@ -901,7 +951,7 @@ erts_decode_dist_ext(Eterm** hpp, goto error; ep++; } - ep = dec_term(edep, hpp, ep, off_heap, &obj); + ep = dec_term(edep, factory, ep, &obj, NULL); if (!ep) goto error; @@ -910,19 +960,22 @@ erts_decode_dist_ext(Eterm** hpp, return obj; error: + erts_factory_undo(factory); bad_dist_ext(edep); return THE_NON_VALUE; } -Eterm erts_decode_ext(Eterm **hpp, ErlOffHeap *off_heap, byte **ext) +Eterm erts_decode_ext(ErtsHeapFactory* factory, byte **ext) { Eterm obj; byte *ep = *ext; - if (*ep++ != VERSION_MAGIC) + if (*ep++ != VERSION_MAGIC) { + erts_factory_undo(factory); return THE_NON_VALUE; - ep = dec_term(NULL, hpp, ep, off_heap, &obj); + } + ep = dec_term(NULL, factory, ep, &obj, NULL); if (!ep) { #ifdef DEBUG bin_write(ERTS_PRINT_STDERR,NULL,*ext,500); @@ -933,10 +986,10 @@ Eterm erts_decode_ext(Eterm **hpp, ErlOffHeap *off_heap, byte **ext) return obj; } -Eterm erts_decode_ext_ets(Eterm **hpp, ErlOffHeap *off_heap, byte *ext) +Eterm erts_decode_ext_ets(ErtsHeapFactory* factory, byte *ext) { Eterm obj; - ext = dec_term(NULL, hpp, ext, off_heap, &obj); + ext = dec_term(NULL, factory, ext, &obj, NULL); ASSERT(ext); return obj; } @@ -945,9 +998,8 @@ Eterm erts_decode_ext_ets(Eterm **hpp, ErlOffHeap *off_heap, byte *ext) BIF_RETTYPE erts_debug_dist_ext_to_term_2(BIF_ALIST_2) { + ErtsHeapFactory factory; Eterm res; - Eterm *hp; - Eterm *hendp; Sint hsz; ErtsDistExternal ede; Eterm *tp; @@ -994,12 +1046,9 @@ BIF_RETTYPE erts_debug_dist_ext_to_term_2(BIF_ALIST_2) if (hsz < 0) goto badarg; - hp = HAlloc(BIF_P, (Uint) hsz); - hendp = hp + hsz; - - res = erts_decode_dist_ext(&hp, &MSO(BIF_P), &ede); - - HRelease(BIF_P, hendp, hp); + erts_factory_proc_prealloc_init(&factory, BIF_P, hsz); + res = erts_decode_dist_ext(&factory, &ede); + erts_factory_close(&factory); if (is_value(res)) BIF_RET(res); @@ -1009,12 +1058,41 @@ BIF_RETTYPE erts_debug_dist_ext_to_term_2(BIF_ALIST_2) BIF_ERROR(BIF_P, BADARG); } +static BIF_RETTYPE term_to_binary_trap_1(BIF_ALIST_1) +{ + Eterm *tp = tuple_val(BIF_ARG_1); + Eterm Term = tp[1]; + Eterm bt = tp[2]; + Binary *bin = ((ProcBin *) binary_val(bt))->val; + Eterm res = erts_term_to_binary_int(BIF_P, Term, 0, 0,bin); + if (is_tuple(res)) { + ASSERT(BIF_P->flags & F_DISABLE_GC); + BIF_TRAP1(&term_to_binary_trap_export,BIF_P,res); + } else { + if (erts_set_gc_state(BIF_P, 1) + || MSO(BIF_P).overhead > BIN_VHEAP_SZ(BIF_P)) + ERTS_BIF_YIELD_RETURN(BIF_P, res); + else + BIF_RET(res); + } +} + +HIPE_WRAPPER_BIF_DISABLE_GC(term_to_binary, 1) BIF_RETTYPE term_to_binary_1(BIF_ALIST_1) { - return erts_term_to_binary(BIF_P, BIF_ARG_1, 0, TERM_TO_BINARY_DFLAGS); + Eterm res = erts_term_to_binary_int(BIF_P, BIF_ARG_1, 0, TERM_TO_BINARY_DFLAGS, NULL); + if (is_tuple(res)) { + erts_set_gc_state(BIF_P, 0); + BIF_TRAP1(&term_to_binary_trap_export,BIF_P,res); + } else { + ASSERT(!(BIF_P->flags & F_DISABLE_GC)); + BIF_RET(res); + } } +HIPE_WRAPPER_BIF_DISABLE_GC(term_to_binary, 2) + BIF_RETTYPE term_to_binary_2(BIF_ALIST_2) { Process* p = BIF_P; @@ -1022,6 +1100,7 @@ BIF_RETTYPE term_to_binary_2(BIF_ALIST_2) Eterm Flags = BIF_ARG_2; int level = 0; Uint flags = TERM_TO_BINARY_DFLAGS; + Eterm res; while (is_list(Flags)) { Eterm arg = CAR(list_val(Flags)); @@ -1032,10 +1111,10 @@ BIF_RETTYPE term_to_binary_2(BIF_ALIST_2) if (tp[1] == am_minor_version && is_small(tp[2])) { switch (signed_val(tp[2])) { case 0: - flags = TERM_TO_BINARY_DFLAGS; + flags = TERM_TO_BINARY_DFLAGS & ~DFLAG_NEW_FLOATS; break; case 1: - flags = TERM_TO_BINARY_DFLAGS|DFLAG_NEW_FLOATS; + flags = TERM_TO_BINARY_DFLAGS; break; default: goto error; @@ -1058,9 +1137,76 @@ BIF_RETTYPE term_to_binary_2(BIF_ALIST_2) goto error; } - return erts_term_to_binary(p, Term, level, flags); + res = erts_term_to_binary_int(p, Term, level, flags, NULL); + if (is_tuple(res)) { + erts_set_gc_state(p, 0); + BIF_TRAP1(&term_to_binary_trap_export,BIF_P,res); + } else { + ASSERT(!(BIF_P->flags & F_DISABLE_GC)); + BIF_RET(res); + } } + +enum B2TState { /* order is somewhat significant */ + B2TPrepare, + B2TUncompressChunk, + B2TSizeInit, + B2TSize, + B2TDecodeInit, + B2TDecode, + B2TDecodeList, + B2TDecodeTuple, + B2TDecodeString, + B2TDecodeBinary, + + B2TDone, + B2TDecodeFail, + B2TBadArg +}; + +typedef struct { + int heap_size; + int terms; + byte* ep; + int atom_extra_skip; +} B2TSizeContext; + +typedef struct { + byte* ep; + Eterm res; + Eterm* next; + ErtsHeapFactory factory; + int remaining_n; + char* remaining_bytes; + ErtsWStack flat_maps; + ErtsPStack hamt_array; +} B2TDecodeContext; + +typedef struct { + z_stream stream; + byte* dbytes; + Uint dleft; +} B2TUncompressContext; + +typedef struct B2TContext_t { + Sint heap_size; + byte* aligned_alloc; + ErtsBinary2TermState b2ts; + Uint32 flags; + SWord reds; + Eterm trap_bin; + Export *bif; + Eterm arg[2]; + enum B2TState state; + union { + B2TSizeContext sc; + B2TDecodeContext dc; + B2TUncompressContext uc; + } u; +} B2TContext; + + static uLongf binary2term_uncomp_size(byte* data, Sint size) { z_stream stream; @@ -1090,48 +1236,62 @@ static uLongf binary2term_uncomp_size(byte* data, Sint size) return err == Z_STREAM_END ? uncomp_size : 0; } -static ERTS_INLINE Sint -binary2term_prepare(ErtsBinary2TermState *state, byte *data, Sint data_size) +static ERTS_INLINE int +binary2term_prepare(ErtsBinary2TermState *state, byte *data, Sint data_size, + B2TContext* ctx) { - Sint res; byte *bytes = data; Sint size = data_size; state->exttmp = 0; if (size < 1 || *bytes != VERSION_MAGIC) { - error: - if (state->exttmp) - erts_free(ERTS_ALC_T_TMP, state->extp); - state->extp = NULL; - state->exttmp = 0; return -1; } bytes++; size--; if (size < 5 || *bytes != COMPRESSED) { state->extp = bytes; + if (ctx) + ctx->state = B2TSizeInit; } else { uLongf dest_len = (Uint32) get_int32(bytes+1); bytes += 5; size -= 5; if (dest_len > 32*1024*1024 - || (state->extp = erts_alloc_fnf(ERTS_ALC_T_TMP, dest_len)) == NULL) { + || (state->extp = erts_alloc_fnf(ERTS_ALC_T_EXT_TERM_DATA, dest_len)) == NULL) { + /* + * Try avoid out-of-memory crash due to corrupted 'dest_len' + * by checking the actual length of the uncompressed data. + * The only way to do that is to uncompress it. Sad but true. + */ if (dest_len != binary2term_uncomp_size(bytes, size)) { - goto error; + return -1; } - state->extp = erts_alloc(ERTS_ALC_T_TMP, dest_len); + state->extp = erts_alloc(ERTS_ALC_T_EXT_TERM_DATA, dest_len); + ctx->reds -= dest_len; } state->exttmp = 1; - if (erl_zlib_uncompress(state->extp, &dest_len, bytes, size) != Z_OK) - goto error; + if (ctx) { + if (erl_zlib_inflate_start(&ctx->u.uc.stream, bytes, size) != Z_OK) + return -1; + + ctx->u.uc.dbytes = state->extp; + ctx->u.uc.dleft = dest_len; + ctx->state = B2TUncompressChunk; + } + else { + uLongf dlen = dest_len; + if (erl_zlib_uncompress(state->extp, &dlen, bytes, size) != Z_OK + || dlen != dest_len) { + return -1; + } + } size = (Sint) dest_len; } - res = decoded_size(state->extp, state->extp + size, 0); - if (res < 0) - goto error; - return res; + state->extsize = size; + return 0; } static ERTS_INLINE void @@ -1139,19 +1299,21 @@ binary2term_abort(ErtsBinary2TermState *state) { if (state->exttmp) { state->exttmp = 0; - erts_free(ERTS_ALC_T_TMP, state->extp); + erts_free(ERTS_ALC_T_EXT_TERM_DATA, state->extp); } } static ERTS_INLINE Eterm -binary2term_create(ErtsDistExternal *edep, ErtsBinary2TermState *state, Eterm **hpp, ErlOffHeap *ohp) +binary2term_create(ErtsDistExternal *edep, ErtsBinary2TermState *state, + ErtsHeapFactory* factory) { Eterm res; - if (!dec_term(edep, hpp, state->extp, ohp, &res)) + + if (!dec_term(edep, factory, state->extp, &res, NULL)) res = THE_NON_VALUE; if (state->exttmp) { state->exttmp = 0; - erts_free(ERTS_ALC_T_TMP, state->extp); + erts_free(ERTS_ALC_T_EXT_TERM_DATA, state->extp); } return res; } @@ -1159,7 +1321,18 @@ binary2term_create(ErtsDistExternal *edep, ErtsBinary2TermState *state, Eterm ** Sint erts_binary2term_prepare(ErtsBinary2TermState *state, byte *data, Sint data_size) { - return binary2term_prepare(state, data, data_size); + Sint res; + + if (binary2term_prepare(state, data, data_size, NULL) < 0 || + (res=decoded_size(state->extp, state->extp + state->extsize, 0, NULL)) < 0) { + + if (state->exttmp) + erts_free(ERTS_ALC_T_EXT_TERM_DATA, state->extp); + state->extp = NULL; + state->exttmp = 0; + return -1; + } + return res; } void @@ -1169,73 +1342,278 @@ erts_binary2term_abort(ErtsBinary2TermState *state) } Eterm -erts_binary2term_create(ErtsBinary2TermState *state, Eterm **hpp, ErlOffHeap *ohp) +erts_binary2term_create(ErtsBinary2TermState *state, ErtsHeapFactory* factory) { - return binary2term_create(NULL,state, hpp, ohp); + return binary2term_create(NULL,state, factory); } -BIF_RETTYPE binary_to_term_1(BIF_ALIST_1) +static void b2t_destroy_context(B2TContext* context) { - Sint heap_size; - Eterm res; + erts_free_aligned_binary_bytes_extra(context->aligned_alloc, + ERTS_ALC_T_EXT_TERM_DATA); + context->aligned_alloc = NULL; + binary2term_abort(&context->b2ts); + switch (context->state) { + case B2TUncompressChunk: + erl_zlib_inflate_finish(&context->u.uc.stream); + break; + case B2TDecode: + case B2TDecodeList: + case B2TDecodeTuple: + case B2TDecodeString: + case B2TDecodeBinary: + if (context->u.dc.hamt_array.pstart) { + erts_free(context->u.dc.hamt_array.alloc_type, + context->u.dc.hamt_array.pstart); + } + break; + default:; + } +} + +static void b2t_context_destructor(Binary *context_bin) +{ + B2TContext* ctx = (B2TContext*) ERTS_MAGIC_BIN_DATA(context_bin); + ASSERT(ERTS_MAGIC_BIN_DESTRUCTOR(context_bin) == b2t_context_destructor); + + b2t_destroy_context(ctx); +} + +static BIF_RETTYPE binary_to_term_trap_1(BIF_ALIST_1) +{ + Binary *context_bin = ((ProcBin *) binary_val(BIF_ARG_1))->val; + ASSERT(ERTS_MAGIC_BIN_DESTRUCTOR(context_bin) == b2t_context_destructor); + + return binary_to_term_int(BIF_P, 0, THE_NON_VALUE, context_bin, NULL, + THE_NON_VALUE, THE_NON_VALUE); +} + + +#define B2T_BYTES_PER_REDUCTION 128 +#define B2T_MEMCPY_FACTOR 8 + +/* Define for testing */ +/*#define EXTREME_B2T_TRAPPING 1*/ + +#ifdef EXTREME_B2T_TRAPPING +static unsigned b2t_rand(void) +{ + static unsigned prev = 17; + prev = (prev * 214013 + 2531011); + return prev; +} +#endif + + +static B2TContext* b2t_export_context(Process* p, B2TContext* src) +{ + Binary* context_b = erts_create_magic_binary(sizeof(B2TContext), + b2t_context_destructor); + B2TContext* ctx = ERTS_MAGIC_BIN_DATA(context_b); Eterm* hp; - Eterm* endp; - Sint size; - byte* bytes; - byte* temp_alloc = NULL; - ErtsBinary2TermState b2ts; + sys_memcpy(ctx, src, sizeof(B2TContext)); + if (ctx->state >= B2TDecode && ctx->u.dc.next == &src->u.dc.res) { + ctx->u.dc.next = &ctx->u.dc.res; + } + hp = HAlloc(p, PROC_BIN_SIZE); + ctx->trap_bin = erts_mk_magic_binary_term(&hp, &MSO(p), context_b); + return ctx; +} - if ((bytes = erts_get_aligned_binary_bytes(BIF_ARG_1, &temp_alloc)) == NULL) { - error: - erts_free_aligned_binary_bytes(temp_alloc); - BIF_ERROR(BIF_P, BADARG); +static BIF_RETTYPE binary_to_term_int(Process* p, Uint32 flags, Eterm bin, Binary* context_b, + Export *bif_init, Eterm arg0, Eterm arg1) +{ + BIF_RETTYPE ret_val; +#ifdef EXTREME_B2T_TRAPPING + SWord initial_reds = 1 + b2t_rand() % 4; +#else + SWord initial_reds = (Uint)(ERTS_BIF_REDS_LEFT(p) * B2T_BYTES_PER_REDUCTION); +#endif + B2TContext c_buff; + B2TContext *ctx; + int is_first_call; + + if (context_b == NULL) { + /* Setup enough to get started */ + is_first_call = 1; + ctx = &c_buff; + ctx->state = B2TPrepare; + ctx->aligned_alloc = NULL; + ctx->flags = flags; + ctx->bif = bif_init; + ctx->arg[0] = arg0; + ctx->arg[1] = arg1; + IF_DEBUG(ctx->trap_bin = THE_NON_VALUE;) + } else { + is_first_call = 0; + ctx = ERTS_MAGIC_BIN_DATA(context_b); + ASSERT(ctx->state != B2TPrepare); } - size = binary_size(BIF_ARG_1); + ctx->reds = initial_reds; + + do { + switch (ctx->state) { + case B2TPrepare: { + byte* bytes; + Uint bin_size; + bytes = erts_get_aligned_binary_bytes_extra(bin, + &ctx->aligned_alloc, + ERTS_ALC_T_EXT_TERM_DATA, + 0); + if (bytes == NULL) { + ctx->b2ts.exttmp = 0; + ctx->state = B2TBadArg; + break; + } + bin_size = binary_size(bin); + if (ctx->aligned_alloc) { + ctx->reds -= bin_size / 8; + } + if (binary2term_prepare(&ctx->b2ts, bytes, bin_size, ctx) < 0) { + ctx->state = B2TBadArg; + } + break; + } + case B2TUncompressChunk: { + uLongf chunk = ctx->reds; + int zret; + + if (chunk > ctx->u.uc.dleft) + chunk = ctx->u.uc.dleft; + zret = erl_zlib_inflate_chunk(&ctx->u.uc.stream, + ctx->u.uc.dbytes, &chunk); + ctx->u.uc.dbytes += chunk; + ctx->u.uc.dleft -= chunk; + if (zret == Z_OK && ctx->u.uc.dleft > 0) { + ctx->reds = 0; + } + else if (erl_zlib_inflate_finish(&ctx->u.uc.stream) == Z_OK + && zret == Z_STREAM_END + && ctx->u.uc.dleft == 0) { + ctx->reds -= chunk; + ctx->state = B2TSizeInit; + } + else { + ctx->state = B2TBadArg; + } + break; + } + case B2TSizeInit: + ctx->u.sc.ep = NULL; + ctx->state = B2TSize; + /*fall through*/ + case B2TSize: + ctx->heap_size = decoded_size(ctx->b2ts.extp, + ctx->b2ts.extp + ctx->b2ts.extsize, + 0, ctx); + break; + + case B2TDecodeInit: + if (ctx == &c_buff && ctx->b2ts.extsize > ctx->reds) { + /* dec_term will maybe trap, allocate space for magic bin + before result term to make it easy to trim with HRelease. + */ + ctx = b2t_export_context(p, &c_buff); + } + ctx->u.dc.ep = ctx->b2ts.extp; + ctx->u.dc.res = (Eterm) (UWord) NULL; + ctx->u.dc.next = &ctx->u.dc.res; + erts_factory_proc_prealloc_init(&ctx->u.dc.factory, p, ctx->heap_size); + ctx->u.dc.flat_maps.wstart = NULL; + ctx->u.dc.hamt_array.pstart = NULL; + ctx->state = B2TDecode; + /*fall through*/ + case B2TDecode: + case B2TDecodeList: + case B2TDecodeTuple: + case B2TDecodeString: + case B2TDecodeBinary: { + ErtsDistExternal fakedep; + fakedep.flags = ctx->flags; + dec_term(&fakedep, NULL, NULL, NULL, ctx); + break; + } + case B2TDecodeFail: + /*fall through*/ + case B2TBadArg: + BUMP_REDS(p, (initial_reds - ctx->reds) / B2T_BYTES_PER_REDUCTION); - heap_size = binary2term_prepare(&b2ts, bytes, size); - if (heap_size < 0) - goto error; + ASSERT(ctx->bif == bif_export[BIF_binary_to_term_1] + || ctx->bif == bif_export[BIF_binary_to_term_2]); - hp = HAlloc(BIF_P, heap_size); - endp = hp + heap_size; + if (is_first_call) + ERTS_BIF_PREP_ERROR(ret_val, p, BADARG); + else { + erts_set_gc_state(p, 1); + if (is_non_value(ctx->arg[1])) + ERTS_BIF_PREP_ERROR_TRAPPED1(ret_val, p, BADARG, ctx->bif, + ctx->arg[0]); + else + ERTS_BIF_PREP_ERROR_TRAPPED2(ret_val, p, BADARG, ctx->bif, + ctx->arg[0], ctx->arg[1]); + } + b2t_destroy_context(ctx); + return ret_val; - res = binary2term_create(NULL, &b2ts, &hp, &MSO(BIF_P)); + case B2TDone: + b2t_destroy_context(ctx); - erts_free_aligned_binary_bytes(temp_alloc); + if (ctx->u.dc.factory.hp > ctx->u.dc.factory.hp_end) { + erts_exit(ERTS_ERROR_EXIT, ":%s, line %d: heap overrun by %d words(s)\n", + __FILE__, __LINE__, ctx->u.dc.factory.hp - ctx->u.dc.factory.hp_end); + } + erts_factory_close(&ctx->u.dc.factory); - if (hp > endp) { - erl_exit(1, ":%s, line %d: heap overrun by %d words(s)\n", - __FILE__, __LINE__, hp-endp); + if (!is_first_call) { + erts_set_gc_state(p, 1); + } + BUMP_REDS(p, (initial_reds - ctx->reds) / B2T_BYTES_PER_REDUCTION); + ERTS_BIF_PREP_RET(ret_val, ctx->u.dc.res); + return ret_val; + + default: + ASSERT(!"Unknown state in binary_to_term"); + } + }while (ctx->reds > 0 || ctx->state >= B2TDone); + + if (ctx == &c_buff) { + ASSERT(ctx->trap_bin == THE_NON_VALUE); + ctx = b2t_export_context(p, &c_buff); } + ASSERT(ctx->trap_bin != THE_NON_VALUE); - HRelease(BIF_P, endp, hp); + if (is_first_call) { + erts_set_gc_state(p, 0); + } + BUMP_ALL_REDS(p); - if (res == THE_NON_VALUE) - goto error; + ERTS_BIF_PREP_TRAP1(ret_val, &binary_to_term_trap_export, + p, ctx->trap_bin); - return res; + return ret_val; } +HIPE_WRAPPER_BIF_DISABLE_GC(binary_to_term, 1) + +BIF_RETTYPE binary_to_term_1(BIF_ALIST_1) +{ + return binary_to_term_int(BIF_P, 0, BIF_ARG_1, NULL, bif_export[BIF_binary_to_term_1], + BIF_ARG_1, THE_NON_VALUE); +} + +HIPE_WRAPPER_BIF_DISABLE_GC(binary_to_term, 2) + BIF_RETTYPE binary_to_term_2(BIF_ALIST_2) { - Sint heap_size; - Eterm res; Eterm opts; Eterm opt; - Eterm* hp; - Eterm* endp; - Sint size; - byte* bytes; - byte* temp_alloc = NULL; - ErtsBinary2TermState b2ts; - ErtsDistExternal fakedep; + Uint32 flags = 0; - fakedep.flags = 0; opts = BIF_ARG_2; while (is_list(opts)) { opt = CAR(list_val(opts)); if (opt == am_safe) { - fakedep.flags |= ERTS_DIST_EXT_BTT_SAFE; + flags |= ERTS_DIST_EXT_BTT_SAFE; } else { goto error; @@ -1246,35 +1624,11 @@ BIF_RETTYPE binary_to_term_2(BIF_ALIST_2) if (is_not_nil(opts)) goto error; - if ((bytes = erts_get_aligned_binary_bytes(BIF_ARG_1, &temp_alloc)) == NULL) { - error: - erts_free_aligned_binary_bytes(temp_alloc); - BIF_ERROR(BIF_P, BADARG); - } - size = binary_size(BIF_ARG_1); - - heap_size = binary2term_prepare(&b2ts, bytes, size); - if (heap_size < 0) - goto error; - - hp = HAlloc(BIF_P, heap_size); - endp = hp + heap_size; + return binary_to_term_int(BIF_P, flags, BIF_ARG_1, NULL, bif_export[BIF_binary_to_term_2], + BIF_ARG_1, BIF_ARG_2); - res = binary2term_create(&fakedep, &b2ts, &hp, &MSO(BIF_P)); - - erts_free_aligned_binary_bytes(temp_alloc); - - if (hp > endp) { - erl_exit(1, ":%s, line %d: heap overrun by %d words(s)\n", - __FILE__, __LINE__, hp-endp); - } - - HRelease(BIF_P, endp, hp); - - if (res == THE_NON_VALUE) - goto error; - - return res; +error: + BIF_ERROR(BIF_P, BADARG); } Eterm @@ -1306,9 +1660,9 @@ external_size_2(BIF_ALIST_2) if (tp[1] == am_minor_version && is_small(tp[2])) { switch (signed_val(tp[2])) { case 0: + flags &= ~DFLAG_NEW_FLOATS; break; case 1: - flags |= DFLAG_NEW_FLOATS; break; default: goto error; @@ -1335,16 +1689,13 @@ external_size_2(BIF_ALIST_2) } } -Eterm -erts_term_to_binary(Process* p, Eterm Term, int level, Uint flags) +static Eterm +erts_term_to_binary_simple(Process* p, Eterm Term, Uint size, int level, Uint flags) { - Uint size; Eterm bin; size_t real_size; byte* endp; - size = encode_size_struct2(NULL, Term, flags) + 1 /* VERSION_MAGIC */; - if (level != 0) { byte buf[256]; byte* bytes = buf; @@ -1357,12 +1708,12 @@ erts_term_to_binary(Process* p, Eterm Term, int level, Uint flags) if ((endp = enc_term(NULL, Term, bytes, flags, NULL)) == NULL) { - erl_exit(1, "%s, line %d: bad term: %x\n", + erts_exit(ERTS_ERROR_EXIT, "%s, line %d: bad term: %x\n", __FILE__, __LINE__, Term); } real_size = endp - bytes; if (real_size > size) { - erl_exit(1, "%s, line %d: buffer overflow: %d word(s)\n", + erts_exit(ERTS_ERROR_EXIT, "%s, line %d: buffer overflow: %d word(s)\n", __FILE__, __LINE__, real_size - size); } @@ -1402,18 +1753,304 @@ erts_term_to_binary(Process* p, Eterm Term, int level, Uint flags) bytes[0] = VERSION_MAGIC; if ((endp = enc_term(NULL, Term, bytes+1, flags, NULL)) == NULL) { - erl_exit(1, "%s, line %d: bad term: %x\n", + erts_exit(ERTS_ERROR_EXIT, "%s, line %d: bad term: %x\n", __FILE__, __LINE__, Term); } real_size = endp - bytes; if (real_size > size) { - erl_exit(1, "%s, line %d: buffer overflow: %d word(s)\n", + erts_exit(ERTS_ERROR_EXIT, "%s, line %d: buffer overflow: %d word(s)\n", __FILE__, __LINE__, endp - (bytes + size)); } return erts_realloc_binary(bin, real_size); } } +Eterm +erts_term_to_binary(Process* p, Eterm Term, int level, Uint flags) { + Uint size; + size = encode_size_struct2(NULL, Term, flags) + 1 /* VERSION_MAGIC */; + return erts_term_to_binary_simple(p, Term, size, level, flags); +} + +/* Define EXTREME_TTB_TRAPPING for testing in dist.h */ + +#ifndef EXTREME_TTB_TRAPPING +#define TERM_TO_BINARY_COMPRESS_CHUNK (1 << 18) +#else +#define TERM_TO_BINARY_COMPRESS_CHUNK 10 +#endif +#define TERM_TO_BINARY_MEMCPY_FACTOR 8 + +static void ttb_context_destructor(Binary *context_bin) +{ + TTBContext *context = ERTS_MAGIC_BIN_DATA(context_bin); + if (context->alive) { + context->alive = 0; + switch (context->state) { + case TTBSize: + DESTROY_SAVED_WSTACK(&context->s.sc.wstack); + break; + case TTBEncode: + DESTROY_SAVED_WSTACK(&context->s.ec.wstack); + if (context->s.ec.result_bin != NULL) { /* Set to NULL if ever made alive! */ + ASSERT(erts_refc_read(&(context->s.ec.result_bin->refc),0) == 0); + erts_bin_free(context->s.ec.result_bin); + context->s.ec.result_bin = NULL; + } + break; + case TTBCompress: + erl_zlib_deflate_finish(&(context->s.cc.stream)); + + if (context->s.cc.destination_bin != NULL) { /* Set to NULL if ever made alive! */ + ASSERT(erts_refc_read(&(context->s.cc.destination_bin->refc),0) == 0); + erts_bin_free(context->s.cc.destination_bin); + context->s.cc.destination_bin = NULL; + } + + if (context->s.cc.result_bin != NULL) { /* Set to NULL if ever made alive! */ + ASSERT(erts_refc_read(&(context->s.cc.result_bin->refc),0) == 0); + erts_bin_free(context->s.cc.result_bin); + context->s.cc.result_bin = NULL; + } + break; + } + } +} + +static Eterm erts_term_to_binary_int(Process* p, Eterm Term, int level, Uint flags, + Binary *context_b) +{ + Eterm *hp; + Eterm res; + Eterm c_term; +#ifndef EXTREME_TTB_TRAPPING + Sint reds = (Sint) (ERTS_BIF_REDS_LEFT(p) * TERM_TO_BINARY_LOOP_FACTOR); +#else + Sint reds = 20; /* For testing */ +#endif + Sint initial_reds = reds; + TTBContext c_buff; + TTBContext *context = &c_buff; + +#define EXPORT_CONTEXT() \ + do { \ + if (context_b == NULL) { \ + context_b = erts_create_magic_binary(sizeof(TTBContext), \ + ttb_context_destructor); \ + context = ERTS_MAGIC_BIN_DATA(context_b); \ + memcpy(context,&c_buff,sizeof(TTBContext)); \ + } \ + } while (0) + +#define RETURN_STATE() \ + do { \ + hp = HAlloc(p, PROC_BIN_SIZE+3); \ + c_term = erts_mk_magic_binary_term(&hp, &MSO(p), context_b); \ + res = TUPLE2(hp, Term, c_term); \ + BUMP_ALL_REDS(p); \ + return res; \ + } while (0); + + + if (context_b == NULL) { + /* Setup enough to get started */ + context->state = TTBSize; + context->alive = 1; + context->s.sc.wstack.wstart = NULL; + context->s.sc.flags = flags; + context->s.sc.level = level; + } else { + context = ERTS_MAGIC_BIN_DATA(context_b); + } + /* Initialization done, now we will go through the states */ + for (;;) { + switch (context->state) { + case TTBSize: + { + Uint size; + Binary *result_bin; + int level; + Uint flags; + /* Try for fast path */ + if (encode_size_struct_int(&context->s.sc, NULL, Term, + context->s.sc.flags, &reds, &size) < 0) { + EXPORT_CONTEXT(); + /* Same state */ + RETURN_STATE(); + } + ++size; /* VERSION_MAGIC */ + /* Move these to next state */ + flags = context->s.sc.flags; + level = context->s.sc.level; + if (size <= ERL_ONHEAP_BIN_LIMIT) { + /* Finish in one go */ + res = erts_term_to_binary_simple(p, Term, size, + level, flags); + BUMP_REDS(p, 1); + return res; + } + + result_bin = erts_bin_nrml_alloc(size); + erts_refc_init(&result_bin->refc, 0); + result_bin->orig_bytes[0] = VERSION_MAGIC; + /* Next state immediately, no need to export context */ + context->state = TTBEncode; + context->s.ec.flags = flags; + context->s.ec.level = level; + context->s.ec.wstack.wstart = NULL; + context->s.ec.result_bin = result_bin; + break; + } + case TTBEncode: + { + byte *endp; + byte *bytes = (byte *) context->s.ec.result_bin->orig_bytes; + size_t real_size; + Binary *result_bin; + + flags = context->s.ec.flags; + if (enc_term_int(&context->s.ec, NULL,Term, bytes+1, flags, NULL, &reds, &endp) < 0) { + EXPORT_CONTEXT(); + RETURN_STATE(); + } + real_size = endp - bytes; + result_bin = erts_bin_realloc(context->s.ec.result_bin,real_size); + level = context->s.ec.level; + BUMP_REDS(p, (initial_reds - reds) / TERM_TO_BINARY_LOOP_FACTOR); + if (level == 0 || real_size < 6) { /* We are done */ + ProcBin* pb; + return_normal: + context->s.ec.result_bin = NULL; + context->alive = 0; + pb = (ProcBin *) HAlloc(p, PROC_BIN_SIZE); + pb->thing_word = HEADER_PROC_BIN; + pb->size = real_size; + pb->next = MSO(p).first; + MSO(p).first = (struct erl_off_heap_header*)pb; + pb->val = result_bin; + pb->bytes = (byte*) result_bin->orig_bytes; + pb->flags = 0; + OH_OVERHEAD(&(MSO(p)), pb->size / sizeof(Eterm)); + erts_refc_inc(&result_bin->refc, 1); + if (context_b && erts_refc_read(&context_b->refc,0) == 0) { + erts_bin_free(context_b); + } + return make_binary(pb); + } + /* Continue with compression... */ + /* To make absolutely sure that zlib does not barf on a reallocated context, + we make sure it's "exported" before doing anything compession-like */ + EXPORT_CONTEXT(); + bytes = (byte *) result_bin->orig_bytes; /* result_bin is reallocated */ + if (erl_zlib_deflate_start(&(context->s.cc.stream),bytes+1,real_size-1,level) + != Z_OK) { + goto return_normal; + } + context->state = TTBCompress; + context->s.cc.real_size = real_size; + context->s.cc.result_bin = result_bin; + + result_bin = erts_bin_nrml_alloc(real_size); + erts_refc_init(&result_bin->refc, 0); + result_bin->orig_bytes[0] = VERSION_MAGIC; + + context->s.cc.destination_bin = result_bin; + context->s.cc.dest_len = 0; + context->s.cc.dbytes = (byte *) result_bin->orig_bytes+6; + break; + } + case TTBCompress: + { + uLongf tot_dest_len = context->s.cc.real_size - 6; + uLongf left = (tot_dest_len - context->s.cc.dest_len); + uLongf this_time = (left > TERM_TO_BINARY_COMPRESS_CHUNK) ? + TERM_TO_BINARY_COMPRESS_CHUNK : + left; + Binary *result_bin; + ProcBin *pb; + Uint max = (ERTS_BIF_REDS_LEFT(p) * TERM_TO_BINARY_COMPRESS_CHUNK) / CONTEXT_REDS; + + if (max < this_time) { + this_time = max + 1; /* do not set this_time to 0 */ + } + + res = erl_zlib_deflate_chunk(&(context->s.cc.stream), context->s.cc.dbytes, &this_time); + context->s.cc.dbytes += this_time; + context->s.cc.dest_len += this_time; + switch (res) { + case Z_OK: + if (context->s.cc.dest_len >= tot_dest_len) { + goto no_use_compressing; + } + RETURN_STATE(); + case Z_STREAM_END: + { + byte *dbytes = (byte *) context->s.cc.destination_bin->orig_bytes + 1; + + dbytes[0] = COMPRESSED; + put_int32(context->s.cc.real_size-1,dbytes+1); + erl_zlib_deflate_finish(&(context->s.cc.stream)); + result_bin = erts_bin_realloc(context->s.cc.destination_bin, + context->s.cc.dest_len+6); + context->s.cc.destination_bin = NULL; + pb = (ProcBin *) HAlloc(p, PROC_BIN_SIZE); + pb->thing_word = HEADER_PROC_BIN; + pb->size = context->s.cc.dest_len+6; + pb->next = MSO(p).first; + MSO(p).first = (struct erl_off_heap_header*)pb; + pb->val = result_bin; + pb->bytes = (byte*) result_bin->orig_bytes; + pb->flags = 0; + OH_OVERHEAD(&(MSO(p)), pb->size / sizeof(Eterm)); + erts_refc_inc(&result_bin->refc, 1); + erts_bin_free(context->s.cc.result_bin); + context->s.cc.result_bin = NULL; + context->alive = 0; + BUMP_REDS(p, (this_time * CONTEXT_REDS) / TERM_TO_BINARY_COMPRESS_CHUNK); + if (context_b && erts_refc_read(&context_b->refc,0) == 0) { + erts_bin_free(context_b); + } + return make_binary(pb); + } + default: /* Compression error, revert to uncompressed binary (still in + context) */ + no_use_compressing: + result_bin = context->s.cc.result_bin; + context->s.cc.result_bin = NULL; + pb = (ProcBin *) HAlloc(p, PROC_BIN_SIZE); + pb->thing_word = HEADER_PROC_BIN; + pb->size = context->s.cc.real_size; + pb->next = MSO(p).first; + MSO(p).first = (struct erl_off_heap_header*)pb; + pb->val = result_bin; + pb->bytes = (byte*) result_bin->orig_bytes; + pb->flags = 0; + OH_OVERHEAD(&(MSO(p)), pb->size / sizeof(Eterm)); + erts_refc_inc(&result_bin->refc, 1); + erl_zlib_deflate_finish(&(context->s.cc.stream)); + erts_bin_free(context->s.cc.destination_bin); + context->s.cc.destination_bin = NULL; + context->alive = 0; + BUMP_REDS(p, (this_time * CONTEXT_REDS) / TERM_TO_BINARY_COMPRESS_CHUNK); + if (context_b && erts_refc_read(&context_b->refc,0) == 0) { + erts_bin_free(context_b); + } + return make_binary(pb); + } + } + } + } +#undef EXPORT_CONTEXT +#undef RETURN_STATE +} + + + + + + + + /* * This function fills ext with the external format of atom. * If it's an old atom we just supply an index, otherwise @@ -1619,7 +2256,7 @@ static ERTS_INLINE ErlNode* dec_get_node(Eterm sysname, Uint creation) } static byte* -dec_pid(ErtsDistExternal *edep, Eterm** hpp, byte* ep, ErlOffHeap* off_heap, Eterm* objp) +dec_pid(ErtsDistExternal *edep, ErtsHeapFactory* factory, byte* ep, Eterm* objp) { Eterm sysname; Uint data; @@ -1658,15 +2295,15 @@ dec_pid(ErtsDistExternal *edep, Eterm** hpp, byte* ep, ErlOffHeap* off_heap, Ete if(node == erts_this_node) { *objp = make_internal_pid(data); } else { - ExternalThing *etp = (ExternalThing *) *hpp; - *hpp += EXTERNAL_THING_HEAD_SIZE + 1; + ExternalThing *etp = (ExternalThing *) factory->hp; + factory->hp += EXTERNAL_THING_HEAD_SIZE + 1; etp->header = make_external_pid_header(1); - etp->next = off_heap->first; + etp->next = factory->off_heap->first; etp->node = node; etp->data.ui[0] = data; - off_heap->first = (struct erl_off_heap_header*) etp; + factory->off_heap->first = (struct erl_off_heap_header*) etp; *objp = make_external_pid(etp); } return ep; @@ -1676,12 +2313,24 @@ dec_pid(ErtsDistExternal *edep, Eterm** hpp, byte* ep, ErlOffHeap* off_heap, Ete #define ENC_TERM ((Eterm) 0) #define ENC_ONE_CONS ((Eterm) 1) #define ENC_PATCH_FUN_SIZE ((Eterm) 2) -#define ENC_LAST_ARRAY_ELEMENT ((Eterm) 3) +#define ENC_BIN_COPY ((Eterm) 3) +#define ENC_MAP_PAIR ((Eterm) 4) +#define ENC_HASHMAP_NODE ((Eterm) 5) +#define ENC_LAST_ARRAY_ELEMENT ((Eterm) 6) static byte* enc_term(ErtsAtomCacheMap *acmp, Eterm obj, byte* ep, Uint32 dflags, struct erl_off_heap_header** off_heap) { + byte *res; + (void) enc_term_int(NULL, acmp, obj, ep, dflags, off_heap, NULL, &res); + return res; +} + +static int +enc_term_int(TTBEncodeContext* ctx, ErtsAtomCacheMap *acmp, Eterm obj, byte* ep, Uint32 dflags, + struct erl_off_heap_header** off_heap, Sint *reds, byte **res) +{ DECLARE_WSTACK(s); Uint n; Uint i; @@ -1689,11 +2338,26 @@ enc_term(ErtsAtomCacheMap *acmp, Eterm obj, byte* ep, Uint32 dflags, Uint* ptr; Eterm val; FloatDef f; + Sint r = 0; #if HALFWORD_HEAP UWord wobj; #endif + if (ctx) { + WSTACK_CHANGE_ALLOCATOR(s, ERTS_ALC_T_SAVED_ESTACK); + r = *reds; + + if (ctx->wstack.wstart) { /* restore saved stacks and byte pointer */ + WSTACK_RESTORE(s, &ctx->wstack); + ep = ctx->ep; + obj = ctx->obj; + if (is_non_value(obj)) { + goto outer_loop; + } + } + } + goto L_jump_start; outer_loop: @@ -1714,8 +2378,8 @@ enc_term(ErtsAtomCacheMap *acmp, Eterm obj, byte* ep, Uint32 dflags, obj = CAR(cons); tl = CDR(cons); - WSTACK_PUSH(s, is_list(tl) ? ENC_ONE_CONS : ENC_TERM); - WSTACK_PUSH(s, tl); + WSTACK_PUSH2(s, (is_list(tl) ? ENC_ONE_CONS : ENC_TERM), + tl); } break; case ENC_PATCH_FUN_SIZE: @@ -1728,7 +2392,48 @@ enc_term(ErtsAtomCacheMap *acmp, Eterm obj, byte* ep, Uint32 dflags, put_int32(ep - size_p, size_p); } goto outer_loop; + case ENC_BIN_COPY: { + Uint bits = (Uint)obj; + Uint bitoffs = WSTACK_POP(s); + byte* bytes = (byte*) WSTACK_POP(s); + byte* dst = (byte*) WSTACK_POP(s); + if (bits > r * (TERM_TO_BINARY_MEMCPY_FACTOR * 8)) { + Uint n = r * TERM_TO_BINARY_MEMCPY_FACTOR; + WSTACK_PUSH5(s, (UWord)(dst + n), (UWord)(bytes + n), bitoffs, + ENC_BIN_COPY, bits - 8*n); + bits = 8*n; + copy_binary_to_buffer(dst, 0, bytes, bitoffs, bits); + obj = THE_NON_VALUE; + r = 0; /* yield */ + break; + } else { + copy_binary_to_buffer(dst, 0, bytes, bitoffs, bits); + r -= bits / (TERM_TO_BINARY_MEMCPY_FACTOR * 8); + goto outer_loop; + } + } + case ENC_MAP_PAIR: { + Uint pairs_left = obj; + Eterm *vptr = (Eterm*) WSTACK_POP(s); + Eterm *kptr = (Eterm*) WSTACK_POP(s); + + obj = *kptr; + if (--pairs_left > 0) { + WSTACK_PUSH4(s, (UWord)(kptr+1), (UWord)(vptr+1), + ENC_MAP_PAIR, pairs_left); + } + WSTACK_PUSH2(s, ENC_TERM, *vptr); + break; + } + case ENC_HASHMAP_NODE: + if (is_list(obj)) { /* leaf node [K|V] */ + ptr = list_val(obj); + WSTACK_PUSH2(s, ENC_TERM, CDR(ptr)); + obj = CAR(ptr); + } + break; case ENC_LAST_ARRAY_ELEMENT: + /* obj is the tuple */ { #if HALFWORD_HEAP Eterm* ptr = (Eterm *) wobj; @@ -1746,13 +2451,20 @@ enc_term(ErtsAtomCacheMap *acmp, Eterm obj, byte* ep, Uint32 dflags, Eterm* ptr = (Eterm *) obj; #endif obj = *ptr++; - WSTACK_PUSH(s, val-1); - WSTACK_PUSH(s, (UWord) ptr); + WSTACK_PUSH2(s, val-1, (UWord)ptr); } break; } L_jump_start: + + if (ctx && --r <= 0) { + *reds = 0; + ctx->obj = obj; + ctx->ep = ep; + WSTACK_SAVE(s, &ctx->wstack); + return -1; + } switch(tag_val_def(obj)) { case NIL_DEF: *ep++ = NIL_EXT; @@ -1896,16 +2608,64 @@ enc_term(ErtsAtomCacheMap *acmp, Eterm obj, byte* ep, Uint32 dflags, ep += 4; } if (i > 0) { - WSTACK_PUSH(s, ENC_LAST_ARRAY_ELEMENT+i-1); - WSTACK_PUSH(s, (UWord) ptr); + WSTACK_PUSH2(s, ENC_LAST_ARRAY_ELEMENT+i-1, (UWord)ptr); } break; + case MAP_DEF: + if (is_flatmap(obj)) { + flatmap_t *mp = (flatmap_t*)flatmap_val(obj); + Uint size = flatmap_get_size(mp); + + *ep++ = MAP_EXT; + put_int32(size, ep); ep += 4; + + if (size > 0) { + Eterm *kptr = flatmap_get_keys(mp); + Eterm *vptr = flatmap_get_values(mp); + + WSTACK_PUSH4(s, (UWord)kptr, (UWord)vptr, + ENC_MAP_PAIR, size); + } + } else { + Eterm hdr; + Uint node_sz; + ptr = boxed_val(obj); + hdr = *ptr; + ASSERT(is_header(hdr)); + switch(hdr & _HEADER_MAP_SUBTAG_MASK) { + case HAMT_SUBTAG_HEAD_ARRAY: + *ep++ = MAP_EXT; + ptr++; + put_int32(*ptr, ep); ep += 4; + node_sz = 16; + break; + case HAMT_SUBTAG_HEAD_BITMAP: + *ep++ = MAP_EXT; + ptr++; + put_int32(*ptr, ep); ep += 4; + /*fall through*/ + case HAMT_SUBTAG_NODE_BITMAP: + node_sz = hashmap_bitcount(MAP_HEADER_VAL(hdr)); + ASSERT(node_sz < 17); + break; + default: + erts_exit(ERTS_ERROR_EXIT, "bad header\r\n"); + } + + ptr++; + WSTACK_RESERVE(s, node_sz*2); + while(node_sz--) { + WSTACK_FAST_PUSH(s, ENC_HASHMAP_NODE); + WSTACK_FAST_PUSH(s, *ptr++); + } + } + break; case FLOAT_DEF: GET_DOUBLE(obj, f); if (dflags & DFLAG_NEW_FLOATS) { *ep++ = NEW_FLOAT_EXT; -#ifdef WORDS_BIGENDIAN +#if defined(WORDS_BIGENDIAN) || defined(DOUBLE_MIDDLE_ENDIAN) put_int32(f.fw[0], ep); ep += 4; put_int32(f.fw[1], ep); @@ -1934,6 +2694,7 @@ enc_term(ErtsAtomCacheMap *acmp, Eterm obj, byte* ep, Uint32 dflags, Uint bitoffs; Uint bitsize; byte* bytes; + byte* data_dst; ERTS_GET_BINARY_BYTES(obj, bytes, bitoffs, bitsize); if (dflags & DFLAG_INTERNAL_TAGS) { @@ -1979,7 +2740,7 @@ enc_term(ErtsAtomCacheMap *acmp, Eterm obj, byte* ep, Uint32 dflags, j = binary_size(obj); put_int32(j, ep); ep += 4; - copy_binary_to_buffer(ep, 0, bytes, bitoffs, 8*j); + data_dst = ep; ep += j; } else if (dflags & DFLAG_BIT_BINARIES) { /* Bit-level binary. */ @@ -1989,7 +2750,7 @@ enc_term(ErtsAtomCacheMap *acmp, Eterm obj, byte* ep, Uint32 dflags, ep += 4; *ep++ = bitsize; ep[j] = 0; /* Zero unused bits at end of binary */ - copy_binary_to_buffer(ep, 0, bytes, bitoffs, 8*j+bitsize); + data_dst = ep; ep += j + 1; } else { /* @@ -2003,11 +2764,18 @@ enc_term(ErtsAtomCacheMap *acmp, Eterm obj, byte* ep, Uint32 dflags, put_int32((j+1), ep); ep += 4; ep[j] = 0; /* Zero unused bits at end of binary */ - copy_binary_to_buffer(ep, 0, bytes, bitoffs, 8*j+bitsize); + data_dst = ep; ep += j+1; *ep++ = SMALL_INTEGER_EXT; *ep++ = bitsize; } + if (ctx && j > r * TERM_TO_BINARY_MEMCPY_FACTOR) { + WSTACK_PUSH5(s, (UWord)data_dst, (UWord)bytes, bitoffs, + ENC_BIN_COPY, 8*j + bitsize); + } else { + copy_binary_to_buffer(data_dst, 0, bytes, bitoffs, + 8 * j + bitsize); + } } break; case EXPORT_DEF: @@ -2036,13 +2804,12 @@ enc_term(ErtsAtomCacheMap *acmp, Eterm obj, byte* ep, Uint32 dflags, case FUN_DEF: { ErlFunThing* funp = (ErlFunThing *) fun_val(obj); + int ei; if ((dflags & DFLAG_NEW_FUN_TAGS) != 0) { - int ei; - *ep++ = NEW_FUN_EXT; - WSTACK_PUSH(s, ENC_PATCH_FUN_SIZE); - WSTACK_PUSH(s, (UWord) ep); /* Position for patching in size */ + WSTACK_PUSH2(s, ENC_PATCH_FUN_SIZE, + (UWord) ep); /* Position for patching in size */ ep += 4; *ep = funp->arity; ep += 1; @@ -2056,16 +2823,6 @@ enc_term(ErtsAtomCacheMap *acmp, Eterm obj, byte* ep, Uint32 dflags, ep = enc_term(acmp, make_small(funp->fe->old_index), ep, dflags, off_heap); ep = enc_term(acmp, make_small(funp->fe->old_uniq), ep, dflags, off_heap); ep = enc_pid(acmp, funp->creator, ep, dflags); - - fun_env: - for (ei = funp->num_free-1; ei > 0; ei--) { - WSTACK_PUSH(s, ENC_TERM); - WSTACK_PUSH(s, (UWord) funp->env[ei]); - } - if (funp->num_free != 0) { - obj = funp->env[0]; - goto L_jump_start; - } } else { /* * Communicating with an obsolete erl_interface or @@ -2097,14 +2854,25 @@ enc_term(ErtsAtomCacheMap *acmp, Eterm obj, byte* ep, Uint32 dflags, *ep++ = SMALL_TUPLE_EXT; put_int8(funp->num_free, ep); ep += 1; - goto fun_env; + } + for (ei = funp->num_free-1; ei > 0; ei--) { + WSTACK_PUSH2(s, ENC_TERM, (UWord) funp->env[ei]); + } + if (funp->num_free != 0) { + obj = funp->env[0]; + goto L_jump_start; } } break; } } DESTROY_WSTACK(s); - return ep; + if (ctx) { + ASSERT(ctx->wstack.wstart == NULL); + *reds = r; + } + *res = ep; + return 0; } static @@ -2146,49 +2914,134 @@ is_external_string(Eterm list, int* p_is_string) return len; } -/* Assumes that the ones to undo are preluding the list. */ -static void -undo_offheap_in_area(ErlOffHeap* off_heap, Eterm* start, Eterm* end) -{ - const Uint area_sz = (end - start) * sizeof(Eterm); - struct erl_off_heap_header* hdr; - struct erl_off_heap_header** hdr_nextp = NULL; - for (hdr = off_heap->first; ; hdr=hdr->next) { - if (!in_area(hdr, start, area_sz)) { - if (hdr_nextp != NULL) { - *hdr_nextp = NULL; - erts_cleanup_offheap(off_heap); - off_heap->first = hdr; - } - break; - } - hdr_nextp = &hdr->next; - } +struct dec_term_hamt +{ + Eterm* objp; /* write result here */ + Uint size; /* nr of leafs */ + Eterm* leaf_array; +}; - /* Assert that the ones to undo were indeed preluding the list. */ -#ifdef DEBUG - for (hdr = off_heap->first; hdr != NULL; hdr = hdr->next) { - ASSERT(!in_area(hdr, start, area_sz)); - } -#endif /* DEBUG */ -} /* Decode term from external format into *objp. -** On failure return NULL and (R13B04) *hpp will be unchanged. +** On failure calls erts_factory_undo() and returns NULL */ static byte* -dec_term(ErtsDistExternal *edep, Eterm** hpp, byte* ep, ErlOffHeap* off_heap, Eterm* objp) +dec_term(ErtsDistExternal *edep, + ErtsHeapFactory* factory, + byte* ep, + Eterm* objp, + B2TContext* ctx) { - Eterm* hp_saved = *hpp; +#define PSTACK_TYPE struct dec_term_hamt + PSTACK_DECLARE(hamt_array, 5); int n; ErtsAtomEncoding char_enc; - register Eterm* hp = *hpp; /* Please don't take the address of hp */ - Eterm* next = objp; + register Eterm* hp; /* Please don't take the address of hp */ + DECLARE_WSTACK(flat_maps); /* for preprocessing of small maps */ + Eterm* next; + SWord reds; +#ifdef DEBUG + Eterm* dbg_resultp = ctx ? &ctx->u.dc.res : objp; +#endif - *next = (Eterm) (UWord) NULL; + if (ctx) { + reds = ctx->reds; + next = ctx->u.dc.next; + ep = ctx->u.dc.ep; + factory = &ctx->u.dc.factory; + + if (ctx->state != B2TDecode) { + int n_limit = reds; + + n = ctx->u.dc.remaining_n; + if (ctx->state == B2TDecodeBinary) { + n_limit *= B2T_MEMCPY_FACTOR; + ASSERT(n_limit >= reds); + reds -= n / B2T_MEMCPY_FACTOR; + } + else + reds -= n; + + if (n > n_limit) { + ctx->u.dc.remaining_n -= n_limit; + n = n_limit; + reds = 0; + } + else { + ctx->u.dc.remaining_n = 0; + } + + switch (ctx->state) { + case B2TDecodeList: + objp = next - 2; + while (n > 0) { + objp[0] = (Eterm) COMPRESS_POINTER(next); + objp[1] = make_list(next); + next = objp; + objp -= 2; + n--; + } + break; + + case B2TDecodeTuple: + objp = next - 1; + while (n-- > 0) { + objp[0] = (Eterm) COMPRESS_POINTER(next); + next = objp; + objp--; + } + break; + + case B2TDecodeString: + hp = factory->hp; + hp[-1] = make_list(hp); /* overwrite the premature NIL */ + while (n-- > 0) { + hp[0] = make_small(*ep++); + hp[1] = make_list(hp+2); + hp += 2; + } + hp[-1] = NIL; + factory->hp = hp; + break; + + case B2TDecodeBinary: + sys_memcpy(ctx->u.dc.remaining_bytes, ep, n); + ctx->u.dc.remaining_bytes += n; + ep += n; + break; + + default: + ASSERT(!"Unknown state"); + } + if (!ctx->u.dc.remaining_n) { + ctx->state = B2TDecode; + } + if (reds <= 0) { + ctx->u.dc.next = next; + ctx->u.dc.ep = ep; + ctx->reds = 0; + return NULL; + } + } + PSTACK_CHANGE_ALLOCATOR(hamt_array, ERTS_ALC_T_SAVED_ESTACK); + WSTACK_CHANGE_ALLOCATOR(flat_maps, ERTS_ALC_T_SAVED_ESTACK); + if (ctx->u.dc.hamt_array.pstart) { + PSTACK_RESTORE(hamt_array, &ctx->u.dc.hamt_array); + } + if (ctx->u.dc.flat_maps.wstart) { + WSTACK_RESTORE(flat_maps, &ctx->u.dc.flat_maps); + } + } + else { + reds = ERTS_SWORD_MAX; + next = objp; + *next = (Eterm) (UWord) NULL; + } + hp = factory->hp; while (next != NULL) { + objp = next; next = (Eterm *) EXPAND_POINTER(*objp); @@ -2247,6 +3100,8 @@ dec_term(ErtsDistExternal *edep, Eterm** hpp, byte* ep, ErlOffHeap* off_heap, Et big = make_small(0); } else { big = bytes_to_big(first, n, neg, hp); + if (is_nil(big)) + goto error; if (is_big(big)) { hp += big_arity(big) + 1; } @@ -2308,7 +3163,16 @@ dec_term_atom_common: *objp = make_tuple(hp); *hp++ = make_arityval(n); hp += n; - objp = hp - 1; + objp = hp - 1; + if (ctx) { + if (reds < n) { + ASSERT(reds > 0); + ctx->state = B2TDecodeTuple; + ctx->u.dc.remaining_n = n - reds; + n = reds; + } + reds -= n; + } while (n-- > 0) { objp[0] = (Eterm) COMPRESS_POINTER(next); next = objp; @@ -2326,17 +3190,27 @@ dec_term_atom_common: break; } *objp = make_list(hp); - hp += 2*n; + hp += 2 * n; objp = hp - 2; objp[0] = (Eterm) COMPRESS_POINTER((objp+1)); objp[1] = (Eterm) COMPRESS_POINTER(next); next = objp; objp -= 2; - while (--n > 0) { + n--; + if (ctx) { + if (reds < n) { + ctx->state = B2TDecodeList; + ctx->u.dc.remaining_n = n - reds; + n = reds; + } + reds -= n; + } + while (n > 0) { objp[0] = (Eterm) COMPRESS_POINTER(next); - objp[1] = make_list(objp + 2); + objp[1] = make_list(next); next = objp; objp -= 2; + n--; } break; case STRING_EXT: @@ -2347,6 +3221,14 @@ dec_term_atom_common: break; } *objp = make_list(hp); + if (ctx) { + if (reds < n) { + ctx->state = B2TDecodeString; + ctx->u.dc.remaining_n = n - reds; + n = reds; + } + reds -= n; + } while (n-- > 0) { hp[0] = make_small(*ep++); hp[1] = make_list(hp+2); @@ -2374,7 +3256,7 @@ dec_term_atom_common: volatile unsigned long *fpexnp = erts_get_current_fp_exception(); #endif -#ifdef WORDS_BIGENDIAN +#if defined(WORDS_BIGENDIAN) || defined(DOUBLE_MIDDLE_ENDIAN) ff.fw[0] = get_int32(ep); ep += 4; ff.fw[1] = get_int32(ep); @@ -2393,9 +3275,9 @@ dec_term_atom_common: break; } case PID_EXT: - *hpp = hp; - ep = dec_pid(edep, hpp, ep, off_heap, objp); - hp = *hpp; + factory->hp = hp; + ep = dec_pid(edep, factory, ep, objp); + hp = factory->hp; if (ep == NULL) { goto error; } @@ -2429,11 +3311,11 @@ dec_term_atom_common: hp += EXTERNAL_THING_HEAD_SIZE + 1; etp->header = make_external_port_header(1); - etp->next = off_heap->first; + etp->next = factory->off_heap->first; etp->node = node; etp->data.ui[0] = num; - off_heap->first = (struct erl_off_heap_header*)etp; + factory->off_heap->first = (struct erl_off_heap_header*)etp; *objp = make_external_port(etp); } @@ -2513,10 +3395,10 @@ dec_term_atom_common: #else etp->header = make_external_ref_header(ref_words); #endif - etp->next = off_heap->first; + etp->next = factory->off_heap->first; etp->node = node; - off_heap->first = (struct erl_off_heap_header*)etp; + factory->off_heap->first = (struct erl_off_heap_header*)etp; *objp = make_external_ref(etp); ref_num = &(etp->data.ui32[0]); } @@ -2540,7 +3422,7 @@ dec_term_atom_common: n = get_int32(ep); ep += 4; - if (n <= ERL_ONHEAP_BIN_LIMIT) { + if ((unsigned)n <= ERL_ONHEAP_BIN_LIMIT) { ErlHeapBin* hb = (ErlHeapBin *) hp; hb->thing_word = header_heap_bin(n); @@ -2551,21 +3433,32 @@ dec_term_atom_common: } else { Binary* dbin = erts_bin_nrml_alloc(n); ProcBin* pb; - dbin->flags = 0; - dbin->orig_size = n; erts_refc_init(&dbin->refc, 1); - sys_memcpy(dbin->orig_bytes, ep, n); pb = (ProcBin *) hp; hp += PROC_BIN_SIZE; pb->thing_word = HEADER_PROC_BIN; pb->size = n; - pb->next = off_heap->first; - off_heap->first = (struct erl_off_heap_header*)pb; + pb->next = factory->off_heap->first; + factory->off_heap->first = (struct erl_off_heap_header*)pb; + OH_OVERHEAD(factory->off_heap, pb->size / sizeof(Eterm)); pb->val = dbin; pb->bytes = (byte*) dbin->orig_bytes; pb->flags = 0; *objp = make_binary(pb); - } + if (ctx) { + int n_limit = reds * B2T_MEMCPY_FACTOR; + if (n > n_limit) { + ctx->state = B2TDecodeBinary; + ctx->u.dc.remaining_n = n - n_limit; + ctx->u.dc.remaining_bytes = dbin->orig_bytes + n_limit; + n = n_limit; + reds = 0; + } + else + reds -= n / B2T_MEMCPY_FACTOR; + } + sys_memcpy(dbin->orig_bytes, ep, n); + } ep += n; break; } @@ -2577,8 +3470,10 @@ dec_term_atom_common: n = get_int32(ep); bitsize = ep[4]; - ep += 5; - if (n <= ERL_ONHEAP_BIN_LIMIT) { + if (((bitsize==0) != (n==0)) || bitsize > 8) + goto error; + ep += 5; + if ((unsigned)n <= ERL_ONHEAP_BIN_LIMIT) { ErlHeapBin* hb = (ErlHeapBin *) hp; hb->thing_word = header_heap_bin(n); @@ -2586,29 +3481,44 @@ dec_term_atom_common: sys_memcpy(hb->data, ep, n); bin = make_binary(hb); hp += heap_bin_size(n); + ep += n; } else { Binary* dbin = erts_bin_nrml_alloc(n); ProcBin* pb; - dbin->flags = 0; - dbin->orig_size = n; + erts_refc_init(&dbin->refc, 1); - sys_memcpy(dbin->orig_bytes, ep, n); pb = (ProcBin *) hp; pb->thing_word = HEADER_PROC_BIN; pb->size = n; - pb->next = off_heap->first; - off_heap->first = (struct erl_off_heap_header*)pb; + pb->next = factory->off_heap->first; + factory->off_heap->first = (struct erl_off_heap_header*)pb; + OH_OVERHEAD(factory->off_heap, pb->size / sizeof(Eterm)); pb->val = dbin; pb->bytes = (byte*) dbin->orig_bytes; pb->flags = 0; bin = make_binary(pb); hp += PROC_BIN_SIZE; - } - ep += n; - if (bitsize == 0) { + if (ctx) { + int n_limit = reds * B2T_MEMCPY_FACTOR; + if (n > n_limit) { + ctx->state = B2TDecodeBinary; + ctx->u.dc.remaining_n = n - n_limit; + ctx->u.dc.remaining_bytes = dbin->orig_bytes + n_limit; + n = n_limit; + reds = 0; + } + else + reds -= n / B2T_MEMCPY_FACTOR; + } + sys_memcpy(dbin->orig_bytes, ep, n); + ep += n; + n = pb->size; + } + + if (bitsize == 8 || n == 0) { *objp = bin; } else { - sb = (ErlSubBin *) hp; + sb = (ErlSubBin *)hp; sb->thing_word = HEADER_SUB_BIN; sb->orig = bin; sb->size = n - 1; @@ -2634,9 +3544,9 @@ dec_term_atom_common: if ((ep = dec_atom(edep, ep, &name)) == NULL) { goto error; } - *hpp = hp; - ep = dec_term(edep, hpp, ep, off_heap, &temp); - hp = *hpp; + factory->hp = hp; + ep = dec_term(edep, factory, ep, &temp, NULL); + hp = factory->hp; if (ep == NULL) { goto error; } @@ -2662,6 +3572,61 @@ dec_term_atom_common: break; } break; + case MAP_EXT: + { + Uint32 size,n; + Eterm *kptr,*vptr; + Eterm keys; + + size = get_int32(ep); ep += 4; + + if (size <= MAP_SMALL_MAP_LIMIT) { + flatmap_t *mp; + + keys = make_tuple(hp); + *hp++ = make_arityval(size); + hp += size; + kptr = hp - 1; + + mp = (flatmap_t*)hp; + hp += MAP_HEADER_FLATMAP_SZ; + hp += size; + vptr = hp - 1; + + /* kptr, last word for keys + * vptr, last word for values + */ + + WSTACK_PUSH(flat_maps, (UWord)mp); + mp->thing_word = MAP_HEADER_FLATMAP; + mp->size = size; + mp->keys = keys; + *objp = make_flatmap(mp); + + for (n = size; n; n--) { + *vptr = (Eterm) COMPRESS_POINTER(next); + *kptr = (Eterm) COMPRESS_POINTER(vptr); + next = kptr; + vptr--; + kptr--; + } + } + else { /* Make hamt */ + struct dec_term_hamt* hamt = PSTACK_PUSH(hamt_array); + + hamt->objp = objp; + hamt->size = size; + hamt->leaf_array = hp; + + for (n = size; n; n--) { + CDR(hp) = (Eterm) COMPRESS_POINTER(next); + CAR(hp) = (Eterm) COMPRESS_POINTER(&CDR(hp)); + next = &CAR(hp); + hp += 2; + } + } + } + break; case NEW_FUN_EXT: { ErlFunThing* funp = (ErlFunThing *) hp; @@ -2693,9 +3658,9 @@ dec_term_atom_common: if ((ep = dec_atom(edep, ep, &module)) == NULL) { goto error; } - *hpp = hp; + factory->hp = hp; /* Index */ - if ((ep = dec_term(edep, hpp, ep, off_heap, &temp)) == NULL) { + if ((ep = dec_term(edep, factory, ep, &temp, NULL)) == NULL) { goto error; } if (!is_small(temp)) { @@ -2704,7 +3669,7 @@ dec_term_atom_common: old_index = unsigned_val(temp); /* Uniq */ - if ((ep = dec_term(edep, hpp, ep, off_heap, &temp)) == NULL) { + if ((ep = dec_term(edep, factory, ep, &temp, NULL)) == NULL) { goto error; } if (!is_small(temp)) { @@ -2716,8 +3681,8 @@ dec_term_atom_common: * It is safe to link the fun into the fun list only when * no more validity tests can fail. */ - funp->next = off_heap->first; - off_heap->first = (struct erl_off_heap_header*)funp; + funp->next = factory->off_heap->first; + factory->off_heap->first = (struct erl_off_heap_header*)funp; funp->fe = erts_put_fun_entry2(module, old_uniq, old_index, uniq, index, arity); @@ -2728,7 +3693,7 @@ dec_term_atom_common: } funp->native_address = funp->fe->native_address; #endif - hp = *hpp; + hp = factory->hp; /* Environment */ for (i = num_free-1; i >= 0; i--) { @@ -2754,14 +3719,14 @@ dec_term_atom_common: ep += 4; hp += ERL_FUN_SIZE; hp += num_free; - *hpp = hp; + factory->hp = hp; funp->thing_word = HEADER_FUN; funp->num_free = num_free; *objp = make_fun(funp); /* Creator pid */ if (*ep != PID_EXT - || (ep = dec_pid(edep, hpp, ++ep, off_heap, + || (ep = dec_pid(edep, factory, ++ep, &funp->creator))==NULL) { goto error; } @@ -2772,7 +3737,7 @@ dec_term_atom_common: } /* Index */ - if ((ep = dec_term(edep, hpp, ep, off_heap, &temp)) == NULL) { + if ((ep = dec_term(edep, factory, ep, &temp, NULL)) == NULL) { goto error; } if (!is_small(temp)) { @@ -2781,7 +3746,7 @@ dec_term_atom_common: old_index = unsigned_val(temp); /* Uniq */ - if ((ep = dec_term(edep, hpp, ep, off_heap, &temp)) == NULL) { + if ((ep = dec_term(edep, factory, ep, &temp, NULL)) == NULL) { goto error; } if (!is_small(temp)) { @@ -2792,8 +3757,8 @@ dec_term_atom_common: * It is safe to link the fun into the fun list only when * no more validity tests can fail. */ - funp->next = off_heap->first; - off_heap->first = (struct erl_off_heap_header*)funp; + funp->next = factory->off_heap->first; + factory->off_heap->first = (struct erl_off_heap_header*)funp; old_uniq = unsigned_val(temp); funp->fe = erts_put_fun_entry(module, old_uniq, old_index); @@ -2801,7 +3766,7 @@ dec_term_atom_common: #ifdef HIPE funp->native_address = funp->fe->native_address; #endif - hp = *hpp; + hp = factory->hp; /* Environment */ for (i = num_free-1; i >= 0; i--) { @@ -2835,8 +3800,9 @@ dec_term_atom_common: erts_refc_inc(&pb->val->refc, 1); hp += PROC_BIN_SIZE; - pb->next = off_heap->first; - off_heap->first = (struct erl_off_heap_header*)pb; + pb->next = factory->off_heap->first; + factory->off_heap->first = (struct erl_off_heap_header*)pb; + OH_OVERHEAD(factory->off_heap, pb->size / sizeof(Eterm)); pb->flags = 0; *objp = make_binary(pb); break; @@ -2852,8 +3818,9 @@ dec_term_atom_common: erts_refc_inc(&pb->val->refc, 1); hp += PROC_BIN_SIZE; - pb->next = off_heap->first; - off_heap->first = (struct erl_off_heap_header*)pb; + pb->next = factory->off_heap->first; + factory->off_heap->first = (struct erl_off_heap_header*)pb; + OH_OVERHEAD(factory->off_heap, pb->size / sizeof(Eterm)); pb->flags = 0; sub = (ErlSubBin*)hp; @@ -2871,83 +3838,139 @@ dec_term_atom_common: } default: - error: - /* UNDO: - * Must unlink all off-heap objects that may have been - * linked into the process. - */ - if (hp < *hpp) { /* Sometimes we used hp and sometimes *hpp */ - hp = *hpp; /* the largest must be the freshest */ - } - undo_offheap_in_area(off_heap, hp_saved, hp); - *hpp = hp_saved; - return NULL; + goto error; } + + if (--reds <= 0) { + if (ctx) { + if (next || ctx->state != B2TDecode) { + ctx->u.dc.ep = ep; + ctx->u.dc.next = next; + ctx->u.dc.factory.hp = hp; + if (!WSTACK_ISEMPTY(flat_maps)) { + WSTACK_SAVE(flat_maps, &ctx->u.dc.flat_maps); + } + if (!PSTACK_IS_EMPTY(hamt_array)) { + PSTACK_SAVE(hamt_array, &ctx->u.dc.hamt_array); + } + ctx->reds = 0; + return NULL; + } + } + else { + reds = ERTS_SWORD_MAX; + } + } + } + + ASSERT(hp <= factory->hp_end + || (factory->mode == FACTORY_CLOSED && is_immed(*dbg_resultp))); + factory->hp = hp; + /* + * From here on factory may produce (more) heap fragments + */ + + if (!PSTACK_IS_EMPTY(hamt_array)) { + do { + struct dec_term_hamt* hamt = PSTACK_TOP(hamt_array); + + *hamt->objp = erts_hashmap_from_array(factory, + hamt->leaf_array, + hamt->size, + 1); + if (is_non_value(*hamt->objp)) + goto error_hamt; + + (void) PSTACK_POP(hamt_array); + } while (!PSTACK_IS_EMPTY(hamt_array)); + PSTACK_DESTROY(hamt_array); + } + + /* Iterate through all the (flat)maps and check for validity and sort keys + * - done here for when we know it is complete. + */ + + while(!WSTACK_ISEMPTY(flat_maps)) { + next = (Eterm *)WSTACK_POP(flat_maps); + if (!erts_validate_and_sort_flatmap((flatmap_t*)next)) + goto error; + } + WSTACK_DESTROY(flat_maps); + + ASSERT((Eterm*)EXPAND_POINTER(*dbg_resultp) != NULL); + + if (ctx) { + ctx->state = B2TDone; + ctx->reds = reds; } - *hpp = hp; + return ep; + +error: + /* UNDO: + * Must unlink all off-heap objects that may have been + * linked into the process. + */ + if (factory->hp < hp) { /* Sometimes we used hp and sometimes factory->hp */ + factory->hp = hp; /* the largest must be the freshest */ + } +error_hamt: + erts_factory_undo(factory); + PSTACK_DESTROY(hamt_array); + if (ctx) { + ctx->state = B2TDecodeFail; + ctx->reds = reds; + } + WSTACK_DESTROY(flat_maps); + + return NULL; } /* returns the number of bytes needed to encode an object to a sequence of bytes N.B. That this must agree with to_external2() above!!! (except for cached atoms) */ +static Uint encode_size_struct2(ErtsAtomCacheMap *acmp, Eterm obj, unsigned dflags) { + Uint res; + (void) encode_size_struct_int(NULL, acmp, obj, dflags, NULL, &res); + return res; +} -static Uint -encode_size_struct2(ErtsAtomCacheMap *acmp, Eterm obj, unsigned dflags) +static int +encode_size_struct_int(TTBSizeContext* ctx, ErtsAtomCacheMap *acmp, Eterm obj, + unsigned dflags, Sint *reds, Uint *res) { DECLARE_WSTACK(s); Uint m, i, arity; Uint result = 0; -#if HALFWORD_HEAP - UWord wobj = 0; -#endif + Sint r = 0; - goto L_jump_start; + if (ctx) { + WSTACK_CHANGE_ALLOCATOR(s, ERTS_ALC_T_SAVED_ESTACK); + r = *reds; - outer_loop: - while (!WSTACK_ISEMPTY(s)) { -#if HALFWORD_HEAP - obj = (Eterm) (wobj = WSTACK_POP(s)); -#else - obj = WSTACK_POP(s); -#endif - handle_popped_obj: - if (is_CP(obj)) { /* Does not look for CP, looks for "no tag" */ -#if HALFWORD_HEAP - Eterm* ptr = (Eterm *) wobj; -#else - Eterm* ptr = (Eterm *) obj; -#endif - /* - * Pointer into a tuple. - */ - obj = *ptr--; - if (!is_header(obj)) { - WSTACK_PUSH(s, (UWord)ptr); - } else { - /* Reached tuple header */ - ASSERT(header_is_arityval(obj)); - goto outer_loop; - } - } else if (is_list(obj)) { - Eterm* cons = list_val(obj); - Eterm tl; + if (ctx->wstack.wstart) { /* restore saved stack */ + WSTACK_RESTORE(s, &ctx->wstack); + result = ctx->result; + obj = ctx->obj; + } + } - tl = CDR(cons); - obj = CAR(cons); - WSTACK_PUSH(s, tl); - } else if (is_nil(obj)) { - result++; - goto outer_loop; - } else { - /* - * Other term (in the tail of a non-proper list or - * in a fun's environment). - */ +#define LIST_TAIL_OP ((0 << _TAG_PRIMARY_SIZE) | TAG_PRIMARY_HEADER) +#define TERM_ARRAY_OP(N) (((N) << _TAG_PRIMARY_SIZE) | TAG_PRIMARY_HEADER) +#define TERM_ARRAY_OP_DEC(OP) ((OP) - (1 << _TAG_PRIMARY_SIZE)) + + + for (;;) { + ASSERT(!is_header(obj)); + + if (ctx && --r == 0) { + *reds = r; + ctx->obj = obj; + ctx->result = result; + WSTACK_SAVE(s, &ctx->wstack); + return -1; } - - L_jump_start: switch (tag_val_def(obj)) { case NIL_DEF: result++; @@ -3028,26 +4051,79 @@ encode_size_struct2(ErtsAtomCacheMap *acmp, Eterm obj, unsigned dflags) result += m + 2 + 1; } else { result += 5; - goto handle_popped_obj; + WSTACK_PUSH2(s, (UWord)CDR(list_val(obj)), (UWord)LIST_TAIL_OP); + obj = CAR(list_val(obj)); + continue; /* big loop */ } break; case TUPLE_DEF: { Eterm* ptr = tuple_val(obj); - arity = arityval(*ptr); if (arity <= 0xff) { result += 1 + 1; } else { result += 1 + 4; } - ptr += arity; -#if HALFWORD_HEAP - obj = (Eterm) (wobj = (UWord) ptr); -#else - obj = (Eterm) ptr; -#endif - goto handle_popped_obj; + if (arity > 1) { + WSTACK_PUSH2(s, (UWord) (ptr + 2), + (UWord) TERM_ARRAY_OP(arity-1)); + } + else if (arity == 0) { + break; + } + obj = ptr[1]; + continue; /* big loop */ + } + case MAP_DEF: + if (is_flatmap(obj)) { + flatmap_t *mp = (flatmap_t*)flatmap_val(obj); + Uint size = flatmap_get_size(mp); + + result += 1 + 4; /* tag + 4 bytes size */ + + if (size) { + WSTACK_PUSH4(s, (UWord) flatmap_get_values(mp), + (UWord) TERM_ARRAY_OP(size), + (UWord) flatmap_get_keys(mp), + (UWord) TERM_ARRAY_OP(size)); + } + } else { + Eterm *ptr; + Eterm hdr; + Uint node_sz; + ptr = boxed_val(obj); + hdr = *ptr; + ASSERT(is_header(hdr)); + switch(hdr & _HEADER_MAP_SUBTAG_MASK) { + case HAMT_SUBTAG_HEAD_ARRAY: + ptr++; + node_sz = 16; + result += 1 + 4; /* tag + 4 bytes size */ + break; + case HAMT_SUBTAG_HEAD_BITMAP: + ptr++; + result += 1 + 4; /* tag + 4 bytes size */ + /*fall through*/ + case HAMT_SUBTAG_NODE_BITMAP: + node_sz = hashmap_bitcount(MAP_HEADER_VAL(hdr)); + ASSERT(node_sz < 17); + break; + default: + erts_exit(ERTS_ERROR_EXIT, "bad header\r\n"); + } + + ptr++; + WSTACK_RESERVE(s, node_sz*2); + while(node_sz--) { + if (is_list(*ptr)) { + WSTACK_FAST_PUSH(s, CAR(list_val(*ptr))); + WSTACK_FAST_PUSH(s, CDR(list_val(*ptr))); + } else { + WSTACK_FAST_PUSH(s, *ptr); + } + ptr++; + } } break; case FLOAT_DEF: @@ -3100,25 +4176,13 @@ encode_size_struct2(ErtsAtomCacheMap *acmp, Eterm obj, unsigned dflags) result += 2 * (1 + 4); /* Index + Uniq */ result += 1 + (funp->num_free < 0x100 ? 1 : 4); } - for (i = 1; i < funp->num_free; i++) { - obj = funp->env[i]; - - if (is_not_list(obj)) { - /* Push any non-list terms on the stack */ - WSTACK_PUSH(s, obj); - } else { - /* Lists must be handled specially. */ - if ((m = is_string(obj)) && (m < MAX_STRING_LEN)) { - result += m + 2 + 1; - } else { - result += 5; - WSTACK_PUSH(s, obj); - } - } + if (funp->num_free > 1) { + WSTACK_PUSH2(s, (UWord) (funp->env + 1), + (UWord) TERM_ARRAY_OP(funp->num_free-1)); } if (funp->num_free != 0) { obj = funp->env[0]; - goto L_jump_start; + continue; /* big loop */ } break; } @@ -3138,39 +4202,83 @@ encode_size_struct2(ErtsAtomCacheMap *acmp, Eterm obj, unsigned dflags) break; default: - erl_exit(1,"Internal data structure error (in encode_size_struct2)%x\n", + erts_exit(ERTS_ERROR_EXIT,"Internal data structure error (in encode_size_struct2)%x\n", obj); } + + if (WSTACK_ISEMPTY(s)) { + break; + } + obj = (Eterm) WSTACK_POP(s); + + if (is_header(obj)) { + switch (obj) { + case LIST_TAIL_OP: + obj = (Eterm) WSTACK_POP(s); + if (is_list(obj)) { + Eterm* cons = list_val(obj); + + WSTACK_PUSH2(s, (UWord)CDR(cons), (UWord)LIST_TAIL_OP); + obj = CAR(cons); + } + break; + + case TERM_ARRAY_OP(1): + obj = *(Eterm*)WSTACK_POP(s); + break; + default: { /* TERM_ARRAY_OP(N) when N > 1 */ + Eterm* ptr = (Eterm*) WSTACK_POP(s); + WSTACK_PUSH2(s, (UWord) (ptr+1), + (UWord) TERM_ARRAY_OP_DEC(obj)); + obj = *ptr; + } + } + } } - DESTROY_WSTACK(s); - return result; + WSTACK_DESTROY(s); + if (ctx) { + ASSERT(ctx->wstack.wstart == NULL); + *reds = r; + } + *res = result; + return 0; } -static int is_valid_utf8_atom(byte* bytes, Uint nbytes) -{ - byte* err_pos; - Uint num_chars; - /*SVERK Do we really need to validate correct utf8? */ - return nbytes <= MAX_ATOM_SZ_LIMIT - && erts_analyze_utf8(bytes, nbytes, &err_pos, &num_chars, NULL) == ERTS_UTF8_OK - && num_chars <= MAX_ATOM_CHARACTERS; -} static Sint -decoded_size(byte *ep, byte* endp, int internal_tags) +decoded_size(byte *ep, byte* endp, int internal_tags, B2TContext* ctx) { - int heap_size = 0; + int heap_size; int terms; - int atom_extra_skip = 0; + int atom_extra_skip; Uint n; + SWord reds; + + if (ctx) { + reds = ctx->reds; + if (ctx->u.sc.ep) { + heap_size = ctx->u.sc.heap_size; + terms = ctx->u.sc.terms; + ep = ctx->u.sc.ep; + atom_extra_skip = ctx->u.sc.atom_extra_skip; + goto init_done; + } + } + else + reds = 0; /* not used but compiler warns anyway */ + + heap_size = 0; + terms = 1; + atom_extra_skip = 0; +init_done: #define SKIP(sz) \ do { \ if ((sz) <= endp-ep) { \ ep += (sz); \ - } else { return -1; }; \ + } else { goto error; }; \ } while (0) #define SKIP2(sz1, sz2) \ @@ -3178,31 +4286,32 @@ decoded_size(byte *ep, byte* endp, int internal_tags) Uint sz = (sz1) + (sz2); \ if (sz1 < sz && (sz) <= endp-ep) { \ ep += (sz); \ - } else { return -1; } \ + } else { goto error; } \ } while (0) #define CHKSIZE(sz) \ do { \ - if ((sz) > endp-ep) { return -1; } \ + if ((sz) > endp-ep) { goto error; } \ } while (0) #define ADDTERMS(n) \ do { \ int before = terms; \ terms += (n); \ - if (terms < before) return -1; \ + if (terms < before) goto error; \ } while (0) - - for (terms=1; terms > 0; terms--) { - int tag; - + ASSERT(terms > 0); + do { + int tag; CHKSIZE(1); tag = ep++[0]; switch (tag) { case INTEGER_EXT: SKIP(4); +#if !defined(ARCH_64) || HALFWORD_HEAP heap_size += BIG_UINT_HEAP_SIZE; +#endif break; case SMALL_INTEGER_EXT: SKIP(1); @@ -3217,7 +4326,7 @@ decoded_size(byte *ep, byte* endp, int internal_tags) CHKSIZE(4); n = get_int32(ep); if (n > BIG_ARITY_MAX*sizeof(ErtsDigit)) { - return -1; + goto error; } SKIP2(n,4+1); /* skip, size,sign,digits */ heap_size += 1+1+(n+sizeof(Eterm)-1)/sizeof(Eterm); /* XXX: 1 too much? */ @@ -3226,7 +4335,7 @@ decoded_size(byte *ep, byte* endp, int internal_tags) CHKSIZE(2); n = get_int16(ep); if (n > MAX_ATOM_CHARACTERS) { - return -1; + goto error; } SKIP(n+2+atom_extra_skip); atom_extra_skip = 0; @@ -3235,8 +4344,8 @@ decoded_size(byte *ep, byte* endp, int internal_tags) CHKSIZE(2); n = get_int16(ep); ep += 2; - if (!is_valid_utf8_atom(ep, n)) { - return -1; + if (n > MAX_ATOM_SZ_LIMIT) { + goto error; } SKIP(n+atom_extra_skip); atom_extra_skip = 0; @@ -3245,7 +4354,7 @@ decoded_size(byte *ep, byte* endp, int internal_tags) CHKSIZE(1); n = get_int8(ep); if (n > MAX_ATOM_CHARACTERS) { - return -1; + goto error; } SKIP(n+1+atom_extra_skip); atom_extra_skip = 0; @@ -3254,8 +4363,8 @@ decoded_size(byte *ep, byte* endp, int internal_tags) CHKSIZE(1); n = get_int8(ep); ep++; - if (!is_valid_utf8_atom(ep, n)) { - return -1; + if (n > MAX_ATOM_SZ_LIMIT) { + goto error; } SKIP(n+atom_extra_skip); atom_extra_skip = 0; @@ -3284,7 +4393,7 @@ decoded_size(byte *ep, byte* endp, int internal_tags) id_words = get_int16(ep); if (id_words > ERTS_MAX_REF_NUMBERS) - return -1; + goto error; ep += 2; atom_extra_skip = 1 + 4*id_words; @@ -3326,6 +4435,17 @@ decoded_size(byte *ep, byte* endp, int internal_tags) ADDTERMS(n); heap_size += n + 1; break; + case MAP_EXT: + CHKSIZE(4); + n = get_int32(ep); + ep += 4; + ADDTERMS(2*n); + if (n <= MAP_SMALL_MAP_LIMIT) { + heap_size += 3 + n + 1 + n; + } else { + heap_size += HASHMAP_ESTIMATED_HEAP_SIZE(n); + } + break; case STRING_EXT: CHKSIZE(2); n = get_int16(ep); @@ -3386,7 +4506,7 @@ decoded_size(byte *ep, byte* endp, int internal_tags) num_free = get_int32(ep); ep += 4; if (num_free > MAX_ARG) { - return -1; + goto error; } terms += 4 + num_free; heap_size += ERL_FUN_SIZE + num_free; @@ -3403,24 +4523,47 @@ decoded_size(byte *ep, byte* endp, int internal_tags) case BINARY_INTERNAL_REF: if (!internal_tags) { - return -1; + goto error; } SKIP(sizeof(ProcBin)); heap_size += PROC_BIN_SIZE; break; case BIT_BINARY_INTERNAL_REF: if (!internal_tags) { - return -1; + goto error; } SKIP(2+sizeof(ProcBin)); heap_size += PROC_BIN_SIZE + ERL_SUB_BIN_SIZE; break; default: - return -1; + goto error; } - } + terms--; + + if (ctx && --reds <= 0 && terms > 0) { + ctx->u.sc.heap_size = heap_size; + ctx->u.sc.terms = terms; + ctx->u.sc.ep = ep; + ctx->u.sc.atom_extra_skip = atom_extra_skip; + ctx->reds = 0; + return 0; + } + }while (terms > 0); + /* 'terms' may be non-zero if it has wrapped around */ - return terms==0 ? heap_size : -1; + if (terms == 0) { + if (ctx) { + ctx->state = B2TDecodeInit; + ctx->reds = reds; + } + return heap_size; + } + +error: + if (ctx) { + ctx->state = B2TBadArg; + } + return -1; #undef SKIP #undef SKIP2 #undef CHKSIZE diff --git a/erts/emulator/beam/external.h b/erts/emulator/beam/external.h index e37d47919e..d12051c6b4 100644 --- a/erts/emulator/beam/external.h +++ b/erts/emulator/beam/external.h @@ -1,18 +1,19 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 1996-2013. All Rights Reserved. + * Copyright Ericsson AB 1996-2014. All Rights Reserved. * - * The contents of this file are subject to the Erlang Public License, - * Version 1.1, (the "License"); you may not use this file except in - * compliance with the License. You should have received a copy of the - * Erlang Public License along with this software. If not, it can be - * retrieved online at http://www.erlang.org/. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Software distributed under the License is distributed on an "AS IS" - * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See - * the License for the specific language governing rights and limitations - * under the License. + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. * * %CopyrightEnd% */ @@ -50,6 +51,7 @@ #define LARGE_BIG_EXT 'o' #define NEW_FUN_EXT 'p' #define EXPORT_EXT 'q' +#define MAP_EXT 't' #define FUN_EXT 'u' #define ATOM_UTF8_EXT 'v' #define SMALL_ATOM_UTF8_EXT 'w' @@ -138,29 +140,35 @@ typedef struct { #define ERTS_DIST_EXT_SIZE(EDEP) \ (sizeof(ErtsDistExternal) \ - (((EDEP)->flags & ERTS_DIST_EXT_ATOM_TRANS_TAB) \ - ? (ASSERT_EXPR(0 <= (EDEP)->attab.size \ - && (EDEP)->attab.size <= ERTS_ATOM_CACHE_SIZE), \ + ? (ASSERT(0 <= (EDEP)->attab.size \ + && (EDEP)->attab.size <= ERTS_ATOM_CACHE_SIZE), \ sizeof(Eterm)*(ERTS_ATOM_CACHE_SIZE - (EDEP)->attab.size)) \ : sizeof(ErtsAtomTranslationTable))) typedef struct { byte *extp; int exttmp; + Uint extsize; + Uint heap_size; } ErtsBinary2TermState; + /* -------------------------------------------------------------------------- */ void erts_init_atom_cache_map(ErtsAtomCacheMap *); void erts_reset_atom_cache_map(ErtsAtomCacheMap *); void erts_destroy_atom_cache_map(ErtsAtomCacheMap *); void erts_finalize_atom_cache_map(ErtsAtomCacheMap *, Uint32); -Uint erts_encode_ext_dist_header_size(ErtsAtomCacheMap *); Uint erts_encode_ext_dist_header_size(ErtsAtomCacheMap *); byte *erts_encode_ext_dist_header_setup(byte *, ErtsAtomCacheMap *); byte *erts_encode_ext_dist_header_finalize(byte *, ErtsAtomCache *, Uint32); -Uint erts_encode_dist_ext_size(Eterm, Uint32, ErtsAtomCacheMap *); -void erts_encode_dist_ext(Eterm, byte **, Uint32, ErtsAtomCacheMap *); +struct erts_dsig_send_context; +int erts_encode_dist_ext_size(Eterm, Uint32, ErtsAtomCacheMap*, Uint* szp); +int erts_encode_dist_ext_size_int(Eterm term, struct erts_dsig_send_context* ctx, Uint* szp); +struct TTBEncodeContext_; +int erts_encode_dist_ext(Eterm, byte **, Uint32, ErtsAtomCacheMap *, + struct TTBEncodeContext_ *, Sint* reds); Uint erts_encode_ext_size(Eterm); Uint erts_encode_ext_size_2(Eterm, unsigned); @@ -179,18 +187,18 @@ void erts_destroy_dist_ext_copy(ErtsDistExternal *); int erts_prepare_dist_ext(ErtsDistExternal *, byte *, Uint, DistEntry *, ErtsAtomCache *); Sint erts_decode_dist_ext_size(ErtsDistExternal *); -Eterm erts_decode_dist_ext(Eterm **, ErlOffHeap *, ErtsDistExternal *); +Eterm erts_decode_dist_ext(ErtsHeapFactory* factory, ErtsDistExternal *); Sint erts_decode_ext_size(byte*, Uint); Sint erts_decode_ext_size_ets(byte*, Uint); -Eterm erts_decode_ext(Eterm **, ErlOffHeap *, byte**); -Eterm erts_decode_ext_ets(Eterm **, ErlOffHeap *, byte*); +Eterm erts_decode_ext(ErtsHeapFactory*, byte**); +Eterm erts_decode_ext_ets(ErtsHeapFactory*, byte*); Eterm erts_term_to_binary(Process* p, Eterm Term, int level, Uint flags); Sint erts_binary2term_prepare(ErtsBinary2TermState *, byte *, Sint); void erts_binary2term_abort(ErtsBinary2TermState *); -Eterm erts_binary2term_create(ErtsBinary2TermState *, Eterm **hpp, ErlOffHeap *); +Eterm erts_binary2term_create(ErtsBinary2TermState *, ErtsHeapFactory*); int erts_debug_max_atom_out_cache_index(void); int erts_debug_atom_to_out_cache_index(Eterm); diff --git a/erts/emulator/beam/global.h b/erts/emulator/beam/global.h index 012c1c7e6a..1b8595fe57 100755..100644 --- a/erts/emulator/beam/global.h +++ b/erts/emulator/beam/global.h @@ -1,18 +1,19 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 1996-2013. All Rights Reserved. + * Copyright Ericsson AB 1996-2014. All Rights Reserved. * - * The contents of this file are subject to the Erlang Public License, - * Version 1.1, (the "License"); you may not use this file except in - * compliance with the License. You should have received a copy of the - * Erlang Public License along with this software. If not, it can be - * retrieved online at http://www.erlang.org/. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Software distributed under the License is distributed on an "AS IS" - * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See - * the License for the specific language governing rights and limitations - * under the License. + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. * * %CopyrightEnd% */ @@ -41,6 +42,7 @@ #include "error.h" #include "erl_utils.h" #include "erl_port.h" +#include "erl_gc.h" struct enif_environment_t /* ErlNifEnv */ { @@ -51,6 +53,7 @@ struct enif_environment_t /* ErlNifEnv */ ErlHeapFragment* heap_frag; int fpe_was_unmasked; struct enif_tmp_obj_t* tmp_obj_list; + int exception_thrown; /* boolean */ }; extern void erts_pre_nif(struct enif_environment_t*, Process*, struct erl_module_nif*); @@ -160,6 +163,7 @@ struct erts_driver_t_ { void (*ready_async)(ErlDrvData drv_data, ErlDrvThreadData thread_data); /* Might be NULL */ void (*process_exit)(ErlDrvData drv_data, ErlDrvMonitor *monitor); void (*stop_select)(ErlDrvEvent event, void*); /* Might be NULL */ + void (*emergency_close)(ErlDrvData drv_data); /* Might be NULL */ }; extern erts_driver_t *driver_list; @@ -186,11 +190,6 @@ extern void erts_ddll_remove_monitor(Process *p, extern Eterm erts_ddll_monitor_driver(Process *p, Eterm description, ErtsProcLocks plocks); -/* - * Max no. of drivers (linked in and dynamically loaded). Each table - * entry uses 4 bytes. - */ -#define DRIVER_TAB_SIZE 32 /* ** Just like the driver binary but with initial flags @@ -232,9 +231,23 @@ typedef struct { ERTS_INTERNAL_BINARY_FIELDS SWord orig_size; void (*destructor)(Binary *); - char magic_bin_data[1]; + union { + struct { + ERTS_BINARY_STRUCT_ALIGNMENT + char data[1]; + } aligned; + struct { + char data[1]; + } unaligned; + } u; } ErtsMagicBinary; +#ifdef ARCH_32 +#define ERTS_MAGIC_BIN_BYTES_TO_ALIGN 4 +#else +#define ERTS_MAGIC_BIN_BYTES_TO_ALIGN 0 +#endif + typedef union { Binary binary; ErtsMagicBinary magic_binary; @@ -254,15 +267,30 @@ typedef union { #define ERTS_MAGIC_BIN_DESTRUCTOR(BP) \ ((ErtsBinary *) (BP))->magic_binary.destructor #define ERTS_MAGIC_BIN_DATA(BP) \ - ((void *) ((ErtsBinary *) (BP))->magic_binary.magic_bin_data) -#define ERTS_MAGIC_BIN_DATA_SIZE(BP) \ - ((BP)->orig_size - sizeof(void (*)(Binary *))) + ((void *) ((ErtsBinary *) (BP))->magic_binary.u.aligned.data) +#define ERTS_MAGIC_DATA_OFFSET \ + (offsetof(ErtsMagicBinary,u.aligned.data) - offsetof(Binary,orig_bytes)) #define ERTS_MAGIC_BIN_ORIG_SIZE(Sz) \ - (sizeof(void (*)(Binary *)) + (Sz)) + (ERTS_MAGIC_DATA_OFFSET + (Sz)) #define ERTS_MAGIC_BIN_SIZE(Sz) \ - (offsetof(ErtsMagicBinary,magic_bin_data) + (Sz)) -#define ERTS_MAGIC_BIN_FROM_DATA(DATA) \ - ((ErtsBinary*)((char*)(DATA) - offsetof(ErtsMagicBinary,magic_bin_data))) + (offsetof(ErtsMagicBinary,u.aligned.data) + (Sz)) + +/* On 32-bit arch these macro variants will save memory + by not forcing 8-byte alignment for the magic payload. +*/ +#define ERTS_MAGIC_BIN_UNALIGNED_DATA(BP) \ + ((void *) ((ErtsBinary *) (BP))->magic_binary.u.unaligned.data) +#define ERTS_MAGIC_UNALIGNED_DATA_OFFSET \ + (offsetof(ErtsMagicBinary,u.unaligned.data) - offsetof(Binary,orig_bytes)) +#define ERTS_MAGIC_BIN_UNALIGNED_DATA_SIZE(BP) \ + ((BP)->orig_size - ERTS_MAGIC_UNALIGNED_DATA_OFFSET) +#define ERTS_MAGIC_BIN_UNALIGNED_ORIG_SIZE(Sz) \ + (ERTS_MAGIC_UNALIGNED_DATA_OFFSET + (Sz)) +#define ERTS_MAGIC_BIN_UNALIGNED_SIZE(Sz) \ + (offsetof(ErtsMagicBinary,u.unaligned.data) + (Sz)) +#define ERTS_MAGIC_BIN_FROM_UNALIGNED_DATA(DATA) \ + ((ErtsBinary*)((char*)(DATA) - offsetof(ErtsMagicBinary,u.unaligned.data))) + #define Binary2ErlDrvBinary(B) (&((ErtsBinary *) (B))->driver.binary) #define ErlDrvBinary2Binary(D) ((Binary *) \ @@ -351,8 +379,6 @@ extern Uint display_items; /* no of items to display in traces etc */ extern int erts_backtrace_depth; extern erts_smp_atomic32_t erts_max_gen_gcs; -extern int erts_disable_tolerant_timeofday; - extern int bif_reductions; /* reductions + fcalls (when doing call_bif) */ extern int stackdump_on_exit; @@ -375,113 +401,452 @@ extern int stackdump_on_exit; * DESTROY_ESTACK(Stack) */ +typedef struct ErtsEStack_ { + Eterm* start; + Eterm* sp; + Eterm* end; + Eterm* edefault; + ErtsAlcType_t alloc_type; +}ErtsEStack; -void erl_grow_stack(Eterm** start, Eterm** sp, Eterm** end); -#define ESTK_CONCAT(a,b) a##b -#define ESTK_SUBSCRIPT(s,i) *((Eterm *)((byte *)ESTK_CONCAT(s,_start) + (i))) #define DEF_ESTACK_SIZE (16) -#define DECLARE_ESTACK(s) \ - Eterm ESTK_CONCAT(s,_default_stack)[DEF_ESTACK_SIZE]; \ - Eterm* ESTK_CONCAT(s,_start) = ESTK_CONCAT(s,_default_stack); \ - Eterm* ESTK_CONCAT(s,_sp) = ESTK_CONCAT(s,_start); \ - Eterm* ESTK_CONCAT(s,_end) = ESTK_CONCAT(s,_start) + DEF_ESTACK_SIZE +void erl_grow_estack(ErtsEStack*, Uint need); +#define ESTK_CONCAT(a,b) a##b +#define ESTK_DEF_STACK(s) ESTK_CONCAT(s,_default_estack) + +#define DECLARE_ESTACK(s) \ + Eterm ESTK_DEF_STACK(s)[DEF_ESTACK_SIZE]; \ + ErtsEStack s = { \ + ESTK_DEF_STACK(s), /* start */ \ + ESTK_DEF_STACK(s), /* sp */ \ + ESTK_DEF_STACK(s) + DEF_ESTACK_SIZE, /* end */ \ + ESTK_DEF_STACK(s), /* default */ \ + ERTS_ALC_T_ESTACK /* alloc_type */ \ + } -#define DESTROY_ESTACK(s) \ +#define ESTACK_CHANGE_ALLOCATOR(s,t) \ do { \ - if (ESTK_CONCAT(s,_start) != ESTK_CONCAT(s,_default_stack)) { \ - erts_free(ERTS_ALC_T_ESTACK, ESTK_CONCAT(s,_start)); \ + if ((s).start != ESTK_DEF_STACK(s)) { \ + erts_exit(ERTS_ERROR_EXIT, "Internal error - trying to change allocator " \ + "type of active estack\n"); \ } \ + (s).alloc_type = (t); \ + } while (0) + +#define DESTROY_ESTACK(s) \ +do { \ + if ((s).start != ESTK_DEF_STACK(s)) { \ + erts_free((s).alloc_type, (s).start); \ + } \ } while(0) -#define ESTACK_PUSH(s, x) \ -do { \ - if (ESTK_CONCAT(s,_sp) == ESTK_CONCAT(s,_end)) { \ - erl_grow_stack(&ESTK_CONCAT(s,_start), &ESTK_CONCAT(s,_sp), \ - &ESTK_CONCAT(s,_end)); \ - } \ - *ESTK_CONCAT(s,_sp)++ = (x); \ + +/* + * Do not free the stack after this, it may have pointers into what + * was saved in 'dst'. + */ +#define ESTACK_SAVE(s,dst)\ +do {\ + if ((s).start == ESTK_DEF_STACK(s)) {\ + UWord _wsz = ESTACK_COUNT(s);\ + (dst)->start = erts_alloc((s).alloc_type,\ + DEF_ESTACK_SIZE * sizeof(Eterm));\ + memcpy((dst)->start, (s).start,_wsz*sizeof(Eterm));\ + (dst)->sp = (dst)->start + _wsz;\ + (dst)->end = (dst)->start + DEF_ESTACK_SIZE;\ + (dst)->edefault = NULL;\ + (dst)->alloc_type = (s).alloc_type;\ + } else\ + *(dst) = (s);\ + } while (0) + +#define DESTROY_SAVED_ESTACK(estack)\ +do {\ + if ((estack)->start) {\ + erts_free((estack)->alloc_type, (estack)->start);\ + (estack)->start = NULL;\ + }\ } while(0) -#define ESTACK_PUSH2(s, x, y) \ -do { \ - if (ESTK_CONCAT(s,_sp) > ESTK_CONCAT(s,_end) - 2) { \ - erl_grow_stack(&ESTK_CONCAT(s,_start), &ESTK_CONCAT(s,_sp), \ - &ESTK_CONCAT(s,_end)); \ - } \ - *ESTK_CONCAT(s,_sp)++ = (x); \ - *ESTK_CONCAT(s,_sp)++ = (y); \ +#define CLEAR_SAVED_ESTACK(estack) ((void) ((estack)->start = NULL)) + +/* + * Use on empty stack, only the allocator can be changed before this. + * The src stack is reset to NULL. + */ +#define ESTACK_RESTORE(s, src) \ +do { \ + ASSERT((s).start == ESTK_DEF_STACK(s)); \ + (s) = *(src); /* struct copy */ \ + (src)->start = NULL; \ + ASSERT((s).sp >= (s).start); \ + ASSERT((s).sp <= (s).end); \ +} while (0) + +#define ESTACK_IS_STATIC(s) ((s).start == ESTK_DEF_STACK(s)) + +#define ESTACK_PUSH(s, x) \ +do { \ + if ((s).sp == (s).end) { \ + erl_grow_estack(&(s), 1); \ + } \ + *(s).sp++ = (x); \ } while(0) -#define ESTACK_PUSH3(s, x, y, z) \ -do { \ - if (ESTK_CONCAT(s,_sp) > ESTK_CONCAT(s,_end) - 3) { \ - erl_grow_stack(&ESTK_CONCAT(s,_start), &ESTK_CONCAT(s,_sp), \ - &ESTK_CONCAT(s,_end)); \ - } \ - *ESTK_CONCAT(s,_sp)++ = (x); \ - *ESTK_CONCAT(s,_sp)++ = (y); \ - *ESTK_CONCAT(s,_sp)++ = (z); \ +#define ESTACK_PUSH2(s, x, y) \ +do { \ + if ((s).sp > (s).end - 2) { \ + erl_grow_estack(&(s), 2); \ + } \ + *(s).sp++ = (x); \ + *(s).sp++ = (y); \ } while(0) -#define ESTACK_COUNT(s) (ESTK_CONCAT(s,_sp) - ESTK_CONCAT(s,_start)) +#define ESTACK_PUSH3(s, x, y, z) \ +do { \ + if ((s).sp > (s).end - 3) { \ + erl_grow_estack(&s, 3); \ + } \ + *(s).sp++ = (x); \ + *(s).sp++ = (y); \ + *(s).sp++ = (z); \ +} while(0) -#define ESTACK_ISEMPTY(s) (ESTK_CONCAT(s,_sp) == ESTK_CONCAT(s,_start)) -#define ESTACK_POP(s) (*(--ESTK_CONCAT(s,_sp))) +#define ESTACK_PUSH4(s, E1, E2, E3, E4) \ +do { \ + if ((s).sp > (s).end - 4) { \ + erl_grow_estack(&s, 4); \ + } \ + *(s).sp++ = (E1); \ + *(s).sp++ = (E2); \ + *(s).sp++ = (E3); \ + *(s).sp++ = (E4); \ +} while(0) +#define ESTACK_RESERVE(s, push_cnt) \ +do { \ + if ((s).sp > (s).end - (push_cnt)) { \ + erl_grow_estack(&(s), (push_cnt)); \ + } \ +} while(0) + +/* Must be preceded by ESTACK_RESERVE */ +#define ESTACK_FAST_PUSH(s, x) \ +do { \ + ASSERT((s).sp < (s).end); \ + *s.sp++ = (x); \ +} while(0) + +#define ESTACK_COUNT(s) ((s).sp - (s).start) +#define ESTACK_ISEMPTY(s) ((s).sp == (s).start) +#define ESTACK_POP(s) (*(--(s).sp)) + + +/* + * WSTACK: same as ESTACK but with UWord instead of Eterm + */ + +typedef struct ErtsWStack_ { + UWord* wstart; + UWord* wsp; + UWord* wend; + UWord* wdefault; + ErtsAlcType_t alloc_type; +}ErtsWStack; -void erl_grow_wstack(UWord** start, UWord** sp, UWord** end); -#define WSTK_CONCAT(a,b) a##b -#define WSTK_SUBSCRIPT(s,i) *((UWord *)((byte *)WSTK_CONCAT(s,_start) + (i))) #define DEF_WSTACK_SIZE (16) -#define DECLARE_WSTACK(s) \ - UWord WSTK_CONCAT(s,_default_stack)[DEF_WSTACK_SIZE]; \ - UWord* WSTK_CONCAT(s,_start) = WSTK_CONCAT(s,_default_stack); \ - UWord* WSTK_CONCAT(s,_sp) = WSTK_CONCAT(s,_start); \ - UWord* WSTK_CONCAT(s,_end) = WSTK_CONCAT(s,_start) + DEF_WSTACK_SIZE +void erl_grow_wstack(ErtsWStack*, Uint need); +#define WSTK_CONCAT(a,b) a##b +#define WSTK_DEF_STACK(s) WSTK_CONCAT(s,_default_wstack) + +#define WSTACK_DECLARE(s) \ + UWord WSTK_DEF_STACK(s)[DEF_WSTACK_SIZE]; \ + ErtsWStack s = { \ + WSTK_DEF_STACK(s), /* wstart */ \ + WSTK_DEF_STACK(s), /* wsp */ \ + WSTK_DEF_STACK(s) + DEF_WSTACK_SIZE, /* wend */ \ + WSTK_DEF_STACK(s), /* wdflt */ \ + ERTS_ALC_T_ESTACK /* alloc_type */ \ + } +#define DECLARE_WSTACK WSTACK_DECLARE + +typedef struct ErtsDynamicWStack_ { + UWord default_stack[DEF_WSTACK_SIZE]; + ErtsWStack ws; +}ErtsDynamicWStack; + +#define WSTACK_INIT(dwsp, ALC_TYPE) \ +do { \ + (dwsp)->ws.wstart = (dwsp)->default_stack; \ + (dwsp)->ws.wsp = (dwsp)->default_stack; \ + (dwsp)->ws.wend = (dwsp)->default_stack + DEF_WSTACK_SIZE;\ + (dwsp)->ws.wdefault = (dwsp)->default_stack; \ + (dwsp)->ws.alloc_type = ALC_TYPE; \ +} while (0) -#define DESTROY_WSTACK(s) \ +#define WSTACK_CHANGE_ALLOCATOR(s,t) \ do { \ - if (WSTK_CONCAT(s,_start) != WSTK_CONCAT(s,_default_stack)) { \ - erts_free(ERTS_ALC_T_ESTACK, WSTK_CONCAT(s,_start)); \ + if (s.wstart != WSTK_DEF_STACK(s)) { \ + erts_exit(ERTS_ERROR_EXIT, "Internal error - trying to change allocator " \ + "type of active wstack\n"); \ } \ + s.alloc_type = (t); \ + } while (0) + +#define WSTACK_DESTROY(s) \ +do { \ + if (s.wstart != s.wdefault) { \ + erts_free(s.alloc_type, s.wstart); \ + } \ } while(0) +#define DESTROY_WSTACK WSTACK_DESTROY -#define WSTACK_PUSH(s, x) \ -do { \ - if (WSTK_CONCAT(s,_sp) == WSTK_CONCAT(s,_end)) { \ - erl_grow_wstack(&WSTK_CONCAT(s,_start), &WSTK_CONCAT(s,_sp), \ - &WSTK_CONCAT(s,_end)); \ - } \ - *WSTK_CONCAT(s,_sp)++ = (x); \ +#define WSTACK_DEBUG(s) \ + do { \ + fprintf(stderr, "wstack size = %ld\r\n", s.wsp - s.wstart); \ + fprintf(stderr, "wstack wstart = %p\r\n", s.wstart); \ + fprintf(stderr, "wstack wsp = %p\r\n", s.wsp); \ + } while(0) + +/* + * Do not free the stack after this, it may have pointers into what + * was saved in 'dst'. + */ +#define WSTACK_SAVE(s,dst)\ +do {\ + if (s.wstart == WSTK_DEF_STACK(s)) {\ + UWord _wsz = WSTACK_COUNT(s);\ + (dst)->wstart = erts_alloc(s.alloc_type,\ + DEF_WSTACK_SIZE * sizeof(UWord));\ + memcpy((dst)->wstart, s.wstart,_wsz*sizeof(UWord));\ + (dst)->wsp = (dst)->wstart + _wsz;\ + (dst)->wend = (dst)->wstart + DEF_WSTACK_SIZE;\ + (dst)->wdefault = NULL;\ + (dst)->alloc_type = s.alloc_type;\ + } else\ + *(dst) = s;\ + } while (0) + +#define DESTROY_SAVED_WSTACK(wstack)\ +do {\ + if ((wstack)->wstart) {\ + erts_free((wstack)->alloc_type, (wstack)->wstart);\ + (wstack)->wstart = NULL;\ + }\ } while(0) -#define WSTACK_PUSH2(s, x, y) \ -do { \ - if (WSTK_CONCAT(s,_sp) > WSTK_CONCAT(s,_end) - 2) { \ - erl_grow_wstack(&WSTK_CONCAT(s,_start), &WSTK_CONCAT(s,_sp), \ - &WSTK_CONCAT(s,_end)); \ - } \ - *WSTK_CONCAT(s,_sp)++ = (x); \ - *WSTK_CONCAT(s,_sp)++ = (y); \ +#define CLEAR_SAVED_WSTACK(wstack) ((void) ((wstack)->wstart = NULL)) + +/* + * Use on empty stack, only the allocator can be changed before this. + * The src stack is reset to NULL. + */ +#define WSTACK_RESTORE(s, src) \ +do { \ + ASSERT(s.wstart == WSTK_DEF_STACK(s)); \ + s = *(src); /* struct copy */ \ + (src)->wstart = NULL; \ + ASSERT(s.wsp >= s.wstart); \ + ASSERT(s.wsp <= s.wend); \ +} while (0) + +#define WSTACK_IS_STATIC(s) (s.wstart == WSTK_DEF_STACK(s)) + +#define WSTACK_PUSH(s, x) \ +do { \ + if (s.wsp == s.wend) { \ + erl_grow_wstack(&s, 1); \ + } \ + *s.wsp++ = (x); \ +} while(0) + +#define WSTACK_PUSH2(s, x, y) \ +do { \ + if (s.wsp > s.wend - 2) { \ + erl_grow_wstack(&s, 2); \ + } \ + *s.wsp++ = (x); \ + *s.wsp++ = (y); \ +} while(0) + +#define WSTACK_PUSH3(s, x, y, z) \ +do { \ + if (s.wsp > s.wend - 3) { \ + erl_grow_wstack(&s, 3); \ + } \ + *s.wsp++ = (x); \ + *s.wsp++ = (y); \ + *s.wsp++ = (z); \ +} while(0) + +#define WSTACK_PUSH4(s, A1, A2, A3, A4) \ +do { \ + if (s.wsp > s.wend - 4) { \ + erl_grow_wstack(&s, 4); \ + } \ + *s.wsp++ = (A1); \ + *s.wsp++ = (A2); \ + *s.wsp++ = (A3); \ + *s.wsp++ = (A4); \ +} while(0) + +#define WSTACK_PUSH5(s, A1, A2, A3, A4, A5) \ +do { \ + if (s.wsp > s.wend - 5) { \ + erl_grow_wstack(&s, 5); \ + } \ + *s.wsp++ = (A1); \ + *s.wsp++ = (A2); \ + *s.wsp++ = (A3); \ + *s.wsp++ = (A4); \ + *s.wsp++ = (A5); \ +} while(0) + +#define WSTACK_PUSH6(s, A1, A2, A3, A4, A5, A6) \ +do { \ + if (s.wsp > s.wend - 6) { \ + erl_grow_wstack(&s, 6); \ + } \ + *s.wsp++ = (A1); \ + *s.wsp++ = (A2); \ + *s.wsp++ = (A3); \ + *s.wsp++ = (A4); \ + *s.wsp++ = (A5); \ + *s.wsp++ = (A6); \ } while(0) -#define WSTACK_PUSH3(s, x, y, z) \ +#define WSTACK_RESERVE(s, push_cnt) \ +do { \ + if (s.wsp > s.wend - (push_cnt)) { \ + erl_grow_wstack(&s, (push_cnt)); \ + } \ +} while(0) + +/* Must be preceded by WSTACK_RESERVE */ +#define WSTACK_FAST_PUSH(s, x) \ +do { \ + ASSERT(s.wsp < s.wend); \ + *s.wsp++ = (x); \ +} while(0) + +#define WSTACK_COUNT(s) (s.wsp - s.wstart) +#define WSTACK_ISEMPTY(s) (s.wsp == s.wstart) +#define WSTACK_POP(s) ((ASSERT(s.wsp > s.wstart)),*(--s.wsp)) + +#define WSTACK_ROLLBACK(s, count) (ASSERT(WSTACK_COUNT(s) >= (count)), \ + s.wsp = s.wstart + (count)) + +/* PSTACK - Stack of any type. + * Usage: + * { + * #define PSTACK_TYPE MyType + * PSTACK_DECLARE(s,16); + * MyType *sp = PSTACK_PUSH(s); + * + * sp->x = .... + * sp->y = .... + * sp = PSTACK_PUSH(s); + * ... + * sp = PSTACK_POP(s); + * if (PSTACK_IS_EMPTY(s)) { + * // sp is invalid when stack is empty after pop + * } + * + * PSTACK_DESTROY(s); + * } + */ + + +typedef struct ErtsPStack_ { + byte* pstart; + byte* psp; + byte* pend; + ErtsAlcType_t alloc_type; +}ErtsPStack; + +void erl_grow_pstack(ErtsPStack* s, void* default_pstack, unsigned need_bytes); +#define PSTK_CONCAT(a,b) a##b +#define PSTK_DEF_STACK(s) PSTK_CONCAT(s,_default_pstack) + +#define PSTACK_DECLARE(s, DEF_PSTACK_SIZE) \ +PSTACK_TYPE PSTK_DEF_STACK(s)[DEF_PSTACK_SIZE]; \ +ErtsPStack s = { (byte*)PSTK_DEF_STACK(s), /* pstart */ \ + (byte*)(PSTK_DEF_STACK(s) - 1), /* psp */ \ + (byte*)(PSTK_DEF_STACK(s) + (DEF_PSTACK_SIZE)), /* pend */\ + ERTS_ALC_T_ESTACK /* alloc_type */ \ +} + +#define PSTACK_CHANGE_ALLOCATOR(s,t) \ do { \ - if (WSTK_CONCAT(s,_sp) > WSTK_CONCAT(s,_end) - 3) { \ - erl_grow_wstack(&WSTK_CONCAT(s,_start), &WSTK_CONCAT(s,_sp), \ - &WSTK_CONCAT(s,_end)); \ + if (s.pstart != (byte*)PSTK_DEF_STACK(s)) { \ + erts_exit(ERTS_ERROR_EXIT, "Internal error - trying to change allocator " \ + "type of active pstack\n"); \ } \ - *WSTK_CONCAT(s,_sp)++ = (x); \ - *WSTK_CONCAT(s,_sp)++ = (y); \ - *WSTK_CONCAT(s,_sp)++ = (z); \ + s.alloc_type = (t); \ + } while (0) + +#define PSTACK_DESTROY(s) \ +do { \ + if (s.pstart != (byte*)PSTK_DEF_STACK(s)) { \ + erts_free(s.alloc_type, s.pstart); \ + } \ } while(0) -#define WSTACK_COUNT(s) (WSTK_CONCAT(s,_sp) - WSTK_CONCAT(s,_start)) +#define PSTACK_IS_EMPTY(s) (s.psp < s.pstart) + +#define PSTACK_COUNT(s) (((PSTACK_TYPE*)s.psp + 1) - (PSTACK_TYPE*)s.pstart) + +#define PSTACK_TOP(s) (ASSERT(!PSTACK_IS_EMPTY(s)), (PSTACK_TYPE*)(s.psp)) + +#define PSTACK_PUSH(s) \ + (s.psp += sizeof(PSTACK_TYPE), \ + ((s.psp == s.pend) ? erl_grow_pstack(&s, PSTK_DEF_STACK(s), \ + sizeof(PSTACK_TYPE)) : (void)0), \ + ((PSTACK_TYPE*) s.psp)) + +#define PSTACK_POP(s) ((PSTACK_TYPE*) (s.psp -= sizeof(PSTACK_TYPE))) + +/* + * Do not free the stack after this, it may have pointers into what + * was saved in 'dst'. + */ +#define PSTACK_SAVE(s,dst)\ +do {\ + if (s.pstart == (byte*)PSTK_DEF_STACK(s)) {\ + UWord _pbytes = PSTACK_COUNT(s) * sizeof(PSTACK_TYPE);\ + (dst)->pstart = erts_alloc(s.alloc_type,\ + sizeof(PSTK_DEF_STACK(s)));\ + sys_memcpy((dst)->pstart, s.pstart, _pbytes);\ + (dst)->psp = (dst)->pstart + _pbytes - sizeof(PSTACK_TYPE);\ + (dst)->pend = (dst)->pstart + sizeof(PSTK_DEF_STACK(s));\ + (dst)->alloc_type = s.alloc_type;\ + } else\ + *(dst) = s;\ + } while (0) + +/* + * Use on empty stack, only the allocator can be changed before this. + * The src stack is reset to NULL. + */ +#define PSTACK_RESTORE(s, src) \ +do { \ + ASSERT(s.pstart == (byte*)PSTK_DEF_STACK(s)); \ + s = *(src); /* struct copy */ \ + (src)->pstart = NULL; \ + ASSERT(s.psp >= (s.pstart - sizeof(PSTACK_TYPE))); \ + ASSERT(s.psp < s.pend); \ +} while (0) + +#define PSTACK_DESTROY_SAVED(pstack)\ +do {\ + if ((pstack)->pstart) {\ + erts_free((pstack)->alloc_type, (pstack)->pstart);\ + (pstack)->pstart = NULL;\ + }\ +} while(0) -#define WSTACK_ISEMPTY(s) (WSTK_CONCAT(s,_sp) == WSTK_CONCAT(s,_start)) -#define WSTACK_POP(s) (*(--WSTK_CONCAT(s,_sp))) /* binary.c */ @@ -503,9 +868,6 @@ erts_bld_port_info(Eterm **hpp, void erts_bif_info_init(void); /* bif.c */ -Eterm erts_make_ref(Process *); -Eterm erts_make_ref_in_buffer(Eterm buffer[REF_THING_SIZE]); -void erts_make_ref_in_array(Uint32 ref[ERTS_MAX_REF_NUMBERS]); ERTS_GLB_INLINE Eterm erts_proc_store_ref(Process *c_p, Uint32 ref[ERTS_MAX_REF_NUMBERS]); @@ -537,6 +899,10 @@ Eterm erl_send(Process *p, Eterm to, Eterm msg); Eterm erl_is_function(Process* p, Eterm arg1, Eterm arg2); +/* beam_bif_load.c */ +Eterm erts_check_process_code(Process *c_p, Eterm module, int allow_gc, int *redsp); + + /* beam_load.c */ typedef struct { BeamInstr* current; /* Pointer to: Mod, Name, Arity */ @@ -581,10 +947,13 @@ void print_process_info(int, void *, Process*); void info(int, void *); void loaded(int, void *); +/* erl_arith.c */ +double erts_get_positive_zero_float(void); + /* config.c */ -__decl_noreturn void __noreturn erl_exit(int n, char*, ...); -__decl_noreturn void __noreturn erl_exit_flush_async(int n, char*, ...); +__decl_noreturn void __noreturn erts_exit(int n, char*, ...); +__decl_noreturn void __noreturn erts_flush_async_exit(int n, char*, ...); void erl_error(char*, va_list); /* copy.c */ @@ -686,29 +1055,6 @@ void MD5Init(MD5_CTX *); void MD5Update(MD5_CTX *, unsigned char *, unsigned int); void MD5Final(unsigned char [16], MD5_CTX *); -/* ggc.c */ - - -typedef struct { - Uint garbage_collections; - Uint reclaimed; -} ErtsGCInfo; - -void erts_gc_info(ErtsGCInfo *gcip); -void erts_init_gc(void); -int erts_garbage_collect(Process*, int, Eterm*, int); -void erts_garbage_collect_hibernate(Process* p); -Eterm erts_gc_after_bif_call(Process* p, Eterm result, Eterm* regs, Uint arity); -void erts_garbage_collect_literals(Process* p, Eterm* literals, - Uint lit_size, - struct erl_off_heap_header* oh); -Uint erts_next_heap_size(Uint, Uint); -Eterm erts_heap_sizes(Process* p); - -void erts_offset_off_heap(ErlOffHeap *, Sint, Eterm*, Eterm*); -void erts_offset_heap_ptr(Eterm*, Uint, Sint, Eterm*, Eterm*); -void erts_offset_heap(Eterm*, Uint, Sint, Eterm*, Eterm*); -void erts_free_heap_frags(Process* p); /* io.c */ @@ -724,41 +1070,50 @@ int erts_add_driver_entry(ErlDrvEntry *drv, DE_Handle *handle, int driver_list_l void erts_destroy_driver(erts_driver_t *drv); int erts_save_suspend_process_on_port(Port*, Process*); Port *erts_open_driver(erts_driver_t*, Eterm, char*, SysDriverOpts*, int *, int *); -void erts_init_io(int, int); +void erts_init_io(int, int, int); void erts_raw_port_command(Port*, byte*, Uint); void driver_report_exit(ErlDrvPort, int); LineBuf* allocate_linebuf(int); int async_ready(Port *, void*); -ErtsPortNames *erts_get_port_names(Eterm); +ErtsPortNames *erts_get_port_names(Eterm, ErlDrvPort); void erts_free_port_names(ErtsPortNames *); Uint erts_port_ioq_size(Port *pp); -void erts_stale_drv_select(Eterm, ErlDrvEvent, int, int); +void erts_stale_drv_select(Eterm, ErlDrvPort, ErlDrvEvent, int, int); Port *erts_get_heart_port(void); +void erts_emergency_close_ports(void); #if defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_COUNT) void erts_lcnt_enable_io_lock_count(int enable); #endif +/* driver_tab.c */ +typedef void *(*ErtsStaticNifInitFPtr)(void); +ErtsStaticNifInitFPtr erts_static_nif_get_nif_init(const char *name, int len); +int erts_is_static_nif(void *handle); +void erts_init_static_drivers(void); + /* erl_drv_thread.c */ void erl_drv_thr_init(void); /* utils.c */ void erts_cleanup_offheap(ErlOffHeap *offheap); +Uint64 erts_timestamp_millis(void); + Export* erts_find_function(Eterm, Eterm, unsigned int, ErtsCodeIndex); Eterm store_external_or_ref_in_proc_(Process *, Eterm); Eterm store_external_or_ref_(Uint **, ErlOffHeap*, Eterm); #define NC_HEAP_SIZE(NC) \ - (ASSERT_EXPR(is_node_container((NC))), \ + (ASSERT(is_node_container((NC))), \ IS_CONST((NC)) ? 0 : (thing_arityval(*boxed_val((NC))) + 1)) #define STORE_NC(Hpp, ETpp, NC) \ - (ASSERT_EXPR(is_node_container((NC))), \ + (ASSERT(is_node_container((NC))), \ IS_CONST((NC)) ? (NC) : store_external_or_ref_((Hpp), (ETpp), (NC))) #define STORE_NC_IN_PROC(Pp, NC) \ - (ASSERT_EXPR(is_node_container((NC))), \ + (ASSERT(is_node_container((NC))), \ IS_CONST((NC)) ? (NC) : store_external_or_ref_in_proc_((Pp), (NC))) /* duplicates from big.h */ @@ -777,6 +1132,12 @@ Sint erts_re_set_loop_limit(Sint limit); void erts_init_bif_binary(void); Sint erts_binary_set_loop_limit(Sint limit); +/* external.c */ +void erts_init_external(void); + +/* erl_map.c */ +void erts_init_map(void); + /* erl_unicode.c */ void erts_init_unicode(void); Sint erts_unicode_set_loop_limit(Sint limit); @@ -794,6 +1155,17 @@ char *erts_convert_filename_to_native(Eterm name, char *statbuf, ErtsAlcType_t alloc_type, int allow_empty, int allow_atom, Sint *used /* out */); +char *erts_convert_filename_to_encoding(Eterm name, char *statbuf, + size_t statbuf_size, + ErtsAlcType_t alloc_type, + int allow_empty, int allow_atom, + int encoding, + Sint *used /* out */, + Uint extra); +char* erts_convert_filename_to_wchar(byte* bytes, Uint size, + char *statbuf, size_t statbuf_size, + ErtsAlcType_t alloc_type, Sint* used, + Uint extra_wchars); Eterm erts_convert_native_to_filename(Process *p, byte *bytes); Eterm erts_utf8_to_list(Process *p, Uint num, byte *bytes, Uint sz, Uint left, Uint *num_built, Uint *num_eaten, Eterm tail); @@ -816,20 +1188,67 @@ struct Sint_buf { }; char* Sint_to_buf(Sint, struct Sint_buf*); +#define ERTS_IOLIST_STATE_INITER(C_P, OBJ) \ + {(C_P), 0, 0, (OBJ), {NULL, NULL, NULL, ERTS_ALC_T_INVALID}, 0, 0} + +#define ERTS_IOLIST_STATE_MOVE(TO, FROM) \ + sys_memcpy((void *) (TO), (void *) (FROM), sizeof(ErtsIOListState)) + +#define ERTS_IOLIST_SIZE_YIELDS_COUNT_PER_RED 8 + +typedef struct { + Process *c_p; + ErlDrvSizeT size; + Uint offs; + Eterm obj; + ErtsEStack estack; + int reds_left; + int have_size; +} ErtsIOListState; + +#define ERTS_IOLIST2BUF_STATE_INITER(C_P, OBJ) \ + {ERTS_IOLIST_STATE_INITER((C_P), (OBJ)), {NULL, 0, 0, 0}, NULL, 0, NULL, 0} + +#define ERTS_IOLIST2BUF_STATE_MOVE(TO, FROM) \ + sys_memcpy((void *) (TO), (void *) (FROM), sizeof(ErtsIOList2BufState)) + +#define ERTS_IOLIST_TO_BUF_BYTES_PER_YIELD_COUNT 32 +#define ERTS_IOLIST_TO_BUF_YIELD_COUNT_PER_RED 8 +#define ERTS_IOLIST_TO_BUF_BYTES_PER_RED \ + (ERTS_IOLIST_TO_BUF_YIELD_COUNT_PER_RED*ERTS_IOLIST_TO_BUF_BYTES_PER_YIELD_COUNT) + +typedef struct { + ErtsIOListState iolist; + struct { + byte *bptr; + size_t size; + Uint bitoffs; + Uint bitsize; + } bcopy; + char *buf; + ErlDrvSizeT len; + Eterm *objp; + int offset; +} ErtsIOList2BufState; + #define ERTS_IOLIST_OK 0 #define ERTS_IOLIST_OVERFLOW 1 #define ERTS_IOLIST_TYPE 2 +#define ERTS_IOLIST_YIELD 3 -Eterm buf_to_intlist(Eterm**, char*, size_t, Eterm); /* most callers pass plain char*'s */ +Eterm buf_to_intlist(Eterm**, const char*, size_t, Eterm); /* most callers pass plain char*'s */ #define ERTS_IOLIST_TO_BUF_OVERFLOW (~((ErlDrvSizeT) 0)) #define ERTS_IOLIST_TO_BUF_TYPE_ERROR (~((ErlDrvSizeT) 1)) +#define ERTS_IOLIST_TO_BUF_YIELD (~((ErlDrvSizeT) 2)) #define ERTS_IOLIST_TO_BUF_FAILED(R) \ - (((R) & (~((ErlDrvSizeT) 1))) == (~((ErlDrvSizeT) 1))) + (((R) & (~((ErlDrvSizeT) 3))) == (~((ErlDrvSizeT) 3))) #define ERTS_IOLIST_TO_BUF_SUCCEEDED(R) \ (!ERTS_IOLIST_TO_BUF_FAILED((R))) ErlDrvSizeT erts_iolist_to_buf(Eterm, char*, ErlDrvSizeT); +ErlDrvSizeT erts_iolist_to_buf_yielding(ErtsIOList2BufState *); +int erts_iolist_size_yielding(ErtsIOListState *state); int erts_iolist_size(Eterm, ErlDrvSizeT *); int is_string(Eterm); void erl_at_exit(void (*) (void*), void*); @@ -863,6 +1282,7 @@ Eterm erts_gc_length_1(Process* p, Eterm* reg, Uint live); Eterm erts_gc_size_1(Process* p, Eterm* reg, Uint live); Eterm erts_gc_bit_size_1(Process* p, Eterm* reg, Uint live); Eterm erts_gc_byte_size_1(Process* p, Eterm* reg, Uint live); +Eterm erts_gc_map_size_1(Process* p, Eterm* reg, Uint live); Eterm erts_gc_abs_1(Process* p, Eterm* reg, Uint live); Eterm erts_gc_float_1(Process* p, Eterm* reg, Uint live); Eterm erts_gc_round_1(Process* p, Eterm* reg, Uint live); @@ -901,9 +1321,10 @@ Eterm erts_match_set_lint(Process *p, Eterm matchexpr); extern void erts_match_set_release_result(Process* p); enum erts_pam_run_flags { - ERTS_PAM_TMP_RESULT=0, - ERTS_PAM_COPY_RESULT=1, - ERTS_PAM_CONTIGUOUS_TUPLE=2 + ERTS_PAM_TMP_RESULT=1, + ERTS_PAM_COPY_RESULT=2, + ERTS_PAM_CONTIGUOUS_TUPLE=4, + ERTS_PAM_IGNORE_TRACE_SILENT=8 }; extern Eterm erts_match_set_run(Process *p, Binary *mpsp, Eterm *args, int num_args, @@ -925,6 +1346,8 @@ extern erts_driver_t vanilla_driver; extern erts_driver_t spawn_driver; extern erts_driver_t fd_driver; +int erts_beam_jump_table(void); + /* Should maybe be placed in erl_message.h, but then we get an include mess. */ ERTS_GLB_INLINE Eterm * erts_alloc_message_heap_state(Uint size, @@ -969,12 +1392,13 @@ erts_alloc_message_heap_state(Uint size, state = erts_smp_atomic32_read_acqb(&receiver->state); if (statep) *statep = state; - if (state & (ERTS_PSFLG_EXITING|ERTS_PSFLG_PENDING_EXIT)) + if (state & (ERTS_PSFLG_EXITING + | ERTS_PSFLG_PENDING_EXIT)) goto allocate_in_mbuf; #endif if (size > (Uint) INT_MAX) - erl_exit(ERTS_ABORT_EXIT, "HUGE size (%beu)\n", size); + erts_exit(ERTS_ABORT_EXIT, "HUGE size (%beu)\n", size); if ( #if defined(ERTS_SMP) @@ -989,8 +1413,14 @@ erts_alloc_message_heap_state(Uint size, state = erts_smp_atomic32_read_nob(&receiver->state); if (statep) *statep = state; - if ((state & (ERTS_PSFLG_EXITING|ERTS_PSFLG_PENDING_EXIT)) + if ((state & (ERTS_PSFLG_EXITING + | ERTS_PSFLG_PENDING_EXIT)) + || (receiver->flags & F_DISABLE_GC) || HEAP_LIMIT(receiver) - HEAP_TOP(receiver) <= size) { + /* + * The heap is either potentially in an inconsistent + * state, or not large enough. + */ #ifdef ERTS_SMP if (locked_main) { *receiver_locks &= ~ERTS_PROC_LOCK_MAIN; @@ -1100,6 +1530,13 @@ erts_alloc_message_heap(Uint size, # define UnUseTmpHeapNoproc(Size) /* Nothing */ #endif /* HEAP_ON_C_STACK */ +ERTS_GLB_INLINE void dtrace_pid_str(Eterm pid, char *process_buf); +ERTS_GLB_INLINE void dtrace_proc_str(Process *process, char *process_buf); +ERTS_GLB_INLINE void dtrace_port_str(Port *port, char *port_buf); +ERTS_GLB_INLINE void dtrace_fun_decode(Process *process, + Eterm module, Eterm function, int arity, + char *process_buf, char *mfa_buf); + #if ERTS_GLB_INLINE_INCL_FUNC_DEF #include "dtrace-wrapper.h" diff --git a/erts/emulator/beam/hash.c b/erts/emulator/beam/hash.c index afaf32f8ce..895fe657d1 100644 --- a/erts/emulator/beam/hash.c +++ b/erts/emulator/beam/hash.c @@ -3,16 +3,17 @@ * * Copyright Ericsson AB 1996-2009. All Rights Reserved. * - * The contents of this file are subject to the Erlang Public License, - * Version 1.1, (the "License"); you may not use this file except in - * compliance with the License. You should have received a copy of the - * Erlang Public License along with this software. If not, it can be - * retrieved online at http://www.erlang.org/. - * - * Software distributed under the License is distributed on an "AS IS" - * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See - * the License for the specific language governing rights and limitations - * under the License. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. * * %CopyrightEnd% */ @@ -132,7 +133,7 @@ Hash* hash_init(ErtsAlcType_t type, Hash* h, char* name, int size, HashFunctions while (h_size_table[ix] != -1 && h_size_table[ix] < size) ix++; if (h_size_table[ix] == -1) - erl_exit(1, "panic: too large hash table size (%d)\n", size); + erts_exit(ERTS_ERROR_EXIT, "panic: too large hash table size (%d)\n", size); size = h_size_table[ix]; sz = size*sizeof(HashBucket*); diff --git a/erts/emulator/beam/hash.h b/erts/emulator/beam/hash.h index 6dd66fc9b3..87fdb360e3 100644 --- a/erts/emulator/beam/hash.h +++ b/erts/emulator/beam/hash.h @@ -3,16 +3,17 @@ * * Copyright Ericsson AB 1996-2009. All Rights Reserved. * - * The contents of this file are subject to the Erlang Public License, - * Version 1.1, (the "License"); you may not use this file except in - * compliance with the License. You should have received a copy of the - * Erlang Public License along with this software. If not, it can be - * retrieved online at http://www.erlang.org/. - * - * Software distributed under the License is distributed on an "AS IS" - * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See - * the License for the specific language governing rights and limitations - * under the License. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. * * %CopyrightEnd% */ diff --git a/erts/emulator/beam/index.c b/erts/emulator/beam/index.c index 79c3ecf1b3..5f6ea14732 100644 --- a/erts/emulator/beam/index.c +++ b/erts/emulator/beam/index.c @@ -3,16 +3,17 @@ * * Copyright Ericsson AB 1996-2012. All Rights Reserved. * - * The contents of this file are subject to the Erlang Public License, - * Version 1.1, (the "License"); you may not use this file except in - * compliance with the License. You should have received a copy of the - * Erlang Public License along with this software. If not, it can be - * retrieved online at http://www.erlang.org/. - * - * Software distributed under the License is distributed on an "AS IS" - * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See - * the License for the specific language governing rights and limitations - * under the License. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. * * %CopyrightEnd% */ @@ -83,7 +84,7 @@ index_put_entry(IndexTable* t, void* tmpl) Uint sz; if (ix >= t->limit) { /* A core dump is unnecessary */ - erl_exit(ERTS_DUMP_EXIT, "no more index entries in %s (max=%d)\n", + erts_exit(ERTS_DUMP_EXIT, "no more index entries in %s (max=%d)\n", t->htable.name, t->limit); } sz = INDEX_PAGE_SIZE*sizeof(IndexSlot*); @@ -122,7 +123,7 @@ void erts_index_merge(Hash* src, IndexTable* dst) ix = dst->entries++; if (ix >= dst->size) { if (ix >= dst->limit) { - erl_exit(1, "no more index entries in %s (max=%d)\n", + erts_exit(ERTS_ERROR_EXIT, "no more index entries in %s (max=%d)\n", dst->htable.name, dst->limit); } sz = INDEX_PAGE_SIZE*sizeof(IndexSlot*); diff --git a/erts/emulator/beam/index.h b/erts/emulator/beam/index.h index 537bc11056..14fab41026 100644 --- a/erts/emulator/beam/index.h +++ b/erts/emulator/beam/index.h @@ -3,16 +3,17 @@ * * Copyright Ericsson AB 1996-2013. All Rights Reserved. * - * The contents of this file are subject to the Erlang Public License, - * Version 1.1, (the "License"); you may not use this file except in - * compliance with the License. You should have received a copy of the - * Erlang Public License along with this software. If not, it can be - * retrieved online at http://www.erlang.org/. - * - * Software distributed under the License is distributed on an "AS IS" - * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See - * the License for the specific language governing rights and limitations - * under the License. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. * * %CopyrightEnd% */ diff --git a/erts/emulator/beam/io.c b/erts/emulator/beam/io.c index 9a67e05bfa..fbcb0c31fc 100644 --- a/erts/emulator/beam/io.c +++ b/erts/emulator/beam/io.c @@ -1,18 +1,19 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 1996-2013. All Rights Reserved. + * Copyright Ericsson AB 1996-2014. All Rights Reserved. * - * The contents of this file are subject to the Erlang Public License, - * Version 1.1, (the "License"); you may not use this file except in - * compliance with the License. You should have received a copy of the - * Erlang Public License along with this software. If not, it can be - * retrieved online at http://www.erlang.org/. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Software distributed under the License is distributed on an "AS IS" - * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See - * the License for the specific language governing rights and limitations - * under the License. + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. * * %CopyrightEnd% */ @@ -46,9 +47,15 @@ #define ERTS_WANT_EXTERNAL_TAGS #include "external.h" #include "dtrace-wrapper.h" +#include "erl_map.h" +#include "erl_bif_unique.h" +#include "erl_hl_timer.h" +#include "erl_time.h" extern ErlDrvEntry fd_driver_entry; +#ifndef __OSE__ extern ErlDrvEntry vanilla_driver_entry; +#endif extern ErlDrvEntry spawn_driver_entry; extern ErlDrvEntry *driver_tab[]; /* table of static drivers, only used during initialization */ @@ -60,8 +67,6 @@ static erts_smp_tsd_key_t driver_list_last_error_key; /* Save last DDLL error o per thread basis (for BC interfaces) */ ErtsPTab erts_port erts_align_attribute(ERTS_CACHE_LINE_SIZE); /* The port table */ -erts_smp_atomic_t erts_bytes_out; /* No bytes sent out of the system */ -erts_smp_atomic_t erts_bytes_in; /* No bytes gotten into the system */ const ErlDrvTermData driver_term_nil = (ErlDrvTermData)NIL; @@ -75,6 +80,9 @@ int erts_port_synchronous_ops = 0; int erts_port_schedule_all_ops = 0; int erts_port_parallelism = 0; +static erts_atomic64_t bytes_in; +static erts_atomic64_t bytes_out; + static void deliver_result(Eterm sender, Eterm pid, Eterm res); static int init_driver(erts_driver_t *, ErlDrvEntry *, DE_Handle *); static void terminate_port(Port *p); @@ -244,11 +252,13 @@ static ERTS_INLINE void port_init_instr(Port *prt ASSERT(prt->drv_ptr && prt->lock); if (!prt->drv_ptr->lock) { char *lock_str = "port_lock"; + erts_mtx_init_locked_x(prt->lock, lock_str, id, #ifdef ERTS_ENABLE_LOCK_COUNT - if (!(erts_lcnt_rt_options & ERTS_LCNT_OPT_PORTLOCK)) - lock_str = NULL; + (erts_lcnt_rt_options & ERTS_LCNT_OPT_PORTLOCK) +#else + 0 #endif - erts_mtx_init_locked_x(prt->lock, lock_str, id); + ); } #endif erts_port_task_init_sched(&prt->sched, id); @@ -364,9 +374,8 @@ static Port *create_port(char *name, ERTS_P_LINKS(prt) = NULL; ERTS_P_MONITORS(prt) = NULL; prt->linebuf = NULL; - prt->bp = NULL; prt->suspended = NULL; - prt->data = am_undefined; + erts_init_port_data(prt); prt->port_data_lock = NULL; prt->control_flags = 0; prt->bytes_in = 0; @@ -374,11 +383,7 @@ static Port *create_port(char *name, prt->dist_entry = NULL; ERTS_PORT_INIT_CONNECTED(prt, pid); prt->common.u.alive.reg = NULL; -#ifdef ERTS_SMP - prt->common.u.alive.ptimer = NULL; -#else - sys_memset(&prt->common.u.alive.tm, 0, sizeof(ErlTimer)); -#endif + ERTS_PTMR_INIT(prt); erts_port_task_handle_init(&prt->timeout_task); prt->psd = NULL; prt->drv_data = (SWord) 0; @@ -387,7 +392,7 @@ static Port *create_port(char *name, /* Set default tracing */ erts_get_default_tracing(&ERTS_TRACE_FLAGS(prt), &ERTS_TRACER_PROC(prt)); - ASSERT(((char *) prt) == ((char *) &prt->common)); + ERTS_CT_ASSERT(offsetof(Port,common) == 0); #if !ERTS_PORT_INIT_INSTR_NEED_ID /* @@ -458,11 +463,7 @@ erts_port_free(Port *prt) | ERTS_PORT_SFLG_FREE)); ASSERT(state & ERTS_PORT_SFLG_PORT_DEBUG); -#ifdef ERTS_SMP - ERTS_LC_ASSERT(erts_atomic32_read_nob(&prt->common.refc) == 0); -#else - ERTS_LC_ASSERT(erts_atomic32_read_nob(&prt->refc) == 0); -#endif + ERTS_LC_ASSERT(erts_atomic_read_nob(&prt->common.refc.atmc) == 0); erts_port_task_fini_sched(&prt->sched); @@ -731,11 +732,7 @@ erts_open_driver(erts_driver_t* driver, /* Pointer to driver. */ /* * Must clean up the port. */ -#ifdef ERTS_SMP - erts_cancel_smp_ptimer(port->common.u.alive.ptimer); -#else - erts_cancel_timer(&(port->common.u.alive.tm)); -#endif + erts_cancel_port_timer(port); stopq(port); if (port->linebuf != NULL) { erts_free(ERTS_ALC_T_LINEBUF, @@ -910,8 +907,8 @@ int erts_port_handle_xports(Port *prt) (iov)->iov_base = (ptr); \ (iov)->iov_len = (len); \ if (sizeof((iov)->iov_len) < sizeof(len) \ - /* Check if (len) overflowed (iov)->iov_len */ \ - && ((len) >> (sizeof((iov)->iov_len)*CHAR_BIT)) != 0) { \ + /* Check if (len) overflowed (iov)->iov_len */ \ + && (iov)->iov_len != (len)) { \ goto L_overflow; \ } \ *(bv)++ = (bin); \ @@ -1214,9 +1211,10 @@ typedef struct { static ERTS_INLINE ErtsTryImmDrvCallResult try_imm_drv_call(ErtsTryImmDrvCallState *sp) { + unsigned int prof_runnable_ports; ErtsTryImmDrvCallResult res; int reds_left_in; - erts_aint32_t invalid_state, invalid_sched_flags; + erts_aint32_t act, exp, invalid_state, invalid_sched_flags; Port *prt = sp->port; Process *c_p = sp->c_p; @@ -1243,18 +1241,39 @@ try_imm_drv_call(ErtsTryImmDrvCallState *sp) goto locked_fail; } - sp->sched_flags = erts_smp_atomic32_read_nob(&prt->sched.flags); - if (sp->sched_flags & invalid_sched_flags) { - res = ERTS_TRY_IMM_DRV_CALL_INVALID_SCHED_FLAGS; - goto locked_fail; - } + prof_runnable_ports = erts_system_profile_flags.runnable_ports; + if (prof_runnable_ports) + erts_port_task_sched_lock(&prt->sched); + act = erts_smp_atomic32_read_nob(&prt->sched.flags); + + do { + erts_aint32_t new; + + if (act & invalid_sched_flags) { + res = ERTS_TRY_IMM_DRV_CALL_INVALID_SCHED_FLAGS; + sp->sched_flags = act; + goto locked_fail; + } + exp = act; + new = act | ERTS_PTS_FLG_EXEC_IMM; + act = erts_smp_atomic32_cmpxchg_mb(&prt->sched.flags, new, exp); + } while (act != exp); + + sp->sched_flags = act; if (!c_p) reds_left_in = CONTEXT_REDS/10; else { if (IS_TRACED_FL(c_p, F_TRACE_SCHED_PROCS)) trace_virtual_sched(c_p, am_out); + /* + * No status lock held while sending runnable + * proc trace messages. It is however not needed + * in this case, since only this thread can send + * such messages for this process until the process + * has been scheduled out. + */ if (erts_system_profile_flags.runnable_procs && erts_system_profile_flags.exclusive) profile_runnable_proc(c_p, am_inactive); @@ -1269,11 +1288,14 @@ try_imm_drv_call(ErtsTryImmDrvCallState *sp) ERTS_SMP_CHK_NO_PROC_LOCKS; - if (IS_TRACED_FL(prt, F_TRACE_SCHED_PORTS)) - trace_sched_ports_where(prt, am_in, sp->port_op); - if (erts_system_profile_flags.runnable_ports - && !erts_port_is_scheduled(prt)) - profile_runnable_port(prt, am_active); + if (prof_runnable_ports | IS_TRACED_FL(prt, F_TRACE_SCHED_PORTS)) { + if (prof_runnable_ports && !(act & (ERTS_PTS_FLG_IN_RUNQ|ERTS_PTS_FLG_EXEC))) + profile_runnable_port(prt, am_active); + if (IS_TRACED_FL(prt, F_TRACE_SCHED_PORTS)) + trace_sched_ports_where(prt, am_in, sp->port_op); + if (prof_runnable_ports) + erts_port_task_sched_unlock(&prt->sched); + } sp->fpe_was_unmasked = erts_block_fpe(); @@ -1290,17 +1312,31 @@ finalize_imm_drv_call(ErtsTryImmDrvCallState *sp) int reds; Port *prt = sp->port; Process *c_p = sp->c_p; + erts_aint32_t act; + unsigned int prof_runnable_ports; reds = prt->reds; reds += erts_port_driver_callback_epilogue(prt, NULL); erts_unblock_fpe(sp->fpe_was_unmasked); - if (IS_TRACED_FL(prt, F_TRACE_SCHED_PORTS)) - trace_sched_ports_where(prt, am_out, sp->port_op); - if (erts_system_profile_flags.runnable_ports - && !erts_port_is_scheduled(prt)) - profile_runnable_port(prt, am_inactive); + prof_runnable_ports = erts_system_profile_flags.runnable_ports; + if (prof_runnable_ports) + erts_port_task_sched_lock(&prt->sched); + + act = erts_smp_atomic32_read_band_mb(&prt->sched.flags, + ~ERTS_PTS_FLG_EXEC_IMM); + ERTS_SMP_LC_ASSERT(act & ERTS_PTS_FLG_EXEC_IMM); + + if (prof_runnable_ports | IS_TRACED_FL(prt, F_TRACE_SCHED_PORTS)) { + if (IS_TRACED_FL(prt, F_TRACE_SCHED_PORTS)) + trace_sched_ports_where(prt, am_out, sp->port_op); + if (prof_runnable_ports) { + if (!(act & (ERTS_PTS_FLG_IN_RUNQ|ERTS_PTS_FLG_EXEC))) + profile_runnable_port(prt, am_inactive); + erts_port_task_sched_unlock(&prt->sched); + } + } erts_port_release(prt); @@ -1315,6 +1351,13 @@ finalize_imm_drv_call(ErtsTryImmDrvCallState *sp) if (IS_TRACED_FL(c_p, F_TRACE_SCHED_PROCS)) trace_virtual_sched(c_p, am_in); + /* + * No status lock held while sending runnable + * proc trace messages. It is however not needed + * in this case, since only this thread can send + * such messages for this process until the process + * has been scheduled out. + */ if (erts_system_profile_flags.runnable_procs && erts_system_profile_flags.exclusive) profile_runnable_proc(c_p, am_active); @@ -1331,7 +1374,7 @@ force_imm_drv_call(ErtsTryImmDrvCallState *sp) erts_aint32_t invalid_state; Port *prt = sp->port; - ASSERT(ERTS_IS_CRASH_DUMPING) + ASSERT(ERTS_IS_CRASH_DUMPING); ASSERT(is_atom(sp->port_op)); invalid_state = sp->state; @@ -1355,39 +1398,22 @@ finalize_force_imm_drv_call(ErtsTryImmDrvCallState *sp) static ERTS_INLINE void queue_port_sched_op_reply(Process *rp, ErtsProcLocks *rp_locksp, - Eterm *hp_start, - Eterm *hp, - Uint h_size, - ErlHeapFragment* bp, + ErtsHeapFactory* factory, Uint32 *ref_num, Eterm msg) { - Eterm ref = make_internal_ref(hp); + Eterm* hp = erts_produce_heap(factory, ERTS_QUEUE_PORT_SCHED_OP_REPLY_SIZE, 0); + Eterm ref; + + ref= make_internal_ref(hp); write_ref_thing(hp, ref_num[0], ref_num[1], ref_num[2]); hp += REF_THING_SIZE; msg = TUPLE2(hp, ref, msg); - hp += 3; - if (!bp) { - HRelease(rp, hp_start + h_size, hp); - } - else { - Uint used_h_size = hp - hp_start; - ASSERT(h_size >= used_h_size); - if (h_size > used_h_size) - bp = erts_resize_message_buffer(bp, used_h_size, &msg, 1); - } + erts_factory_trim_and_close(factory, &msg, 1); - erts_queue_message(rp, - rp_locksp, - bp, - msg, - NIL -#ifdef USE_VM_PROBES - , NIL -#endif - ); + erts_queue_message(rp, rp_locksp, factory->heap_frags, msg, NIL); } static void @@ -1397,9 +1423,10 @@ port_sched_op_reply(Eterm to, Uint32 *ref_num, Eterm msg) if (rp) { ErlOffHeap *ohp; ErlHeapFragment* bp; + ErtsHeapFactory factory; Eterm msg_copy; Uint hsz, msg_sz; - Eterm *hp, *hp_start; + Eterm *hp; ErtsProcLocks rp_locks = 0; hsz = ERTS_QUEUE_PORT_SCHED_OP_REPLY_SIZE; @@ -1410,22 +1437,22 @@ port_sched_op_reply(Eterm to, Uint32 *ref_num, Eterm msg) hsz += msg_sz; } - hp_start = hp = erts_alloc_message_heap(hsz, + hp = erts_alloc_message_heap(hsz, &bp, &ohp, rp, &rp_locks); + erts_factory_message_init(&factory, rp, hp, bp); if (is_immed(msg)) msg_copy = msg; - else + else { msg_copy = copy_struct(msg, msg_sz, &hp, ohp); + factory.hp = hp; + } queue_port_sched_op_reply(rp, &rp_locks, - hp_start, - hp, - hsz, - bp, + &factory, ref_num, msg_copy); @@ -1442,6 +1469,7 @@ erts_schedule_proc2port_signal(Process *c_p, Eterm *refp, ErtsProc2PortSigData *sigdp, int task_flags, + ErtsPortTaskHandle *pthp, ErtsProc2PortSigCallback callback) { int sched_res; @@ -1491,7 +1519,7 @@ erts_schedule_proc2port_signal(Process *c_p, /* Schedule port close call for later execution... */ sched_res = erts_port_task_schedule(prt->common.id, - NULL, + pthp, ERTS_PORT_TASK_PROC_SIG, sigdp, callback, @@ -1520,12 +1548,10 @@ erts_schedule_proc2port_signal(Process *c_p, } static ERTS_INLINE void -send_badsig(Port *prt) -{ +send_badsig(Port *prt) { ErtsProcLocks rp_locks = ERTS_PROC_LOCKS_XSIG_SEND; Process* rp; Eterm connected = ERTS_PORT_GET_CONNECTED(prt); - ERTS_SMP_CHK_NO_PROC_LOCKS; ERTS_LC_ASSERT(erts_get_scheduler_id()); @@ -1545,15 +1571,13 @@ send_badsig(Port *prt) 0); if (rp_locks) erts_smp_proc_unlock(rp, rp_locks); - } -} + } /* exit sent */ +} /* send_badsig */ static void -badsig_received(int bang_op, - Port *prt, +badsig_received(int bang_op, Port *prt, erts_aint32_t state, - int bad_output_value) -{ + int bad_output_value) { /* * if (bang_op) * we are part of a "Prt ! Something" operation @@ -1569,12 +1593,12 @@ badsig_received(int bang_op, } if (bang_op) send_badsig(prt); - } -} + } /* not invalid */ +} /* behaved accordingly */ static int -port_badsig(Port *prt, erts_aint32_t state, int op, ErtsProc2PortSigData *sigdp) -{ +port_badsig(Port *prt, erts_aint32_t state, int op, + ErtsProc2PortSigData *sigdp) { if (op == ERTS_PROC2PORT_SIG_EXEC) badsig_received(sigdp->flags & ERTS_P2P_SIG_DATA_FLG_BANG_OP, prt, @@ -1583,16 +1607,14 @@ port_badsig(Port *prt, erts_aint32_t state, int op, ErtsProc2PortSigData *sigdp) if (sigdp->flags & ERTS_P2P_SIG_DATA_FLG_REPLY) port_sched_op_reply(sigdp->caller, sigdp->ref, am_badarg); return ERTS_PORT_REDS_BADSIG; -} - - -/* - * bad_port_signal() will +} /* port_badsig */ +/* bad_port_signal() will * - preserve signal order of signals. * - send a 'badsig' exit signal to connected process if 'from' is an * internal pid and the port is alive when the bad signal reaches * it. */ + static ErtsPortOpResult bad_port_signal(Process *c_p, int flags, @@ -1640,6 +1662,7 @@ bad_port_signal(Process *c_p, refp, sigdp, 0, + NULL, port_badsig); } @@ -1666,6 +1689,7 @@ call_driver_outputv(int bang_op, if (bang_op && from != ERTS_PORT_GET_CONNECTED(prt)) send_badsig(prt); else { + ErtsSchedulerData *esdp = erts_get_scheduler_data(); ErlDrvSizeT size = evp->size; ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(prt) @@ -1683,7 +1707,10 @@ call_driver_outputv(int bang_op, prt->caller = NIL; prt->bytes_out += size; - erts_smp_atomic_add_nob(&erts_bytes_out, size); + if (esdp) + esdp->io.out += (Uint64) size; + else + erts_atomic64_add_nob(&bytes_out, (erts_aint64_t) size); } } @@ -1763,7 +1790,7 @@ call_driver_output(int bang_op, if (bang_op && from != ERTS_PORT_GET_CONNECTED(prt)) send_badsig(prt); else { - + ErtsSchedulerData *esdp = erts_get_scheduler_data(); ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(prt) || ERTS_IS_CRASH_DUMPING); @@ -1779,7 +1806,10 @@ call_driver_output(int bang_op, prt->caller = NIL; prt->bytes_out += size; - erts_smp_atomic_add_nob(&erts_bytes_out, size); + if (esdp) + esdp->io.out += (Uint64) size; + else + erts_atomic64_add_nob(&bytes_out, (erts_aint64_t) size); } } @@ -1849,8 +1879,11 @@ erts_port_output(Process *c_p, ErlIOVec *evp = NULL; char *buf = NULL; int force_immediate_call = (flags & ERTS_PORT_SIG_FLG_FORCE_IMM_CALL); + int async_nosuspend; + ErtsPortTaskHandle *ns_pthp; ASSERT((flags & ~(ERTS_PORT_SIG_FLG_BANG_OP + | ERTS_PORT_SIG_FLG_ASYNC | ERTS_PORT_SIG_FLG_NOSUSPEND | ERTS_PORT_SIG_FLG_FORCE | ERTS_PORT_SIG_FLG_FORCE_IMM_CALL)) == 0); @@ -1872,6 +1905,12 @@ erts_port_output(Process *c_p, ? ERTS_PORT_OP_DROPPED : ERTS_PORT_OP_BUSY); + async_nosuspend = ((flags & (ERTS_PORT_SIG_FLG_ASYNC + | ERTS_PORT_SIG_FLG_NOSUSPEND + | ERTS_PORT_SIG_FLG_FORCE)) + == (ERTS_PORT_SIG_FLG_ASYNC + | ERTS_PORT_SIG_FLG_NOSUSPEND)); + try_call = (force_immediate_call /* crash dumping */ || !(sched_flags & (invalid_flags | ERTS_PTS_FLGS_FORCE_SCHEDULE_OP))); @@ -2006,6 +2045,15 @@ erts_port_output(Process *c_p, return ERTS_PORT_OP_DONE; case ERTS_TRY_IMM_DRV_CALL_INVALID_SCHED_FLAGS: sched_flags = try_call_state.sched_flags; + if (async_nosuspend + && (sched_flags & (busy_flgs|ERTS_PTS_FLG_EXIT))) { + driver_free_binary(cbin); + if (evp != &ev) + erts_free(ERTS_ALC_T_TMP, evp); + return ((sched_flags & ERTS_PTS_FLG_EXIT) + ? ERTS_PORT_OP_DROPPED + : ERTS_PORT_OP_BUSY); + } case ERTS_TRY_IMM_DRV_CALL_BUSY_LOCK: /* Schedule outputv() call instead... */ break; @@ -2153,6 +2201,13 @@ erts_port_output(Process *c_p, return ERTS_PORT_OP_DONE; case ERTS_TRY_IMM_DRV_CALL_INVALID_SCHED_FLAGS: sched_flags = try_call_state.sched_flags; + if (async_nosuspend + && (sched_flags & (busy_flgs|ERTS_PTS_FLG_EXIT))) { + erts_free(ERTS_ALC_T_TMP, buf); + return ((sched_flags & ERTS_PTS_FLG_EXIT) + ? ERTS_PORT_OP_DROPPED + : ERTS_PORT_OP_BUSY); + } case ERTS_TRY_IMM_DRV_CALL_BUSY_LOCK: /* Schedule outputv() call instead... */ break; @@ -2174,20 +2229,33 @@ erts_port_output(Process *c_p, task_flags = ERTS_PT_FLG_WAIT_BUSY; sigdp->flags |= flags; + ns_pthp = NULL; if (flags & (ERTS_P2P_SIG_DATA_FLG_FORCE|ERTS_P2P_SIG_DATA_FLG_NOSUSPEND)) { task_flags = 0; if (flags & ERTS_P2P_SIG_DATA_FLG_FORCE) sigdp->flags &= ~ERTS_P2P_SIG_DATA_FLG_NOSUSPEND; + else if (async_nosuspend) { + ErtsSchedulerData *esdp = (c_p + ? ERTS_PROC_GET_SCHDATA(c_p) + : erts_get_scheduler_data()); + ASSERT(esdp); + ns_pthp = &esdp->nosuspend_port_task_handle; + sigdp->flags &= ~ERTS_P2P_SIG_DATA_FLG_NOSUSPEND; + } else if (flags & ERTS_P2P_SIG_DATA_FLG_NOSUSPEND) task_flags = ERTS_PT_FLG_NOSUSPEND; } + ASSERT(ns_pthp || !async_nosuspend); + ASSERT(async_nosuspend || !ns_pthp); + res = erts_schedule_proc2port_signal(c_p, prt, c_p ? c_p->common.id : ERTS_INVALID_PID, refp, sigdp, task_flags, + ns_pthp, port_sig_callback); if (res != ERTS_PORT_OP_SCHEDULED) { @@ -2198,9 +2266,23 @@ erts_port_output(Process *c_p, return res; } - if (!(sched_flags & ERTS_PTS_FLG_EXIT) && (sched_flags & busy_flgs)) - return ERTS_PORT_OP_BUSY_SCHEDULED; - + if (!(flags & ERTS_PORT_SIG_FLG_FORCE)) { + sched_flags = erts_smp_atomic32_read_acqb(&prt->sched.flags); + if (!(sched_flags & ERTS_PTS_FLG_BUSY_PORT)) { + if (async_nosuspend) + erts_port_task_tmp_handle_detach(ns_pthp); + } + else { + if (!async_nosuspend) + return ERTS_PORT_OP_BUSY_SCHEDULED; + else { + if (erts_port_task_abort(ns_pthp) == 0) + return ERTS_PORT_OP_BUSY; + else + erts_port_task_tmp_handle_detach(ns_pthp); + } + } + } return res; bad_value: @@ -2296,6 +2378,7 @@ erts_port_exit(Process *c_p, ErlHeapFragment *bp = NULL; ASSERT((flags & ~(ERTS_PORT_SIG_FLG_BANG_OP + | ERTS_PORT_SIG_FLG_ASYNC | ERTS_PORT_SIG_FLG_BROKEN_LINK | ERTS_PORT_SIG_FLG_FORCE_SCHED)) == 0); @@ -2356,6 +2439,7 @@ erts_port_exit(Process *c_p, refp, sigdp, 0, + NULL, port_sig_exit); if (res == ERTS_PORT_OP_DROPPED) { @@ -2424,7 +2508,7 @@ set_port_connected(int bang_op, DTRACE_CHARBUF(newprocess_str, DTRACE_TERM_BUF_SIZE); dtrace_pid_str(connect, process_str); - erts_snprintf(port_str, sizeof(port_str), "%T", prt->common.id); + erts_snprintf(port_str, sizeof(DTRACE_CHARBUF_NAME(port_str)), "%T", prt->common.id); dtrace_proc_str(rp, newprocess_str); DTRACE4(port_connect, process_str, port_str, prt->name, newprocess_str); } @@ -2471,7 +2555,8 @@ erts_port_connect(Process *c_p, !refp, am_connect); - ASSERT((flags & ~ERTS_PORT_SIG_FLG_BANG_OP) == 0); + ASSERT((flags & ~(ERTS_PORT_SIG_FLG_BANG_OP + | ERTS_PORT_SIG_FLG_ASYNC)) == 0); if (is_not_internal_pid(connect)) connect_id = NIL; /* Fail in op (for signal order) */ @@ -2510,6 +2595,7 @@ erts_port_connect(Process *c_p, refp, sigdp, 0, + NULL, port_sig_connect); } @@ -2566,6 +2652,7 @@ erts_port_unlink(Process *c_p, Port *prt, Eterm from, Eterm *refp) refp, sigdp, 0, + NULL, port_sig_unlink); } @@ -2655,18 +2742,29 @@ erts_port_link(Process *c_p, Port *prt, Eterm to, Eterm *refp) refp, sigdp, 0, + NULL, port_sig_link); } void erts_init_io(int port_tab_size, - int port_tab_size_ignore_files) + int port_tab_size_ignore_files, + int legacy_port_tab) { ErlDrvEntry** dp; + UWord common_element_size; erts_smp_rwmtx_opt_t drv_list_rwmtx_opts = ERTS_SMP_RWMTX_OPT_DEFAULT_INITER; drv_list_rwmtx_opts.type = ERTS_SMP_RWMTX_TYPE_EXTREMELY_FREQUENT_READ; drv_list_rwmtx_opts.lived = ERTS_SMP_RWMTX_LONG_LIVED; + erts_atomic64_init_nob(&bytes_in, 0); + erts_atomic64_init_nob(&bytes_out, 0); + + common_element_size = ERTS_ALC_DATA_ALIGN_SIZE(sizeof(Port)); + common_element_size += ERTS_ALC_DATA_ALIGN_SIZE(sizeof(ErtsPortTaskBusyPortQ)); + common_element_size += 10; /* name */ #ifdef ERTS_SMP + common_element_size += sizeof(erts_mtx_t); + init_xports_list_alloc(); #endif @@ -2687,18 +2785,20 @@ void erts_init_io(int port_tab_size, &drv_list_rwmtx_opts, "driver_list"); driver_list = NULL; - erts_smp_tsd_key_create(&driver_list_lock_status_key); - erts_smp_tsd_key_create(&driver_list_last_error_key); + erts_smp_tsd_key_create(&driver_list_lock_status_key, + "erts_driver_list_lock_status_key"); + erts_smp_tsd_key_create(&driver_list_last_error_key, + "erts_driver_list_last_error_key"); erts_ptab_init_table(&erts_port, ERTS_ALC_T_PORT_TABLE, NULL, (ErtsPTabElementCommon *) &erts_invalid_port.common, port_tab_size, - "port_table"); - - erts_smp_atomic_init_nob(&erts_bytes_out, 0); - erts_smp_atomic_init_nob(&erts_bytes_in, 0); + common_element_size, /* Doesn't need to be excact */ + "port_table", + legacy_port_tab, + 1); sys_init_io(); @@ -2706,8 +2806,11 @@ void erts_init_io(int port_tab_size, erts_smp_rwmtx_rwlock(&erts_driver_list_lock); init_driver(&fd_driver, &fd_driver_entry, NULL); +#ifndef __OSE__ init_driver(&vanilla_driver, &vanilla_driver_entry, NULL); +#endif init_driver(&spawn_driver, &spawn_driver_entry, NULL); + erts_init_static_drivers(); for (dp = driver_tab; *dp != NULL; dp++) erts_add_driver_entry(*dp, NULL, 1); @@ -2716,7 +2819,6 @@ void erts_init_io(int port_tab_size, } #if defined(ERTS_ENABLE_LOCK_COUNT) && defined(ERTS_SMP) - static ERTS_INLINE void lcnt_enable_drv_lock_count(erts_driver_t *dp, int enable) { if (dp->lock) { @@ -2756,25 +2858,26 @@ static ERTS_INLINE void lcnt_enable_port_lock_count(Port *prt, int enable) } } -void erts_lcnt_enable_io_lock_count(int enable) -{ +void erts_lcnt_enable_io_lock_count(int enable) { erts_driver_t *dp; - int i, max = erts_ptab_max(&erts_port); + int ix, max = erts_ptab_max(&erts_port); + Port *prt; - for (i = 0; i < max; i++) { - Port *prt = erts_pix2port(i); - if (prt) + for (ix = 0; ix < max; ix++) { + if ((prt = erts_pix2port(ix)) != NULL) { lcnt_enable_port_lock_count(prt, enable); - } + } + } /* for all ports */ lcnt_enable_drv_lock_count(&vanilla_driver, enable); lcnt_enable_drv_lock_count(&spawn_driver, enable); lcnt_enable_drv_lock_count(&fd_driver, enable); - for (dp = driver_list; dp; dp = dp->next) + /* enable lock counting in all drivers */ + for (dp = driver_list; dp; dp = dp->next) { lcnt_enable_drv_lock_count(dp, enable); -} -#endif - + } +} /* enable/disable lock counting of ports */ +#endif /* defined(ERTS_ENABLE_LOCK_COUNT) && defined(ERTS_SMP) */ /* * Buffering of data when using line oriented I/O on ports */ @@ -2959,7 +3062,7 @@ deliver_result(Eterm sender, Eterm pid, Eterm res) rp = (scheduler ? erts_proc_lookup(pid) - : erts_pid2proc_opt(NULL, 0, pid, 0, ERTS_P2P_FLG_SMP_INC_REFC)); + : erts_pid2proc_opt(NULL, 0, pid, 0, ERTS_P2P_FLG_INC_REFC)); if (rp) { Eterm tuple; @@ -2972,16 +3075,12 @@ deliver_result(Eterm sender, Eterm pid, Eterm res) hp = erts_alloc_message_heap(sz_res + 3, &bp, &ohp, rp, &rp_locks); res = copy_struct(res, sz_res, &hp, ohp); tuple = TUPLE2(hp, sender, res); - erts_queue_message(rp, &rp_locks, bp, tuple, NIL -#ifdef USE_VM_PROBES - , NIL -#endif - ); + erts_queue_message(rp, &rp_locks, bp, tuple, NIL); if (rp_locks) erts_smp_proc_unlock(rp, rp_locks); if (!scheduler) - erts_smp_proc_dec_refc(rp); + erts_proc_dec_refc(rp); } } @@ -3025,7 +3124,7 @@ static void deliver_read_message(Port* prt, erts_aint32_t state, Eterm to, rp = (scheduler ? erts_proc_lookup(to) - : erts_pid2proc_opt(NULL, 0, to, 0, ERTS_P2P_FLG_SMP_INC_REFC)); + : erts_pid2proc_opt(NULL, 0, to, 0, ERTS_P2P_FLG_INC_REFC)); if (!rp) return; @@ -3040,8 +3139,6 @@ static void deliver_read_message(Port* prt, erts_aint32_t state, Eterm to, Binary* bptr; bptr = erts_bin_nrml_alloc(len); - bptr->flags = 0; - bptr->orig_size = len; erts_refc_init(&bptr->refc, 1); sys_memcpy(bptr->orig_bytes, buf, len); @@ -3074,15 +3171,11 @@ static void deliver_read_message(Port* prt, erts_aint32_t state, Eterm to, tuple = TUPLE2(hp, prt->common.id, tuple); hp += 3; - erts_queue_message(rp, &rp_locks, bp, tuple, am_undefined -#ifdef USE_VM_PROBES - , NIL -#endif - ); + erts_queue_message(rp, &rp_locks, bp, tuple, am_undefined); if (rp_locks) erts_smp_proc_unlock(rp, rp_locks); if (!scheduler) - erts_smp_proc_dec_refc(rp); + erts_proc_dec_refc(rp); } /* @@ -3168,7 +3261,7 @@ deliver_vec_message(Port* prt, /* Port */ rp = (scheduler ? erts_proc_lookup(to) - : erts_pid2proc_opt(NULL, 0, to, 0, ERTS_P2P_FLG_SMP_INC_REFC)); + : erts_pid2proc_opt(NULL, 0, to, 0, ERTS_P2P_FLG_INC_REFC)); if (!rp) return; @@ -3245,14 +3338,10 @@ deliver_vec_message(Port* prt, /* Port */ tuple = TUPLE2(hp, prt->common.id, tuple); hp += 3; - erts_queue_message(rp, &rp_locks, bp, tuple, am_undefined -#ifdef USE_VM_PROBES - , NIL -#endif - ); + erts_queue_message(rp, &rp_locks, bp, tuple, am_undefined); erts_smp_proc_unlock(rp, rp_locks); if (!scheduler) - erts_smp_proc_dec_refc(rp); + erts_proc_dec_refc(rp); } @@ -3342,11 +3431,8 @@ terminate_port(Port *prt) send_closed_port_id = NIL; } -#ifdef ERTS_SMP - erts_cancel_smp_ptimer(prt->common.u.alive.ptimer); -#else - erts_cancel_timer(&prt->common.u.alive.tm); -#endif + if (ERTS_PTMR_IS_SET(prt)) + erts_cancel_port_timer(prt); drv = prt->drv_ptr; if ((drv != NULL) && (drv->stop != NULL)) { @@ -3375,11 +3461,8 @@ terminate_port(Port *prt) erts_free(ERTS_ALC_T_LINEBUF, (void *) prt->linebuf); prt->linebuf = NULL; } - if (prt->bp != NULL) { - free_message_buffer(prt->bp); - prt->bp = NULL; - prt->data = am_undefined; - } + + erts_cleanup_port_data(prt); if (prt->psd) erts_free(ERTS_ALC_T_PRTSD, prt->psd); @@ -3395,7 +3478,7 @@ terminate_port(Port *prt) if ((state & ERTS_PORT_SFLG_HALT) && (erts_smp_atomic32_dec_read_nob(&erts_halt_progress) == 0)) { erts_port_release(prt); /* We will exit and never return */ - erl_exit_flush_async(erts_halt_code, ""); + erts_flush_async_exit(erts_halt_code, ""); } if (is_internal_port(send_closed_port_id)) deliver_result(send_closed_port_id, connected_id, am_closed); @@ -3527,9 +3610,9 @@ erts_deliver_port_exit(Port *p, Eterm from, Eterm reason, int send_closed) DTRACE_CHARBUF(port_str, DTRACE_TERM_BUF_SIZE); DTRACE_CHARBUF(rreason_str, 64); - erts_snprintf(from_str, sizeof(from_str), "%T", from); + erts_snprintf(from_str, sizeof(DTRACE_CHARBUF_NAME(from_str)), "%T", from); dtrace_port_str(p, port_str); - erts_snprintf(rreason_str, sizeof(rreason_str), "%T", rreason); + erts_snprintf(rreason_str, sizeof(DTRACE_CHARBUF_NAME(rreason_str)), "%T", rreason); DTRACE4(port_exit, from_str, port_str, p->name, rreason_str); } #endif @@ -3547,6 +3630,8 @@ erts_deliver_port_exit(Port *p, Eterm from, Eterm reason, int send_closed) if (send_closed) set_state_flags |= ERTS_PORT_SFLG_SEND_CLOSED; + erts_port_task_sched_enter_exiting_state(&p->sched); + state = erts_atomic32_read_bor_mb(&p->state, set_state_flags); state |= set_state_flags; @@ -3629,6 +3714,10 @@ erts_port_command(Process *c_p, ASSERT(port); flags |= ERTS_PORT_SIG_FLG_BANG_OP; + if (!erts_port_synchronous_ops) { + flags |= ERTS_PORT_SIG_FLG_ASYNC; + refp = NULL; + } if (is_tuple_arity(command, 2)) { Eterm cntd; @@ -3636,21 +3725,14 @@ erts_port_command(Process *c_p, cntd = tp[1]; if (is_internal_pid(cntd)) { if (tp[2] == am_close) { - if (!erts_port_synchronous_ops) - refp = NULL; flags &= ~ERTS_PORT_SIG_FLG_NOSUSPEND; return erts_port_exit(c_p, flags, port, cntd, am_normal, refp); } else if (is_tuple_arity(tp[2], 2)) { tp = tuple_val(tp[2]); if (tp[1] == am_command) { - if (!(flags & ERTS_PORT_SIG_FLG_NOSUSPEND) - && !erts_port_synchronous_ops) - refp = NULL; return erts_port_output(c_p, flags, port, cntd, tp[2], refp); } else if (tp[1] == am_connect) { - if (!erts_port_synchronous_ops) - refp = NULL; flags &= ~ERTS_PORT_SIG_FLG_NOSUSPEND; return erts_port_connect(c_p, flags, port, cntd, tp[2], refp); } @@ -3659,8 +3741,6 @@ erts_port_command(Process *c_p, } /* badsig */ - if (!erts_port_synchronous_ops) - refp = NULL; flags &= ~ERTS_PORT_SIG_FLG_NOSUSPEND; return bad_port_signal(c_p, flags, port, c_p->common.id, refp, am_command); } @@ -3823,12 +3903,13 @@ port_sig_control(Port *prt, if (res == ERTS_PORT_OP_DONE) { Eterm msg; - Eterm *hp, *hp_start; + Eterm *hp; ErlHeapFragment *bp; ErlOffHeap *ohp; + ErtsHeapFactory factory; Process *rp; ErtsProcLocks rp_locks = 0; - Uint hsz; + Uint hsz, rsz; int control_flags; rp = erts_proc_lookup_raw(sigdp->caller); @@ -3837,17 +3918,19 @@ port_sig_control(Port *prt, control_flags = prt->control_flags; - hsz = ERTS_QUEUE_PORT_SCHED_OP_REPLY_SIZE; - hsz += port_control_result_size(control_flags, + rsz = port_control_result_size(control_flags, resp_bufp, &resp_size, &resp_buf[0]); + hsz = rsz + ERTS_QUEUE_PORT_SCHED_OP_REPLY_SIZE; + - hp_start = hp = erts_alloc_message_heap(hsz, + hp = erts_alloc_message_heap(hsz, &bp, &ohp, rp, &rp_locks); + erts_factory_message_init(&factory, rp, hp, bp); msg = write_port_control_result(control_flags, resp_bufp, @@ -3856,13 +3939,11 @@ port_sig_control(Port *prt, &hp, bp, ohp); + factory.hp = hp; queue_port_sched_op_reply(rp, &rp_locks, - hp_start, - hp, - hsz, - bp, + &factory, sigdp->ref, msg); @@ -3980,6 +4061,9 @@ erts_port_control(Process* c_p, size, &resp_bufp, &resp_size); + + control_flags = prt->control_flags; + finalize_imm_drv_call(&try_call_state); if (tmp_alloced) erts_free(ERTS_ALC_T_TMP, bufp); @@ -3987,8 +4071,6 @@ erts_port_control(Process* c_p, return ERTS_PORT_OP_BADARG; } - control_flags = prt->control_flags; - hsz = port_control_result_size(control_flags, resp_bufp, &resp_size, @@ -4029,7 +4111,7 @@ erts_port_control(Process* c_p, copy = 1; else { binp = ((ProcBin *) ebinp)->val; - ASSERT(bufp < bufp + size); + ASSERT(bufp <= bufp + size); ASSERT(binp->orig_bytes <= bufp && bufp + size <= binp->orig_bytes + binp->orig_size); erts_refc_inc(&binp->refc, 1); @@ -4057,6 +4139,7 @@ erts_port_control(Process* c_p, retvalp, sigdp, 0, + NULL, port_sig_control); if (res != ERTS_PORT_OP_SCHEDULED) { cleanup_scheduled_control(binp, bufp); @@ -4147,11 +4230,9 @@ port_sig_call(Port *prt, if (res == ERTS_PORT_OP_DONE) { Eterm msg; Eterm *hp; - ErlHeapFragment *bp; - ErlOffHeap *ohp; Process *rp; ErtsProcLocks rp_locks = 0; - Uint hsz; + Sint hsz; rp = erts_proc_lookup_raw(sigdp->caller); if (!rp) @@ -4159,29 +4240,31 @@ port_sig_call(Port *prt, hsz = erts_decode_ext_size((byte *) resp_bufp, resp_size); if (hsz >= 0) { - Eterm *hp_start; + ErlHeapFragment* bp; + ErlOffHeap* ohp; + ErtsHeapFactory factory; byte *endp; hsz += 3; /* ok tuple */ hsz += ERTS_QUEUE_PORT_SCHED_OP_REPLY_SIZE; - hp_start = hp = erts_alloc_message_heap(hsz, - &bp, - &ohp, - rp, - &rp_locks); + hp = erts_alloc_message_heap(hsz, + &bp, + &ohp, + rp, + &rp_locks); endp = (byte *) resp_bufp; - msg = erts_decode_ext(&hp, ohp, &endp); + erts_factory_message_init(&factory, rp, hp, bp); + msg = erts_decode_ext(&factory, &endp); if (is_value(msg)) { + hp = erts_produce_heap(&factory, + 3, + ERTS_QUEUE_PORT_SCHED_OP_REPLY_SIZE); msg = TUPLE2(hp, am_ok, msg); - hp += 3; queue_port_sched_op_reply(rp, &rp_locks, - hp_start, - hp, - hsz, - bp, + &factory, sigdp->ref, msg); @@ -4189,8 +4272,6 @@ port_sig_call(Port *prt, erts_smp_proc_unlock(rp, rp_locks); goto done; } - if (bp) - free_message_buffer(bp); if (rp_locks) erts_smp_proc_unlock(rp, rp_locks); } @@ -4267,10 +4348,11 @@ erts_port_call(Process* c_p, try_call_res = try_imm_drv_call(&try_call_state); switch (try_call_res) { case ERTS_TRY_IMM_DRV_CALL_OK: { - Eterm *hp, *hp_end; - Uint hsz; + ErtsHeapFactory factory; + Sint hsz; unsigned ret_flags = 0U; Eterm term; + Eterm* hp; res = call_driver_call(c_p->common.id, prt, @@ -4290,15 +4372,14 @@ erts_port_call(Process* c_p, if (hsz < 0) return ERTS_PORT_OP_BADARG; hsz += 3; - hp = HAlloc(c_p, hsz); - hp_end = hp + hsz; + erts_factory_proc_prealloc_init(&factory, c_p, hsz); endp = (byte *) resp_bufp; - term = erts_decode_ext(&hp, &MSO(c_p), &endp); + term = erts_decode_ext(&factory, &endp); if (term == THE_NON_VALUE) return ERTS_PORT_OP_BADARG; + hp = erts_produce_heap(&factory,3,0); *retvalp = TUPLE2(hp, am_ok, term); - hp += 3; - HRelease(c_p, hp_end, hp); + erts_factory_close(&factory); if (resp_bufp != &resp_buf[0] && !(ret_flags & DRIVER_CALL_KEEP_BUFFER)) driver_free(resp_bufp); @@ -4337,6 +4418,7 @@ erts_port_call(Process* c_p, retvalp, sigdp, 0, + NULL, port_sig_call); if (res != ERTS_PORT_OP_SCHEDULED) { cleanup_scheduled_call(bufp); @@ -4375,7 +4457,7 @@ make_port_info_term(Eterm **hpp_start, int len; int start; static Eterm item[] = ERTS_PORT_INFO_1_ITEMS; - static Eterm value[sizeof(item)/sizeof(item[0])]; + Eterm value[sizeof(item)/sizeof(item[0])]; start = 0; len = sizeof(item)/sizeof(item[0]); @@ -4432,12 +4514,11 @@ port_sig_info(Port *prt, prt, sigdp->u.info.item); if (is_value(value)) { + ErtsHeapFactory factory; + erts_factory_message_init(&factory, NULL, hp, bp); queue_port_sched_op_reply(rp, &rp_locks, - hp_start, - hp, - hsz, - bp, + &factory, sigdp->ref, value); } @@ -4503,228 +4584,106 @@ erts_port_info(Process* c_p, retvalp, sigdp, 0, + NULL, port_sig_info); } -static int -port_sig_set_data(Port *prt, - erts_aint32_t state, - int op, - ErtsProc2PortSigData *sigdp) -{ - ASSERT(sigdp->flags & ERTS_P2P_SIG_DATA_FLG_REPLY); - - if (op == ERTS_PROC2PORT_SIG_EXEC) { - if (prt->bp) - free_message_buffer(prt->bp); - prt->bp = sigdp->u.set_data.bp; - prt->data = sigdp->u.set_data.data; - port_sched_op_reply(sigdp->caller, sigdp->ref, am_true); - } - else { - if (sigdp->u.set_data.bp) - free_message_buffer(sigdp->u.set_data.bp); - port_sched_op_reply(sigdp->caller, sigdp->ref, am_badarg); - } - return ERTS_PORT_REDS_SET_DATA; -} - -ErtsPortOpResult -erts_port_set_data(Process* c_p, - Port *prt, - Eterm data, - Eterm *refp) -{ - ErtsPortOpResult res; - Eterm set_data; - ErlHeapFragment *bp; - ErtsProc2PortSigData *sigdp; - ErtsTryImmDrvCallResult try_call_res; - ErtsTryImmDrvCallState try_call_state - = ERTS_INIT_TRY_IMM_DRV_CALL_STATE( - c_p, - prt, - ERTS_PORT_SFLGS_INVALID_LOOKUP, - 0, - !refp, - am_set_data); - - if (is_immed(data)) { - set_data = data; - bp = NULL; - } - else { - Eterm *hp; - Uint sz = size_object(data); - bp = new_message_buffer(sz); - hp = bp->mem; - set_data = copy_struct(data, sz, &hp, &bp->off_heap); - } - - try_call_res = try_imm_drv_call(&try_call_state); - switch (try_call_res) { - case ERTS_TRY_IMM_DRV_CALL_OK: - if (prt->bp) - free_message_buffer(prt->bp); - prt->bp = bp; - prt->data = set_data; - finalize_imm_drv_call(&try_call_state); - BUMP_REDS(c_p, ERTS_PORT_REDS_SET_DATA); - return ERTS_PORT_OP_DONE; - case ERTS_TRY_IMM_DRV_CALL_INVALID_PORT: - return ERTS_PORT_OP_DROPPED; - case ERTS_TRY_IMM_DRV_CALL_INVALID_SCHED_FLAGS: - case ERTS_TRY_IMM_DRV_CALL_BUSY_LOCK: - /* Schedule call instead... */ - break; - } - - sigdp = erts_port_task_alloc_p2p_sig_data(); - sigdp->flags = ERTS_P2P_SIG_TYPE_SET_DATA; - sigdp->u.set_data.data = set_data; - sigdp->u.set_data.bp = bp; - - res = erts_schedule_proc2port_signal(c_p, - prt, - c_p->common.id, - refp, - sigdp, - 0, - port_sig_set_data); - if (res != ERTS_PORT_OP_SCHEDULED && bp) - free_message_buffer(bp); - return res; -} +typedef struct { + Uint sched_id; + Eterm pid; + Uint32 refn[ERTS_REF_NUMBERS]; + erts_smp_atomic32_t refc; +} ErtsIOBytesReq; -static int -port_sig_get_data(Port *prt, - erts_aint32_t state, - int op, - ErtsProc2PortSigData *sigdp) +static void +reply_io_bytes(void *vreq) { - ASSERT(sigdp->flags & ERTS_P2P_SIG_DATA_FLG_REPLY); - if (op != ERTS_PROC2PORT_SIG_EXEC) - port_sched_op_reply(sigdp->caller, sigdp->ref, am_badarg); - else { - Process *rp; - ErtsProcLocks rp_locks = 0; + ErtsIOBytesReq *req = (ErtsIOBytesReq *) vreq; + Process *rp; - rp = erts_proc_lookup_raw(sigdp->caller); - if (rp) { - Uint hsz; - Eterm *hp, *hp_start; - Eterm data, msg; - ErlHeapFragment *bp; - ErlOffHeap *ohp; + rp = erts_proc_lookup(req->pid); + if (rp) { + ErlOffHeap *ohp = NULL; + ErlHeapFragment *bp = NULL; + ErtsProcLocks rp_locks; + Eterm ref, msg, ein, eout, *hp; + Uint64 in, out; + Uint hsz; + ErtsSchedulerData *esdp = erts_get_scheduler_data(); + Uint sched_id = esdp->no; + in = esdp->io.in; + out = esdp->io.out; + if (req->sched_id != sched_id) + rp_locks = 0; + else { + in += (Uint64) erts_atomic64_read_nob(&bytes_in); + out += (Uint64) erts_atomic64_read_nob(&bytes_out); + rp_locks = ERTS_PROC_LOCK_MAIN; + } - hsz = ERTS_QUEUE_PORT_SCHED_OP_REPLY_SIZE; - hsz += 3; - if (prt->bp) - hsz += prt->bp->used_size; + hsz = 5 /* 4-tuple */ + REF_THING_SIZE; - hp_start = hp = erts_alloc_message_heap(hsz, - &bp, - &ohp, - rp, - &rp_locks); + erts_bld_uint64(NULL, &hsz, in); + erts_bld_uint64(NULL, &hsz, out); - if (is_immed(prt->data)) - data = prt->data; - else - data = copy_struct(prt->data, - prt->bp->used_size, - &hp, - &bp->off_heap); + hp = erts_alloc_message_heap(hsz, &bp, &ohp, rp, &rp_locks); + ref = make_internal_ref(hp); + write_ref_thing(hp, req->refn[0], req->refn[1], req->refn[2]); + hp += REF_THING_SIZE; + ein = erts_bld_uint64(&hp, NULL, in); + eout = erts_bld_uint64(&hp, NULL, out); - msg = TUPLE2(hp, am_ok, data); - hp += 3; + msg = TUPLE4(hp, ref, make_small(sched_id), ein, eout); + erts_queue_message(rp, &rp_locks, bp, msg, NIL); - queue_port_sched_op_reply(rp, - &rp_locks, - hp_start, - hp, - hsz, - bp, - sigdp->ref, - msg); - if (rp_locks) - erts_smp_proc_unlock(rp, rp_locks); - } + if (req->sched_id == sched_id) + rp_locks &= ~ERTS_PROC_LOCK_MAIN; + if (rp_locks) + erts_smp_proc_unlock(rp, rp_locks); } - return ERTS_PORT_REDS_GET_DATA; + + if (erts_smp_atomic32_dec_read_nob(&req->refc) == 0) + erts_free(ERTS_ALC_T_IOB_REQ, req); } -ErtsPortOpResult -erts_port_get_data(Process* c_p, - Port *prt, - Eterm *retvalp) +Eterm +erts_request_io_bytes(Process *c_p) { - ErtsProc2PortSigData *sigdp; - ErtsTryImmDrvCallResult try_call_res; - ErtsTryImmDrvCallState try_call_state - = ERTS_INIT_TRY_IMM_DRV_CALL_STATE( - c_p, - prt, - ERTS_PORT_SFLGS_INVALID_LOOKUP, - 0, - 0, - am_get_data); + Uint *hp; + Eterm ref; + Uint32 *refn; + ErtsSchedulerData *esdp = ERTS_PROC_GET_SCHDATA(c_p); + ErtsIOBytesReq *req = erts_alloc(ERTS_ALC_T_IOB_REQ, + sizeof(ErtsIOBytesReq)); + + hp = HAlloc(c_p, REF_THING_SIZE); + ref = erts_sched_make_ref_in_buffer(esdp, hp); + refn = internal_ref_numbers(ref); + + req->sched_id = esdp->no; + req->pid = c_p->common.id; + req->refn[0] = refn[0]; + req->refn[1] = refn[1]; + req->refn[2] = refn[2]; + erts_smp_atomic32_init_nob(&req->refc, + (erts_aint32_t) erts_no_schedulers); - try_call_res = try_imm_drv_call(&try_call_state); - switch (try_call_res) { - case ERTS_TRY_IMM_DRV_CALL_OK: { - Eterm *hp; - Eterm data; - ErlHeapFragment *bp; - Uint sz; - if (is_immed(prt->data)) { - bp = NULL; - data = prt->data; - } - else { - bp = new_message_buffer(prt->bp->used_size); - data = copy_struct(prt->data, - prt->bp->used_size, - &hp, - &bp->off_heap); - } - finalize_imm_drv_call(&try_call_state); - if (is_immed(data)) - sz = 0; - else - sz = bp->used_size; - - hp = HAlloc(c_p, sz + 3); - if (is_not_immed(data)) { - data = copy_struct(data, bp->used_size, &hp, &MSO(c_p)); - free_message_buffer(bp); - } - *retvalp = TUPLE2(hp, am_ok, data); - BUMP_REDS(c_p, ERTS_PORT_REDS_GET_DATA); - return ERTS_PORT_OP_DONE; - } - case ERTS_TRY_IMM_DRV_CALL_INVALID_PORT: - return ERTS_PORT_OP_DROPPED; - case ERTS_TRY_IMM_DRV_CALL_INVALID_SCHED_FLAGS: - case ERTS_TRY_IMM_DRV_CALL_BUSY_LOCK: - /* Schedule call instead... */ - break; - } +#ifdef ERTS_SMP + if (erts_no_schedulers > 1) + erts_schedule_multi_misc_aux_work(1, + erts_no_schedulers, + reply_io_bytes, + (void *) req); +#endif - sigdp = erts_port_task_alloc_p2p_sig_data(); - sigdp->flags = ERTS_P2P_SIG_TYPE_GET_DATA; + reply_io_bytes((void *) req); - return erts_schedule_proc2port_signal(c_p, - prt, - c_p->common.id, - retvalp, - sigdp, - 0, - port_sig_get_data); + return ref; } + typedef struct { int to; void *arg; @@ -4815,7 +4774,7 @@ set_busy_port(ErlDrvPort dprt, int on) #ifdef USE_VM_PROBES if (DTRACE_ENABLED(port_busy)) { - erts_snprintf(port_str, sizeof(port_str), + erts_snprintf(port_str, sizeof(DTRACE_CHARBUF_NAME(port_str)), "%T", prt->common.id); DTRACE1(port_busy, port_str); } @@ -4828,7 +4787,7 @@ set_busy_port(ErlDrvPort dprt, int on) #ifdef USE_VM_PROBES if (DTRACE_ENABLED(port_not_busy)) { - erts_snprintf(port_str, sizeof(port_str), + erts_snprintf(port_str, sizeof(DTRACE_CHARBUF_NAME(port_str)), "%T", prt->common.id); DTRACE1(port_not_busy, port_str); } @@ -4880,9 +4839,9 @@ erts_port_resume_procs(Port *prt) DTRACE_CHARBUF(pid_str, 16); ErtsProcList* plp2 = plp; - erts_snprintf(port_str, sizeof(port_str), "%T", prt->common.id); + erts_snprintf(port_str, sizeof(DTRACE_CHARBUF_NAME(port_str)), "%T", prt->common.id); while (plp2 != NULL) { - erts_snprintf(pid_str, sizeof(pid_str), "%T", plp2->pid); + erts_snprintf(pid_str, sizeof(DTRACE_CHARBUF_NAME(pid_str)), "%T", plp2->pid); DTRACE2(process_port_unblocked, pid_str, port_str); } } @@ -4934,7 +4893,7 @@ void erts_raw_port_command(Port* p, byte* buf, Uint len) ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(p)); if (len > (Uint) INT_MAX) - erl_exit(ERTS_ABORT_EXIT, + erts_exit(ERTS_ABORT_EXIT, "Absurdly large data buffer (%beu bytes) passed to" "output callback of %s driver.\n", len, @@ -4981,7 +4940,8 @@ int async_ready(Port *p, void* data) static void report_missing_drv_callback(Port *p, char *drv_type, char *callback) { - ErtsPortNames *pnp = erts_get_port_names(p->common.id); + ErtsPortNames *pnp = erts_get_port_names(p->common.id, + ERTS_Port2ErlDrvPort(p)); char *unknown = "<unknown>"; char *drv_name = pnp->driver_name ? pnp->driver_name : unknown; char *prt_name = pnp->name ? pnp->name : unknown; @@ -4996,15 +4956,25 @@ report_missing_drv_callback(Port *p, char *drv_type, char *callback) void erts_stale_drv_select(Eterm port, + ErlDrvPort drv_port, ErlDrvEvent hndl, int mode, int deselect) { char *type; - ErlDrvPort drv_port = ERTS_Port2ErlDrvPort(erts_port_lookup_raw(port)); - ErtsPortNames *pnp = erts_get_port_names(port); + ErtsPortNames *pnp; erts_dsprintf_buf_t *dsbufp; + if (drv_port == ERTS_INVALID_ERL_DRV_PORT) { + Port *prt = erts_port_lookup_raw(port); + if (prt) + drv_port = ERTS_Port2ErlDrvPort(prt); + else + drv_port = ERTS_INVALID_ERL_DRV_PORT; + } + + pnp = erts_get_port_names(port, drv_port); + switch (mode) { case ERL_DRV_READ | ERL_DRV_WRITE: type = "Input/Output"; @@ -5039,12 +5009,16 @@ erts_stale_drv_select(Eterm port, } ErtsPortNames * -erts_get_port_names(Eterm id) +erts_get_port_names(Eterm id, ErlDrvPort drv_port) { - Port *prt = erts_port_lookup_raw(id); + Port *prt; ErtsPortNames *pnp; ASSERT(is_nil(id) || is_internal_port(id)); + prt = ERTS_ErlDrvPort2Port(drv_port); + if (prt == ERTS_INVALID_ERL_DRV_PORT) + prt = erts_port_lookup_raw(id); + if (!prt) { pnp = erts_alloc(ERTS_ALC_T_PORT_NAMES, sizeof(ErtsPortNames)); pnp->name = NULL; @@ -5056,6 +5030,7 @@ erts_get_port_names(Eterm id) size_t pnp_len = sizeof(ErtsPortNames); #ifndef DEBUG pnp_len += 100; /* In most cases 100 characters will be enough... */ + ASSERT(prt->common.id == id); #endif pnp = erts_alloc(ERTS_ALC_T_PORT_NAMES, pnp_len); do { @@ -5098,24 +5073,6 @@ erts_free_port_names(ErtsPortNames *pnp) erts_free(ERTS_ALC_T_PORT_NAMES, pnp); } -static void schedule_port_timeout(Port *p) -{ - /* - * Scheduling of port timeouts can be done without port locking, but - * since the task handle is stored in the port structure and the ptimer - * structure is protected by the port lock we require the port to be - * locked for now... - * - * TODO: Implement scheduling of port timeouts without locking - * the port. - * /Rickard - */ - ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(p)); - erts_port_task_schedule(p->common.id, - &p->timeout_task, - ERTS_PORT_TASK_TIMEOUT); -} - ErlDrvTermData driver_mk_term_nil(void) { return driver_term_nil; @@ -5144,7 +5101,7 @@ void driver_report_exit(ErlDrvPort ix, int status) rp = (scheduler ? erts_proc_lookup(pid) - : erts_pid2proc_opt(NULL, 0, pid, 0, ERTS_P2P_FLG_SMP_INC_REFC)); + : erts_pid2proc_opt(NULL, 0, pid, 0, ERTS_P2P_FLG_INC_REFC)); if (!rp) return; @@ -5154,15 +5111,11 @@ void driver_report_exit(ErlDrvPort ix, int status) hp += 3; tuple = TUPLE2(hp, prt->common.id, tuple); - erts_queue_message(rp, &rp_locks, bp, tuple, am_undefined -#ifdef USE_VM_PROBES - , NIL -#endif - ); + erts_queue_message(rp, &rp_locks, bp, tuple, am_undefined); erts_smp_proc_unlock(rp, rp_locks); if (!scheduler) - erts_smp_proc_dec_refc(rp); + erts_proc_dec_refc(rp); } #define ERTS_B2T_STATES_DEF_STATES_SZ 5 @@ -5254,22 +5207,27 @@ cleanup_b2t_states(struct b2t_states__ *b2tsp) static int driver_deliver_term(Eterm to, ErlDrvTermData* data, int len) { +#define HEAP_EXTRA 200 #define ERTS_DDT_FAIL do { res = -1; goto done; } while (0) Uint need = 0; int depth = 0; int res; - Eterm *hp = NULL, *hp_start = NULL, *hp_end = NULL; ErlDrvTermData* ptr; ErlDrvTermData* ptr_end; DECLARE_ESTACK(stack); - Eterm mess = NIL; /* keeps compiler happy */ + Eterm mess; Process* rp = NULL; - ErlHeapFragment *bp = NULL; - ErlOffHeap *ohp; + ErtsHeapFactory factory; ErtsProcLocks rp_locks = 0; struct b2t_states__ b2t; - int scheduler = 1; /* Silence erroneous warning... */ + int scheduler; + int is_heap_need_limited = 1; + ErtsSchedulerData *esdp = erts_get_scheduler_data(); + ERTS_UNDEF(mess,NIL); + ERTS_UNDEF(scheduler,1); + + factory.mode = FACTORY_CLOSED; init_b2t_states(&b2t); /* @@ -5433,14 +5391,34 @@ driver_deliver_term(Eterm to, ErlDrvTermData* data, int len) #ifdef DEBUG b2t.org_ext[b2t.ix] = ext; #endif - hsz = erts_binary2term_prepare(&b2t.state[b2t.ix++], ext, size); + hsz = erts_binary2term_prepare(&b2t.state[b2t.ix], ext, size); if (hsz < 0) ERTS_DDT_FAIL; /* Invalid data */ + b2t.state[b2t.ix++].heap_size = hsz; need += hsz; ptr += 2; depth++; + if (size > MAP_SMALL_MAP_LIMIT*3) { /* may contain big map */ + is_heap_need_limited = 0; + } + break; + } + case ERL_DRV_MAP: { /* int */ + ERTS_DDT_CHK_ENOUGH_ARGS(1); + if ((int) ptr[0] < 0) ERTS_DDT_FAIL; + if (ptr[0] > MAP_SMALL_MAP_LIMIT) { + need += HASHMAP_ESTIMATED_HEAP_SIZE(ptr[0]); + is_heap_need_limited = 0; + } else { + need += MAP_HEADER_FLATMAP_SZ + 1 + 2*ptr[0]; + } + depth -= 2*ptr[0]; + if (depth < 0) ERTS_DDT_FAIL; + ptr++; + depth++; break; } + default: ERTS_DDT_FAIL; } @@ -5463,14 +5441,23 @@ driver_deliver_term(Eterm to, ErlDrvTermData* data, int len) scheduler = erts_get_scheduler_id() != 0; rp = (scheduler ? erts_proc_lookup(to) - : erts_pid2proc_opt(NULL, 0, to, 0, ERTS_P2P_FLG_SMP_INC_REFC)); + : erts_pid2proc_opt(NULL, 0, to, 0, ERTS_P2P_FLG_INC_REFC)); if (!rp) { res = 0; goto done; } - hp_start = hp = erts_alloc_message_heap(need, &bp, &ohp, rp, &rp_locks); - hp_end = hp + need; + /* Try copy directly to destination heap if we know there are no big maps */ + if (is_heap_need_limited) { + ErlOffHeap *ohp; + ErlHeapFragment* bp; + Eterm* hp = erts_alloc_message_heap(need, &bp, &ohp, rp, &rp_locks); + erts_factory_message_init(&factory, rp, hp, bp); + } + else { + erts_factory_message_init(&factory, NULL, NULL, + new_message_buffer(need)); + } /* * Interpret the instructions and build the term. @@ -5491,13 +5478,15 @@ driver_deliver_term(Eterm to, ErlDrvTermData* data, int len) case ERL_DRV_INT: /* signed int argument */ #if HALFWORD_HEAP - mess = erts_bld_sint64(&hp, NULL, (Sint64)ptr[0]); + erts_reserve_heap(&factory, BIG_NEED_SIZE(2)); + mess = erts_bld_sint64(&factory.hp, NULL, (Sint64)ptr[0]); #else + erts_reserve_heap(&factory, BIG_UINT_HEAP_SIZE); if (IS_SSMALL((Sint)ptr[0])) mess = make_small((Sint)ptr[0]); else { - mess = small_to_big((Sint)ptr[0], hp); - hp += BIG_UINT_HEAP_SIZE; + mess = small_to_big((Sint)ptr[0], factory.hp); + factory.hp += BIG_UINT_HEAP_SIZE; } #endif ptr++; @@ -5505,25 +5494,29 @@ driver_deliver_term(Eterm to, ErlDrvTermData* data, int len) case ERL_DRV_UINT: /* unsigned int argument */ #if HALFWORD_HEAP - mess = erts_bld_uint64(&hp, NULL, (Uint64)ptr[0]); + erts_reserve_heap(&factory, BIG_NEED_FOR_BITS(64)); + mess = erts_bld_uint64(&factory.hp, NULL, (Uint64)ptr[0]); #else + erts_reserve_heap(&factory, BIG_UINT_HEAP_SIZE); if (IS_USMALL(0, (Uint)ptr[0])) mess = make_small((Uint)ptr[0]); else { - mess = uint_to_big((Uint)ptr[0], hp); - hp += BIG_UINT_HEAP_SIZE; + mess = uint_to_big((Uint)ptr[0], factory.hp); + factory.hp += BIG_UINT_HEAP_SIZE; } #endif ptr++; break; case ERL_DRV_INT64: /* pointer to unsigned 64-bit int argument */ - mess = erts_bld_sint64(&hp, NULL, *((Sint64 *) ptr[0])); + erts_reserve_heap(&factory, BIG_NEED_FOR_BITS(64)); + mess = erts_bld_sint64(&factory.hp, NULL, *((Sint64 *) ptr[0])); ptr++; break; case ERL_DRV_UINT64: /* pointer to unsigned 64-bit int argument */ - mess = erts_bld_uint64(&hp, NULL, *((Uint64 *) ptr[0])); + erts_reserve_heap(&factory, BIG_NEED_FOR_BITS(64)); + mess = erts_bld_uint64(&factory.hp, NULL, *((Uint64 *) ptr[0])); ptr++; break; @@ -5537,11 +5530,14 @@ driver_deliver_term(Eterm to, ErlDrvTermData* data, int len) Uint size = ptr[1]; Uint offset = ptr[2]; - erts_smp_atomic_add_nob(&erts_bytes_in, (erts_aint_t) size); + if (esdp) + esdp->io.in += (Uint64) size; + else + erts_atomic64_add_nob(&bytes_in, (erts_aint64_t) size); if (size <= ERL_ONHEAP_BIN_LIMIT) { - ErlHeapBin* hbp = (ErlHeapBin *) hp; - hp += heap_bin_size(size); + ErlHeapBin* hbp = (ErlHeapBin *) erts_produce_heap(&factory, + heap_bin_size(size), HEAP_EXTRA); hbp->thing_word = header_heap_bin(size); hbp->size = size; if (size > 0) { @@ -5550,18 +5546,18 @@ driver_deliver_term(Eterm to, ErlDrvTermData* data, int len) mess = make_binary(hbp); } else { - ProcBin* pb = (ProcBin *) hp; + ProcBin* pb = (ProcBin *) erts_produce_heap(&factory, + PROC_BIN_SIZE, HEAP_EXTRA); driver_binary_inc_refc(b); /* caller will free binary */ pb->thing_word = HEADER_PROC_BIN; pb->size = size; - pb->next = ohp->first; - ohp->first = (struct erl_off_heap_header*)pb; + pb->next = factory.off_heap->first; + factory.off_heap->first = (struct erl_off_heap_header*)pb; pb->val = ErlDrvBinary2Binary(b); pb->bytes = ((byte*) b->orig_bytes) + offset; pb->flags = 0; mess = make_binary(pb); - hp += PROC_BIN_SIZE; - OH_OVERHEAD(ohp, pb->size / sizeof(Eterm)); + OH_OVERHEAD(factory.off_heap, pb->size / sizeof(Eterm)); } ptr += 3; break; @@ -5571,11 +5567,15 @@ driver_deliver_term(Eterm to, ErlDrvTermData* data, int len) byte *bufp = (byte *) ptr[0]; Uint size = (Uint) ptr[1]; - erts_smp_atomic_add_nob(&erts_bytes_in, (erts_aint_t) size); + if (esdp) + esdp->io.in += (Uint64) size; + else + erts_atomic64_add_nob(&bytes_in, (erts_aint64_t) size); if (size <= ERL_ONHEAP_BIN_LIMIT) { - ErlHeapBin* hbp = (ErlHeapBin *) hp; - hp += heap_bin_size(size); + ErlHeapBin* hbp = (ErlHeapBin *) erts_produce_heap(&factory, + heap_bin_size(size), + HEAP_EXTRA); hbp->thing_word = header_heap_bin(size); hbp->size = size; if (size > 0) { @@ -5588,20 +5588,18 @@ driver_deliver_term(Eterm to, ErlDrvTermData* data, int len) ProcBin* pbp; Binary* bp = erts_bin_nrml_alloc(size); ASSERT(bufp); - bp->flags = 0; - bp->orig_size = (SWord) size; erts_refc_init(&bp->refc, 1); sys_memcpy((void *) bp->orig_bytes, (void *) bufp, size); - pbp = (ProcBin *) hp; - hp += PROC_BIN_SIZE; + pbp = (ProcBin *) erts_produce_heap(&factory, + PROC_BIN_SIZE, HEAP_EXTRA); pbp->thing_word = HEADER_PROC_BIN; pbp->size = size; - pbp->next = ohp->first; - ohp->first = (struct erl_off_heap_header*)pbp; + pbp->next = factory.off_heap->first; + factory.off_heap->first = (struct erl_off_heap_header*)pbp; pbp->val = bp; pbp->bytes = (byte*) bp->orig_bytes; pbp->flags = 0; - OH_OVERHEAD(ohp, pbp->size / sizeof(Eterm)); + OH_OVERHEAD(factory.off_heap, pbp->size / sizeof(Eterm)); mess = make_binary(pbp); } ptr += 2; @@ -5609,14 +5607,19 @@ driver_deliver_term(Eterm to, ErlDrvTermData* data, int len) } case ERL_DRV_STRING: /* char*, length */ - erts_smp_atomic_add_nob(&erts_bytes_in, (erts_aint_t) ptr[1]); - mess = buf_to_intlist(&hp, (char*)ptr[0], ptr[1], NIL); + if (esdp) + esdp->io.in += (Uint64) ptr[1]; + else + erts_atomic64_add_nob(&bytes_in, (erts_aint64_t) ptr[1]); + erts_reserve_heap(&factory, 2*ptr[1]); + mess = buf_to_intlist(&factory.hp, (char*)ptr[0], ptr[1], NIL); ptr += 2; break; case ERL_DRV_STRING_CONS: /* char*, length */ mess = ESTACK_POP(stack); - mess = buf_to_intlist(&hp, (char*)ptr[0], ptr[1], mess); + erts_reserve_heap(&factory, 2*ptr[1]); + mess = buf_to_intlist(&factory.hp, (char*)ptr[0], ptr[1], mess); ptr += 2; break; @@ -5625,11 +5628,12 @@ driver_deliver_term(Eterm to, ErlDrvTermData* data, int len) mess = ESTACK_POP(stack); i--; + erts_reserve_heap(&factory, 2*i); while(i > 0) { Eterm hd = ESTACK_POP(stack); - mess = CONS(hp, hd, mess); - hp += 2; + mess = CONS(factory.hp, hd, mess); + factory.hp += 2; i--; } ptr++; @@ -5638,13 +5642,12 @@ driver_deliver_term(Eterm to, ErlDrvTermData* data, int len) case ERL_DRV_TUPLE: { /* int */ int size = (int)ptr[0]; - Eterm* tp = hp; + Eterm* tp = erts_produce_heap(&factory, size+1, HEAP_EXTRA); *tp = make_arityval(size); mess = make_tuple(tp); tp += size; /* point at last element */ - hp = tp+1; /* advance "heap" pointer */ while(size--) { *tp-- = ESTACK_POP(stack); @@ -5660,23 +5663,69 @@ driver_deliver_term(Eterm to, ErlDrvTermData* data, int len) case ERL_DRV_FLOAT: { /* double * */ FloatDef f; + Eterm* fp = erts_produce_heap(&factory, FLOAT_SIZE_OBJECT, HEAP_EXTRA); - mess = make_float(hp); + mess = make_float(fp); f.fd = *((double *) ptr[0]); - PUT_DOUBLE(f, hp); - hp += FLOAT_SIZE_OBJECT; + if (!erts_isfinite(f.fd)) + ERTS_DDT_FAIL; + PUT_DOUBLE(f, fp); ptr++; break; } case ERL_DRV_EXT2TERM: /* char *ext, int size */ ASSERT(b2t.org_ext[b2t.ix] == (byte *) ptr[0]); - mess = erts_binary2term_create(&b2t.state[b2t.ix++], &hp, ohp); + + erts_reserve_heap(&factory, b2t.state[b2t.ix].heap_size); + mess = erts_binary2term_create(&b2t.state[b2t.ix++], &factory); if (mess == THE_NON_VALUE) ERTS_DDT_FAIL; ptr += 2; break; + case ERL_DRV_MAP: { /* int */ + int size = (int)ptr[0]; + if (size > MAP_SMALL_MAP_LIMIT) { + int ix = 2*size; + Eterm* leafs; + + erts_produce_heap(&factory, ix, HEAP_EXTRA); + leafs = factory.hp; + while(ix--) { *--leafs = ESTACK_POP(stack); } + + mess = erts_hashmap_from_array(&factory, leafs, size, 1); + if (is_non_value(mess)) + ERTS_DDT_FAIL; + } else { + Eterm* vp; + flatmap_t *mp; + Eterm* tp = erts_produce_heap(&factory, + 2*size + 1 + MAP_HEADER_FLATMAP_SZ, + HEAP_EXTRA); + + *tp = make_arityval(size); + + mp = (flatmap_t*) (tp + 1 + size); + mp->thing_word = MAP_HEADER_FLATMAP; + mp->size = size; + mp->keys = make_tuple(tp); + mess = make_flatmap(mp); + + tp += size; /* point at last key */ + vp = factory.hp - 1; /* point at last value */ + + while(size--) { + *vp-- = ESTACK_POP(stack); + *tp-- = ESTACK_POP(stack); + } + if (!erts_validate_and_sort_flatmap(mp)) + ERTS_DDT_FAIL; + } + ptr++; + break; + } + } ESTACK_PUSH(stack, mess); } @@ -5687,42 +5736,28 @@ driver_deliver_term(Eterm to, ErlDrvTermData* data, int len) if (res > 0) { mess = ESTACK_POP(stack); /* get resulting value */ - if (bp) - bp = erts_resize_message_buffer(bp, hp - hp_start, &mess, 1); - else { - ASSERT(hp); - HRelease(rp, hp_end, hp); - } + erts_factory_close(&factory); /* send message */ - erts_queue_message(rp, &rp_locks, bp, mess, am_undefined -#ifdef USE_VM_PROBES - , NIL -#endif - ); + erts_queue_message(rp, &rp_locks, factory.heap_frags, mess, am_undefined); } else { if (b2t.ix > b2t.used) b2t.used = b2t.ix; for (b2t.ix = 0; b2t.ix < b2t.used; b2t.ix++) erts_binary2term_abort(&b2t.state[b2t.ix]); - if (bp) - free_message_buffer(bp); - else if (hp) { - HRelease(rp, hp_end, hp); - } + erts_factory_undo(&factory); } -#ifdef ERTS_SMP if (rp) { if (rp_locks) erts_smp_proc_unlock(rp, rp_locks); if (!scheduler) - erts_smp_proc_dec_refc(rp); + erts_proc_dec_refc(rp); } -#endif cleanup_b2t_states(&b2t); DESTROY_ESTACK(stack); return res; #undef ERTS_DDT_FAIL +#undef HEAP_EXTRA } static ERTS_INLINE int @@ -5851,6 +5886,7 @@ int driver_output_binary(ErlDrvPort ix, char* hbuf, ErlDrvSizeT hlen, { erts_aint32_t state; Port* prt = erts_drvport2port_state(ix, &state); + ErtsSchedulerData *esdp = erts_get_scheduler_data(); ERTS_SMP_CHK_NO_PROC_LOCKS; @@ -5861,7 +5897,10 @@ int driver_output_binary(ErlDrvPort ix, char* hbuf, ErlDrvSizeT hlen, return 0; prt->bytes_in += (hlen + len); - erts_smp_atomic_add_nob(&erts_bytes_in, (erts_aint_t) (hlen + len)); + if (esdp) + esdp->io.in += (Uint64) (hlen + len); + else + erts_atomic64_add_nob(&bytes_in, (erts_aint64_t) (hlen + len)); if (state & ERTS_PORT_SFLG_DISTRIBUTION) { return erts_net_message(prt, prt->dist_entry, @@ -5886,6 +5925,7 @@ int driver_output2(ErlDrvPort ix, char* hbuf, ErlDrvSizeT hlen, { erts_aint32_t state; Port* prt = erts_drvport2port_state(ix, &state); + ErtsSchedulerData *esdp = erts_get_scheduler_data(); ERTS_SMP_CHK_NO_PROC_LOCKS; @@ -5897,7 +5937,10 @@ int driver_output2(ErlDrvPort ix, char* hbuf, ErlDrvSizeT hlen, return 0; prt->bytes_in += (hlen + len); - erts_smp_atomic_add_nob(&erts_bytes_in, (erts_aint_t) (hlen + len)); + if (esdp) + esdp->io.in += (Uint64) (hlen + len); + else + erts_atomic64_add_nob(&bytes_in, (erts_aint64_t) (hlen + len)); if (state & ERTS_PORT_SFLG_DISTRIBUTION) { if (len == 0) return erts_net_message(prt, @@ -5937,6 +5980,7 @@ int driver_outputv(ErlDrvPort ix, char* hbuf, ErlDrvSizeT hlen, ErlDrvBinary** binv; Port* prt; erts_aint32_t state; + ErtsSchedulerData *esdp = erts_get_scheduler_data(); ERTS_SMP_CHK_NO_PROC_LOCKS; @@ -5945,10 +5989,6 @@ int driver_outputv(ErlDrvPort ix, char* hbuf, ErlDrvSizeT hlen, return driver_output2(ix, hbuf, hlen, NULL, 0); size = vec->size - skip; /* Size of remaining bytes in vector */ - ASSERT(hlen >= 0); /* debug only */ - if (hlen < 0) - hlen = 0; - prt = erts_drvport2port_state(ix, &state); if (prt == ERTS_INVALID_ERL_DRV_PORT) return -1; @@ -5971,7 +6011,7 @@ int driver_outputv(ErlDrvPort ix, char* hbuf, ErlDrvSizeT hlen, binv++; n--; } else { - iov->iov_base += skip; + iov->iov_base = ((char *)(iov->iov_base)) + skip; iov->iov_len -= skip; skip = 0; } @@ -5979,7 +6019,10 @@ int driver_outputv(ErlDrvPort ix, char* hbuf, ErlDrvSizeT hlen, /* XXX handle distribution !!! */ prt->bytes_in += (hlen + size); - erts_smp_atomic_add_nob(&erts_bytes_in, (erts_aint_t) (hlen + size)); + if (esdp) + esdp->io.in += (Uint64) (hlen + size); + else + erts_atomic64_add_nob(&bytes_in, (erts_aint64_t) (hlen + size)); deliver_vec_message(prt, ERTS_PORT_GET_CONNECTED(prt), hbuf, hlen, binv, iov, n, size); return 0; @@ -6055,9 +6098,7 @@ driver_alloc_binary(ErlDrvSizeT size) bin = erts_bin_drv_alloc_fnf((Uint) size); if (!bin) return NULL; /* The driver write must take action */ - bin->flags = BIN_FLAG_DRV; erts_refc_init(&bin->refc, 1); - bin->orig_size = (SWord) size; return Binary2ErlDrvBinary(bin); } @@ -6087,7 +6128,6 @@ ErlDrvBinary* driver_realloc_binary(ErlDrvBinary* bin, ErlDrvSizeT size) if (!newbin) return NULL; - newbin->orig_size = size; return Binary2ErlDrvBinary(newbin); } @@ -6231,7 +6271,7 @@ driver_pdl_create(ErlDrvPort dp) return NULL; pdl = erts_alloc(ERTS_ALC_T_PORT_DATA_LOCK, sizeof(struct erl_drv_port_data_lock)); - erts_mtx_init(&pdl->mtx, "port_data_lock"); + erts_mtx_init_x(&pdl->mtx, "port_data_lock", pp->common.id, 1); pdl_init_refc(pdl); erts_port_inc_refc(pp); pdl->prt = pp; @@ -6416,7 +6456,7 @@ int driver_enqv(ErlDrvPort ix, ErlIOVec* vec, ErlDrvSizeT skip) n--; } else { - iov->iov_base += skip; + iov->iov_base = ((char *)(iov->iov_base)) + skip; iov->iov_len -= skip; skip = 0; } @@ -6481,7 +6521,7 @@ int driver_pushqv(ErlDrvPort ix, ErlIOVec* vec, ErlDrvSizeT skip) n--; } else { - iov->iov_base += skip; + iov->iov_base = ((char *)(iov->iov_base)) + skip; iov->iov_len -= skip; skip = 0; } @@ -6540,7 +6580,7 @@ ErlDrvSizeT driver_deq(ErlDrvPort ix, ErlDrvSizeT size) q->v_head++; } else { - q->v_head->iov_base += size; + q->v_head->iov_base = ((char *)(q->v_head->iov_base)) + size; q->v_head->iov_len -= size; size = 0; } @@ -6671,18 +6711,6 @@ int driver_pushq(ErlDrvPort ix, char* buffer, ErlDrvSizeT len) return code; } -static ERTS_INLINE void -drv_cancel_timer(Port *prt) -{ -#ifdef ERTS_SMP - erts_cancel_smp_ptimer(prt->common.u.alive.ptimer); -#else - erts_cancel_timer(&prt->common.u.alive.tm); -#endif - if (erts_port_task_is_scheduled(&prt->timeout_task)) - erts_port_task_abort(&prt->timeout_task); -} - int driver_set_timer(ErlDrvPort ix, unsigned long t) { Port* prt = erts_drvport2port(ix); @@ -6694,19 +6722,8 @@ int driver_set_timer(ErlDrvPort ix, unsigned long t) if (prt->drv_ptr->timeout == NULL) return -1; - drv_cancel_timer(prt); -#ifdef ERTS_SMP - erts_create_smp_ptimer(&prt->common.u.alive.ptimer, - prt->common.id, - (ErlTimeoutProc) schedule_port_timeout, - t); -#else - erts_set_timer(&prt->common.u.alive.tm, - (ErlTimeoutProc) schedule_port_timeout, - NULL, - prt, - t); -#endif + + erts_set_port_timer(prt, (Sint64) t); return 0; } @@ -6716,28 +6733,28 @@ int driver_cancel_timer(ErlDrvPort ix) if (prt == ERTS_INVALID_ERL_DRV_PORT) return -1; ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(prt)); - drv_cancel_timer(prt); + erts_cancel_port_timer(prt); return 0; } - int driver_read_timer(ErlDrvPort ix, unsigned long* t) { Port* prt = erts_drvport2port(ix); + Sint64 left; ERTS_SMP_CHK_NO_PROC_LOCKS; if (prt == ERTS_INVALID_ERL_DRV_PORT) return -1; ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(prt)); -#ifdef ERTS_SMP - *t = (prt->common.u.alive.ptimer - ? erts_time_left(&prt->common.u.alive.ptimer->timer.tm) - : 0); -#else - *t = erts_time_left(&prt->common.u.alive.tm); -#endif + + left = erts_read_port_timer(prt); + if (left < 0) + left = 0; + + *t = (unsigned long) left; + return 0; } @@ -6757,11 +6774,33 @@ driver_get_now(ErlDrvNowData *now_data) return 0; } +ErlDrvTime +erl_drv_monotonic_time(ErlDrvTimeUnit time_unit) +{ + return (ErlDrvTime) erts_napi_monotonic_time((int) time_unit); +} + +ErlDrvTime +erl_drv_time_offset(ErlDrvTimeUnit time_unit) +{ + return (ErlDrvTime) erts_napi_time_offset((int) time_unit); +} + +ErlDrvTime +erl_drv_convert_time_unit(ErlDrvTime val, + ErlDrvTimeUnit from, + ErlDrvTimeUnit to) +{ + return (ErlDrvTime) erts_napi_convert_time_unit((ErtsMonotonicTime) val, + (int) from, + (int) to); +} + static void ref_to_driver_monitor(Eterm ref, ErlDrvMonitor *mon) { RefThing *refp; ASSERT(is_internal_ref(ref)); - ASSERT(sizeof(RefThing) <= sizeof(ErlDrvMonitor)); + ERTS_CT_ASSERT(sizeof(RefThing) <= sizeof(ErlDrvMonitor)); refp = ref_thing_ptr(ref); memset(mon,0,sizeof(ErlDrvMonitor)); memcpy(mon,refp,sizeof(RefThing)); @@ -7213,7 +7252,7 @@ void *driver_dl_open(char * path) int res; int *last_error_p = erts_smp_tsd_get(driver_list_last_error_key); int locked = maybe_lock_driver_list(); - if ((res = erts_sys_ddll_open(path, &ptr)) == 0) { + if ((res = erts_sys_ddll_open(path, &ptr, NULL)) == 0) { maybe_unlock_driver_list(locked); return ptr; } else { @@ -7268,7 +7307,7 @@ char *driver_dl_error(void) #define ERL_DRV_SYS_INFO_SIZE(LAST_FIELD) \ - (((size_t) &((ErlDrvSysInfo *) 0)->LAST_FIELD) \ + (offsetof(ErlDrvSysInfo, LAST_FIELD) \ + sizeof(((ErlDrvSysInfo *) 0)->LAST_FIELD)) void @@ -7284,7 +7323,7 @@ driver_system_info(ErlDrvSysInfo *sip, size_t si_size) * of ErlDrvSysInfo (introduced in driver version 1.0). */ if (!sip || si_size < ERL_DRV_SYS_INFO_SIZE(smp_support)) - erl_exit(1, + erts_exit(ERTS_ERROR_EXIT, "driver_system_info(%p, %ld) called with invalid arguments\n", sip, si_size); @@ -7330,6 +7369,18 @@ driver_system_info(ErlDrvSysInfo *sip, size_t si_size) sip->nif_major_version = ERL_NIF_MAJOR_VERSION; sip->nif_minor_version = ERL_NIF_MINOR_VERSION; } + /* + * 'dirty_scheduler_support' is the last field in the 4th version + * (driver version 3.1, NIF version 2.7) + */ + if (si_size >= ERL_DRV_SYS_INFO_SIZE(dirty_scheduler_support)) { +#if defined(ERL_NIF_DIRTY_SCHEDULER_SUPPORT) && defined(USE_THREADS) + sip->dirty_scheduler_support = 1; +#else + sip->dirty_scheduler_support = 0; +#endif + } + } @@ -7393,6 +7444,8 @@ no_stop_select_callback(ErlDrvEvent event, void* private) erts_send_error_to_logger_nogl(dsbufp); } +#define IS_DRIVER_VERSION_GE(DE,MAJOR,MINOR) \ + ((DE)->major_version >= (MAJOR) && (DE)->minor_version >= (MINOR)) static int init_driver(erts_driver_t *drv, ErlDrvEntry *de, DE_Handle *handle) @@ -7416,10 +7469,11 @@ init_driver(erts_driver_t *drv, ErlDrvEntry *de, DE_Handle *handle) erts_atom_put((byte *) drv->name, sys_strlen(drv->name), ERTS_ATOM_ENC_LATIN1, - 1) + 1), #else - NIL + NIL, #endif + 1 ); } #endif @@ -7439,6 +7493,7 @@ init_driver(erts_driver_t *drv, ErlDrvEntry *de, DE_Handle *handle) drv->timeout = de->timeout ? de->timeout : no_timeout_callback; drv->ready_async = de->ready_async; drv->process_exit = de->process_exit; + drv->emergency_close = IS_DRIVER_VERSION_GE(de,3,2) ? de->emergency_close : NULL; if (de->stop_select) drv->stop_select = de->stop_select; else @@ -7457,6 +7512,8 @@ init_driver(erts_driver_t *drv, ErlDrvEntry *de, DE_Handle *handle) } } +#undef IS_DRIVER_VERSION_GE + void erts_destroy_driver(erts_driver_t *drv) { @@ -7573,15 +7630,15 @@ int null_func(void) } int -erl_drv_putenv(char *key, char *value) +erl_drv_putenv(const char *key, char *value) { - return erts_sys_putenv_raw(key, value); + return erts_sys_putenv_raw((char*)key, value); } int -erl_drv_getenv(char *key, char *value, size_t *value_size) +erl_drv_getenv(const char *key, char *value, size_t *value_size) { - return erts_sys_getenv_raw(key, value, value_size); + return erts_sys_getenv_raw((char*)key, value, value_size); } /* get heart_port @@ -7600,7 +7657,7 @@ Port *erts_get_heart_port(void) if (!port) continue; /* only examine undead or alive ports */ - if (erts_atomic32_read_nob(&port->state) & ERTS_PORT_SFLGS_DEAD) + if (erts_atomic32_read_nob(&port->state) & ERTS_PORT_SFLGS_INVALID_DRIVER_LOOKUP) continue; /* immediate atom compare */ reg = port->common.u.alive.reg; @@ -7611,3 +7668,23 @@ Port *erts_get_heart_port(void) return NULL; } + +void erts_emergency_close_ports(void) +{ + int ix, max = erts_ptab_max(&erts_port); + + for (ix = 0; ix < max; ix++) { + Port *port = erts_pix2port(ix); + + if (!port) + continue; + /* only examine undead or alive ports */ + if (erts_atomic32_read_nob(&port->state) & ERTS_PORT_SFLGS_INVALID_DRIVER_LOOKUP) + continue; + + /* emergency close socket */ + if (port->drv_ptr->emergency_close) { + port->drv_ptr->emergency_close((ErlDrvData) port->drv_data); + } + } +} diff --git a/erts/emulator/beam/module.c b/erts/emulator/beam/module.c index daa6e136c5..86dd3b5aac 100644 --- a/erts/emulator/beam/module.c +++ b/erts/emulator/beam/module.c @@ -3,16 +3,17 @@ * * Copyright Ericsson AB 1996-2013. All Rights Reserved. * - * The contents of this file are subject to the Erlang Public License, - * Version 1.1, (the "License"); you may not use this file except in - * compliance with the License. You should have received a copy of the - * Erlang Public License along with this software. If not, it can be - * retrieved online at http://www.erlang.org/. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Software distributed under the License is distributed on an "AS IS" - * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See - * the License for the specific language governing rights and limitations - * under the License. + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. * * %CopyrightEnd% */ diff --git a/erts/emulator/beam/module.h b/erts/emulator/beam/module.h index 5235528e98..c8a6351b04 100644 --- a/erts/emulator/beam/module.h +++ b/erts/emulator/beam/module.h @@ -3,16 +3,17 @@ * * Copyright Ericsson AB 1996-2013. All Rights Reserved. * - * The contents of this file are subject to the Erlang Public License, - * Version 1.1, (the "License"); you may not use this file except in - * compliance with the License. You should have received a copy of the - * Erlang Public License along with this software. If not, it can be - * retrieved online at http://www.erlang.org/. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Software distributed under the License is distributed on an "AS IS" - * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See - * the License for the specific language governing rights and limitations - * under the License. + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. * * %CopyrightEnd% */ diff --git a/erts/emulator/beam/ops.tab b/erts/emulator/beam/ops.tab index 1e5ae46bfa..1d32e72247 100644 --- a/erts/emulator/beam/ops.tab +++ b/erts/emulator/beam/ops.tab @@ -3,16 +3,17 @@ # # Copyright Ericsson AB 1997-2013. All Rights Reserved. # -# The contents of this file are subject to the Erlang Public License, -# Version 1.1, (the "License"); you may not use this file except in -# compliance with the License. You should have received a copy of the -# Erlang Public License along with this software. If not, it can be -# retrieved online at http://www.erlang.org/. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# Software distributed under the License is distributed on an "AS IS" -# basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See -# the License for the specific language governing rights and limitations -# under the License. +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. # # %CopyrightEnd% # @@ -166,22 +167,26 @@ is_tuple Fail=f S | select_tuple_arity S=d Fail=f Size=u Rest=* => \ select_tuple_arity S=d Fail=f Size=u Rest=* => \ gen_select_tuple_arity(S, Fail, Size, Rest) -i_select_val r f I -i_select_val x f I -i_select_val y f I +i_select_val_bins r f I +i_select_val_bins x f I +i_select_val_bins y f I -i_select_val2 r f c f c f -i_select_val2 x f c f c f -i_select_val2 y f c f c f +i_select_val_lins r f I +i_select_val_lins x f I +i_select_val_lins y f I -i_select_tuple_arity2 r f A f A f -i_select_tuple_arity2 x f A f A f -i_select_tuple_arity2 y f A f A f +i_select_val2 r f c c f f +i_select_val2 x f c c f f +i_select_val2 y f c c f f i_select_tuple_arity r f I i_select_tuple_arity x f I i_select_tuple_arity y f I +i_select_tuple_arity2 r f A A f f +i_select_tuple_arity2 x f A A f f +i_select_tuple_arity2 y f A A f f + i_jump_on_val_zero r f I i_jump_on_val_zero x f I i_jump_on_val_zero y f I @@ -294,10 +299,49 @@ move_jump f c move_jump f x move_jump f y + +# Movement to and from the stack is common +# Try to pack as much as we can into one instruction + +# Window move +move_window/5 +move_window/6 + +# x -> y + +move S1=r S2=y | move X1=x Y1=y => move2 S1 S2 X1 Y1 + +move X1=x Y1=y | move X2=x Y2=y | move X3=x Y3=y | succ(Y1,Y2) | succ(Y2,Y3) => \ + move_window X1 X2 X3 Y1 Y3 + +move_window X1=x X2=x X3=x Y1=y Y3=y | move X4=x Y4=y | succ(Y3,Y4) => \ + move_window X1 X2 X3 X4 Y1 Y4 + +move_window X1=x X2=x X3=x X4=x Y1=y Y4=y | move X5=x Y5=y | succ(Y4,Y5) => \ + move_window5 X1 X2 X3 X4 X5 Y1 + +move_window X1=x X2=x X3=x Y1=y Y3=y => move_window3 X1 X2 X3 Y1 +move_window X1=x X2=x X3=x X4=x Y1=y Y4=y => move_window4 X1 X2 X3 X4 Y1 + +move_window3 x x x y +move_window4 x x x x y +move_window5 x x x x x y + move X1=x Y1=y | move X2=x Y2=y => move2 X1 Y1 X2 Y2 move Y1=y X1=x | move Y2=y X2=x => move2 Y1 X1 Y2 X2 move X1=x X2=x | move X3=x X4=x => move2 X1 X2 X3 X4 +move S1=x S2=r | move S3=x S4=x => move2 S1 S2 S3 S4 +move S1=x S2=r | move X1=x Y1=y => move2 S1 S2 X1 Y1 +move S1=y S2=r | move X1=x Y1=y => move2 S1 S2 X1 Y1 + +move Y1=y X1=x | move S1=r D1=x => move2 Y1 X1 S1 D1 +move S1=r D1=x | move Y1=y X1=x => move2 S1 D1 Y1 X1 + +move2 X1=x Y1=y X2=x Y2=y | move X3=x Y3=y => move3 X1 Y1 X2 Y2 X3 Y3 +move2 Y1=y X1=x Y2=y X2=x | move Y3=y X3=x => move3 Y1 X1 Y2 X2 Y3 X3 +move2 X1=x X2=x X3=x X4=x | move X5=x X6=x => move3 X1 X2 X3 X4 X5 X6 + move C=aiq X=x==1 => move_x1 C move C=aiq X=x==2 => move_x2 C @@ -309,6 +353,20 @@ move2 x y x y move2 y x y x move2 x x x x +move2 x r x x + +move2 x r x y +move2 r y x y +move2 y r x y + +move2 r x y x +move2 y x r x + +%macro: move3 Move3 +move3 x y x y x y +move3 y x y x y x +move3 x x x x x x + # The compiler almost never generates a "move Literal y(Y)" instruction, # so let's cheat if we encounter one. move S=n D=y => init D @@ -388,14 +446,59 @@ i_is_ne_exact_literal x f c i_is_ne_exact_literal y f c # +# Common Compare Specializations +# We don't do all of them since we want +# to keep the instruction set small-ish +# + +is_eq_exact Lbl S1=xy S2=r => is_eq_exact Lbl S2 S1 +is_eq_exact Lbl S1=rx S2=xy => i_is_eq_exact_spec Lbl S1 S2 +%macro: i_is_eq_exact_spec EqualExact -fail_action + +i_is_eq_exact_spec f x x +i_is_eq_exact_spec f x y +i_is_eq_exact_spec f r x +i_is_eq_exact_spec f r y +%cold +i_is_eq_exact_spec f r r +%hot + +is_lt Lbl S1=rxc S2=rxc => i_is_lt_spec Lbl S1 S2 + +%macro: i_is_lt_spec IsLessThan -fail_action + +i_is_lt_spec f x x +i_is_lt_spec f x r +i_is_lt_spec f x c +i_is_lt_spec f r x +i_is_lt_spec f r c +i_is_lt_spec f c x +i_is_lt_spec f c r +%cold +i_is_lt_spec f r r +i_is_lt_spec f c c +%hot + +is_ge Lbl S1=xc S2=xc => i_is_ge_spec Lbl S1 S2 + +%macro: i_is_ge_spec IsGreaterEqual -fail_action + +i_is_ge_spec f x x +i_is_ge_spec f x c +i_is_ge_spec f c x +%cold +i_is_ge_spec f c c +%hot + +# # All other comparisons. # is_eq_exact Lbl S1 S2 => i_fetch S1 S2 | i_is_eq_exact Lbl is_ne_exact Lbl S1 S2 => i_fetch S1 S2 | i_is_ne_exact Lbl -is_ge Lbl S1 S2 => i_fetch S1 S2 | i_is_ge Lbl is_lt Lbl S1 S2 => i_fetch S1 S2 | i_is_lt Lbl +is_ge Lbl S1 S2 => i_fetch S1 S2 | i_is_ge Lbl is_eq Lbl S1 S2 => i_fetch S1 S2 | i_is_eq Lbl is_ne Lbl S1 S2 => i_fetch S1 S2 | i_is_ne Lbl @@ -489,7 +592,6 @@ put_list s s d %hot %macro: i_fetch FetchArgs -pack -i_fetch c c i_fetch c r i_fetch c x i_fetch c y @@ -506,6 +608,7 @@ i_fetch y x i_fetch y y %cold +i_fetch c c i_fetch s s %hot @@ -763,17 +866,17 @@ allocate_init t I y ################################################################# # -# The BIFs erlang:check_process_code/2 must be called like a function, +# The BIFs erts_internal:check_process_code/2 must be called like a function, # to ensure that c_p->i (program counter) is set correctly (an ordinary # BIF call doesn't set it). # -call_ext u==2 Bif=u$bif:erlang:check_process_code/2 => i_call_ext Bif -call_ext_last u==2 Bif=u$bif:erlang:check_process_code/2 D => i_call_ext_last Bif D -call_ext_only u==2 Bif=u$bif:erlang:check_process_code/2 => i_call_ext_only Bif +call_ext u==2 Bif=u$bif:erts_internal:check_process_code/2 => i_call_ext Bif +call_ext_last u==2 Bif=u$bif:erts_internal:check_process_code/2 D => i_call_ext_last Bif D +call_ext_only u==2 Bif=u$bif:erts_internal:check_process_code/2 => i_call_ext_only Bif # -# The BIFs erlang:garbage_collect/0,1 must be called like functions, +# The BIFs erlang:garbage_collect/0 must be called like a function, # to allow them to invoke the garbage collector. (The stack pointer must # be saved and p->arity must be zeroed, which is not done on ordinary BIF calls.) # @@ -782,10 +885,6 @@ call_ext u==0 Bif=u$bif:erlang:garbage_collect/0 => i_call_ext Bif call_ext_last u==0 Bif=u$bif:erlang:garbage_collect/0 D => i_call_ext_last Bif D call_ext_only u==0 Bif=u$bif:erlang:garbage_collect/0 => i_call_ext_only Bif -call_ext u==1 Bif=u$bif:erlang:garbage_collect/1 => i_call_ext Bif -call_ext_last u==1 Bif=u$bif:erlang:garbage_collect/1 D => i_call_ext_last Bif D -call_ext_only u==1 Bif=u$bif:erlang:garbage_collect/1 => i_call_ext_only Bif - # # put/2 and erase/1 must be able to do garbage collection, so we must call # them like functions. @@ -1470,6 +1569,81 @@ apply I apply_last I P # +# Map instructions in R17. +# + +sorted_put_map_assoc/5 +put_map_assoc F Map Dst Live Size Rest=* | map_key_sort(Size, Rest) => \ + sorted_put_map_assoc F Map Dst Live Size Rest + +sorted_put_map_exact/5 +put_map_exact F Map Dst Live Size Rest=* | map_key_sort(Size, Rest) => \ + sorted_put_map_exact F Map Dst Live Size Rest + +sorted_put_map_assoc j Map Dst Live Size Rest=* | is_empty_map(Map) => \ + new_map Dst Live Size Rest +sorted_put_map_assoc F Src=s Dst Live Size Rest=* => \ + update_map_assoc F Src Dst Live Size Rest +sorted_put_map_assoc F Src Dst Live Size Rest=* => \ + move Src x | update_map_assoc F x Dst Live Size Rest + +sorted_put_map_exact F Src=s Dst Live Size Rest=* => \ + update_map_exact F Src Dst Live Size Rest +sorted_put_map_exact F Src Dst Live Size Rest=* => \ + move Src x | update_map_exact F x Dst Live Size Rest + +new_map d I I +update_map_assoc j s d I I +update_map_exact j s d I I + +is_map Fail Lit=q | literal_is_map(Lit) => +is_map Fail cq => jump Fail + +%macro: is_map IsMap -fail_action +is_map f r +is_map f x +is_map f y + +## Transform has_map_fields #{ K1 := _, K2 := _ } to has_map_elements + +has_map_fields Fail Src Size Rest=* => \ + gen_has_map_fields(Fail, Src, Size, Rest) + +## Transform get_map_elements(s) #{ K1 := V1, K2 := V2 } + +get_map_elements Fail Src=rxy Size=u==2 Rest=* => \ + gen_get_map_element(Fail, Src, Size, Rest) +get_map_elements Fail Src Size Rest=* | map_key_sort(Size, Rest) => \ + gen_get_map_elements(Fail, Src, Size, Rest) + +i_get_map_elements f s I + +i_get_map_element Fail Src=rxy Key=ry Dst => \ + move Key x | i_get_map_element Fail Src x Dst + +%macro: i_get_map_element_hash GetMapElementHash -fail_action +i_get_map_element_hash f r c I r +i_get_map_element_hash f x c I r +i_get_map_element_hash f y c I r +i_get_map_element_hash f r c I x +i_get_map_element_hash f x c I x +i_get_map_element_hash f y c I x +i_get_map_element_hash f r c I y +i_get_map_element_hash f x c I y +i_get_map_element_hash f y c I y + +%macro: i_get_map_element GetMapElement -fail_action +i_get_map_element f r x r +i_get_map_element f x x r +i_get_map_element f y x r +i_get_map_element f r x x +i_get_map_element f x x x +i_get_map_element f y x x +i_get_map_element f r x y +i_get_map_element f x x y +i_get_map_element f y x y + +# # Optimize addition and subtraction of small literals using # the i_increment/4 instruction (in bodies, not in guards). # @@ -1487,17 +1661,21 @@ gc_bif2 p Live u$bif:erlang:sminus/2 Reg=d Int=i Dst | \ # GCing arithmetic instructions. # +gc_bif2 Fail I u$bif:erlang:splus/2 S1=x S2=x Dst=d => i_plus Fail I S1 S2 Dst gc_bif2 Fail I u$bif:erlang:splus/2 S1 S2 Dst=d => i_fetch S1 S2 | i_plus Fail I Dst +gc_bif2 Fail I u$bif:erlang:sminus/2 S1=x S2=x Dst=d => i_minus Fail I S1 S2 Dst gc_bif2 Fail I u$bif:erlang:sminus/2 S1 S2 Dst=d => i_fetch S1 S2 | i_minus Fail I Dst gc_bif2 Fail I u$bif:erlang:stimes/2 S1 S2 Dst=d => i_fetch S1 S2 | i_times Fail I Dst gc_bif2 Fail I u$bif:erlang:div/2 S1 S2 Dst=d => i_fetch S1 S2 | i_m_div Fail I Dst gc_bif2 Fail I u$bif:erlang:intdiv/2 S1 S2 Dst=d => i_fetch S1 S2 | i_int_div Fail I Dst +gc_bif2 Fail I u$bif:erlang:rem/2 S1=x S2=x Dst=d => i_rem Fail I S1 S2 Dst gc_bif2 Fail I u$bif:erlang:rem/2 S1 S2 Dst=d => i_fetch S1 S2 | i_rem Fail I Dst gc_bif2 Fail I u$bif:erlang:bsl/2 S1 S2 Dst=d => i_fetch S1 S2 | i_bsl Fail I Dst gc_bif2 Fail I u$bif:erlang:bsr/2 S1 S2 Dst=d => i_fetch S1 S2 | i_bsr Fail I Dst +gc_bif2 Fail I u$bif:erlang:band/2 S1=x S2=c Dst=d => i_band Fail I S1 S2 Dst gc_bif2 Fail I u$bif:erlang:band/2 S1 S2 Dst=d => i_fetch S1 S2 | i_band Fail I Dst gc_bif2 Fail I u$bif:erlang:bor/2 S1 S2 Dst=d => i_fetch S1 S2 | i_bor Fail I Dst gc_bif2 Fail I u$bif:erlang:bxor/2 S1 S2 Dst=d => i_fetch S1 S2 | i_bxor Fail I Dst @@ -1511,16 +1689,20 @@ i_increment r I I d i_increment x I I d i_increment y I I d +i_plus j I x x d i_plus j I d +i_minus j I x x d i_minus j I d i_times j I d i_m_div j I d i_int_div j I d +i_rem j I x x d i_rem j I d i_bsl j I d i_bsr j I d +i_band j I x c d i_band j I d i_bor j I d i_bxor j I d diff --git a/erts/emulator/beam/packet_parser.c b/erts/emulator/beam/packet_parser.c index db0e78b1a7..a737a86f14 100644 --- a/erts/emulator/beam/packet_parser.c +++ b/erts/emulator/beam/packet_parser.c @@ -3,16 +3,17 @@ * * Copyright Ericsson AB 2008-2013. All Rights Reserved. * - * The contents of this file are subject to the Erlang Public License, - * Version 1.1, (the "License"); you may not use this file except in - * compliance with the License. You should have received a copy of the - * Erlang Public License along with this software. If not, it can be - * retrieved online at http://www.erlang.org/. - * - * Software distributed under the License is distributed on an "AS IS" - * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See - * the License for the specific language governing rights and limitations - * under the License. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. * * %CopyrightEnd% */ @@ -255,6 +256,7 @@ int packet_get_length(enum PacketParseType htype, const char* ptr, unsigned n, /* Bytes read so far */ unsigned max_plen, /* Max packet length, 0=no limit */ unsigned trunc_len, /* Truncate (lines) if longer, 0=no limit */ + char delimiter, /* Line delimiting character */ int* statep) /* Protocol specific state */ { unsigned hlen, plen; @@ -298,9 +300,9 @@ int packet_get_length(enum PacketParseType htype, goto remain; case TCP_PB_LINE_LF: { - /* TCP_PB_LINE_LF: [Data ... \n] */ + /* TCP_PB_LINE_LF: [Data ... Delimiter] */ const char* ptr2; - if ((ptr2 = memchr(ptr, '\n', n)) == NULL) { + if ((ptr2 = memchr(ptr, delimiter, n)) == NULL) { if (n > max_plen && max_plen != 0) { /* packet full */ DEBUGF((" => packet full (no NL)=%d\r\n", n)); goto error; diff --git a/erts/emulator/beam/packet_parser.h b/erts/emulator/beam/packet_parser.h index 1c3a9aa3da..717d905fad 100644 --- a/erts/emulator/beam/packet_parser.h +++ b/erts/emulator/beam/packet_parser.h @@ -3,16 +3,17 @@ * * Copyright Ericsson AB 2008-2009. All Rights Reserved. * - * The contents of this file are subject to the Erlang Public License, - * Version 1.1, (the "License"); you may not use this file except in - * compliance with the License. You should have received a copy of the - * Erlang Public License along with this software. If not, it can be - * retrieved online at http://www.erlang.org/. - * - * Software distributed under the License is distributed on an "AS IS" - * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See - * the License for the specific language governing rights and limitations - * under the License. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. * * %CopyrightEnd% */ @@ -104,7 +105,8 @@ int packet_get_length(enum PacketParseType htype, const char* ptr, unsigned n, /* Bytes read so far */ unsigned max_plen, /* Packet max length, 0=no limit */ unsigned trunc_len, /* Truncate (lines) if longer, 0=no limit */ - int* statep); /* Internal protocol state */ + char delimiter, /* Line delimiting character */ + int* statep); /* Internal protocol state */ ERTS_GLB_INLINE void packet_get_body(enum PacketParseType htype, diff --git a/erts/emulator/beam/register.c b/erts/emulator/beam/register.c index c626cb2780..020e61d136 100644 --- a/erts/emulator/beam/register.c +++ b/erts/emulator/beam/register.c @@ -3,16 +3,17 @@ * * Copyright Ericsson AB 1996-2013. All Rights Reserved. * - * The contents of this file are subject to the Erlang Public License, - * Version 1.1, (the "License"); you may not use this file except in - * compliance with the License. You should have received a copy of the - * Erlang Public License along with this software. If not, it can be - * retrieved online at http://www.erlang.org/. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Software distributed under the License is distributed on an "AS IS" - * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See - * the License for the specific language governing rights and limitations - * under the License. + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. * * %CopyrightEnd% */ @@ -124,7 +125,7 @@ static RegProc* reg_alloc(RegProc *tmpl) { RegProc* obj = (RegProc*) erts_alloc(ERTS_ALC_T_REG_PROC, sizeof(RegProc)); if (!obj) { - erl_exit(1, "Can't allocate %d bytes of memory\n", sizeof(RegProc)); + erts_exit(ERTS_ERROR_EXIT, "Can't allocate %d bytes of memory\n", sizeof(RegProc)); } obj->name = tmpl->name; obj->p = tmpl->p; @@ -269,7 +270,10 @@ erts_whereis_name_to_id(Process *c_p, Eterm name) #ifdef ERTS_SMP ErtsProcLocks c_p_locks = c_p ? ERTS_PROC_LOCK_MAIN : 0; - ERTS_SMP_CHK_HAVE_ONLY_MAIN_PROC_LOCK(c_p); +#ifdef ERTS_ENABLE_LOCK_CHECK + if (c_p) ERTS_SMP_CHK_HAVE_ONLY_MAIN_PROC_LOCK(c_p); +#endif + reg_safe_read_lock(c_p, &c_p_locks); if (c_p && !c_p_locks) erts_smp_proc_lock(c_p, ERTS_PROC_LOCK_MAIN); @@ -380,8 +384,6 @@ erts_whereis_name(Process *c_p, erts_smp_proc_unlock(rp->p, need_locks); *proc = NULL; } - if (*proc && (flags & ERTS_P2P_FLG_SMP_INC_REFC)) - erts_smp_proc_inc_refc(rp->p); } #else if (rp->p @@ -390,6 +392,8 @@ erts_whereis_name(Process *c_p, else *proc = NULL; #endif + if (*proc && (flags & ERTS_P2P_FLG_INC_REFC)) + erts_proc_inc_refc(*proc); } } diff --git a/erts/emulator/beam/register.h b/erts/emulator/beam/register.h index 7170463375..144536f34b 100644 --- a/erts/emulator/beam/register.h +++ b/erts/emulator/beam/register.h @@ -3,16 +3,17 @@ * * Copyright Ericsson AB 1996-2012. All Rights Reserved. * - * The contents of this file are subject to the Erlang Public License, - * Version 1.1, (the "License"); you may not use this file except in - * compliance with the License. You should have received a copy of the - * Erlang Public License along with this software. If not, it can be - * retrieved online at http://www.erlang.org/. - * - * Software distributed under the License is distributed on an "AS IS" - * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See - * the License for the specific language governing rights and limitations - * under the License. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. * * %CopyrightEnd% */ diff --git a/erts/emulator/beam/safe_hash.c b/erts/emulator/beam/safe_hash.c index 3326e5cc2a..3f039c8dfd 100644 --- a/erts/emulator/beam/safe_hash.c +++ b/erts/emulator/beam/safe_hash.c @@ -3,16 +3,17 @@ * * Copyright Ericsson AB 2008-2011. All Rights Reserved. * - * The contents of this file are subject to the Erlang Public License, - * Version 1.1, (the "License"); you may not use this file except in - * compliance with the License. You should have received a copy of the - * Erlang Public License along with this software. If not, it can be - * retrieved online at http://www.erlang.org/. - * - * Software distributed under the License is distributed on an "AS IS" - * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See - * the License for the specific language governing rights and limitations - * under the License. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. * * %CopyrightEnd% */ diff --git a/erts/emulator/beam/safe_hash.h b/erts/emulator/beam/safe_hash.h index c691126ef9..a11370813c 100644 --- a/erts/emulator/beam/safe_hash.h +++ b/erts/emulator/beam/safe_hash.h @@ -3,16 +3,17 @@ * * Copyright Ericsson AB 2008-2009. All Rights Reserved. * - * The contents of this file are subject to the Erlang Public License, - * Version 1.1, (the "License"); you may not use this file except in - * compliance with the License. You should have received a copy of the - * Erlang Public License along with this software. If not, it can be - * retrieved online at http://www.erlang.org/. - * - * Software distributed under the License is distributed on an "AS IS" - * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See - * the License for the specific language governing rights and limitations - * under the License. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. * * %CopyrightEnd% */ diff --git a/erts/emulator/beam/sys.h b/erts/emulator/beam/sys.h index 05bff430e3..6497a1e648 100644 --- a/erts/emulator/beam/sys.h +++ b/erts/emulator/beam/sys.h @@ -3,16 +3,17 @@ * * Copyright Ericsson AB 1996-2013. All Rights Reserved. * - * The contents of this file are subject to the Erlang Public License, - * Version 1.1, (the "License"); you may not use this file except in - * compliance with the License. You should have received a copy of the - * Erlang Public License along with this software. If not, it can be - * retrieved online at http://www.erlang.org/. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Software distributed under the License is distributed on an "AS IS" - * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See - * the License for the specific language governing rights and limitations - * under the License. + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. * * %CopyrightEnd% */ @@ -20,6 +21,41 @@ #ifndef __SYS_H__ #define __SYS_H__ +#ifdef ERTS_INLINE +# ifndef ERTS_CAN_INLINE +# define ERTS_CAN_INLINE 1 +# endif +#else +# if defined(__GNUC__) +# define ERTS_CAN_INLINE 1 +# define ERTS_INLINE __inline__ +# elif defined(__WIN32__) +# define ERTS_CAN_INLINE 1 +# define ERTS_INLINE __inline +# else +# define ERTS_CAN_INLINE 0 +# define ERTS_INLINE +# endif +#endif + +#if defined(DEBUG) || defined(ERTS_ENABLE_LOCK_CHECK) +# undef ERTS_CAN_INLINE +# define ERTS_CAN_INLINE 0 +# undef ERTS_INLINE +# define ERTS_INLINE +#endif + +#if ERTS_CAN_INLINE +#define ERTS_GLB_INLINE static ERTS_INLINE +#else +#define ERTS_GLB_INLINE +#endif + +#if ERTS_CAN_INLINE || defined(ERTS_DO_INCL_GLB_INLINE_FUNC_DEF) +# define ERTS_GLB_INLINE_INCL_FUNC_DEF 1 +#else +# define ERTS_GLB_INLINE_INCL_FUNC_DEF 0 +#endif #if defined(VALGRIND) && !defined(NO_FPE_SIGNALS) # define NO_FPE_SIGNALS @@ -38,6 +74,8 @@ #if defined (__WIN32__) # include "erl_win_sys.h" +#elif defined (__OSE__) +# include "erl_ose_sys.h" #else # include "erl_unix_sys.h" #ifndef UNIX @@ -64,26 +102,13 @@ */ #ifndef ERTS_SYS_FD_TYPE +#define ERTS_SYS_FD_INVALID ((ErtsSysFdType) -1) typedef int ErtsSysFdType; #else -typedef ERTS_SYS_FD_TYPE ErtsSysFdType; +#ifndef ERTS_SYS_FD_INVALID +# error missing ERTS_SYS_FD_INVALID #endif - -#ifdef ERTS_INLINE -# ifndef ERTS_CAN_INLINE -# define ERTS_CAN_INLINE 1 -# endif -#else -# if defined(__GNUC__) -# define ERTS_CAN_INLINE 1 -# define ERTS_INLINE __inline__ -# elif defined(__WIN32__) -# define ERTS_CAN_INLINE 1 -# define ERTS_INLINE __inline -# else -# define ERTS_CAN_INLINE 0 -# define ERTS_INLINE -# endif +typedef ERTS_SYS_FD_TYPE ErtsSysFdType; #endif #if !defined(__GNUC__) @@ -126,42 +151,70 @@ typedef ERTS_SYS_FD_TYPE ErtsSysFdType; # endif #endif -#if defined(DEBUG) || defined(ERTS_ENABLE_LOCK_CHECK) -# undef ERTS_CAN_INLINE -# define ERTS_CAN_INLINE 0 -# undef ERTS_INLINE -# define ERTS_INLINE +#define ERTS_MK_VSN_INT(Major, Minor, Build) \ + ((((Major) & 0x3ff) << 20) | (((Minor) & 0x3ff) << 10) | ((Build) & 0x3ff)) + +#ifndef ERTS_EXIT_AFTER_DUMP +# define ERTS_EXIT_AFTER_DUMP exit #endif -#if ERTS_CAN_INLINE -#define ERTS_GLB_INLINE static ERTS_INLINE +/* In VC++, noreturn is a declspec that has to be before the types, + * but in GNUC it is an att ribute to be placed between return type + * and function name, hence __decl_noreturn <types> __noreturn <function name> + * + * at some platforms (e.g. Android) __noreturn is defined at sys/cdef.h + */ +#if __GNUC__ +# define __decl_noreturn +# ifndef __noreturn +# define __noreturn __attribute__((noreturn)) +# endif #else -#define ERTS_GLB_INLINE +# if defined(__WIN32__) && defined(_MSC_VER) +# define __noreturn +# define __decl_noreturn __declspec(noreturn) +# else +# define __noreturn +# define __decl_noreturn +# endif #endif -#if ERTS_CAN_INLINE || defined(ERTS_DO_INCL_GLB_INLINE_FUNC_DEF) -# define ERTS_GLB_INLINE_INCL_FUNC_DEF 1 +#define ERTS_ASSERT(e) \ + ((void) ((e) ? 1 : (erl_assert_error(#e, __func__, __FILE__, __LINE__), 0))) + +__decl_noreturn void __noreturn erl_assert_error(const char* expr, const char *func, + const char* file, int line); + +#ifdef DEBUG +# define ASSERT(e) ERTS_ASSERT(e) #else -# define ERTS_GLB_INLINE_INCL_FUNC_DEF 0 +# define ASSERT(e) ((void) 1) #endif -#ifndef ERTS_EXIT_AFTER_DUMP -# define ERTS_EXIT_AFTER_DUMP exit +/* ERTS_UNDEF can be used to silence false warnings about + * "variable may be used uninitialized" while keeping the variable + * marked as undefined by valgrind. + */ +#ifdef VALGRIND +# define ERTS_UNDEF(V,I) +#else +# define ERTS_UNDEF(V,I) V = I #endif -#ifdef DEBUG -# define ASSERT(e) \ - if (e) { \ - ; \ - } else { \ - erl_assert_error(#e, __FILE__, __LINE__); \ - } -# define ASSERT_EXPR(e) \ - ((void) ((e) ? 1 : (erl_assert_error(#e, __FILE__, __LINE__), 0))) -void erl_assert_error(char* expr, char* file, int line); +/* + * Compile time assert + * (the actual compiler error msg can be a bit confusing) + */ +#if ERTS_AT_LEAST_GCC_VSN__(3,1,1) +# define ERTS_CT_ASSERT(e) \ + do { \ + enum { compile_time_assert__ = __builtin_choose_expr((e),0,(void)0) }; \ + } while(0) #else -# define ASSERT(e) -# define ASSERT_EXPR(e) ((void) 1) +# define ERTS_CT_ASSERT(e) \ + do { \ + enum { compile_time_assert__ = 1/(e) }; \ + } while (0) #endif /* @@ -197,23 +250,6 @@ void erl_assert_error(char* expr, char* file, int line); # define erts_align_attribute(SZ) #endif -/* In VC++, noreturn is a declspec that has to be before the types, - * but in GNUC it is an att ribute to be placed between return type - * and function name, hence __decl_noreturn <types> __noreturn <function name> - */ -#if __GNUC__ -# define __decl_noreturn -# define __noreturn __attribute__((noreturn)) -#else -# if defined(__WIN32__) && defined(_MSC_VER) -# define __noreturn -# define __decl_noreturn __declspec(noreturn) -# else -# define __noreturn -# define __decl_noreturn -# endif -#endif - /* ** Data types: ** @@ -271,6 +307,7 @@ void erl_assert_error(char* expr, char* file, int line); typedef unsigned int Eterm; typedef unsigned int Uint; typedef int Sint; +#define ERTS_UINT_MAX UINT_MAX #define ERTS_SIZEOF_ETERM SIZEOF_INT #define ErtsStrToSint strtol #else @@ -282,16 +319,22 @@ typedef unsigned long UWord; typedef long SWord; #define SWORD_CONSTANT(Const) Const##L #define UWORD_CONSTANT(Const) Const##UL +#define ERTS_UWORD_MAX ULONG_MAX +#define ERTS_SWORD_MAX LONG_MAX #elif SIZEOF_VOID_P == SIZEOF_INT typedef unsigned int UWord; typedef int SWord; #define SWORD_CONSTANT(Const) Const #define UWORD_CONSTANT(Const) Const##U +#define ERTS_UWORD_MAX UINT_MAX +#define ERTS_SWORD_MAX INT_MAX #elif SIZEOF_VOID_P == SIZEOF_LONG_LONG typedef unsigned long long UWord; typedef long long SWord; #define SWORD_CONSTANT(Const) Const##LL #define UWORD_CONSTANT(Const) Const##ULL +#define ERTS_UWORD_MAX ULLONG_MAX +#define ERTS_SWORD_MAX LLONG_MAX #else #error Found no appropriate type to use for 'Eterm', 'Uint' and 'Sint' #endif @@ -304,6 +347,8 @@ typedef unsigned long Uint; typedef long Sint; #define SWORD_CONSTANT(Const) Const##L #define UWORD_CONSTANT(Const) Const##UL +#define ERTS_UWORD_MAX ULONG_MAX +#define ERTS_SWORD_MAX LONG_MAX #define ERTS_SIZEOF_ETERM SIZEOF_LONG #define ErtsStrToSint strtol #elif SIZEOF_VOID_P == SIZEOF_INT @@ -312,6 +357,8 @@ typedef unsigned int Uint; typedef int Sint; #define SWORD_CONSTANT(Const) Const #define UWORD_CONSTANT(Const) Const##U +#define ERTS_UWORD_MAX UINT_MAX +#define ERTS_SWORD_MAX INT_MAX #define ERTS_SIZEOF_ETERM SIZEOF_INT #define ErtsStrToSint strtol #elif SIZEOF_VOID_P == SIZEOF_LONG_LONG @@ -320,6 +367,8 @@ typedef unsigned long long Uint; typedef long long Sint; #define SWORD_CONSTANT(Const) Const##LL #define UWORD_CONSTANT(Const) Const##ULL +#define ERTS_UWORD_MAX ULLONG_MAX +#define ERTS_SWORD_MAX LLONG_MAX #define ERTS_SIZEOF_ETERM SIZEOF_LONG_LONG #if defined(__WIN32__) #define ErtsStrToSint _strtoi64 @@ -332,23 +381,52 @@ typedef long long Sint; typedef Uint UWord; typedef Sint SWord; +#define ERTS_UINT_MAX ERTS_UWORD_MAX #endif /* HALFWORD_HEAP */ typedef UWord BeamInstr; #ifndef HAVE_INT64 -#if SIZEOF_LONG == 8 -#define HAVE_INT64 1 +# if SIZEOF_LONG == 8 +# define HAVE_INT64 1 typedef unsigned long Uint64; typedef long Sint64; -#elif SIZEOF_LONG_LONG == 8 -#define HAVE_INT64 1 +# ifdef ULONG_MAX +# define ERTS_UINT64_MAX ULONG_MAX +# endif +# ifdef LONG_MAX +# define ERTS_SINT64_MAX LONG_MAX +# endif +# ifdef LONG_MIN +# define ERTS_SINT64_MIN LONG_MIN +# endif +# elif SIZEOF_LONG_LONG == 8 +# define HAVE_INT64 1 typedef unsigned long long Uint64; typedef long long Sint64; -#else -#define HAVE_INT64 0 +# ifdef ULLONG_MAX +# define ERTS_UINT64_MAX ULLONG_MAX +# endif +# ifdef LLONG_MAX +# define ERTS_SINT64_MAX LLONG_MAX +# endif +# ifdef LLONG_MIN +# define ERTS_SINT64_MIN LLONG_MIN +# endif +# else +# error "No 64-bit integer type found" +# endif +#endif + +#ifndef ERTS_UINT64_MAX +# define ERTS_UINT64_MAX (~((Uint64) 0)) +#endif +#ifndef ERTS_SINT64_MAX +# define ERTS_SINT64_MAX ((Sint64) ((((Uint64) 1) << 63)-1)) #endif +#ifndef ERTS_SINT64_MIN +# define ERTS_SINT64_MIN (-1*(((Sint64) 1) << 63)) #endif #if SIZEOF_LONG == 4 @@ -484,7 +562,7 @@ extern volatile int erts_writing_erl_crash_dump; # define NO_ERF # define NO_ERFC /* This definition doesn't take NaN into account, but matherr() gets those */ -# define finite(x) (fabs(x) != HUGE_VAL) +# define isfinite(x) (fabs(x) != HUGE_VAL) # define USE_MATHERR # define HAVE_FINITE #endif @@ -533,15 +611,16 @@ static unsigned long zero_value = 0, one_value = 1; # endif /* !__WIN32__ */ #endif /* WANT_NONBLOCKING */ -__decl_noreturn void __noreturn erl_exit(int n, char*, ...); +__decl_noreturn void __noreturn erts_exit(int n, char*, ...); -/* Some special erl_exit() codes: */ -#define ERTS_INTR_EXIT INT_MIN /* called from signal handler */ -#define ERTS_ABORT_EXIT (INT_MIN + 1) /* no crash dump; only abort() */ -#define ERTS_DUMP_EXIT (INT_MIN + 2) /* crash dump; then exit() */ +/* Some special erts_exit() codes: */ +#define ERTS_INTR_EXIT -1 /* called from signal handler */ +#define ERTS_ABORT_EXIT -2 /* no crash dump; only abort() */ +#define ERTS_DUMP_EXIT -3 /* crash dump; then exit() */ +#define ERTS_ERROR_EXIT -4 /* crash dump; then abort() */ #define ERTS_INTERNAL_ERROR(What) \ - erl_exit(ERTS_ABORT_EXIT, "%s:%d:%s(): Internal error: %s\n", \ + erts_exit(ERTS_ABORT_EXIT, "%s:%d:%s(): Internal error: %s\n", \ __FILE__, __LINE__, __func__, What) Eterm erts_check_io_info(void *p); @@ -556,7 +635,6 @@ Uint erts_sys_misc_mem_sz(void); /* Io constants to erts_print and erts_putc */ #define ERTS_PRINT_STDERR (2) #define ERTS_PRINT_STDOUT (1) -#define ERTS_PRINT_INVALID (0) /* Don't want to use 0 since CBUF was 0 */ #define ERTS_PRINT_FILE (-1) #define ERTS_PRINT_SBUF (-2) #define ERTS_PRINT_SNBUF (-3) @@ -579,6 +657,7 @@ erts_dsprintf_buf_t *erts_create_logger_dsbuf(void); int erts_send_info_to_logger(Eterm, erts_dsprintf_buf_t *); int erts_send_warning_to_logger(Eterm, erts_dsprintf_buf_t *); int erts_send_error_to_logger(Eterm, erts_dsprintf_buf_t *); +int erts_send_error_term_to_logger(Eterm, erts_dsprintf_buf_t *, Eterm); int erts_send_info_to_logger_str(Eterm, char *); int erts_send_warning_to_logger_str(Eterm, char *); int erts_send_error_to_logger_str(Eterm, char *); @@ -625,14 +704,37 @@ extern char *erts_default_arg0; extern char os_type[]; -extern int sys_init_time(void); +typedef struct { + int have_os_monotonic_time; + int have_corrected_os_monotonic_time; + ErtsMonotonicTime os_monotonic_time_unit; + ErtsMonotonicTime sys_clock_resolution; + struct { + Uint64 resolution; + char *func; + char *clock_id; + int locked_use; + int extended; + } os_monotonic_time_info; + struct { + Uint64 resolution; + char *func; + char *clock_id; + int locked_use; + } os_system_time_info; +} ErtsSysInitTimeResult; + +#define ERTS_SYS_INIT_TIME_RESULT_INITER \ + {0, 0, (ErtsMonotonicTime) -1, (ErtsMonotonicTime) 1} + +extern void erts_init_sys_time_sup(void); +extern void sys_init_time(ErtsSysInitTimeResult *); +extern void erts_late_sys_init_time(void); extern void erts_deliver_time(void); extern void erts_time_remaining(SysTimeval *); -extern int erts_init_time_sup(void); extern void erts_sys_init_float(void); extern void erts_thread_init_float(void); extern void erts_thread_disable_fpe(void); - ERTS_GLB_INLINE int erts_block_fpe(void); ERTS_GLB_INLINE void erts_unblock_fpe(int); @@ -658,8 +760,7 @@ typedef struct { #define ERTS_SYS_DDLL_ERROR_INIT {NULL} extern void erts_sys_ddll_free_error(ErtsSysDdllError*); extern void erl_sys_ddll_init(void); /* to initialize mutexes etc */ -extern int erts_sys_ddll_open2(const char *path, void **handle, ErtsSysDdllError*); -#define erts_sys_ddll_open(P,H) erts_sys_ddll_open2(P,H,NULL) +extern int erts_sys_ddll_open(const char *path, void **handle, ErtsSysDdllError*); extern int erts_sys_ddll_open_noext(char *path, void **handle, ErtsSysDdllError*); extern int erts_sys_ddll_load_driver_init(void *handle, void **function); extern int erts_sys_ddll_load_nif_init(void *handle, void **function,ErtsSysDdllError*); @@ -676,11 +777,9 @@ extern char *erts_sys_ddll_error(int code); /* * System interfaces for startup. */ -#include "erl_time.h" - void erts_sys_schedule_interrupt(int set); #ifdef ERTS_SMP -void erts_sys_schedule_interrupt_timed(int set, erts_short_time_t msec); +void erts_sys_schedule_interrupt_timed(int, ErtsMonotonicTime); void erts_sys_main_thread(void); #endif @@ -719,6 +818,12 @@ int univ_to_local( int local_to_univ(Sint *year, Sint *month, Sint *day, Sint *hour, Sint *minute, Sint *second, int isdst); void get_now(Uint*, Uint*, Uint*); +struct ErtsSchedulerData_; +ErtsMonotonicTime erts_get_monotonic_time(struct ErtsSchedulerData_ *); +ErtsMonotonicTime erts_get_time_offset(void); +void +erts_make_timestamp_value(Uint* megasec, Uint* sec, Uint* microsec, + ErtsMonotonicTime mtime, ErtsMonotonicTime offset); void get_sys_now(Uint*, Uint*, Uint*); void set_break_quit(void (*)(void), void (*)(void)); @@ -728,6 +833,16 @@ void init_getenv_state(GETENV_STATE *); char * getenv_string(GETENV_STATE *); void fini_getenv_state(GETENV_STATE *); +#define HAVE_ERTS_CHECK_IO_DEBUG +typedef struct { + int no_used_fds; + int no_driver_select_structs; + int no_driver_event_structs; +} ErtsCheckIoDebugInfo; +int erts_check_io_debug(ErtsCheckIoDebugInfo *ip); + +int erts_sys_is_area_readable(char *start, char *stop); + /* xxxP */ #define SYS_DEFAULT_FLOAT_DECIMALS 20 void init_sys_float(void); @@ -749,11 +864,18 @@ int erts_sys_getenv(char *key, char *value, size_t *size); int erts_sys_getenv_raw(char *key, char *value, size_t *size); /* erts_sys_getenv__() is only allowed to be used in early init phase */ int erts_sys_getenv__(char *key, char *value, size_t *size); +/* erst_sys_unsetenv() returns 0 on success and a value != 0 on failure. */ +int erts_sys_unsetenv(char *key); /* Easier to use, but not as efficient, environment functions */ char *erts_read_env(char *key); void erts_free_read_env(void *value); +#if defined(ERTS_THR_HAVE_SIG_FUNCS) && !defined(ETHR_UNUSABLE_SIGUSRX) +extern void sys_thr_resume(erts_tid_t tid); +extern void sys_thr_suspend(erts_tid_t tid); +#endif + /* utils.c */ /* Options to sys_alloc_opt */ @@ -812,7 +934,7 @@ erts_refc_inc(erts_refc_t *refcp, erts_aint_t min_val) #ifdef ERTS_REFC_DEBUG erts_aint_t val = erts_smp_atomic_inc_read_nob((erts_smp_atomic_t *) refcp); if (val < min_val) - erl_exit(ERTS_ABORT_EXIT, + erts_exit(ERTS_ABORT_EXIT, "erts_refc_inc(): Bad refc found (refc=%ld < %ld)!\n", val, min_val); #else @@ -826,7 +948,7 @@ erts_refc_inctest(erts_refc_t *refcp, erts_aint_t min_val) erts_aint_t val = erts_smp_atomic_inc_read_nob((erts_smp_atomic_t *) refcp); #ifdef ERTS_REFC_DEBUG if (val < min_val) - erl_exit(ERTS_ABORT_EXIT, + erts_exit(ERTS_ABORT_EXIT, "erts_refc_inctest(): Bad refc found (refc=%ld < %ld)!\n", val, min_val); #endif @@ -839,7 +961,7 @@ erts_refc_dec(erts_refc_t *refcp, erts_aint_t min_val) #ifdef ERTS_REFC_DEBUG erts_aint_t val = erts_smp_atomic_dec_read_nob((erts_smp_atomic_t *) refcp); if (val < min_val) - erl_exit(ERTS_ABORT_EXIT, + erts_exit(ERTS_ABORT_EXIT, "erts_refc_dec(): Bad refc found (refc=%ld < %ld)!\n", val, min_val); #else @@ -853,7 +975,7 @@ erts_refc_dectest(erts_refc_t *refcp, erts_aint_t min_val) erts_aint_t val = erts_smp_atomic_dec_read_nob((erts_smp_atomic_t *) refcp); #ifdef ERTS_REFC_DEBUG if (val < min_val) - erl_exit(ERTS_ABORT_EXIT, + erts_exit(ERTS_ABORT_EXIT, "erts_refc_dectest(): Bad refc found (refc=%ld < %ld)!\n", val, min_val); #endif @@ -866,7 +988,7 @@ erts_refc_add(erts_refc_t *refcp, erts_aint_t diff, erts_aint_t min_val) #ifdef ERTS_REFC_DEBUG erts_aint_t val = erts_smp_atomic_add_read_nob((erts_smp_atomic_t *) refcp, diff); if (val < min_val) - erl_exit(ERTS_ABORT_EXIT, + erts_exit(ERTS_ABORT_EXIT, "erts_refc_add(%ld): Bad refc found (refc=%ld < %ld)!\n", diff, val, min_val); #else @@ -880,7 +1002,7 @@ erts_refc_read(erts_refc_t *refcp, erts_aint_t min_val) erts_aint_t val = erts_smp_atomic_read_nob((erts_smp_atomic_t *) refcp); #ifdef ERTS_REFC_DEBUG if (val < min_val) - erl_exit(ERTS_ABORT_EXIT, + erts_exit(ERTS_ABORT_EXIT, "erts_refc_read(): Bad refc found (refc=%ld < %ld)!\n", val, min_val); #endif @@ -1012,6 +1134,9 @@ void erl_bin_write(unsigned char *, int, int); #define ERTS_SMALL_ABS(Small) labs(Small) #endif +#ifndef ERTS_HAVE_ERTS_SYS_ALIGNED_ALLOC +# define ERTS_HAVE_ERTS_SYS_ALIGNED_ALLOC 0 +#endif #ifdef __WIN32__ void call_break_handler(void); diff --git a/erts/emulator/beam/time.c b/erts/emulator/beam/time.c index 2fd8e0cf00..988338ed81 100644 --- a/erts/emulator/beam/time.c +++ b/erts/emulator/beam/time.c @@ -3,16 +3,17 @@ * * Copyright Ericsson AB 1996-2013. All Rights Reserved. * - * The contents of this file are subject to the Erlang Public License, - * Version 1.1, (the "License"); you may not use this file except in - * compliance with the License. You should have received a copy of the - * Erlang Public License along with this software. If not, it can be - * retrieved online at http://www.erlang.org/. - * - * Software distributed under the License is distributed on an "AS IS" - * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See - * the License for the specific language governing rights and limitations - * under the License. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. * * %CopyrightEnd% */ @@ -76,6 +77,11 @@ #include "sys.h" #include "erl_vm.h" #include "global.h" +#define ERTS_WANT_TIMER_WHEEL_API +#include "erl_time.h" + +#define ERTS_MONOTONIC_DAY ERTS_SEC_TO_MONOTONIC(60*60*24) +#define ERTS_CLKTCKS_DAY ERTS_MONOTONIC_TO_CLKTCKS(ERTS_MONOTONIC_DAY) #ifdef ERTS_ENABLE_LOCK_CHECK #define ASSERT_NO_LOCKED_LOCKS erts_lc_check_exact(NULL, 0) @@ -83,26 +89,25 @@ #define ASSERT_NO_LOCKED_LOCKS #endif -static erts_smp_mtx_t tiw_lock; - - -/* BEGIN tiw_lock protected variables -** -** The individual timer cells in tiw are also protected by the same mutex. -*/ +#if 0 +# define ERTS_TW_DEBUG +#endif +#if defined(DEBUG) && !defined(ERTS_TW_DEBUG) +# define ERTS_TW_DEBUG +#endif -#ifdef SMALL_MEMORY -#define TIW_SIZE 8192 +#undef ERTS_TW_ASSERT +#if defined(ERTS_TW_DEBUG) +# define ERTS_TW_ASSERT(E) ERTS_ASSERT(E) #else -#define TIW_SIZE 65536 /* timing wheel size (should be a power of 2) */ +# define ERTS_TW_ASSERT(E) ((void) 1) #endif -static ErlTimer** tiw; /* the timing wheel, allocated in init_time() */ -static Uint tiw_pos; /* current position in wheel */ -static Uint tiw_nto; /* number of timeouts in wheel */ -static Uint tiw_min; -static ErlTimer *tiw_min_ptr; -/* END tiw_lock protected variables */ +#ifdef ERTS_TW_DEBUG +# define ERTS_TWHEEL_BUMP_YIELD_LIMIT 5 +#else +# define ERTS_TWHEEL_BUMP_YIELD_LIMIT 100 +#endif /* Actual interval time chosen by sys_init_time() */ @@ -114,392 +119,580 @@ static int tiw_itime; /* Constant after init */ # define TIW_ITIME tiw_itime #endif -erts_smp_atomic32_t do_time; /* set at clock interrupt */ -static ERTS_INLINE erts_short_time_t do_time_read(void) -{ - return erts_smp_atomic32_read_acqb(&do_time); -} - -static ERTS_INLINE erts_short_time_t do_time_update(void) +struct ErtsTimerWheel_ { + ErtsTWheelTimer *w[ERTS_TIW_SIZE]; + ErtsMonotonicTime pos; + Uint nto; + struct { + ErtsTWheelTimer *head; + ErtsTWheelTimer *tail; + Uint nto; + } at_once; + int yield_slot; + int yield_slots_left; + int yield_start_pos; + ErtsTWheelTimer sentinel; + int true_next_timeout_time; + ErtsMonotonicTime next_timeout_time; +}; + +static ERTS_INLINE ErtsMonotonicTime +find_next_timeout(ErtsSchedulerData *esdp, + ErtsTimerWheel *tiw, + int search_all, + ErtsMonotonicTime curr_time, /* When !search_all */ + ErtsMonotonicTime max_search_time) /* When !search_all */ { - return do_time_read(); -} - -static ERTS_INLINE void do_time_init(void) -{ - erts_smp_atomic32_init_nob(&do_time, 0); -} + int start_ix, tiw_pos_ix; + ErtsTWheelTimer *p; + int true_min_timeout = 0; + ErtsMonotonicTime min_timeout, min_timeout_pos, slot_timeout_pos; + + if (tiw->nto == 0) { /* no timeouts in wheel */ + if (!search_all) + min_timeout_pos = tiw->pos; + else { + curr_time = erts_get_monotonic_time(esdp); + tiw->pos = min_timeout_pos = ERTS_MONOTONIC_TO_CLKTCKS(curr_time); + } + min_timeout_pos += ERTS_MONOTONIC_TO_CLKTCKS(ERTS_MONOTONIC_DAY); + goto found_next; + } -/* get the time (in units of TIW_ITIME) to the next timeout, - or -1 if there are no timeouts */ + slot_timeout_pos = min_timeout_pos = tiw->pos; + if (search_all) + min_timeout_pos += ERTS_MONOTONIC_TO_CLKTCKS(ERTS_MONOTONIC_DAY); + else + min_timeout_pos = ERTS_MONOTONIC_TO_CLKTCKS(curr_time + max_search_time); -static erts_short_time_t next_time_internal(void) /* PRE: tiw_lock taken by caller */ -{ - int i, tm, nto; - Uint32 min; - ErlTimer* p; - erts_short_time_t dt; - - if (tiw_nto == 0) - return -1; /* no timeouts in wheel */ + start_ix = tiw_pos_ix = (int) (tiw->pos & (ERTS_TIW_SIZE-1)); - if (tiw_min_ptr) { - min = tiw_min; - dt = do_time_read(); - return ((min >= dt) ? (min - dt) : 0); - } - - /* start going through wheel to find next timeout */ - tm = nto = 0; - min = (Uint32) -1; /* max Uint32 */ - i = tiw_pos; do { - p = tiw[i]; - while (p != NULL) { - nto++; - if (p->count == 0) { - /* found next timeout */ - dt = do_time_read(); - /* p->count is zero */ - tiw_min_ptr = p; - tiw_min = tm; - return ((tm >= dt) ? (tm - dt) : 0); - } else { - /* keep shortest time in 'min' */ - if (tm + p->count*TIW_SIZE < min) { - min = tm + p->count*TIW_SIZE; - tiw_min_ptr = p; - tiw_min = min; + if (++slot_timeout_pos >= min_timeout_pos) + break; + + p = tiw->w[tiw_pos_ix]; + + if (p) { + ErtsTWheelTimer *end = p; + + do { + ErtsMonotonicTime timeout_pos; + timeout_pos = p->timeout_pos; + if (min_timeout_pos > timeout_pos) { + true_min_timeout = 1; + min_timeout_pos = timeout_pos; + if (min_timeout_pos <= slot_timeout_pos) + goto found_next; } - } - p = p->next; + p = p->next; + } while (p != end); } - /* when we have found all timeouts the shortest time will be in min */ - if (nto == tiw_nto) break; - tm++; - i = (i + 1) % TIW_SIZE; - } while (i != tiw_pos); - dt = do_time_read(); - if (min <= (Uint32) dt) - return 0; - if ((min - (Uint32) dt) > (Uint32) ERTS_SHORT_TIME_T_MAX) - return ERTS_SHORT_TIME_T_MAX; - return (erts_short_time_t) (min - (Uint32) dt); + + tiw_pos_ix++; + if (tiw_pos_ix == ERTS_TIW_SIZE) + tiw_pos_ix = 0; + } while (start_ix != tiw_pos_ix); + +found_next: + + min_timeout = ERTS_CLKTCKS_TO_MONOTONIC(min_timeout_pos); + tiw->next_timeout_time = min_timeout; + tiw->true_next_timeout_time = true_min_timeout; + + return min_timeout; } -static void remove_timer(ErlTimer *p) { - /* first */ - if (!p->prev) { - tiw[p->slot] = p->next; - if(p->next) - p->next->prev = NULL; - } else { - p->prev->next = p->next; +static ERTS_INLINE void +insert_timer_into_slot(ErtsTimerWheel *tiw, int slot, ErtsTWheelTimer *p) +{ + ERTS_TW_ASSERT(slot >= 0); + ERTS_TW_ASSERT(slot < ERTS_TIW_SIZE); + p->slot = slot; + if (!tiw->w[slot]) { + tiw->w[slot] = p; + p->next = p; + p->prev = p; + } + else { + ErtsTWheelTimer *next, *prev; + next = tiw->w[slot]; + prev = next->prev; + p->next = next; + p->prev = prev; + prev->next = p; + next->prev = p; } +} - /* last */ - if (!p->next) { +static ERTS_INLINE void +remove_timer(ErtsTimerWheel *tiw, ErtsTWheelTimer *p) +{ + int slot = p->slot; + ERTS_TW_ASSERT(slot != ERTS_TWHEEL_SLOT_INACTIVE); + + if (slot >= 0) { + /* + * Timer in wheel or in circular + * list of timers currently beeing + * triggered (referred by sentinel). + */ + ERTS_TW_ASSERT(slot < ERTS_TIW_SIZE); + + if (p->next == p) { + ERTS_TW_ASSERT(tiw->w[slot] == p); + tiw->w[slot] = NULL; + } + else { + if (tiw->w[slot] == p) + tiw->w[slot] = p->next; + p->prev->next = p->next; + p->next->prev = p->prev; + } + } + else { + /* Timer in "at once" queue... */ + ERTS_TW_ASSERT(slot == ERTS_TWHEEL_SLOT_AT_ONCE); if (p->prev) - p->prev->next = NULL; - } else { - p->next->prev = p->prev; + p->prev->next = p->next; + else { + ERTS_TW_ASSERT(tiw->at_once.head == p); + tiw->at_once.head = p->next; + } + if (p->next) + p->next->prev = p->prev; + else { + ERTS_TW_ASSERT(tiw->at_once.tail == p); + tiw->at_once.tail = p->prev; + } + ERTS_TW_ASSERT(tiw->at_once.nto > 0); + tiw->at_once.nto--; } - p->next = NULL; - p->prev = NULL; - /* Make sure cancel callback isn't called */ - p->active = 0; - tiw_nto--; + p->slot = ERTS_TWHEEL_SLOT_INACTIVE; + + tiw->nto--; } -/* Private export to erl_time_sup.c */ -erts_short_time_t erts_next_time(void) +ErtsMonotonicTime +erts_check_next_timeout_time(ErtsSchedulerData *esdp) { - erts_short_time_t ret; + ErtsTimerWheel *tiw = esdp->timer_wheel; + if (tiw->true_next_timeout_time) + return tiw->next_timeout_time; + return find_next_timeout(esdp, tiw, 1, 0, 0); +} + +#ifndef ERTS_TW_DEBUG +#define ERTS_DBG_CHK_SAFE_TO_SKIP_TO(TIW, TO) ((void) 0) +#else +#define ERTS_DBG_CHK_SAFE_TO_SKIP_TO(TIW, TO) debug_check_safe_to_skip_to((TIW), (TO)) +static void +debug_check_safe_to_skip_to(ErtsTimerWheel *tiw, ErtsMonotonicTime skip_to_pos) +{ + int slots, ix; + ErtsTWheelTimer *tmr; + ErtsMonotonicTime tmp; + + ix = (int) (tiw->pos & (ERTS_TIW_SIZE-1)); + tmp = skip_to_pos - tiw->pos; + ERTS_TW_ASSERT(tmp >= 0); + if (tmp < (ErtsMonotonicTime) ERTS_TIW_SIZE) + slots = (int) tmp; + else + slots = ERTS_TIW_SIZE; + + while (slots > 0) { + tmr = tiw->w[ix]; + if (tmr) { + ErtsTWheelTimer *end = tmr; + do { + ERTS_TW_ASSERT(tmr->timeout_pos > skip_to_pos); + tmr = tmr->next; + } while (tmr != end); + } + ix++; + if (ix == ERTS_TIW_SIZE) + ix = 0; + slots--; + } +} +#endif - erts_smp_mtx_lock(&tiw_lock); - (void)do_time_update(); - ret = next_time_internal(); - erts_smp_mtx_unlock(&tiw_lock); - return ret; +static ERTS_INLINE void +timeout_timer(ErtsTWheelTimer *p) +{ + ErlTimeoutProc timeout; + void *arg; + p->slot = ERTS_TWHEEL_SLOT_INACTIVE; + timeout = p->u.func.timeout; + arg = p->u.func.arg; + (*timeout)(arg); + ASSERT_NO_LOCKED_LOCKS; } -static ERTS_INLINE void bump_timer_internal(erts_short_time_t dt) /* PRE: tiw_lock is write-locked */ +void +erts_bump_timers(ErtsTimerWheel *tiw, ErtsMonotonicTime curr_time) { - Uint keep_pos; - Uint count; - ErlTimer *p, **prev, *timeout_head, **timeout_tail; - Uint dtime = (Uint) dt; - - /* no need to bump the position if there aren't any timeouts */ - if (tiw_nto == 0) { - erts_smp_mtx_unlock(&tiw_lock); - return; + int tiw_pos_ix, slots, yielded_slot_restarted, yield_count; + ErtsMonotonicTime bump_to, tmp_slots, old_pos; + + yield_count = ERTS_TWHEEL_BUMP_YIELD_LIMIT; + + /* + * In order to be fair we always continue with work + * where we left off when restarting after a yield. + */ + + if (tiw->yield_slot >= 0) { + yielded_slot_restarted = 1; + tiw_pos_ix = tiw->yield_slot; + slots = tiw->yield_slots_left; + bump_to = tiw->pos; + old_pos = tiw->yield_start_pos; + goto restart_yielded_slot; } - /* if do_time > TIW_SIZE we want to go around just once */ - count = (Uint)(dtime / TIW_SIZE) + 1; - keep_pos = (tiw_pos + dtime) % TIW_SIZE; - if (dtime > TIW_SIZE) dtime = TIW_SIZE; - - timeout_head = NULL; - timeout_tail = &timeout_head; - while (dtime > 0) { - /* this is to decrease the counters with the right amount */ - /* when dtime >= TIW_SIZE */ - if (tiw_pos == keep_pos) count--; - prev = &tiw[tiw_pos]; - while ((p = *prev) != NULL) { - ASSERT( p != p->next); - if (p->count < count) { /* we have a timeout */ - /* remove min time */ - if (tiw_min_ptr == p) { - tiw_min_ptr = NULL; - tiw_min = 0; + do { + + yielded_slot_restarted = 0; + + bump_to = ERTS_MONOTONIC_TO_CLKTCKS(curr_time); + + while (1) { + ErtsTWheelTimer *p; + + old_pos = tiw->pos; + + if (tiw->nto == 0) { + empty_wheel: + ERTS_DBG_CHK_SAFE_TO_SKIP_TO(tiw, bump_to); + tiw->true_next_timeout_time = 0; + tiw->next_timeout_time = curr_time + ERTS_MONOTONIC_DAY; + tiw->pos = bump_to; + tiw->yield_slot = ERTS_TWHEEL_SLOT_INACTIVE; + return; + } + + p = tiw->at_once.head; + while (p) { + if (--yield_count <= 0) { + ERTS_TW_ASSERT(tiw->nto > 0); + ERTS_TW_ASSERT(tiw->at_once.nto > 0); + tiw->yield_slot = ERTS_TWHEEL_SLOT_AT_ONCE; + tiw->true_next_timeout_time = 1; + tiw->next_timeout_time = ERTS_CLKTCKS_TO_MONOTONIC(old_pos); + return; } - /* Remove from list */ - remove_timer(p); - *timeout_tail = p; /* Insert in timeout queue */ - timeout_tail = &p->next; + ERTS_TW_ASSERT(tiw->nto > 0); + ERTS_TW_ASSERT(tiw->at_once.nto > 0); + tiw->nto--; + tiw->at_once.nto--; + tiw->at_once.head = p->next; + if (p->next) + p->next->prev = NULL; + else + tiw->at_once.tail = NULL; + + timeout_timer(p); + + p = tiw->at_once.head; } - else { - /* no timeout, just decrease counter */ - p->count -= count; - prev = &p->next; + + if (tiw->pos >= bump_to) + break; + + if (tiw->nto == 0) + goto empty_wheel; + + if (tiw->true_next_timeout_time) { + ErtsMonotonicTime skip_until_pos; + /* + * No need inspecting slots where we know no timeouts + * to trigger should reside. + */ + + skip_until_pos = ERTS_MONOTONIC_TO_CLKTCKS(tiw->next_timeout_time); + if (skip_until_pos > bump_to) + skip_until_pos = bump_to; + + skip_until_pos--; + + if (skip_until_pos > tiw->pos) { + ERTS_DBG_CHK_SAFE_TO_SKIP_TO(tiw, skip_until_pos); + + tiw->pos = skip_until_pos; + } + } + + tiw_pos_ix = (int) ((tiw->pos+1) & (ERTS_TIW_SIZE-1)); + tmp_slots = (bump_to - tiw->pos); + if (tmp_slots < (ErtsMonotonicTime) ERTS_TIW_SIZE) + slots = (int) tmp_slots; + else + slots = ERTS_TIW_SIZE; + + tiw->pos = bump_to; + + while (slots > 0) { + + p = tiw->w[tiw_pos_ix]; + if (p) { + if (p->next == p) { + ERTS_TW_ASSERT(tiw->sentinel.next == &tiw->sentinel); + ERTS_TW_ASSERT(tiw->sentinel.prev == &tiw->sentinel); + } + else { + tiw->sentinel.next = p->next; + tiw->sentinel.prev = p->prev; + tiw->sentinel.next->prev = &tiw->sentinel; + tiw->sentinel.prev->next = &tiw->sentinel; + } + tiw->w[tiw_pos_ix] = NULL; + + while (1) { + + if (p->timeout_pos > bump_to) { + /* Very unusual case... */ + ++yield_count; + insert_timer_into_slot(tiw, tiw_pos_ix, p); + } + else { + /* Normal case... */ + timeout_timer(p); + tiw->nto--; + } + + restart_yielded_slot: + + p = tiw->sentinel.next; + if (p == &tiw->sentinel) { + ERTS_TW_ASSERT(tiw->sentinel.prev == &tiw->sentinel); + break; + } + + if (--yield_count <= 0) { + tiw->true_next_timeout_time = 1; + tiw->next_timeout_time = ERTS_CLKTCKS_TO_MONOTONIC(old_pos); + tiw->yield_slot = tiw_pos_ix; + tiw->yield_slots_left = slots; + tiw->yield_start_pos = old_pos; + return; /* Yield! */ + } + + tiw->sentinel.next = p->next; + p->next->prev = &tiw->sentinel; + } + } + tiw_pos_ix++; + if (tiw_pos_ix == ERTS_TIW_SIZE) + tiw_pos_ix = 0; + slots--; } } - tiw_pos = (tiw_pos + 1) % TIW_SIZE; - dtime--; - } - tiw_pos = keep_pos; - if (tiw_min_ptr) - tiw_min -= dt; - - erts_smp_mtx_unlock(&tiw_lock); - - /* Call timedout timers callbacks */ - while (timeout_head) { - p = timeout_head; - timeout_head = p->next; - /* Here comes hairy use of the timer fields! - * They are reset without having the lock. - * It is assumed that no code but this will - * accesses any field until the ->timeout - * callback is called. - */ - p->next = NULL; - p->prev = NULL; - p->slot = 0; - (*p->timeout)(p->arg); - } -} -void erts_bump_timer(erts_short_time_t dt) /* dt is value from do_time */ -{ - erts_smp_mtx_lock(&tiw_lock); - bump_timer_internal(dt); + } while (yielded_slot_restarted); + + tiw->yield_slot = ERTS_TWHEEL_SLOT_INACTIVE; + tiw->true_next_timeout_time = 0; + tiw->next_timeout_time = curr_time + ERTS_MONOTONIC_DAY; + + /* Search at most two seconds ahead... */ + (void) find_next_timeout(NULL, tiw, 0, curr_time, ERTS_SEC_TO_MONOTONIC(2)); } Uint erts_timer_wheel_memory_size(void) { - return (Uint) TIW_SIZE * sizeof(ErlTimer*); + return sizeof(ErtsTimerWheel)*erts_no_schedulers; +} + +ErtsTimerWheel * +erts_create_timer_wheel(ErtsSchedulerData *esdp) +{ + ErtsMonotonicTime mtime; + int i; + ErtsTimerWheel *tiw; + tiw = erts_alloc_permanent_cache_aligned(ERTS_ALC_T_TIMER_WHEEL, + sizeof(ErtsTimerWheel)); + for(i = 0; i < ERTS_TIW_SIZE; i++) + tiw->w[i] = NULL; + + mtime = erts_get_monotonic_time(esdp); + tiw->pos = ERTS_MONOTONIC_TO_CLKTCKS(mtime); + tiw->nto = 0; + tiw->at_once.head = NULL; + tiw->at_once.tail = NULL; + tiw->at_once.nto = 0; + tiw->yield_slot = ERTS_TWHEEL_SLOT_INACTIVE; + tiw->true_next_timeout_time = 0; + tiw->next_timeout_time = mtime + ERTS_MONOTONIC_DAY; + tiw->sentinel.next = &tiw->sentinel; + tiw->sentinel.prev = &tiw->sentinel; + tiw->sentinel.u.func.timeout = NULL; + tiw->sentinel.u.func.cancel = NULL; + tiw->sentinel.u.func.arg = NULL; + return tiw; +} + +ErtsNextTimeoutRef +erts_get_next_timeout_reference(ErtsTimerWheel *tiw) +{ + return (ErtsNextTimeoutRef) &tiw->next_timeout_time; } + /* this routine links the time cells into a free list at the start and sets the time queue as empty */ void -erts_init_time(void) +erts_init_time(int time_correction, ErtsTimeWarpMode time_warp_mode) { - int i, itime; + int itime; /* system dependent init; must be done before do_time_init() if timer thread is enabled */ - itime = erts_init_time_sup(); + itime = erts_init_time_sup(time_correction, time_warp_mode); #ifdef TIW_ITIME_IS_CONSTANT if (itime != TIW_ITIME) { - erl_exit(ERTS_ABORT_EXIT, "timer resolution mismatch %d != %d", itime, TIW_ITIME); + erts_exit(ERTS_ABORT_EXIT, "timer resolution mismatch %d != %d", itime, TIW_ITIME); } #else tiw_itime = itime; #endif +} - erts_smp_mtx_init(&tiw_lock, "timer_wheel"); +void +erts_twheel_set_timer(ErtsTimerWheel *tiw, + ErtsTWheelTimer *p, ErlTimeoutProc timeout, + ErlCancelProc cancel, void *arg, + ErtsMonotonicTime timeout_pos) +{ + ErtsMonotonicTime timeout_time; - tiw = (ErlTimer**) erts_alloc(ERTS_ALC_T_TIMER_WHEEL, - TIW_SIZE * sizeof(ErlTimer*)); - for(i = 0; i < TIW_SIZE; i++) - tiw[i] = NULL; - do_time_init(); - tiw_pos = tiw_nto = 0; - tiw_min_ptr = NULL; - tiw_min = 0; -} + p->u.func.timeout = timeout; + p->u.func.cancel = cancel; + p->u.func.arg = arg; + ERTS_TW_ASSERT(p->slot == ERTS_TWHEEL_SLOT_INACTIVE); + if (timeout_pos <= tiw->pos) { + tiw->nto++; + tiw->at_once.nto++; + p->next = NULL; + p->prev = tiw->at_once.tail; + if (tiw->at_once.tail) { + ERTS_TW_ASSERT(tiw->at_once.head); + tiw->at_once.tail->next = p; + } + else { + ERTS_TW_ASSERT(!tiw->at_once.head); + tiw->at_once.head = p; + } + tiw->at_once.tail = p; + p->timeout_pos = tiw->pos; + p->slot = ERTS_TWHEEL_SLOT_AT_ONCE; + timeout_time = ERTS_CLKTCKS_TO_MONOTONIC(tiw->pos); + } + else { + int slot; + /* calculate slot */ + slot = (int) (timeout_pos & (ERTS_TIW_SIZE-1)); -/* -** Insert a process into the time queue, with a timeout 't' -*/ -static void -insert_timer(ErlTimer* p, Uint t) -{ - Uint tm; - Uint64 ticks; + insert_timer_into_slot(tiw, slot, p); - /* The current slot (tiw_pos) in timing wheel is the next slot to be - * be processed. Hence no extra time tick is needed. - * - * (x + y - 1)/y is precisely the "number of bins" formula. - */ - ticks = (t + (TIW_ITIME - 1)) / TIW_ITIME; + tiw->nto++; - /* - * Ticks must be a Uint64, or the addition may overflow here, - * resulting in an incorrect value for p->count below. - */ - ticks += do_time_update(); /* Add backlog of unprocessed time */ - - /* calculate slot */ - tm = (ticks + tiw_pos) % TIW_SIZE; - p->slot = (Uint) tm; - p->count = (Uint) (ticks / TIW_SIZE); - - /* insert at head of list at slot */ - p->next = tiw[tm]; - p->prev = NULL; - if (p->next != NULL) - p->next->prev = p; - tiw[tm] = p; - - - /* insert min time */ - if ((tiw_nto == 0) || ((tiw_min_ptr != NULL) && (ticks < tiw_min))) { - tiw_min = ticks; - tiw_min_ptr = p; - } - if ((tiw_min_ptr == p) && (ticks > tiw_min)) { - /* some other timer might be 'min' now */ - tiw_min = 0; - tiw_min_ptr = NULL; + timeout_time = ERTS_CLKTCKS_TO_MONOTONIC(timeout_pos); + p->timeout_pos = timeout_pos; } - tiw_nto++; + if (timeout_time < tiw->next_timeout_time) { + tiw->true_next_timeout_time = 1; + tiw->next_timeout_time = timeout_time; + } } void -erts_set_timer(ErlTimer* p, ErlTimeoutProc timeout, ErlCancelProc cancel, - void* arg, Uint t) +erts_twheel_cancel_timer(ErtsTimerWheel *tiw, ErtsTWheelTimer *p) { - - erts_deliver_time(); - erts_smp_mtx_lock(&tiw_lock); - if (p->active) { /* XXX assert ? */ - erts_smp_mtx_unlock(&tiw_lock); - return; + if (p->slot != ERTS_TWHEEL_SLOT_INACTIVE) { + ErlCancelProc cancel; + void *arg; + remove_timer(tiw, p); + cancel = p->u.func.cancel; + arg = p->u.func.arg; + if (cancel) + (*cancel)(arg); } - p->timeout = timeout; - p->cancel = cancel; - p->arg = arg; - p->active = 1; - insert_timer(p, t); - erts_smp_mtx_unlock(&tiw_lock); -#if defined(ERTS_SMP) - if (t <= (Uint) ERTS_SHORT_TIME_T_MAX) - erts_sys_schedule_interrupt_timed(1, (erts_short_time_t) t); -#endif } void -erts_cancel_timer(ErlTimer* p) +erts_twheel_debug_foreach(ErtsTimerWheel *tiw, + void (*tclbk)(void *), + void (*func)(void *, + ErtsMonotonicTime, + void *), + void *arg) { - erts_smp_mtx_lock(&tiw_lock); - if (!p->active) { /* allow repeated cancel (drivers) */ - erts_smp_mtx_unlock(&tiw_lock); - return; - } - - /* is it the 'min' timer, remove min */ - if (p == tiw_min_ptr) { - tiw_min_ptr = NULL; - tiw_min = 0; + ErtsTWheelTimer *tmr; + int ix; + + tmr = tiw->sentinel.next; + while (tmr != &tiw->sentinel) { + if (tmr->u.func.timeout == tclbk) + (*func)(arg, tmr->timeout_pos, tmr->u.func.arg); + tmr = tmr->next; } - remove_timer(p); - p->slot = p->count = 0; - - if (p->cancel != NULL) { - erts_smp_mtx_unlock(&tiw_lock); - (*p->cancel)(p->arg); - return; + for (tmr = tiw->at_once.head; tmr; tmr = tmr->next) { + if (tmr->u.func.timeout == tclbk) + (*func)(arg, tmr->timeout_pos, tmr->u.func.arg); } - erts_smp_mtx_unlock(&tiw_lock); -} - -/* - Returns the amount of time left in ms until the timer 'p' is triggered. - 0 is returned if 'p' isn't active. - 0 is returned also if the timer is overdue (i.e., would have triggered - immediately if it hadn't been cancelled). -*/ -Uint -erts_time_left(ErlTimer *p) -{ - Uint left; - erts_short_time_t dt; - erts_smp_mtx_lock(&tiw_lock); - - if (!p->active) { - erts_smp_mtx_unlock(&tiw_lock); - return 0; + for (ix = 0; ix < ERTS_TIW_SIZE; ix++) { + tmr = tiw->w[ix]; + if (tmr) { + do { + if (tmr->u.func.timeout == tclbk) + (*func)(arg, tmr->timeout_pos, tmr->u.func.arg); + tmr = tmr->next; + } while (tmr != tiw->w[ix]); + } } - - if (p->slot < tiw_pos) - left = (p->count + 1) * TIW_SIZE + p->slot - tiw_pos; - else - left = p->count * TIW_SIZE + p->slot - tiw_pos; - dt = do_time_read(); - if (left < dt) - left = 0; - else - left -= dt; - - erts_smp_mtx_unlock(&tiw_lock); - - return (Uint) left * TIW_ITIME; } -#ifdef DEBUG +#ifdef ERTS_TW_DEBUG void erts_p_slpq(void) { + erts_printf("Not yet implemented...\n"); +#if 0 + ErtsMonotonicTime current_time = erts_get_monotonic_time(NULL); int i; - ErlTimer* p; + ErtsTWheelTimer* p; - erts_smp_mtx_lock(&tiw_lock); - /* print the whole wheel, starting at the current position */ - erts_printf("\ntiw_pos = %d tiw_nto %d\n", tiw_pos, tiw_nto); - i = tiw_pos; - if (tiw[i] != NULL) { + erts_printf("\ncurrent time = %bps tiw_pos = %d tiw_nto %d\n", + current_time, tiw->pos, tiw->nto); + i = tiw->pos; + if (tiw->w[i] != NULL) { erts_printf("%d:\n", i); - for(p = tiw[i]; p != NULL; p = p->next) { - erts_printf(" (count %d, slot %d)\n", - p->count, p->slot); + for(p = tiw->w[i]; p != NULL; p = p->next) { + erts_printf(" (timeout time %bps, slot %d)\n", + ERTS_CLKTCKS_TO_MONOTONIC(p->timeout_pos), + p->slot); } } - for(i = (i+1)%TIW_SIZE; i != tiw_pos; i = (i+1)%TIW_SIZE) { - if (tiw[i] != NULL) { + for(i = ((i+1) & (ERTS_TIW_SIZE-1)); i != (tiw->pos & (ERTS_TIW_SIZE-1)); i = ((i+1) & (ERTS_TIW_SIZE-1))) { + if (tiw->w[i] != NULL) { erts_printf("%d:\n", i); - for(p = tiw[i]; p != NULL; p = p->next) { - erts_printf(" (count %d, slot %d)\n", - p->count, p->slot); + for(p = tiw->w[i]; p != NULL; p = p->next) { + erts_printf(" (timeout time %bps, slot %d)\n", + ERTS_CLKTCKS_TO_MONOTONIC(p->timeout_pos), p->slot); } } } - - erts_smp_mtx_unlock(&tiw_lock); +#endif } -#endif /* DEBUG */ +#endif /* ERTS_TW_DEBUG */ diff --git a/erts/emulator/beam/utils.c b/erts/emulator/beam/utils.c index a8f15fdc38..b9ce70e364 100644 --- a/erts/emulator/beam/utils.c +++ b/erts/emulator/beam/utils.c @@ -1,18 +1,19 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 1996-2013. All Rights Reserved. + * Copyright Ericsson AB 1996-2014. All Rights Reserved. * - * The contents of this file are subject to the Erlang Public License, - * Version 1.1, (the "License"); you may not use this file except in - * compliance with the License. You should have received a copy of the - * Erlang Public License along with this software. If not, it can be - * retrieved online at http://www.erlang.org/. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Software distributed under the License is distributed on an "AS IS" - * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See - * the License for the specific language governing rights and limitations - * under the License. + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. * * %CopyrightEnd% */ @@ -31,6 +32,7 @@ #include "bif.h" #include "erl_binary.h" #include "erl_bits.h" +#include "erl_map.h" #include "packet_parser.h" #include "erl_gc.h" #define ERTS_WANT_DB_INTERNAL__ @@ -47,6 +49,13 @@ #include "erl_sched_spec_pre_alloc.h" #include "beam_bp.h" #include "erl_ptab.h" +#include "erl_check_io.h" +#include "erl_bif_unique.h" +#define ERTS_WANT_TIMER_WHEEL_API +#include "erl_time.h" +#ifdef HIPE +# include "hipe_mode_switch.h" +#endif #undef M_TRIM_THRESHOLD #undef M_TOP_PAD @@ -185,39 +194,79 @@ erts_set_hole_marker(Eterm* ptr, Uint sz) * Helper function for the ESTACK macros defined in global.h. */ void -erl_grow_stack(Eterm** start, Eterm** sp, Eterm** end) +erl_grow_estack(ErtsEStack* s, Uint need) { - Uint old_size = (*end - *start); - Uint new_size = old_size * 2; - Uint sp_offs = *sp - *start; - if (new_size > 2 * DEF_ESTACK_SIZE) { - *start = erts_realloc(ERTS_ALC_T_ESTACK, (void *) *start, new_size*sizeof(Eterm)); + Uint old_size = (s->end - s->start); + Uint new_size; + Uint sp_offs = s->sp - s->start; + + if (need < old_size) + new_size = 2*old_size; + else + new_size = ((need / old_size) + 2) * old_size; + + if (s->start != s->edefault) { + s->start = erts_realloc(s->alloc_type, s->start, + new_size*sizeof(Eterm)); } else { - Eterm* new_ptr = erts_alloc(ERTS_ALC_T_ESTACK, new_size*sizeof(Eterm)); - sys_memcpy(new_ptr, *start, old_size*sizeof(Eterm)); - *start = new_ptr; + Eterm* new_ptr = erts_alloc(s->alloc_type, new_size*sizeof(Eterm)); + sys_memcpy(new_ptr, s->start, old_size*sizeof(Eterm)); + s->start = new_ptr; } - *end = *start + new_size; - *sp = *start + sp_offs; + s->end = s->start + new_size; + s->sp = s->start + sp_offs; } /* - * Helper function for the ESTACK macros defined in global.h. + * Helper function for the WSTACK macros defined in global.h. + */ +void +erl_grow_wstack(ErtsWStack* s, Uint need) +{ + Uint old_size = (s->wend - s->wstart); + Uint new_size; + Uint sp_offs = s->wsp - s->wstart; + + if (need < old_size) + new_size = 2 * old_size; + else + new_size = ((need / old_size) + 2) * old_size; + + if (s->wstart != s->wdefault) { + s->wstart = erts_realloc(s->alloc_type, s->wstart, + new_size*sizeof(UWord)); + } else { + UWord* new_ptr = erts_alloc(s->alloc_type, new_size*sizeof(UWord)); + sys_memcpy(new_ptr, s->wstart, old_size*sizeof(UWord)); + s->wstart = new_ptr; + } + s->wend = s->wstart + new_size; + s->wsp = s->wstart + sp_offs; +} + +/* + * Helper function for the PSTACK macros defined in global.h. */ void -erl_grow_wstack(UWord** start, UWord** sp, UWord** end) +erl_grow_pstack(ErtsPStack* s, void* default_pstack, unsigned need_bytes) { - Uint old_size = (*end - *start); - Uint new_size = old_size * 2; - Uint sp_offs = *sp - *start; - if (new_size > 2 * DEF_ESTACK_SIZE) { - *start = erts_realloc(ERTS_ALC_T_ESTACK, (void *) *start, new_size*sizeof(UWord)); + Uint old_size = s->pend - s->pstart; + Uint new_size; + Uint sp_offs = s->psp - s->pstart; + + if (need_bytes < old_size) + new_size = 2 * old_size; + else + new_size = ((need_bytes / old_size) + 2) * old_size; + + if (s->pstart != default_pstack) { + s->pstart = erts_realloc(s->alloc_type, s->pstart, new_size); } else { - UWord* new_ptr = erts_alloc(ERTS_ALC_T_ESTACK, new_size*sizeof(UWord)); - sys_memcpy(new_ptr, *start, old_size*sizeof(UWord)); - *start = new_ptr; + byte* new_ptr = erts_alloc(s->alloc_type, new_size); + sys_memcpy(new_ptr, s->pstart, old_size); + s->pstart = new_ptr; } - *end = *start + new_size; - *sp = *start + sp_offs; + s->pend = s->pstart + new_size; + s->psp = s->pstart + sp_offs; } /* CTYPE macros */ @@ -255,7 +304,7 @@ erl_grow_wstack(UWord** start, UWord** sp, UWord** end) * Returns -1 if not a proper list (i.e. not terminated with NIL) */ int -list_length(Eterm list) +erts_list_length(Eterm list) { int i = 0; @@ -307,6 +356,17 @@ int erts_fit_in_bits_int32(Sint32 value) return fit_in_bits((Sint64) (Uint32) value, 4); } +int erts_fit_in_bits_uint(Uint value) +{ +#if ERTS_SIZEOF_ETERM == 4 + return fit_in_bits((Sint64) (Uint32) value, 4); +#elif ERTS_SIZEOF_ETERM == 8 + return fit_in_bits(value, 5); +#else +# error "No way, Jose" +#endif +} + int erts_print(int to, void *arg, char *format, ...) { @@ -339,9 +399,6 @@ erts_print(int to, void *arg, char *format, ...) case ERTS_PRINT_DSBUF: res = erts_vdsprintf((erts_dsprintf_buf_t *) arg, format, arg_list); break; - case ERTS_PRINT_INVALID: - res = -EINVAL; - break; default: res = erts_vfdprintf((int) to, format, arg_list); break; @@ -576,8 +633,8 @@ erts_bld_2tup_list(Uint **hpp, Uint *szp, } Eterm -erts_bld_atom_uint_2tup_list(Uint **hpp, Uint *szp, - Sint length, Eterm atoms[], Uint uints[]) +erts_bld_atom_uword_2tup_list(Uint **hpp, Uint *szp, + Sint length, Eterm atoms[], UWord uints[]) { Sint i; Eterm res = THE_NON_VALUE; @@ -703,7 +760,7 @@ erts_bld_atom_2uint_3tup_list(Uint **hpp, Uint *szp, Sint length, ** If N < 0, Y = FUNNY_NUMBER4 else Y = FUNNY_NUMBER3. ** The hash value is Y*h(J) mod 2^32 where h(J) is calculated like ** h(0) = <initial hash> -** h(i) = h(i-i)*X + B(i-1) +** h(i) = h(i-1)*X + B(i-1) ** The above should hold regardless of internal representation. ** Pids are hashed like small numbers but with differrent constants, as are ** ports. @@ -732,6 +789,8 @@ erts_bld_atom_2uint_3tup_list(Uint **hpp, Uint *szp, Sint length, #define FUNNY_NUMBER10 268440479 #define FUNNY_NUMBER11 268440577 #define FUNNY_NUMBER12 268440581 +#define FUNNY_NUMBER13 268440593 +#define FUNNY_NUMBER14 268440611 static Uint32 hash_binary_bytes(Eterm bin, Uint sz, Uint32 hash) @@ -782,11 +841,10 @@ Uint32 make_hash(Eterm term_arg) Eterm hash = 0; unsigned op; - /* Must not collide with the real tag_val_def's: */ -#define MAKE_HASH_TUPLE_OP 0x10 -#define MAKE_HASH_FUN_OP 0x11 -#define MAKE_HASH_CDR_PRE_OP 0x12 -#define MAKE_HASH_CDR_POST_OP 0x13 +#define MAKE_HASH_TUPLE_OP (FIRST_VACANT_TAG_DEF) +#define MAKE_HASH_TERM_ARRAY_OP (FIRST_VACANT_TAG_DEF+1) +#define MAKE_HASH_CDR_PRE_OP (FIRST_VACANT_TAG_DEF+2) +#define MAKE_HASH_CDR_POST_OP (FIRST_VACANT_TAG_DEF+3) /* ** Convenience macro for calculating a bytewise hash on an unsigned 32 bit @@ -875,7 +933,7 @@ tail_recur: hash = hash*FUNNY_NUMBER2 + funp->fe->old_uniq; if (num_free > 0) { if (num_free > 1) { - WSTACK_PUSH3(stack, (UWord) &funp->env[1], (num_free-1), MAKE_HASH_FUN_OP); + WSTACK_PUSH3(stack, (UWord) &funp->env[1], (num_free-1), MAKE_HASH_TERM_ARRAY_OP); } term = funp->env[0]; goto tail_recur; @@ -896,12 +954,15 @@ tail_recur: UINT32_HASH_RET(external_ref_numbers(term)[0],FUNNY_NUMBER9,FUNNY_NUMBER10); case FLOAT_DEF: { - FloatDef ff; - GET_DOUBLE(term, ff); - hash = hash*FUNNY_NUMBER6 + (ff.fw[0] ^ ff.fw[1]); - break; + FloatDef ff; + GET_DOUBLE(term, ff); + if (ff.fd == 0.0f) { + /* ensure positive 0.0 */ + ff.fd = erts_get_positive_zero_float(); + } + hash = hash*FUNNY_NUMBER6 + (ff.fw[0] ^ ff.fw[1]); + break; } - case MAKE_HASH_CDR_PRE_OP: term = (Eterm) WSTACK_POP(stack); if (is_not_list(term)) { @@ -965,6 +1026,9 @@ tail_recur: hash *= is_neg ? FUNNY_NUMBER4 : FUNNY_NUMBER3; break; } + case MAP_DEF: + hash = hash*FUNNY_NUMBER13 + FUNNY_NUMBER14 + make_hash2(term); + break; case TUPLE_DEF: { Eterm* ptr = tuple_val(term); @@ -974,7 +1038,7 @@ tail_recur: op = MAKE_HASH_TUPLE_OP; }/*fall through*/ case MAKE_HASH_TUPLE_OP: - case MAKE_HASH_FUN_OP: + case MAKE_HASH_TERM_ARRAY_OP: { Uint i = (Uint) WSTACK_POP(stack); Eterm* ptr = (Eterm*) WSTACK_POP(stack); @@ -991,7 +1055,7 @@ tail_recur: } default: - erl_exit(1, "Invalid tag in make_hash(0x%X,0x%X)\n", term, op); + erts_exit(ERTS_ERROR_EXIT, "Invalid tag in make_hash(0x%X,0x%X)\n", term, op); return 0; } if (WSTACK_ISEMPTY(stack)) break; @@ -1068,9 +1132,12 @@ Uint32 make_hash2(Eterm term) { Uint32 hash; + Uint32 hash_xor_pairs; DeclareTmpHeapNoproc(tmp_big,2); -/* (HCONST * {2, ..., 14}) mod 2^32 */ + ERTS_UNDEF(hash_xor_pairs, 0); + +/* (HCONST * {2, ..., 22}) mod 2^32 */ #define HCONST_2 0x3c6ef372UL #define HCONST_3 0xdaa66d2bUL #define HCONST_4 0x78dde6e4UL @@ -1085,6 +1152,17 @@ make_hash2(Eterm term) #define HCONST_13 0x08d12e65UL #define HCONST_14 0xa708a81eUL #define HCONST_15 0x454021d7UL +#define HCONST_16 0xe3779b90UL +#define HCONST_17 0x81af1549UL +#define HCONST_18 0x1fe68f02UL +#define HCONST_19 0xbe1e08bbUL +#define HCONST_20 0x5c558274UL +#define HCONST_21 0xfa8cfc2dUL +#define HCONST_22 0x98c475e6UL + +#define HASH_MAP_TAIL (_make_header(1,_TAG_HEADER_REF)) +#define HASH_MAP_PAIR (_make_header(2,_TAG_HEADER_REF)) +#define HASH_CDR (_make_header(3,_TAG_HEADER_REF)) #define UINT32_HASH_2(Expr1, Expr2, AConst) \ do { \ @@ -1107,6 +1185,13 @@ make_hash2(Eterm term) } while(0) #define IS_SSMALL28(x) (((Uint) (((x) >> (28-1)) + 1)) < 2) + +#ifdef ARCH_64 +# define POINTER_HASH(Ptr, AConst) UINT32_HASH_2((Uint32)(UWord)(Ptr), (((UWord)(Ptr)) >> 32), AConst) +#else +# define POINTER_HASH(Ptr, AConst) UINT32_HASH(Ptr, AConst) +#endif + /* Optimization. Simple cases before declaration of estack. */ if (primary_tag(term) == TAG_PRIMARY_IMMED1) { switch (term & _TAG_IMMED1_MASK) { @@ -1161,9 +1246,9 @@ make_hash2(Eterm term) if (c > 0) UINT32_HASH(sh, HCONST_4); if (is_list(term)) { - term = *ptr; - tmp = *++ptr; - ESTACK_PUSH(s, tmp); + tmp = CDR(ptr); + ESTACK_PUSH(s, tmp); + term = CAR(ptr); } } break; @@ -1180,17 +1265,90 @@ make_hash2(Eterm term) UINT32_HASH(arity, HCONST_9); if (arity == 0) /* Empty tuple */ goto hash2_common; - for (i = arity; i >= 2; i--) { - tmp = elem[i]; - ESTACK_PUSH(s, tmp); + for (i = arity; ; i--) { + term = elem[i]; + if (i == 1) + break; + ESTACK_PUSH(s, term); } - term = elem[1]; } break; + case MAP_SUBTAG: + { + Eterm* ptr = boxed_val(term) + 1; + Uint size; + int i; + switch (hdr & _HEADER_MAP_SUBTAG_MASK) { + case HAMT_SUBTAG_HEAD_FLATMAP: + { + flatmap_t *mp = (flatmap_t *)flatmap_val(term); + Eterm *ks = flatmap_get_keys(mp); + Eterm *vs = flatmap_get_values(mp); + size = flatmap_get_size(mp); + UINT32_HASH(size, HCONST_16); + if (size == 0) + goto hash2_common; + + /* We want a portable hash function that is *independent* of + * the order in which keys and values are encountered. + * We therefore calculate context independent hashes for all . + * key-value pairs and then xor them together. + */ + ESTACK_PUSH(s, hash_xor_pairs); + ESTACK_PUSH(s, hash); + ESTACK_PUSH(s, HASH_MAP_TAIL); + hash = 0; + hash_xor_pairs = 0; + for (i = size - 1; i >= 0; i--) { + ESTACK_PUSH(s, HASH_MAP_PAIR); + ESTACK_PUSH(s, vs[i]); + ESTACK_PUSH(s, ks[i]); + } + goto hash2_common; + } + + case HAMT_SUBTAG_HEAD_ARRAY: + case HAMT_SUBTAG_HEAD_BITMAP: + size = *ptr++; + UINT32_HASH(size, HCONST_16); + if (size == 0) + goto hash2_common; + ESTACK_PUSH(s, hash_xor_pairs); + ESTACK_PUSH(s, hash); + ESTACK_PUSH(s, HASH_MAP_TAIL); + hash = 0; + hash_xor_pairs = 0; + } + switch (hdr & _HEADER_MAP_SUBTAG_MASK) { + case HAMT_SUBTAG_HEAD_ARRAY: + i = 16; + break; + case HAMT_SUBTAG_HEAD_BITMAP: + case HAMT_SUBTAG_NODE_BITMAP: + i = hashmap_bitcount(MAP_HEADER_VAL(hdr)); + break; + default: + erts_exit(ERTS_ERROR_EXIT, "bad header"); + } + while (i) { + if (is_list(*ptr)) { + Eterm* cons = list_val(*ptr); + ESTACK_PUSH(s, HASH_MAP_PAIR); + ESTACK_PUSH(s, CDR(cons)); + ESTACK_PUSH(s, CAR(cons)); + } + else { + ASSERT(is_boxed(*ptr)); + ESTACK_PUSH(s, *ptr); + } + i--; ptr++; + } + goto hash2_common; + } + break; case EXPORT_SUBTAG: { Export* ep = *((Export **) (export_val(term) + 1)); - UINT32_HASH_2 (ep->code[2], atom_tab(atom_val(ep->code[0]))->slot.bucket.hvalue, @@ -1205,7 +1363,6 @@ make_hash2(Eterm term) { ErlFunThing* funp = (ErlFunThing *) fun_val(term); Uint num_free = funp->num_free; - UINT32_HASH_2 (num_free, atom_tab(atom_val(funp->fe->module))->slot.bucket.hvalue, @@ -1286,7 +1443,8 @@ make_hash2(Eterm term) do { Uint t; Uint32 x, y; - t = i < n ? BIG_DIGIT(ptr, i++) : 0; + ASSERT(i < n); + t = BIG_DIGIT(ptr, i++); x = t & 0xffffffff; y = t >> 32; UINT32_HASH_2(x, y, con); @@ -1319,7 +1477,11 @@ make_hash2(Eterm term) { FloatDef ff; GET_DOUBLE(term, ff); -#if defined(WORDS_BIGENDIAN) + if (ff.fd == 0.0f) { + /* ensure positive 0.0 */ + ff.fd = erts_get_positive_zero_float(); + } +#if defined(WORDS_BIGENDIAN) || defined(DOUBLE_MIDDLE_ENDIAN) UINT32_HASH_2(ff.fw[0], ff.fw[1], HCONST_12); #else UINT32_HASH_2(ff.fw[1], ff.fw[0], HCONST_12); @@ -1329,7 +1491,7 @@ make_hash2(Eterm term) break; default: - erl_exit(1, "Invalid tag in make_hash2(0x%X)\n", term); + erts_exit(ERTS_ERROR_EXIT, "Invalid tag in make_hash2(0x%X)\n", term); } } break; @@ -1360,7 +1522,7 @@ make_hash2(Eterm term) UINT32_HASH(NIL_DEF, HCONST_2); goto hash2_common; default: - erl_exit(1, "Invalid tag in make_hash2(0x%X)\n", term); + erts_exit(ERTS_ERROR_EXIT, "Invalid tag in make_hash2(0x%X)\n", term); } case _TAG_IMMED1_SMALL: { @@ -1376,17 +1538,427 @@ make_hash2(Eterm term) } break; default: - erl_exit(1, "Invalid tag in make_hash2(0x%X)\n", term); + erts_exit(ERTS_ERROR_EXIT, "Invalid tag in make_hash2(0x%X)\n", term); hash2_common: + + /* Uint32 hash always has the hash value of the previous term, + * compounded or otherwise. + */ + if (ESTACK_ISEMPTY(s)) { DESTROY_ESTACK(s); UnUseTmpHeapNoproc(2); return hash; } + + term = ESTACK_POP(s); + + switch (term) { + case HASH_MAP_TAIL: { + hash = (Uint32) ESTACK_POP(s); + UINT32_HASH(hash_xor_pairs, HCONST_19); + hash_xor_pairs = (Uint32) ESTACK_POP(s); + goto hash2_common; + } + case HASH_MAP_PAIR: + hash_xor_pairs ^= hash; + hash = 0; + goto hash2_common; + default: + break; + } + } + } + } +} + +/* Term hash function for internal use. + * + * Limitation #1: Is not "portable" in any way between different VM instances. + * + * Limitation #2: The hash value is only valid as long as the term exists + * somewhere in the VM. Why? Because external pids, ports and refs are hashed + * by mixing the node *pointer* value. If a node disappears and later reappears + * with a new ErlNode struct, externals from that node will hash different than + * before. + * + * One IMPORTANT property must hold (for hamt). + * EVERY BIT of the term that is significant for equality (see EQ) + * MUST BE USED AS INPUT FOR THE HASH. Two different terms must always have a + * chance of hashing different when salted: hash([Salt|A]) vs hash([Salt|B]). + * + * This is why we can not use cached hash values for atoms for example. + * + */ + +#define CONST_HASH(AConst) \ +do { /* Lightweight mixing of constant (type info) */ \ + hash ^= AConst; \ + hash = (hash << 17) ^ (hash >> (32-17)); \ +} while (0) + +Uint32 +make_internal_hash(Eterm term) +{ + Uint32 hash; + Uint32 hash_xor_pairs; + + ERTS_UNDEF(hash_xor_pairs, 0); + + /* Optimization. Simple cases before declaration of estack. */ + if (primary_tag(term) == TAG_PRIMARY_IMMED1) { + hash = 0; + #if ERTS_SIZEOF_ETERM == 8 + UINT32_HASH_2((Uint32)term, (Uint32)(term >> 32), HCONST); + #elif ERTS_SIZEOF_ETERM == 4 + UINT32_HASH(term, HCONST); + #else + # error "No you don't" + #endif + return hash; + } + { + Eterm tmp; + DECLARE_ESTACK(s); + + hash = 0; + for (;;) { + switch (primary_tag(term)) { + case TAG_PRIMARY_LIST: + { + int c = 0; + Uint32 sh = 0; + Eterm* ptr = list_val(term); + while (is_byte(*ptr)) { + /* Optimization for strings. */ + sh = (sh << 8) + unsigned_val(*ptr); + if (c == 3) { + UINT32_HASH(sh, HCONST_4); + c = sh = 0; + } else { + c++; + } + term = CDR(ptr); + if (is_not_list(term)) + break; + ptr = list_val(term); + } + if (c > 0) + UINT32_HASH_2(sh, (Uint32)c, HCONST_22); + + if (is_list(term)) { + tmp = CDR(ptr); + CONST_HASH(HCONST_17); /* Hash CAR in cons cell */ + ESTACK_PUSH(s, tmp); + if (is_not_list(tmp)) { + ESTACK_PUSH(s, HASH_CDR); + } + term = CAR(ptr); + } + } + break; + case TAG_PRIMARY_BOXED: + { + Eterm hdr = *boxed_val(term); + ASSERT(is_header(hdr)); + switch (hdr & _TAG_HEADER_MASK) { + case ARITYVAL_SUBTAG: + { + int i; + int arity = header_arity(hdr); + Eterm* elem = tuple_val(term); + UINT32_HASH(arity, HCONST_9); + if (arity == 0) /* Empty tuple */ + goto pop_next; + for (i = arity; ; i--) { + term = elem[i]; + if (i == 1) + break; + ESTACK_PUSH(s, term); + } + } + break; + + case MAP_SUBTAG: + { + Eterm* ptr = boxed_val(term) + 1; + Uint size; + int i; + switch (hdr & _HEADER_MAP_SUBTAG_MASK) { + case HAMT_SUBTAG_HEAD_FLATMAP: + { + flatmap_t *mp = (flatmap_t *)flatmap_val(term); + Eterm *ks = flatmap_get_keys(mp); + Eterm *vs = flatmap_get_values(mp); + size = flatmap_get_size(mp); + UINT32_HASH(size, HCONST_16); + if (size == 0) + goto pop_next; + + /* We want a hash function that is *independent* of + * the order in which keys and values are encountered. + * We therefore calculate context independent hashes for all . + * key-value pairs and then xor them together. + */ + ESTACK_PUSH(s, hash_xor_pairs); + ESTACK_PUSH(s, hash); + ESTACK_PUSH(s, HASH_MAP_TAIL); + hash = 0; + hash_xor_pairs = 0; + for (i = size - 1; i >= 0; i--) { + ESTACK_PUSH(s, HASH_MAP_PAIR); + ESTACK_PUSH(s, vs[i]); + ESTACK_PUSH(s, ks[i]); + } + goto pop_next; + } + case HAMT_SUBTAG_HEAD_ARRAY: + case HAMT_SUBTAG_HEAD_BITMAP: + size = *ptr++; + UINT32_HASH(size, HCONST_16); + if (size == 0) + goto pop_next; + ESTACK_PUSH(s, hash_xor_pairs); + ESTACK_PUSH(s, hash); + ESTACK_PUSH(s, HASH_MAP_TAIL); + hash = 0; + hash_xor_pairs = 0; + } + switch (hdr & _HEADER_MAP_SUBTAG_MASK) { + case HAMT_SUBTAG_HEAD_ARRAY: + i = 16; + break; + case HAMT_SUBTAG_HEAD_BITMAP: + case HAMT_SUBTAG_NODE_BITMAP: + i = hashmap_bitcount(MAP_HEADER_VAL(hdr)); + break; + default: + erts_exit(ERTS_ERROR_EXIT, "bad header"); + } + while (i) { + if (is_list(*ptr)) { + Eterm* cons = list_val(*ptr); + ESTACK_PUSH(s, HASH_MAP_PAIR); + ESTACK_PUSH(s, CDR(cons)); + ESTACK_PUSH(s, CAR(cons)); + } + else { + ASSERT(is_boxed(*ptr)); + ESTACK_PUSH(s, *ptr); + } + i--; ptr++; + } + goto pop_next; + } + break; + case EXPORT_SUBTAG: + { + Export* ep = *((Export **) (export_val(term) + 1)); + /* Assumes Export entries never moves */ + POINTER_HASH(ep, HCONST_14); + goto pop_next; + } + + case FUN_SUBTAG: + { + ErlFunThing* funp = (ErlFunThing *) fun_val(term); + Uint num_free = funp->num_free; + UINT32_HASH_2(num_free, funp->fe->module, HCONST_20); + UINT32_HASH_2(funp->fe->old_index, funp->fe->old_uniq, HCONST_21); + if (num_free == 0) { + goto pop_next; + } else { + Eterm* bptr = funp->env + num_free - 1; + while (num_free-- > 1) { + term = *bptr--; + ESTACK_PUSH(s, term); + } + term = *bptr; + } + } + break; + case REFC_BINARY_SUBTAG: + case HEAP_BINARY_SUBTAG: + case SUB_BINARY_SUBTAG: + { + byte* bptr; + unsigned sz = binary_size(term); + Uint32 con = HCONST_13 + hash; + Uint bitoffs; + Uint bitsize; + + ERTS_GET_BINARY_BYTES(term, bptr, bitoffs, bitsize); + if (sz == 0 && bitsize == 0) { + hash = con; + } else { + if (bitoffs == 0) { + hash = block_hash(bptr, sz, con); + if (bitsize > 0) { + UINT32_HASH_2(bitsize, (bptr[sz] >> (8 - bitsize)), + HCONST_15); + } + } else { + byte* buf = (byte *) erts_alloc(ERTS_ALC_T_TMP, + sz + (bitsize != 0)); + erts_copy_bits(bptr, bitoffs, 1, buf, 0, 1, sz*8+bitsize); + hash = block_hash(buf, sz, con); + if (bitsize > 0) { + UINT32_HASH_2(bitsize, (buf[sz] >> (8 - bitsize)), + HCONST_15); + } + erts_free(ERTS_ALC_T_TMP, (void *) buf); + } + } + goto pop_next; + } + break; + case POS_BIG_SUBTAG: + case NEG_BIG_SUBTAG: + { + Eterm* ptr = big_val(term); + Uint i = 0; + Uint n = BIG_SIZE(ptr); + Uint32 con = BIG_SIGN(ptr) ? HCONST_10 : HCONST_11; +#if D_EXP == 16 + do { + Uint32 x, y; + x = i < n ? BIG_DIGIT(ptr, i++) : 0; + x += (Uint32)(i < n ? BIG_DIGIT(ptr, i++) : 0) << 16; + y = i < n ? BIG_DIGIT(ptr, i++) : 0; + y += (Uint32)(i < n ? BIG_DIGIT(ptr, i++) : 0) << 16; + UINT32_HASH_2(x, y, con); + } while (i < n); +#elif D_EXP == 32 + do { + Uint32 x, y; + x = i < n ? BIG_DIGIT(ptr, i++) : 0; + y = i < n ? BIG_DIGIT(ptr, i++) : 0; + UINT32_HASH_2(x, y, con); + } while (i < n); +#elif D_EXP == 64 + do { + Uint t; + Uint32 x, y; + ASSERT(i < n); + t = BIG_DIGIT(ptr, i++); + x = t & 0xffffffff; + y = t >> 32; + UINT32_HASH_2(x, y, con); + } while (i < n); +#else +#error "unsupported D_EXP size" +#endif + goto pop_next; + } + break; + case REF_SUBTAG: + UINT32_HASH(internal_ref_numbers(term)[0], HCONST_7); + ASSERT(internal_ref_no_of_numbers(term) == 3); + UINT32_HASH_2(internal_ref_numbers(term)[1], + internal_ref_numbers(term)[2], HCONST_8); + goto pop_next; + + case EXTERNAL_REF_SUBTAG: + { + ExternalThing* thing = external_thing_ptr(term); + + ASSERT(external_thing_ref_no_of_numbers(thing) == 3); + /* See limitation #2 */ + #ifdef ARCH_64 + POINTER_HASH(thing->node, HCONST_7); + UINT32_HASH(external_thing_ref_numbers(thing)[0], HCONST_7); + #else + UINT32_HASH_2(thing->node, + external_thing_ref_numbers(thing)[0], HCONST_7); + #endif + UINT32_HASH_2(external_thing_ref_numbers(thing)[1], + external_thing_ref_numbers(thing)[2], HCONST_8); + goto pop_next; + } + case EXTERNAL_PID_SUBTAG: { + ExternalThing* thing = external_thing_ptr(term); + /* See limitation #2 */ + #ifdef ARCH_64 + POINTER_HASH(thing->node, HCONST_5); + UINT32_HASH(thing->data.ui[0], HCONST_5); + #else + UINT32_HASH_2(thing->node, thing->data.ui[0], HCONST_5); + #endif + goto pop_next; + } + case EXTERNAL_PORT_SUBTAG: { + ExternalThing* thing = external_thing_ptr(term); + /* See limitation #2 */ + #ifdef ARCH_64 + POINTER_HASH(thing->node, HCONST_6); + UINT32_HASH(thing->data.ui[0], HCONST_6); + #else + UINT32_HASH_2(thing->node, thing->data.ui[0], HCONST_6); + #endif + goto pop_next; + } + case FLOAT_SUBTAG: + { + FloatDef ff; + GET_DOUBLE(term, ff); + if (ff.fd == 0.0f) { + /* ensure positive 0.0 */ + ff.fd = erts_get_positive_zero_float(); + } + UINT32_HASH_2(ff.fw[0], ff.fw[1], HCONST_12); + goto pop_next; + } + default: + erts_exit(ERTS_ERROR_EXIT, "Invalid tag in make_hash2(0x%X)\n", term); + } + } + break; + case TAG_PRIMARY_IMMED1: + #if ERTS_SIZEOF_ETERM == 8 + UINT32_HASH_2((Uint32)term, (Uint32)(term >> 32), HCONST); + #else + UINT32_HASH(term, HCONST); + #endif + goto pop_next; + + default: + erts_exit(ERTS_ERROR_EXIT, "Invalid tag in make_hash2(0x%X)\n", term); + + pop_next: + if (ESTACK_ISEMPTY(s)) { + DESTROY_ESTACK(s); + return hash; + } + term = ESTACK_POP(s); + + switch (term) { + case HASH_MAP_TAIL: { + hash = (Uint32) ESTACK_POP(s); + UINT32_HASH(hash_xor_pairs, HCONST_19); + hash_xor_pairs = (Uint32) ESTACK_POP(s); + goto pop_next; + } + case HASH_MAP_PAIR: + hash_xor_pairs ^= hash; + hash = 0; + goto pop_next; + + case HASH_CDR: + CONST_HASH(HCONST_18); /* Hash CDR i cons cell */ + goto pop_next; + default: + break; + } } } } + +#undef CONST_HASH +#undef HASH_MAP_TAIL +#undef HASH_MAP_PAIR +#undef HASH_CDR + #undef UINT32_HASH_2 #undef UINT32_HASH #undef SINT32_HASH @@ -1488,7 +2060,7 @@ tail_recur: hash = hash*FUNNY_NUMBER2 + funp->fe->old_uniq; if (num_free > 0) { if (num_free > 1) { - WSTACK_PUSH3(stack, (UWord) &funp->env[1], (num_free-1), MAKE_HASH_FUN_OP); + WSTACK_PUSH3(stack, (UWord) &funp->env[1], (num_free-1), MAKE_HASH_TERM_ARRAY_OP); } term = funp->env[0]; goto tail_recur; @@ -1516,12 +2088,15 @@ tail_recur: break; case FLOAT_DEF: { - FloatDef ff; - GET_DOUBLE(term, ff); - hash = hash*FUNNY_NUMBER6 + (ff.fw[0] ^ ff.fw[1]); + FloatDef ff; + GET_DOUBLE(term, ff); + if (ff.fd == 0.0f) { + /* ensure positive 0.0 */ + ff.fd = erts_get_positive_zero_float(); + } + hash = hash*FUNNY_NUMBER6 + (ff.fw[0] ^ ff.fw[1]); } break; - case MAKE_HASH_CDR_PRE_OP: term = (Eterm) WSTACK_POP(stack); if (is_not_list(term)) { @@ -1601,6 +2176,9 @@ tail_recur: } break; + case MAP_DEF: + hash = hash*FUNNY_NUMBER13 + FUNNY_NUMBER14 + make_hash2(term); + break; case TUPLE_DEF: { Eterm* ptr = tuple_val(term); @@ -1610,7 +2188,7 @@ tail_recur: op = MAKE_HASH_TUPLE_OP; }/*fall through*/ case MAKE_HASH_TUPLE_OP: - case MAKE_HASH_FUN_OP: + case MAKE_HASH_TERM_ARRAY_OP: { Uint i = (Uint) WSTACK_POP(stack); Eterm* ptr = (Eterm*) WSTACK_POP(stack); @@ -1627,7 +2205,7 @@ tail_recur: } default: - erl_exit(1, "Invalid tag in make_broken_hash\n"); + erts_exit(ERTS_ERROR_EXIT, "Invalid tag in make_broken_hash\n"); return 0; } if (WSTACK_ISEMPTY(stack)) break; @@ -1638,106 +2216,162 @@ tail_recur: return hash; #undef MAKE_HASH_TUPLE_OP -#undef MAKE_HASH_FUN_OP +#undef MAKE_HASH_TERM_ARRAY_OP #undef MAKE_HASH_CDR_PRE_OP #undef MAKE_HASH_CDR_POST_OP } -static int do_send_to_logger(Eterm tag, Eterm gleader, char *buf, int len) +static Eterm +do_allocate_logger_message(Eterm gleader, Eterm **hp, ErlOffHeap **ohp, + ErlHeapFragment **bp, Process **p, Uint sz) { - /* error_logger ! - {notify,{info_msg,gleader,{emulator,"~s~n",[<message as list>]}}} | - {notify,{error,gleader,{emulator,"~s~n",[<message as list>]}}} | - {notify,{warning_msg,gleader,{emulator,"~s~n",[<message as list>}]}} */ - Eterm* hp; - Uint sz; Uint gl_sz; - Eterm gl; - Eterm list,plist,format,tuple1,tuple2,tuple3; - ErlOffHeap *ohp; - ErlHeapFragment *bp = NULL; -#if !defined(ERTS_SMP) - Process *p; -#endif - - ASSERT(is_atom(tag)); - - if (len <= 0) { - return -1; - } + gl_sz = IS_CONST(gleader) ? 0 : size_object(gleader); + sz = sz + gl_sz; #ifndef ERTS_SMP #ifdef USE_THREADS - p = NULL; if (erts_get_scheduler_data()) /* Must be scheduler thread */ #endif { - p = erts_whereis_process(NULL, 0, am_error_logger, 0, 0); - if (p) { - erts_aint32_t state = erts_smp_atomic32_read_acqb(&p->state); - if (state & ERTS_PSFLG_RUNNING) - p = NULL; + *p = erts_whereis_process(NULL, 0, am_error_logger, 0, 0); + if (*p) { + erts_aint32_t state = erts_smp_atomic32_read_acqb(&(*p)->state); + if (state & (ERTS_PSFLG_RUNNING|ERTS_PSFLG_RUNNING_SYS)) + *p = NULL; } } - if (!p) { - /* buf *always* points to a null terminated string */ - erts_fprintf(stderr, "(no error logger present) %T: \"%s\"\n", - tag, buf); - return 0; + if (!*p) { + return NIL; } - /* So we have an error logger, lets build the message */ -#endif - gl_sz = IS_CONST(gleader) ? 0 : size_object(gleader); - sz = len * 2 /* message list */+ 2 /* cons surrounding message list */ - + gl_sz + - 3 /*outer 2-tuple*/ + 4 /* middle 3-tuple */ + 4 /*inner 3-tuple */ + - 8 /* "~s~n" */; -#ifndef ERTS_SMP - if (sz <= HeapWordsLeft(p)) { - ohp = &MSO(p); - hp = HEAP_TOP(p); - HEAP_TOP(p) += sz; + /* So we have an error logger, lets build the message */ + if (sz <= HeapWordsLeft(*p)) { + *ohp = &MSO(*p); + *hp = HEAP_TOP(*p); + HEAP_TOP(*p) += sz; } else { #endif - bp = new_message_buffer(sz); - ohp = &bp->off_heap; - hp = bp->mem; + *bp = new_message_buffer(sz); + *ohp = &(*bp)->off_heap; + *hp = (*bp)->mem; #ifndef ERTS_SMP } #endif - gl = (is_nil(gleader) + + return (is_nil(gleader) ? am_noproc : (IS_CONST(gleader) ? gleader - : copy_struct(gleader,gl_sz,&hp,ohp))); - list = buf_to_intlist(&hp, buf, len, NIL); - plist = CONS(hp,list,NIL); - hp += 2; - format = buf_to_intlist(&hp, "~s~n", 4, NIL); - tuple1 = TUPLE3(hp, am_emulator, format, plist); - hp += 4; - tuple2 = TUPLE3(hp, tag, gl, tuple1); - hp += 4; - tuple3 = TUPLE2(hp, am_notify, tuple2); + : copy_struct(gleader,gl_sz,hp,*ohp))); +} + +static void do_send_logger_message(Eterm *hp, ErlOffHeap *ohp, ErlHeapFragment *bp, + Process *p, Eterm message) +{ #ifdef HARDDEBUG - erts_fprintf(stderr, "%T\n", tuple3); + erts_fprintf(stderr, "%T\n", message); #endif #ifdef ERTS_SMP { Eterm from = erts_get_current_pid(); if (is_not_internal_pid(from)) from = NIL; - erts_queue_error_logger_message(from, tuple3, bp); + erts_queue_error_logger_message(from, message, bp); } #else - erts_queue_message(p, NULL /* only used for smp build */, bp, tuple3, NIL -#ifdef USE_VM_PROBES - , NIL -#endif - ); + erts_queue_message(p, NULL /* only used for smp build */, bp, message, NIL); #endif +} + +/* error_logger ! + {notify,{info_msg,gleader,{emulator,format,[args]}}} | + {notify,{error,gleader,{emulator,format,[args]}}} | + {notify,{warning_msg,gleader,{emulator,format,[args}]}} */ +static int do_send_to_logger(Eterm tag, Eterm gleader, char *buf, int len) +{ + Uint sz; + Eterm gl; + Eterm list,args,format,tuple1,tuple2,tuple3; + + Eterm *hp = NULL; + ErlOffHeap *ohp = NULL; + ErlHeapFragment *bp = NULL; + Process *p = NULL; + + ASSERT(is_atom(tag)); + + if (len <= 0) { + return -1; + } + + sz = len * 2 /* message list */ + 2 /* cons surrounding message list */ + + 3 /*outer 2-tuple*/ + 4 /* middle 3-tuple */ + 4 /*inner 3-tuple */ + + 8 /* "~s~n" */; + + /* gleader size is accounted and allocated next */ + gl = do_allocate_logger_message(gleader, &hp, &ohp, &bp, &p, sz); + + if(is_nil(gl)) { + /* buf *always* points to a null terminated string */ + erts_fprintf(stderr, "(no error logger present) %T: \"%s\"\n", + tag, buf); + return 0; + } + + list = buf_to_intlist(&hp, buf, len, NIL); + args = CONS(hp,list,NIL); + hp += 2; + format = buf_to_intlist(&hp, "~s~n", 4, NIL); + tuple1 = TUPLE3(hp, am_emulator, format, args); + hp += 4; + tuple2 = TUPLE3(hp, tag, gl, tuple1); + hp += 4; + tuple3 = TUPLE2(hp, am_notify, tuple2); + + do_send_logger_message(hp, ohp, bp, p, tuple3); + return 0; +} + +static int do_send_term_to_logger(Eterm tag, Eterm gleader, + char *buf, int len, Eterm args) +{ + Uint sz; + Eterm gl; + Uint args_sz; + Eterm format,tuple1,tuple2,tuple3; + + Eterm *hp = NULL; + ErlOffHeap *ohp = NULL; + ErlHeapFragment *bp = NULL; + Process *p = NULL; + + ASSERT(is_atom(tag)); + + args_sz = size_object(args); + sz = len * 2 /* format */ + args_sz + + 3 /*outer 2-tuple*/ + 4 /* middle 3-tuple */ + 4 /*inner 3-tuple */; + + /* gleader size is accounted and allocated next */ + gl = do_allocate_logger_message(gleader, &hp, &ohp, &bp, &p, sz); + + if(is_nil(gl)) { + /* buf *always* points to a null terminated string */ + erts_fprintf(stderr, "(no error logger present) %T: \"%s\" %T\n", + tag, buf, args); + return 0; + } + + format = buf_to_intlist(&hp, buf, len, NIL); + args = copy_struct(args, args_sz, &hp, ohp); + tuple1 = TUPLE3(hp, am_emulator, format, args); + hp += 4; + tuple2 = TUPLE3(hp, tag, gl, tuple1); + hp += 4; + tuple3 = TUPLE2(hp, am_notify, tuple2); + + do_send_logger_message(hp, ohp, bp, p, tuple3); return 0; } @@ -1765,6 +2399,12 @@ send_error_to_logger(Eterm gleader, char *buf, int len) return do_send_to_logger(am_error, gleader, buf, len); } +static ERTS_INLINE int +send_error_term_to_logger(Eterm gleader, char *buf, int len, Eterm args) +{ + return do_send_term_to_logger(am_error, gleader, buf, len, args); +} + #define LOGGER_DSBUF_INC_SZ 256 static erts_dsprintf_buf_t * @@ -1840,6 +2480,15 @@ erts_send_error_to_logger(Eterm gleader, erts_dsprintf_buf_t *dsbufp) } int +erts_send_error_term_to_logger(Eterm gleader, erts_dsprintf_buf_t *dsbufp, Eterm args) +{ + int res; + res = send_error_term_to_logger(gleader, dsbufp->str, dsbufp->str_len, args); + destroy_logger_dsbuf(dsbufp); + return res; +} + +int erts_send_info_to_logger_str(Eterm gleader, char *str) { return send_info_to_logger(gleader, str, sys_strlen(str)); @@ -2196,6 +2845,46 @@ tailrecur_ne: } break; /* not equal */ } + case MAP_SUBTAG: + if (is_flatmap_rel(a, a_base)) { + aa = flatmap_val_rel(a, a_base); + if (!is_boxed(b) || *boxed_val_rel(b,b_base) != *aa) + goto not_equal; + bb = flatmap_val_rel(b,b_base); + sz = flatmap_get_size((flatmap_t*)aa); + + if (sz != flatmap_get_size((flatmap_t*)bb)) goto not_equal; + if (sz == 0) goto pop_next; + + aa += 2; + bb += 2; + sz += 1; /* increment for tuple-keys */ + goto term_array; + + } else { + if (!is_boxed(b) || *boxed_val_rel(b,b_base) != hdr) + goto not_equal; + + aa = hashmap_val_rel(a, a_base) + 1; + bb = hashmap_val_rel(b, b_base) + 1; + switch (hdr & _HEADER_MAP_SUBTAG_MASK) { + case HAMT_SUBTAG_HEAD_ARRAY: + aa++; bb++; + sz = 16; + break; + case HAMT_SUBTAG_HEAD_BITMAP: + aa++; bb++; + case HAMT_SUBTAG_NODE_BITMAP: + sz = hashmap_bitcount(MAP_HEADER_VAL(hdr)); + ASSERT(sz > 0 && sz < 17); + break; + default: + erts_exit(ERTS_ERROR_EXIT, "Unknown hashmap subsubtag\n"); + } + goto term_array; + } + default: + ASSERT(!"Unknown boxed subtab in EQ"); } break; } @@ -2279,7 +2968,7 @@ static int cmpbytes(byte *s1, int l1, byte *s2, int l2) * * According to the Erlang Standard, types are orderered as follows: * numbers < (characters) < atoms < refs < funs < ports < pids < - * tuples < [] < conses < binaries. + * tuples < maps < [] < conses < binaries. * * Note that characters are currently not implemented. * @@ -2299,13 +2988,74 @@ static int cmp_atoms(Eterm a, Eterm b) bb->name+3, bb->len-3); } +#if !HALFWORD_HEAP +/* cmp(Eterm a, Eterm b) + * For compatibility with HiPE - arith-based compare. + */ +Sint cmp(Eterm a, Eterm b) +{ + return erts_cmp(a, b, 0, 0); +} +#endif + #if HALFWORD_HEAP -Sint cmp_rel(Eterm a, Eterm* a_base, Eterm b, Eterm* b_base) +static Sint erts_cmp_compound_rel_opt(Eterm a, Eterm* a_base, + Eterm b, Eterm* b_base, + int exact, int eq_only); #else -Sint cmp(Eterm a, Eterm b) +static Sint erts_cmp_compound(Eterm a, Eterm b, int exact, int eq_only); +#endif + +#if HALFWORD_HEAP +Sint erts_cmp_rel_opt(Eterm a, Eterm* a_base, + Eterm b, Eterm* b_base, + int exact, int eq_only) +#else +Sint erts_cmp(Eterm a, Eterm b, int exact, int eq_only) #endif { - DECLARE_WSTACK(stack); + if (is_atom(a) && is_atom(b)) { + return cmp_atoms(a, b); + } else if (is_both_small(a, b)) { + return (signed_val(a) - signed_val(b)); + } else if (is_float_rel(a, a_base) && is_float_rel(b, b_base)) { + FloatDef af, bf; + GET_DOUBLE_REL(a, af, a_base); + GET_DOUBLE_REL(b, bf, b_base); + return float_comp(af.fd, bf.fd); + } +#if HALFWORD_HEAP + return erts_cmp_compound_rel_opt(a,a_base,b,b_base,exact,eq_only); +#else + return erts_cmp_compound(a,b,exact,eq_only); +#endif +} + + +/* erts_cmp(Eterm a, Eterm b, int exact) + * exact = 1 -> term-based compare + * exact = 0 -> arith-based compare + */ +#if HALFWORD_HEAP +static Sint erts_cmp_compound_rel_opt(Eterm a, Eterm* a_base, + Eterm b, Eterm* b_base, + int exact, int eq_only) +#else +static Sint erts_cmp_compound(Eterm a, Eterm b, int exact, int eq_only) +#endif +{ +#define PSTACK_TYPE struct erts_cmp_hashmap_state + struct erts_cmp_hashmap_state { + Sint wstack_rollback; + int was_exact; + Eterm *ap; + Eterm *bp; + Eterm min_key; + Sint cmp_res; /* result so far -1,0,+1 */ + }; + PSTACK_DECLARE(hmap_stack, 1); + WSTACK_DECLARE(stack); + WSTACK_DECLARE(b_stack); /* only used by hashmaps */ Eterm* aa; Eterm* bb; int i; @@ -2321,6 +3071,26 @@ Sint cmp(Eterm a, Eterm b) Uint32 *anum; Uint32 *bnum; +/* The WSTACK contains naked Eterms and Operations marked with header-tags */ +#define OP_BITS 4 +#define OP_MASK 0xF +#define TERM_ARRAY_OP 0 +#define SWITCH_EXACT_OFF_OP 1 +#define HASHMAP_PHASE1_ARE_KEYS_EQUAL 2 +#define HASHMAP_PHASE1_IS_MIN_KEY 3 +#define HASHMAP_PHASE1_CMP_VALUES 4 +#define HASHMAP_PHASE2_ARE_KEYS_EQUAL 5 +#define HASHMAP_PHASE2_IS_MIN_KEY_A 6 +#define HASHMAP_PHASE2_IS_MIN_KEY_B 7 + + +#define OP_WORD(OP) (((OP) << _TAG_PRIMARY_SIZE) | TAG_PRIMARY_HEADER) +#define TERM_ARRAY_OP_WORD(SZ) OP_WORD(((SZ) << OP_BITS) | TERM_ARRAY_OP) + +#define GET_OP(WORD) (ASSERT(is_header(WORD)), ((WORD) >> _TAG_PRIMARY_SIZE) & OP_MASK) +#define GET_OP_ARG(WORD) (ASSERT(is_header(WORD)), ((WORD) >> (OP_BITS + _TAG_PRIMARY_SIZE))) + + #define RETURN_NEQ(cmp) { j=(cmp); ASSERT(j != 0); goto not_equal; } #define ON_CMP_GOTO(cmp) if ((j=(cmp)) == 0) goto pop_next; else goto not_equal @@ -2336,6 +3106,8 @@ Sint cmp(Eterm a, Eterm b) } while (0) +bodyrecur: + j = 0; tailrecur: if (is_same(a,a_base,b,b_base)) { /* Equal values or pointers. */ goto pop_next; @@ -2462,7 +3234,96 @@ tailrecur_ne: ++aa; ++bb; goto term_array; - + case (_TAG_HEADER_MAP >> _TAG_PRIMARY_SIZE) : + { + struct erts_cmp_hashmap_state* sp; + if (is_flatmap_header(ahdr)) { + if (!is_flatmap_rel(b,b_base)) { + if (is_hashmap_rel(b,b_base)) { + aa = (Eterm *)flatmap_val_rel(a,a_base); + i = flatmap_get_size((flatmap_t*)aa) - hashmap_size_rel(b,b_base); + ASSERT(i != 0); + RETURN_NEQ(i); + } + a_tag = MAP_DEF; + goto mixed_types; + } + aa = (Eterm *)flatmap_val_rel(a,a_base); + bb = (Eterm *)flatmap_val_rel(b,b_base); + + i = flatmap_get_size((flatmap_t*)aa); + if (i != flatmap_get_size((flatmap_t*)bb)) { + RETURN_NEQ((int)(i - flatmap_get_size((flatmap_t*)bb))); + } + if (i == 0) { + goto pop_next; + } + aa += 2; + bb += 2; + if (exact) { + i += 1; /* increment for tuple-keys */ + goto term_array; + } + else { + /* Value array */ + WSTACK_PUSH3(stack,(UWord)(bb+1),(UWord)(aa+1),TERM_ARRAY_OP_WORD(i)); + /* Switch back from 'exact' key compare */ + WSTACK_PUSH(stack,OP_WORD(SWITCH_EXACT_OFF_OP)); + /* Now do 'exact' compare of key tuples */ + a = *aa; + b = *bb; + exact = 1; + goto bodyrecur; + } + } + if (!is_hashmap_rel(b,b_base)) { + if (is_flatmap_rel(b,b_base)) { + bb = (Eterm *)flatmap_val_rel(b,b_base); + i = hashmap_size_rel(a,a_base) - flatmap_get_size((flatmap_t*)bb); + ASSERT(i != 0); + RETURN_NEQ(i); + } + a_tag = MAP_DEF; + goto mixed_types; + } + i = hashmap_size_rel(a,a_base) - hashmap_size_rel(b,b_base); + if (i) { + RETURN_NEQ(i); + } + if (hashmap_size_rel(a,a_base) == 0) { + goto pop_next; + } + + /* Hashmap compare strategy: + Phase 1. While keys are identical + Do synchronous stepping through leafs of both trees in hash + order. Maintain value compare result of minimal key. + + Phase 2. If key diff was found in phase 1 + Ignore values from now on. + Continue iterate trees by always advancing the one + lagging behind hash-wise. Identical keys are skipped. + A minimal key can only be candidate as tie-breaker if we + have passed that hash value in the other tree (which means + the key did not exist in the other tree). + */ + + sp = PSTACK_PUSH(hmap_stack); + hashmap_iterator_init(&stack, a, 0); + hashmap_iterator_init(&b_stack, b, 0); + sp->ap = hashmap_iterator_next(&stack); + sp->bp = hashmap_iterator_next(&b_stack); + sp->cmp_res = 0; + ASSERT(sp->ap && sp->bp); + + a = CAR(sp->ap); + b = CAR(sp->bp); + sp->was_exact = exact; + exact = 1; + WSTACK_PUSH(stack, OP_WORD(HASHMAP_PHASE1_ARE_KEYS_EQUAL)); + sp->wstack_rollback = WSTACK_COUNT(stack); + goto bodyrecur; + } case (_TAG_HEADER_FLOAT >> _TAG_PRIMARY_SIZE): if (!is_float_rel(b,b_base)) { a_tag = FLOAT_DEF; @@ -2686,11 +3547,6 @@ tailrecur_ne: { FloatDef f1, f2; Eterm big; -#if HEAP_ON_C_STACK - Eterm big_buf[CMP_TMP_HEAP_SIZE]; /* If HEAP_ON_C_STACK */ -#else - Eterm *big_buf = erts_get_scheduler_data()->cmp_tmp_heap; -#endif #if HALFWORD_HEAP Wterm aw = is_immed(a) ? a : rterm2wterm(a,a_base); Wterm bw = is_immed(b) ? b : rterm2wterm(b,b_base); @@ -2701,6 +3557,8 @@ tailrecur_ne: #define MAX_LOSSLESS_FLOAT ((double)((1LL << 53) - 2)) #define MIN_LOSSLESS_FLOAT ((double)(((1LL << 53) - 2)*-1)) #define BIG_ARITY_FLOAT_MAX (1024 / D_EXP) /* arity of max float as a bignum */ + Eterm big_buf[BIG_NEED_SIZE(BIG_ARITY_FLOAT_MAX)]; + b_tag = tag_val_def(bw); switch(_NUMBER_CODE(a_tag, b_tag)) { @@ -2711,86 +3569,95 @@ tailrecur_ne: j = big_sign(aw) ? -1 : 1; break; case SMALL_FLOAT: + if (exact) goto exact_fall_through; GET_DOUBLE(bw, f2); if (f2.fd < MAX_LOSSLESS_FLOAT && f2.fd > MIN_LOSSLESS_FLOAT) { - // Float is within the no loss limit + /* Float is within the no loss limit */ f1.fd = signed_val(aw); j = float_comp(f1.fd, f2.fd); + } #if ERTS_SIZEOF_ETERM == 8 - } else if (f2.fd > (double) (MAX_SMALL + 1)) { - // Float is a positive bignum, i.e. bigger + else if (f2.fd > (double) (MAX_SMALL + 1)) { + /* Float is a positive bignum, i.e. bigger */ j = -1; } else if (f2.fd < (double) (MIN_SMALL - 1)) { - // Float is a negative bignum, i.e. smaller + /* Float is a negative bignum, i.e. smaller */ j = 1; - } else { // Float is a Sint but less precise + } else { + /* Float is a Sint but less precise */ j = signed_val(aw) - (Sint) f2.fd; } #else - } else { - // If float is positive it is bigger than small + else { + /* If float is positive it is bigger than small */ j = (f2.fd > 0.0) ? -1 : 1; } -#endif // ERTS_SIZEOF_ETERM == 8 +#endif /* ERTS_SIZEOF_ETERM == 8 */ break; case FLOAT_BIG: + if (exact) goto exact_fall_through; { Wterm tmp = aw; aw = bw; bw = tmp; }/* fall through */ case BIG_FLOAT: + if (exact) goto exact_fall_through; GET_DOUBLE(bw, f2); if ((f2.fd < (double) (MAX_SMALL + 1)) && (f2.fd > (double) (MIN_SMALL - 1))) { - // Float is a Sint + /* Float is a Sint */ j = big_sign(aw) ? -1 : 1; } else if (big_arity(aw) > BIG_ARITY_FLOAT_MAX || pow(2.0,(big_arity(aw)-1)*D_EXP) > fabs(f2.fd)) { - // If bignum size shows that it is bigger than the abs float + /* If bignum size shows that it is bigger than the abs float */ j = big_sign(aw) ? -1 : 1; } else if (big_arity(aw) < BIG_ARITY_FLOAT_MAX && (pow(2.0,(big_arity(aw))*D_EXP)-1.0) < fabs(f2.fd)) { - // If bignum size shows that it is smaller than the abs float + /* If bignum size shows that it is smaller than the abs float */ j = f2.fd < 0 ? 1 : -1; } else if (f2.fd < MAX_LOSSLESS_FLOAT && f2.fd > MIN_LOSSLESS_FLOAT) { - // Float is within the no loss limit + /* Float is within the no loss limit */ if (big_to_double(aw, &f1.fd) < 0) { j = big_sign(aw) ? -1 : 1; } else { j = float_comp(f1.fd, f2.fd); } } else { - big = double_to_big(f2.fd, big_buf); - j = big_comp(aw, big); + big = double_to_big(f2.fd, big_buf, sizeof(big_buf)/sizeof(Eterm)); + j = big_comp(aw, rterm2wterm(big,big_buf)); } if (_NUMBER_CODE(a_tag, b_tag) == FLOAT_BIG) { j = -j; } break; case FLOAT_SMALL: + if (exact) goto exact_fall_through; GET_DOUBLE(aw, f1); if (f1.fd < MAX_LOSSLESS_FLOAT && f1.fd > MIN_LOSSLESS_FLOAT) { - // Float is within the no loss limit + /* Float is within the no loss limit */ f2.fd = signed_val(bw); j = float_comp(f1.fd, f2.fd); + } #if ERTS_SIZEOF_ETERM == 8 - } else if (f1.fd > (double) (MAX_SMALL + 1)) { - // Float is a positive bignum, i.e. bigger + else if (f1.fd > (double) (MAX_SMALL + 1)) { + /* Float is a positive bignum, i.e. bigger */ j = 1; } else if (f1.fd < (double) (MIN_SMALL - 1)) { - // Float is a negative bignum, i.e. smaller + /* Float is a negative bignum, i.e. smaller */ j = -1; - } else { // Float is a Sint but less precise it + } else { + /* Float is a Sint but less precise it */ j = (Sint) f1.fd - signed_val(bw); } #else - } else { - // If float is positive it is bigger than small + else { + /* If float is positive it is bigger than small */ j = (f1.fd > 0.0) ? 1 : -1; } -#endif // ERTS_SIZEOF_ETERM == 8 +#endif /* ERTS_SIZEOF_ETERM == 8 */ break; +exact_fall_through: default: j = b_tag - a_tag; } @@ -2816,8 +3683,7 @@ term_array: /* arrays in 'aa' and 'bb', length in 'i' */ goto not_equal; } } else { - /* (ab)Use TAG_PRIMARY_HEADER to recognize a term_array */ - WSTACK_PUSH3(stack, i, (UWord)bb, (UWord)aa | TAG_PRIMARY_HEADER); + WSTACK_PUSH3(stack, (UWord)bb, (UWord)aa, TERM_ARRAY_OP_WORD(i)); goto tailrecur_ne; } } @@ -2829,22 +3695,179 @@ term_array: /* arrays in 'aa' and 'bb', length in 'i' */ pop_next: if (!WSTACK_ISEMPTY(stack)) { UWord something = WSTACK_POP(stack); - if (primary_tag((Eterm) something) == TAG_PRIMARY_HEADER) { /* a term_array */ - aa = (Eterm*) something; - bb = (Eterm*) WSTACK_POP(stack); - i = WSTACK_POP(stack); - goto term_array; + struct erts_cmp_hashmap_state* sp; + if (primary_tag((Eterm) something) == TAG_PRIMARY_HEADER) { /* an operation */ + switch (GET_OP(something)) { + case TERM_ARRAY_OP: + i = GET_OP_ARG(something); + aa = (Eterm*)WSTACK_POP(stack); + bb = (Eterm*) WSTACK_POP(stack); + goto term_array; + + case SWITCH_EXACT_OFF_OP: + /* Done with exact compare of map keys, switch back */ + ASSERT(exact); + exact = 0; + goto pop_next; + + case HASHMAP_PHASE1_ARE_KEYS_EQUAL: { + sp = PSTACK_TOP(hmap_stack); + if (j) { + /* Key diff found, enter phase 2 */ + if (hashmap_key_hash_cmp(sp->ap, sp->bp) < 0) { + sp->min_key = CAR(sp->ap); + sp->cmp_res = -1; + sp->ap = hashmap_iterator_next(&stack); + } + else { + sp->min_key = CAR(sp->bp); + sp->cmp_res = 1; + sp->bp = hashmap_iterator_next(&b_stack); + } + exact = 1; /* only exact key compares in phase 2 */ + goto case_HASHMAP_PHASE2_LOOP; + } + + /* No key diff found so far, compare values if min key */ + + if (sp->cmp_res) { + a = CAR(sp->ap); + b = sp->min_key; + exact = 1; + WSTACK_PUSH(stack, OP_WORD(HASHMAP_PHASE1_IS_MIN_KEY)); + sp->wstack_rollback = WSTACK_COUNT(stack); + goto bodyrecur; + } + /* no min key-value found yet */ + a = CDR(sp->ap); + b = CDR(sp->bp); + exact = sp->was_exact; + WSTACK_PUSH(stack, OP_WORD(HASHMAP_PHASE1_CMP_VALUES)); + sp->wstack_rollback = WSTACK_COUNT(stack); + goto bodyrecur; + } + case HASHMAP_PHASE1_IS_MIN_KEY: + sp = PSTACK_TOP(hmap_stack); + if (j < 0) { + a = CDR(sp->ap); + b = CDR(sp->bp); + exact = sp->was_exact; + WSTACK_PUSH(stack, OP_WORD(HASHMAP_PHASE1_CMP_VALUES)); + sp->wstack_rollback = WSTACK_COUNT(stack); + goto bodyrecur; + } + goto case_HASHMAP_PHASE1_LOOP; + + case HASHMAP_PHASE1_CMP_VALUES: + sp = PSTACK_TOP(hmap_stack); + if (j) { + sp->cmp_res = j; + sp->min_key = CAR(sp->ap); + } + case_HASHMAP_PHASE1_LOOP: + sp->ap = hashmap_iterator_next(&stack); + sp->bp = hashmap_iterator_next(&b_stack); + if (!sp->ap) { + /* end of maps with identical keys */ + ASSERT(!sp->bp); + j = sp->cmp_res; + exact = sp->was_exact; + (void) PSTACK_POP(hmap_stack); + ON_CMP_GOTO(j); + } + a = CAR(sp->ap); + b = CAR(sp->bp); + exact = 1; + WSTACK_PUSH(stack, OP_WORD(HASHMAP_PHASE1_ARE_KEYS_EQUAL)); + sp->wstack_rollback = WSTACK_COUNT(stack); + goto bodyrecur; + + case_HASHMAP_PHASE2_LOOP: + if (sp->ap && sp->bp) { + a = CAR(sp->ap); + b = CAR(sp->bp); + ASSERT(exact); + WSTACK_PUSH(stack, OP_WORD(HASHMAP_PHASE2_ARE_KEYS_EQUAL)); + sp->wstack_rollback = WSTACK_COUNT(stack); + goto bodyrecur; + } + goto case_HASHMAP_PHASE2_NEXT_STEP; + + case HASHMAP_PHASE2_ARE_KEYS_EQUAL: + sp = PSTACK_TOP(hmap_stack); + if (j == 0) { + /* keys are equal, skip them */ + sp->ap = hashmap_iterator_next(&stack); + sp->bp = hashmap_iterator_next(&b_stack); + goto case_HASHMAP_PHASE2_LOOP; + } + /* fall through */ + case_HASHMAP_PHASE2_NEXT_STEP: + if (sp->ap || sp->bp) { + if (hashmap_key_hash_cmp(sp->ap, sp->bp) < 0) { + ASSERT(sp->ap); + a = CAR(sp->ap); + b = sp->min_key; + ASSERT(exact); + WSTACK_PUSH(stack, OP_WORD(HASHMAP_PHASE2_IS_MIN_KEY_A)); + } + else { /* hash_cmp > 0 */ + ASSERT(sp->bp); + a = CAR(sp->bp); + b = sp->min_key; + ASSERT(exact); + WSTACK_PUSH(stack, OP_WORD(HASHMAP_PHASE2_IS_MIN_KEY_B)); + } + sp->wstack_rollback = WSTACK_COUNT(stack); + goto bodyrecur; + } + /* End of both maps */ + j = sp->cmp_res; + exact = sp->was_exact; + (void) PSTACK_POP(hmap_stack); + ON_CMP_GOTO(j); + + case HASHMAP_PHASE2_IS_MIN_KEY_A: + sp = PSTACK_TOP(hmap_stack); + if (j < 0) { + sp->min_key = CAR(sp->ap); + sp->cmp_res = -1; + } + sp->ap = hashmap_iterator_next(&stack); + goto case_HASHMAP_PHASE2_LOOP; + + case HASHMAP_PHASE2_IS_MIN_KEY_B: + sp = PSTACK_TOP(hmap_stack); + if (j < 0) { + sp->min_key = CAR(sp->bp); + sp->cmp_res = 1; + } + sp->bp = hashmap_iterator_next(&b_stack); + goto case_HASHMAP_PHASE2_LOOP; + + default: + ASSERT(!"Invalid cmp op"); + } /* switch */ } a = (Eterm) something; b = (Eterm) WSTACK_POP(stack); goto tailrecur; } - DESTROY_WSTACK(stack); + ASSERT(PSTACK_IS_EMPTY(hmap_stack)); + PSTACK_DESTROY(hmap_stack); + WSTACK_DESTROY(stack); + WSTACK_DESTROY(b_stack); return 0; not_equal: - DESTROY_ESTACK(stack); + if (!PSTACK_IS_EMPTY(hmap_stack) && !eq_only) { + WSTACK_ROLLBACK(stack, PSTACK_TOP(hmap_stack)->wstack_rollback); + goto pop_next; + } + PSTACK_DESTROY(hmap_stack); + WSTACK_DESTROY(stack); + WSTACK_DESTROY(b_stack); return j; #undef CMP_NODES @@ -2981,7 +4004,7 @@ char* Sint_to_buf(Sint n, struct Sint_buf *buf) */ Eterm -buf_to_intlist(Eterm** hpp, char *buf, size_t len, Eterm tail) +buf_to_intlist(Eterm** hpp, const char *buf, size_t len, Eterm tail) { Eterm* hp = *hpp; size_t i = len; @@ -3019,118 +4042,349 @@ buf_to_intlist(Eterm** hpp, char *buf, size_t len, Eterm tail) ** Return remaining bytes in buffer on success ** ERTS_IOLIST_TO_BUF_OVERFLOW on overflow ** ERTS_IOLIST_TO_BUF_TYPE_ERROR on type error (including that result would not be a whole number of bytes) +** +** Note! +** Do not detect indata errors in this fiunction that are not detected by erts_iolist_size! +** +** A caller should be able to rely on a successful return from erts_iolist_to_buf +** if erts_iolist_size is previously successfully called and erts_iolist_to_buf +** is called with a buffer at least as large as the value given by erts_iolist_size. +** */ -ErlDrvSizeT erts_iolist_to_buf(Eterm obj, char* buf, ErlDrvSizeT alloced_len) +typedef enum { + ERTS_IL2B_BCOPY_OK, + ERTS_IL2B_BCOPY_YIELD, + ERTS_IL2B_BCOPY_OVERFLOW, + ERTS_IL2B_BCOPY_TYPE_ERROR +} ErtsIL2BBCopyRes; + +static ErtsIL2BBCopyRes +iolist_to_buf_bcopy(ErtsIOList2BufState *state, Eterm obj, int *yield_countp); + +static ERTS_INLINE ErlDrvSizeT +iolist_to_buf(const int yield_support, + ErtsIOList2BufState *state, + Eterm obj, + char* buf, + ErlDrvSizeT alloced_len) { - ErlDrvSizeT len = (ErlDrvSizeT) alloced_len; - Eterm* objp; +#undef IOLIST_TO_BUF_BCOPY +#define IOLIST_TO_BUF_BCOPY(CONSP) \ +do { \ + size_t size = binary_size(obj); \ + if (size > 0) { \ + Uint bitsize; \ + byte* bptr; \ + Uint bitoffs; \ + Uint num_bits; \ + if (yield_support) { \ + size_t max_size = ERTS_IOLIST_TO_BUF_BYTES_PER_YIELD_COUNT; \ + if (yield_count > 0) \ + max_size *= yield_count+1; \ + if (size > max_size) { \ + state->objp = CONSP; \ + goto L_bcopy_yield; \ + } \ + if (size >= ERTS_IOLIST_TO_BUF_BYTES_PER_YIELD_COUNT) { \ + int cost = (int) size; \ + cost /= ERTS_IOLIST_TO_BUF_BYTES_PER_YIELD_COUNT; \ + yield_count -= cost; \ + } \ + } \ + if (len < size) \ + goto L_overflow; \ + ERTS_GET_BINARY_BYTES(obj, bptr, bitoffs, bitsize); \ + if (bitsize != 0) \ + goto L_type_error; \ + num_bits = 8*size; \ + copy_binary_to_buffer(buf, 0, bptr, bitoffs, num_bits); \ + buf += size; \ + len -= size; \ + } \ +} while (0) + + ErlDrvSizeT res, len; + Eterm* objp = NULL; + int init_yield_count; + int yield_count; DECLARE_ESTACK(s); - goto L_again; - - while (!ESTACK_ISEMPTY(s)) { - obj = ESTACK_POP(s); - L_again: - if (is_list(obj)) { - L_iter_list: - objp = list_val(obj); - obj = CAR(objp); - if (is_byte(obj)) { - if (len == 0) { - goto L_overflow; - } - *buf++ = unsigned_val(obj); - len--; - } else if (is_binary(obj)) { - byte* bptr; - size_t size = binary_size(obj); - Uint bitsize; - Uint bitoffs; - Uint num_bits; - - if (len < size) { + + len = (ErlDrvSizeT) alloced_len; + + if (!yield_support) { + yield_count = init_yield_count = 0; /* Shut up faulty warning... >:-( */ + goto L_again; + } + else { + + if (state->iolist.reds_left <= 0) + return ERTS_IOLIST_TO_BUF_YIELD; + + ESTACK_CHANGE_ALLOCATOR(s, ERTS_ALC_T_SAVED_ESTACK); + init_yield_count = (ERTS_IOLIST_TO_BUF_YIELD_COUNT_PER_RED + * state->iolist.reds_left); + yield_count = init_yield_count; + + if (!state->iolist.estack.start) + goto L_again; + else { + int chk_stack; + /* Restart; restore state... */ + ESTACK_RESTORE(s, &state->iolist.estack); + + if (!state->bcopy.bptr) + chk_stack = 0; + else { + chk_stack = 1; + switch (iolist_to_buf_bcopy(state, THE_NON_VALUE, &yield_count)) { + case ERTS_IL2B_BCOPY_OK: + break; + case ERTS_IL2B_BCOPY_YIELD: + BUMP_ALL_REDS(state->iolist.c_p); + state->iolist.reds_left = 0; + ESTACK_SAVE(s, &state->iolist.estack); + return ERTS_IOLIST_TO_BUF_YIELD; + case ERTS_IL2B_BCOPY_OVERFLOW: goto L_overflow; - } - ERTS_GET_BINARY_BYTES(obj, bptr, bitoffs, bitsize); - if (bitsize != 0) { + case ERTS_IL2B_BCOPY_TYPE_ERROR: goto L_type_error; } - num_bits = 8*size; - copy_binary_to_buffer(buf, 0, bptr, bitoffs, num_bits); - buf += size; - len -= size; - } else if (is_list(obj)) { - ESTACK_PUSH(s, CDR(objp)); - goto L_iter_list; /* on head */ - } else if (is_not_nil(obj)) { - goto L_type_error; } - obj = CDR(objp); - if (is_list(obj)) { - goto L_iter_list; /* on tail */ - } else if (is_binary(obj)) { - byte* bptr; - size_t size = binary_size(obj); - Uint bitsize; - Uint bitoffs; - Uint num_bits; - if (len < size) { - goto L_overflow; + obj = state->iolist.obj; + buf = state->buf; + len = state->len; + objp = state->objp; + state->objp = NULL; + if (objp) + goto L_tail; + if (!chk_stack) + goto L_again; + /* check stack */ + } + } + + while (!ESTACK_ISEMPTY(s)) { + obj = ESTACK_POP(s); + L_again: + if (is_list(obj)) { + while (1) { /* Tail loop */ + while (1) { /* Head loop */ + if (yield_support && --yield_count <= 0) + goto L_yield; + objp = list_val(obj); + obj = CAR(objp); + if (is_byte(obj)) { + if (len == 0) { + goto L_overflow; + } + *buf++ = unsigned_val(obj); + len--; + } else if (is_binary(obj)) { + IOLIST_TO_BUF_BCOPY(objp); + } else if (is_list(obj)) { + ESTACK_PUSH(s, CDR(objp)); + continue; /* Head loop */ + } else if (is_not_nil(obj)) { + goto L_type_error; + } + break; } - ERTS_GET_BINARY_BYTES(obj, bptr, bitoffs, bitsize); - if (bitsize != 0) { + + L_tail: + + obj = CDR(objp); + + if (is_list(obj)) { + continue; /* Tail loop */ + } else if (is_binary(obj)) { + IOLIST_TO_BUF_BCOPY(NULL); + } else if (is_not_nil(obj)) { goto L_type_error; } - num_bits = 8*size; - copy_binary_to_buffer(buf, 0, bptr, bitoffs, num_bits); - buf += size; - len -= size; - } else if (is_not_nil(obj)) { - goto L_type_error; + break; } } else if (is_binary(obj)) { - byte* bptr; - size_t size = binary_size(obj); - Uint bitsize; - Uint bitoffs; - Uint num_bits; - if (len < size) { - goto L_overflow; - } - ERTS_GET_BINARY_BYTES(obj, bptr, bitoffs, bitsize); - if (bitsize != 0) { - goto L_type_error; - } - num_bits = 8*size; - copy_binary_to_buffer(buf, 0, bptr, bitoffs, num_bits); - buf += size; - len -= size; + IOLIST_TO_BUF_BCOPY(NULL); } else if (is_not_nil(obj)) { goto L_type_error; - } + } else if (yield_support && --yield_count <= 0) + goto L_yield; } + res = len; + + L_return: + DESTROY_ESTACK(s); - return len; + + if (yield_support) { + int reds; + CLEAR_SAVED_ESTACK(&state->iolist.estack); + reds = ((init_yield_count - yield_count - 1) + / ERTS_IOLIST_TO_BUF_YIELD_COUNT_PER_RED) + 1; + BUMP_REDS(state->iolist.c_p, reds); + state->iolist.reds_left -= reds; + if (state->iolist.reds_left < 0) + state->iolist.reds_left = 0; + } + + + return res; L_type_error: - DESTROY_ESTACK(s); - return ERTS_IOLIST_TO_BUF_TYPE_ERROR; + res = ERTS_IOLIST_TO_BUF_TYPE_ERROR; + goto L_return; L_overflow: - DESTROY_ESTACK(s); - return ERTS_IOLIST_TO_BUF_OVERFLOW; + res = ERTS_IOLIST_TO_BUF_OVERFLOW; + goto L_return; + + L_bcopy_yield: + + state->buf = buf; + state->len = len; + + switch (iolist_to_buf_bcopy(state, obj, &yield_count)) { + case ERTS_IL2B_BCOPY_OK: + ERTS_INTERNAL_ERROR("Missing yield"); + case ERTS_IL2B_BCOPY_YIELD: + BUMP_ALL_REDS(state->iolist.c_p); + state->iolist.reds_left = 0; + ESTACK_SAVE(s, &state->iolist.estack); + return ERTS_IOLIST_TO_BUF_YIELD; + case ERTS_IL2B_BCOPY_OVERFLOW: + goto L_overflow; + case ERTS_IL2B_BCOPY_TYPE_ERROR: + goto L_type_error; + } + + L_yield: + + BUMP_ALL_REDS(state->iolist.c_p); + state->iolist.reds_left = 0; + state->iolist.obj = obj; + state->buf = buf; + state->len = len; + ESTACK_SAVE(s, &state->iolist.estack); + return ERTS_IOLIST_TO_BUF_YIELD; + +#undef IOLIST_TO_BUF_BCOPY +} + +static ErtsIL2BBCopyRes +iolist_to_buf_bcopy(ErtsIOList2BufState *state, Eterm obj, int *yield_countp) +{ + ErtsIL2BBCopyRes res; + char *buf = state->buf; + ErlDrvSizeT len = state->len; + byte* bptr; + size_t size; + size_t max_size; + Uint bitoffs; + Uint num_bits; + int yield_count = *yield_countp; + + if (state->bcopy.bptr) { + bptr = state->bcopy.bptr; + size = state->bcopy.size; + bitoffs = state->bcopy.bitoffs; + state->bcopy.bptr = NULL; + } + else { + Uint bitsize; + + ASSERT(is_binary(obj)); + + size = binary_size(obj); + if (size <= 0) + return ERTS_IL2B_BCOPY_OK; + + if (len < size) + return ERTS_IL2B_BCOPY_OVERFLOW; + + ERTS_GET_BINARY_BYTES(obj, bptr, bitoffs, bitsize); + if (bitsize != 0) + return ERTS_IL2B_BCOPY_TYPE_ERROR; + } + + ASSERT(size > 0); + max_size = (size_t) ERTS_IOLIST_TO_BUF_BYTES_PER_YIELD_COUNT; + if (yield_count > 0) + max_size *= (size_t) (yield_count+1); + + if (size <= max_size) { + if (size >= ERTS_IOLIST_TO_BUF_BYTES_PER_YIELD_COUNT) { + int cost = (int) size; + cost /= ERTS_IOLIST_TO_BUF_BYTES_PER_YIELD_COUNT; + yield_count -= cost; + } + res = ERTS_IL2B_BCOPY_OK; + } + else { + ASSERT(0 < max_size && max_size < size); + yield_count = 0; + state->bcopy.bptr = bptr + max_size; + state->bcopy.bitoffs = bitoffs; + state->bcopy.size = size - max_size; + size = max_size; + res = ERTS_IL2B_BCOPY_YIELD; + } + + num_bits = 8*size; + copy_binary_to_buffer(buf, 0, bptr, bitoffs, num_bits); + state->buf += size; + state->len -= size; + *yield_countp = yield_count; + + return res; +} + +ErlDrvSizeT erts_iolist_to_buf_yielding(ErtsIOList2BufState *state) +{ + return iolist_to_buf(1, state, state->iolist.obj, state->buf, state->len); +} + +ErlDrvSizeT erts_iolist_to_buf(Eterm obj, char* buf, ErlDrvSizeT alloced_len) +{ + return iolist_to_buf(0, NULL, obj, buf, alloced_len); } /* * Return 0 if successful, and non-zero if unsuccessful. + * + * It is vital that if erts_iolist_to_buf would return an error for + * any type of term data, this function should do so as well. + * Any input term error detected in erts_iolist_to_buf should also + * be detected in this function! */ -int erts_iolist_size(Eterm obj, ErlDrvSizeT* sizep) + +static ERTS_INLINE int +iolist_size(const int yield_support, ErtsIOListState *state, Eterm obj, ErlDrvSizeT* sizep) { + int res, init_yield_count, yield_count; Eterm* objp; - Uint size = 0; /* Intentionally Uint due to halfword heap */ + Uint size = (Uint) *sizep; /* Intentionally Uint due to halfword heap */ DECLARE_ESTACK(s); + + if (!yield_support) + yield_count = init_yield_count = 0; /* Shut up faulty warning... >:-( */ + else { + if (state->reds_left <= 0) + return ERTS_IOLIST_YIELD; + ESTACK_CHANGE_ALLOCATOR(s, ERTS_ALC_T_SAVED_ESTACK); + init_yield_count = ERTS_IOLIST_SIZE_YIELDS_COUNT_PER_RED; + init_yield_count *= state->reds_left; + yield_count = init_yield_count; + if (state->estack.start) { + /* Restart; restore state... */ + ESTACK_RESTORE(s, &state->estack); + size = (Uint) state->size; + obj = state->obj; + } + } + goto L_again; #define SAFE_ADD(Var, Val) \ @@ -3146,51 +4400,101 @@ int erts_iolist_size(Eterm obj, ErlDrvSizeT* sizep) obj = ESTACK_POP(s); L_again: if (is_list(obj)) { - L_iter_list: - objp = list_val(obj); - /* Head */ - obj = CAR(objp); - if (is_byte(obj)) { - size++; - if (size == 0) { - goto L_overflow_error; + while (1) { /* Tail loop */ + while (1) { /* Head loop */ + if (yield_support && --yield_count <= 0) + goto L_yield; + objp = list_val(obj); + /* Head */ + obj = CAR(objp); + if (is_byte(obj)) { + size++; + if (size == 0) { + goto L_overflow_error; + } + } else if (is_binary(obj) && binary_bitsize(obj) == 0) { + SAFE_ADD(size, binary_size(obj)); + } else if (is_list(obj)) { + ESTACK_PUSH(s, CDR(objp)); + continue; /* Head loop */ + } else if (is_not_nil(obj)) { + goto L_type_error; + } + break; } - } else if (is_binary(obj) && binary_bitsize(obj) == 0) { - SAFE_ADD(size, binary_size(obj)); - } else if (is_list(obj)) { - ESTACK_PUSH(s, CDR(objp)); - goto L_iter_list; /* on head */ - } else if (is_not_nil(obj)) { - goto L_type_error; + /* Tail */ + obj = CDR(objp); + if (is_list(obj)) + continue; /* Tail loop */ + else if (is_binary(obj) && binary_bitsize(obj) == 0) { + SAFE_ADD(size, binary_size(obj)); + } else if (is_not_nil(obj)) { + goto L_type_error; + } + break; } - /* Tail */ - obj = CDR(objp); - if (is_list(obj)) - goto L_iter_list; /* on tail */ - else if (is_binary(obj) && binary_bitsize(obj) == 0) { + } else { + if (yield_support && --yield_count <= 0) + goto L_yield; + if (is_binary(obj) && binary_bitsize(obj) == 0) { /* Tail was binary */ SAFE_ADD(size, binary_size(obj)); } else if (is_not_nil(obj)) { goto L_type_error; } - } else if (is_binary(obj) && binary_bitsize(obj) == 0) { /* Tail was binary */ - SAFE_ADD(size, binary_size(obj)); - } else if (is_not_nil(obj)) { - goto L_type_error; } } #undef SAFE_ADD - DESTROY_ESTACK(s); *sizep = (ErlDrvSizeT) size; - return ERTS_IOLIST_OK; - L_overflow_error: + res = ERTS_IOLIST_OK; + + L_return: + DESTROY_ESTACK(s); - return ERTS_IOLIST_OVERFLOW; + + if (yield_support) { + int yc, reds; + CLEAR_SAVED_ESTACK(&state->estack); + yc = init_yield_count - yield_count; + reds = ((yc - 1) / ERTS_IOLIST_SIZE_YIELDS_COUNT_PER_RED) + 1; + BUMP_REDS(state->c_p, reds); + state->reds_left -= reds; + state->size = (ErlDrvSizeT) size; + state->have_size = 1; + } + + return res; + + L_overflow_error: + res = ERTS_IOLIST_OVERFLOW; + size = 0; + goto L_return; L_type_error: - DESTROY_ESTACK(s); - return ERTS_IOLIST_TYPE; + res = ERTS_IOLIST_TYPE; + size = 0; + goto L_return; + + L_yield: + BUMP_ALL_REDS(state->c_p); + state->reds_left = 0; + state->size = size; + state->obj = obj; + ESTACK_SAVE(s, &state->estack); + return ERTS_IOLIST_YIELD; +} + +int erts_iolist_size_yielding(ErtsIOListState *state) +{ + ErlDrvSizeT size = state->size; + return iolist_size(1, state, state->obj, &size); +} + +int erts_iolist_size(Eterm obj, ErlDrvSizeT* sizep) +{ + *sizep = 0; + return iolist_size(0, NULL, obj, sizep); } /* return 0 if item is not a non-empty flat list of bytes */ @@ -3213,145 +4517,6 @@ is_string(Eterm list) return 0; } -#ifdef ERTS_SMP - -/* - * Process and Port timers in smp case - */ - -ERTS_SCHED_PREF_PRE_ALLOC_IMPL(ptimer_pre, ErtsSmpPTimer, 1000) - -#define ERTS_PTMR_FLGS_ALLCD_SIZE \ - 2 -#define ERTS_PTMR_FLGS_ALLCD_MASK \ - ((((Uint32) 1) << ERTS_PTMR_FLGS_ALLCD_SIZE) - 1) - -#define ERTS_PTMR_FLGS_PREALLCD ((Uint32) 1) -#define ERTS_PTMR_FLGS_SLALLCD ((Uint32) 2) -#define ERTS_PTMR_FLGS_LLALLCD ((Uint32) 3) -#define ERTS_PTMR_FLG_CANCELLED (((Uint32) 1) << (ERTS_PTMR_FLGS_ALLCD_SIZE+0)) - -static void -init_ptimers(void) -{ - init_ptimer_pre_alloc(); -} - -static ERTS_INLINE void -free_ptimer(ErtsSmpPTimer *ptimer) -{ - switch (ptimer->timer.flags & ERTS_PTMR_FLGS_ALLCD_MASK) { - case ERTS_PTMR_FLGS_PREALLCD: - (void) ptimer_pre_free(ptimer); - break; - case ERTS_PTMR_FLGS_SLALLCD: - erts_free(ERTS_ALC_T_SL_PTIMER, (void *) ptimer); - break; - case ERTS_PTMR_FLGS_LLALLCD: - erts_free(ERTS_ALC_T_LL_PTIMER, (void *) ptimer); - break; - default: - erl_exit(ERTS_ABORT_EXIT, - "Internal error: Bad ptimer alloc type\n"); - break; - } -} - -/* Callback for process timeout cancelled */ -static void -ptimer_cancelled(ErtsSmpPTimer *ptimer) -{ - free_ptimer(ptimer); -} - -/* Callback for process timeout */ -static void -ptimer_timeout(ErtsSmpPTimer *ptimer) -{ - if (is_internal_pid(ptimer->timer.id)) { - Process *p; - p = erts_pid2proc_opt(NULL, - 0, - ptimer->timer.id, - ERTS_PROC_LOCK_MAIN|ERTS_PROC_LOCK_STATUS, - ERTS_P2P_FLG_ALLOW_OTHER_X); - if (p) { - if (!ERTS_PROC_IS_EXITING(p) - && !(ptimer->timer.flags & ERTS_PTMR_FLG_CANCELLED)) { - ASSERT(*ptimer->timer.timer_ref == ptimer); - *ptimer->timer.timer_ref = NULL; - (*ptimer->timer.timeout_func)(p); - } - erts_smp_proc_unlock(p, ERTS_PROC_LOCK_MAIN|ERTS_PROC_LOCK_STATUS); - } - } - else { - Port *p; - ASSERT(is_internal_port(ptimer->timer.id)); - p = erts_id2port_sflgs(ptimer->timer.id, - NULL, - 0, - ERTS_PORT_SFLGS_DEAD); - if (p) { - if (!(ptimer->timer.flags & ERTS_PTMR_FLG_CANCELLED)) { - ASSERT(*ptimer->timer.timer_ref == ptimer); - *ptimer->timer.timer_ref = NULL; - (*ptimer->timer.timeout_func)(p); - } - erts_port_release(p); - } - } - free_ptimer(ptimer); -} - -void -erts_create_smp_ptimer(ErtsSmpPTimer **timer_ref, - Eterm id, - ErlTimeoutProc timeout_func, - Uint timeout) -{ - ErtsSmpPTimer *res = ptimer_pre_alloc(); - if (res) - res->timer.flags = ERTS_PTMR_FLGS_PREALLCD; - else { - if (timeout < ERTS_ALC_MIN_LONG_LIVED_TIME) { - res = erts_alloc(ERTS_ALC_T_SL_PTIMER, sizeof(ErtsSmpPTimer)); - res->timer.flags = ERTS_PTMR_FLGS_SLALLCD; - } - else { - res = erts_alloc(ERTS_ALC_T_LL_PTIMER, sizeof(ErtsSmpPTimer)); - res->timer.flags = ERTS_PTMR_FLGS_LLALLCD; - } - } - res->timer.timeout_func = timeout_func; - res->timer.timer_ref = timer_ref; - res->timer.id = id; - res->timer.tm.active = 0; /* MUST be initalized */ - - ASSERT(!*timer_ref); - - *timer_ref = res; - - erts_set_timer(&res->timer.tm, - (ErlTimeoutProc) ptimer_timeout, - (ErlCancelProc) ptimer_cancelled, - (void*) res, - timeout); -} - -void -erts_cancel_smp_ptimer(ErtsSmpPTimer *ptimer) -{ - if (ptimer) { - ASSERT(*ptimer->timer.timer_ref == ptimer); - *ptimer->timer.timer_ref = NULL; - ptimer->timer.flags |= ERTS_PTMR_FLG_CANCELLED; - erts_cancel_timer(&ptimer->timer.tm); - } -} - -#endif - static int trim_threshold; static int top_pad; static int mmap_threshold; @@ -3361,9 +4526,7 @@ Uint tot_bin_allocated; void erts_init_utils(void) { -#ifdef ERTS_SMP - init_ptimers(); -#endif + } void erts_init_utils_mem(void) @@ -3464,6 +4627,294 @@ erts_free_read_env(void *value) erts_free(ERTS_ALC_T_TMP, value); } + +typedef struct { + size_t sz; + char *ptr; +} ErtsEmuArg; + +typedef struct { + int argc; + ErtsEmuArg *arg; + size_t no_bytes; +} ErtsEmuArgs; + +ErtsEmuArgs saved_emu_args = {0}; + +void +erts_save_emu_args(int argc, char **argv) +{ +#ifdef DEBUG + char *end_ptr; +#endif + char *ptr; + int i; + size_t arg_sz[100]; + size_t size; + + ASSERT(!saved_emu_args.argc); + + size = sizeof(ErtsEmuArg)*argc; + for (i = 0; i < argc; i++) { + size_t sz = sys_strlen(argv[i]); + if (i < sizeof(arg_sz)/sizeof(arg_sz[0])) + arg_sz[i] = sz; + size += sz+1; + } + ptr = (char *) malloc(size); + if (!ptr) { + ERTS_INTERNAL_ERROR("malloc failed to allocate memory!"); + } +#ifdef DEBUG + end_ptr = ptr + size; +#endif + saved_emu_args.arg = (ErtsEmuArg *) ptr; + ptr += sizeof(ErtsEmuArg)*argc; + saved_emu_args.argc = argc; + saved_emu_args.no_bytes = 0; + for (i = 0; i < argc; i++) { + size_t sz; + if (i < sizeof(arg_sz)/sizeof(arg_sz[0])) + sz = arg_sz[i]; + else + sz = sys_strlen(argv[i]); + saved_emu_args.arg[i].ptr = ptr; + saved_emu_args.arg[i].sz = sz; + saved_emu_args.no_bytes += sz; + ptr += sz+1; + sys_strcpy(saved_emu_args.arg[i].ptr, argv[i]); + } + ASSERT(ptr == end_ptr); +} + +Eterm +erts_get_emu_args(Process *c_p) +{ +#ifdef DEBUG + Eterm *end_hp; +#endif + int i; + Uint hsz; + Eterm *hp, res; + + hsz = saved_emu_args.no_bytes*2; + hsz += saved_emu_args.argc*2; + + hp = HAlloc(c_p, hsz); +#ifdef DEBUG + end_hp = hp + hsz; +#endif + res = NIL; + + for (i = saved_emu_args.argc-1; i >= 0; i--) { + Eterm arg = buf_to_intlist(&hp, + saved_emu_args.arg[i].ptr, + saved_emu_args.arg[i].sz, + NIL); + res = CONS(hp, arg, res); + hp += 2; + } + + ASSERT(hp == end_hp); + + return res; +} + + +Eterm +erts_get_ethread_info(Process *c_p) +{ + Uint sz, *szp; + Eterm res, *hp, **hpp, *end_hp = NULL; + + sz = 0; + szp = &sz; + hpp = NULL; + + while (1) { + Eterm tup, list, name; +#if defined(ETHR_NATIVE_ATOMIC32_IMPL) \ + || defined(ETHR_NATIVE_ATOMIC64_IMPL) \ + || defined(ETHR_NATIVE_DW_ATOMIC_IMPL) + char buf[1024]; + int i; + char **str; +#endif + + res = NIL; + +#ifdef ETHR_X86_MEMBAR_H__ + + tup = erts_bld_tuple(hpp, szp, 2, + erts_bld_string(hpp, szp, "sse2"), +#ifdef ETHR_X86_RUNTIME_CONF_HAVE_SSE2__ + erts_bld_string(hpp, szp, + (ETHR_X86_RUNTIME_CONF_HAVE_SSE2__ + ? "yes" : "no")) +#else + erts_bld_string(hpp, szp, "yes") +#endif + ); + res = erts_bld_cons(hpp, szp, tup, res); + + tup = erts_bld_tuple(hpp, szp, 2, + erts_bld_string(hpp, szp, + "x86" +#ifdef ARCH_64 + "_64" +#endif + " OOO"), + erts_bld_string(hpp, szp, +#ifdef ETHR_X86_OUT_OF_ORDER + "yes" +#else + "no" +#endif + )); + + res = erts_bld_cons(hpp, szp, tup, res); +#endif + +#ifdef ETHR_SPARC_V9_MEMBAR_H__ + + tup = erts_bld_tuple(hpp, szp, 2, + erts_bld_string(hpp, szp, "Sparc V9"), + erts_bld_string(hpp, szp, +#if defined(ETHR_SPARC_TSO) + "TSO" +#elif defined(ETHR_SPARC_PSO) + "PSO" +#elif defined(ETHR_SPARC_RMO) + "RMO" +#else + "undefined" +#endif + )); + + res = erts_bld_cons(hpp, szp, tup, res); + +#endif + +#ifdef ETHR_PPC_MEMBAR_H__ + + tup = erts_bld_tuple(hpp, szp, 2, + erts_bld_string(hpp, szp, "lwsync"), + erts_bld_string(hpp, szp, +#if defined(ETHR_PPC_HAVE_LWSYNC) + "yes" +#elif defined(ETHR_PPC_HAVE_NO_LWSYNC) + "no" +#elif defined(ETHR_PPC_RUNTIME_CONF_HAVE_LWSYNC__) + ETHR_PPC_RUNTIME_CONF_HAVE_LWSYNC__ ? "yes" : "no" +#else + "undefined" +#endif + )); + + res = erts_bld_cons(hpp, szp, tup, res); + +#endif + + tup = erts_bld_tuple(hpp, szp, 2, + erts_bld_string(hpp, szp, "Native rw-spinlocks"), +#ifdef ETHR_NATIVE_RWSPINLOCK_IMPL + erts_bld_string(hpp, szp, ETHR_NATIVE_RWSPINLOCK_IMPL) +#else + erts_bld_string(hpp, szp, "no") +#endif + ); + res = erts_bld_cons(hpp, szp, tup, res); + + tup = erts_bld_tuple(hpp, szp, 2, + erts_bld_string(hpp, szp, "Native spinlocks"), +#ifdef ETHR_NATIVE_SPINLOCK_IMPL + erts_bld_string(hpp, szp, ETHR_NATIVE_SPINLOCK_IMPL) +#else + erts_bld_string(hpp, szp, "no") +#endif + ); + res = erts_bld_cons(hpp, szp, tup, res); + + + list = NIL; +#ifdef ETHR_NATIVE_DW_ATOMIC_IMPL + if (ethr_have_native_dw_atomic()) { + name = erts_bld_string(hpp, szp, ETHR_NATIVE_DW_ATOMIC_IMPL); + str = ethr_native_dw_atomic_ops(); + for (i = 0; str[i]; i++) { + erts_snprintf(buf, sizeof(buf), "ethr_native_dw_atomic_%s()", str[i]); + list = erts_bld_cons(hpp, szp, + erts_bld_string(hpp, szp, buf), + list); + } + str = ethr_native_su_dw_atomic_ops(); + for (i = 0; str[i]; i++) { + erts_snprintf(buf, sizeof(buf), "ethr_native_su_dw_atomic_%s()", str[i]); + list = erts_bld_cons(hpp, szp, + erts_bld_string(hpp, szp, buf), + list); + } + } + else +#endif + name = erts_bld_string(hpp, szp, "no"); + + tup = erts_bld_tuple(hpp, szp, 3, + erts_bld_string(hpp, szp, "Double word native atomics"), + name, + list); + res = erts_bld_cons(hpp, szp, tup, res); + + list = NIL; +#ifdef ETHR_NATIVE_ATOMIC64_IMPL + name = erts_bld_string(hpp, szp, ETHR_NATIVE_ATOMIC64_IMPL); + str = ethr_native_atomic64_ops(); + for (i = 0; str[i]; i++) { + erts_snprintf(buf, sizeof(buf), "ethr_native_atomic64_%s()", str[i]); + list = erts_bld_cons(hpp, szp, + erts_bld_string(hpp, szp, buf), + list); + } +#else + name = erts_bld_string(hpp, szp, "no"); +#endif + tup = erts_bld_tuple(hpp, szp, 3, + erts_bld_string(hpp, szp, "64-bit native atomics"), + name, + list); + res = erts_bld_cons(hpp, szp, tup, res); + + list = NIL; +#ifdef ETHR_NATIVE_ATOMIC32_IMPL + name = erts_bld_string(hpp, szp, ETHR_NATIVE_ATOMIC32_IMPL); + str = ethr_native_atomic32_ops(); + for (i = 0; str[i]; i++) { + erts_snprintf(buf, sizeof(buf), "ethr_native_atomic32_%s()", str[i]); + list = erts_bld_cons(hpp, szp, + erts_bld_string(hpp, szp, buf), + list); + } +#else + name = erts_bld_string(hpp, szp, "no"); +#endif + tup = erts_bld_tuple(hpp, szp, 3, + erts_bld_string(hpp, szp, "32-bit native atomics"), + name, + list); + res = erts_bld_cons(hpp, szp, tup, res); + + if (hpp) { + HRelease(c_p, end_hp, *hpp) + return res; + } + + hp = HAlloc(c_p, sz); + end_hp = hp + sz; + hpp = &hp; + szp = NULL; + } +} + /* * To be used to silence unused result warnings, but do not abuse it. */ @@ -3478,19 +4929,7 @@ void erts_silence_warn_unused_result(long unused) void erts_interval_init(erts_interval_t *icp) { -#ifdef ARCH_64 - erts_atomic_init_nob(&icp->counter.atomic, 0); -#else - erts_dw_aint_t dw; -#ifdef ETHR_SU_DW_NAINT_T__ - dw.dw_sint = 0; -#else - dw.sint[ERTS_DW_AINT_HIGH_WORD] = 0; - dw.sint[ERTS_DW_AINT_LOW_WORD] = 0; -#endif - erts_dw_atomic_init_nob(&icp->counter.atomic, &dw); - -#endif + erts_atomic64_init_nob(&icp->counter.atomic, 0); #ifdef DEBUG icp->smp_api = 0; #endif @@ -3512,55 +4951,13 @@ erts_smp_interval_init(erts_interval_t *icp) static ERTS_INLINE Uint64 step_interval_nob(erts_interval_t *icp) { -#ifdef ARCH_64 - return (Uint64) erts_atomic_inc_read_nob(&icp->counter.atomic); -#else - erts_dw_aint_t exp; - - erts_dw_atomic_read_nob(&icp->counter.atomic, &exp); - while (1) { - erts_dw_aint_t new = exp; - -#ifdef ETHR_SU_DW_NAINT_T__ - new.dw_sint++; -#else - new.sint[ERTS_DW_AINT_LOW_WORD]++; - if (new.sint[ERTS_DW_AINT_LOW_WORD] == 0) - new.sint[ERTS_DW_AINT_HIGH_WORD]++; -#endif - - if (erts_dw_atomic_cmpxchg_nob(&icp->counter.atomic, &new, &exp)) - return erts_interval_dw_aint_to_val__(&new); - - } -#endif + return (Uint64) erts_atomic64_inc_read_nob(&icp->counter.atomic); } static ERTS_INLINE Uint64 step_interval_relb(erts_interval_t *icp) { -#ifdef ARCH_64 - return (Uint64) erts_atomic_inc_read_relb(&icp->counter.atomic); -#else - erts_dw_aint_t exp; - - erts_dw_atomic_read_nob(&icp->counter.atomic, &exp); - while (1) { - erts_dw_aint_t new = exp; - -#ifdef ETHR_SU_DW_NAINT_T__ - new.dw_sint++; -#else - new.sint[ERTS_DW_AINT_LOW_WORD]++; - if (new.sint[ERTS_DW_AINT_LOW_WORD] == 0) - new.sint[ERTS_DW_AINT_HIGH_WORD]++; -#endif - - if (erts_dw_atomic_cmpxchg_relb(&icp->counter.atomic, &new, &exp)) - return erts_interval_dw_aint_to_val__(&new); - - } -#endif + return (Uint64) erts_atomic64_inc_read_relb(&icp->counter.atomic); } @@ -3568,38 +4965,10 @@ static ERTS_INLINE Uint64 ensure_later_interval_nob(erts_interval_t *icp, Uint64 ic) { Uint64 curr_ic; -#ifdef ARCH_64 - curr_ic = (Uint64) erts_atomic_read_nob(&icp->counter.atomic); - if (curr_ic > ic) - return curr_ic; - return (Uint64) erts_atomic_inc_read_nob(&icp->counter.atomic); -#else - erts_dw_aint_t exp; - - erts_dw_atomic_read_nob(&icp->counter.atomic, &exp); - curr_ic = erts_interval_dw_aint_to_val__(&exp); + curr_ic = (Uint64) erts_atomic64_read_nob(&icp->counter.atomic); if (curr_ic > ic) return curr_ic; - - while (1) { - erts_dw_aint_t new = exp; - -#ifdef ETHR_SU_DW_NAINT_T__ - new.dw_sint++; -#else - new.sint[ERTS_DW_AINT_LOW_WORD]++; - if (new.sint[ERTS_DW_AINT_LOW_WORD] == 0) - new.sint[ERTS_DW_AINT_HIGH_WORD]++; -#endif - - if (erts_dw_atomic_cmpxchg_nob(&icp->counter.atomic, &new, &exp)) - return erts_interval_dw_aint_to_val__(&new); - - curr_ic = erts_interval_dw_aint_to_val__(&exp); - if (curr_ic > ic) - return curr_ic; - } -#endif + return (Uint64) erts_atomic64_inc_read_nob(&icp->counter.atomic); } @@ -3607,38 +4976,10 @@ static ERTS_INLINE Uint64 ensure_later_interval_acqb(erts_interval_t *icp, Uint64 ic) { Uint64 curr_ic; -#ifdef ARCH_64 - curr_ic = (Uint64) erts_atomic_read_acqb(&icp->counter.atomic); - if (curr_ic > ic) - return curr_ic; - return (Uint64) erts_atomic_inc_read_acqb(&icp->counter.atomic); -#else - erts_dw_aint_t exp; - - erts_dw_atomic_read_acqb(&icp->counter.atomic, &exp); - curr_ic = erts_interval_dw_aint_to_val__(&exp); + curr_ic = (Uint64) erts_atomic64_read_acqb(&icp->counter.atomic); if (curr_ic > ic) return curr_ic; - - while (1) { - erts_dw_aint_t new = exp; - -#ifdef ETHR_SU_DW_NAINT_T__ - new.dw_sint++; -#else - new.sint[ERTS_DW_AINT_LOW_WORD]++; - if (new.sint[ERTS_DW_AINT_LOW_WORD] == 0) - new.sint[ERTS_DW_AINT_HIGH_WORD]++; -#endif - - if (erts_dw_atomic_cmpxchg_acqb(&icp->counter.atomic, &new, &exp)) - return erts_interval_dw_aint_to_val__(&new); - - curr_ic = erts_interval_dw_aint_to_val__(&exp); - if (curr_ic > ic) - return curr_ic; - } -#endif + return (Uint64) erts_atomic64_inc_read_acqb(&icp->counter.atomic); } Uint64 @@ -3719,6 +5060,23 @@ erts_smp_ensure_later_interval_acqb(erts_interval_t *icp, Uint64 ic) #endif } +/* + * A millisecond timestamp without time correction where there's no hrtime + * - for tracing on "long" things... + */ +Uint64 erts_timestamp_millis(void) +{ +#ifdef ERTS_HAVE_OS_MONOTONIC_TIME_SUPPORT + return ERTS_MONOTONIC_TO_MSEC(erts_os_monotonic_time()); +#else + Uint64 res; + SysTimeval tv; + sys_gettimeofday(&tv); + res = (Uint64) tv.tv_sec*1000000; + res += (Uint64) tv.tv_usec; + return (res / 1000); +#endif +} #ifdef DEBUG /* diff --git a/erts/emulator/beam/version.h b/erts/emulator/beam/version.h index 3952c751b7..004725eaf5 100644 --- a/erts/emulator/beam/version.h +++ b/erts/emulator/beam/version.h @@ -3,16 +3,17 @@ * * Copyright Ericsson AB 1996-2009. All Rights Reserved. * - * The contents of this file are subject to the Erlang Public License, - * Version 1.1, (the "License"); you may not use this file except in - * compliance with the License. You should have received a copy of the - * Erlang Public License along with this software. If not, it can be - * retrieved online at http://www.erlang.org/. - * - * Software distributed under the License is distributed on an "AS IS" - * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See - * the License for the specific language governing rights and limitations - * under the License. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. * * %CopyrightEnd% */ |