diff options
Diffstat (limited to 'erts/emulator/beam')
171 files changed, 40215 insertions, 18579 deletions
diff --git a/erts/emulator/beam/atom.c b/erts/emulator/beam/atom.c index 84d2d5e3ed..099c00bcf6 100644 --- a/erts/emulator/beam/atom.c +++ b/erts/emulator/beam/atom.c @@ -3,16 +3,17 @@ * * Copyright Ericsson AB 1996-2013. All Rights Reserved. * - * The contents of this file are subject to the Erlang Public License, - * Version 1.1, (the "License"); you may not use this file except in - * compliance with the License. You should have received a copy of the - * Erlang Public License along with this software. If not, it can be - * retrieved online at http://www.erlang.org/. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Software distributed under the License is distributed on an "AS IS" - * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See - * the License for the specific language governing rights and limitations - * under the License. + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. * * %CopyrightEnd% */ @@ -175,7 +176,7 @@ atom_alloc(Atom* tmpl) /* * Precompute ordinal value of first 3 bytes + 7 bits. - * This is used by utils.c:cmp_atoms(). + * This is used by utils.c:erts_cmp_atoms(). * We cannot use the full 32 bits of the first 4 bytes, * since we use the sign of the difference between two * ordinal values to represent their relative order. @@ -434,6 +435,9 @@ init_atom_table(void) f.cmp = (HCMP_FUN) atom_cmp; f.alloc = (HALLOC_FUN) atom_alloc; f.free = (HFREE_FUN) atom_free; + f.meta_alloc = (HMALLOC_FUN) erts_alloc; + f.meta_free = (HMFREE_FUN) erts_free; + f.meta_print = (HMPRINT_FUN) erts_print; atom_text_pos = NULL; atom_text_end = NULL; diff --git a/erts/emulator/beam/atom.h b/erts/emulator/beam/atom.h index 5904ae0f7e..2c002ca92f 100644 --- a/erts/emulator/beam/atom.h +++ b/erts/emulator/beam/atom.h @@ -3,16 +3,17 @@ * * Copyright Ericsson AB 1996-2013. All Rights Reserved. * - * The contents of this file are subject to the Erlang Public License, - * Version 1.1, (the "License"); you may not use this file except in - * compliance with the License. You should have received a copy of the - * Erlang Public License along with this software. If not, it can be - * retrieved online at http://www.erlang.org/. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Software distributed under the License is distributed on an "AS IS" - * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See - * the License for the specific language governing rights and limitations - * under the License. + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. * * %CopyrightEnd% */ @@ -128,6 +129,7 @@ typedef enum { (erts_is_atom_utf8_bytes((byte *) LSTR, sizeof(LSTR) - 1, (TERM))) #define ERTS_DECL_AM(S) Eterm AM_ ## S = am_atom_put(#S, sizeof(#S) - 1) #define ERTS_INIT_AM(S) AM_ ## S = am_atom_put(#S, sizeof(#S) - 1) +#define ERTS_MAKE_AM(Str) am_atom_put(Str, sizeof(Str) - 1) int atom_table_size(void); /* number of elements */ int atom_table_sz(void); /* table size in bytes, excluding stored objects */ diff --git a/erts/emulator/beam/atom.names b/erts/emulator/beam/atom.names index d28e519ae1..169b071cd7 100644 --- a/erts/emulator/beam/atom.names +++ b/erts/emulator/beam/atom.names @@ -3,16 +3,17 @@ # # Copyright Ericsson AB 1996-2013. All Rights Reserved. # -# The contents of this file are subject to the Erlang Public License, -# Version 1.1, (the "License"); you may not use this file except in -# compliance with the License. You should have received a copy of the -# Erlang Public License along with this software. If not, it can be -# retrieved online at http://www.erlang.org/. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# Software distributed under the License is distributed on an "AS IS" -# basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See -# the License for the specific language governing rights and limitations -# under the License. +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. # # %CopyrightEnd% # @@ -68,7 +69,9 @@ atom aborted atom abs_path atom absoluteURI atom ac +atom accessor atom active +atom active_tasks atom all atom all_but_first atom all_names @@ -94,31 +97,37 @@ atom args atom arg0 atom arity atom asn1 +atom async atom asynchronous atom atom atom atom_used atom attributes +atom await_microstate_accounting_modifications atom await_port_send_result atom await_proc_exit +atom await_result atom await_sched_wall_time_modifications atom awaiting_load atom awaiting_unload atom backtrace backtrace_depth -atom badarg badarith badarity badfile badmatch badsig badfun +atom badarg badarith badarity badfile badfun badkey badmap badmatch badsig atom bag atom band atom big atom bif_return_trap +atom bif_timer_server atom binary atom binary_bin_to_list_trap atom binary_copy_trap +atom binary_find_trap atom binary_longest_prefix_trap atom binary_longest_suffix_trap -atom binary_match_trap -atom binary_matches_trap +atom binary_to_list_continue atom binary_to_term_trap atom block +atom block_normal atom blocked +atom blocked_normal atom bm atom bnot atom bor @@ -143,9 +152,11 @@ atom catchlevel atom cd atom cdr atom cflags +atom CHANGE='CHANGE' atom characters_to_binary_int atom characters_to_list_int atom clear +atom clock_service atom close atom closed atom code @@ -155,6 +166,7 @@ atom compat_rel atom compile atom compressed atom config_h +atom convert_time_unit atom connect atom connected atom connection_closed @@ -163,6 +175,7 @@ atom const atom context_switches atom control atom copy +atom counters atom cpu atom cpu_timestamp atom cr @@ -179,7 +192,9 @@ atom dexit atom depth atom dgroup_leader atom dictionary +atom dirty_cpu atom dirty_cpu_schedulers_online +atom dirty_io atom disable_trace atom disabled atom display_items @@ -197,9 +212,12 @@ atom dotall atom driver atom driver_options atom dsend +atom dsend_continue_trap atom dunlink atom duplicate_bag +atom duplicated atom dupnames +atom einval atom elib_malloc atom emulator atom enable_trace @@ -235,7 +253,7 @@ atom first atom firstline atom flags atom flush -atom flush_monitor_message +atom flush_monitor_messages atom force atom format_cpu_topology atom free @@ -247,6 +265,7 @@ atom functions atom function_clause atom garbage_collecting atom garbage_collection +atom garbage_collection_info atom gc_end atom gc_start atom Ge='>=' @@ -256,7 +275,10 @@ atom get_seq_token atom get_tcw atom getenv atom gather_gc_info_result +atom gather_io_bytes +atom gather_microstate_accounting_result atom gather_sched_wall_time_result +atom gather_system_check_result atom getting_linked atom getting_unlinked atom global @@ -278,7 +300,6 @@ atom http httph https http_response http_request http_header http_eoh http_error atom id atom if_clause atom ignore -atom imports atom in atom in_exiting atom inactive @@ -288,6 +309,7 @@ atom index atom infinity atom info atom info_msg +atom init atom initial_call atom input atom internal @@ -311,10 +333,12 @@ atom ldflags atom Le='=<' atom lf atom line +atom line_delimiter atom line_length atom linked_in_driver atom links atom list +atom list_to_binary_continue atom little atom loaded atom load_cancelled @@ -333,31 +357,42 @@ atom max atom maximum atom max_tables max_processes atom mbuf_size +atom md5 atom memory atom memory_internal atom memory_types atom message atom message_binary +atom message_queue_data atom message_queue_len atom messages +atom merge_trap atom meta atom meta_match_spec +atom micro_seconds +atom microstate_accounting +atom milli_seconds atom min_heap_size atom min_bin_vheap_size atom minor_version atom Minus='-' +atom mixed atom module atom module_info atom monitored_by atom monitor atom monitor_nodes atom monitors +atom monotonic +atom monotonic_timestamp atom more atom multi_scheduling atom multiline +atom nano_seconds atom name atom named_table atom namelist +atom native atom native_addresses atom Neq='=/=' atom Neqeq='/=' @@ -405,10 +440,12 @@ atom notify atom notsup atom nouse_stdio atom objects +atom off_heap atom offset atom ok atom old_heap_block_size atom old_heap_size +atom on_heap atom on_load atom open atom open_error @@ -419,13 +456,6 @@ atom orelse atom os_pid atom os_type atom os_version -atom ose_bg_proc -atom ose_int_proc -atom ose_phantom -atom ose_pri_proc -atom ose_process_prio -atom ose_process_type -atom ose_ti_proc atom out atom out_exited atom out_exiting @@ -448,6 +478,7 @@ atom ports atom port_count atom port_limit atom port_op +atom positive atom print atom priority atom private @@ -492,6 +523,7 @@ atom return_from atom return_to atom return_trace atom run_queue +atom run_queue_lengths atom runnable atom runnable_ports atom runnable_procs @@ -507,6 +539,7 @@ atom schedulers_online atom scheme atom scientific atom scope +atom seconds atom sensitive atom sequential_tracer atom sequential_trace_token @@ -536,6 +569,7 @@ atom static atom stderr_to_stdout atom stop atom stream +atom strict_monotonic_timestamp atom sunrm atom suspend atom suspended @@ -552,17 +586,22 @@ atom term_to_binary_trap atom this atom thread_pool_size atom threads +atom time_offset atom timeout atom timeout_value atom Times='*' atom timestamp atom total +atom total_active_tasks atom total_heap_size +atom total_run_queue_lengths atom tpkt atom trace trace_ts traced atom trace_control_word atom tracer atom trap_exit +atom trim +atom trim_all atom try_clause atom true atom tuple @@ -578,6 +617,7 @@ atom use_stdio atom used atom utf8 atom unblock +atom unblock_normal atom uniq atom unless_suspending atom unloaded diff --git a/erts/emulator/beam/beam_bif_load.c b/erts/emulator/beam/beam_bif_load.c index df1983a83d..0b47fc3586 100644 --- a/erts/emulator/beam/beam_bif_load.c +++ b/erts/emulator/beam/beam_bif_load.c @@ -3,16 +3,17 @@ * * Copyright Ericsson AB 1999-2013. All Rights Reserved. * - * The contents of this file are subject to the Erlang Public License, - * Version 1.1, (the "License"); you may not use this file except in - * compliance with the License. You should have received a copy of the - * Erlang Public License along with this software. If not, it can be - * retrieved online at http://www.erlang.org/. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Software distributed under the License is distributed on an "AS IS" - * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See - * the License for the specific language governing rights and limitations - * under the License. + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. * * %CopyrightEnd% */ @@ -33,18 +34,16 @@ #include "beam_catches.h" #include "erl_binary.h" #include "erl_nif.h" +#include "erl_bits.h" #include "erl_thr_progress.h" static void set_default_trace_pattern(Eterm module); -static Eterm check_process_code(Process* rp, Module* modp, int allow_gc, int *redsp); +static Eterm check_process_code(Process* rp, Module* modp, Uint flags, int *redsp); static void delete_code(Module* modp); -static void decrement_refc(BeamInstr* code); -static int is_native(BeamInstr* code); +static void decrement_refc(BeamCodeHeader*); static int any_heap_ref_ptrs(Eterm* start, Eterm* end, char* mod_start, Uint mod_size); static int any_heap_refs(Eterm* start, Eterm* end, char* mod_start, Uint mod_size); - - BIF_RETTYPE code_is_module_native_1(BIF_ALIST_1) { Module* modp; @@ -59,8 +58,8 @@ BIF_RETTYPE code_is_module_native_1(BIF_ALIST_1) return am_undefined; } erts_rlock_old_code(code_ix); - res = ((modp->curr.code && is_native(modp->curr.code)) || - (modp->old.code != 0 && is_native(modp->old.code))) ? + res = (erts_is_module_native(modp->curr.code_hdr) || + erts_is_module_native(modp->old.code_hdr)) ? am_true : am_false; erts_runlock_old_code(code_ix); return res; @@ -82,12 +81,12 @@ BIF_RETTYPE code_make_stub_module_3(BIF_ALIST_3) modp = erts_get_module(BIF_ARG_1, erts_active_code_ix()); if (modp && modp->curr.num_breakpoints > 0) { - ASSERT(modp->curr.code != NULL); + ASSERT(modp->curr.code_hdr != NULL); erts_clear_module_break(modp); ASSERT(modp->curr.num_breakpoints == 0); } - erts_start_staging_code_ix(); + erts_start_staging_code_ix(1); res = erts_make_stub_module(BIF_P, BIF_ARG_1, BIF_ARG_2, BIF_ARG_3); @@ -140,6 +139,25 @@ prepare_loading_2(BIF_ALIST_2) BIF_RET(res); } +BIF_RETTYPE +has_prepared_code_on_load_1(BIF_ALIST_1) +{ + Eterm res; + ProcBin* pb; + + if (!ERTS_TERM_IS_MAGIC_BINARY(BIF_ARG_1)) { + error: + BIF_ERROR(BIF_P, BADARG); + } + + pb = (ProcBin*) binary_val(BIF_ARG_1); + res = erts_has_code_on_load(pb->val); + if (res == NIL) { + goto error; + } + BIF_RET(res); +} + struct m { Binary* code; Eterm module; @@ -155,7 +173,7 @@ static struct /* Protected by code_write_permission */ { Process* stager; ErtsThrPrgrLaterOp lop; -}commiter_state; +} committer_state; #endif static Eterm @@ -164,14 +182,13 @@ exception_list(Process* p, Eterm tag, struct m* mp, Sint exceptions) Eterm* hp = HAlloc(p, 3 + 2*exceptions); Eterm res = NIL; - mp += exceptions - 1; while (exceptions > 0) { if (mp->exception) { res = CONS(hp, mp->module, res); hp += 2; exceptions--; } - mp--; + mp++; } return TUPLE2(hp, tag, res); } @@ -180,8 +197,8 @@ exception_list(Process* p, Eterm tag, struct m* mp, Sint exceptions) BIF_RETTYPE finish_loading_1(BIF_ALIST_1) { - int i; - int n; + Sint i; + Sint n; struct m* p = NULL; Uint exceptions; Eterm res; @@ -202,9 +219,13 @@ finish_loading_1(BIF_ALIST_1) */ n = erts_list_length(BIF_ARG_1); - if (n == -1) { - ERTS_BIF_PREP_ERROR(res, BIF_P, BADARG); - goto done; + if (n < 0) { + badarg: + if (p) { + erts_free(ERTS_ALC_T_LOADER_TMP, p); + } + erts_release_code_write_permission(); + BIF_ERROR(BIF_P, BADARG); } p = erts_alloc(ERTS_ALC_T_LOADER_TMP, n*sizeof(struct m)); @@ -219,29 +240,32 @@ finish_loading_1(BIF_ALIST_1) ProcBin* pb; if (!ERTS_TERM_IS_MAGIC_BINARY(term)) { - ERTS_BIF_PREP_ERROR(res, BIF_P, BADARG); - goto done; + goto badarg; } pb = (ProcBin*) binary_val(term); p[i].code = pb->val; p[i].module = erts_module_for_prepared_code(p[i].code); if (p[i].module == NIL) { - ERTS_BIF_PREP_ERROR(res, BIF_P, BADARG); - goto done; + goto badarg; } BIF_ARG_1 = CDR(cons); } /* * Since we cannot handle atomic loading of a group of modules - * if one or more of them uses on_load, we will only allow one - * element in the list. This limitation is intended to be - * lifted in the future. + * if one or more of them uses on_load, we will only allow + * more than one element in the list if none of the modules + * have an on_load function. */ if (n > 1) { - ERTS_BIF_PREP_ERROR(res, BIF_P, SYSTEM_LIMIT); - goto done; + for (i = 0; i < n; i++) { + if (erts_has_code_on_load(p[i].code) == am_true) { + erts_free(ERTS_ALC_T_LOADER_TMP, p); + erts_release_code_write_permission(); + BIF_ERROR(BIF_P, SYSTEM_LIMIT); + } + } } /* @@ -253,11 +277,27 @@ finish_loading_1(BIF_ALIST_1) */ res = am_ok; - erts_start_staging_code_ix(); + erts_start_staging_code_ix(n); for (i = 0; i < n; i++) { p[i].modp = erts_put_module(p[i].module); + p[i].modp->seen = 0; + } + + exceptions = 0; + for (i = 0; i < n; i++) { + p[i].exception = 0; + if (p[i].modp->seen) { + p[i].exception = 1; + exceptions++; + } + p[i].modp->seen = 1; + } + if (exceptions) { + res = exception_list(BIF_P, am_duplicated, p, exceptions); + goto done; } + for (i = 0; i < n; i++) { if (p[i].modp->curr.num_breakpoints > 0 || p[i].modp->curr.num_traced_exports > 0 || @@ -282,7 +322,7 @@ finish_loading_1(BIF_ALIST_1) exceptions = 0; for (i = 0; i < n; i++) { p[i].exception = 0; - if (p[i].modp->curr.code && p[i].modp->old.code) { + if (p[i].modp->curr.code_hdr && p[i].modp->old.code_hdr) { p[i].exception = 1; exceptions++; } @@ -368,10 +408,10 @@ staging_epilogue(Process* c_p, int commit, Eterm res, int is_blocking, * schedulers to read active code_ix in a safe way while executing * without any memory barriers at all. */ - ASSERT(commiter_state.stager == NULL); - commiter_state.stager = c_p; - erts_schedule_thr_prgr_later_op(smp_code_ix_commiter, NULL, &commiter_state.lop); - erts_smp_proc_inc_refc(c_p); + ASSERT(committer_state.stager == NULL); + committer_state.stager = c_p; + erts_schedule_thr_prgr_later_op(smp_code_ix_commiter, NULL, &committer_state.lop); + erts_proc_inc_refc(c_p); erts_suspend(c_p, ERTS_PROC_LOCK_MAIN, NULL); /* * smp_code_ix_commiter() will do the rest "later" @@ -386,11 +426,11 @@ staging_epilogue(Process* c_p, int commit, Eterm res, int is_blocking, #ifdef ERTS_SMP static void smp_code_ix_commiter(void* null) { - Process* p = commiter_state.stager; + Process* p = committer_state.stager; erts_commit_staging_code_ix(); #ifdef DEBUG - commiter_state.stager = NULL; + committer_state.stager = NULL; #endif erts_release_code_write_permission(); erts_smp_proc_lock(p, ERTS_PROC_LOCK_STATUS); @@ -398,7 +438,7 @@ static void smp_code_ix_commiter(void* null) erts_resume(p, ERTS_PROC_LOCK_STATUS); } erts_smp_proc_unlock(p, ERTS_PROC_LOCK_STATUS); - erts_smp_proc_dec_refc(p); + erts_proc_dec_refc(p); } #endif /* ERTS_SMP */ @@ -418,7 +458,7 @@ check_old_code_1(BIF_ALIST_1) modp = erts_get_module(BIF_ARG_1, code_ix); if (modp != NULL) { erts_rlock_old_code(code_ix); - if (modp->old.code != NULL) { + if (modp->old.code_hdr) { res = am_true; } erts_runlock_old_code(code_ix); @@ -427,7 +467,7 @@ check_old_code_1(BIF_ALIST_1) } Eterm -erts_check_process_code(Process *c_p, Eterm module, int allow_gc, int *redsp) +erts_check_process_code(Process *c_p, Eterm module, Uint flags, int *redsp) { Module* modp; Eterm res; @@ -442,7 +482,8 @@ erts_check_process_code(Process *c_p, Eterm module, int allow_gc, int *redsp) if (!modp) return am_false; erts_rlock_old_code(code_ix); - res = modp->old.code ? check_process_code(c_p, modp, allow_gc, redsp) : am_false; + res = (!modp->old.code_hdr ? am_false : + check_process_code(c_p, modp, flags, redsp)); erts_runlock_old_code(code_ix); return res; @@ -451,49 +492,21 @@ erts_check_process_code(Process *c_p, Eterm module, int allow_gc, int *redsp) BIF_RETTYPE erts_internal_check_process_code_2(BIF_ALIST_2) { int reds = 0; + Uint flags; Eterm res; - Eterm olist = BIF_ARG_2; - int allow_gc = 1; if (is_not_atom(BIF_ARG_1)) goto badarg; - while (is_list(olist)) { - Eterm *lp = list_val(olist); - Eterm opt = CAR(lp); - if (is_tuple(opt)) { - Eterm* tp = tuple_val(opt); - switch (arityval(tp[0])) { - case 2: - switch (tp[1]) { - case am_allow_gc: - switch (tp[2]) { - case am_false: - allow_gc = 0; - break; - case am_true: - allow_gc = 1; - break; - default: - goto badarg; - } - break; - default: - goto badarg; - } - break; - default: - goto badarg; - } - } - else - goto badarg; - olist = CDR(lp); + if (is_not_small(BIF_ARG_2)) + goto badarg; + + flags = unsigned_val(BIF_ARG_2); + if (flags & ~ERTS_CPC_ALL) { + goto badarg; } - if (is_not_nil(olist)) - goto badarg; - res = erts_check_process_code(BIF_P, BIF_ARG_1, allow_gc, &reds); + res = erts_check_process_code(BIF_P, BIF_ARG_1, flags, &reds); ASSERT(is_value(res)); @@ -520,13 +533,13 @@ BIF_RETTYPE delete_module_1(BIF_ALIST_1) } { - erts_start_staging_code_ix(); + erts_start_staging_code_ix(0); code_ix = erts_staging_code_ix(); modp = erts_get_module(BIF_ARG_1, code_ix); if (!modp) { res = am_undefined; } - else if (modp->old.code != 0) { + else if (modp->old.code_hdr) { erts_dsprintf_buf_t *dsbufp = erts_create_logger_dsbuf(); erts_dsprintf(dsbufp, "Module %T must be purged before loading\n", BIF_ARG_1); @@ -564,8 +577,8 @@ BIF_RETTYPE module_loaded_1(BIF_ALIST_1) } code_ix = erts_active_code_ix(); if ((modp = erts_get_module(BIF_ARG_1, code_ix)) != NULL) { - if (modp->curr.code != NULL - && modp->curr.code[MI_ON_LOAD_FUNCTION_PTR] == 0) { + if (modp->curr.code_hdr + && modp->curr.code_hdr->on_load_function_ptr == NULL) { res = am_true; } } @@ -612,8 +625,8 @@ BIF_RETTYPE call_on_load_function_1(BIF_ALIST_1) { Module* modp = erts_get_module(BIF_ARG_1, erts_active_code_ix()); - if (modp && modp->curr.code) { - BIF_TRAP_CODE_PTR_0(BIF_P, modp->curr.code[MI_ON_LOAD_FUNCTION_PTR]); + if (modp && modp->curr.code_hdr) { + BIF_TRAP_CODE_PTR_0(BIF_P, modp->curr.code_hdr->on_load_function_ptr); } else { BIF_ERROR(BIF_P, BADARG); @@ -624,7 +637,6 @@ BIF_RETTYPE finish_after_on_load_2(BIF_ALIST_2) { ErtsCodeIndex code_ix; Module* modp; - Eterm on_load; if (!erts_try_seize_code_write_permission(BIF_P)) { ERTS_BIF_YIELD2(bif_export[BIF_finish_after_on_load_2], @@ -639,14 +651,14 @@ BIF_RETTYPE finish_after_on_load_2(BIF_ALIST_2) code_ix = erts_active_code_ix(); modp = erts_get_module(BIF_ARG_1, code_ix); - if (!modp || modp->curr.code == 0) { + if (!modp || !modp->curr.code_hdr) { error: erts_smp_thr_progress_unblock(); erts_smp_proc_lock(BIF_P, ERTS_PROC_LOCK_MAIN); erts_release_code_write_permission(); BIF_ERROR(BIF_P, BADARG); } - if ((on_load = modp->curr.code[MI_ON_LOAD_FUNCTION_PTR]) == 0) { + if (modp->curr.code_hdr->on_load_function_ptr == NULL) { goto error; } if (BIF_ARG_2 != am_false && BIF_ARG_2 != am_true) { @@ -668,7 +680,7 @@ BIF_RETTYPE finish_after_on_load_2(BIF_ALIST_2) ep->code[4] = 0; } } - modp->curr.code[MI_ON_LOAD_FUNCTION_PTR] = 0; + modp->curr.code_hdr->on_load_function_ptr = NULL; set_default_trace_pattern(BIF_ARG_1); } else if (BIF_ARG_2 == am_false) { BeamInstr* code; @@ -680,13 +692,16 @@ BIF_RETTYPE finish_after_on_load_2(BIF_ALIST_2) * the current code; the old code is not touched. */ erts_total_code_size -= modp->curr.code_length; - code = modp->curr.code; - end = (BeamInstr *)((char *)code + modp->curr.code_length); + code = (BeamInstr*) modp->curr.code_hdr; + end = (BeamInstr *) ((char *)code + modp->curr.code_length); erts_cleanup_funs_on_purge(code, end); beam_catches_delmod(modp->curr.catches, code, modp->curr.code_length, erts_active_code_ix()); - erts_free(ERTS_ALC_T_CODE, (void *) code); - modp->curr.code = NULL; + if (modp->curr.code_hdr->literals_start) { + erts_free(ERTS_ALC_T_LITERAL, modp->curr.code_hdr->literals_start); + } + erts_free(ERTS_ALC_T_CODE, modp->curr.code_hdr); + modp->curr.code_hdr = NULL; modp->curr.code_length = 0; modp->curr.catches = BEAM_CATCHES_NIL; erts_remove_from_ranges(code); @@ -722,31 +737,50 @@ set_default_trace_pattern(Eterm module) } } +static ERTS_INLINE int +check_mod_funs(Process *p, ErlOffHeap *off_heap, char *area, size_t area_size) +{ + struct erl_off_heap_header* oh; + for (oh = off_heap->first; oh; oh = oh->next) { + if (thing_subtag(oh->thing_word) == FUN_SUBTAG) { + ErlFunThing* funp = (ErlFunThing*) oh; + if (ErtsInArea(funp->fe->address, area, area_size)) + return !0; + } + } + return 0; +} + + static Eterm -check_process_code(Process* rp, Module* modp, int allow_gc, int *redsp) +check_process_code(Process* rp, Module* modp, Uint flags, int *redsp) { BeamInstr* start; + char* literals; + Uint lit_bsize; char* mod_start; Uint mod_size; - BeamInstr* end; Eterm* sp; - struct erl_off_heap_header* oh; int done_gc = 0; + int need_gc = 0; + ErtsMessage *msgp; + ErlHeapFragment *hfrag; -#define INSIDE(a) (start <= (a) && (a) < end) +#define ERTS_ORDINARY_GC__ (1 << 0) +#define ERTS_LITERAL_GC__ (1 << 1) /* * Pick up limits for the module. */ - start = modp->old.code; - end = (BeamInstr *)((char *)start + modp->old.code_length); + start = (BeamInstr*) modp->old.code_hdr; mod_start = (char *) start; mod_size = modp->old.code_length; /* * Check if current instruction or continuation pointer points into module. */ - if (INSIDE(rp->i) || INSIDE(rp->cp)) { + if (ErtsInArea(rp->i, mod_start, mod_size) + || ErtsInArea(rp->cp, mod_start, mod_size)) { return am_true; } @@ -754,7 +788,7 @@ check_process_code(Process* rp, Module* modp, int allow_gc, int *redsp) * Check all continuation pointers stored on the stack. */ for (sp = rp->stop; sp < STACK_START(rp); sp++) { - if (is_CP(*sp) && INSIDE(cp_val(*sp))) { + if (is_CP(*sp) && ErtsInArea(cp_val(*sp), mod_start, mod_size)) { return am_true; } } @@ -768,15 +802,15 @@ check_process_code(Process* rp, Module* modp, int allow_gc, int *redsp) struct StackTrace *s; ASSERT(is_list(rp->ftrace)); s = (struct StackTrace *) big_val(CDR(list_val(rp->ftrace))); - if ((s->pc && INSIDE(s->pc)) || - (s->current && INSIDE(s->current))) { + if ((s->pc && ErtsInArea(s->pc, mod_start, mod_size)) || + (s->current && ErtsInArea(s->current, mod_start, mod_size))) { rp->freason = EXC_NULL; rp->fvalue = NIL; rp->ftrace = NIL; } else { int i; for (i = 0; i < s->depth; i++) { - if (INSIDE(s->trace[i])) { + if (ErtsInArea(s->trace[i], mod_start, mod_size)) { rp->freason = EXC_NULL; rp->fvalue = NIL; rp->ftrace = NIL; @@ -797,111 +831,147 @@ check_process_code(Process* rp, Module* modp, int allow_gc, int *redsp) } /* - * See if there are funs that refer to the old version of the module. + * Message queue can contains funs, but (at least currently) no + * constants. If we got references to this module from the message + * queue, a GC cannot remove these... */ - rescan: - for (oh = MSO(rp).first; oh; oh = oh->next) { - if (thing_subtag(oh->thing_word) == FUN_SUBTAG) { - ErlFunThing* funp = (ErlFunThing*) oh; + erts_smp_proc_lock(rp, ERTS_PROC_LOCK_MSGQ); + ERTS_SMP_MSGQ_MV_INQ2PRIVQ(rp); + erts_smp_proc_unlock(rp, ERTS_PROC_LOCK_MSGQ); - if (INSIDE((BeamInstr *) funp->fe->address)) { - if (done_gc) { - return am_true; - } else { - if (!allow_gc) - return am_aborted; - /* - * Try to get rid of this fun by garbage collecting. - * Clear both fvalue and ftrace to make sure they - * don't hold any funs. - */ - rp->freason = EXC_NULL; - rp->fvalue = NIL; - rp->ftrace = NIL; - done_gc = 1; - FLAGS(rp) |= F_NEED_FULLSWEEP; - *redsp += erts_garbage_collect(rp, 0, rp->arg_reg, rp->arity); - goto rescan; - } - } + literals = (char*) modp->old.code_hdr->literals_start; + lit_bsize = (char*) modp->old.code_hdr->literals_end - literals; + + for (msgp = rp->msg.first; msgp; msgp = msgp->next) { + if (msgp->data.attached == ERTS_MSG_COMBINED_HFRAG) + hfrag = &msgp->hfrag; + else if (is_value(ERL_MESSAGE_TERM(msgp)) && msgp->data.heap_frag) + hfrag = msgp->data.heap_frag; + else + continue; + for (; hfrag; hfrag = hfrag->next) { + if (check_mod_funs(rp, &hfrag->off_heap, mod_start, mod_size)) + return am_true; + /* Should not contain any constants... */ + ASSERT(!any_heap_refs(&hfrag->mem[0], + &hfrag->mem[hfrag->used_size], + literals, + lit_bsize)); } } - /* - * See if there are constants inside the module referenced by the process. - */ - done_gc = 0; - for (;;) { - ErlMessage* mp; - - if (any_heap_ref_ptrs(&rp->fvalue, &rp->fvalue+1, mod_start, mod_size)) { + while (1) { + + /* Check heap, stack etc... */ + if (check_mod_funs(rp, &rp->off_heap, mod_start, mod_size)) + goto try_gc; + if (!(flags & ERTS_CPC_COPY_LITERALS)) { + /* Process ok. May contain old literals but we will be called + * again before module is purged. + */ + return am_false; + } + if (any_heap_ref_ptrs(&rp->fvalue, &rp->fvalue+1, literals, lit_bsize)) { rp->freason = EXC_NULL; rp->fvalue = NIL; rp->ftrace = NIL; } - if (any_heap_ref_ptrs(rp->stop, rp->hend, mod_start, mod_size)) { - goto need_gc; - } - if (any_heap_refs(rp->heap, rp->htop, mod_start, mod_size)) { - goto need_gc; - } - - if (any_heap_refs(rp->old_heap, rp->old_htop, mod_start, mod_size)) { - goto need_gc; + if (any_heap_ref_ptrs(rp->stop, rp->hend, literals, lit_bsize)) + goto try_literal_gc; + if (any_heap_refs(rp->heap, rp->htop, literals, lit_bsize)) + goto try_literal_gc; + if (any_heap_refs(rp->old_heap, rp->old_htop, literals, lit_bsize)) + goto try_literal_gc; + + /* Check dictionary */ + if (rp->dictionary) { + Eterm* start = ERTS_PD_START(rp->dictionary); + Eterm* end = start + ERTS_PD_SIZE(rp->dictionary); + + if (any_heap_ref_ptrs(start, end, literals, lit_bsize)) + goto try_literal_gc; } - if (rp->dictionary != NULL) { - Eterm* start = rp->dictionary->data; - Eterm* end = start + rp->dictionary->used; + /* Check heap fragments */ + for (hfrag = rp->mbuf; hfrag; hfrag = hfrag->next) { + Eterm *hp, *hp_end; + /* Off heap lists should already have been moved into process */ + ASSERT(!check_mod_funs(rp, &hfrag->off_heap, mod_start, mod_size)); - if (any_heap_ref_ptrs(start, end, mod_start, mod_size)) { - goto need_gc; - } + hp = &hfrag->mem[0]; + hp_end = &hfrag->mem[hfrag->used_size]; + if (any_heap_refs(hp, hp_end, literals, lit_bsize)) + goto try_literal_gc; } - for (mp = rp->msg.first; mp != NULL; mp = mp->next) { - if (any_heap_ref_ptrs(mp->m, mp->m+2, mod_start, mod_size)) { - goto need_gc; +#ifdef DEBUG + /* + * Message buffer fragments should not have any references + * to constants, and off heap lists should already have + * been moved into process off heap structure. + */ + for (msgp = rp->msg_frag; msgp; msgp = msgp->next) { + if (msgp->data.attached == ERTS_MSG_COMBINED_HFRAG) + hfrag = &msgp->hfrag; + else + hfrag = msgp->data.heap_frag; + for (; hfrag; hfrag = hfrag->next) { + Eterm *hp, *hp_end; + ASSERT(!check_mod_funs(rp, &hfrag->off_heap, mod_start, mod_size)); + + hp = &hfrag->mem[0]; + hp_end = &hfrag->mem[hfrag->used_size]; + ASSERT(!any_heap_refs(hp, hp_end, literals, lit_bsize)); } } - break; - need_gc: - if (done_gc) { +#endif + + return am_false; + + try_literal_gc: + need_gc |= ERTS_LITERAL_GC__; + + try_gc: + need_gc |= ERTS_ORDINARY_GC__; + + if ((done_gc & need_gc) == need_gc) return am_true; - } else { - Eterm* literals; - Uint lit_size; - struct erl_off_heap_header* oh; - if (!allow_gc) - return am_aborted; + if (!(flags & ERTS_CPC_ALLOW_GC)) + return am_aborted; - /* - * Try to get rid of constants by by garbage collecting. - * Clear both fvalue and ftrace. - */ - rp->freason = EXC_NULL; - rp->fvalue = NIL; - rp->ftrace = NIL; - done_gc = 1; + need_gc &= ~done_gc; + + /* + * Try to get rid of constants by by garbage collecting. + * Clear both fvalue and ftrace. + */ + + rp->freason = EXC_NULL; + rp->fvalue = NIL; + rp->ftrace = NIL; + + if (need_gc & ERTS_ORDINARY_GC__) { FLAGS(rp) |= F_NEED_FULLSWEEP; - *redsp += erts_garbage_collect(rp, 0, rp->arg_reg, rp->arity); - literals = (Eterm *) modp->old.code[MI_LITERALS_START]; - lit_size = (Eterm *) modp->old.code[MI_LITERALS_END] - literals; - oh = (struct erl_off_heap_header *) - modp->old.code[MI_LITERALS_OFF_HEAP]; - *redsp += lit_size / 10; /* Need, better value... */ - erts_garbage_collect_literals(rp, literals, lit_size, oh); + *redsp += erts_garbage_collect_nobump(rp, 0, rp->arg_reg, rp->arity); + done_gc |= ERTS_ORDINARY_GC__; + } + if (need_gc & ERTS_LITERAL_GC__) { + struct erl_off_heap_header* oh; + oh = modp->old.code_hdr->literals_off_heap; + *redsp += lit_bsize / 64; /* Need, better value... */ + erts_garbage_collect_literals(rp, (Eterm*)literals, lit_bsize, oh); + done_gc |= ERTS_LITERAL_GC__; } + need_gc = 0; } - return am_false; -#undef INSIDE -} -#define in_area(ptr,start,nbytes) \ - ((UWord)((char*)(ptr) - (char*)(start)) < (nbytes)) +#undef ERTS_ORDINARY_GC__ +#undef ERTS_LITERAL_GC__ + +} static int any_heap_ref_ptrs(Eterm* start, Eterm* end, char* mod_start, Uint mod_size) @@ -914,7 +984,7 @@ any_heap_ref_ptrs(Eterm* start, Eterm* end, char* mod_start, Uint mod_size) switch (primary_tag(val)) { case TAG_PRIMARY_BOXED: case TAG_PRIMARY_LIST: - if (in_area(EXPAND_POINTER(val), mod_start, mod_size)) { + if (ErtsInArea(val, mod_start, mod_size)) { return 1; } break; @@ -934,13 +1004,21 @@ any_heap_refs(Eterm* start, Eterm* end, char* mod_start, Uint mod_size) switch (primary_tag(val)) { case TAG_PRIMARY_BOXED: case TAG_PRIMARY_LIST: - if (in_area(EXPAND_POINTER(val), mod_start, mod_size)) { + if (ErtsInArea(val, mod_start, mod_size)) { return 1; } break; case TAG_PRIMARY_HEADER: if (!header_is_transparent(val)) { - Eterm* new_p = p + thing_arityval(val); + Eterm* new_p; + if (header_is_bin_matchstate(val)) { + ErlBinMatchState *ms = (ErlBinMatchState*) p; + ErlBinMatchBuffer *mb = &(ms->mb); + if (ErtsInArea(mb->orig, mod_start, mod_size)) { + return 1; + } + } + new_p = p + thing_arityval(val); ASSERT(start <= new_p && new_p < end); p = new_p; } @@ -951,7 +1029,104 @@ any_heap_refs(Eterm* start, Eterm* end, char* mod_start, Uint mod_size) #undef in_area -BIF_RETTYPE purge_module_1(BIF_ALIST_1) +#ifdef ERTS_SMP +static void copy_literals_commit(void*); +#endif + +copy_literals_t erts_clrange = {NULL, 0, THE_NON_VALUE}; + +/* copy literals + * + * copy_literals.ptr = LitPtr + * copy_literals.sz = LitSz + * ------ THR PROG COMMIT ----- + * + * - check process code + * - check process code + * ... + * copy_literals.ptr = NULL + * copy_literals.sz = 0 + * ------ THR PROG COMMIT ----- + * ... + */ + + +BIF_RETTYPE erts_internal_copy_literals_2(BIF_ALIST_2) +{ + ErtsCodeIndex code_ix; + Eterm res = am_true; + + if (is_not_atom(BIF_ARG_1) || (am_true != BIF_ARG_2 && am_false != BIF_ARG_2)) { + BIF_ERROR(BIF_P, BADARG); + } + + if (!erts_try_seize_code_write_permission(BIF_P)) { + ERTS_BIF_YIELD2(bif_export[BIF_erts_internal_copy_literals_2], + BIF_P, BIF_ARG_1, BIF_ARG_2); + } + + code_ix = erts_active_code_ix(); + + if (BIF_ARG_2 == am_true) { + Module* modp = erts_get_module(BIF_ARG_1, code_ix); + if (!modp || !modp->old.code_hdr) { + res = am_false; + goto done; + } + if (erts_clrange.ptr != NULL + && !(BIF_P->static_flags & ERTS_STC_FLG_SYSTEM_PROC)) { + res = am_aborted; + goto done; + } + erts_clrange.ptr = modp->old.code_hdr->literals_start; + erts_clrange.sz = modp->old.code_hdr->literals_end - erts_clrange.ptr; + erts_clrange.pid = BIF_P->common.id; + } else if (BIF_ARG_2 == am_false) { + if (erts_clrange.pid != BIF_P->common.id) { + res = am_false; + goto done; + } + erts_clrange.ptr = NULL; + erts_clrange.sz = 0; + erts_clrange.pid = THE_NON_VALUE; + } + +#ifdef ERTS_SMP + ASSERT(committer_state.stager == NULL); + committer_state.stager = BIF_P; + erts_schedule_thr_prgr_later_op(copy_literals_commit, NULL, &committer_state.lop); + erts_proc_inc_refc(BIF_P); + erts_suspend(BIF_P, ERTS_PROC_LOCK_MAIN, NULL); + ERTS_BIF_YIELD_RETURN(BIF_P, am_true); +#endif +done: + erts_release_code_write_permission(); + BIF_RET(res); +} + +#ifdef ERTS_SMP +static void copy_literals_commit(void* null) { + Process* p = committer_state.stager; +#ifdef DEBUG + committer_state.stager = NULL; +#endif + erts_release_code_write_permission(); + erts_smp_proc_lock(p, ERTS_PROC_LOCK_STATUS); + if (!ERTS_PROC_IS_EXITING(p)) { + erts_resume(p, ERTS_PROC_LOCK_STATUS); + } + erts_smp_proc_unlock(p, ERTS_PROC_LOCK_STATUS); + erts_proc_dec_refc(p); +} +#endif /* ERTS_SMP */ + + +/* Do the actualy module purging and return: + * true for success + * false if no such old module + * BADARG if not an atom + */ +BIF_RETTYPE erts_internal_purge_module_1(BIF_ALIST_1) { ErtsCodeIndex code_ix; BeamInstr* code; @@ -965,7 +1140,8 @@ BIF_RETTYPE purge_module_1(BIF_ALIST_1) } if (!erts_try_seize_code_write_permission(BIF_P)) { - ERTS_BIF_YIELD1(bif_export[BIF_purge_module_1], BIF_P, BIF_ARG_1); + ERTS_BIF_YIELD1(bif_export[BIF_erts_internal_purge_module_1], + BIF_P, BIF_ARG_1); } code_ix = erts_active_code_ix(); @@ -975,7 +1151,7 @@ BIF_RETTYPE purge_module_1(BIF_ALIST_1) */ if ((modp = erts_get_module(BIF_ARG_1, code_ix)) == NULL) { - ERTS_BIF_PREP_ERROR(ret, BIF_P, BADARG); + ERTS_BIF_PREP_RET(ret, am_false); } else { erts_rwlock_old_code(code_ix); @@ -983,8 +1159,8 @@ BIF_RETTYPE purge_module_1(BIF_ALIST_1) /* * Any code to purge? */ - if (modp->old.code == 0) { - ERTS_BIF_PREP_ERROR(ret, BIF_P, BADARG); + if (!modp->old.code_hdr) { + ERTS_BIF_PREP_RET(ret, am_false); } else { /* @@ -1006,14 +1182,17 @@ BIF_RETTYPE purge_module_1(BIF_ALIST_1) */ ASSERT(erts_total_code_size >= modp->old.code_length); erts_total_code_size -= modp->old.code_length; - code = modp->old.code; + code = (BeamInstr*) modp->old.code_hdr; end = (BeamInstr *)((char *)code + modp->old.code_length); erts_cleanup_funs_on_purge(code, end); beam_catches_delmod(modp->old.catches, code, modp->old.code_length, code_ix); - decrement_refc(code); + decrement_refc(modp->old.code_hdr); + if (modp->old.code_hdr->literals_start) { + erts_free(ERTS_ALC_T_LITERAL, modp->old.code_hdr->literals_start); + } erts_free(ERTS_ALC_T_CODE, (void *) code); - modp->old.code = NULL; + modp->old.code_hdr = NULL; modp->old.code_length = 0; modp->old.catches = BEAM_CATCHES_NIL; erts_remove_from_ranges(code); @@ -1030,10 +1209,9 @@ BIF_RETTYPE purge_module_1(BIF_ALIST_1) } static void -decrement_refc(BeamInstr* code) +decrement_refc(BeamCodeHeader* code_hdr) { - struct erl_off_heap_header* oh = - (struct erl_off_heap_header *) code[MI_LITERALS_OFF_HEAP]; + struct erl_off_heap_header* oh = code_hdr->literals_off_heap; while (oh) { Binary* bptr; @@ -1082,7 +1260,7 @@ delete_code(Module* modp) ASSERT(modp->curr.num_breakpoints == 0); ASSERT(modp->curr.num_traced_exports == 0); modp->old = modp->curr; - modp->curr.code = NULL; + modp->curr.code_hdr = NULL; modp->curr.code_length = 0; modp->curr.catches = BEAM_CATCHES_NIL; modp->curr.nif = NULL; @@ -1099,32 +1277,10 @@ beam_make_current_old(Process *c_p, ErtsProcLocks c_p_locks, Eterm module) * if not, delete old code; error if old code already exists. */ - if (modp->curr.code != NULL && modp->old.code != NULL) { + if (modp->curr.code_hdr && modp->old.code_hdr) { return am_not_purged; - } else if (modp->old.code == NULL) { /* Make the current version old. */ + } else if (!modp->old.code_hdr) { /* Make the current version old. */ delete_code(modp); } return NIL; } - -static int -is_native(BeamInstr* code) -{ - Uint i, num_functions = code[MI_NUM_FUNCTIONS]; - - /* Check NativeAdress of first real function in module - */ - for (i=0; i<num_functions; i++) { - BeamInstr* func_info = (BeamInstr *) code[MI_FUNCTIONS+i]; - Eterm name = (Eterm) func_info[3]; - - if (is_atom(name)) { - return func_info[1] != 0; - } - else ASSERT(is_nil(name)); /* ignore BIF stubs */ - } - /* Not a single non-BIF function? */ - return 0; -} - - diff --git a/erts/emulator/beam/beam_bp.c b/erts/emulator/beam/beam_bp.c index 4e711c89e0..74c9d3ee53 100644 --- a/erts/emulator/beam/beam_bp.c +++ b/erts/emulator/beam/beam_bp.c @@ -3,16 +3,17 @@ * * Copyright Ericsson AB 2000-2013. All Rights Reserved. * - * The contents of this file are subject to the Erlang Public License, - * Version 1.1, (the "License"); you may not use this file except in - * compliance with the License. You should have received a copy of the - * Erlang Public License along with this software. If not, it can be - * retrieved online at http://www.erlang.org/. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Software distributed under the License is distributed on an "AS IS" - * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See - * the License for the specific language governing rights and limitations - * under the License. + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. * * %CopyrightEnd% */ @@ -74,6 +75,16 @@ extern BeamInstr beam_return_time_trace[1]; /* OpCode(i_return_time_trace) */ erts_smp_atomic32_t erts_active_bp_index; erts_smp_atomic32_t erts_staging_bp_index; +/* + * Inlined helpers + */ + +static ERTS_INLINE ErtsMonotonicTime +get_mtime(Process *c_p) +{ + return erts_get_monotonic_time(ERTS_PROC_GET_SCHDATA(c_p)); +} + /* ************************************************************************* ** Local prototypes */ @@ -96,9 +107,6 @@ static int clear_function_break(BeamInstr *pc, Uint break_flags); static BpDataTime* get_time_break(BeamInstr *pc); static GenericBpData* check_break(BeamInstr *pc, Uint break_flags); -static void bp_time_diff(bp_data_time_item_t *item, - process_breakpoint_time_t *pbt, - Uint ms, Uint s, Uint us); static void bp_meta_unref(BpMetaPid* bmp); static void bp_count_unref(BpCount* bcp); @@ -109,13 +117,8 @@ static void uninstall_breakpoint(BeamInstr* pc); /* bp_hash */ #define BP_TIME_ADD(pi0, pi1) \ do { \ - Uint r; \ (pi0)->count += (pi1)->count; \ - (pi0)->s_time += (pi1)->s_time; \ - (pi0)->us_time += (pi1)->us_time; \ - r = (pi0)->us_time / 1000000; \ - (pi0)->s_time += r; \ - (pi0)->us_time = (pi0)->us_time % 1000000; \ + (pi0)->time += (pi1)->time; \ } while(0) static void bp_hash_init(bp_time_hash_t *hash, Uint n); @@ -151,8 +154,8 @@ erts_bp_match_functions(BpFunctions* f, Eterm mfa[3], int specified) num_modules = 0; for (current = 0; current < max_modules; current++) { modp = module_code(current, code_ix); - if (modp->curr.code) { - max_funcs += modp->curr.code[MI_NUM_FUNCTIONS]; + if (modp->curr.code_hdr) { + max_funcs += modp->curr.code_hdr->num_functions; module[num_modules++] = modp; } } @@ -160,9 +163,9 @@ erts_bp_match_functions(BpFunctions* f, Eterm mfa[3], int specified) f->matching = (BpFunction *) Alloc(max_funcs*sizeof(BpFunction)); i = 0; for (current = 0; current < num_modules; current++) { - BeamInstr** code_base = (BeamInstr **) module[current]->curr.code; + BeamCodeHeader* code_hdr = module[current]->curr.code_hdr; BeamInstr* code; - Uint num_functions = (Uint)(UWord) code_base[MI_NUM_FUNCTIONS]; + Uint num_functions = (Uint)(UWord) code_hdr->num_functions; Uint fi; if (specified > 0) { @@ -176,7 +179,7 @@ erts_bp_match_functions(BpFunctions* f, Eterm mfa[3], int specified) BeamInstr* pc; int wi; - code = code_base[MI_FUNCTIONS+fi]; + code = code_hdr->functions[fi]; ASSERT(code[0] == (BeamInstr) BeamOp(op_i_func_info_IaaI)); pc = code+5; if (erts_is_native_break(pc)) { @@ -546,21 +549,21 @@ erts_clear_all_breaks(BpFunctions* f) int erts_clear_module_break(Module *modp) { - BeamInstr** code_base; + BeamCodeHeader* code_hdr; Uint n; Uint i; ERTS_SMP_LC_ASSERT(erts_smp_thr_progress_is_blocking()); ASSERT(modp); - code_base = (BeamInstr **) modp->curr.code; - if (code_base == NULL) { + code_hdr = modp->curr.code_hdr; + if (!code_hdr) { return 0; } - n = (Uint)(UWord) code_base[MI_NUM_FUNCTIONS]; + n = (Uint)(UWord) code_hdr->num_functions; for (i = 0; i < n; ++i) { BeamInstr* pc; - pc = code_base[MI_FUNCTIONS+i] + 5; + pc = code_hdr->functions[i] + 5; if (erts_is_native_break(pc)) { continue; } @@ -572,7 +575,7 @@ erts_clear_module_break(Module *modp) { for (i = 0; i < n; ++i) { BeamInstr* pc; - pc = code_base[MI_FUNCTIONS+i] + 5; + pc = code_hdr->functions[i] + 5; if (erts_is_native_break(pc)) { continue; } @@ -861,7 +864,7 @@ do_call_trace(Process* c_p, BeamInstr* I, Eterm* reg, Eterm* cpp; int return_to_trace = 0; BeamInstr w; - BeamInstr *cp_save; + BeamInstr *cp_save = c_p->cp; Uint32 flags; Uint need = 0; Eterm* E = c_p->stop; @@ -947,7 +950,7 @@ do_call_trace(Process* c_p, BeamInstr* I, Eterm* reg, void erts_trace_time_call(Process* c_p, BeamInstr* I, BpDataTime* bdt) { - Uint ms,s,us; + ErtsMonotonicTime time; process_breakpoint_time_t *pbt = NULL; bp_data_time_item_t sitem, *item = NULL; bp_time_hash_t *h = NULL; @@ -960,7 +963,7 @@ erts_trace_time_call(Process* c_p, BeamInstr* I, BpDataTime* bdt) * from the process psd */ pbt = ERTS_PROC_GET_CALL_TIME(c_p); - get_sys_now(&ms, &s, &us); + time = get_mtime(c_p); /* get pbt * timestamp = t0 @@ -971,11 +974,11 @@ erts_trace_time_call(Process* c_p, BeamInstr* I, BpDataTime* bdt) if (pbt == 0) { /* First call of process to instrumented function */ pbt = Alloc(sizeof(process_breakpoint_time_t)); - (void) ERTS_PROC_SET_CALL_TIME(c_p, ERTS_PROC_LOCK_MAIN, pbt); + (void) ERTS_PROC_SET_CALL_TIME(c_p, pbt); } else { ASSERT(pbt->pc); /* add time to previous code */ - bp_time_diff(&sitem, pbt, ms, s, us); + sitem.time = time - pbt->time; sitem.pid = c_p->common.id; sitem.count = 0; @@ -1001,8 +1004,7 @@ erts_trace_time_call(Process* c_p, BeamInstr* I, BpDataTime* bdt) /* Add count to this code */ sitem.pid = c_p->common.id; sitem.count = 1; - sitem.s_time = 0; - sitem.us_time = 0; + sitem.time = 0; /* this breakpoint */ ASSERT(bdt); @@ -1019,15 +1021,13 @@ erts_trace_time_call(Process* c_p, BeamInstr* I, BpDataTime* bdt) } pbt->pc = I; - pbt->ms = ms; - pbt->s = s; - pbt->us = us; + pbt->time = time; } void erts_trace_time_return(Process *p, BeamInstr *pc) { - Uint ms,s,us; + ErtsMonotonicTime time; process_breakpoint_time_t *pbt = NULL; bp_data_time_item_t sitem, *item = NULL; bp_time_hash_t *h = NULL; @@ -1040,7 +1040,7 @@ erts_trace_time_return(Process *p, BeamInstr *pc) * from the process psd */ pbt = ERTS_PROC_GET_CALL_TIME(p); - get_sys_now(&ms,&s,&us); + time = get_mtime(p); /* get pbt * lookup bdt from code @@ -1056,7 +1056,7 @@ erts_trace_time_return(Process *p, BeamInstr *pc) */ ASSERT(pbt->pc); - bp_time_diff(&sitem, pbt, ms, s, us); + sitem.time = time - pbt->time; sitem.pid = p->common.id; sitem.count = 0; @@ -1079,9 +1079,7 @@ erts_trace_time_return(Process *p, BeamInstr *pc) } pbt->pc = pc; - pbt->ms = ms; - pbt->s = s; - pbt->us = us; + pbt->time = time; } } @@ -1182,10 +1180,14 @@ int erts_is_time_break(Process *p, BeamInstr *pc, Eterm *retval) { for(ix = 0; ix < hash.n; ix++) { item = &(hash.item[ix]); if (item->pid != NIL) { + ErtsMonotonicTime sec, usec; + usec = ERTS_MONOTONIC_TO_USEC(item->time); + sec = usec / 1000000; + usec = usec - sec*1000000; t = TUPLE4(hp, item->pid, make_small(item->count), - make_small(item->s_time), - make_small(item->us_time)); + make_small((Uint) sec), + make_small((Uint) usec)); hp += 5; *retval = CONS(hp, t, *retval); hp += 2; } @@ -1203,17 +1205,17 @@ int erts_is_time_break(Process *p, BeamInstr *pc, Eterm *retval) { BeamInstr * erts_find_local_func(Eterm mfa[3]) { Module *modp; - BeamInstr** code_base; + BeamCodeHeader* code_hdr; BeamInstr* code_ptr; Uint i,n; if ((modp = erts_get_module(mfa[0], erts_active_code_ix())) == NULL) return NULL; - if ((code_base = (BeamInstr **) modp->curr.code) == NULL) + if ((code_hdr = modp->curr.code_hdr) == NULL) return NULL; - n = (BeamInstr) code_base[MI_NUM_FUNCTIONS]; + n = (BeamInstr) code_hdr->num_functions; for (i = 0; i < n; ++i) { - code_ptr = code_base[MI_FUNCTIONS+i]; + code_ptr = code_hdr->functions[i]; ASSERT(((BeamInstr) BeamOp(op_i_func_info_IaaI)) == code_ptr[0]); ASSERT(mfa[0] == ((Eterm) code_ptr[2]) || is_nil((Eterm) code_ptr[2])); @@ -1265,8 +1267,7 @@ static void bp_hash_rehash(bp_time_hash_t *hash, Uint n) { } item[hval].pid = hash->item[ix].pid; item[hval].count = hash->item[ix].count; - item[hval].s_time = hash->item[ix].s_time; - item[hval].us_time = hash->item[ix].us_time; + item[hval].time = hash->item[ix].time; } } @@ -1314,8 +1315,7 @@ static ERTS_INLINE bp_data_time_item_t * bp_hash_put(bp_time_hash_t *hash, bp_da item = &(hash->item[hval]); item->pid = sitem->pid; - item->s_time = sitem->s_time; - item->us_time = sitem->us_time; + item->time = sitem->time; item->count = sitem->count; hash->used++; @@ -1329,41 +1329,7 @@ static void bp_hash_delete(bp_time_hash_t *hash) { hash->item = NULL; } -static void bp_time_diff(bp_data_time_item_t *item, /* out */ - process_breakpoint_time_t *pbt, /* in */ - Uint ms, Uint s, Uint us) { - int ds,dus; -#ifdef DEBUG - int dms; - - - dms = ms - pbt->ms; -#endif - ds = s - pbt->s; - dus = us - pbt->us; - - /* get_sys_now may return zero difftime, - * this is ok. - */ - -#ifdef DEBUG - ASSERT(dms >= 0 || ds >= 0 || dus >= 0); -#endif - - if (dus < 0) { - dus += 1000000; - ds -= 1; - } - if (ds < 0) { - ds += 1000000; - } - - item->s_time = ds; - item->us_time = dus; -} - void erts_schedule_time_break(Process *p, Uint schedule) { - Uint ms, s, us; process_breakpoint_time_t *pbt = NULL; bp_data_time_item_t sitem, *item = NULL; bp_time_hash_t *h = NULL; @@ -1386,8 +1352,7 @@ void erts_schedule_time_break(Process *p, Uint schedule) { pbdt = get_time_break(pbt->pc); if (pbdt) { - get_sys_now(&ms,&s,&us); - bp_time_diff(&sitem, pbt, ms, s, us); + sitem.time = get_mtime(p) - pbt->time; sitem.pid = p->common.id; sitem.count = 0; @@ -1409,10 +1374,7 @@ void erts_schedule_time_break(Process *p, Uint schedule) { * timestamp it and remove the previous * timestamp in the psd. */ - get_sys_now(&ms,&s,&us); - pbt->ms = ms; - pbt->s = s; - pbt->us = us; + pbt->time = get_mtime(p); break; default : ASSERT(0); @@ -1636,9 +1598,7 @@ bp_time_unref(BpDataTime* bdt) h_p = erts_pid2proc(NULL, 0, item->pid, ERTS_PROC_LOCK_MAIN); if (h_p) { - pbt = ERTS_PROC_SET_CALL_TIME(h_p, - ERTS_PROC_LOCK_MAIN, - NULL); + pbt = ERTS_PROC_SET_CALL_TIME(h_p, NULL); if (pbt) { Free(pbt); } diff --git a/erts/emulator/beam/beam_bp.h b/erts/emulator/beam/beam_bp.h index b061401863..2b89d6fc71 100644 --- a/erts/emulator/beam/beam_bp.h +++ b/erts/emulator/beam/beam_bp.h @@ -3,16 +3,17 @@ * * Copyright Ericsson AB 2000-2013. All Rights Reserved. * - * The contents of this file are subject to the Erlang Public License, - * Version 1.1, (the "License"); you may not use this file except in - * compliance with the License. You should have received a copy of the - * Erlang Public License along with this software. If not, it can be - * retrieved online at http://www.erlang.org/. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Software distributed under the License is distributed on an "AS IS" - * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See - * the License for the specific language governing rights and limitations - * under the License. + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. * * %CopyrightEnd% */ @@ -28,8 +29,7 @@ typedef struct { Eterm pid; Sint count; - Uint s_time; - Uint us_time; + ErtsMonotonicTime time; } bp_data_time_item_t; typedef struct { @@ -45,9 +45,7 @@ typedef struct bp_data_time { /* Call time */ } BpDataTime; typedef struct { - Uint ms; - Uint s; - Uint us; + ErtsMonotonicTime time; BeamInstr *pc; } process_breakpoint_time_t; /* used within psd */ diff --git a/erts/emulator/beam/beam_catches.c b/erts/emulator/beam/beam_catches.c index d374d0469e..7a1f4901aa 100644 --- a/erts/emulator/beam/beam_catches.c +++ b/erts/emulator/beam/beam_catches.c @@ -3,16 +3,17 @@ * * Copyright Ericsson AB 2000-2013. All Rights Reserved. * - * The contents of this file are subject to the Erlang Public License, - * Version 1.1, (the "License"); you may not use this file except in - * compliance with the License. You should have received a copy of the - * Erlang Public License along with this software. If not, it can be - * retrieved online at http://www.erlang.org/. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Software distributed under the License is distributed on an "AS IS" - * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See - * the License for the specific language governing rights and limitations - * under the License. + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. * * %CopyrightEnd% */ @@ -142,7 +143,7 @@ BeamInstr *beam_catches_car(unsigned i) struct bc_pool* p = &bccix[erts_active_code_ix()]; if (i >= p->tabsize ) { - erl_exit(1, "beam_catches_delmod: index %#x is out of range\r\n", i); + erts_exit(ERTS_ERROR_EXIT, "beam_catches_delmod: index %#x is out of range\r\n", i); } return p->beam_catches[i].cp; } @@ -156,10 +157,10 @@ void beam_catches_delmod(unsigned head, BeamInstr *code, unsigned code_bytes, ASSERT((code_ix == erts_active_code_ix()) != bccix[erts_staging_code_ix()].is_staging); for(i = head; i != (unsigned)-1;) { if (i >= p->tabsize) { - erl_exit(1, "beam_catches_delmod: index %#x is out of range\r\n", i); + erts_exit(ERTS_ERROR_EXIT, "beam_catches_delmod: index %#x is out of range\r\n", i); } if( (char*)p->beam_catches[i].cp - (char*)code >= code_bytes ) { - erl_exit(1, + erts_exit(ERTS_ERROR_EXIT, "beam_catches_delmod: item %#x has cp %p which is not " "in module's range [%p,%p[\r\n", i, p->beam_catches[i].cp, code, ((char*)code + code_bytes)); diff --git a/erts/emulator/beam/beam_catches.h b/erts/emulator/beam/beam_catches.h index 51ef463b2f..59ee64d033 100644 --- a/erts/emulator/beam/beam_catches.h +++ b/erts/emulator/beam/beam_catches.h @@ -3,16 +3,17 @@ * * Copyright Ericsson AB 2000-2013. All Rights Reserved. * - * The contents of this file are subject to the Erlang Public License, - * Version 1.1, (the "License"); you may not use this file except in - * compliance with the License. You should have received a copy of the - * Erlang Public License along with this software. If not, it can be - * retrieved online at http://www.erlang.org/. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Software distributed under the License is distributed on an "AS IS" - * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See - * the License for the specific language governing rights and limitations - * under the License. + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. * * %CopyrightEnd% */ diff --git a/erts/emulator/beam/beam_debug.c b/erts/emulator/beam/beam_debug.c index a3cd08834f..e37bd4d78c 100644 --- a/erts/emulator/beam/beam_debug.c +++ b/erts/emulator/beam/beam_debug.c @@ -3,16 +3,17 @@ * * Copyright Ericsson AB 1998-2013. All Rights Reserved. * - * The contents of this file are subject to the Erlang Public License, - * Version 1.1, (the "License"); you may not use this file except in - * compliance with the License. You should have received a copy of the - * Erlang Public License along with this software. If not, it can be - * retrieved online at http://www.erlang.org/. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Software distributed under the License is distributed on an "AS IS" - * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See - * the License for the specific language governing rights and limitations - * under the License. + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. * * %CopyrightEnd% */ @@ -72,13 +73,47 @@ erts_debug_flat_size_1(BIF_ALIST_1) } } +BIF_RETTYPE +erts_debug_size_shared_1(BIF_ALIST_1) +{ + Process* p = BIF_P; + Eterm term = BIF_ARG_1; + Uint size = size_shared(term); + + if (IS_USMALL(0, size)) { + BIF_RET(make_small(size)); + } else { + Eterm* hp = HAlloc(p, BIG_UINT_HEAP_SIZE); + BIF_RET(uint_to_big(size, hp)); + } +} + +BIF_RETTYPE +erts_debug_copy_shared_1(BIF_ALIST_1) +{ + Process* p = BIF_P; + Eterm term = BIF_ARG_1; + Uint size; + Eterm* hp; + Eterm copy; + erts_shcopy_t info; + INITIALIZE_SHCOPY(info); + + size = copy_shared_calculate(term, &info); + if (size > 0) { + hp = HAlloc(p, size); + } + copy = copy_shared_perform(term, size, &info, &hp, &p->off_heap); + DESTROY_SHCOPY(info); + BIF_RET(copy); +} BIF_RETTYPE erts_debug_breakpoint_2(BIF_ALIST_2) { Process* p = BIF_P; Eterm MFA = BIF_ARG_1; - Eterm bool = BIF_ARG_2; + Eterm boolean = BIF_ARG_2; Eterm* tp; Eterm mfa[3]; int i; @@ -86,7 +121,7 @@ erts_debug_breakpoint_2(BIF_ALIST_2) Eterm res; BpFunctions f; - if (bool != am_true && bool != am_false) + if (boolean != am_true && boolean != am_false) goto error; if (is_not_tuple(MFA)) { @@ -123,7 +158,7 @@ erts_debug_breakpoint_2(BIF_ALIST_2) erts_smp_thr_progress_block(); erts_bp_match_functions(&f, mfa, specified); - if (bool == am_true) { + if (boolean == am_true) { erts_set_debug_break(&f); erts_install_breakpoints(&f); erts_commit_staged_bp(); @@ -207,7 +242,7 @@ erts_debug_disassemble_1(BIF_ALIST_1) Eterm bin; Eterm mfa; BeamInstr* funcinfo = NULL; /* Initialized to eliminate warning. */ - BeamInstr* code_base; + BeamCodeHeader* code_hdr; BeamInstr* code_ptr = NULL; /* Initialized to eliminate warning. */ BeamInstr instr; BeamInstr uaddr; @@ -257,12 +292,12 @@ erts_debug_disassemble_1(BIF_ALIST_1) */ code_ptr = ((BeamInstr *) ep->addressv[code_ix]) - 5; funcinfo = code_ptr+2; - } else if (modp == NULL || (code_base = modp->curr.code) == NULL) { + } else if (modp == NULL || (code_hdr = modp->curr.code_hdr) == NULL) { BIF_RET(am_undef); } else { - n = code_base[MI_NUM_FUNCTIONS]; + n = code_hdr->num_functions; for (i = 0; i < n; i++) { - code_ptr = (BeamInstr *) code_base[MI_FUNCTIONS+i]; + code_ptr = code_hdr->functions[i]; if (code_ptr[3] == name && code_ptr[4] == arity) { funcinfo = code_ptr+2; break; @@ -297,8 +332,8 @@ erts_debug_disassemble_1(BIF_ALIST_1) (void) erts_bld_uword(NULL, &hsz, (BeamInstr) code_ptr); hp = HAlloc(p, hsz); addr = erts_bld_uword(&hp, NULL, (BeamInstr) code_ptr); - ASSERT(is_atom(funcinfo[0])); - ASSERT(is_atom(funcinfo[1])); + ASSERT(is_atom(funcinfo[0]) || funcinfo[0] == NIL); + ASSERT(is_atom(funcinfo[1]) || funcinfo[1] == NIL); mfa = TUPLE3(hp, (Eterm) funcinfo[0], (Eterm) funcinfo[1], make_small((Eterm) funcinfo[2])); hp += 4; return TUPLE3(hp, addr, bin, mfa); @@ -431,39 +466,33 @@ print_op(int to, void *to_arg, int op, int size, BeamInstr* addr) while (*sign) { switch (*sign) { case 'r': /* x(0) */ - erts_print(to, to_arg, "x(0)"); + erts_print(to, to_arg, "r(0)"); break; case 'x': /* x(N) */ - if (reg_index(ap[0]) == 0) { - erts_print(to, to_arg, "x[0]"); - } else { - erts_print(to, to_arg, "x(%d)", reg_index(ap[0])); + { + Uint n = ap[0] / sizeof(Eterm); + erts_print(to, to_arg, "x(%d)", n); + ap++; } - ap++; break; case 'y': /* y(N) */ - erts_print(to, to_arg, "y(%d)", reg_index(ap[0]) - CP_SIZE); - ap++; + { + Uint n = ap[0] / sizeof(Eterm) - CP_SIZE; + erts_print(to, to_arg, "y(%d)", n); + ap++; + } break; case 'n': /* Nil */ erts_print(to, to_arg, "[]"); break; case 's': /* Any source (tagged constant or register) */ - tag = beam_reg_tag(*ap); - if (tag == X_REG_DEF) { - if (reg_index(*ap) == 0) { - erts_print(to, to_arg, "x[0]"); - } else { - erts_print(to, to_arg, "x(%d)", reg_index(*ap)); - } - ap++; - break; - } else if (tag == Y_REG_DEF) { - erts_print(to, to_arg, "y(%d)", reg_index(*ap) - CP_SIZE); + tag = loader_tag(*ap); + if (tag == LOADER_X_REG) { + erts_print(to, to_arg, "x(%d)", loader_x_reg_index(*ap)); ap++; break; - } else if (tag == R_REG_DEF) { - erts_print(to, to_arg, "x(0)"); + } else if (tag == LOADER_Y_REG) { + erts_print(to, to_arg, "y(%d)", loader_y_reg_index(*ap) - CP_SIZE); ap++; break; } @@ -480,20 +509,12 @@ print_op(int to, void *to_arg, int op, int size, BeamInstr* addr) ap++; break; case 'd': /* Destination (x(0), x(N), y(N)) */ - switch (beam_reg_tag(*ap)) { - case X_REG_DEF: - if (reg_index(*ap) == 0) { - erts_print(to, to_arg, "x[0]"); - } else { - erts_print(to, to_arg, "x(%d)", reg_index(*ap)); - } - break; - case Y_REG_DEF: - erts_print(to, to_arg, "y(%d)", reg_index(*ap) - CP_SIZE); - break; - case R_REG_DEF: - erts_print(to, to_arg, "x(0)"); - break; + if (*ap & 1) { + erts_print(to, to_arg, "y(%d)", + *ap / sizeof(Eterm) - CP_SIZE); + } else { + erts_print(to, to_arg, "x(%d)", + *ap / sizeof(Eterm)); } ap++; break; @@ -560,7 +581,7 @@ print_op(int to, void *to_arg, int op, int size, BeamInstr* addr) ap++; break; case 'l': /* fr(N) */ - erts_print(to, to_arg, "fr(%d)", reg_index(ap[0])); + erts_print(to, to_arg, "fr(%d)", loader_reg_index(ap[0])); ap++; break; default: @@ -579,36 +600,62 @@ print_op(int to, void *to_arg, int op, int size, BeamInstr* addr) unpacked = ap; ap = addr + size; switch (op) { - case op_i_select_val_rfI: - case op_i_select_val_xfI: - case op_i_select_val_yfI: + case op_i_select_val_lins_xfI: + case op_i_select_val_lins_yfI: { int n = ap[-1]; + int ix = n; - while (n > 0) { - erts_print(to, to_arg, "%T f(" HEXF ") ", (Eterm) ap[0], ap[1]); - ap += 2; - size += 2; - n--; + while (ix--) { + erts_print(to, to_arg, "%T ", (Eterm) ap[0]); + ap++; + size++; + } + ix = n; + while (ix--) { + erts_print(to, to_arg, "f(" HEXF ") ", (Eterm) ap[0]); + ap++; + size++; } } break; - case op_i_select_tuple_arity_rfI: - case op_i_select_tuple_arity_xfI: - case op_i_select_tuple_arity_yfI: + case op_i_select_val_bins_xfI: + case op_i_select_val_bins_yfI: { int n = ap[-1]; while (n > 0) { - Uint arity = arityval(ap[0]); - erts_print(to, to_arg, " {%d} f(" HEXF ")", arity, ap[1]); + erts_print(to, to_arg, "%T f(" HEXF ") ", (Eterm) ap[0], ap[1]); ap += 2; size += 2; n--; } } break; - case op_i_jump_on_val_rfII: + case op_i_select_tuple_arity_xfI: + case op_i_select_tuple_arity_yfI: + { + int n = ap[-1]; + int ix = n - 1; /* without sentinel */ + + while (ix--) { + Uint arity = arityval(ap[0]); + erts_print(to, to_arg, "{%d} ", arity, ap[1]); + ap++; + size++; + } + /* print sentinel */ + erts_print(to, to_arg, "{%T} ", ap[0], ap[1]); + ap++; + size++; + ix = n; + while (ix--) { + erts_print(to, to_arg, "f(" HEXF ") ", ap[0]); + ap++; + size++; + } + } + break; case op_i_jump_on_val_xfII: case op_i_jump_on_val_yfII: { @@ -620,7 +667,6 @@ print_op(int to, void *to_arg, int op, int size, BeamInstr* addr) } } break; - case op_i_jump_on_val_zero_rfI: case op_i_jump_on_val_zero_xfI: case op_i_jump_on_val_zero_yfI: { @@ -632,30 +678,47 @@ print_op(int to, void *to_arg, int op, int size, BeamInstr* addr) } } break; - case op_i_put_tuple_rI: case op_i_put_tuple_xI: case op_i_put_tuple_yI: - case op_new_map_jdII: + case op_new_map_dII: case op_update_map_assoc_jsdII: case op_update_map_exact_jsdII: - case op_i_has_map_fields_fsI: - case op_i_get_map_elements_fsI: { int n = unpacked[-1]; while (n > 0) { - if (!is_header(ap[0])) { + switch (loader_tag(ap[0])) { + case LOADER_X_REG: + erts_print(to, to_arg, " x(%d)", loader_x_reg_index(ap[0])); + break; + case LOADER_Y_REG: + erts_print(to, to_arg, " x(%d)", loader_y_reg_index(ap[0])); + break; + default: erts_print(to, to_arg, " %T", (Eterm) ap[0]); + break; + } + ap++, size++, n--; + } + } + break; + case op_i_get_map_elements_fsI: + { + int n = unpacked[-1]; + + while (n > 0) { + if (n % 3 == 1) { + erts_print(to, to_arg, " %X", ap[0]); } else { - switch ((ap[0] >> 2) & 0x03) { - case R_REG_DEF: - erts_print(to, to_arg, " x(0)"); + switch (loader_tag(ap[0])) { + case LOADER_X_REG: + erts_print(to, to_arg, " x(%d)", loader_x_reg_index(ap[0])); break; - case X_REG_DEF: - erts_print(to, to_arg, " x(%d)", ap[0] >> 4); + case LOADER_Y_REG: + erts_print(to, to_arg, " y(%d)", loader_y_reg_index(ap[0])); break; - case Y_REG_DEF: - erts_print(to, to_arg, " y(%d)", ap[0] >> 4); + default: + erts_print(to, to_arg, " %T", (Eterm) ap[0]); break; } } diff --git a/erts/emulator/beam/beam_emu.c b/erts/emulator/beam/beam_emu.c index 1026e5f649..a390422040 100644 --- a/erts/emulator/beam/beam_emu.c +++ b/erts/emulator/beam/beam_emu.c @@ -3,16 +3,17 @@ * * Copyright Ericsson AB 1996-2014. All Rights Reserved. * - * The contents of this file are subject to the Erlang Public License, - * Version 1.1, (the "License"); you may not use this file except in - * compliance with the License. You should have received a copy of the - * Erlang Public License along with this software. If not, it can be - * retrieved online at http://www.erlang.org/. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Software distributed under the License is distributed on an "AS IS" - * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See - * the License for the specific language governing rights and limitations - * under the License. + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. * * %CopyrightEnd% */ @@ -98,10 +99,7 @@ do { \ do { \ int i_; \ int Arity_ = PC[-1]; \ - if (Arity_ > 0) { \ - CHECK_TERM(r(0)); \ - } \ - for (i_ = 1; i_ < Arity_; i_++) { \ + for (i_ = 0; i_ < Arity_; i_++) { \ CHECK_TERM(x(i_)); \ } \ } while (0) @@ -115,6 +113,9 @@ do { \ #define MAX(x, y) (((x) > (y)) ? (x) : (y)) #endif +#define GET_BIF_MODULE(p) ((Eterm) (((Export *) p)->code[0])) +#define GET_BIF_FUNCTION(p) ((Eterm) (((Export *) p)->code[1])) +#define GET_BIF_ARITY(p) ((Eterm) (((Export *) p)->code[2])) #define GET_BIF_ADDRESS(p) ((BifFunction) (((Export *) p)->code[4])) #define TermWords(t) (((t) / (sizeof(BeamInstr)/sizeof(Eterm))) + !!((t) % (sizeof(BeamInstr)/sizeof(Eterm)))) @@ -150,25 +151,21 @@ do { \ ASSERT(VALID_INSTR(* (Eterm *)(ip))); \ I = (ip) -#define FetchArgs(S1, S2) tmp_arg1 = (S1); tmp_arg2 = (S2) +/* + * Register target (X or Y register). + */ +#define REG_TARGET(Target) (*(((Target) & 1) ? &yb(Target-1) : &xb(Target))) /* * Store a result into a register given a destination descriptor. */ -#define StoreResult(Result, DestDesc) \ - do { \ - Eterm stb_reg; \ - stb_reg = (DestDesc); \ - CHECK_TERM(Result); \ - switch (beam_reg_tag(stb_reg)) { \ - case R_REG_DEF: \ - r(0) = (Result); break; \ - case X_REG_DEF: \ - xb(x_reg_offset(stb_reg)) = (Result); break; \ - default: \ - yb(y_reg_offset(stb_reg)) = (Result); break; \ - } \ +#define StoreResult(Result, DestDesc) \ + do { \ + Eterm stb_reg; \ + stb_reg = (DestDesc); \ + CHECK_TERM(Result); \ + REG_TARGET(stb_reg) = (Result); \ } while (0) #define StoreSimpleDest(Src, Dest) Dest = (Src) @@ -179,22 +176,16 @@ do { \ * be just before the next instruction. */ -#define StoreBifResult(Dst, Result) \ - do { \ - BeamInstr* stb_next; \ - Eterm stb_reg; \ - stb_reg = Arg(Dst); \ - I += (Dst) + 2; \ - stb_next = (BeamInstr *) *I; \ - CHECK_TERM(Result); \ - switch (beam_reg_tag(stb_reg)) { \ - case R_REG_DEF: \ - r(0) = (Result); Goto(stb_next); \ - case X_REG_DEF: \ - xb(x_reg_offset(stb_reg)) = (Result); Goto(stb_next); \ - default: \ - yb(y_reg_offset(stb_reg)) = (Result); Goto(stb_next); \ - } \ +#define StoreBifResult(Dst, Result) \ + do { \ + BeamInstr* stb_next; \ + Eterm stb_reg; \ + stb_reg = Arg(Dst); \ + I += (Dst) + 2; \ + stb_next = (BeamInstr *) *I; \ + CHECK_TERM(Result); \ + REG_TARGET(stb_reg) = (Result); \ + Goto(stb_next); \ } while (0) #define ClauseFail() goto jump_f @@ -241,10 +232,6 @@ BeamInstr beam_return_time_trace[1]; /* OpCode(i_return_time_trace) */ void** beam_ops; #endif -#ifndef ERTS_SMP /* Not supported with smp emulator */ -extern int count_instructions; -#endif - #define SWAPIN \ HTOP = HEAP_TOP(c_p); \ E = c_p->stop @@ -253,6 +240,14 @@ extern int count_instructions; HEAP_TOP(c_p) = HTOP; \ c_p->stop = E +#define HEAVY_SWAPIN \ + SWAPIN; \ + FCALLS = c_p->fcalls + +#define HEAVY_SWAPOUT \ + SWAPOUT; \ + c_p->fcalls = FCALLS + /* * Use LIGHT_SWAPOUT when the called function * will call HeapOnlyAlloc() (and never HAlloc()). @@ -296,7 +291,7 @@ extern int count_instructions; #define Ib(N) (N) #define x(N) reg[N] #define y(N) E[N] -#define r(N) x##N +#define r(N) x(N) /* * Makes sure that there are StackNeed + HeapNeed + 1 words available @@ -312,12 +307,10 @@ extern int count_instructions; needed = (StackNeed) + 1; \ if (E - HTOP < (needed + (HeapNeed))) { \ SWAPOUT; \ - reg[0] = r(0); \ PROCESS_MAIN_CHK_LOCKS(c_p); \ - FCALLS -= erts_garbage_collect(c_p, needed + (HeapNeed), reg, (M)); \ + FCALLS -= erts_garbage_collect_nobump(c_p, needed + (HeapNeed), reg, (M)); \ ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p); \ PROCESS_MAIN_CHK_LOCKS(c_p); \ - r(0) = reg[0]; \ SWAPIN; \ } \ E -= needed; \ @@ -366,12 +359,10 @@ extern int count_instructions; unsigned need = (Nh); \ if ((E - HTOP < need) || (MSO(c_p).overhead + (VNh) >= BIN_VHEAP_SZ(c_p))) {\ SWAPOUT; \ - reg[0] = r(0); \ PROCESS_MAIN_CHK_LOCKS(c_p); \ - FCALLS -= erts_garbage_collect(c_p, need, reg, (Live)); \ + FCALLS -= erts_garbage_collect_nobump(c_p, need, reg, (Live)); \ ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p); \ PROCESS_MAIN_CHK_LOCKS(c_p); \ - r(0) = reg[0]; \ SWAPIN; \ } \ HEAP_SPACE_VERIFIED(need); \ @@ -389,12 +380,10 @@ extern int count_instructions; unsigned need = (Nh); \ if (E - HTOP < need) { \ SWAPOUT; \ - reg[0] = r(0); \ PROCESS_MAIN_CHK_LOCKS(c_p); \ - FCALLS -= erts_garbage_collect(c_p, need, reg, (Live)); \ + FCALLS -= erts_garbage_collect_nobump(c_p, need, reg, (Live));\ ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p); \ PROCESS_MAIN_CHK_LOCKS(c_p); \ - r(0) = reg[0]; \ SWAPIN; \ } \ HEAP_SPACE_VERIFIED(need); \ @@ -411,15 +400,11 @@ extern int count_instructions; unsigned need = (Nh); \ if (E - HTOP < need) { \ SWAPOUT; \ - reg[0] = r(0); \ reg[Live] = Extra; \ PROCESS_MAIN_CHK_LOCKS(c_p); \ - FCALLS -= erts_garbage_collect(c_p, need, reg, (Live)+1); \ + FCALLS -= erts_garbage_collect_nobump(c_p, need, reg, (Live)+1); \ ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p); \ PROCESS_MAIN_CHK_LOCKS(c_p); \ - if (Live > 0) { \ - r(0) = reg[0]; \ - } \ Extra = reg[Live]; \ SWAPIN; \ } \ @@ -441,10 +426,9 @@ extern int count_instructions; #define MakeFun(FunP, NumFree) \ do { \ - SWAPOUT; \ - reg[0] = r(0); \ + HEAVY_SWAPOUT; \ r(0) = new_fun(c_p, reg, (ErlFunEntry *) FunP, NumFree); \ - SWAPIN; \ + HEAVY_SWAPIN; \ } while (0) #define PutTuple(Dst, Arity) \ @@ -534,15 +518,19 @@ extern int count_instructions; ASSERT(VALID_INSTR(Dst)); \ Goto(Dst) -#define GetR(pos, tr) \ - do { \ - tr = Arg(pos); \ - switch (beam_reg_tag(tr)) { \ - case R_REG_DEF: tr = r(0); break; \ - case X_REG_DEF: tr = xb(x_reg_offset(tr)); break; \ - case Y_REG_DEF: ASSERT(y_reg_offset(tr) >= 1); tr = yb(y_reg_offset(tr)); break; \ - } \ - CHECK_TERM(tr); \ +#define GetR(pos, tr) \ + do { \ + tr = Arg(pos); \ + switch (loader_tag(tr)) { \ + case LOADER_X_REG: \ + tr = x(loader_x_reg_index(tr)); \ + break; \ + case LOADER_Y_REG: \ + ASSERT(loader_y_reg_index(tr) >= 1); \ + tr = y(loader_y_reg_index(tr)); \ + break; \ + } \ + CHECK_TERM(tr); \ } while (0) #define GetArg1(N, Dst) GetR((N), Dst) @@ -560,23 +548,93 @@ extern int count_instructions; HTOP += 2; \ } while (0) +#define Swap(R1, R2) \ + do { \ + Eterm V = R1; \ + R1 = R2; \ + R2 = V; \ + } while (0) + +#define SwapTemp(R1, R2, Tmp) \ + do { \ + Eterm V = R1; \ + R1 = R2; \ + R2 = Tmp = V; \ + } while (0) + #define Move(Src, Dst, Store) \ do { \ Eterm term = (Src); \ Store(term, Dst); \ } while (0) -#define Move2(src1, dst1, src2, dst2) dst1 = (src1); dst2 = (src2) +#define Move2Par(S1, D1, S2, D2) \ + do { \ + Eterm V1, V2; \ + V1 = (S1); V2 = (S2); D1 = V1; D2 = V2; \ + } while (0) + +#define MoveShift(Src, SD, D) \ + do { \ + Eterm V; \ + V = Src; D = SD; SD = V; \ + } while (0) + +#define MoveDup(Src, D1, D2) \ + do { \ + D1 = D2 = (Src); \ + } while (0) + +#define Move3(S1, D1, S2, D2, S3, D3) D1 = (S1); D2 = (S2); D3 = (S3) + +#define MoveWindow3(S1, S2, S3, D) \ + do { \ + Eterm xt0, xt1, xt2; \ + Eterm *y = &D; \ + xt0 = S1; \ + xt1 = S2; \ + xt2 = S3; \ + y[0] = xt0; \ + y[1] = xt1; \ + y[2] = xt2; \ + } while (0) + +#define MoveWindow4(S1, S2, S3, S4, D) \ + do { \ + Eterm xt0, xt1, xt2, xt3; \ + Eterm *y = &D; \ + xt0 = S1; \ + xt1 = S2; \ + xt2 = S3; \ + xt3 = S4; \ + y[0] = xt0; \ + y[1] = xt1; \ + y[2] = xt2; \ + y[3] = xt3; \ + } while (0) -#define MoveGenDest(src, dstp) \ - if ((dstp) == NULL) { r(0) = (src); } else { *(dstp) = src; } +#define MoveWindow5(S1, S2, S3, S4, S5, D) \ + do { \ + Eterm xt0, xt1, xt2, xt3, xt4; \ + Eterm *y = &D; \ + xt0 = S1; \ + xt1 = S2; \ + xt2 = S3; \ + xt3 = S4; \ + xt4 = S5; \ + y[0] = xt0; \ + y[1] = xt1; \ + y[2] = xt2; \ + y[3] = xt3; \ + y[4] = xt4; \ + } while (0) -#define MoveReturn(Src, Dest) \ - (Dest) = (Src); \ - I = c_p->cp; \ - ASSERT(VALID_INSTR(*c_p->cp)); \ - c_p->cp = 0; \ - CHECK_TERM(r(0)); \ +#define MoveReturn(Src) \ + x(0) = (Src); \ + I = c_p->cp; \ + ASSERT(VALID_INSTR(*c_p->cp)); \ + c_p->cp = 0; \ + CHECK_TERM(r(0)); \ Goto(*I) #define DeallocateReturn(Deallocate) \ @@ -588,26 +646,26 @@ extern int count_instructions; Goto(*I); \ } while (0) -#define MoveDeallocateReturn(Src, Dest, Deallocate) \ - (Dest) = (Src); \ +#define MoveDeallocateReturn(Src, Deallocate) \ + x(0) = (Src); \ DeallocateReturn(Deallocate) -#define MoveCall(Src, Dest, CallDest, Size) \ - (Dest) = (Src); \ +#define MoveCall(Src, CallDest, Size) \ + x(0) = (Src); \ SET_CP(c_p, I+Size+1); \ - SET_I((BeamInstr *) CallDest); \ + SET_I((BeamInstr *) CallDest); \ Dispatch(); -#define MoveCallLast(Src, Dest, CallDest, Deallocate) \ - (Dest) = (Src); \ - RESTORE_CP(E); \ - E = ADD_BYTE_OFFSET(E, (Deallocate)); \ - SET_I((BeamInstr *) CallDest); \ +#define MoveCallLast(Src, CallDest, Deallocate) \ + x(0) = (Src); \ + RESTORE_CP(E); \ + E = ADD_BYTE_OFFSET(E, (Deallocate)); \ + SET_I((BeamInstr *) CallDest); \ Dispatch(); -#define MoveCallOnly(Src, Dest, CallDest) \ - (Dest) = (Src); \ - SET_I((BeamInstr *) CallDest); \ +#define MoveCallOnly(Src, CallDest) \ + x(0) = (Src); \ + SET_I((BeamInstr *) CallDest); \ Dispatch(); #define MoveJump(Src) \ @@ -615,57 +673,58 @@ extern int count_instructions; SET_I((BeamInstr *) Arg(0)); \ Goto(*I); -#define GetList(Src, H, T) do { \ - Eterm* tmp_ptr = list_val(Src); \ - H = CAR(tmp_ptr); \ - T = CDR(tmp_ptr); } while (0) - -#define GetTupleElement(Src, Element, Dest) \ - do { \ - tmp_arg1 = (Eterm) COMPRESS_POINTER(((unsigned char *) tuple_val(Src)) + \ - (Element)); \ - (Dest) = (*(Eterm *) EXPAND_POINTER(tmp_arg1)); \ - } while (0) - -#define ExtractNextElement(Dest) \ - tmp_arg1 += sizeof(Eterm); \ - (Dest) = (* (Eterm *) (((unsigned char *) EXPAND_POINTER(tmp_arg1)))) - -#define ExtractNextElement2(Dest) \ - do { \ - Eterm* ene_dstp = &(Dest); \ - ene_dstp[0] = ((Eterm *) EXPAND_POINTER(tmp_arg1))[1]; \ - ene_dstp[1] = ((Eterm *) EXPAND_POINTER(tmp_arg1))[2]; \ - tmp_arg1 += sizeof(Eterm) + sizeof(Eterm); \ +#define GetList(Src, H, T) \ + do { \ + Eterm* tmp_ptr = list_val(Src); \ + Eterm hd, tl; \ + hd = CAR(tmp_ptr); \ + tl = CDR(tmp_ptr); \ + H = hd; T = tl; \ } while (0) -#define ExtractNextElement3(Dest) \ - do { \ - Eterm* ene_dstp = &(Dest); \ - ene_dstp[0] = ((Eterm *) EXPAND_POINTER(tmp_arg1))[1]; \ - ene_dstp[1] = ((Eterm *) EXPAND_POINTER(tmp_arg1))[2]; \ - ene_dstp[2] = ((Eterm *) EXPAND_POINTER(tmp_arg1))[3]; \ - tmp_arg1 += 3*sizeof(Eterm); \ +#define GetTupleElement(Src, Element, Dest) \ + do { \ + Eterm* src; \ + src = ADD_BYTE_OFFSET(tuple_val(Src), (Element)); \ + (Dest) = *src; \ } while (0) -#define ExtractNextElement4(Dest) \ - do { \ - Eterm* ene_dstp = &(Dest); \ - ene_dstp[0] = ((Eterm *) EXPAND_POINTER(tmp_arg1))[1]; \ - ene_dstp[1] = ((Eterm *) EXPAND_POINTER(tmp_arg1))[2]; \ - ene_dstp[2] = ((Eterm *) EXPAND_POINTER(tmp_arg1))[3]; \ - ene_dstp[3] = ((Eterm *) EXPAND_POINTER(tmp_arg1))[4]; \ - tmp_arg1 += 4*sizeof(Eterm); \ +#define GetTupleElement2(Src, Element, Dest) \ + do { \ + Eterm* src; \ + Eterm* dst; \ + Eterm E1, E2; \ + src = ADD_BYTE_OFFSET(tuple_val(Src), (Element)); \ + dst = &(Dest); \ + E1 = src[0]; \ + E2 = src[1]; \ + dst[0] = E1; \ + dst[1] = E2; \ } while (0) -#define ExtractElement(Element, Dest) \ - do { \ - tmp_arg1 += (Element); \ - (Dest) = (* (Eterm *) EXPAND_POINTER(tmp_arg1)); \ +#define GetTupleElement3(Src, Element, Dest) \ + do { \ + Eterm* src; \ + Eterm* dst; \ + Eterm E1, E2, E3; \ + src = ADD_BYTE_OFFSET(tuple_val(Src), (Element)); \ + dst = &(Dest); \ + E1 = src[0]; \ + E2 = src[1]; \ + E3 = src[2]; \ + dst[0] = E1; \ + dst[1] = E2; \ + dst[2] = E3; \ } while (0) #define EqualImmed(X, Y, Action) if (X != Y) { Action; } #define NotEqualImmed(X, Y, Action) if (X == Y) { Action; } +#define EqualExact(X, Y, Action) if (!EQ(X,Y)) { Action; } +#define NotEqualExact(X, Y, Action) if (EQ(X,Y)) { Action; } +#define Equal(X, Y, Action) CMP_EQ_ACTION(X,Y,Action) +#define NotEqual(X, Y, Action) CMP_NE_ACTION(X,Y,Action) +#define IsLessThan(X, Y, Action) CMP_LT_ACTION(X,Y,Action) +#define IsGreaterEqual(X, Y, Action) CMP_GE_ACTION(X,Y,Action) #define IsFloat(Src, Fail) if (is_not_float(Src)) { Fail; } @@ -689,23 +748,29 @@ extern int count_instructions; if (is_not_list(Src)) { Fail; } \ A(Need, Alive) -#define IsNonemptyListTestHeap(Src, Need, Alive, Fail) \ - if (is_not_list(Src)) { Fail; } \ +#define IsNonemptyListTestHeap(Need, Alive, Fail) \ + if (is_not_list(x(0))) { Fail; } \ TestHeap(Need, Alive) +#define IsNonemptyListGetList(Src, H, T, Fail) \ + if (is_not_list(Src)) { \ + Fail; \ + } else { \ + Eterm* tmp_ptr = list_val(Src); \ + Eterm hd, tl; \ + hd = CAR(tmp_ptr); \ + tl = CDR(tmp_ptr); \ + H = hd; T = tl; \ + } + #define IsTuple(X, Action) if (is_not_tuple(X)) Action -#define IsArity(Pointer, Arity, Fail) \ - if (*(Eterm *) \ - EXPAND_POINTER(tmp_arg1 = (Eterm) \ - COMPRESS_POINTER(tuple_val(Pointer))) != (Arity)) \ - { \ - Fail; \ +#define IsArity(Pointer, Arity, Fail) \ + if (*tuple_val(Pointer) != (Arity)) { \ + Fail; \ } -#define IsMap(Src, Fail) if (is_not_map(Src)) { Fail; } - -#define HasMapField(Src, Key, Fail) if (has_not_map_field(Src, Key)) { Fail; } +#define IsMap(Src, Fail) if (!is_map(Src)) { Fail; } #define GetMapElement(Src, Key, Dst, Fail) \ do { \ @@ -716,6 +781,15 @@ extern int count_instructions; Dst = _res; \ } while (0) +#define GetMapElementHash(Src, Key, Hx, Dst, Fail) \ + do { \ + Eterm _res = get_map_element_hash(Src, Key, Hx); \ + if (is_non_value(_res)) { \ + Fail; \ + } \ + Dst = _res; \ + } while (0) + #define IsFunction(X, Action) \ do { \ if ( !(is_any_fun(X)) ) { \ @@ -730,15 +804,21 @@ extern int count_instructions; } \ } while (0) -#define IsTupleOfArity(Src, Arity, Fail) \ - do { \ - if (is_not_tuple(Src) || \ - *(Eterm *) \ - EXPAND_POINTER(tmp_arg1 = \ - (Eterm) COMPRESS_POINTER(tuple_val(Src))) != Arity) { \ - Fail; \ - } \ +#ifdef DEBUG +#define IsTupleOfArity(Src, Arityval, Fail) \ + do { \ + if (!(is_tuple(Src) && *tuple_val(Src) == Arityval)) { \ + Fail; \ + } \ + } while (0) +#else +#define IsTupleOfArity(Src, Arityval, Fail) \ + do { \ + if (!(is_boxed(Src) && *tuple_val(Src) == Arityval)) { \ + Fail; \ + } \ } while (0) +#endif #define IsBoolean(X, Fail) if ((X) != am_true && (X) != am_false) { Fail; } @@ -748,7 +828,7 @@ extern int count_instructions; #define IsBitstring(Src, Fail) \ if (is_not_binary(Src)) { Fail; } -#if defined(ARCH_64) && !HALFWORD_HEAP +#if defined(ARCH_64) #define BsSafeMul(A, B, Fail, Target) \ do { Uint64 _res = (A) * (B); \ if (_res / B != A) { Fail; } \ @@ -765,6 +845,7 @@ extern int count_instructions; #define BsGetFieldSize(Bits, Unit, Fail, Target) \ do { \ Sint _signed_size; Uint _uint_size; \ + Uint temp_bits; \ if (is_small(Bits)) { \ _signed_size = signed_val(Bits); \ if (_signed_size < 0) { Fail; } \ @@ -779,6 +860,7 @@ extern int count_instructions; #define BsGetUncheckedFieldSize(Bits, Unit, Fail, Target) \ do { \ Sint _signed_size; Uint _uint_size; \ + Uint temp_bits; \ if (is_small(Bits)) { \ _signed_size = signed_val(Bits); \ if (_signed_size < 0) { Fail; } \ @@ -964,8 +1046,8 @@ static Eterm update_map_assoc(Process* p, Eterm* reg, Eterm map, BeamInstr* I) NOINLINE; static Eterm update_map_exact(Process* p, Eterm* reg, Eterm map, BeamInstr* I) NOINLINE; -static int has_not_map_field(Eterm map, Eterm key); static Eterm get_map_element(Eterm map, Eterm key); +static Eterm get_map_element_hash(Eterm map, Eterm key, Uint32 hx); /* * Functions not directly called by process_main(). OK to inline. @@ -990,23 +1072,17 @@ init_emulator(void) */ #if defined(__GNUC__) && defined(sparc) && !defined(DEBUG) -# define REG_x0 asm("%l0") # define REG_xregs asm("%l1") # define REG_htop asm("%l2") # define REG_stop asm("%l3") # define REG_I asm("%l4") # define REG_fcalls asm("%l5") -# define REG_tmp_arg1 asm("%l6") -# define REG_tmp_arg2 asm("%l7") #else -# define REG_x0 # define REG_xregs # define REG_htop # define REG_stop # define REG_I # define REG_fcalls -# define REG_tmp_arg1 -# define REG_tmp_arg2 #endif #ifdef USE_VM_PROBES @@ -1081,16 +1157,32 @@ init_emulator(void) DTRACE2(nif_return, process_name, mfa); \ } -#else /* USE_VM_PROBES */ - -#define DTRACE_LOCAL_CALL(p, m, f, a) do {} while (0) -#define DTRACE_GLOBAL_CALL(p, m, f, a) do {} while (0) -#define DTRACE_RETURN(p, m, f, a) do {} while (0) -#define DTRACE_BIF_ENTRY(p, m, f, a) do {} while (0) -#define DTRACE_BIF_RETURN(p, m, f, a) do {} while (0) -#define DTRACE_NIF_ENTRY(p, m, f, a) do {} while (0) -#define DTRACE_NIF_RETURN(p, m, f, a) do {} while (0) +#define DTRACE_GLOBAL_CALL_FROM_EXPORT(p,e) \ + do { \ + if (DTRACE_ENABLED(global_function_entry)) { \ + BeamInstr* fp = (BeamInstr *) (((Export *) (e))->addressv[erts_active_code_ix()]); \ + DTRACE_GLOBAL_CALL((p), (Eterm)fp[-3], (Eterm)fp[-2], fp[-1]); \ + } \ + } while(0) + +#define DTRACE_RETURN_FROM_PC(p) \ + do { \ + BeamInstr* fp; \ + if (DTRACE_ENABLED(function_return) && (fp = find_function_from_pc((p)->cp))) { \ + DTRACE_RETURN((p), (Eterm)fp[0], (Eterm)fp[1], (Uint)fp[2]); \ + } \ + } while(0) +#else /* USE_VM_PROBES */ +#define DTRACE_LOCAL_CALL(p, m, f, a) do {} while (0) +#define DTRACE_GLOBAL_CALL(p, m, f, a) do {} while (0) +#define DTRACE_GLOBAL_CALL_FROM_EXPORT(p, e) do {} while (0) +#define DTRACE_RETURN(p, m, f, a) do {} while (0) +#define DTRACE_RETURN_FROM_PC(p) do {} while (0) +#define DTRACE_BIF_ENTRY(p, m, f, a) do {} while (0) +#define DTRACE_BIF_RETURN(p, m, f, a) do {} while (0) +#define DTRACE_NIF_ENTRY(p, m, f, a) do {} while (0) +#define DTRACE_NIF_RETURN(p, m, f, a) do {} while (0) #endif /* USE_VM_PROBES */ /* @@ -1108,11 +1200,6 @@ void process_main(void) ERTS_DECLARE_DUMMY(Eterm pid); #endif - /* - * X register zero; also called r(0) - */ - register Eterm x0 REG_x0 = NIL; - /* Pointer to X registers: x(1)..x(N); reg[0] is used when doing GC, * in all other cases x0 is used. */ @@ -1140,17 +1227,6 @@ void process_main(void) register Sint FCALLS REG_fcalls = 0; /* - * Temporaries used for picking up arguments for instructions. - */ - register Eterm tmp_arg1 REG_tmp_arg1 = NIL; - register Eterm tmp_arg2 REG_tmp_arg2 = NIL; -#if HEAP_ON_C_STACK - Eterm tmp_big[2]; /* Temporary buffer for small bignums if HEAP_ON_C_STACK. */ -#else - Eterm *tmp_big; /* Temporary buffer for small bignums if !HEAP_ON_C_STACK. */ -#endif - - /* * X registers and floating point registers are located in * scheduler specific data. */ @@ -1161,24 +1237,23 @@ void process_main(void) */ int neg_o_reds = 0; - Eterm (*arith_func)(Process* p, Eterm* reg, Uint live); - -#ifndef NO_JUMP_TABLE - static void* opcodes[] = { DEFINE_OPCODES }; #ifdef ERTS_OPCODE_COUNTER_SUPPORT static void* counting_opcodes[] = { DEFINE_COUNTING_OPCODES }; -#endif +#else +#ifndef NO_JUMP_TABLE + static void* opcodes[] = { DEFINE_OPCODES }; #else int Go; #endif - - Uint temp_bits; /* Temporary used by BsSkipBits2 & BsGetInteger2 */ +#endif Eterm pt_arity; /* Used by do_put_tuple */ Uint64 start_time = 0; /* Monitor long schedule */ BeamInstr* start_time_i = NULL; + ERTS_MSACC_DECLARE_CACHE_X() /* a cached value of the tsd pointer for msacc */ + ERL_BITS_DECLARE_STATEP; /* Has to be last declaration */ @@ -1222,9 +1297,6 @@ void process_main(void) PROCESS_MAIN_CHK_LOCKS(c_p); ERTS_SMP_UNREQ_PROC_MAIN_LOCK(c_p); -#if HALFWORD_HEAP - ASSERT(erts_get_scheduler_data()->num_tmp_heap_used == 0); -#endif ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p); c_p = schedule(c_p, reds_used); ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p); @@ -1235,6 +1307,8 @@ void process_main(void) ERTS_SMP_REQ_PROC_MAIN_LOCK(c_p); PROCESS_MAIN_CHK_LOCKS(c_p); + ERTS_MSACC_UPDATE_CACHE_X(); + if (erts_system_monitor_long_schedule != 0) { start_time = erts_timestamp_millis(); start_time_i = c_p->i; @@ -1242,9 +1316,6 @@ void process_main(void) reg = ERTS_PROC_GET_SCHDATA(c_p)->x_reg_array; freg = ERTS_PROC_GET_SCHDATA(c_p)->f_reg_array; -#if !HEAP_ON_C_STACK - tmp_big = ERTS_PROC_GET_SCHDATA(c_p)->beam_emu_tmp_heap; -#endif ERL_BITS_RELOAD_STATEP(c_p); { int reds; @@ -1253,7 +1324,7 @@ void process_main(void) int i; argp = c_p->arg_reg; - for (i = c_p->arity - 1; i > 0; i--) { + for (i = c_p->arity - 1; i >= 0; i--) { reg[i] = argp[i]; CHECK_TERM(reg[i]); } @@ -1277,12 +1348,6 @@ void process_main(void) } next = (BeamInstr *) *I; - r(0) = c_p->arg_reg[0]; -#ifdef HARDDEBUG - if (c_p->arity > 0) { - CHECK_TERM(r(0)); - } -#endif SWAPIN; ASSERT(VALID_INSTR(next)); @@ -1321,26 +1386,24 @@ void process_main(void) #endif #include "beam_hot.h" -#define STORE_ARITH_RESULT(res) StoreBifResult(2, (res)); -#define ARITH_FUNC(name) erts_gc_##name - { Eterm increment_reg_val; Eterm increment_val; Uint live; Eterm result; - OpCase(i_increment_yIId): - increment_reg_val = yb(Arg(0)); + OpCase(i_increment_rIId): + increment_reg_val = x(0); + I--; goto do_increment; OpCase(i_increment_xIId): increment_reg_val = xb(Arg(0)); goto do_increment; - OpCase(i_increment_rIId): - increment_reg_val = r(0); - I--; + OpCase(i_increment_yIId): + increment_reg_val = yb(Arg(0)); + goto do_increment; do_increment: increment_val = Arg(1); @@ -1349,137 +1412,133 @@ void process_main(void) ASSERT(MY_IS_SSMALL(i) == IS_SSMALL(i)); if (MY_IS_SSMALL(i)) { result = make_small(i); - store_result: StoreBifResult(3, result); } } live = Arg(2); - SWAPOUT; - reg[0] = r(0); + HEAVY_SWAPOUT; reg[live] = increment_reg_val; reg[live+1] = make_small(increment_val); result = erts_gc_mixed_plus(c_p, reg, live); - r(0) = reg[0]; - SWAPIN; + HEAVY_SWAPIN; ERTS_HOLE_CHECK(c_p); if (is_value(result)) { - goto store_result; + StoreBifResult(3, result); } ASSERT(c_p->freason != BADMATCH || is_value(c_p->fvalue)); goto find_func_info; } - - OpCase(i_plus_jId): + +#define DO_OUTLINED_ARITH_2(name, Op1, Op2) \ + do { \ + Eterm result; \ + Uint live = Arg(1); \ + \ + HEAVY_SWAPOUT; \ + reg[live] = Op1; \ + reg[live+1] = Op2; \ + result = erts_gc_##name(c_p, reg, live); \ + HEAVY_SWAPIN; \ + ERTS_HOLE_CHECK(c_p); \ + if (is_value(result)) { \ + StoreBifResult(4, result); \ + } \ + goto lb_Cl_error; \ + } while (0) + { + Eterm PlusOp1, PlusOp2; Eterm result; - if (is_both_small(tmp_arg1, tmp_arg2)) { - Sint i = signed_val(tmp_arg1) + signed_val(tmp_arg2); + OpCase(i_plus_jIxxd): + PlusOp1 = xb(Arg(2)); + PlusOp2 = xb(Arg(3)); + goto do_plus; + + OpCase(i_plus_jIxyd): + PlusOp1 = xb(Arg(2)); + PlusOp2 = yb(Arg(3)); + goto do_plus; + + OpCase(i_plus_jIssd): + GetArg2(2, PlusOp1, PlusOp2); + goto do_plus; + + do_plus: + if (is_both_small(PlusOp1, PlusOp2)) { + Sint i = signed_val(PlusOp1) + signed_val(PlusOp2); ASSERT(MY_IS_SSMALL(i) == IS_SSMALL(i)); if (MY_IS_SSMALL(i)) { result = make_small(i); - STORE_ARITH_RESULT(result); + StoreBifResult(4, result); } - } - arith_func = ARITH_FUNC(mixed_plus); - goto do_big_arith2; + DO_OUTLINED_ARITH_2(mixed_plus, PlusOp1, PlusOp2); } - OpCase(i_minus_jId): { + Eterm MinusOp1, MinusOp2; Eterm result; - if (is_both_small(tmp_arg1, tmp_arg2)) { - Sint i = signed_val(tmp_arg1) - signed_val(tmp_arg2); + OpCase(i_minus_jIxxd): + MinusOp1 = xb(Arg(2)); + MinusOp2 = xb(Arg(3)); + goto do_minus; + + OpCase(i_minus_jIssd): + GetArg2(2, MinusOp1, MinusOp2); + goto do_minus; + + do_minus: + if (is_both_small(MinusOp1, MinusOp2)) { + Sint i = signed_val(MinusOp1) - signed_val(MinusOp2); ASSERT(MY_IS_SSMALL(i) == IS_SSMALL(i)); if (MY_IS_SSMALL(i)) { result = make_small(i); - STORE_ARITH_RESULT(result); + StoreBifResult(4, result); } } - arith_func = ARITH_FUNC(mixed_minus); - goto do_big_arith2; + DO_OUTLINED_ARITH_2(mixed_minus, MinusOp1, MinusOp2); } - OpCase(i_is_lt_f): - if (CMP_GE(tmp_arg1, tmp_arg2)) { - ClauseFail(); - } - Next(1); - - OpCase(i_is_ge_f): - if (CMP_LT(tmp_arg1, tmp_arg2)) { - ClauseFail(); - } - Next(1); - - OpCase(i_is_eq_f): - if (CMP_NE(tmp_arg1, tmp_arg2)) { - ClauseFail(); - } - Next(1); - - OpCase(i_is_ne_f): - if (CMP_EQ(tmp_arg1, tmp_arg2)) { - ClauseFail(); - } - Next(1); - - OpCase(i_is_eq_exact_f): - if (!EQ(tmp_arg1, tmp_arg2)) { - ClauseFail(); - } - Next(1); - { Eterm is_eq_exact_lit_val; - OpCase(i_is_eq_exact_literal_xfc): - is_eq_exact_lit_val = xb(Arg(0)); - I++; + OpCase(i_is_eq_exact_literal_fxc): + is_eq_exact_lit_val = xb(Arg(1)); goto do_is_eq_exact_literal; - OpCase(i_is_eq_exact_literal_yfc): - is_eq_exact_lit_val = yb(Arg(0)); - I++; + OpCase(i_is_eq_exact_literal_fyc): + is_eq_exact_lit_val = yb(Arg(1)); goto do_is_eq_exact_literal; - OpCase(i_is_eq_exact_literal_rfc): - is_eq_exact_lit_val = r(0); - do_is_eq_exact_literal: - if (!eq(Arg(1), is_eq_exact_lit_val)) { + if (!eq(Arg(2), is_eq_exact_lit_val)) { ClauseFail(); } - Next(2); + Next(3); } { Eterm is_ne_exact_lit_val; - OpCase(i_is_ne_exact_literal_xfc): - is_ne_exact_lit_val = xb(Arg(0)); - I++; + OpCase(i_is_ne_exact_literal_fxc): + is_ne_exact_lit_val = xb(Arg(1)); goto do_is_ne_exact_literal; - OpCase(i_is_ne_exact_literal_yfc): - is_ne_exact_lit_val = yb(Arg(0)); - I++; + OpCase(i_is_ne_exact_literal_fyc): + is_ne_exact_lit_val = yb(Arg(1)); goto do_is_ne_exact_literal; - OpCase(i_is_ne_exact_literal_rfc): - is_ne_exact_lit_val = r(0); - do_is_ne_exact_literal: - if (eq(Arg(1), is_ne_exact_lit_val)) { + if (eq(Arg(2), is_ne_exact_lit_val)) { ClauseFail(); } - Next(2); + Next(3); } - OpCase(i_move_call_only_fcr): { + OpCase(i_move_call_only_fc): { r(0) = Arg(1); } /* FALL THROUGH */ @@ -1489,7 +1548,7 @@ void process_main(void) Dispatch(); } - OpCase(i_move_call_last_fPcr): { + OpCase(i_move_call_last_fPc): { r(0) = Arg(2); } /* FALL THROUGH */ @@ -1501,7 +1560,7 @@ void process_main(void) Dispatch(); } - OpCase(i_move_call_crf): { + OpCase(i_move_call_cf): { r(0) = Arg(0); I++; } @@ -1513,7 +1572,7 @@ void process_main(void) Dispatch(); } - OpCase(i_move_call_ext_last_ePcr): { + OpCase(i_move_call_ext_last_ePc): { r(0) = Arg(2); } /* FALL THROUGH */ @@ -1526,40 +1585,25 @@ void process_main(void) * is not loaded, it points to code which will invoke the error handler * (see lb_call_error_handler below). */ -#ifdef USE_VM_CALL_PROBES - if (DTRACE_ENABLED(global_function_entry)) { - BeamInstr* fp = (BeamInstr *) (((Export *) Arg(0))->addressv[erts_active_code_ix()]); - DTRACE_GLOBAL_CALL(c_p, (Eterm)fp[-3], (Eterm)fp[-2], fp[-1]); - } -#endif + DTRACE_GLOBAL_CALL_FROM_EXPORT(c_p, Arg(0)); Dispatchx(); - OpCase(i_move_call_ext_cre): { + OpCase(i_move_call_ext_ce): { r(0) = Arg(0); I++; } /* FALL THROUGH */ OpCase(i_call_ext_e): SET_CP(c_p, I+2); -#ifdef USE_VM_CALL_PROBES - if (DTRACE_ENABLED(global_function_entry)) { - BeamInstr* fp = (BeamInstr *) (((Export *) Arg(0))->addressv[erts_active_code_ix()]); - DTRACE_GLOBAL_CALL(c_p, (Eterm)fp[-3], (Eterm)fp[-2], fp[-1]); - } -#endif + DTRACE_GLOBAL_CALL_FROM_EXPORT(c_p, Arg(0)); Dispatchx(); - OpCase(i_move_call_ext_only_ecr): { + OpCase(i_move_call_ext_only_ec): { r(0) = Arg(1); } /* FALL THROUGH */ OpCase(i_call_ext_only_e): -#ifdef USE_VM_CALL_PROBES - if (DTRACE_ENABLED(global_function_entry)) { - BeamInstr* fp = (BeamInstr *) (((Export *) Arg(0))->addressv[erts_active_code_ix()]); - DTRACE_GLOBAL_CALL(c_p, (Eterm)fp[-3], (Eterm)fp[-2], fp[-1]); - } -#endif + DTRACE_GLOBAL_CALL_FROM_EXPORT(c_p, Arg(0)); Dispatchx(); OpCase(init_y): { @@ -1593,18 +1637,9 @@ void process_main(void) Next(1); } - OpCase(return): { -#ifdef USE_VM_CALL_PROBES - BeamInstr* fptr; -#endif SET_I(c_p->cp); - -#ifdef USE_VM_CALL_PROBES - if (DTRACE_ENABLED(function_return) && (fptr = find_function_from_pc(c_p->cp))) { - DTRACE_RETURN(c_p, (Eterm)fptr[0], (Eterm)fptr[1], (Uint)fptr[2]); - } -#endif + DTRACE_RETURN_FROM_PC(c_p); /* * We must clear the CP to make sure that a stale value do not * create a false module dependcy preventing code upgrading. @@ -1629,17 +1664,11 @@ void process_main(void) PRE_BIF_SWAPOUT(c_p); c_p->fcalls = FCALLS - 1; - reg[0] = r(0); result = erl_send(c_p, r(0), x(1)); PreFetch(0, next); ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p); ERTS_SMP_REQ_PROC_MAIN_LOCK(c_p); PROCESS_MAIN_CHK_LOCKS(c_p); - if (c_p->mbuf || MSO(c_p).overhead >= BIN_VHEAP_SZ(c_p)) { - result = erts_gc_after_bif_call(c_p, result, reg, 2); - r(0) = reg[0]; - E = c_p->stop; - } HTOP = HEAP_TOP(c_p); FCALLS = c_p->fcalls; if (is_value(result)) { @@ -1650,7 +1679,6 @@ void process_main(void) SET_CP(c_p, I+1); SET_I(c_p->i); SWAPIN; - r(0) = reg[0]; Dispatch(); } goto find_func_info; @@ -1660,29 +1688,23 @@ void process_main(void) Eterm element_index; Eterm element_tuple; - OpCase(i_element_xjsd): - element_tuple = xb(Arg(0)); - I++; + OpCase(i_element_jxsd): + element_tuple = xb(Arg(1)); goto do_element; - OpCase(i_element_yjsd): - element_tuple = yb(Arg(0)); - I++; + OpCase(i_element_jysd): + element_tuple = yb(Arg(1)); goto do_element; - OpCase(i_element_rjsd): - element_tuple = r(0); - /* Fall through */ - do_element: - GetArg1(1, element_index); + GetArg1(2, element_index); if (is_small(element_index) && is_tuple(element_tuple)) { Eterm* tp = tuple_val(element_tuple); if ((signed_val(element_index) >= 1) && (signed_val(element_index) <= arityval(*tp))) { Eterm result = tp[signed_val(element_index)]; - StoreBifResult(2, result); + StoreBifResult(3, result); } } } @@ -1696,29 +1718,24 @@ void process_main(void) { Eterm fast_element_tuple; - OpCase(i_fast_element_rjId): - fast_element_tuple = r(0); + OpCase(i_fast_element_jxId): + fast_element_tuple = xb(Arg(1)); + goto do_fast_element; + + OpCase(i_fast_element_jyId): + fast_element_tuple = yb(Arg(1)); + goto do_fast_element; do_fast_element: if (is_tuple(fast_element_tuple)) { Eterm* tp = tuple_val(fast_element_tuple); - Eterm pos = Arg(1); /* Untagged integer >= 1 */ + Eterm pos = Arg(2); /* Untagged integer >= 1 */ if (pos <= arityval(*tp)) { Eterm result = tp[pos]; - StoreBifResult(2, result); + StoreBifResult(3, result); } } goto badarg; - - OpCase(i_fast_element_xjId): - fast_element_tuple = xb(Arg(0)); - I++; - goto do_fast_element; - - OpCase(i_fast_element_yjId): - fast_element_tuple = yb(Arg(0)); - I++; - goto do_fast_element; } OpCase(catch_yf): @@ -1739,11 +1756,10 @@ void process_main(void) SWAPIN; } /* only x(2) is included in the rootset here */ - if (E - HTOP < 3 || c_p->mbuf) { /* Force GC in case add_stacktrace() - * created heap fragments */ + if (E - HTOP < 3) { SWAPOUT; PROCESS_MAIN_CHK_LOCKS(c_p); - FCALLS -= erts_garbage_collect(c_p, 3, reg+2, 1); + FCALLS -= erts_garbage_collect_nobump(c_p, 3, reg+2, 1); ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p); PROCESS_MAIN_CHK_LOCKS(c_p); SWAPIN; @@ -1824,13 +1840,20 @@ void process_main(void) * Pick up the next message and place it in x(0). * If no message, jump to a wait or wait_timeout instruction. */ - OpCase(i_loop_rec_fr): + OpCase(i_loop_rec_f): { BeamInstr *next; - ErlMessage* msgp; + ErtsMessage* msgp; - loop_rec__: + /* + * We need to disable GC while matching messages + * in the queue. This since messages with data outside + * the heap will be corrupted by a GC. + */ + ASSERT(!(c_p->flags & F_DELAY_GC)); + c_p->flags |= F_DELAY_GC; + loop_rec__: PROCESS_MAIN_CHK_LOCKS(c_p); msgp = PEEK_MESSAGE(c_p); @@ -1842,6 +1865,7 @@ void process_main(void) if (ERTS_PROC_PENDING_EXIT(c_p)) { erts_smp_proc_unlock(c_p, ERTS_PROC_LOCKS_MSG_RECEIVE); SWAPOUT; + c_p->flags &= ~F_DELAY_GC; goto do_schedule; /* Will be rescheduled for exit */ } ERTS_SMP_MSGQ_MV_INQ2PRIVQ(c_p); @@ -1851,32 +1875,27 @@ void process_main(void) else #endif { + c_p->flags &= ~F_DELAY_GC; SET_I((BeamInstr *) Arg(0)); Goto(*I); /* Jump to a wait or wait_timeout instruction */ } } - ErtsMoveMsgAttachmentIntoProc(msgp, c_p, E, HTOP, FCALLS, - { - SWAPOUT; - reg[0] = r(0); - PROCESS_MAIN_CHK_LOCKS(c_p); - }, - { - ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p); - PROCESS_MAIN_CHK_LOCKS(c_p); - r(0) = reg[0]; - SWAPIN; - }); if (is_non_value(ERL_MESSAGE_TERM(msgp))) { - /* - * A corrupt distribution message that we weren't able to decode; - * remove it... - */ - ASSERT(!msgp->data.attached); - /* TODO: Add DTrace probe for this bad message situation? */ - UNLINK_MESSAGE(c_p, msgp); - free_message(msgp); - goto loop_rec__; + SWAPOUT; /* erts_decode_dist_message() may write to heap... */ + if (!erts_decode_dist_message(c_p, ERTS_PROC_LOCK_MAIN, msgp, 0)) { + /* + * A corrupt distribution message that we weren't able to decode; + * remove it... + */ + /* No swapin should be needed */ + ASSERT(HTOP == c_p->htop && E == c_p->stop); + /* TODO: Add DTrace probe for this bad message situation? */ + UNLINK_MESSAGE(c_p, msgp); + msgp->next = NULL; + erts_cleanup_messages(msgp); + goto loop_rec__; + } + SWAPIN; } PreFetch(1, next); r(0) = ERL_MESSAGE_TERM(msgp); @@ -1888,8 +1907,7 @@ void process_main(void) */ OpCase(remove_message): { BeamInstr *next; - ErlMessage* msgp; - + ErtsMessage* msgp; PROCESS_MAIN_CHK_LOCKS(c_p); PreFetch(0, next); @@ -1903,20 +1921,7 @@ void process_main(void) if (DT_UTAG(c_p) != NIL) { if (DT_UTAG_FLAGS(c_p) & DT_UTAG_PERMANENT) { SEQ_TRACE_TOKEN(c_p) = am_have_dt_utag; -#ifdef DTRACE_TAG_HARDDEBUG - if (DT_UTAG_FLAGS(c_p) & DT_UTAG_SPREADING) - erts_fprintf(stderr, - "Dtrace -> (%T) stop spreading " - "tag %T with message %T\r\n", - c_p->common.id,DT_UTAG(c_p),ERL_MESSAGE_TERM(msgp)); -#endif } else { -#ifdef DTRACE_TAG_HARDDEBUG - erts_fprintf(stderr, - "Dtrace -> (%T) kill tag %T with " - "message %T\r\n", - c_p->common.id,DT_UTAG(c_p),ERL_MESSAGE_TERM(msgp)); -#endif DT_UTAG(c_p) = NIL; SEQ_TRACE_TOKEN(c_p) = NIL; } @@ -1936,12 +1941,6 @@ void process_main(void) DT_UTAG(c_p) = ERL_MESSAGE_DT_UTAG(msgp); } DT_UTAG_FLAGS(c_p) |= DT_UTAG_SPREADING; -#ifdef DTRACE_TAG_HARDDEBUG - erts_fprintf(stderr, - "Dtrace -> (%T) receive tag (%T) " - "with message %T\r\n", - c_p->common.id, DT_UTAG(c_p), ERL_MESSAGE_TERM(msgp)); -#endif } else { #endif ASSERT(is_tuple(SEQ_TRACE_TOKEN(c_p))); @@ -1971,7 +1970,7 @@ void process_main(void) dtrace_proc_str(c_p, receiver_name); token2 = SEQ_TRACE_TOKEN(c_p); - if (token2 != NIL && token2 != am_have_dt_utag) { + if (have_seqtrace(token2)) { tok_label = signed_val(SEQ_TRACE_T_LABEL(token2)); tok_lastcnt = signed_val(SEQ_TRACE_T_LASTCNT(token2)); tok_serial = signed_val(SEQ_TRACE_T_SERIAL(token2)); @@ -1984,11 +1983,21 @@ void process_main(void) UNLINK_MESSAGE(c_p, msgp); JOIN_MESSAGE(c_p); CANCEL_TIMER(c_p); - free_message(msgp); + + erts_save_message_in_proc(c_p, msgp); + c_p->flags &= ~F_DELAY_GC; + + if (ERTS_IS_GC_DESIRED_INTERNAL(c_p, HTOP, E)) { + /* + * We want to GC soon but we leave a few + * reductions giving the message some time + * to turn into garbage. + */ + ERTS_VBUMP_LEAVE_REDS_INTERNAL(c_p, 5, FCALLS); + } ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p); PROCESS_MAIN_CHK_LOCKS(c_p); - NextPF(0, next); } @@ -1997,9 +2006,22 @@ void process_main(void) * message didn't match), then jump to the loop_rec instruction. */ OpCase(loop_rec_end_f): { + + ASSERT(c_p->flags & F_DELAY_GC); + SET_I((BeamInstr *) Arg(0)); SAVE_MESSAGE(c_p); - goto loop_rec__; + if (FCALLS > 0 || FCALLS > neg_o_reds) { + FCALLS--; + goto loop_rec__; + } + + c_p->flags &= ~F_DELAY_GC; + c_p->i = I; + SWAPOUT; + c_p->arity = 0; + c_p->current = NULL; + goto do_schedule; } /* * Prepare to wait for a message or a timeout, whichever occurs first. @@ -2027,44 +2049,30 @@ void process_main(void) } GetArg1(1, timeout_value); if (timeout_value != make_small(0)) { -#if !defined(ARCH_64) || HALFWORD_HEAP - Uint time_val; -#endif - if (is_small(timeout_value) && signed_val(timeout_value) > 0 && -#if defined(ARCH_64) && !HALFWORD_HEAP - ((unsigned_val(timeout_value) >> 32) == 0) -#else - 1 -#endif - ) { - /* - * The timer routiner will set c_p->i to the value in - * c_p->def_arg_reg[0]. Note that it is safe to use this - * location because there are no living x registers in - * a receive statement. - * Note that for the halfword emulator, the two first elements - * of the array are used. - */ - BeamInstr** pi = (BeamInstr**) c_p->def_arg_reg; - *pi = I+3; - set_timer(c_p, unsigned_val(timeout_value)); - } else if (timeout_value == am_infinity) { + if (timeout_value == am_infinity) c_p->flags |= F_TIMO; -#if !defined(ARCH_64) || HALFWORD_HEAP - } else if (term_to_Uint(timeout_value, &time_val)) { - BeamInstr** pi = (BeamInstr**) c_p->def_arg_reg; - *pi = I+3; - set_timer(c_p, time_val); -#endif - } else { /* Wrong time */ - OpCase(i_wait_error_locked): { - erts_smp_proc_unlock(c_p, ERTS_PROC_LOCKS_MSG_RECEIVE); - /* Fall through */ + else { + int tres = erts_set_proc_timer_term(c_p, timeout_value); + if (tres == 0) { + /* + * The timer routiner will set c_p->i to the value in + * c_p->def_arg_reg[0]. Note that it is safe to use this + * location because there are no living x registers in + * a receive statement. + */ + BeamInstr** pi = (BeamInstr**) c_p->def_arg_reg; + *pi = I+3; } + else { /* Wrong time */ + OpCase(i_wait_error_locked): { + erts_smp_proc_unlock(c_p, ERTS_PROC_LOCKS_MSG_RECEIVE); + /* Fall through */ + } OpCase(i_wait_error): { - c_p->freason = EXC_TIMEOUT_VALUE; - goto find_func_info; + c_p->freason = EXC_TIMEOUT_VALUE; + goto find_func_info; + } } } @@ -2082,6 +2090,22 @@ void process_main(void) OpCase(wait_f): wait2: { +#ifndef ERTS_SMP + if (ERTS_PROC_IS_EXITING(c_p)) { + /* + * I non smp case: + * + * Currently executing process might be sent an exit + * signal if it is traced by a port that it also is + * linked to, and the port terminates during the + * trace. In this case we do *not* want to clear + * the active flag, which will make the process hang + * in limbo forever. + */ + SWAPOUT; + goto do_schedule; + } +#endif c_p->i = (BeamInstr *) Arg(0); /* L1 */ SWAPOUT; c_p->arity = 0; @@ -2113,7 +2137,7 @@ void process_main(void) if ((c_p->flags & (F_INSLPQUEUE | F_TIMO)) == 0) { BeamInstr** p = (BeamInstr **) c_p->def_arg_reg; *p = I+3; - set_timer(c_p, Arg(1)); + erts_set_proc_timer_uword(c_p, Arg(1)); } goto wait2; } @@ -2141,22 +2165,17 @@ void process_main(void) NextPF(0, next); } - { Eterm select_val2; - OpCase(i_select_tuple_arity2_yfAfAf): + OpCase(i_select_tuple_arity2_yfAAff): select_val2 = yb(Arg(0)); goto do_select_tuple_arity2; - OpCase(i_select_tuple_arity2_xfAfAf): + OpCase(i_select_tuple_arity2_xfAAff): select_val2 = xb(Arg(0)); goto do_select_tuple_arity2; - OpCase(i_select_tuple_arity2_rfAfAf): - select_val2 = r(0); - I--; - do_select_tuple_arity2: if (is_not_tuple(select_val2)) { goto select_val2_fail; @@ -2164,22 +2183,18 @@ void process_main(void) select_val2 = *tuple_val(select_val2); goto do_select_val2; - OpCase(i_select_val2_yfcfcf): + OpCase(i_select_val2_yfccff): select_val2 = yb(Arg(0)); goto do_select_val2; - OpCase(i_select_val2_xfcfcf): + OpCase(i_select_val2_xfccff): select_val2 = xb(Arg(0)); goto do_select_val2; - OpCase(i_select_val2_rfcfcf): - select_val2 = r(0); - I--; - do_select_val2: if (select_val2 == Arg(2)) { - I += 2; - } else if (select_val2 == Arg(4)) { + I += 3; + } else if (select_val2 == Arg(3)) { I += 4; } @@ -2199,30 +2214,48 @@ void process_main(void) select_val = yb(Arg(0)); goto do_select_tuple_arity; - OpCase(i_select_tuple_arity_rfI): - select_val = r(0); - I--; - do_select_tuple_arity: if (is_tuple(select_val)) { select_val = *tuple_val(select_val); - goto do_binary_search; + goto do_linear_search; } SET_I((BeamInstr *) Arg(1)); Goto(*I); - OpCase(i_select_val_xfI): + OpCase(i_select_val_lins_xfI): + select_val = xb(Arg(0)); + goto do_linear_search; + + OpCase(i_select_val_lins_yfI): + select_val = yb(Arg(0)); + goto do_linear_search; + + do_linear_search: { + BeamInstr *vs = &Arg(3); + int ix = 0; + + for(;;) { + if (vs[ix+0] >= select_val) { ix += 0; break; } + if (vs[ix+1] >= select_val) { ix += 1; break; } + ix += 2; + } + + if (vs[ix] == select_val) { + I += ix + Arg(2) + 2; + } + + SET_I((BeamInstr *) Arg(1)); + Goto(*I); + } + + OpCase(i_select_val_bins_xfI): select_val = xb(Arg(0)); goto do_binary_search; - OpCase(i_select_val_yfI): + OpCase(i_select_val_bins_yfI): select_val = yb(Arg(0)); goto do_binary_search; - OpCase(i_select_val_rfI): - select_val = r(0); - I--; - do_binary_search: { struct Pairs { @@ -2283,10 +2316,6 @@ void process_main(void) jump_on_val_zero_index = xb(Arg(0)); goto do_jump_on_val_zero_index; - OpCase(i_jump_on_val_zero_rfI): - jump_on_val_zero_index = r(0); - I--; - do_jump_on_val_zero_index: if (is_small(jump_on_val_zero_index)) { jump_on_val_zero_index = signed_val(jump_on_val_zero_index); @@ -2311,10 +2340,6 @@ void process_main(void) jump_on_val_index = xb(Arg(0)); goto do_jump_on_val_index; - OpCase(i_jump_on_val_rfII): - jump_on_val_index = r(0); - I--; - do_jump_on_val_index: if (is_small(jump_on_val_index)) { jump_on_val_index = (Uint) (signed_val(jump_on_val_index) - Arg(3)); @@ -2334,15 +2359,12 @@ void process_main(void) do { Eterm term = *I++; - switch (term & _TAG_IMMED1_MASK) { - case (R_REG_DEF << _TAG_PRIMARY_SIZE) | TAG_PRIMARY_HEADER: - *hp++ = r(0); + switch (loader_tag(term)) { + case LOADER_X_REG: + *hp++ = x(loader_x_reg_index(term)); break; - case (X_REG_DEF << _TAG_PRIMARY_SIZE) | TAG_PRIMARY_HEADER: - *hp++ = x(term >> _TAG_IMMED1_SIZE); - break; - case (Y_REG_DEF << _TAG_PRIMARY_SIZE) | TAG_PRIMARY_HEADER: - *hp++ = y(term >> _TAG_IMMED1_SIZE); + case LOADER_Y_REG: + *hp++ = y(loader_y_reg_index(term)); break; default: *hp++ = term; @@ -2353,93 +2375,35 @@ void process_main(void) Goto(*I); } - OpCase(new_map_jdII): { + OpCase(new_map_dII): { Eterm res; - x(0) = r(0); - SWAPOUT; - res = new_map(c_p, reg, I); - SWAPIN; - r(0) = x(0); - StoreResult(res, Arg(1)); - Next(4+Arg(3)); - } - - OpCase(i_has_map_fields_fsI): { - map_t* mp; - Eterm map; - Eterm field; - Eterm *ks; - BeamInstr* fs; - Uint sz,n; - - GetArg1(1, map); - - /* this instruction assumes Arg1 is a map, - * i.e. that it follows a test is_map if needed. - */ - - mp = (map_t *)map_val(map); - sz = map_get_size(mp); - - if (sz == 0) { - SET_I((BeamInstr *) Arg(0)); - goto has_map_fields_fail; - } - - ks = map_get_keys(mp); - n = (Uint)Arg(2); - fs = &Arg(3); /* pattern fields */ - - ASSERT(n>0); - - while(sz) { - field = (Eterm)*fs; - if (EQ(field,*ks)) { - n--; - fs++; - if (n == 0) break; - } - ks++; sz--; - } - - if (n) { - SET_I((BeamInstr *) Arg(0)); - goto has_map_fields_fail; - } - - I += 4 + Arg(2); -has_map_fields_fail: - ASSERT(VALID_INSTR(*I)); - Goto(*I); - } - -#define PUT_TERM_REG(term, desc) \ -do { \ - switch ((desc) & _TAG_IMMED1_MASK) { \ - case (R_REG_DEF << _TAG_PRIMARY_SIZE) | TAG_PRIMARY_HEADER: \ - r(0) = (term); \ - break; \ - case (X_REG_DEF << _TAG_PRIMARY_SIZE) | TAG_PRIMARY_HEADER: \ - x((desc) >> _TAG_IMMED1_SIZE) = (term); \ - break; \ - case (Y_REG_DEF << _TAG_PRIMARY_SIZE) | TAG_PRIMARY_HEADER: \ - y((desc) >> _TAG_IMMED1_SIZE) = (term); \ - break; \ - default: \ - ASSERT(0); \ - break; \ - } \ + HEAVY_SWAPOUT; + res = new_map(c_p, reg, I-1); + HEAVY_SWAPIN; + StoreResult(res, Arg(0)); + Next(3+Arg(2)); + } + +#define PUT_TERM_REG(term, desc) \ +do { \ + switch (loader_tag(desc)) { \ + case LOADER_X_REG: \ + x(loader_x_reg_index(desc)) = (term); \ + break; \ + case LOADER_Y_REG: \ + y(loader_y_reg_index(desc)) = (term); \ + break; \ + default: \ + ASSERT(0); \ + break; \ + } \ } while(0) OpCase(i_get_map_elements_fsI): { Eterm map; - map_t *mp; - Eterm field; - Eterm *ks; - Eterm *vs; BeamInstr *fs; - Uint sz,n; + Uint sz, n; GetArg1(1, map); @@ -2447,41 +2411,55 @@ do { \ * i.e. that it follows a test is_map if needed. */ - mp = (map_t *)map_val(map); - sz = map_get_size(mp); + n = (Uint)Arg(2) / 3; + fs = &Arg(3); /* pattern fields and target registers */ + + if (is_flatmap(map)) { + flatmap_t *mp; + Eterm *ks; + Eterm *vs; - if (sz == 0) { - SET_I((BeamInstr *) Arg(0)); - goto get_map_elements_fail; - } + mp = (flatmap_t *)flatmap_val(map); + sz = flatmap_get_size(mp); - n = (Uint)Arg(2) / 2; - fs = &Arg(3); /* pattern fields and target registers */ - ks = map_get_keys(mp); - vs = map_get_values(mp); + if (sz == 0) { + ClauseFail(); + } - while(sz) { - field = (Eterm)*fs; - if (EQ(field,*ks)) { - PUT_TERM_REG(*vs, fs[1]); - n--; - fs += 2; - /* no more values to fetch, we are done */ - if (n == 0) break; + ks = flatmap_get_keys(mp); + vs = flatmap_get_values(mp); + + while(sz) { + if (EQ((Eterm) fs[0], *ks)) { + PUT_TERM_REG(*vs, fs[1]); + n--; + fs += 3; + /* no more values to fetch, we are done */ + if (n == 0) { + I = fs; + Next(-1); + } + } + ks++, sz--, vs++; } - ks++; sz--; - vs++; - } - if (n) { - SET_I((BeamInstr *) Arg(0)); - goto get_map_elements_fail; + ClauseFail(); + } else { + const Eterm *v; + Uint32 hx; + ASSERT(is_hashmap(map)); + while(n--) { + hx = fs[2]; + ASSERT(hx == hashmap_make_hash((Eterm)fs[0])); + if ((v = erts_hashmap_get(hx, (Eterm)fs[0], map)) == NULL) { + ClauseFail(); + } + PUT_TERM_REG(*v, fs[1]); + fs += 3; + } + I = fs; + Next(-1); } - - I += 4 + Arg(2); -get_map_elements_fail: - ASSERT(VALID_INSTR(*I)); - Goto(*I); } #undef PUT_TERM_REG @@ -2490,16 +2468,20 @@ get_map_elements_fail: Eterm map; GetArg1(1, map); - x(0) = r(0); - SWAPOUT; + HEAVY_SWAPOUT; res = update_map_assoc(c_p, reg, map, I); - SWAPIN; + HEAVY_SWAPIN; if (is_value(res)) { - r(0) = x(0); StoreResult(res, Arg(2)); Next(5+Arg(4)); } else { - goto badarg; + /* + * This can only happen if the code was compiled + * with the compiler in OTP 17. + */ + c_p->freason = BADMAP; + c_p->fvalue = map; + goto lb_Cl_error; } } @@ -2508,16 +2490,14 @@ get_map_elements_fail: Eterm map; GetArg1(1, map); - x(0) = r(0); - SWAPOUT; + HEAVY_SWAPOUT; res = update_map_exact(c_p, reg, map, I); - SWAPIN; + HEAVY_SWAPIN; if (is_value(res)) { - r(0) = x(0); StoreResult(res, Arg(2)); Next(5+Arg(4)); } else { - goto badarg; + goto lb_Cl_error; } } @@ -2595,13 +2575,10 @@ get_map_elements_fail: { typedef Eterm (*GcBifFunction)(Process*, Eterm*, Uint); GcBifFunction bf; - Eterm arg; Eterm result; Uint live = (Uint) Arg(3); - GetArg1(2, arg); - reg[0] = r(0); - reg[live] = arg; + GetArg1(2, x(live)); bf = (GcBifFunction) Arg(1); c_p->fcalls = FCALLS; SWAPOUT; @@ -2612,7 +2589,6 @@ get_map_elements_fail: ERTS_SMP_REQ_PROC_MAIN_LOCK(c_p); PROCESS_MAIN_CHK_LOCKS(c_p); SWAPIN; - r(0) = reg[0]; ERTS_HOLE_CHECK(c_p); FCALLS = c_p->fcalls; if (is_value(result)) { @@ -2622,12 +2598,12 @@ get_map_elements_fail: SET_I((BeamInstr *) Arg(0)); Goto(*I); } - reg[0] = arg; + x(0) = x(live); I = handle_error(c_p, I, reg, translate_gc_bif((void *) bf)); goto post_error_handling; } - OpCase(i_gc_bif2_jIId): /* Note, one less parameter than the i_gc_bif1 + OpCase(i_gc_bif2_jIIssd): /* Note, one less parameter than the i_gc_bif1 and i_gc_bif3 */ { typedef Eterm (*GcBifFunction)(Process*, Eterm*, Uint); @@ -2635,9 +2611,13 @@ get_map_elements_fail: Eterm result; Uint live = (Uint) Arg(2); - reg[0] = r(0); - reg[live++] = tmp_arg1; - reg[live] = tmp_arg2; + GetArg2(3, x(live), x(live+1)); + /* + * XXX This calling convention does not make sense. 'live' + * should point out the first argument, not the second + * (i.e. 'live' should not be incremented below). + */ + live++; bf = (GcBifFunction) Arg(1); c_p->fcalls = FCALLS; SWAPOUT; @@ -2648,35 +2628,37 @@ get_map_elements_fail: ERTS_SMP_REQ_PROC_MAIN_LOCK(c_p); PROCESS_MAIN_CHK_LOCKS(c_p); SWAPIN; - r(0) = reg[0]; ERTS_HOLE_CHECK(c_p); FCALLS = c_p->fcalls; if (is_value(result)) { - StoreBifResult(3, result); + StoreBifResult(5, result); } if (Arg(0) != 0) { SET_I((BeamInstr *) Arg(0)); Goto(*I); } - reg[0] = tmp_arg1; - reg[1] = tmp_arg2; + live--; + x(0) = x(live); + x(1) = x(live+1); I = handle_error(c_p, I, reg, translate_gc_bif((void *) bf)); goto post_error_handling; } - OpCase(i_gc_bif3_jIsId): + OpCase(i_gc_bif3_jIIssd): { typedef Eterm (*GcBifFunction)(Process*, Eterm*, Uint); GcBifFunction bf; - Eterm arg; Eterm result; - Uint live = (Uint) Arg(3); + Uint live = (Uint) Arg(2); - GetArg1(2, arg); - reg[0] = r(0); - reg[live++] = arg; - reg[live++] = tmp_arg1; - reg[live] = tmp_arg2; + x(live) = x(SCRATCH_X_REG); + GetArg2(3, x(live+1), x(live+2)); + /* + * XXX This calling convention does not make sense. 'live' + * should point out the first argument, not the third + * (i.e. 'live' should not be incremented below). + */ + live += 2; bf = (GcBifFunction) Arg(1); c_p->fcalls = FCALLS; SWAPOUT; @@ -2687,19 +2669,19 @@ get_map_elements_fail: ERTS_SMP_REQ_PROC_MAIN_LOCK(c_p); PROCESS_MAIN_CHK_LOCKS(c_p); SWAPIN; - r(0) = reg[0]; ERTS_HOLE_CHECK(c_p); FCALLS = c_p->fcalls; if (is_value(result)) { - StoreBifResult(4, result); + StoreBifResult(5, result); } if (Arg(0) != 0) { SET_I((BeamInstr *) Arg(0)); Goto(*I); } - reg[0] = arg; - reg[1] = tmp_arg1; - reg[2] = tmp_arg2; + live -= 2; + x(0) = x(live); + x(1) = x(live+1); + x(2) = x(live+2); I = handle_error(c_p, I, reg, translate_gc_bif((void *) bf)); goto post_error_handling; } @@ -2707,12 +2689,13 @@ get_map_elements_fail: /* * Guards bifs and, or, xor in guards. */ - OpCase(i_bif2_fbd): + OpCase(i_bif2_fbssd): { - Eterm tmp_reg[2] = {tmp_arg1, tmp_arg2}; + Eterm tmp_reg[2]; Eterm (*bf)(Process*, Eterm*); Eterm result; + GetArg2(2, tmp_reg[0], tmp_reg[1]); bf = (BifFunction) Arg(1); c_p->fcalls = FCALLS; PROCESS_MAIN_CHK_LOCKS(c_p); @@ -2724,7 +2707,7 @@ get_map_elements_fail: ERTS_HOLE_CHECK(c_p); FCALLS = c_p->fcalls; if (is_value(result)) { - StoreBifResult(2, result); + StoreBifResult(4, result); } SET_I((BeamInstr *) Arg(0)); Goto(*I); @@ -2733,12 +2716,13 @@ get_map_elements_fail: /* * Guards bifs and, or, xor, relational operators in body. */ - OpCase(i_bif2_body_bd): + OpCase(i_bif2_body_bssd): { - Eterm tmp_reg[2] = {tmp_arg1, tmp_arg2}; + Eterm tmp_reg[2]; Eterm (*bf)(Process*, Eterm*); Eterm result; + GetArg2(1, tmp_reg[0], tmp_reg[1]); bf = (BifFunction) Arg(0); PROCESS_MAIN_CHK_LOCKS(c_p); ASSERT(!ERTS_PROC_IS_EXITING(c_p)); @@ -2749,10 +2733,10 @@ get_map_elements_fail: ERTS_HOLE_CHECK(c_p); if (is_value(result)) { ASSERT(!is_CP(result)); - StoreBifResult(1, result); + StoreBifResult(3, result); } - reg[0] = tmp_arg1; - reg[1] = tmp_arg2; + reg[0] = tmp_reg[0]; + reg[1] = tmp_reg[1]; SWAPOUT; I = handle_error(c_p, I, reg, bf); goto post_error_handling; @@ -2764,9 +2748,20 @@ get_map_elements_fail: */ OpCase(call_bif_e): { - Eterm (*bf)(Process*, Eterm*, BeamInstr*) = GET_BIF_ADDRESS(Arg(0)); + Eterm (*bf)(Process*, Eterm*, BeamInstr*); Eterm result; BeamInstr *next; + ErlHeapFragment *live_hf_end; + + if (ERTS_MSACC_IS_ENABLED_CACHED_X()) { + if (GET_BIF_MODULE(Arg(0)) == am_ets) { + ERTS_MSACC_SET_STATE_CACHED_M_X(ERTS_MSACC_STATE_ETS); + } else { + ERTS_MSACC_SET_STATE_CACHED_M_X(ERTS_MSACC_STATE_BIF); + } + } + + bf = GET_BIF_ADDRESS(Arg(0)); PRE_BIF_SWAPOUT(c_p); c_p->fcalls = FCALLS - 1; @@ -2775,20 +2770,27 @@ get_map_elements_fail: } PreFetch(1, next); ASSERT(!ERTS_PROC_IS_EXITING(c_p)); - reg[0] = r(0); + ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p); + live_hf_end = c_p->mbuf; result = (*bf)(c_p, reg, I); ASSERT(!ERTS_PROC_IS_EXITING(c_p) || is_non_value(result)); ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p); ERTS_HOLE_CHECK(c_p); ERTS_SMP_REQ_PROC_MAIN_LOCK(c_p); - PROCESS_MAIN_CHK_LOCKS(c_p); - if (c_p->mbuf || MSO(c_p).overhead >= BIN_VHEAP_SZ(c_p)) { + if (ERTS_IS_GC_DESIRED(c_p)) { Uint arity = ((Export *)Arg(0))->code[2]; - result = erts_gc_after_bif_call(c_p, result, reg, arity); + result = erts_gc_after_bif_call_lhf(c_p, live_hf_end, result, reg, arity); E = c_p->stop; } + PROCESS_MAIN_CHK_LOCKS(c_p); HTOP = HEAP_TOP(c_p); FCALLS = c_p->fcalls; + /* We have to update the cache if we are enabled in order + to make sure no book keeping is done after we disabled + msacc. We don't always do this as it is quite expensive. */ + if (ERTS_MSACC_IS_ENABLED_CACHED_X()) + ERTS_MSACC_UPDATE_CACHE_X(); + ERTS_MSACC_SET_STATE_CACHED_M_X(ERTS_MSACC_STATE_EMULATOR); if (is_value(result)) { r(0) = result; CHECK_TERM(r(0)); @@ -2797,7 +2799,6 @@ get_map_elements_fail: SET_CP(c_p, I+2); SET_I(c_p->i); SWAPIN; - r(0) = reg[0]; Dispatch(); } @@ -2813,82 +2814,81 @@ get_map_elements_fail: * Arithmetic operations. */ - OpCase(i_times_jId): + OpCase(i_times_jIssd): { - arith_func = ARITH_FUNC(mixed_times); - goto do_big_arith2; + Eterm Op1, Op2; + GetArg2(2, Op1, Op2); + DO_OUTLINED_ARITH_2(mixed_times, Op1, Op2); } - OpCase(i_m_div_jId): + OpCase(i_m_div_jIssd): { - arith_func = ARITH_FUNC(mixed_div); - goto do_big_arith2; + Eterm Op1, Op2; + GetArg2(2, Op1, Op2); + DO_OUTLINED_ARITH_2(mixed_div, Op1, Op2); } - OpCase(i_int_div_jId): + OpCase(i_int_div_jIssd): { - Eterm result; + Eterm Op1, Op2; - if (tmp_arg2 == SMALL_ZERO) { + GetArg2(2, Op1, Op2); + if (Op2 == SMALL_ZERO) { goto badarith; - } else if (is_both_small(tmp_arg1, tmp_arg2)) { - Sint ires = signed_val(tmp_arg1) / signed_val(tmp_arg2); + } else if (is_both_small(Op1, Op2)) { + Sint ires = signed_val(Op1) / signed_val(Op2); if (MY_IS_SSMALL(ires)) { - result = make_small(ires); - STORE_ARITH_RESULT(result); + Eterm result = make_small(ires); + StoreBifResult(4, result); } } - arith_func = ARITH_FUNC(int_div); - goto do_big_arith2; + DO_OUTLINED_ARITH_2(int_div, Op1, Op2); } - OpCase(i_rem_jId): { - Eterm result; - - if (tmp_arg2 == SMALL_ZERO) { - goto badarith; - } else if (is_both_small(tmp_arg1, tmp_arg2)) { - result = make_small(signed_val(tmp_arg1) % signed_val(tmp_arg2)); - STORE_ARITH_RESULT(result); + Eterm RemOp1, RemOp2; + + OpCase(i_rem_jIxxd): + RemOp1 = xb(Arg(2)); + RemOp2 = xb(Arg(3)); + goto do_rem; + + OpCase(i_rem_jIssd): + GetArg2(2, RemOp1, RemOp2); + goto do_rem; + + do_rem: + if (RemOp2 == SMALL_ZERO) { + goto badarith; + } else if (is_both_small(RemOp1, RemOp2)) { + Eterm result = make_small(signed_val(RemOp1) % signed_val(RemOp2)); + StoreBifResult(4, result); } else { - arith_func = ARITH_FUNC(int_rem); - goto do_big_arith2; + DO_OUTLINED_ARITH_2(int_rem, RemOp1, RemOp2); } } - OpCase(i_band_jId): { - Eterm result; - - if (is_both_small(tmp_arg1, tmp_arg2)) { - /* - * No need to untag -- TAG & TAG == TAG. - */ - result = tmp_arg1 & tmp_arg2; - STORE_ARITH_RESULT(result); + Eterm BandOp1, BandOp2; + + OpCase(i_band_jIxcd): + BandOp1 = xb(Arg(2)); + BandOp2 = Arg(3); + goto do_band; + + OpCase(i_band_jIssd): + GetArg2(2, BandOp1, BandOp2); + goto do_band; + + do_band: + if (is_both_small(BandOp1, BandOp2)) { + /* + * No need to untag -- TAG & TAG == TAG. + */ + Eterm result = BandOp1 & BandOp2; + StoreBifResult(4, result); } - arith_func = ARITH_FUNC(band); - goto do_big_arith2; - } - - do_big_arith2: - { - Eterm result; - Uint live = Arg(1); - - SWAPOUT; - reg[0] = r(0); - reg[live] = tmp_arg1; - reg[live+1] = tmp_arg2; - result = arith_func(c_p, reg, live); - r(0) = reg[0]; - SWAPIN; - ERTS_HOLE_CHECK(c_p); - if (is_value(result)) { - STORE_ARITH_RESULT(result); - } - goto lb_Cl_error; + DO_OUTLINED_ARITH_2(band, BandOp1, BandOp2); } /* @@ -2909,97 +2909,105 @@ get_map_elements_fail: goto find_func_info; } - OpCase(i_bor_jId): + OpCase(i_bor_jIssd): { - Eterm result; + Eterm Op1, Op2; - if (is_both_small(tmp_arg1, tmp_arg2)) { + GetArg2(2, Op1, Op2); + if (is_both_small(Op1, Op2)) { /* * No need to untag -- TAG | TAG == TAG. */ - result = tmp_arg1 | tmp_arg2; - STORE_ARITH_RESULT(result); + Eterm result = Op1 | Op2; + StoreBifResult(4, result); } - arith_func = ARITH_FUNC(bor); - goto do_big_arith2; + DO_OUTLINED_ARITH_2(bor, Op1, Op2); } - OpCase(i_bxor_jId): + OpCase(i_bxor_jIssd): { - Eterm result; + Eterm Op1, Op2; - if (is_both_small(tmp_arg1, tmp_arg2)) { + GetArg2(2, Op1, Op2); + if (is_both_small(Op1, Op2)) { /* * We could extract the tag from one argument, but a tag extraction * could mean a shift. Therefore, play it safe here. */ - result = make_small(signed_val(tmp_arg1) ^ signed_val(tmp_arg2)); - STORE_ARITH_RESULT(result); + Eterm result = make_small(signed_val(Op1) ^ signed_val(Op2)); + StoreBifResult(4, result); } - arith_func = ARITH_FUNC(bxor); - goto do_big_arith2; + DO_OUTLINED_ARITH_2(bxor, Op1, Op2); } { + Eterm Op1, Op2; Sint i; Sint ires; Eterm* bigp; + Eterm tmp_big[2]; - OpCase(i_bsr_jId): - if (is_small(tmp_arg2)) { - i = -signed_val(tmp_arg2); - if (is_small(tmp_arg1)) { + OpCase(i_bsr_jIssd): + GetArg2(2, Op1, Op2); + if (is_small(Op2)) { + i = -signed_val(Op2); + if (is_small(Op1)) { goto small_shift; - } else if (is_big(tmp_arg1)) { + } else if (is_big(Op1)) { if (i == 0) { - StoreBifResult(2, tmp_arg1); + StoreBifResult(4, Op1); } goto big_shift; } - } else if (is_big(tmp_arg2)) { + } else if (is_big(Op2)) { /* * N bsr NegativeBigNum == N bsl MAX_SMALL * N bsr PositiveBigNum == N bsl MIN_SMALL */ - tmp_arg2 = make_small(bignum_header_is_neg(*big_val(tmp_arg2)) ? + Op2 = make_small(bignum_header_is_neg(*big_val(Op2)) ? MAX_SMALL : MIN_SMALL); goto do_bsl; } goto badarith; - OpCase(i_bsl_jId): + OpCase(i_bsl_jIssd): + GetArg2(2, Op1, Op2); + do_bsl: - if (is_small(tmp_arg2)) { - i = signed_val(tmp_arg2); + if (is_small(Op2)) { + i = signed_val(Op2); - if (is_small(tmp_arg1)) { + if (is_small(Op1)) { small_shift: - ires = signed_val(tmp_arg1); + ires = signed_val(Op1); if (i == 0 || ires == 0) { - StoreBifResult(2, tmp_arg1); + StoreBifResult(4, Op1); } else if (i < 0) { /* Right shift */ i = -i; if (i >= SMALL_BITS-1) { - tmp_arg1 = (ires < 0) ? SMALL_MINUS_ONE : SMALL_ZERO; + Op1 = (ires < 0) ? SMALL_MINUS_ONE : SMALL_ZERO; } else { - tmp_arg1 = make_small(ires >> i); + Op1 = make_small(ires >> i); } - StoreBifResult(2, tmp_arg1); + StoreBifResult(4, Op1); } else if (i < SMALL_BITS-1) { /* Left shift */ if ((ires > 0 && ((~(Uint)0 << ((SMALL_BITS-1)-i)) & ires) == 0) || ((~(Uint)0 << ((SMALL_BITS-1)-i)) & ~ires) == 0) { - tmp_arg1 = make_small(ires << i); - StoreBifResult(2, tmp_arg1); + Op1 = make_small(ires << i); + StoreBifResult(4, Op1); } } - tmp_arg1 = small_to_big(ires, tmp_big); + Op1 = small_to_big(ires, tmp_big); +#ifdef TAG_LITERAL_PTR + Op1 |= TAG_LITERAL_PTR; +#endif big_shift: if (i > 0) { /* Left shift. */ - ires = big_size(tmp_arg1) + (i / D_EXP); + ires = big_size(Op1) + (i / D_EXP); } else { /* Right shift. */ - ires = big_size(tmp_arg1); + ires = big_size(Op1); if (ires <= (-i / D_EXP)) ires = 3; /* ??? */ else @@ -3017,14 +3025,14 @@ get_map_elements_fail: c_p->freason = SYSTEM_LIMIT; goto lb_Cl_error; } - TestHeapPreserve(ires+1, Arg(1), tmp_arg1); + TestHeapPreserve(ires+1, Arg(1), Op1); bigp = HTOP; - tmp_arg1 = big_lshift(tmp_arg1, i, bigp); - if (is_big(tmp_arg1)) { + Op1 = big_lshift(Op1, i, bigp); + if (is_big(Op1)) { HTOP += bignum_header_arity(*HTOP) + 1; } HEAP_SPACE_VERIFIED(0); - if (is_nil(tmp_arg1)) { + if (is_nil(Op1)) { /* * This result must have been only slight larger * than allowed since it wasn't caught by the @@ -3034,25 +3042,25 @@ get_map_elements_fail: goto lb_Cl_error; } ERTS_HOLE_CHECK(c_p); - StoreBifResult(2, tmp_arg1); + StoreBifResult(4, Op1); } - } else if (is_big(tmp_arg1)) { + } else if (is_big(Op1)) { if (i == 0) { - StoreBifResult(2, tmp_arg1); + StoreBifResult(4, Op1); } goto big_shift; } - } else if (is_big(tmp_arg2)) { - if (bignum_header_is_neg(*big_val(tmp_arg2))) { + } else if (is_big(Op2)) { + if (bignum_header_is_neg(*big_val(Op2))) { /* * N bsl NegativeBigNum is either 0 or -1, depending on * the sign of N. Since we don't believe this case * is common, do the calculation with the minimum * amount of code. */ - tmp_arg2 = make_small(MIN_SMALL); + Op2 = make_small(MIN_SMALL); goto do_bsl; - } else if (is_small(tmp_arg1) || is_big(tmp_arg1)) { + } else if (is_small(Op1) || is_big(Op1)) { /* * N bsl PositiveBigNum is too large to represent. */ @@ -3076,12 +3084,10 @@ get_map_elements_fail: bnot_val = make_small(~signed_val(bnot_val)); } else { Uint live = Arg(2); - SWAPOUT; - reg[0] = r(0); + HEAVY_SWAPOUT; reg[live] = bnot_val; bnot_val = erts_gc_bnot(c_p, reg, live); - r(0) = reg[0]; - SWAPIN; + HEAVY_SWAPIN; ERTS_HOLE_CHECK(c_p); if (is_nil(bnot_val)) { goto lb_Cl_error; @@ -3096,11 +3102,10 @@ get_map_elements_fail: OpCase(i_apply): { BeamInstr *next; - SWAPOUT; + HEAVY_SWAPOUT; next = apply(c_p, r(0), x(1), x(2), reg); - SWAPIN; + HEAVY_SWAPIN; if (next != NULL) { - r(0) = reg[0]; SET_CP(c_p, I+1); SET_I(next); Dispatch(); @@ -3111,12 +3116,11 @@ get_map_elements_fail: OpCase(i_apply_last_P): { BeamInstr *next; - SWAPOUT; + HEAVY_SWAPOUT; next = apply(c_p, r(0), x(1), x(2), reg); - SWAPIN; + HEAVY_SWAPIN; if (next != NULL) { - r(0) = reg[0]; - SET_CP(c_p, (BeamInstr *) EXPAND_POINTER(E[0])); + SET_CP(c_p, (BeamInstr *) E[0]); E = ADD_BYTE_OFFSET(E, Arg(0)); SET_I(next); Dispatch(); @@ -3127,11 +3131,10 @@ get_map_elements_fail: OpCase(i_apply_only): { BeamInstr *next; - SWAPOUT; + HEAVY_SWAPOUT; next = apply(c_p, r(0), x(1), x(2), reg); - SWAPIN; + HEAVY_SWAPIN; if (next != NULL) { - r(0) = reg[0]; SET_I(next); Dispatch(); } @@ -3142,12 +3145,10 @@ get_map_elements_fail: OpCase(apply_I): { BeamInstr *next; - reg[0] = r(0); - SWAPOUT; + HEAVY_SWAPOUT; next = fixed_apply(c_p, reg, Arg(0)); - SWAPIN; + HEAVY_SWAPIN; if (next != NULL) { - r(0) = reg[0]; SET_CP(c_p, I+2); SET_I(next); Dispatch(); @@ -3159,13 +3160,11 @@ get_map_elements_fail: OpCase(apply_last_IP): { BeamInstr *next; - reg[0] = r(0); - SWAPOUT; + HEAVY_SWAPOUT; next = fixed_apply(c_p, reg, Arg(0)); - SWAPIN; + HEAVY_SWAPIN; if (next != NULL) { - r(0) = reg[0]; - SET_CP(c_p, (BeamInstr *) EXPAND_POINTER(E[0])); + SET_CP(c_p, (BeamInstr *) E[0]); E = ADD_BYTE_OFFSET(E, Arg(1)); SET_I(next); Dispatch(); @@ -3177,11 +3176,10 @@ get_map_elements_fail: OpCase(i_apply_fun): { BeamInstr *next; - SWAPOUT; + HEAVY_SWAPOUT; next = apply_fun(c_p, r(0), x(1), reg); - SWAPIN; + HEAVY_SWAPIN; if (next != NULL) { - r(0) = reg[0]; SET_CP(c_p, I+1); SET_I(next); Dispatchfun(); @@ -3192,12 +3190,11 @@ get_map_elements_fail: OpCase(i_apply_fun_last_P): { BeamInstr *next; - SWAPOUT; + HEAVY_SWAPOUT; next = apply_fun(c_p, r(0), x(1), reg); - SWAPIN; + HEAVY_SWAPIN; if (next != NULL) { - r(0) = reg[0]; - SET_CP(c_p, (BeamInstr *) EXPAND_POINTER(E[0])); + SET_CP(c_p, (BeamInstr *) E[0]); E = ADD_BYTE_OFFSET(E, Arg(0)); SET_I(next); Dispatchfun(); @@ -3208,11 +3205,10 @@ get_map_elements_fail: OpCase(i_apply_fun_only): { BeamInstr *next; - SWAPOUT; + HEAVY_SWAPOUT; next = apply_fun(c_p, r(0), x(1), reg); - SWAPIN; + HEAVY_SWAPIN; if (next != NULL) { - r(0) = reg[0]; SET_I(next); Dispatchfun(); } @@ -3222,13 +3218,10 @@ get_map_elements_fail: OpCase(i_call_fun_I): { BeamInstr *next; - SWAPOUT; - reg[0] = r(0); - + HEAVY_SWAPOUT; next = call_fun(c_p, Arg(0), reg, THE_NON_VALUE); - SWAPIN; + HEAVY_SWAPIN; if (next != NULL) { - r(0) = reg[0]; SET_CP(c_p, I+2); SET_I(next); Dispatchfun(); @@ -3239,13 +3232,11 @@ get_map_elements_fail: OpCase(i_call_fun_last_IP): { BeamInstr *next; - SWAPOUT; - reg[0] = r(0); + HEAVY_SWAPOUT; next = call_fun(c_p, Arg(0), reg, THE_NON_VALUE); - SWAPIN; + HEAVY_SWAPIN; if (next != NULL) { - r(0) = reg[0]; - SET_CP(c_p, (BeamInstr *) EXPAND_POINTER(E[0])); + SET_CP(c_p, (BeamInstr *) E[0]); E = ADD_BYTE_OFFSET(E, Arg(1)); SET_I(next); Dispatchfun(); @@ -3329,10 +3320,9 @@ get_map_elements_fail: */ argp = c_p->arg_reg; - for (i = c_p->arity - 1; i > 0; i--) { + for (i = c_p->arity - 1; i >= 0; i--) { argp[i] = reg[i]; } - c_p->arg_reg[0] = r(0); SWAPOUT; c_p->i = I; goto do_schedule1; @@ -3345,19 +3335,14 @@ get_map_elements_fail: Eterm* p; PreFetch(3, next); - GetArg2(0, element, tuple); + GetArg1(0, element); + tuple = REG_TARGET(Arg(1)); ASSERT(is_tuple(tuple)); p = (Eterm *) ((unsigned char *) tuple_val(tuple) + Arg(2)); *p = element; NextPF(3, next); } - OpCase(i_is_ne_exact_f): - if (EQ(tmp_arg1, tmp_arg2)) { - ClauseFail(); - } - Next(1); - OpCase(normal_exit): { SWAPOUT; c_p->freason = EXC_NORMAL; @@ -3424,25 +3409,14 @@ get_map_elements_fail: { Eterm badmatch_val; - OpCase(badmatch_y): - badmatch_val = yb(Arg(0)); - goto do_badmatch; - OpCase(badmatch_x): badmatch_val = xb(Arg(0)); - goto do_badmatch; - - OpCase(badmatch_r): - badmatch_val = r(0); - - do_badmatch: c_p->fvalue = badmatch_val; c_p->freason = BADMATCH; } /* Fall through here */ find_func_info: { - reg[0] = r(0); SWAPOUT; I = handle_error(c_p, I, reg, NULL); goto post_error_handling; @@ -3459,11 +3433,9 @@ get_map_elements_fail: * code[3]: &&call_error_handler * code[4]: Not used */ - SWAPOUT; - reg[0] = r(0); + HEAVY_SWAPOUT; I = call_error_handler(c_p, I-3, reg, am_undefined_function); - r(0) = reg[0]; - SWAPIN; + HEAVY_SWAPIN; if (I) { Goto(*I); } @@ -3471,18 +3443,13 @@ get_map_elements_fail: /* Fall through */ OpCase(error_action_code): { handle_error: - reg[0] = r(0); SWAPOUT; I = handle_error(c_p, NULL, reg, NULL); post_error_handling: if (I == 0) { goto do_schedule; } else { - r(0) = reg[0]; ASSERT(!is_value(r(0))); - if (c_p->mbuf) { - erts_garbage_collect(c_p, 0, reg+1, 3); - } SWAPIN; Goto(*I); } @@ -3503,8 +3470,12 @@ get_map_elements_fail: * I[0]: &&call_nif * I[1]: Function pointer to NIF function * I[2]: Pointer to erl_module_nif + * I[3]: Function pointer to dirty NIF */ BifFunction vbf; + ErlHeapFragment *live_hf_end; + + ERTS_MSACC_SET_STATE_CACHED_M_X(ERTS_MSACC_STATE_NIF); DTRACE_NIF_ENTRY(c_p, (Eterm)I[-3], (Eterm)I[-2], (Uint)I[-1]); c_p->current = I-3; /* current and vbf set to please handle_error */ @@ -3520,21 +3491,18 @@ get_map_elements_fail: NifF* fp = vbf = (NifF*) I[1]; struct enif_environment_t env; erts_pre_nif(&env, c_p, (struct erl_module_nif*)I[2]); - reg[0] = r(0); + live_hf_end = c_p->mbuf; nif_bif_result = (*fp)(&env, bif_nif_arity, reg); + if (env.exception_thrown) + nif_bif_result = THE_NON_VALUE; erts_post_nif(&env); -#ifdef ERTS_DIRTY_SCHEDULERS - if (is_non_value(nif_bif_result) && c_p->freason == TRAP) { - Export* ep = ERTS_PROC_GET_DIRTY_SCHED_TRAP_EXPORT(c_p); - ep->code[0] = I[-3]; - ep->code[1] = I[-2]; - } -#endif } ASSERT(!ERTS_PROC_IS_EXITING(c_p) || is_non_value(nif_bif_result)); PROCESS_MAIN_CHK_LOCKS(c_p); ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p); + ERTS_MSACC_SET_STATE_CACHED_M_X(ERTS_MSACC_STATE_EMULATOR); + DTRACE_NIF_RETURN(c_p, (Eterm)I[-3], (Eterm)I[-2], (Uint)I[-1]); goto apply_bif_or_nif_epilogue; @@ -3549,6 +3517,13 @@ get_map_elements_fail: * code[3]: &&apply_bif * code[4]: Function pointer to BIF function */ + if (ERTS_MSACC_IS_ENABLED_CACHED_X()) { + if ((Eterm)I[-3] == am_ets) { + ERTS_MSACC_SET_STATE_CACHED_M_X(ERTS_MSACC_STATE_ETS); + } else { + ERTS_MSACC_SET_STATE_CACHED_M_X(ERTS_MSACC_STATE_BIF); + } + } c_p->current = I-3; /* In case we apply process_info/1,2 or load_nif/1 */ c_p->i = I; /* In case we apply check_process_code/2. */ @@ -3562,27 +3537,34 @@ get_map_elements_fail: vbf = (BifFunction) Arg(0); PROCESS_MAIN_CHK_LOCKS(c_p); bif_nif_arity = I[-1]; - ASSERT(bif_nif_arity <= 3); + ASSERT(bif_nif_arity <= 4); ERTS_SMP_UNREQ_PROC_MAIN_LOCK(c_p); - reg[0] = r(0); + ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p); { Eterm (*bf)(Process*, Eterm*, BeamInstr*) = vbf; ASSERT(!ERTS_PROC_IS_EXITING(c_p)); + live_hf_end = c_p->mbuf; nif_bif_result = (*bf)(c_p, reg, I); ASSERT(!ERTS_PROC_IS_EXITING(c_p) || is_non_value(nif_bif_result)); ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p); PROCESS_MAIN_CHK_LOCKS(c_p); } - + /* We have to update the cache if we are enabled in order + to make sure no book keeping is done after we disabled + msacc. We don't always do this as it is quite expensive. */ + if (ERTS_MSACC_IS_ENABLED_CACHED_X()) + ERTS_MSACC_UPDATE_CACHE_X(); + ERTS_MSACC_SET_STATE_CACHED_M_X(ERTS_MSACC_STATE_EMULATOR); DTRACE_BIF_RETURN(c_p, (Eterm)I[-3], (Eterm)I[-2], (Uint)I[-1]); apply_bif_or_nif_epilogue: ERTS_SMP_REQ_PROC_MAIN_LOCK(c_p); ERTS_HOLE_CHECK(c_p); - if (c_p->mbuf) { - nif_bif_result = erts_gc_after_bif_call(c_p, nif_bif_result, - reg, bif_nif_arity); + if (ERTS_IS_GC_DESIRED(c_p)) { + nif_bif_result = erts_gc_after_bif_call_lhf(c_p, live_hf_end, + nif_bif_result, + reg, bif_nif_arity); } SWAPIN; /* There might have been a garbage collection. */ FCALLS = c_p->fcalls; @@ -3594,7 +3576,6 @@ get_map_elements_fail: Goto(*I); } else if (c_p->freason == TRAP) { SET_I(c_p->i); - r(0) = reg[0]; if (c_p->flags & F_HIBERNATE_SCHED) { c_p->flags &= ~F_HIBERNATE_SCHED; goto do_schedule; @@ -3616,21 +3597,21 @@ get_map_elements_fail: StoreBifResult(1, result); } + OpCase(i_get_hash_cId): + { + Eterm arg; + Eterm result; + + GetArg1(0, arg); + result = erts_pd_hash_get_with_hx(c_p, Arg(1), arg); + StoreBifResult(2, result); + } + { Eterm case_end_val; OpCase(case_end_x): case_end_val = xb(Arg(0)); - goto do_case_end; - - OpCase(case_end_y): - case_end_val = yb(Arg(0)); - goto do_case_end; - - OpCase(case_end_r): - case_end_val = r(0); - - do_case_end: c_p->fvalue = case_end_val; c_p->freason = EXC_CASE_CLAUSE; goto find_func_info; @@ -3678,19 +3659,13 @@ get_map_elements_fail: goto do_bs_init_bits_known; } - OpCase(i_bs_init_bits_fail_heap_IjId): { - /* tmp_arg1 was fetched by an i_fetch instruction */ - num_bits_term = tmp_arg1; - alloc = Arg(0); - I++; + OpCase(i_bs_init_bits_fail_heap_sIjId): { + GetArg1(0, num_bits_term); + alloc = Arg(1); + I += 2; goto do_bs_init_bits; } - OpCase(i_bs_init_bits_fail_rjId): { - num_bits_term = r(0); - alloc = 0; - goto do_bs_init_bits; - } OpCase(i_bs_init_bits_fail_yjId): { num_bits_term = yb(Arg(0)); I++; @@ -3788,8 +3763,6 @@ get_map_elements_fail: * Allocate the binary struct itself. */ bptr = erts_bin_nrml_alloc(num_bytes); - bptr->flags = 0; - bptr->orig_size = num_bytes; erts_refc_init(&bptr->refc, 1); erts_current_bin = (byte *) bptr->orig_bytes; @@ -3812,52 +3785,48 @@ get_map_elements_fail: } { - OpCase(i_bs_init_fail_heap_IjId): { - /* tmp_arg1 was fetched by an i_fetch instruction */ - tmp_arg2 = Arg(0); - I++; - goto do_bs_init; - } + Eterm BsOp1, BsOp2; - OpCase(i_bs_init_fail_rjId): { - tmp_arg1 = r(0); - tmp_arg2 = 0; + OpCase(i_bs_init_fail_heap_sIjId): { + GetArg1(0, BsOp1); + BsOp2 = Arg(1); + I += 2; goto do_bs_init; } OpCase(i_bs_init_fail_yjId): { - tmp_arg1 = yb(Arg(0)); - tmp_arg2 = 0; + BsOp1 = yb(Arg(0)); + BsOp2 = 0; I++; goto do_bs_init; } OpCase(i_bs_init_fail_xjId): { - tmp_arg1 = xb(Arg(0)); - tmp_arg2 = 0; + BsOp1 = xb(Arg(0)); + BsOp2 = 0; I++; } /* FALL THROUGH */ do_bs_init: - if (is_small(tmp_arg1)) { - Sint size = signed_val(tmp_arg1); + if (is_small(BsOp1)) { + Sint size = signed_val(BsOp1); if (size < 0) { goto badarg; } - tmp_arg1 = (Eterm) size; + BsOp1 = (Eterm) size; } else { Uint bytes; - if (!term_to_Uint(tmp_arg1, &bytes)) { + if (!term_to_Uint(BsOp1, &bytes)) { c_p->freason = bytes; goto lb_Cl_error; } if ((bytes >> (8*sizeof(Uint)-3)) != 0) { goto system_limit; } - tmp_arg1 = (Eterm) bytes; + BsOp1 = (Eterm) bytes; } - if (tmp_arg1 <= ERL_ONHEAP_BIN_LIMIT) { + if (BsOp1 <= ERL_ONHEAP_BIN_LIMIT) { goto do_heap_bin_alloc; } else { goto do_proc_bin_alloc; @@ -3865,15 +3834,15 @@ get_map_elements_fail: OpCase(i_bs_init_heap_IIId): { - tmp_arg1 = Arg(0); - tmp_arg2 = Arg(1); + BsOp1 = Arg(0); + BsOp2 = Arg(1); I++; goto do_proc_bin_alloc; } OpCase(i_bs_init_IId): { - tmp_arg1 = Arg(0); - tmp_arg2 = 0; + BsOp1 = Arg(0); + BsOp2 = 0; } /* FALL THROUGH */ do_proc_bin_alloc: { @@ -3882,15 +3851,13 @@ get_map_elements_fail: erts_bin_offset = 0; erts_writable_bin = 0; - TestBinVHeap(tmp_arg1 / sizeof(Eterm), - tmp_arg2 + PROC_BIN_SIZE + ERL_SUB_BIN_SIZE, Arg(1)); + TestBinVHeap(BsOp1 / sizeof(Eterm), + BsOp2 + PROC_BIN_SIZE + ERL_SUB_BIN_SIZE, Arg(1)); /* * Allocate the binary struct itself. */ - bptr = erts_bin_nrml_alloc(tmp_arg1); - bptr->flags = 0; - bptr->orig_size = tmp_arg1; + bptr = erts_bin_nrml_alloc(BsOp1); erts_refc_init(&bptr->refc, 1); erts_current_bin = (byte *) bptr->orig_bytes; @@ -3900,28 +3867,28 @@ get_map_elements_fail: pb = (ProcBin *) HTOP; HTOP += PROC_BIN_SIZE; pb->thing_word = HEADER_PROC_BIN; - pb->size = tmp_arg1; + pb->size = BsOp1; pb->next = MSO(c_p).first; MSO(c_p).first = (struct erl_off_heap_header*) pb; pb->val = bptr; pb->bytes = (byte*) bptr->orig_bytes; pb->flags = 0; - OH_OVERHEAD(&(MSO(c_p)), tmp_arg1 / sizeof(Eterm)); + OH_OVERHEAD(&(MSO(c_p)), BsOp1 / sizeof(Eterm)); StoreBifResult(2, make_binary(pb)); } OpCase(i_bs_init_heap_bin_heap_IIId): { - tmp_arg1 = Arg(0); - tmp_arg2 = Arg(1); + BsOp1 = Arg(0); + BsOp2 = Arg(1); I++; goto do_heap_bin_alloc; } OpCase(i_bs_init_heap_bin_IId): { - tmp_arg1 = Arg(0); - tmp_arg2 = 0; + BsOp1 = Arg(0); + BsOp2 = 0; } /* Fall through */ do_heap_bin_alloc: @@ -3929,33 +3896,36 @@ get_map_elements_fail: ErlHeapBin* hb; Uint bin_need; - bin_need = heap_bin_size(tmp_arg1); + bin_need = heap_bin_size(BsOp1); erts_bin_offset = 0; erts_writable_bin = 0; - TestHeap(bin_need+tmp_arg2+ERL_SUB_BIN_SIZE, Arg(1)); + TestHeap(bin_need+BsOp2+ERL_SUB_BIN_SIZE, Arg(1)); hb = (ErlHeapBin *) HTOP; HTOP += bin_need; - hb->thing_word = header_heap_bin(tmp_arg1); - hb->size = tmp_arg1; + hb->thing_word = header_heap_bin(BsOp1); + hb->size = BsOp1; erts_current_bin = (byte *) hb->data; - tmp_arg1 = make_binary(hb); - StoreBifResult(2, tmp_arg1); + BsOp1 = make_binary(hb); + StoreBifResult(2, BsOp1); } } - OpCase(i_bs_add_jId): { - Uint Unit = Arg(1); - if (is_both_small(tmp_arg1, tmp_arg2)) { - Sint Arg1 = signed_val(tmp_arg1); - Sint Arg2 = signed_val(tmp_arg2); + OpCase(bs_add_jssId): { + Eterm Op1, Op2; + Uint Unit = Arg(3); + + GetArg2(1, Op1, Op2); + if (is_both_small(Op1, Op2)) { + Sint Arg1 = signed_val(Op1); + Sint Arg2 = signed_val(Op2); if (Arg1 >= 0 && Arg2 >= 0) { - BsSafeMul(Arg2, Unit, goto system_limit, tmp_arg1); - tmp_arg1 += Arg1; + BsSafeMul(Arg2, Unit, goto system_limit, Op1); + Op1 += Arg1; store_bs_add_result: - if (MY_IS_SSMALL((Sint) tmp_arg1)) { - tmp_arg1 = make_small(tmp_arg1); + if (Op1 <= MAX_SMALL) { + Op1 = make_small(Op1); } else { /* * May generate a heap fragment, but in this @@ -3967,10 +3937,10 @@ get_map_elements_fail: * references (such as the heap). */ SWAPOUT; - tmp_arg1 = erts_make_integer(tmp_arg1, c_p); + Op1 = erts_make_integer(Op1, c_p); HTOP = HEAP_TOP(c_p); } - StoreBifResult(2, tmp_arg1); + StoreBifResult(4, Op1); } goto badarg; } else { @@ -3993,16 +3963,16 @@ get_map_elements_fail: * an Uint, the reason is SYSTEM_LIMIT. */ - if (!term_to_Uint(tmp_arg1, &a)) { + if (!term_to_Uint(Op1, &a)) { if (a == BADARG) { goto badarg; } - if (!term_to_Uint(tmp_arg2, &b)) { + if (!term_to_Uint(Op2, &b)) { c_p->freason = b; goto lb_Cl_error; } goto system_limit; - } else if (!term_to_Uint(tmp_arg2, &b)) { + } else if (!term_to_Uint(Op2, &b)) { c_p->freason = b; goto lb_Cl_error; } @@ -4012,8 +3982,8 @@ get_map_elements_fail: */ BsSafeMul(b, Unit, goto system_limit, c); - tmp_arg1 = a + c; - if (tmp_arg1 < a) { + Op1 = a + c; + if (Op1 < a) { /* * If the result is less than one of the * arguments, there must have been an overflow. @@ -4035,52 +4005,47 @@ get_map_elements_fail: } /* - * tmp_arg1 = Number of bytes to build - * tmp_arg2 = Source binary - * Operands: Fail ExtraHeap Live Unit Dst + * x(SCRATCH_X_REG); + * Operands: Fail ExtraHeap Live Unit Size Dst */ - OpCase(i_bs_append_jIIId): { + OpCase(i_bs_append_jIIIsd): { Uint live = Arg(2); Uint res; + Eterm Size; - SWAPOUT; - reg[0] = r(0); - reg[live] = tmp_arg2; - res = erts_bs_append(c_p, reg, live, tmp_arg1, Arg(1), Arg(3)); - r(0) = reg[0]; - SWAPIN; + GetArg1(4, Size); + HEAVY_SWAPOUT; + reg[live] = x(SCRATCH_X_REG); + res = erts_bs_append(c_p, reg, live, Size, Arg(1), Arg(3)); + HEAVY_SWAPIN; if (is_non_value(res)) { /* c_p->freason is already set (may be either BADARG or SYSTEM_LIMIT). */ goto lb_Cl_error; } - StoreBifResult(4, res); + StoreBifResult(5, res); } /* - * tmp_arg1 = Number of bytes to build - * tmp_arg2 = Source binary - * Operands: Fail Unit Dst + * Operands: Fail Size Src Unit Dst */ - OpCase(i_bs_private_append_jId): { + OpCase(i_bs_private_append_jIssd): { Eterm res; + Eterm Size, Src; - res = erts_bs_private_append(c_p, tmp_arg2, tmp_arg1, Arg(1)); + GetArg2(2, Size, Src); + res = erts_bs_private_append(c_p, Src, Size, Arg(1)); if (is_non_value(res)) { /* c_p->freason is already set (may be either BADARG or SYSTEM_LIMIT). */ goto lb_Cl_error; } - StoreBifResult(2, res); + StoreBifResult(4, res); } - /* - * tmp_arg1 = Initial size of writable binary - * Operands: Live Dst - */ OpCase(bs_init_writable): { - SWAPOUT; + HEAVY_SWAPOUT; r(0) = erts_bs_init_writable(c_p, r(0)); - SWAPIN; + HEAVY_SWAPIN; Next(0); } @@ -4171,26 +4136,29 @@ get_map_elements_fail: /* * Only used for validating a value matched out. - * - * tmp_arg1 = Integer to validate - * tmp_arg2 = Match context */ - OpCase(i_bs_validate_unicode_retract_j): { - /* - * There is no need to untag the integer, but it IS necessary - * to make sure it is small (a bignum pointer could fall in - * the valid range). - */ - if (is_not_small(tmp_arg1) || tmp_arg1 > make_small(0x10FFFFUL) || - (make_small(0xD800UL) <= tmp_arg1 && - tmp_arg1 <= make_small(0xDFFFUL))) { - ErlBinMatchBuffer *mb = ms_matchbuffer(tmp_arg2); + OpCase(i_bs_validate_unicode_retract_jss): { + Eterm i; /* Integer to validate */ - mb->offset -= 32; - goto badarg; - } - Next(1); - } + /* + * There is no need to untag the integer, but it IS necessary + * to make sure it is small (a bignum pointer could fall in + * the valid range). + */ + + GetArg1(1, i); + if (is_not_small(i) || i > make_small(0x10FFFFUL) || + (make_small(0xD800UL) <= i && i <= make_small(0xDFFFUL))) { + Eterm ms; /* Match context */ + ErlBinMatchBuffer* mb; + + GetArg1(2, ms); + mb = ms_matchbuffer(ms); + mb->offset -= 32; + goto badarg; + } + Next(3); + } /* * Matching of binaries. @@ -4202,9 +4170,6 @@ get_map_elements_fail: Uint slots; Eterm context; - OpCase(i_bs_start_match2_rfIId): { - context = r(0); - do_start_match: slots = Arg(2); if (!is_boxed(context)) { @@ -4251,7 +4216,7 @@ get_map_elements_fail: ClauseFail(); } NextPF(4, next); - } + OpCase(i_bs_start_match2_xfIId): { context = xb(Arg(0)); I++; @@ -4264,18 +4229,6 @@ get_map_elements_fail: } } - OpCase(bs_test_zero_tail2_fr): { - BeamInstr *next; - ErlBinMatchBuffer *_mb; - - PreFetch(1, next); - _mb = (ErlBinMatchBuffer*) ms_matchbuffer(r(0)); - if (_mb->size != _mb->offset) { - ClauseFail(); - } - NextPF(1, next); - } - OpCase(bs_test_zero_tail2_fx): { BeamInstr *next; ErlBinMatchBuffer *_mb; @@ -4288,16 +4241,6 @@ get_map_elements_fail: NextPF(2, next); } - OpCase(bs_test_tail_imm2_frI): { - BeamInstr *next; - ErlBinMatchBuffer *_mb; - PreFetch(2, next); - _mb = ms_matchbuffer(r(0)); - if (_mb->size - _mb->offset != Arg(1)) { - ClauseFail(); - } - NextPF(2, next); - } OpCase(bs_test_tail_imm2_fxI): { BeamInstr *next; ErlBinMatchBuffer *_mb; @@ -4309,16 +4252,6 @@ get_map_elements_fail: NextPF(3, next); } - OpCase(bs_test_unit_frI): { - BeamInstr *next; - ErlBinMatchBuffer *_mb; - PreFetch(2, next); - _mb = ms_matchbuffer(r(0)); - if ((_mb->size - _mb->offset) % Arg(1)) { - ClauseFail(); - } - NextPF(2, next); - } OpCase(bs_test_unit_fxI): { BeamInstr *next; ErlBinMatchBuffer *_mb; @@ -4330,16 +4263,6 @@ get_map_elements_fail: NextPF(3, next); } - OpCase(bs_test_unit8_fr): { - BeamInstr *next; - ErlBinMatchBuffer *_mb; - PreFetch(1, next); - _mb = ms_matchbuffer(r(0)); - if ((_mb->size - _mb->offset) & 7) { - ClauseFail(); - } - NextPF(1, next); - } OpCase(bs_test_unit8_fx): { BeamInstr *next; ErlBinMatchBuffer *_mb; @@ -4354,19 +4277,11 @@ get_map_elements_fail: { Eterm bs_get_integer8_context; - OpCase(i_bs_get_integer_8_rfd): { - bs_get_integer8_context = r(0); - goto do_bs_get_integer_8; - } - OpCase(i_bs_get_integer_8_xfd): { - bs_get_integer8_context = xb(Arg(0)); - I++; - } - - do_bs_get_integer_8: { ErlBinMatchBuffer *_mb; Eterm _result; + bs_get_integer8_context = xb(Arg(0)); + I++; _mb = ms_matchbuffer(bs_get_integer8_context); if (_mb->size - _mb->offset < 8) { ClauseFail(); @@ -4384,15 +4299,10 @@ get_map_elements_fail: { Eterm bs_get_integer_16_context; - OpCase(i_bs_get_integer_16_rfd): - bs_get_integer_16_context = r(0); - goto do_bs_get_integer_16; - OpCase(i_bs_get_integer_16_xfd): bs_get_integer_16_context = xb(Arg(0)); I++; - do_bs_get_integer_16: { ErlBinMatchBuffer *_mb; Eterm _result; @@ -4413,17 +4323,10 @@ get_map_elements_fail: { Eterm bs_get_integer_32_context; - OpCase(i_bs_get_integer_32_rfId): - bs_get_integer_32_context = r(0); - goto do_bs_get_integer_32; - - OpCase(i_bs_get_integer_32_xfId): bs_get_integer_32_context = xb(Arg(0)); I++; - - do_bs_get_integer_32: { ErlBinMatchBuffer *_mb; Uint32 _integer; @@ -4436,11 +4339,11 @@ get_map_elements_fail: _integer = get_int32(_mb->base + _mb->offset/8); } _mb->offset += 32; -#if !defined(ARCH_64) || HALFWORD_HEAP +#if !defined(ARCH_64) if (IS_USMALL(0, _integer)) { #endif _result = make_small(_integer); -#if !defined(ARCH_64) || HALFWORD_HEAP +#if !defined(ARCH_64) } else { TestHeap(BIG_UINT_HEAP_SIZE, Arg(1)); _result = uint_to_big((Uint) _integer, HTOP); @@ -4452,103 +4355,82 @@ get_map_elements_fail: } } - /* Operands: Size Live Fail Flags Dst */ - OpCase(i_bs_get_integer_imm_rIIfId): { - tmp_arg1 = r(0); - /* Operands: Size Live Fail Flags Dst */ - goto do_bs_get_integer_imm_test_heap; - } + { + Eterm Ms, Sz; - /* Operands: x(Reg) Size Live Fail Flags Dst */ + /* Operands: x(Reg) Size Live Fail Flags Dst */ OpCase(i_bs_get_integer_imm_xIIfId): { - tmp_arg1 = xb(Arg(0)); - I++; - /* Operands: Size Live Fail Flags Dst */ - goto do_bs_get_integer_imm_test_heap; - } - - /* - * tmp_arg1 = match context - * Operands: Size Live Fail Flags Dst - */ - do_bs_get_integer_imm_test_heap: { - Uint wordsneeded; - tmp_arg2 = Arg(0); - wordsneeded = 1+WSIZE(NBYTES(tmp_arg2)); - TestHeapPreserve(wordsneeded, Arg(1), tmp_arg1); - I += 2; - /* Operands: Fail Flags Dst */ - goto do_bs_get_integer_imm; - } - - /* Operands: Size Fail Flags Dst */ - OpCase(i_bs_get_integer_small_imm_rIfId): { - tmp_arg1 = r(0); - tmp_arg2 = Arg(0); - I++; - /* Operands: Fail Flags Dst */ - goto do_bs_get_integer_imm; - } + Uint wordsneeded; + Ms = xb(Arg(0)); + Sz = Arg(1); + wordsneeded = 1+WSIZE(NBYTES(Sz)); + TestHeapPreserve(wordsneeded, Arg(2), Ms); + I += 3; + /* Operands: Fail Flags Dst */ + goto do_bs_get_integer_imm; + } - /* Operands: x(Reg) Size Fail Flags Dst */ + /* Operands: x(Reg) Size Fail Flags Dst */ OpCase(i_bs_get_integer_small_imm_xIfId): { - tmp_arg1 = xb(Arg(0)); - tmp_arg2 = Arg(1); - I += 2; - /* Operands: Fail Flags Dst */ - goto do_bs_get_integer_imm; - } + Ms = xb(Arg(0)); + Sz = Arg(1); + I += 2; + /* Operands: Fail Flags Dst */ + goto do_bs_get_integer_imm; + } - /* - * tmp_arg1 = match context - * tmp_arg2 = size of field - * Operands: Fail Flags Dst - */ + /* + * Ms = match context + * Sz = size of field + * Operands: Fail Flags Dst + */ do_bs_get_integer_imm: { - ErlBinMatchBuffer* mb; - Eterm result; + ErlBinMatchBuffer* mb; + Eterm result; - mb = ms_matchbuffer(tmp_arg1); - LIGHT_SWAPOUT; - result = erts_bs_get_integer_2(c_p, tmp_arg2, Arg(1), mb); - LIGHT_SWAPIN; - HEAP_SPACE_VERIFIED(0); - if (is_non_value(result)) { - ClauseFail(); + mb = ms_matchbuffer(Ms); + LIGHT_SWAPOUT; + result = erts_bs_get_integer_2(c_p, Sz, Arg(1), mb); + LIGHT_SWAPIN; + HEAP_SPACE_VERIFIED(0); + if (is_non_value(result)) { + ClauseFail(); + } + StoreBifResult(2, result); } - StoreBifResult(2, result); } /* - * tmp_arg1 = Match context - * tmp_arg2 = Size field - * Operands: Fail Live FlagsAndUnit Dst + * Operands: Fail Live FlagsAndUnit Ms Sz Dst */ - OpCase(i_bs_get_integer_fIId): { + OpCase(i_bs_get_integer_fIIssd): { Uint flags; Uint size; + Eterm Ms; + Eterm Sz; ErlBinMatchBuffer* mb; Eterm result; flags = Arg(2); - BsGetFieldSize(tmp_arg2, (flags >> 3), ClauseFail(), size); + GetArg2(3, Ms, Sz); + BsGetFieldSize(Sz, (flags >> 3), ClauseFail(), size); if (size >= SMALL_BITS) { Uint wordsneeded; - /* check bits size before potential gc. + /* Check bits size before potential gc. * We do not want a gc and then realize we don't need - * the allocated space (i.e. if the op fails) + * the allocated space (i.e. if the op fails). * - * remember to reacquire the matchbuffer after gc. + * Remember to re-acquire the matchbuffer after gc. */ - mb = ms_matchbuffer(tmp_arg1); + mb = ms_matchbuffer(Ms); if (mb->size - mb->offset < size) { ClauseFail(); } wordsneeded = 1+WSIZE(NBYTES((Uint) size)); - TestHeapPreserve(wordsneeded, Arg(1), tmp_arg1); + TestHeapPreserve(wordsneeded, Arg(1), Ms); } - mb = ms_matchbuffer(tmp_arg1); + mb = ms_matchbuffer(Ms); LIGHT_SWAPOUT; result = erts_bs_get_integer_2(c_p, size, flags, mb); LIGHT_SWAPIN; @@ -4556,18 +4438,13 @@ get_map_elements_fail: if (is_non_value(result)) { ClauseFail(); } - StoreBifResult(3, result); + StoreBifResult(5, result); } { Eterm get_utf8_context; /* Operands: MatchContext Fail Dst */ - OpCase(i_bs_get_utf8_rfd): { - get_utf8_context = r(0); - goto do_bs_get_utf8; - } - OpCase(i_bs_get_utf8_xfd): { get_utf8_context = xb(Arg(0)); I++; @@ -4578,7 +4455,7 @@ get_map_elements_fail: * Operands: Fail Dst */ - do_bs_get_utf8: { + { Eterm result = erts_bs_get_utf8(ms_matchbuffer(get_utf8_context)); if (is_non_value(result)) { ClauseFail(); @@ -4591,12 +4468,7 @@ get_map_elements_fail: Eterm get_utf16_context; /* Operands: MatchContext Fail Flags Dst */ - OpCase(i_bs_get_utf16_rfId): { - get_utf16_context = r(0); - goto do_bs_get_utf16; - } - - OpCase(i_bs_get_utf16_xfId): { + OpCase(i_bs_get_utf16_xfId): { get_utf16_context = xb(Arg(0)); I++; } @@ -4605,7 +4477,7 @@ get_map_elements_fail: * get_utf16_context = match_context * Operands: Fail Flags Dst */ - do_bs_get_utf16: { + { Eterm result = erts_bs_get_utf16(ms_matchbuffer(get_utf16_context), Arg(1)); if (is_non_value(result)) { @@ -4624,26 +4496,10 @@ get_map_elements_fail: Uint orig; Uint hole_size; - OpCase(bs_context_to_binary_r): { - context_to_binary_context = x0; - I -= 2; - goto do_context_to_binary; - } - - /* Unfortunately, inlining can generate this instruction. */ - OpCase(bs_context_to_binary_y): { - context_to_binary_context = yb(Arg(0)); - goto do_context_to_binary0; - } - - OpCase(bs_context_to_binary_x): { + OpCase(bs_context_to_binary_x): context_to_binary_context = xb(Arg(0)); - - do_context_to_binary0: I--; - } - do_context_to_binary: if (is_boxed(context_to_binary_context) && header_is_bin_matchstate(*boxed_val(context_to_binary_context))) { ErlBinMatchState* ms; @@ -4655,17 +4511,11 @@ get_map_elements_fail: } Next(2); - OpCase(i_bs_get_binary_all_reuse_rfI): { - context_to_binary_context = x0; - goto do_bs_get_binary_all_reuse; - } - OpCase(i_bs_get_binary_all_reuse_xfI): { context_to_binary_context = xb(Arg(0)); I++; } - do_bs_get_binary_all_reuse: mb = ms_matchbuffer(context_to_binary_context); size = mb->size - mb->offset; if (size % Arg(1) != 0) { @@ -4693,16 +4543,11 @@ get_map_elements_fail: { Eterm match_string_context; - OpCase(i_bs_match_string_rfII): { - match_string_context = r(0); - goto do_bs_match_string; - } OpCase(i_bs_match_string_xfII): { match_string_context = xb(Arg(0)); I++; } - do_bs_match_string: { BeamInstr *next; byte* bytes; @@ -4730,14 +4575,6 @@ get_map_elements_fail: } } - OpCase(i_bs_save2_rI): { - BeamInstr *next; - ErlBinMatchState *_ms; - PreFetch(1, next); - _ms = (ErlBinMatchState*) boxed_val((Eterm) r(0)); - _ms->save_offset[Arg(0)] = _ms->mb.offset; - NextPF(1, next); - } OpCase(i_bs_save2_xI): { BeamInstr *next; ErlBinMatchState *_ms; @@ -4747,14 +4584,6 @@ get_map_elements_fail: NextPF(2, next); } - OpCase(i_bs_restore2_rI): { - BeamInstr *next; - ErlBinMatchState *_ms; - PreFetch(1, next); - _ms = (ErlBinMatchState*) boxed_val((Eterm) r(0)); - _ms->mb.offset = _ms->save_offset[Arg(0)]; - NextPF(1, next); - } OpCase(i_bs_restore2_xI): { BeamInstr *next; ErlBinMatchState *_ms; @@ -4802,9 +4631,7 @@ get_map_elements_fail: BeamInstr real_I; ASSERT(I[-5] == (BeamInstr) BeamOp(op_i_func_info_IaaI)); SWAPOUT; - reg[0] = r(0); real_I = erts_generic_breakpoint(c_p, I, reg); - r(0) = reg[0]; SWAPIN; ASSERT(VALID_INSTR(real_I)); Goto(real_I); @@ -4864,7 +4691,7 @@ get_map_elements_fail: BeamInstr *next; PreFetch(2, next); - GetR(0, targ1); + targ1 = REG_TARGET(Arg(0)); /* Arg(0) == HEADER_FLONUM */ GET_DOUBLE(targ1, *(FloatDef*)ADD_BYTE_OFFSET(freg, fr)); NextPF(2, next); @@ -4884,7 +4711,7 @@ get_map_elements_fail: Eterm fr = Arg(1); BeamInstr *next; - GetR(0, targ1); + targ1 = REG_TARGET(Arg(0)); PreFetch(2, next); if (is_small(targ1)) { fb(fr) = (double) signed_val(targ1); @@ -4903,7 +4730,7 @@ get_map_elements_fail: #ifdef NO_FPE_SIGNALS OpCase(fclearerror): OpCase(i_fcheckerror): - erl_exit(1, "fclearerror/i_fcheckerror without fpe signals (beam_emu)"); + erts_exit(ERTS_ERROR_EXIT, "fclearerror/i_fcheckerror without fpe signals (beam_emu)"); # define ERTS_NO_FPE_CHECK_INIT ERTS_FP_CHECK_INIT # define ERTS_NO_FPE_ERROR ERTS_FP_ERROR #else @@ -4980,7 +4807,12 @@ get_map_elements_fail: #ifdef HIPE { - unsigned cmd; +#define HIPE_MODE_SWITCH(Cmd) \ + SWAPOUT; \ + c_p->fcalls = FCALLS; \ + c_p->def_arg_reg[4] = -neg_o_reds; \ + c_p = hipe_mode_switch(c_p, Cmd, reg); \ + goto L_post_hipe_mode_switch OpCase(hipe_trap_call): { /* @@ -4993,50 +4825,46 @@ get_map_elements_fail: * ... remainder of original BEAM code */ ASSERT(I[-5] == (Uint) OpCode(i_func_info_IaaI)); - c_p->hipe.ncallee = (void(*)(void)) I[-4]; - cmd = HIPE_MODE_SWITCH_CMD_CALL | (I[-1] << 8); + c_p->hipe.u.ncallee = (void(*)(void)) I[-4]; ++hipe_trap_count; - goto L_hipe_mode_switch; + HIPE_MODE_SWITCH(HIPE_MODE_SWITCH_CMD_CALL | (I[-1] << 8)); } OpCase(hipe_trap_call_closure): { ASSERT(I[-5] == (Uint) OpCode(i_func_info_IaaI)); - c_p->hipe.ncallee = (void(*)(void)) I[-4]; - cmd = HIPE_MODE_SWITCH_CMD_CALL_CLOSURE | (I[-1] << 8); + c_p->hipe.u.ncallee = (void(*)(void)) I[-4]; ++hipe_trap_count; - goto L_hipe_mode_switch; + HIPE_MODE_SWITCH(HIPE_MODE_SWITCH_CMD_CALL_CLOSURE | (I[-1] << 8)); } OpCase(hipe_trap_return): { - cmd = HIPE_MODE_SWITCH_CMD_RETURN; - goto L_hipe_mode_switch; + HIPE_MODE_SWITCH(HIPE_MODE_SWITCH_CMD_RETURN); } OpCase(hipe_trap_throw): { - cmd = HIPE_MODE_SWITCH_CMD_THROW; - goto L_hipe_mode_switch; + HIPE_MODE_SWITCH(HIPE_MODE_SWITCH_CMD_THROW); } OpCase(hipe_trap_resume): { - cmd = HIPE_MODE_SWITCH_CMD_RESUME; - goto L_hipe_mode_switch; + HIPE_MODE_SWITCH(HIPE_MODE_SWITCH_CMD_RESUME); } - L_hipe_mode_switch: - /* XXX: this abuse of def_arg_reg[] is horrid! */ - SWAPOUT; - c_p->fcalls = FCALLS; - c_p->def_arg_reg[4] = -neg_o_reds; - reg[0] = r(0); - c_p = hipe_mode_switch(c_p, cmd, reg); +#undef HIPE_MODE_SWITCH + + L_post_hipe_mode_switch: reg = ERTS_PROC_GET_SCHDATA(c_p)->x_reg_array; freg = ERTS_PROC_GET_SCHDATA(c_p)->f_reg_array; ERL_BITS_RELOAD_STATEP(c_p); + /* XXX: this abuse of def_arg_reg[] is horrid! */ neg_o_reds = -c_p->def_arg_reg[4]; FCALLS = c_p->fcalls; SWAPIN; - switch( c_p->def_arg_reg[3] ) { /* Halfword wont work with hipe yet! */ + switch( c_p->def_arg_reg[3] ) { case HIPE_MODE_SWITCH_RES_RETURN: ASSERT(is_value(reg[0])); - MoveReturn(reg[0], r(0)); - case HIPE_MODE_SWITCH_RES_CALL: + SET_I(c_p->cp); + c_p->cp = 0; + Goto(*I); + case HIPE_MODE_SWITCH_RES_CALL_EXPORTED: + c_p->i = c_p->hipe.u.callee_exp->addressv[erts_active_code_ix()]; + /*fall through*/ + case HIPE_MODE_SWITCH_RES_CALL_BEAM: SET_I(c_p->i); - r(0) = reg[0]; Dispatch(); case HIPE_MODE_SWITCH_RES_CALL_CLOSURE: /* This can be used to call any function value, but currently it's @@ -5045,9 +4873,8 @@ get_map_elements_fail: BeamInstr *next; next = call_fun(c_p, c_p->arity - 1, reg, THE_NON_VALUE); - SWAPIN; + HEAVY_SWAPIN; if (next != NULL) { - r(0) = reg[0]; SET_I(next); Dispatchfun(); } @@ -5058,7 +4885,7 @@ get_map_elements_fail: I = handle_error(c_p, I, reg, NULL); goto post_error_handling; default: - erl_exit(1, "hipe_mode_switch: result %u\n", c_p->def_arg_reg[3]); + erts_exit(ERTS_ERROR_EXIT, "hipe_mode_switch: result %u\n", c_p->def_arg_reg[3]); } } OpCase(hipe_call_count): { @@ -5095,22 +4922,54 @@ get_map_elements_fail: } OpCase(i_hibernate): { - SWAPOUT; + HEAVY_SWAPOUT; if (erts_hibernate(c_p, r(0), x(1), x(2), reg)) { + FCALLS = c_p->fcalls; c_p->flags &= ~F_HIBERNATE_SCHED; goto do_schedule; } else { + HEAVY_SWAPIN; I = handle_error(c_p, I, reg, hibernate_3); goto post_error_handling; } } + /* This is optimised as an instruction because + it has to be very very fast */ + OpCase(i_perf_counter): { + BeamInstr* next; + ErtsSysPerfCounter ts; + PreFetch(0, next); + + ts = erts_sys_perf_counter(); + + if (IS_SSMALL(ts)) { + r(0) = make_small((Sint)ts); + } else { + TestHeap(ERTS_SINT64_HEAP_SIZE(ts),0); + r(0) = make_big(HTOP); +#if defined(ARCH_32) || HALFWORD_HEAP + if (ts >= (((Uint64) 1) << 32)) { + *HTOP = make_pos_bignum_header(2); + BIG_DIGIT(HTOP, 0) = (Uint) (ts & ((Uint) 0xffffffff)); + BIG_DIGIT(HTOP, 1) = (Uint) ((ts >> 32) & ((Uint) 0xffffffff)); + HTOP += 3; + } + else +#endif + { + *HTOP = make_pos_bignum_header(1); + BIG_DIGIT(HTOP, 0) = (Uint) ts; + HTOP += 2; + } + } + NextPF(0, next); + } + OpCase(i_debug_breakpoint): { - SWAPOUT; - reg[0] = r(0); + HEAVY_SWAPOUT; I = call_error_handler(c_p, I-3, reg, am_breakpoint); - r(0) = reg[0]; - SWAPIN; + HEAVY_SWAPIN; if (I) { Goto(*I); } @@ -5138,7 +4997,7 @@ get_map_elements_fail: OpCase(label_L): OpCase(on_load): OpCase(line_I): - erl_exit(1, "meta op\n"); + erts_exit(ERTS_ERROR_EXIT, "meta op\n"); /* * One-time initialization of Beam emulator. @@ -5151,22 +5010,14 @@ get_map_elements_fail: #ifndef NO_JUMP_TABLE #ifdef ERTS_OPCODE_COUNTER_SUPPORT - - /* Are tables correctly generated by beam_makeops? */ - ASSERT(sizeof(counting_opcodes) == sizeof(opcodes)); - - if (count_instructions) { #ifdef DEBUG - counting_opcodes[op_catch_end_y] = LabelAddr(lb_catch_end_y); + counting_opcodes[op_catch_end_y] = LabelAddr(lb_catch_end_y); #endif - counting_opcodes[op_i_func_info_IaaI] = LabelAddr(lb_i_func_info_IaaI); - beam_ops = counting_opcodes; - } - else -#endif /* #ifndef ERTS_OPCODE_COUNTER_SUPPORT */ - { - beam_ops = opcodes; - } + counting_opcodes[op_i_func_info_IaaI] = LabelAddr(lb_i_func_info_IaaI); + beam_ops = counting_opcodes; +#else /* #ifndef ERTS_OPCODE_COUNTER_SUPPORT */ + beam_ops = opcodes; +#endif /* ERTS_OPCODE_COUNTER_SUPPORT */ #endif /* NO_JUMP_TABLE */ em_call_error_handler = OpCode(call_error_handler); @@ -5200,7 +5051,7 @@ get_map_elements_fail: } #ifdef NO_JUMP_TABLE default: - erl_exit(1, "unexpected op code %d\n",Go); + erts_exit(ERTS_ERROR_EXIT, "unexpected op code %d\n",Go); } #endif return; /* Never executed */ @@ -5245,7 +5096,7 @@ translate_gc_bif(void* gcf) } else if (gcf == erts_gc_binary_part_3) { return binary_part_3; } else { - erl_exit(1, "bad gc bif"); + erts_exit(ERTS_ERROR_EXIT, "bad gc bif"); } } @@ -5279,7 +5130,9 @@ Eterm error_atom[NUMBER_EXIT_CODES] = { am_notalive, /* 14 */ am_system_limit, /* 15 */ am_try_clause, /* 16 */ - am_notsup /* 17 */ + am_notsup, /* 17 */ + am_badmap, /* 18 */ + am_badkey, /* 19 */ }; /* @@ -5308,7 +5161,7 @@ handle_error(Process* c_p, BeamInstr* pc, Eterm* reg, BifFunction bf) Eterm* hp; Eterm Value = c_p->fvalue; Eterm Args = am_true; - c_p->i = pc; /* In case we call erl_exit(). */ + c_p->i = pc; /* In case we call erts_exit(). */ ASSERT(c_p->freason != TRAP); /* Should have been handled earlier. */ @@ -5372,7 +5225,7 @@ handle_error(Process* c_p, BeamInstr* pc, Eterm* reg, BifFunction bf) c_p->cp = 0; /* To avoid keeping stale references. */ return new_pc; } - if (c_p->catches > 0) erl_exit(1, "Catch not found"); + if (c_p->catches > 0) erts_exit(ERTS_ERROR_EXIT, "Catch not found"); } ERTS_SMP_UNREQ_PROC_MAIN_LOCK(c_p); terminate_proc(c_p, Value); @@ -5482,18 +5335,35 @@ next_catch(Process* c_p, Eterm *reg) { static void terminate_proc(Process* c_p, Eterm Value) { + Eterm *hp; + Eterm Args = NIL; + /* Add a stacktrace if this is an error. */ if (GET_EXC_CLASS(c_p->freason) == EXTAG_ERROR) { Value = add_stacktrace(c_p, Value, c_p->ftrace); } /* EXF_LOG is a primary exception flag */ if (c_p->freason & EXF_LOG) { + int alive = erts_is_alive; erts_dsprintf_buf_t *dsbufp = erts_create_logger_dsbuf(); - erts_dsprintf(dsbufp, "Error in process %T ", c_p->common.id); - if (erts_is_alive) - erts_dsprintf(dsbufp, "on node %T ", erts_this_node->sysname); - erts_dsprintf(dsbufp,"with exit value: %0.*T\n", display_items, Value); - erts_send_error_to_logger(c_p->group_leader, dsbufp); + + /* Build the format message */ + erts_dsprintf(dsbufp, "Error in process ~p "); + if (alive) + erts_dsprintf(dsbufp, "on node ~p "); + erts_dsprintf(dsbufp, "with exit value:~n~p~n"); + + /* Build the args in reverse order */ + hp = HAlloc(c_p, 2); + Args = CONS(hp, Value, Args); + if (alive) { + hp = HAlloc(c_p, 2); + Args = CONS(hp, erts_this_node->sysname, Args); + } + hp = HAlloc(c_p, 2); + Args = CONS(hp, c_p->common.id, Args); + + erts_send_error_term_to_logger(c_p->group_leader, dsbufp, Args); } /* * If we use a shared heap, the process will be garbage-collected. @@ -5535,6 +5405,8 @@ expand_error_value(Process* c_p, Uint freason, Eterm Value) { case (GET_EXC_INDEX(EXC_TRY_CLAUSE)): case (GET_EXC_INDEX(EXC_BADFUN)): case (GET_EXC_INDEX(EXC_BADARITY)): + case (GET_EXC_INDEX(EXC_BADMAP)): + case (GET_EXC_INDEX(EXC_BADKEY)): /* Some common exceptions: value -> {atom, value} */ ASSERT(is_value(Value)); hp = HAlloc(c_p, 3); @@ -5837,7 +5709,7 @@ build_stacktrace(Process* c_p, Eterm exc) { * (e.g. spawn_link(erlang, abs, [1])). */ if (fi.current == NULL) { - erts_set_current_function(&fi, c_p->initial); + erts_set_current_function(&fi, c_p->u.initial); args = am_true; /* Just in case */ } else { args = get_args_from_exc(exc); @@ -6045,13 +5917,7 @@ apply(Process* p, Eterm module, Eterm function, Eterm args, Eterm* reg) } else if (ERTS_PROC_GET_SAVED_CALLS_BUF(p)) { save_calls(p, ep); } - -#ifdef USE_VM_CALL_PROBES - if (DTRACE_ENABLED(global_function_entry)) { - BeamInstr *fptr = (BeamInstr *) ep->addressv[erts_active_code_ix()]; - DTRACE_GLOBAL_CALL(p, (Eterm)fptr[-3], (Eterm)fptr[-2], (Uint)fptr[-1]); - } -#endif + DTRACE_GLOBAL_CALL_FROM_EXPORT(p, ep); return ep->addressv[erts_active_code_ix()]; } @@ -6100,13 +5966,7 @@ fixed_apply(Process* p, Eterm* reg, Uint arity) } else if (ERTS_PROC_GET_SAVED_CALLS_BUF(p)) { save_calls(p, ep); } - -#ifdef USE_VM_CALL_PROBES - if (DTRACE_ENABLED(global_function_entry)) { - BeamInstr *fptr = (BeamInstr *) ep->addressv[erts_active_code_ix()]; - DTRACE_GLOBAL_CALL(p, (Eterm)fptr[-3], (Eterm)fptr[-2], (Uint)fptr[-1]); - } -#endif + DTRACE_GLOBAL_CALL_FROM_EXPORT(p, ep); return ep->addressv[erts_active_code_ix()]; } @@ -6116,6 +5976,23 @@ erts_hibernate(Process* c_p, Eterm module, Eterm function, Eterm args, Eterm* re int arity; Eterm tmp; +#ifndef ERTS_SMP + if (ERTS_PROC_IS_EXITING(c_p)) { + /* + * I non smp case: + * + * Currently executing process might be sent an exit + * signal if it is traced by a port that it also is + * linked to, and the port terminates during the + * trace. In this case we do *not* want to clear + * the active flag, which will make the process hang + * in limbo forever. Get out of here and terminate + * the process... + */ + return -1; + } +#endif + if (is_not_atom(module) || is_not_atom(function)) { /* * No need to test args here -- done below. @@ -6192,7 +6069,16 @@ erts_hibernate(Process* c_p, Eterm module, Eterm function, Eterm args, Eterm* re ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p); PROCESS_MAIN_CHK_LOCKS(c_p); erts_smp_proc_lock(c_p, ERTS_PROC_LOCK_MSGQ|ERTS_PROC_LOCK_STATUS); -#ifdef ERTS_SMP +#ifndef ERTS_SMP + if (ERTS_PROC_IS_EXITING(c_p)) { + /* + * See comment in the begining of the function... + * + * This second test is needed since gc might be traced. + */ + return -1; + } +#else /* ERTS_SMP */ ERTS_SMP_MSGQ_MV_INQ2PRIVQ(c_p); if (!c_p->msg.len) #endif @@ -6294,7 +6180,7 @@ call_fun(Process* p, /* Current process. */ */ module = fe->module; if ((modp = erts_get_module(module, code_ix)) != NULL - && modp->curr.code != NULL) { + && modp->curr.code_hdr != NULL) { /* * There is a module loaded, but obviously the fun is not * defined in it. We must not call the error_handler @@ -6428,76 +6314,91 @@ new_fun(Process* p, Eterm* reg, ErlFunEntry* fe, int num_free) return make_fun(funp); } -static int has_not_map_field(Eterm map, Eterm key) +static Eterm get_map_element(Eterm map, Eterm key) { - map_t* mp; - Eterm* keys; - Uint i; - Uint n; - - mp = (map_t *)map_val(map); - keys = map_get_keys(mp); - n = map_get_size(mp); - if (is_immed(key)) { - for (i = 0; i < n; i++) { - if (keys[i] == key) { - return 0; + Uint32 hx; + const Eterm *vs; + if (is_flatmap(map)) { + flatmap_t *mp; + Eterm *ks; + Uint i; + Uint n; + + mp = (flatmap_t *)flatmap_val(map); + ks = flatmap_get_keys(mp); + vs = flatmap_get_values(mp); + n = flatmap_get_size(mp); + if (is_immed(key)) { + for (i = 0; i < n; i++) { + if (ks[i] == key) { + return vs[i]; + } } - } - } else { - for (i = 0; i < n; i++) { - if (EQ(keys[i], key)) { - return 0; + } else { + for (i = 0; i < n; i++) { + if (EQ(ks[i], key)) { + return vs[i]; + } } } + return THE_NON_VALUE; } - return 1; + ASSERT(is_hashmap(map)); + hx = hashmap_make_hash(key); + vs = erts_hashmap_get(hx,key,map); + return vs ? *vs : THE_NON_VALUE; } -static Eterm get_map_element(Eterm map, Eterm key) +static Eterm get_map_element_hash(Eterm map, Eterm key, Uint32 hx) { - map_t *mp; - Eterm* ks, *vs; - Uint i; - Uint n; - - mp = (map_t *)map_val(map); - ks = map_get_keys(mp); - vs = map_get_values(mp); - n = map_get_size(mp); - if (is_immed(key)) { - for (i = 0; i < n; i++) { - if (ks[i] == key) { - return vs[i]; + const Eterm *vs; + + if (is_flatmap(map)) { + flatmap_t *mp; + Eterm *ks; + Uint i; + Uint n; + + mp = (flatmap_t *)flatmap_val(map); + ks = flatmap_get_keys(mp); + vs = flatmap_get_values(mp); + n = flatmap_get_size(mp); + if (is_immed(key)) { + for (i = 0; i < n; i++) { + if (ks[i] == key) { + return vs[i]; + } } - } - } else { - for (i = 0; i < n; i++) { - if (EQ(ks[i], key)) { - return vs[i]; + } else { + for (i = 0; i < n; i++) { + if (EQ(ks[i], key)) { + return vs[i]; + } } } + return THE_NON_VALUE; } - return THE_NON_VALUE; + + ASSERT(is_hashmap(map)); + ASSERT(hx == hashmap_make_hash(key)); + vs = erts_hashmap_get(hx, key, map); + return vs ? *vs : THE_NON_VALUE; } -#define GET_TERM(term, dest) \ -do { \ - Eterm src = (Eterm)(term); \ - switch (src & _TAG_IMMED1_MASK) { \ - case (R_REG_DEF << _TAG_PRIMARY_SIZE) | TAG_PRIMARY_HEADER: \ - dest = x(0); \ - break; \ - case (X_REG_DEF << _TAG_PRIMARY_SIZE) | TAG_PRIMARY_HEADER: \ - dest = x(src >> _TAG_IMMED1_SIZE); \ - break; \ - case (Y_REG_DEF << _TAG_PRIMARY_SIZE) | TAG_PRIMARY_HEADER: \ - dest = y(src >> _TAG_IMMED1_SIZE); \ - break; \ - default: \ - dest = src; \ - break; \ - } \ +#define GET_TERM(term, dest) \ +do { \ + Eterm src = (Eterm)(term); \ + switch (loader_tag(src)) { \ + case LOADER_X_REG: \ + dest = x(loader_x_reg_index(src)); \ + break; \ + case LOADER_Y_REG: \ + dest = y(loader_y_reg_index(src)); \ + break; \ + default: \ + dest = src; \ + break; \ + } \ } while(0) @@ -6511,7 +6412,33 @@ new_map(Process* p, Eterm* reg, BeamInstr* I) Eterm *mhp,*thp; Eterm *E; BeamInstr *ptr; - map_t *mp; + flatmap_t *mp; + ErtsHeapFactory factory; + + ptr = &Arg(4); + + if (n > 2*MAP_SMALL_MAP_LIMIT) { + Eterm res; + if (HeapWordsLeft(p) < n) { + erts_garbage_collect(p, n, reg, Arg(2)); + } + + mhp = p->htop; + thp = p->htop; + E = p->stop; + + for (i = 0; i < n/2; i++) { + GET_TERM(*ptr++, *mhp++); + GET_TERM(*ptr++, *mhp++); + } + + p->htop = mhp; + + erts_factory_proc_init(&factory, p); + res = erts_hashmap_from_array(&factory, thp, n/2, 0); + erts_factory_close(&factory); + return res; + } if (HeapWordsLeft(p) < need) { erts_garbage_collect(p, need, reg, Arg(2)); @@ -6520,12 +6447,11 @@ new_map(Process* p, Eterm* reg, BeamInstr* I) thp = p->htop; mhp = thp + 1 + n/2; E = p->stop; - ptr = &Arg(4); keys = make_tuple(thp); *thp++ = make_arityval(n/2); - mp = (map_t *)mhp; mhp += MAP_HEADER_SIZE; - mp->thing_word = MAP_HEADER; + mp = (flatmap_t *)mhp; mhp += MAP_HEADER_FLATMAP_SZ; + mp->thing_word = MAP_HEADER_FLATMAP; mp->size = n/2; mp->keys = keys; @@ -6534,7 +6460,7 @@ new_map(Process* p, Eterm* reg, BeamInstr* I) GET_TERM(*ptr++, *mhp++); } p->htop = mhp; - return make_map(mp); + return make_flatmap(mp); } static Eterm @@ -6544,7 +6470,7 @@ update_map_assoc(Process* p, Eterm* reg, Eterm map, BeamInstr* I) Uint num_old; Uint num_updates; Uint need; - map_t *old_mp, *mp; + flatmap_t *old_mp, *mp; Eterm res; Eterm* hp; Eterm* E; @@ -6554,12 +6480,36 @@ update_map_assoc(Process* p, Eterm* reg, Eterm map, BeamInstr* I) Eterm new_key; Eterm* kp; - if (is_not_map(map)) { - return THE_NON_VALUE; + new_p = &Arg(5); + num_updates = Arg(4) / 2; + + if (is_not_flatmap(map)) { + Uint32 hx; + Eterm val; + + /* apparently the compiler does not emit is_map instructions, + * bad compiler */ + + if (is_not_hashmap(map)) + return THE_NON_VALUE; + + res = map; + E = p->stop; + while(num_updates--) { + /* assoc can't fail */ + GET_TERM(new_p[0], new_key); + GET_TERM(new_p[1], val); + hx = hashmap_make_hash(new_key); + + res = erts_hashmap_insert(p, hx, new_key, val, res, 0); + + new_p += 2; + } + return res; } - old_mp = (map_t *) map_val(map); - num_old = map_get_size(old_mp); + old_mp = (flatmap_t *) flatmap_val(map); + num_old = flatmap_get_size(old_mp); /* * If the old map is empty, create a new map. @@ -6574,14 +6524,13 @@ update_map_assoc(Process* p, Eterm* reg, Eterm map, BeamInstr* I) * update list are new). */ - num_updates = Arg(4) / 2; - need = 2*(num_old+num_updates) + 1 + MAP_HEADER_SIZE; + need = 2*(num_old+num_updates) + 1 + MAP_HEADER_FLATMAP_SZ; if (HeapWordsLeft(p) < need) { Uint live = Arg(3); reg[live] = map; erts_garbage_collect(p, need, reg, live+1); map = reg[live]; - old_mp = (map_t *)map_val(map); + old_mp = (flatmap_t *)flatmap_val(map); } /* @@ -6612,16 +6561,15 @@ update_map_assoc(Process* p, Eterm* reg, Eterm map, BeamInstr* I) kp = p->htop + 1; /* Point to first key */ hp = kp + num_old + num_updates; - res = make_map(hp); - mp = (map_t *)hp; - hp += MAP_HEADER_SIZE; - mp->thing_word = MAP_HEADER; + res = make_flatmap(hp); + mp = (flatmap_t *)hp; + hp += MAP_HEADER_FLATMAP_SZ; + mp->thing_word = MAP_HEADER_FLATMAP; mp->keys = make_tuple(kp-1); - old_vals = map_get_values(old_mp); - old_keys = map_get_keys(old_mp); + old_vals = flatmap_get_values(old_mp); + old_keys = flatmap_get_keys(old_mp); - new_p = &Arg(5); GET_TERM(*new_p, new_key); n = num_updates; @@ -6707,8 +6655,13 @@ update_map_assoc(Process* p, Eterm* reg, Eterm map, BeamInstr* I) n = kp - p->htop - 1; /* Actual number of keys/values */ *p->htop = make_arityval(n); + p->htop = hp; mp->size = n; - p->htop = hp; + + /* The expensive case, need to build a hashmap */ + if (n > MAP_SMALL_MAP_LIMIT) { + res = erts_hashmap_from_ks_and_vs(p,flatmap_get_keys(mp),flatmap_get_values(mp),n); + } return res; } @@ -6723,7 +6676,7 @@ update_map_exact(Process* p, Eterm* reg, Eterm map, BeamInstr* I) Uint i; Uint num_old; Uint need; - map_t *old_mp, *mp; + flatmap_t *old_mp, *mp; Eterm res; Eterm* hp; Eterm* E; @@ -6732,18 +6685,53 @@ update_map_exact(Process* p, Eterm* reg, Eterm map, BeamInstr* I) BeamInstr* new_p; Eterm new_key; - if (is_not_map(map)) { - return THE_NON_VALUE; + new_p = &Arg(5); + n = Arg(4) / 2; /* Number of values to be updated */ + ASSERT(n > 0); + + if (is_not_flatmap(map)) { + Uint32 hx; + Eterm val; + + /* apparently the compiler does not emit is_map instructions, + * bad compiler */ + + if (is_not_hashmap(map)) { + p->freason = BADMAP; + p->fvalue = map; + return THE_NON_VALUE; + } + + res = map; + E = p->stop; + while(n--) { + GET_TERM(new_p[0], new_key); + GET_TERM(new_p[1], val); + hx = hashmap_make_hash(new_key); + + res = erts_hashmap_insert(p, hx, new_key, val, res, 1); + if (is_non_value(res)) { + p->fvalue = new_key; + p->freason = BADKEY; + return res; + } + + new_p += 2; + } + return res; } - old_mp = (map_t *) map_val(map); - num_old = map_get_size(old_mp); + old_mp = (flatmap_t *) flatmap_val(map); + num_old = flatmap_get_size(old_mp); /* - * If the old map is empty, create a new map. + * If the old map is empty, fail. */ if (num_old == 0) { + E = p->stop; + p->freason = BADKEY; + GET_TERM(new_p[0], p->fvalue); return THE_NON_VALUE; } @@ -6751,13 +6739,13 @@ update_map_exact(Process* p, Eterm* reg, Eterm map, BeamInstr* I) * Allocate the exact heap space needed. */ - need = num_old + MAP_HEADER_SIZE; + need = num_old + MAP_HEADER_FLATMAP_SZ; if (HeapWordsLeft(p) < need) { Uint live = Arg(3); reg[live] = map; erts_garbage_collect(p, need, reg, live+1); map = reg[live]; - old_mp = (map_t *)map_val(map); + old_mp = (flatmap_t *)flatmap_val(map); } /* @@ -6767,23 +6755,20 @@ update_map_exact(Process* p, Eterm* reg, Eterm map, BeamInstr* I) hp = p->htop; E = p->stop; - old_vals = map_get_values(old_mp); - old_keys = map_get_keys(old_mp); + old_vals = flatmap_get_values(old_mp); + old_keys = flatmap_get_keys(old_mp); - res = make_map(hp); - mp = (map_t *)hp; - hp += MAP_HEADER_SIZE; - mp->thing_word = MAP_HEADER; + res = make_flatmap(hp); + mp = (flatmap_t *)hp; + hp += MAP_HEADER_FLATMAP_SZ; + mp->thing_word = MAP_HEADER_FLATMAP; mp->size = num_old; mp->keys = old_mp->keys; /* Get array of key/value pairs to be updated */ - new_p = &Arg(5); GET_TERM(*new_p, new_key); /* Update all values */ - n = Arg(4) / 2; /* Number of values to be updated */ - ASSERT(n > 0); for (i = 0; i < num_old; i++) { if (!EQ(*old_keys, new_key)) { /* Not same keys */ @@ -6816,6 +6801,8 @@ update_map_exact(Process* p, Eterm* reg, Eterm map, BeamInstr* I) * update list did not previously exist. */ ASSERT(hp == p->htop + need); + p->freason = BADKEY; + p->fvalue = new_key; return THE_NON_VALUE; } #undef GET_TERM @@ -6837,6 +6824,15 @@ erts_is_builtin(Eterm Mod, Eterm Name, int arity) Export e; Export* ep; + if (Mod == am_erlang && Name == am_apply && arity == 3) { + /* + * Special case. apply/3 is built-in (implemented in C), + * but implemented in a different way than all other + * BIFs. + */ + return 1; + } + e.code[0] = Mod; e.code[1] = Name; e.code[2] = arity; diff --git a/erts/emulator/beam/beam_load.c b/erts/emulator/beam/beam_load.c index e96177cfd9..d3d278fb81 100644 --- a/erts/emulator/beam/beam_load.c +++ b/erts/emulator/beam/beam_load.c @@ -3,16 +3,17 @@ * * Copyright Ericsson AB 1996-2014. All Rights Reserved. * - * The contents of this file are subject to the Erlang Public License, - * Version 1.1, (the "License"); you may not use this file except in - * compliance with the License. You should have received a copy of the - * Erlang Public License along with this software. If not, it can be - * retrieved online at http://www.erlang.org/. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Software distributed under the License is distributed on an "AS IS" - * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See - * the License for the specific language governing rights and limitations - * under the License. + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. * * %CopyrightEnd% */ @@ -36,6 +37,7 @@ #include "beam_catches.h" #include "erl_binary.h" #include "erl_zlib.h" +#include "erl_map.h" #ifdef HIPE #include "hipe_bif0.h" @@ -78,7 +80,7 @@ ErlDrvBinary* erts_gzinflate_buffer(char*, int); typedef struct { Uint value; /* Value of label (NULL if not known yet). */ - Uint patches; /* Index (into code buffer) to first location + Sint patches; /* Index (into code buffer) to first location * which must be patched with the value of this label. */ #ifdef ERTS_SMP @@ -204,10 +206,7 @@ typedef struct { typedef struct { Eterm term; /* The tagged term (in the heap). */ - Uint heap_size; /* (Exact) size on the heap. */ - SWord offset; /* Offset from temporary location to final. */ - ErlOffHeap off_heap; /* Start of linked list of ProcBins. */ - Eterm* heap; /* Heap for term. */ + ErlHeapFragment* heap_frags; } Literal; /* @@ -245,7 +244,7 @@ typedef struct { /* * This structure contains all information about the module being loaded. */ - +#define MD5_SIZE 16 typedef struct LoaderState { /* * The current logical file within the binary. @@ -285,14 +284,15 @@ typedef struct LoaderState { int specific_op; /* Specific opcode (-1 if not found). */ int num_functions; /* Number of functions in module. */ int num_labels; /* Number of labels. */ - int code_buffer_size; /* Size of code buffer in words. */ - BeamInstr* code; /* Loaded code. */ - int ci; /* Current index into loaded code. */ + BeamCodeHeader* hdr; /* Loaded code header */ + BeamInstr* codev; /* Loaded code buffer */ + int codev_size; /* Size of code buffer in words. */ + int ci; /* Current index into loaded code buffer. */ Label* labels; StringPatch* string_patches; /* Linked list of position into string table to patch. */ BeamInstr catches; /* Linked list of catch_yf instructions. */ unsigned loaded_size; /* Final size of code when loaded. */ - byte mod_md5[16]; /* MD5 for module code. */ + byte mod_md5[MD5_SIZE]; /* MD5 for module code. */ int may_load_nif; /* true if NIFs may later be loaded for this module */ int on_load; /* Index in the code for the on_load function * (or 0 if there is no on_load function) @@ -476,10 +476,12 @@ typedef struct LoaderState { static void free_loader_state(Binary* magic); +static ErlHeapFragment* new_literal_fragment(Uint size); +static void free_literal_fragment(ErlHeapFragment*); static void loader_state_dtor(Binary* magic); static Eterm insert_new_code(Process *c_p, ErtsProcLocks c_p_locks, Eterm group_leader, Eterm module, - BeamInstr* code, Uint size); + BeamCodeHeader* code, Uint size); static int init_iff_file(LoaderState* stp, byte* code, Uint size); static int scan_iff_file(LoaderState* stp, Uint* chunk_types, Uint num_types, Uint num_mandatory); @@ -524,11 +526,16 @@ static void new_literal_patch(LoaderState* stp, int pos); static void new_string_patch(LoaderState* stp, int pos); static Uint new_literal(LoaderState* stp, Eterm** hpp, Uint heap_size); static int genopargcompare(GenOpArg* a, GenOpArg* b); -static Eterm exported_from_module(Process* p, Eterm mod); -static Eterm functions_in_module(Process* p, Eterm mod); -static Eterm attributes_for_module(Process* p, Eterm mod); -static Eterm compilation_info_for_module(Process* p, Eterm mod); -static Eterm native_addresses(Process* p, Eterm mod); +static Eterm get_module_info(Process* p, ErtsCodeIndex code_ix, + BeamCodeHeader*, Eterm module, Eterm what); +static Eterm exported_from_module(Process* p, ErtsCodeIndex code_ix, + Eterm mod); +static Eterm functions_in_module(Process* p, BeamCodeHeader*); +static Eterm attributes_for_module(Process* p, BeamCodeHeader*); +static Eterm compilation_info_for_module(Process* p, BeamCodeHeader*); +static Eterm md5_of_module(Process* p, BeamCodeHeader*); +static Eterm has_native(BeamCodeHeader*); +static Eterm native_addresses(Process* p, BeamCodeHeader*); int patch_funentries(Eterm Patchlist); int patch(Eterm Addresses, Uint fe); static int safe_mul(UWord a, UWord b, UWord* resp); @@ -595,6 +602,7 @@ extern void check_allocated_block(Uint type, void *blk); #define CHKBLK(TYPE,BLK) /* nothing */ #endif + Eterm erts_prepare_loading(Binary* magic, Process *c_p, Eterm group_leader, Eterm* modp, byte* code, Uint unloaded_size) @@ -635,19 +643,27 @@ erts_prepare_loading(Binary* magic, Process *c_p, Eterm group_leader, /* * Initialize code area. */ - stp->code_buffer_size = 2048 + stp->num_functions; - stp->code = (BeamInstr *) erts_alloc(ERTS_ALC_T_CODE, - sizeof(BeamInstr) * stp->code_buffer_size); + stp->codev_size = 2048 + stp->num_functions; + stp->hdr = (BeamCodeHeader*) erts_alloc(ERTS_ALC_T_CODE, + (offsetof(BeamCodeHeader,functions) + + sizeof(BeamInstr) * stp->codev_size)); - stp->code[MI_NUM_FUNCTIONS] = stp->num_functions; - stp->ci = MI_FUNCTIONS + stp->num_functions + 1; + stp->hdr->num_functions = stp->num_functions; - stp->code[MI_ATTR_PTR] = 0; - stp->code[MI_ATTR_SIZE] = 0; - stp->code[MI_ATTR_SIZE_ON_HEAP] = 0; - stp->code[MI_COMPILE_PTR] = 0; - stp->code[MI_COMPILE_SIZE] = 0; - stp->code[MI_COMPILE_SIZE_ON_HEAP] = 0; + /* Let the codev array start at functions[0] in order to index + * both function pointers and the loaded code itself that follows. + */ + stp->codev = (BeamInstr*) &stp->hdr->functions; + stp->ci = stp->num_functions + 1; + + stp->hdr->attr_ptr = NULL; + stp->hdr->attr_size = 0; + stp->hdr->attr_size_on_heap = 0; + stp->hdr->compile_ptr = NULL; + stp->hdr->compile_size = 0; + stp->hdr->compile_size_on_heap = 0; + stp->hdr->literals_start = NULL; + stp->hdr->md5_ptr = NULL; /* * Read the atom table. @@ -769,7 +785,7 @@ erts_finish_loading(Binary* magic, Process* c_p, CHKBLK(ERTS_ALC_T_CODE,stp->code); retval = insert_new_code(c_p, c_p_locks, stp->group_leader, stp->module, - stp->code, stp->loaded_size); + stp->hdr, stp->loaded_size); if (retval != NIL) { goto load_error; } @@ -792,7 +808,8 @@ erts_finish_loading(Binary* magic, Process* c_p, debug_dump_code(stp->code,stp->ci); #endif #endif - stp->code = NULL; /* Prevent code from being freed. */ + stp->hdr = NULL; /* Prevent code from being freed. */ + stp->codev = NULL; *modp = stp->module; /* @@ -824,7 +841,8 @@ erts_alloc_loader_state(void) stp->specific_op = -1; stp->genop = NULL; stp->atom = NULL; - stp->code = NULL; + stp->hdr = NULL; + stp->codev = NULL; stp->labels = NULL; stp->import = NULL; stp->export = NULL; @@ -863,13 +881,30 @@ erts_module_for_prepared_code(Binary* magic) return NIL; } stp = ERTS_MAGIC_BIN_DATA(magic); - if (stp->code != 0) { + if (stp->hdr != 0) { return stp->module; } else { return NIL; } } +/* + * Return a non-zero value if the module has an on_load function, + * or 0 if it does not. + */ + +Eterm +erts_has_code_on_load(Binary* magic) +{ + LoaderState* stp; + + if (ERTS_MAGIC_BIN_DESTRUCTOR(magic) != loader_state_dtor) { + return NIL; + } + stp = ERTS_MAGIC_BIN_DATA(magic); + return stp->on_load ? am_true : am_false; +} + static void free_loader_state(Binary* magic) { @@ -879,6 +914,28 @@ free_loader_state(Binary* magic) } } +static ErlHeapFragment* new_literal_fragment(Uint size) +{ + ErlHeapFragment* bp; + bp = (ErlHeapFragment*) ERTS_HEAP_ALLOC(ERTS_ALC_T_PREPARED_CODE, + ERTS_HEAP_FRAG_SIZE(size)); + ERTS_INIT_HEAP_FRAG(bp, size, size); + return bp; +} + +static void free_literal_fragment(ErlHeapFragment* bp) +{ + ASSERT(bp != NULL); + do { + ErlHeapFragment* next_bp = bp->next; + + erts_cleanup_offheap(&bp->off_heap); + ERTS_HEAP_FREE(ERTS_ALC_T_PREPARED_CODE, (void *) bp, + ERTS_HEAP_FRAG_SIZE(bp->size)); + bp = next_bp; + }while (bp != NULL); +} + /* * This destructor function can safely be called multiple times. */ @@ -891,9 +948,13 @@ loader_state_dtor(Binary* magic) driver_free_binary(stp->bin); stp->bin = 0; } - if (stp->code != 0) { - erts_free(ERTS_ALC_T_CODE, stp->code); - stp->code = 0; + if (stp->hdr != 0) { + if (stp->hdr->literals_start) { + erts_free(ERTS_ALC_T_LITERAL, stp->hdr->literals_start); + } + erts_free(ERTS_ALC_T_CODE, stp->hdr); + stp->hdr = 0; + stp->codev = 0; } if (stp->labels != 0) { erts_free(ERTS_ALC_T_PREPARED_CODE, (void *) stp->labels); @@ -918,10 +979,9 @@ loader_state_dtor(Binary* magic) if (stp->literals != 0) { int i; for (i = 0; i < stp->num_literals; i++) { - if (stp->literals[i].heap != 0) { - erts_free(ERTS_ALC_T_PREPARED_CODE, - (void *) stp->literals[i].heap); - stp->literals[i].heap = 0; + if (stp->literals[i].heap_frags != 0) { + free_literal_fragment(stp->literals[i].heap_frags); + stp->literals[i].heap_frags = 0; } } erts_free(ERTS_ALC_T_PREPARED_CODE, (void *) stp->literals); @@ -967,7 +1027,7 @@ loader_state_dtor(Binary* magic) static Eterm insert_new_code(Process *c_p, ErtsProcLocks c_p_locks, - Eterm group_leader, Eterm module, BeamInstr* code, + Eterm group_leader, Eterm module, BeamCodeHeader* code_hdr, Uint size) { Module* modp; @@ -988,7 +1048,7 @@ insert_new_code(Process *c_p, ErtsProcLocks c_p_locks, erts_total_code_size += size; modp = erts_put_module(module); - modp->curr.code = code; + modp->curr.code_hdr = code_hdr; modp->curr.code_length = size; modp->curr.catches = BEAM_CATCHES_NIL; /* Will be filled in later. */ @@ -996,7 +1056,7 @@ insert_new_code(Process *c_p, ErtsProcLocks c_p_locks, * Update ranges (used for finding a function from a PC value). */ - erts_update_ranges(code, size); + erts_update_ranges((BeamInstr*)modp->curr.code_hdr, size); return NIL; } @@ -1345,7 +1405,7 @@ read_export_table(LoaderState* stp) if (value == 0) { LoadError2(stp, "export table entry %d: label %d not resolved", i, n); } - stp->export[i].address = address = stp->code + value; + stp->export[i].address = address = stp->codev + value; /* * Find out if there is a BIF with the same name. @@ -1364,7 +1424,7 @@ read_export_table(LoaderState* stp) * any other functions that walk through all local functions. */ - if (stp->labels[n].patches) { + if (stp->labels[n].patches >= 0) { LoadError3(stp, "there are local calls to the stub for " "the BIF %T:%T/%d", stp->module, func, arity); @@ -1446,6 +1506,7 @@ read_lambda_table(LoaderState* stp) return 0; } + static int read_literal_table(LoaderState* stp) { @@ -1467,7 +1528,7 @@ read_literal_table(LoaderState* stp) stp->allocated_literals = stp->num_literals; for (i = 0; i < stp->num_literals; i++) { - stp->literals[i].heap = 0; + stp->literals[i].heap_frags = 0; } for (i = 0; i < stp->num_literals; i++) { @@ -1475,28 +1536,38 @@ read_literal_table(LoaderState* stp) Sint heap_size; byte* p; Eterm val; - Eterm* hp; + ErtsHeapFactory factory; GetInt(stp, 4, sz); /* Size of external term format. */ GetString(stp, p, sz); if ((heap_size = erts_decode_ext_size(p, sz)) < 0) { LoadError1(stp, "literal %d: bad external format", i); } - hp = stp->literals[i].heap = erts_alloc(ERTS_ALC_T_PREPARED_CODE, - heap_size*sizeof(Eterm)); - stp->literals[i].off_heap.first = 0; - stp->literals[i].off_heap.overhead = 0; - val = erts_decode_ext(&hp, &stp->literals[i].off_heap, &p); - stp->literals[i].heap_size = hp - stp->literals[i].heap; - if (stp->literals[i].heap_size > heap_size) { - erl_exit(1, "overrun by %d word(s) for literal heap, term %d", - stp->literals[i].heap_size - heap_size, i); - } - if (is_non_value(val)) { - LoadError1(stp, "literal %d: bad external format", i); - } - stp->literals[i].term = val; - stp->total_literal_size += stp->literals[i].heap_size; + + if (heap_size > 0) { + erts_factory_heap_frag_init(&factory, + new_literal_fragment(heap_size)); + factory.alloc_type = ERTS_ALC_T_PREPARED_CODE; + val = erts_decode_ext(&factory, &p, 0); + + if (is_non_value(val)) { + LoadError1(stp, "literal %d: bad external format", i); + } + erts_factory_close(&factory); + stp->literals[i].heap_frags = factory.heap_frags; + stp->total_literal_size += erts_used_frag_sz(factory.heap_frags); + } + else { + erts_factory_dummy_init(&factory); + val = erts_decode_ext(&factory, &p, 0); + if (is_non_value(val)) { + LoadError1(stp, "literal %d: bad external format", i); + } + ASSERT(is_immed(val)); + stp->literals[i].heap_frags = NULL; + } + stp->literals[i].term = val; + } erts_free(ERTS_ALC_T_TMP, uncompressed); return 1; @@ -1696,7 +1767,7 @@ read_code_header(LoaderState* stp) stp->num_labels * sizeof(Label)); for (i = 0; i < stp->num_labels; i++) { stp->labels[i].value = 0; - stp->labels[i].patches = 0; + stp->labels[i].patches = -1; #ifdef ERTS_SMP stp->labels[i].looprec_targeted = 0; #endif @@ -1715,13 +1786,14 @@ read_code_header(LoaderState* stp) } else {} #define CodeNeed(w) do { \ - ASSERT(ci <= code_buffer_size); \ - if (code_buffer_size < ci+(w)) { \ - code_buffer_size = 2*ci+(w); \ - stp->code = code = \ - (BeamInstr *) erts_realloc(ERTS_ALC_T_CODE, \ - (void *) code, \ - code_buffer_size * sizeof(BeamInstr)); \ + ASSERT(ci <= codev_size); \ + if (codev_size < ci+(w)) { \ + codev_size = 2*ci+(w); \ + stp->hdr = (BeamCodeHeader*) erts_realloc(ERTS_ALC_T_CODE, \ + (void *) stp->hdr, \ + (offsetof(BeamCodeHeader,functions) \ + + codev_size * sizeof(BeamInstr))); \ + code = stp->codev = (BeamInstr*) &stp->hdr->functions; \ } \ } while (0) @@ -1737,7 +1809,7 @@ load_code(LoaderState* stp) int arg; /* Number of current argument. */ int num_specific; /* Number of specific ops for current. */ BeamInstr* code; - int code_buffer_size; + int codev_size; int specific; Uint last_label = 0; /* Number of last label. */ Uint function_number = 0; @@ -1754,15 +1826,15 @@ load_code(LoaderState* stp) FUNC_INFO_SZ = 5 }; - code = stp->code; - code_buffer_size = stp->code_buffer_size; + code = stp->codev; + codev_size = stp->codev_size; ci = stp->ci; for (;;) { int new_op; GenOp* tmp_op; - ASSERT(ci <= code_buffer_size); + ASSERT(ci <= codev_size); get_next_instr: GetByte(stp, new_op); @@ -1808,9 +1880,7 @@ load_code(LoaderState* stp) case TAG_o: break; case TAG_x: - if (last_op->a[arg].val == 0) { - last_op->a[arg].type = TAG_r; - } else if (last_op->a[arg].val >= MAX_REG) { + if (last_op->a[arg].val >= MAX_REG) { LoadError1(stp, "invalid x register number: %u", last_op->a[arg].val); } @@ -1855,15 +1925,14 @@ load_code(LoaderState* stp) */ { Eterm* hp; -/* XXX:PaN - Halfword should use ARCH_64 variant instead */ -#if !defined(ARCH_64) || HALFWORD_HEAP +#if !defined(ARCH_64) Uint high, low; # endif last_op->a[arg].val = new_literal(stp, &hp, FLOAT_SIZE_OBJECT); hp[0] = HEADER_FLONUM; last_op->a[arg].type = TAG_q; -#if defined(ARCH_64) && !HALFWORD_HEAP +#if defined(ARCH_64) GetInt(stp, 8, hp[1]); # else GetInt(stp, 4, high); @@ -2017,7 +2086,42 @@ load_code(LoaderState* stp) if (((opc[specific].mask[0] & mask[0]) == mask[0]) && ((opc[specific].mask[1] & mask[1]) == mask[1]) && ((opc[specific].mask[2] & mask[2]) == mask[2])) { - break; + + if (!opc[specific].involves_r) { + break; /* No complications - match */ + } + + /* + * The specific operation uses the 'r' operand, + * which is shorthand for x(0). Now things + * get complicated. First we must check whether + * all operands that should be of type 'r' use + * x(0) (as opposed to some other X register). + */ + for (arg = 0; arg < arity; arg++) { + if (opc[specific].involves_r & (1 << arg) && + tmp_op->a[arg].type == TAG_x) { + if (tmp_op->a[arg].val != 0) { + break; /* Other X register than 0 */ + } + } + } + + if (arg == arity) { + /* + * All 'r' operands use x(0) in the generic + * operation. That means a match. Now we + * will need to rewrite the generic instruction + * to actually use 'r' instead of 'x(0)'. + */ + for (arg = 0; arg < arity; arg++) { + if (opc[specific].involves_r & (1 << arg) && + tmp_op->a[arg].type == TAG_x) { + tmp_op->a[arg].type = TAG_r; + } + } + break; /* Match */ + } } specific++; } @@ -2128,14 +2232,11 @@ load_code(LoaderState* stp) break; case 's': /* Any source (tagged constant or register) */ switch (tag) { - case TAG_r: - code[ci++] = make_rreg(); - break; case TAG_x: - code[ci++] = make_xreg(tmp_op->a[arg].val); + code[ci++] = make_loader_x_reg(tmp_op->a[arg].val); break; case TAG_y: - code[ci++] = make_yreg(tmp_op->a[arg].val); + code[ci++] = make_loader_y_reg(tmp_op->a[arg].val); break; case TAG_i: code[ci++] = (BeamInstr) make_small((Uint)tmp_op->a[arg].val); @@ -2146,6 +2247,10 @@ load_code(LoaderState* stp) case TAG_n: code[ci++] = NIL; break; + case TAG_q: + new_literal_patch(stp, ci); + code[ci++] = tmp_op->a[arg].val; + break; default: LoadError1(stp, "bad tag %d for general source", tmp_op->a[arg].type); @@ -2154,14 +2259,11 @@ load_code(LoaderState* stp) break; case 'd': /* Destination (x(0), x(N), y(N) */ switch (tag) { - case TAG_r: - code[ci++] = make_rreg(); - break; case TAG_x: - code[ci++] = make_xreg(tmp_op->a[arg].val); + code[ci++] = tmp_op->a[arg].val * sizeof(Eterm); break; case TAG_y: - code[ci++] = make_yreg(tmp_op->a[arg].val); + code[ci++] = tmp_op->a[arg].val * sizeof(Eterm) + 1; break; default: LoadError1(stp, "bad tag %d for destination", @@ -2318,20 +2420,13 @@ load_code(LoaderState* stp) stp->labels[tmp_op->a[arg].val].patches = ci; ci++; break; - case TAG_r: - CodeNeed(1); - code[ci++] = (R_REG_DEF << _TAG_PRIMARY_SIZE) | - TAG_PRIMARY_HEADER; - break; case TAG_x: CodeNeed(1); - code[ci++] = (tmp_op->a[arg].val << _TAG_IMMED1_SIZE) | - (X_REG_DEF << _TAG_PRIMARY_SIZE) | TAG_PRIMARY_HEADER; + code[ci++] = make_loader_x_reg(tmp_op->a[arg].val); break; case TAG_y: CodeNeed(1); - code[ci++] = (tmp_op->a[arg].val << _TAG_IMMED1_SIZE) | - (Y_REG_DEF << _TAG_PRIMARY_SIZE) | TAG_PRIMARY_HEADER; + code[ci++] = make_loader_y_reg(tmp_op->a[arg].val); break; case TAG_n: CodeNeed(1); @@ -2354,8 +2449,7 @@ load_code(LoaderState* stp) switch (stp->specific_op) { case op_i_func_info_IaaI: { - Uint offset; - + Sint offset; if (function_number >= stp->num_functions) { LoadError1(stp, "too many functions in module (header said %d)", stp->num_functions); @@ -2363,7 +2457,11 @@ load_code(LoaderState* stp) if (stp->may_load_nif) { const int finfo_ix = ci - FUNC_INFO_SZ; - enum { MIN_FUNC_SZ = 3 }; +#ifdef ERTS_DIRTY_SCHEDULERS + enum { MIN_FUNC_SZ = 4 }; +#else + enum { MIN_FUNC_SZ = 3 }; +#endif if (finfo_ix - last_func_start < MIN_FUNC_SZ && last_func_start) { /* Must make room for call_nif op */ int pad = MIN_FUNC_SZ - (finfo_ix - last_func_start); @@ -2393,15 +2491,15 @@ load_code(LoaderState* stp) stp->arity = code[ci-1]; ASSERT(stp->labels[last_label].value == ci - FUNC_INFO_SZ); - offset = MI_FUNCTIONS + function_number; - code[offset] = stp->labels[last_label].patches; + stp->hdr->functions[function_number] = (BeamInstr*) stp->labels[last_label].patches; + offset = function_number; stp->labels[last_label].patches = offset; function_number++; if (stp->arity > MAX_ARG) { LoadError1(stp, "too many arguments: %d", stp->arity); } #ifdef DEBUG - ASSERT(stp->labels[0].patches == 0); /* Should not be referenced. */ + ASSERT(stp->labels[0].patches < 0); /* Should not be referenced. */ for (i = 1; i < stp->num_labels; i++) { ASSERT(stp->labels[i].patches < ci); } @@ -2415,7 +2513,6 @@ load_code(LoaderState* stp) stp->on_load = ci; break; case op_bs_put_string_II: - case op_i_bs_match_string_rfII: case op_i_bs_match_string_xfII: new_string_patch(stp, ci-1); break; @@ -2472,7 +2569,7 @@ load_code(LoaderState* stp) * End of code found. */ case op_int_code_end: - stp->code_buffer_size = code_buffer_size; + stp->codev_size = codev_size; stp->ci = ci; stp->function = THE_NON_VALUE; stp->genop = NULL; @@ -2637,6 +2734,12 @@ same_label(LoaderState* stp, GenOpArg Target, GenOpArg Label) Target.val == Label.val; } +static int +is_killed(LoaderState* stp, GenOpArg Reg, GenOpArg Live) +{ + return Reg.type == TAG_x && Live.type == TAG_u && + Live.val <= Reg.val; +} /* * Generate an instruction for element/2. @@ -2653,17 +2756,17 @@ gen_element(LoaderState* stp, GenOpArg Fail, GenOpArg Index, op->next = NULL; if (Index.type == TAG_i && Index.val > 0 && - (Tuple.type == TAG_r || Tuple.type == TAG_x || Tuple.type == TAG_y)) { + (Tuple.type == TAG_x || Tuple.type == TAG_y)) { op->op = genop_i_fast_element_4; - op->a[0] = Tuple; - op->a[1] = Fail; + op->a[0] = Fail; + op->a[1] = Tuple; op->a[2].type = TAG_u; op->a[2].val = Index.val; op->a[3] = Dst; } else { op->op = genop_i_element_4; - op->a[0] = Tuple; - op->a[1] = Fail; + op->a[0] = Fail; + op->a[1] = Tuple; op->a[2] = Index; op->a[3] = Dst; } @@ -2789,23 +2892,16 @@ gen_get_integer2(LoaderState* stp, GenOpArg Fail, GenOpArg Ms, GenOpArg Live, goto generic; } } else { - GenOp* op2; - NEW_GENOP(stp, op2); - - op->op = genop_i_fetch_2; - op->arity = 2; - op->a[0] = Ms; - op->a[1] = Size; - op->next = op2; - - op2->op = genop_i_bs_get_integer_4; - op2->arity = 4; - op2->a[0] = Fail; - op2->a[1] = Live; - op2->a[2].type = TAG_u; - op2->a[2].val = (Unit.val << 3) | Flags.val; - op2->a[3] = Dst; - op2->next = NULL; + op->op = genop_i_bs_get_integer_6; + op->arity = 6; + op->a[0] = Fail; + op->a[1] = Live; + op->a[2].type = TAG_u; + op->a[2].val = (Unit.val << 3) | Flags.val; + op->a[3] = Ms; + op->a[4] = Size; + op->a[5] = Dst; + op->next = NULL; return op; } op->next = NULL; @@ -3165,7 +3261,11 @@ gen_increment_from_minus(LoaderState* stp, GenOpArg Reg, GenOpArg Integer, static int negation_is_small(LoaderState* stp, GenOpArg Int) { - return Int.type == TAG_i && IS_SSMALL(-Int.val); + /* Check for the rare case of overflow in BeamInstr (UWord) -> Sint + * Cast to the correct type before using IS_SSMALL (Sint) */ + return Int.type == TAG_i && + !(Int.val & ~((((BeamInstr)1) << ((sizeof(Sint)*8)-1))-1)) && + IS_SSMALL(-((Sint)Int.val)); } @@ -3225,14 +3325,14 @@ gen_literal_timeout(LoaderState* stp, GenOpArg Fail, GenOpArg Time) op->a[1].type = TAG_u; if (Time.type == TAG_i && (timeout = Time.val) >= 0 && -#if defined(ARCH_64) && !HALFWORD_HEAP +#if defined(ARCH_64) (timeout >> 32) == 0 #else 1 #endif ) { op->a[1].val = timeout; -#if !defined(ARCH_64) || HALFWORD_HEAP +#if !defined(ARCH_64) } else if (Time.type == TAG_q) { Eterm big; @@ -3249,7 +3349,7 @@ gen_literal_timeout(LoaderState* stp, GenOpArg Fail, GenOpArg Time) } #endif } else { -#if !defined(ARCH_64) || HALFWORD_HEAP +#if !defined(ARCH_64) error: #endif op->op = genop_i_wait_error_0; @@ -3272,14 +3372,14 @@ gen_literal_timeout_locked(LoaderState* stp, GenOpArg Fail, GenOpArg Time) op->a[1].type = TAG_u; if (Time.type == TAG_i && (timeout = Time.val) >= 0 && -#if defined(ARCH_64) && !HALFWORD_HEAP +#if defined(ARCH_64) (timeout >> 32) == 0 #else 1 #endif ) { op->a[1].val = timeout; -#if !defined(ARCH_64) || HALFWORD_HEAP +#if !defined(ARCH_64) } else if (Time.type == TAG_q) { Eterm big; @@ -3296,7 +3396,7 @@ gen_literal_timeout_locked(LoaderState* stp, GenOpArg Fail, GenOpArg Time) } #endif } else { -#if !defined(ARCH_64) || HALFWORD_HEAP +#if !defined(ARCH_64) error: #endif op->op = genop_i_wait_error_locked_0; @@ -3315,9 +3415,10 @@ gen_select_tuple_arity(LoaderState* stp, GenOpArg S, GenOpArg Fail, { GenOp* op; + GenOpArg *tmp; int arity = Size.val + 3; int size = Size.val / 2; - int i; + int i, j, align = 0; /* * Verify the validity of the list. @@ -3332,9 +3433,37 @@ gen_select_tuple_arity(LoaderState* stp, GenOpArg S, GenOpArg Fail, } /* + * Use a special-cased instruction if there are only two values. + */ + if (size == 2) { + NEW_GENOP(stp, op); + op->next = NULL; + op->op = genop_i_select_tuple_arity2_6; + GENOP_ARITY(op, arity - 1); + op->a[0] = S; + op->a[1] = Fail; + op->a[2].type = TAG_u; + op->a[2].val = Rest[0].val; + op->a[3].type = TAG_u; + op->a[3].val = Rest[2].val; + op->a[4] = Rest[1]; + op->a[5] = Rest[3]; + + return op; + } + + /* * Generate the generic instruction. + * Assumption: + * Few different tuple arities to select on (fewer than 20). + * Use linear scan approach. */ + align = 1; + + arity += 2*align; + size += align; + NEW_GENOP(stp, op); op->next = NULL; op->op = genop_i_select_tuple_arity_3; @@ -3342,39 +3471,36 @@ gen_select_tuple_arity(LoaderState* stp, GenOpArg S, GenOpArg Fail, op->a[0] = S; op->a[1] = Fail; op->a[2].type = TAG_u; - op->a[2].val = Size.val / 2; - for (i = 0; i < Size.val; i += 2) { - op->a[i+3].type = TAG_v; - op->a[i+3].val = make_arityval(Rest[i].val); - op->a[i+4] = Rest[i+1]; - } + op->a[2].val = size; - /* - * Sort the values to make them useful for a binary search. - */ + tmp = (GenOpArg *) erts_alloc(ERTS_ALC_T_LOADER_TMP, sizeof(GenOpArg)*(arity-2*align)); - qsort(op->a+3, size, 2*sizeof(GenOpArg), - (int (*)(const void *, const void *)) genopargcompare); -#ifdef DEBUG - for (i = 3; i < arity-2; i += 2) { - ASSERT(op->a[i].val < op->a[i+2].val); + for (i = 3; i < arity - 2*align; i+=2) { + tmp[i-3].type = TAG_v; + tmp[i-3].val = make_arityval(Rest[i-3].val); + tmp[i-2] = Rest[i-2]; } -#endif /* - * Use a special-cased instruction if there are only two values. + * Sort the values to make them useful for a sentinel search */ - if (size == 2) { - op->op = genop_i_select_tuple_arity2_6; - op->arity--; - op->a[2].type = TAG_u; - op->a[2].val = arityval(op->a[3].val); - op->a[3] = op->a[4]; - op->a[4].type = TAG_u; - op->a[4].val = arityval(op->a[5].val); - op->a[5] = op->a[6]; + + qsort(tmp, size - align, 2*sizeof(GenOpArg), + (int (*)(const void *, const void *)) genopargcompare); + + j = 3; + for (i = 3; i < arity - 2*align; i += 2) { + op->a[j] = tmp[i-3]; + op->a[j + size] = tmp[i-2]; + j++; } + erts_free(ERTS_ALC_T_LOADER_TMP, (void *) tmp); + + op->a[j].type = TAG_u; + op->a[j].val = ~((BeamInstr)0); + op->a[j+size] = Fail; + return op; } @@ -3596,45 +3722,109 @@ gen_select_val(LoaderState* stp, GenOpArg S, GenOpArg Fail, GenOpArg Size, GenOpArg* Rest) { GenOp* op; + GenOpArg *tmp; int arity = Size.val + 3; int size = Size.val / 2; - int i; + int i, j, align = 0; + + if (size == 2) { + + /* + * Use a special-cased instruction if there are only two values. + */ + + NEW_GENOP(stp, op); + op->next = NULL; + op->op = genop_i_select_val2_6; + GENOP_ARITY(op, arity - 1); + op->a[0] = S; + op->a[1] = Fail; + op->a[2] = Rest[0]; + op->a[3] = Rest[2]; + op->a[4] = Rest[1]; + op->a[5] = Rest[3]; + + return op; + + } else if (size > 10) { + + /* binary search instruction */ + + NEW_GENOP(stp, op); + op->next = NULL; + op->op = genop_i_select_val_bins_3; + GENOP_ARITY(op, arity); + op->a[0] = S; + op->a[1] = Fail; + op->a[2].type = TAG_u; + op->a[2].val = size; + for (i = 3; i < arity; i++) { + op->a[i] = Rest[i-3]; + } + + /* + * Sort the values to make them useful for a binary search. + */ + + qsort(op->a+3, size, 2*sizeof(GenOpArg), + (int (*)(const void *, const void *)) genopargcompare); +#ifdef DEBUG + for (i = 3; i < arity-2; i += 2) { + ASSERT(op->a[i].val < op->a[i+2].val); + } +#endif + return op; + } + + /* linear search instruction */ + + align = 1; + + arity += 2*align; + size += align; NEW_GENOP(stp, op); op->next = NULL; - op->op = genop_i_select_val_3; + op->op = genop_i_select_val_lins_3; GENOP_ARITY(op, arity); op->a[0] = S; op->a[1] = Fail; op->a[2].type = TAG_u; op->a[2].val = size; - for (i = 3; i < arity; i++) { - op->a[i] = Rest[i-3]; + + tmp = (GenOpArg *) erts_alloc(ERTS_ALC_T_LOADER_TMP, sizeof(GenOpArg)*(arity-2*align)); + + for (i = 3; i < arity - 2*align; i++) { + tmp[i-3] = Rest[i-3]; } /* - * Sort the values to make them useful for a binary search. + * Sort the values to make them useful for a sentinel search */ - qsort(op->a+3, size, 2*sizeof(GenOpArg), - (int (*)(const void *, const void *)) genopargcompare); -#ifdef DEBUG - for (i = 3; i < arity-2; i += 2) { - ASSERT(op->a[i].val < op->a[i+2].val); + qsort(tmp, size - align, 2*sizeof(GenOpArg), + (int (*)(const void *, const void *)) genopargcompare); + + j = 3; + for (i = 3; i < arity - 2*align; i += 2) { + op->a[j] = tmp[i-3]; + op->a[j+size] = tmp[i-2]; + j++; } -#endif - /* - * Use a special-cased instruction if there are only two values. - */ - if (size == 2) { - op->op = genop_i_select_val2_6; - op->arity--; - op->a[2] = op->a[3]; - op->a[3] = op->a[4]; - op->a[4] = op->a[5]; - op->a[5] = op->a[6]; + erts_free(ERTS_ALC_T_LOADER_TMP, (void *) tmp); + + /* add sentinel */ + + op->a[j].type = TAG_u; + op->a[j].val = ~((BeamInstr)0); + op->a[j+size] = Fail; + +#ifdef DEBUG + for (i = 0; i < size - 1; i++) { + ASSERT(op->a[i+3].val <= op->a[i+4].val); } +#endif return op; } @@ -3762,9 +3952,7 @@ gen_make_fun2(LoaderState* stp, GenOpArg idx) /* * Rewrite gc_bifs with one parameter (the common case). Utilized * in ops.tab to rewrite instructions calling bif's in guards - * to use a garbage collecting implementation. The instructions - * are sometimes once again rewritten to handle literals (putting the - * parameter in the mostly unused r[0] before the instruction is executed). + * to use a garbage collecting implementation. */ static GenOp* gen_guard_bif1(LoaderState* stp, GenOpArg Fail, GenOpArg Live, GenOpArg Bif, @@ -3819,10 +4007,6 @@ gen_guard_bif1(LoaderState* stp, GenOpArg Fail, GenOpArg Live, GenOpArg Bif, /* * This is used by the ops.tab rule that rewrites gc_bifs with two parameters. - * The instruction returned is then again rewritten to an i_load instruction - * followed by i_gc_bif2_jIId, to handle literals properly. - * As opposed to the i_gc_bif1_jIsId, the instruction i_gc_bif2_jIId is - * always rewritten, regardless of if there actually are any literals. */ static GenOp* gen_guard_bif2(LoaderState* stp, GenOpArg Fail, GenOpArg Live, GenOpArg Bif, @@ -3849,23 +4033,19 @@ gen_guard_bif2(LoaderState* stp, GenOpArg Fail, GenOpArg Live, GenOpArg Bif, op->a[2].val = stp->import[Bif.val].arity; return op; } - op->op = genop_ii_gc_bif2_6; + op->op = genop_i_gc_bif2_6; op->arity = 6; op->a[0] = Fail; op->a[1].type = TAG_u; - op->a[2] = S1; - op->a[3] = S2; - op->a[4] = Live; + op->a[2] = Live; + op->a[3] = S1; + op->a[4] = S2; op->a[5] = Dst; return op; } /* * This is used by the ops.tab rule that rewrites gc_bifs with three parameters. - * The instruction returned is then again rewritten to a move instruction that - * uses r[0] for temp storage, followed by an i_load instruction, - * followed by i_gc_bif3_jIsId, to handle literals properly. Rewriting - * always occur, as with the gc_bif2 counterpart. */ static GenOp* gen_guard_bif3(LoaderState* stp, GenOpArg Fail, GenOpArg Live, GenOpArg Bif, @@ -3896,10 +4076,10 @@ gen_guard_bif3(LoaderState* stp, GenOpArg Fail, GenOpArg Live, GenOpArg Bif, op->arity = 7; op->a[0] = Fail; op->a[1].type = TAG_u; - op->a[2] = S1; - op->a[3] = S2; - op->a[4] = S3; - op->a[5] = Live; + op->a[2] = Live; + op->a[3] = S1; + op->a[4] = S2; + op->a[5] = S3; op->a[6] = Dst; op->next = NULL; return op; @@ -3955,8 +4135,139 @@ tuple_append_put(LoaderState* stp, GenOpArg Arity, GenOpArg Dst, } /* + * Predicate to test whether the given literal is a map. + */ + +static int +literal_is_map(LoaderState* stp, GenOpArg Lit) +{ + Eterm term; + + ASSERT(Lit.type == TAG_q); + term = stp->literals[Lit.val].term; + return is_map(term); +} + +/* + * Predicate to test whether the given literal is an empty map. + */ + +static int +is_empty_map(LoaderState* stp, GenOpArg Lit) +{ + Eterm term; + + if (Lit.type != TAG_q) { + return 0; + } + term = stp->literals[Lit.val].term; + return is_flatmap(term) && flatmap_get_size(flatmap_val(term)) == 0; +} + +/* + * Pseudo predicate map_key_sort that will sort the Rest operand for + * map instructions as a side effect. + */ + +typedef struct SortGenOpArg { + Eterm term; /* Term to use for comparing */ + GenOpArg arg; /* Original data */ +} SortGenOpArg; + +static int +genopargtermcompare(SortGenOpArg* a, SortGenOpArg* b) +{ + return CMP_TERM(a->term, b->term); +} + +static int +map_key_sort(LoaderState* stp, GenOpArg Size, GenOpArg* Rest) +{ + SortGenOpArg* t; + unsigned size = Size.val; + unsigned i; + + if (size == 2) { + return 1; /* Already sorted. */ + } + + + t = (SortGenOpArg *) erts_alloc(ERTS_ALC_T_TMP, size*sizeof(SortGenOpArg)); + + /* + * Copy original data and sort keys to a temporary array. + */ + for (i = 0; i < size; i += 2) { + t[i].arg = Rest[i]; + switch (Rest[i].type) { + case TAG_a: + t[i].term = Rest[i].val; + ASSERT(is_atom(t[i].term)); + break; + case TAG_i: + t[i].term = make_small(Rest[i].val); + break; + case TAG_n: + t[i].term = NIL; + break; + case TAG_q: + t[i].term = stp->literals[Rest[i].val].term; + break; + default: + /* + * Not a literal key. Not allowed. Only a single + * variable key is allowed in each map instruction. + */ + erts_free(ERTS_ALC_T_TMP, (void *) t); + return 0; + } +#ifdef DEBUG + t[i+1].term = THE_NON_VALUE; +#endif + t[i+1].arg = Rest[i+1]; + } + + /* + * Sort the temporary array. + */ + qsort((void *) t, size / 2, 2 * sizeof(SortGenOpArg), + (int (*)(const void *, const void *)) genopargtermcompare); + + /* + * Copy back the sorted, original data. + */ + for (i = 0; i < size; i++) { + Rest[i] = t[i].arg; + } + + erts_free(ERTS_ALC_T_TMP, (void *) t); + return 1; +} + +static int +hash_genop_arg(LoaderState* stp, GenOpArg Key, Uint32* hx) +{ + switch (Key.type) { + case TAG_a: + *hx = hashmap_make_hash(Key.val); + return 1; + case TAG_i: + *hx = hashmap_make_hash(make_small(Key.val)); + return 1; + case TAG_n: + *hx = hashmap_make_hash(NIL); + return 1; + case TAG_q: + *hx = hashmap_make_hash(stp->literals[Key.val].term); + return 1; + default: + return 0; + } +} + +/* * Replace a get_map_elements with one key to an instruction with one - * element + * element. */ static GenOp* @@ -3964,37 +4275,146 @@ gen_get_map_element(LoaderState* stp, GenOpArg Fail, GenOpArg Src, GenOpArg Size, GenOpArg* Rest) { GenOp* op; + GenOpArg Key; + Uint32 hx = 0; ASSERT(Size.type == TAG_u); NEW_GENOP(stp, op); op->next = NULL; - op->op = genop_get_map_element_4; - op->arity = 4; - op->a[0] = Fail; op->a[1] = Src; op->a[2] = Rest[0]; - op->a[3] = Rest[1]; + + Key = Rest[0]; + if (hash_genop_arg(stp, Key, &hx)) { + op->arity = 5; + op->op = genop_i_get_map_element_hash_5; + op->a[3].type = TAG_u; + op->a[3].val = (BeamInstr) hx; + op->a[4] = Rest[1]; + } else { + op->arity = 4; + op->op = genop_i_get_map_element_4; + op->a[3] = Rest[1]; + } + return op; +} + +static int +hash_internal_genop_arg(LoaderState* stp, GenOpArg Key, Uint32* hx) +{ + switch (Key.type) { + case TAG_a: + *hx = atom_tab(atom_val(Key.val))->slot.bucket.hvalue; + return 1; + case TAG_i: + *hx = Key.val; + return 1; + case TAG_n: + *hx = make_internal_hash(NIL); + return 1; + case TAG_q: + *hx = make_internal_hash(stp->literals[Key.val].term); + return 1; + default: + return 0; + } +} + + +static GenOp* +gen_get(LoaderState* stp, GenOpArg Src, GenOpArg Dst) +{ + GenOp* op; + Uint32 hx = 0; + + NEW_GENOP(stp, op); + op->next = NULL; + if (hash_internal_genop_arg(stp, Src, &hx)) { + op->arity = 3; + op->op = genop_i_get_hash_3; + op->a[0] = Src; + op->a[1].type = TAG_u; + op->a[1].val = (BeamInstr) hx; + op->a[2] = Dst; + } else { + op->arity = 2; + op->op = genop_i_get_2; + op->a[0] = Src; + op->a[1] = Dst; + } return op; } + static GenOp* -gen_has_map_field(LoaderState* stp, GenOpArg Fail, GenOpArg Src, - GenOpArg Size, GenOpArg* Rest) +gen_get_map_elements(LoaderState* stp, GenOpArg Fail, GenOpArg Src, + GenOpArg Size, GenOpArg* Rest) { GenOp* op; + Uint32 hx; + Uint i; + GenOpArg* dst; +#ifdef DEBUG + int good_hash; +#endif ASSERT(Size.type == TAG_u); NEW_GENOP(stp, op); + op->op = genop_i_get_map_elements_3; + GENOP_ARITY(op, 3 + 3*(Size.val/2)); op->next = NULL; - op->op = genop_has_map_field_3; - op->arity = 4; + op->a[0] = Fail; + op->a[1] = Src; + op->a[2].type = TAG_u; + op->a[2].val = 3*(Size.val/2); + + dst = op->a+3; + for (i = 0; i < Size.val / 2; i++) { + dst[0] = Rest[2*i]; + dst[1] = Rest[2*i+1]; +#ifdef DEBUG + good_hash = +#endif + hash_genop_arg(stp, dst[0], &hx); +#ifdef DEBUG + ASSERT(good_hash); +#endif + dst[2].type = TAG_u; + dst[2].val = (BeamInstr) hx; + dst += 3; + } + return op; +} + +static GenOp* +gen_has_map_fields(LoaderState* stp, GenOpArg Fail, GenOpArg Src, + GenOpArg Size, GenOpArg* Rest) +{ + GenOp* op; + Uint i; + Uint n; + + ASSERT(Size.type == TAG_u); + n = Size.val; + + NEW_GENOP(stp, op); + GENOP_ARITY(op, 3 + 2*n); + op->next = NULL; + op->op = genop_get_map_elements_3; op->a[0] = Fail; op->a[1] = Src; - op->a[2] = Rest[0]; + op->a[2].type = TAG_u; + op->a[2].val = 2*n; + + for (i = 0; i < n; i++) { + op->a[3+2*i] = Rest[i]; + op->a[3+2*i+1].type = TAG_x; + op->a[3+2*i+1].val = SCRATCH_X_REG; /* Ignore result */ + } return op; } @@ -4006,8 +4426,8 @@ gen_has_map_field(LoaderState* stp, GenOpArg Fail, GenOpArg Src, static int freeze_code(LoaderState* stp) { - BeamInstr* code = stp->code; - Uint *literal_end = NULL; + BeamCodeHeader* code_hdr = stp->hdr; + BeamInstr* codev = (BeamInstr*) &stp->hdr->functions; int i; byte* str_table; unsigned strtab_size = stp->chunks[STR_CHUNK].size; @@ -4032,128 +4452,76 @@ freeze_code(LoaderState* stp) if (stp->line_instr == 0) { line_size = 0; } else { - line_size = (MI_LINE_FUNC_TAB + (stp->num_functions + 1) + - (stp->current_li+1) + stp->num_fnames) * - sizeof(Eterm) + (stp->current_li+1) * stp->loc_size; + line_size = (offsetof(BeamCodeLineTab,func_tab) + + (stp->num_functions + 1) * sizeof(BeamInstr**) /* func_tab */ + + (stp->current_li + 1) * sizeof(BeamInstr*) /* line items */ + + stp->num_fnames * sizeof(Eterm) /* fname table */ + + (stp->current_li + 1) * stp->loc_size); /* loc_tab */ } - size = (stp->ci * sizeof(BeamInstr)) + - (stp->total_literal_size * sizeof(Eterm)) + - strtab_size + attr_size + compile_size + line_size; + size = offsetof(BeamCodeHeader,functions) + (stp->ci * sizeof(BeamInstr)) + + strtab_size + attr_size + compile_size + MD5_SIZE + line_size; /* * Move the code to its final location. */ - code = (BeamInstr *) erts_realloc(ERTS_ALC_T_CODE, (void *) code, size); - CHKBLK(ERTS_ALC_T_CODE,code); + code_hdr = (BeamCodeHeader*) erts_realloc(ERTS_ALC_T_CODE, (void *) code_hdr, size); + codev = (BeamInstr*) &code_hdr->functions; + CHKBLK(ERTS_ALC_T_CODE,code_hdr); /* * Place a pointer to the op_int_code_end instruction in the * function table in the beginning of the file. */ - code[MI_FUNCTIONS+stp->num_functions] = (BeamInstr) (code + stp->ci - 1); - CHKBLK(ERTS_ALC_T_CODE,code); + code_hdr->functions[stp->num_functions] = (codev + stp->ci - 1); + CHKBLK(ERTS_ALC_T_CODE,code_hdr); /* * Store the pointer to the on_load function. */ if (stp->on_load) { - code[MI_ON_LOAD_FUNCTION_PTR] = (BeamInstr) (code + stp->on_load); + code_hdr->on_load_function_ptr = codev + stp->on_load; } else { - code[MI_ON_LOAD_FUNCTION_PTR] = 0; + code_hdr->on_load_function_ptr = NULL; } - CHKBLK(ERTS_ALC_T_CODE,code); + CHKBLK(ERTS_ALC_T_CODE,code_hdr); - literal_end = (Uint *) (code+stp->ci); /* - * Place the literal heap directly after the code and fix up all - * instructions that refer to it. + * Place the literals in their own allocated heap (for fast range check) + * and fix up all instructions that refer to it. */ { - Uint* ptr; - Uint* low; - Uint* high; + Eterm* ptr; LiteralPatch* lp; - struct erl_off_heap_header* off_heap = 0; - struct erl_off_heap_header** off_heap_last = &off_heap; - - low = (Uint *) (code+stp->ci); - high = low + stp->total_literal_size; - code[MI_LITERALS_START] = (BeamInstr) low; - code[MI_LITERALS_END] = (BeamInstr) high; - ptr = low; - for (i = 0; i < stp->num_literals; i++) { - SWord offset; - struct erl_off_heap_header* t_off_heap; - - sys_memcpy(ptr, stp->literals[i].heap, - stp->literals[i].heap_size*sizeof(Eterm)); - offset = ptr - stp->literals[i].heap; - stp->literals[i].offset = offset; - high = ptr + stp->literals[i].heap_size; - while (ptr < high) { - Eterm val = *ptr; - switch (primary_tag(val)) { - case TAG_PRIMARY_LIST: - case TAG_PRIMARY_BOXED: - *ptr++ = offset_ptr(val, offset); - break; - case TAG_PRIMARY_HEADER: - if (header_is_transparent(val)) { - ptr++; - } else { - if (thing_subtag(val) == REFC_BINARY_SUBTAG) { - struct erl_off_heap_header* oh; - - oh = (struct erl_off_heap_header*) ptr; - if (oh->next) { - Eterm** uptr = (Eterm **) (void *) &oh->next; - *uptr += offset; - } - } - ptr += 1 + thing_arityval(val); - } - break; - default: - ptr++; - break; - } - } - ASSERT(ptr == high); + ErlOffHeap code_off_heap; - /* - * Re-link the off_heap list for this term onto the - * off_heap list for the entire module. - */ - t_off_heap = stp->literals[i].off_heap.first; - if (t_off_heap) { - t_off_heap = (struct erl_off_heap_header *) - offset_ptr((UWord) t_off_heap, offset); - while (t_off_heap) { - *off_heap_last = t_off_heap; - off_heap_last = &t_off_heap->next; - t_off_heap = t_off_heap->next; - } - } + ERTS_INIT_OFF_HEAP(&code_off_heap); + + ptr = (Eterm*)erts_alloc(ERTS_ALC_T_LITERAL, + stp->total_literal_size*sizeof(Eterm)); + code_hdr->literals_start = ptr; + code_hdr->literals_end = ptr + stp->total_literal_size; + for (i = 0; i < stp->num_literals; i++) { + if (is_not_immed(stp->literals[i].term)) { + erts_move_multi_frags(&ptr, &code_off_heap, + stp->literals[i].heap_frags, + &stp->literals[i].term, 1, 1); + ASSERT(erts_is_literal(stp->literals[i].term, + ptr_val(stp->literals[i].term))); + } } - code[MI_LITERALS_OFF_HEAP] = (BeamInstr) off_heap; + code_hdr->literals_off_heap = code_off_heap.first; lp = stp->literal_patches; while (lp != 0) { BeamInstr* op_ptr; - Uint literal; Literal* lit; - op_ptr = code + lp->pos; + op_ptr = codev + lp->pos; lit = &stp->literals[op_ptr[0]]; - literal = lit->term; - if (is_boxed(literal) || is_list(literal)) { - literal = offset_ptr(literal, lit->offset); - } - op_ptr[0] = literal; + op_ptr[0] = lit->term; lp = lp->next; } - literal_end += stp->total_literal_size; } CHKBLK(ERTS_ALC_T_CODE,code); @@ -4161,52 +4529,49 @@ freeze_code(LoaderState* stp) * If there is line information, place it here. */ if (stp->line_instr == 0) { - code[MI_LINE_TABLE] = (BeamInstr) 0; - str_table = (byte *) literal_end; + code_hdr->line_table = NULL; + str_table = (byte *) (codev + stp->ci); } else { - Eterm* line_tab = (Eterm *) literal_end; - Eterm* p; - int ftab_size = stp->num_functions; - int num_instrs = stp->current_li; - Eterm* first_line_item; + BeamCodeLineTab* const line_tab = (BeamCodeLineTab *) (codev+stp->ci); + const int ftab_size = stp->num_functions; + const int num_instrs = stp->current_li; + const BeamInstr** const line_items = + (const BeamInstr**) &line_tab->func_tab[ftab_size + 1]; - code[MI_LINE_TABLE] = (BeamInstr) line_tab; - p = line_tab + MI_LINE_FUNC_TAB; + code_hdr->line_table = line_tab; - first_line_item = (p + ftab_size + 1); for (i = 0; i < ftab_size; i++) { - *p++ = (Eterm) (BeamInstr) (first_line_item + stp->func_line[i]); + line_tab->func_tab[i] = line_items + stp->func_line[i]; } - *p++ = (Eterm) (BeamInstr) (first_line_item + num_instrs); - ASSERT(p == first_line_item); + line_tab->func_tab[i] = line_items + num_instrs; + for (i = 0; i < num_instrs; i++) { - *p++ = (Eterm) (BeamInstr) (code + stp->line_instr[i].pos); + line_items[i] = codev + stp->line_instr[i].pos; } - *p++ = (Eterm) (BeamInstr) (code + stp->ci - 1); + line_items[i] = codev + stp->ci - 1; - line_tab[MI_LINE_FNAME_PTR] = (Eterm) (BeamInstr) p; - memcpy(p, stp->fname, stp->num_fnames*sizeof(Eterm)); - p += stp->num_fnames; + line_tab->fname_ptr = (Eterm*) &line_items[i + 1]; + memcpy(line_tab->fname_ptr, stp->fname, stp->num_fnames*sizeof(Eterm)); - line_tab[MI_LINE_LOC_TAB] = (Eterm) (BeamInstr) p; - line_tab[MI_LINE_LOC_SIZE] = stp->loc_size; + line_tab->loc_size = stp->loc_size; if (stp->loc_size == 2) { - Uint16* locp = (Uint16 *) p; - for (i = 0; i < num_instrs; i++) { + Uint16* locp = (Uint16 *) &line_tab->fname_ptr[stp->num_fnames]; + line_tab->loc_tab.p2 = locp; + for (i = 0; i < num_instrs; i++) { *locp++ = (Uint16) stp->line_instr[i].loc; - } - *locp++ = LINE_INVALID_LOCATION; - str_table = (byte *) locp; + } + *locp++ = LINE_INVALID_LOCATION; + str_table = (byte *) locp; } else { - Uint32* locp = (Uint32 *) p; - ASSERT(stp->loc_size == 4); + Uint32* locp = (Uint32 *) &line_tab->fname_ptr[stp->num_fnames]; + ASSERT(stp->loc_size == 4); + line_tab->loc_tab.p4 = locp; for (i = 0; i < num_instrs; i++) { *locp++ = stp->line_instr[i].loc; } *locp++ = LINE_INVALID_LOCATION; - str_table = (byte *) locp; + str_table = (byte *) locp; } - CHKBLK(ERTS_ALC_T_CODE,code); } @@ -4218,13 +4583,13 @@ freeze_code(LoaderState* stp) if (attr_size) { byte* attr = str_table + strtab_size; sys_memcpy(attr, stp->chunks[ATTR_CHUNK].start, stp->chunks[ATTR_CHUNK].size); - code[MI_ATTR_PTR] = (BeamInstr) attr; - code[MI_ATTR_SIZE] = (BeamInstr) stp->chunks[ATTR_CHUNK].size; + code_hdr->attr_ptr = attr; + code_hdr->attr_size = (BeamInstr) stp->chunks[ATTR_CHUNK].size; decoded_size = erts_decode_ext_size(attr, attr_size); if (decoded_size < 0) { LoadError0(stp, "bad external term representation of module attributes"); } - code[MI_ATTR_SIZE_ON_HEAP] = decoded_size; + code_hdr->attr_size_on_heap = decoded_size; } CHKBLK(ERTS_ALC_T_CODE,code); if (compile_size) { @@ -4234,9 +4599,9 @@ freeze_code(LoaderState* stp) stp->chunks[COMPILE_CHUNK].size); CHKBLK(ERTS_ALC_T_CODE,code); - code[MI_COMPILE_PTR] = (BeamInstr) compile_info; + code_hdr->compile_ptr = compile_info; CHKBLK(ERTS_ALC_T_CODE,code); - code[MI_COMPILE_SIZE] = (BeamInstr) stp->chunks[COMPILE_CHUNK].size; + code_hdr->compile_size = (BeamInstr) stp->chunks[COMPILE_CHUNK].size; CHKBLK(ERTS_ALC_T_CODE,code); decoded_size = erts_decode_ext_size(compile_info, compile_size); CHKBLK(ERTS_ALC_T_CODE,code); @@ -4244,15 +4609,24 @@ freeze_code(LoaderState* stp) LoadError0(stp, "bad external term representation of compilation information"); } CHKBLK(ERTS_ALC_T_CODE,code); - code[MI_COMPILE_SIZE_ON_HEAP] = decoded_size; + code_hdr->compile_size_on_heap = decoded_size; + } + CHKBLK(ERTS_ALC_T_CODE,code); + { + byte* md5_sum = str_table + strtab_size + attr_size + compile_size; + CHKBLK(ERTS_ALC_T_CODE,code); + sys_memcpy(md5_sum, stp->mod_md5, MD5_SIZE); + CHKBLK(ERTS_ALC_T_CODE,code); + code_hdr->md5_ptr = md5_sum; + CHKBLK(ERTS_ALC_T_CODE,code); } CHKBLK(ERTS_ALC_T_CODE,code); /* * Make sure that we have not overflowed the allocated code space. */ - ASSERT(str_table + strtab_size + attr_size + compile_size == - ((byte *) code) + size); + ASSERT(str_table + strtab_size + attr_size + compile_size + MD5_SIZE == + ((byte *) code_hdr) + size); /* * Patch all instructions that refer to the string table. @@ -4264,46 +4638,47 @@ freeze_code(LoaderState* stp) BeamInstr* op_ptr; byte* strp; - op_ptr = code + sp->pos; + op_ptr = codev + sp->pos; strp = str_table + op_ptr[0]; op_ptr[0] = (BeamInstr) strp; sp = sp->next; } } - CHKBLK(ERTS_ALC_T_CODE,code); + CHKBLK(ERTS_ALC_T_CODE,code_hdr); /* * Resolve all labels. */ for (i = 0; i < stp->num_labels; i++) { - Uint this_patch; - Uint next_patch; + Sint this_patch; + Sint next_patch; Uint value = stp->labels[i].value; - if (value == 0 && stp->labels[i].patches != 0) { + if (value == 0 && stp->labels[i].patches >= 0) { LoadError1(stp, "label %d not resolved", i); } ASSERT(value < stp->ci); this_patch = stp->labels[i].patches; - while (this_patch != 0) { + while (this_patch >= 0) { ASSERT(this_patch < stp->ci); - next_patch = code[this_patch]; + next_patch = codev[this_patch]; ASSERT(next_patch < stp->ci); - code[this_patch] = (BeamInstr) (code + value); + codev[this_patch] = (BeamInstr) (codev + value); this_patch = next_patch; } } - CHKBLK(ERTS_ALC_T_CODE,code); + CHKBLK(ERTS_ALC_T_CODE,code_hdr); /* * Save the updated code pointer and code size. */ - stp->code = code; + stp->hdr = code_hdr; + stp->codev = codev; stp->loaded_size = size; - CHKBLK(ERTS_ALC_T_CODE,code); + CHKBLK(ERTS_ALC_T_CODE,code_hdr); return 1; load_error: @@ -4311,7 +4686,8 @@ freeze_code(LoaderState* stp) * Make sure that the caller frees the newly reallocated block, and * not the old one (in case it has moved). */ - stp->code = code; + stp->hdr = code_hdr; + stp->codev = codev; return 0; } @@ -4322,7 +4698,7 @@ final_touch(LoaderState* stp) int on_load = stp->on_load; unsigned catches; Uint index; - BeamInstr* code = stp->code; + BeamInstr* codev = stp->codev; Module* modp; /* @@ -4332,10 +4708,10 @@ final_touch(LoaderState* stp) index = stp->catches; catches = BEAM_CATCHES_NIL; while (index != 0) { - BeamInstr next = code[index]; - code[index] = BeamOpCode(op_catch_yf); - catches = beam_catches_cons((BeamInstr *)code[index+2], catches); - code[index+2] = make_catch(catches); + BeamInstr next = codev[index]; + codev[index] = BeamOpCode(op_catch_yf); + catches = beam_catches_cons((BeamInstr *)codev[index+2], catches); + codev[index+2] = make_catch(catches); index = next; } modp = erts_put_module(stp->module); @@ -4386,8 +4762,8 @@ final_touch(LoaderState* stp) current = stp->import[i].patches; while (current != 0) { ASSERT(current < stp->ci); - next = stp->code[current]; - stp->code[current] = import; + next = stp->codev[current]; + stp->codev[current] = import; current = next; } } @@ -4400,7 +4776,7 @@ final_touch(LoaderState* stp) for (i = 0; i < stp->num_lambdas; i++) { unsigned entry_label = stp->lambdas[i].label; ErlFunEntry* fe = stp->lambdas[i].fe; - BeamInstr* code_ptr = (BeamInstr *) (stp->code + stp->labels[entry_label].value); + BeamInstr* code_ptr = stp->codev + stp->labels[entry_label].value; if (fe->address[0] != 0) { /* @@ -4513,7 +4889,8 @@ transform_engine(LoaderState* st) if (var[i].type != instr->a[ap].type) goto restart; switch (var[i].type) { - case TAG_r: case TAG_n: break; + case TAG_n: + break; default: if (var[i].val != instr->a[ap].val) goto restart; @@ -4967,7 +5344,8 @@ get_tag_and_value(LoaderState* stp, Uint len_code, arity = count/sizeof(Eterm); *result = new_literal(stp, &hp, arity+1); - (void) bytes_to_big(bigbuf, count, neg, hp); + if (is_nil(bytes_to_big(bigbuf, count, neg, hp))) + goto load_error; if (bigbuf != default_buf) { erts_free(ERTS_ALC_T_LOADER_TMP, (void *) bigbuf); @@ -5022,7 +5400,7 @@ new_label(LoaderState* stp) (void *) stp->labels, stp->num_labels * sizeof(Label)); stp->labels[num].value = 0; - stp->labels[num].patches = 0; + stp->labels[num].patches = -1; return num; } @@ -5071,19 +5449,18 @@ new_literal(LoaderState* stp, Eterm** hpp, Uint heap_size) stp->total_literal_size += heap_size; lit = stp->literals + stp->num_literals; - lit->offset = 0; - lit->heap_size = heap_size; - lit->heap = erts_alloc(ERTS_ALC_T_PREPARED_CODE, heap_size*sizeof(Eterm)); - lit->term = make_boxed(lit->heap); - lit->off_heap.first = 0; - lit->off_heap.overhead = 0; - *hpp = lit->heap; + lit->heap_frags = new_literal_fragment(heap_size); + lit->term = make_boxed(lit->heap_frags->mem); + *hpp = lit->heap_frags->mem; return stp->num_literals++; } Eterm erts_module_info_0(Process* p, Eterm module) { + Module* modp; + ErtsCodeIndex code_ix = erts_active_code_ix(); + BeamCodeHeader* code_hdr; Eterm *hp; Eterm list = NIL; Eterm tup; @@ -5092,21 +5469,31 @@ erts_module_info_0(Process* p, Eterm module) return THE_NON_VALUE; } - if (erts_get_module(module, erts_active_code_ix()) == NULL) { + modp = erts_get_module(module, code_ix); + if (modp == NULL) { return THE_NON_VALUE; } + code_hdr = modp->curr.code_hdr; + if (code_hdr == NULL) { + return THE_NON_VALUE; + } + #define BUILD_INFO(What) \ - tup = erts_module_info_1(p, module, What); \ + tup = get_module_info(p, code_ix, code_hdr, module, What); \ hp = HAlloc(p, 5); \ tup = TUPLE2(hp, What, tup); \ hp += 3; \ list = CONS(hp, tup, list) + BUILD_INFO(am_md5); +#ifdef HIPE + BUILD_INFO(am_native); +#endif BUILD_INFO(am_compile); BUILD_INFO(am_attributes); - BUILD_INFO(am_imports); BUILD_INFO(am_exports); + BUILD_INFO(am_module); #undef BUILD_INFO return list; } @@ -5114,20 +5501,47 @@ erts_module_info_0(Process* p, Eterm module) Eterm erts_module_info_1(Process* p, Eterm module, Eterm what) { + Module* modp; + ErtsCodeIndex code_ix = erts_active_code_ix(); + BeamCodeHeader* code_hdr; + + if (is_not_atom(module)) { + return THE_NON_VALUE; + } + + modp = erts_get_module(module, code_ix); + if (modp == NULL) { + return THE_NON_VALUE; + } + + code_hdr = modp->curr.code_hdr; + if (code_hdr == NULL) { + return THE_NON_VALUE; + } + + return get_module_info(p, code_ix, code_hdr, module, what); +} + +static Eterm +get_module_info(Process* p, ErtsCodeIndex code_ix, BeamCodeHeader* code_hdr, + Eterm module, Eterm what) +{ if (what == am_module) { return module; - } else if (what == am_imports) { - return NIL; + } else if (what == am_md5) { + return md5_of_module(p, code_hdr); } else if (what == am_exports) { - return exported_from_module(p, module); + return exported_from_module(p, code_ix, module); } else if (what == am_functions) { - return functions_in_module(p, module); + return functions_in_module(p, code_hdr); } else if (what == am_attributes) { - return attributes_for_module(p, module); + return attributes_for_module(p, code_hdr); } else if (what == am_compile) { - return compilation_info_for_module(p, module); + return compilation_info_for_module(p, code_hdr); } else if (what == am_native_addresses) { - return native_addresses(p, module); + return native_addresses(p, code_hdr); + } else if (what == am_native) { + return has_native(code_hdr); } return THE_NON_VALUE; } @@ -5135,16 +5549,12 @@ erts_module_info_1(Process* p, Eterm module, Eterm what) /* * Builds a list of all functions in the given module: * [{Name, Arity},...] - * - * Returns a tagged term, or 0 on error. */ Eterm functions_in_module(Process* p, /* Process whose heap to use. */ - Eterm mod) /* Tagged atom for module. */ + BeamCodeHeader* code_hdr) { - Module* modp; - BeamInstr* code; int i; Uint num_functions; Uint need; @@ -5152,21 +5562,12 @@ functions_in_module(Process* p, /* Process whose heap to use. */ Eterm* hp_end; Eterm result = NIL; - if (is_not_atom(mod)) { - return THE_NON_VALUE; - } - - modp = erts_get_module(mod, erts_active_code_ix()); - if (modp == NULL) { - return THE_NON_VALUE; - } - code = modp->curr.code; - num_functions = code[MI_NUM_FUNCTIONS]; + num_functions = code_hdr->num_functions; need = 5*num_functions; hp = HAlloc(p, need); hp_end = hp + need; for (i = num_functions-1; i >= 0 ; i--) { - BeamInstr* func_info = (BeamInstr *) code[MI_FUNCTIONS+i]; + BeamInstr* func_info = code_hdr->functions[i]; Eterm name = (Eterm) func_info[3]; int arity = (int) func_info[4]; Eterm tuple; @@ -5188,17 +5589,49 @@ functions_in_module(Process* p, /* Process whose heap to use. */ } /* + * Returns 'true' if mod has any native compiled functions, otherwise 'false' + */ + +static Eterm +has_native(BeamCodeHeader *code_hdr) +{ + Eterm result = am_false; +#ifdef HIPE + if (erts_is_module_native(code_hdr)) { + result = am_true; + } +#endif + return result; +} + +int +erts_is_module_native(BeamCodeHeader* code_hdr) +{ + Uint i, num_functions; + + /* Check NativeAdress of first real function in module */ + if (code_hdr != NULL) { + num_functions = code_hdr->num_functions; + for (i=0; i<num_functions; i++) { + BeamInstr* func_info = (BeamInstr *) code_hdr->functions[i]; + Eterm name = (Eterm) func_info[3]; + if (is_atom(name)) { + return func_info[1] != 0; + } + else ASSERT(is_nil(name)); /* ignore BIF stubs */ + } + } + return 0; +} + +/* * Builds a list of all functions including native addresses. * [{Name,Arity,NativeAddress},...] - * - * Returns a tagged term, or 0 on error. */ static Eterm -native_addresses(Process* p, Eterm mod) +native_addresses(Process* p, BeamCodeHeader* code_hdr) { - Module* modp; - BeamInstr* code; int i; Eterm* hp; Uint num_functions; @@ -5206,22 +5639,12 @@ native_addresses(Process* p, Eterm mod) Eterm* hp_end; Eterm result = NIL; - if (is_not_atom(mod)) { - return THE_NON_VALUE; - } - - modp = erts_get_module(mod, erts_active_code_ix()); - if (modp == NULL) { - return THE_NON_VALUE; - } - - code = modp->curr.code; - num_functions = code[MI_NUM_FUNCTIONS]; + num_functions = code_hdr->num_functions; need = (6+BIG_UINT_HEAP_SIZE)*num_functions; hp = HAlloc(p, need); hp_end = hp + need; for (i = num_functions-1; i >= 0 ; i--) { - BeamInstr* func_info = (BeamInstr *) code[MI_FUNCTIONS+i]; + BeamInstr* func_info = code_hdr->functions[i]; Eterm name = (Eterm) func_info[3]; int arity = (int) func_info[4]; Eterm tuple; @@ -5242,25 +5665,18 @@ native_addresses(Process* p, Eterm mod) /* * Builds a list of all exported functions in the given module: * [{Name, Arity},...] - * - * Returns a tagged term, or 0 on error. */ Eterm exported_from_module(Process* p, /* Process whose heap to use. */ + ErtsCodeIndex code_ix, Eterm mod) /* Tagged atom for module. */ { int i; Eterm* hp = NULL; Eterm* hend = NULL; Eterm result = NIL; - ErtsCodeIndex code_ix; - if (is_not_atom(mod)) { - return THE_NON_VALUE; - } - - code_ix = erts_active_code_ix(); for (i = 0; i < export_list_size(code_ix); i++) { Export* ep = export_list(i,code_ix); @@ -5290,84 +5706,62 @@ exported_from_module(Process* p, /* Process whose heap to use. */ /* * Returns a list of all attributes for the module. - * - * Returns a tagged term, or 0 on error. */ Eterm attributes_for_module(Process* p, /* Process whose heap to use. */ - Eterm mod) /* Tagged atom for module. */ - + BeamCodeHeader* code_hdr) { - Module* modp; - BeamInstr* code; - Eterm* hp; byte* ext; Eterm result = NIL; - Eterm* end; - if (is_not_atom(mod) || (is_not_list(result) && is_not_nil(result))) { - return THE_NON_VALUE; - } - - modp = erts_get_module(mod, erts_active_code_ix()); - if (modp == NULL) { - return THE_NON_VALUE; - } - code = modp->curr.code; - ext = (byte *) code[MI_ATTR_PTR]; + ext = code_hdr->attr_ptr; if (ext != NULL) { - hp = HAlloc(p, code[MI_ATTR_SIZE_ON_HEAP]); - end = hp + code[MI_ATTR_SIZE_ON_HEAP]; - result = erts_decode_ext(&hp, &MSO(p), &ext); + ErtsHeapFactory factory; + erts_factory_proc_prealloc_init(&factory, p, code_hdr->attr_size_on_heap); + result = erts_decode_ext(&factory, &ext, 0); if (is_value(result)) { - ASSERT(hp <= end); + erts_factory_close(&factory); } - HRelease(p,end,hp); } return result; } /* * Returns a list containing compilation information. - * - * Returns a tagged term, or 0 on error. */ Eterm compilation_info_for_module(Process* p, /* Process whose heap to use. */ - Eterm mod) /* Tagged atom for module. */ + BeamCodeHeader* code_hdr) { - Module* modp; - BeamInstr* code; - Eterm* hp; byte* ext; Eterm result = NIL; - Eterm* end; - - if (is_not_atom(mod) || (is_not_list(result) && is_not_nil(result))) { - return THE_NON_VALUE; - } - modp = erts_get_module(mod, erts_active_code_ix()); - if (modp == NULL) { - return THE_NON_VALUE; - } - code = modp->curr.code; - ext = (byte *) code[MI_COMPILE_PTR]; + ext = code_hdr->compile_ptr; if (ext != NULL) { - hp = HAlloc(p, code[MI_COMPILE_SIZE_ON_HEAP]); - end = hp + code[MI_COMPILE_SIZE_ON_HEAP]; - result = erts_decode_ext(&hp, &MSO(p), &ext); + ErtsHeapFactory factory; + erts_factory_proc_prealloc_init(&factory, p, code_hdr->compile_size_on_heap); + result = erts_decode_ext(&factory, &ext, 0); if (is_value(result)) { - ASSERT(hp <= end); + erts_factory_close(&factory); } - HRelease(p,end,hp); } return result; } /* + * Returns the MD5 checksum for a module + */ + +Eterm +md5_of_module(Process* p, /* Process whose heap to use. */ + BeamCodeHeader* code_hdr) +{ + return new_binary(p, code_hdr->md5_ptr, MD5_SIZE); +} + +/* * Build a single {M,F,A,Loction} item to be part of * a stack trace. */ @@ -5543,7 +5937,7 @@ code_module_md5_1(BIF_ALIST_1) res = am_undefined; goto done; } - res = new_binary(p, stp->mod_md5, sizeof(stp->mod_md5)); + res = new_binary(p, stp->mod_md5, MD5_SIZE); done: erts_free_aligned_binary_bytes(temp_alloc); @@ -5563,7 +5957,7 @@ make_stub(BeamInstr* fp, Eterm mod, Eterm func, Uint arity, Uint native, BeamIns fp[4] = arity; #ifdef HIPE if (native) { - fp[5] = BeamOpCode(op_move_return_nr); + fp[5] = BeamOpCode(op_move_return_n); hipe_mfa_save_orig_beam_op(mod, func, arity, fp+5); } #endif @@ -5575,7 +5969,7 @@ static byte* stub_copy_info(LoaderState* stp, int chunk, /* Chunk: ATTR_CHUNK or COMPILE_CHUNK */ byte* info, /* Where to store info. */ - BeamInstr* ptr_word, /* Where to store pointer into info. */ + byte** ptr_word, /* Where to store pointer into info. */ BeamInstr* size_word, /* Where to store size into info. */ BeamInstr* size_on_heap_word) /* Where to store size on heap. */ { @@ -5583,7 +5977,7 @@ stub_copy_info(LoaderState* stp, Uint size = stp->chunks[chunk].size; if (size != 0) { memcpy(info, stp->chunks[chunk].start, size); - *ptr_word = (BeamInstr) info; + *ptr_word = info; decoded_size = erts_decode_ext_size(info, size); if (decoded_size < 0) { return 0; @@ -5836,16 +6230,16 @@ erts_make_stub_module(Process* p, Eterm Mod, Eterm Beam, Eterm Info) LoaderState* stp; BeamInstr Funcs; BeamInstr Patchlist; + Eterm MD5Bin; Eterm* tp; - BeamInstr* code = NULL; - BeamInstr* ptrs; + BeamCodeHeader* code_hdr; + BeamInstr* code_base; BeamInstr* fp; byte* info; - Uint ci; - int n; + Sint n; int code_size; int rval; - int i; + Sint i; byte* temp_alloc = NULL; byte* bytes; Uint size; @@ -5864,12 +6258,15 @@ erts_make_stub_module(Process* p, Eterm Mod, Eterm Beam, Eterm Info) goto error; } tp = tuple_val(Info); - if (tp[0] != make_arityval(2)) { + if (tp[0] != make_arityval(3)) { goto error; } Funcs = tp[1]; - Patchlist = tp[2]; - + Patchlist = tp[2]; + MD5Bin = tp[3]; + if (is_not_binary(MD5Bin) || (binary_size(MD5Bin) != MD5_SIZE)) { + goto error; + } if ((n = erts_list_length(Funcs)) < 0) { goto error; } @@ -5916,37 +6313,40 @@ erts_make_stub_module(Process* p, Eterm Mod, Eterm Beam, Eterm Info) * Allocate memory for the stub module. */ - code_size = ((WORDS_PER_FUNCTION+1)*n + MI_FUNCTIONS + 2) * sizeof(BeamInstr); - code_size += stp->chunks[ATTR_CHUNK].size; - code_size += stp->chunks[COMPILE_CHUNK].size; - code = erts_alloc_fnf(ERTS_ALC_T_CODE, code_size); - if (!code) { + code_size = (offsetof(BeamCodeHeader,functions) + + ((n+1) * sizeof(BeamInstr*)) + + (WORDS_PER_FUNCTION*n + 1) * sizeof(BeamInstr) + + stp->chunks[ATTR_CHUNK].size + + stp->chunks[COMPILE_CHUNK].size + + MD5_SIZE); + code_hdr = erts_alloc_fnf(ERTS_ALC_T_CODE, code_size); + if (!code_hdr) { goto error; } /* - * Initialize code area. + * Initialize code header. */ - code[MI_NUM_FUNCTIONS] = n; - code[MI_ATTR_PTR] = 0; - code[MI_ATTR_SIZE] = 0; - code[MI_ATTR_SIZE_ON_HEAP] = 0; - code[MI_COMPILE_PTR] = 0; - code[MI_COMPILE_SIZE] = 0; - code[MI_COMPILE_SIZE_ON_HEAP] = 0; - code[MI_LITERALS_START] = 0; - code[MI_LITERALS_END] = 0; - code[MI_LITERALS_OFF_HEAP] = 0; - code[MI_ON_LOAD_FUNCTION_PTR] = 0; - ci = MI_FUNCTIONS + n + 1; + code_hdr->num_functions = n; + code_hdr->attr_ptr = NULL; + code_hdr->attr_size = 0; + code_hdr->attr_size_on_heap = 0; + code_hdr->compile_ptr = NULL; + code_hdr->compile_size = 0; + code_hdr->compile_size_on_heap = 0; + code_hdr->literals_start = NULL; + code_hdr->literals_end = NULL; + code_hdr->literals_off_heap = 0; + code_hdr->on_load_function_ptr = NULL; + code_hdr->line_table = NULL; + code_hdr->md5_ptr = NULL; /* * Make stubs for all functions. */ - ptrs = code + MI_FUNCTIONS; - fp = code + ci; + fp = code_base = (BeamInstr*) &code_hdr->functions[n+1]; for (i = 0; i < n; i++) { Eterm* listp; Eterm tuple; @@ -5989,11 +6389,11 @@ erts_make_stub_module(Process* p, Eterm Mod, Eterm Beam, Eterm Info) * Set the pointer and make the stub. Put a return instruction * as the body until we know what kind of trap we should put there. */ - ptrs[i] = (BeamInstr) fp; + code_hdr->functions[i] = fp; #ifdef HIPE op = (Eterm) BeamOpCode(op_hipe_trap_call); /* Might be changed later. */ #else - op = (Eterm) BeamOpCode(op_move_return_nr); + op = (Eterm) BeamOpCode(op_move_return_n); #endif fp = make_stub(fp, Mod, func, arity, (Uint)native_address, op); } @@ -6002,7 +6402,7 @@ erts_make_stub_module(Process* p, Eterm Mod, Eterm Beam, Eterm Info) * Insert the last pointer and the int_code_end instruction. */ - ptrs[i] = (BeamInstr) fp; + code_hdr->functions[i] = fp; *fp++ = (BeamInstr) BeamOp(op_int_code_end); /* @@ -6011,25 +6411,34 @@ erts_make_stub_module(Process* p, Eterm Mod, Eterm Beam, Eterm Info) info = (byte *) fp; info = stub_copy_info(stp, ATTR_CHUNK, info, - code+MI_ATTR_PTR, - code+MI_ATTR_SIZE, - code+MI_ATTR_SIZE_ON_HEAP); + &code_hdr->attr_ptr, + &code_hdr->attr_size, + &code_hdr->attr_size_on_heap); if (info == NULL) { goto error; } info = stub_copy_info(stp, COMPILE_CHUNK, info, - code+MI_COMPILE_PTR, - code+MI_COMPILE_SIZE, - code+MI_COMPILE_SIZE_ON_HEAP); + &code_hdr->compile_ptr, + &code_hdr->compile_size, + &code_hdr->compile_size_on_heap); if (info == NULL) { goto error; } + { + byte *tmp = NULL; + byte *md5 = NULL; + if ((md5 = erts_get_aligned_binary_bytes(MD5Bin, &tmp)) != NULL) { + sys_memcpy(info, md5, MD5_SIZE); + code_hdr->md5_ptr = info; + } + erts_free_aligned_binary_bytes(tmp); + } /* * Insert the module in the module table. */ - rval = insert_new_code(p, 0, p->group_leader, Mod, code, code_size); + rval = insert_new_code(p, 0, p->group_leader, Mod, code_hdr, code_size); if (rval != NIL) { goto error; } @@ -6038,7 +6447,7 @@ erts_make_stub_module(Process* p, Eterm Mod, Eterm Beam, Eterm Info) * Export all stub functions and insert the correct type of HiPE trap. */ - fp = code + ci; + fp = code_base; for (i = 0; i < n; i++) { stub_final_touch(stp, fp); fp += WORDS_PER_FUNCTION; diff --git a/erts/emulator/beam/beam_load.h b/erts/emulator/beam/beam_load.h index bd22b0c4de..22ab71c868 100644 --- a/erts/emulator/beam/beam_load.h +++ b/erts/emulator/beam/beam_load.h @@ -3,16 +3,17 @@ * * Copyright Ericsson AB 1999-2013. All Rights Reserved. * - * The contents of this file are subject to the Erlang Public License, - * Version 1.1, (the "License"); you may not use this file except in - * compliance with the License. You should have received a copy of the - * Erlang Public License along with this software. If not, it can be - * retrieved online at http://www.erlang.org/. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Software distributed under the License is distributed on an "AS IS" - * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See - * the License for the specific language governing rights and limitations - * under the License. + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. * * %CopyrightEnd% */ @@ -26,7 +27,6 @@ Eterm beam_make_current_old(Process *c_p, ErtsProcLocks c_p_locks, Eterm module); - typedef struct gen_op_entry { char* name; int arity; @@ -51,6 +51,7 @@ extern BeamInstr* em_call_error_handler; extern BeamInstr* em_apply_bif; extern BeamInstr* em_call_nif; + /* * The following variables keep a sorted list of address ranges for * each module. It allows us to quickly find a function given an @@ -59,68 +60,81 @@ extern BeamInstr* em_call_nif; /* Total code size in bytes */ extern Uint erts_total_code_size; -/* - * Index into start of code chunks which contains additional information - * about the loaded module. - * - * First number of functions. - */ - -#define MI_NUM_FUNCTIONS 0 - -/* - * The attributes retrieved by Mod:module_info(attributes). - */ - -#define MI_ATTR_PTR 1 -#define MI_ATTR_SIZE 2 -#define MI_ATTR_SIZE_ON_HEAP 3 - -/* - * The compilation information retrieved by Mod:module_info(compile). - */ -#define MI_COMPILE_PTR 4 -#define MI_COMPILE_SIZE 5 -#define MI_COMPILE_SIZE_ON_HEAP 6 +typedef struct BeamCodeLineTab_ BeamCodeLineTab; /* - * Literal area (constant pool). - */ -#define MI_LITERALS_START 7 -#define MI_LITERALS_END 8 -#define MI_LITERALS_OFF_HEAP 9 - - -/* - * Pointer to the on_load function (or NULL if none). - */ -#define MI_ON_LOAD_FUNCTION_PTR 10 - -/* - * Pointer to the line table (or NULL if none). - */ -#define MI_LINE_TABLE 11 - -/* - * Start of function pointer table. This table contains pointers to - * all functions in the module plus an additional pointer just beyond - * the end of the last function. - * - * The actual loaded code (for the first function) start just beyond - * this table. + * Header of code chunks which contains additional information + * about the loaded module. */ - -#define MI_FUNCTIONS 12 +typedef struct beam_code_header { + /* + * Number of functions. + */ + UWord num_functions; + + /* + * The attributes retrieved by Mod:module_info(attributes). + */ + byte* attr_ptr; + UWord attr_size; + UWord attr_size_on_heap; + + /* + * The compilation information retrieved by Mod:module_info(compile). + */ + byte* compile_ptr; + UWord compile_size; + UWord compile_size_on_heap; + + /* + * Literal area (constant pool). + */ + Eterm* literals_start; + Eterm* literals_end; + struct erl_off_heap_header* literals_off_heap; + + /* + * Pointer to the on_load function (or NULL if none). + */ + BeamInstr* on_load_function_ptr; + + /* + * Pointer to the line table (or NULL if none). + */ + BeamCodeLineTab* line_table; + + /* + * Pointer to the module MD5 sum (16 bytes) + */ + byte* md5_ptr; + + /* + * Start of function pointer table. This table contains pointers to + * all functions in the module plus an additional pointer just beyond + * the end of the last function. + * + * The actual loaded code (for the first function) start just beyond + * this table. + */ + BeamInstr* functions[1]; + +}BeamCodeHeader; + +int erts_is_module_native(BeamCodeHeader* code); /* * Layout of the line table. */ - -#define MI_LINE_FNAME_PTR 0 -#define MI_LINE_LOC_TAB 1 -#define MI_LINE_LOC_SIZE 2 -#define MI_LINE_FUNC_TAB 3 +struct BeamCodeLineTab_ { + Eterm* fname_ptr; + int loc_size; + union { + Uint16* p2; + Uint32* p4; + }loc_tab; + const BeamInstr** func_tab[1]; +}; #define LINE_INVALID_LOCATION (0) diff --git a/erts/emulator/beam/beam_ranges.c b/erts/emulator/beam/beam_ranges.c index 0f2d5d0c2a..54c337ee72 100644 --- a/erts/emulator/beam/beam_ranges.c +++ b/erts/emulator/beam/beam_ranges.c @@ -3,16 +3,17 @@ * * Copyright Ericsson AB 2012. All Rights Reserved. * - * The contents of this file are subject to the Erlang Public License, - * Version 1.1, (the "License"); you may not use this file except in - * compliance with the License. You should have received a copy of the - * Erlang Public License along with this software. If not, it can be - * retrieved online at http://www.erlang.org/. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Software distributed under the License is distributed on an "AS IS" - * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See - * the License for the specific language governing rights and limitations - * under the License. + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. * * %CopyrightEnd% */ @@ -36,8 +37,8 @@ typedef struct { #define RANGE_END(R) ((BeamInstr*)erts_smp_atomic_read_nob(&(R)->end)) static Range* find_range(BeamInstr* pc); -static void lookup_loc(FunctionInfo* fi, BeamInstr* pc, - BeamInstr* modp, int idx); +static void lookup_loc(FunctionInfo* fi, const BeamInstr* pc, + BeamCodeHeader*, int idx); /* * The following variables keep a sorted list of address ranges for @@ -52,6 +53,7 @@ struct ranges { }; static struct ranges r[ERTS_NUM_CODE_IX]; static erts_smp_atomic_t mem_used; +static Range* write_ptr; #ifdef HARD_DEBUG static void check_consistency(struct ranges* p) @@ -71,6 +73,17 @@ static void check_consistency(struct ranges* p) # define CHECK(r) #endif /* HARD_DEBUG */ +static int +rangecompare(Range* a, Range* b) +{ + if (a->start < b->start) { + return -1; + } else if (a->start == b->start) { + return 0; + } else { + return 1; + } +} void erts_init_ranges(void) @@ -87,45 +100,70 @@ erts_init_ranges(void) } void -erts_start_staging_ranges(void) +erts_start_staging_ranges(int num_new) { + ErtsCodeIndex src = erts_active_code_ix(); ErtsCodeIndex dst = erts_staging_code_ix(); + Sint need; if (r[dst].modules) { erts_smp_atomic_add_nob(&mem_used, -r[dst].allocated); erts_free(ERTS_ALC_T_MODULE_REFS, r[dst].modules); - r[dst].modules = NULL; } + + need = r[dst].allocated = r[src].n + num_new; + erts_smp_atomic_add_nob(&mem_used, need); + write_ptr = erts_alloc(ERTS_ALC_T_MODULE_REFS, + need * sizeof(Range)); + r[dst].modules = write_ptr; } void erts_end_staging_ranges(int commit) { - ErtsCodeIndex dst = erts_staging_code_ix(); - - if (commit && r[dst].modules == NULL) { + if (commit) { Sint i; - Sint n; - - /* No modules added, just clone src and remove purged code. */ ErtsCodeIndex src = erts_active_code_ix(); + ErtsCodeIndex dst = erts_staging_code_ix(); + Range* mp; + Sint num_inserted; - erts_smp_atomic_add_nob(&mem_used, r[src].n); - r[dst].modules = erts_alloc(ERTS_ALC_T_MODULE_REFS, - r[src].n * sizeof(Range)); - r[dst].allocated = r[src].n; - n = 0; + mp = r[dst].modules; + num_inserted = write_ptr - mp; for (i = 0; i < r[src].n; i++) { Range* rp = r[src].modules+i; if (rp->start < RANGE_END(rp)) { /* Only insert a module that has not been purged. */ - r[dst].modules[n] = *rp; - n++; + write_ptr->start = rp->start; + erts_smp_atomic_init_nob(&write_ptr->end, + (erts_aint_t)(RANGE_END(rp))); + write_ptr++; } } - r[dst].n = n; + + /* + * There are num_inserted new range entries (unsorted) at the + * beginning of the modules array, followed by the old entries + * (sorted). We must now sort the entire array. + */ + + r[dst].n = write_ptr - mp; + if (num_inserted > 1) { + qsort(mp, r[dst].n, sizeof(Range), + (int (*)(const void *, const void *)) rangecompare); + } else if (num_inserted == 1) { + /* Sift the new range into place. This is faster than qsort(). */ + Range t = mp[0]; + for (i = 0; i < r[dst].n-1 && t.start > mp[i+1].start; i++) { + mp[i] = mp[i+1]; + } + mp[i] = t; + } + r[dst].modules = mp; + CHECK(&r[dst]); erts_smp_atomic_set_nob(&r[dst].mid, - (erts_aint_t) (r[dst].modules + n / 2)); + (erts_aint_t) (r[dst].modules + + r[dst].n / 2)); } } @@ -134,82 +172,29 @@ erts_update_ranges(BeamInstr* code, Uint size) { ErtsCodeIndex dst = erts_staging_code_ix(); ErtsCodeIndex src = erts_active_code_ix(); - Sint i; - Sint n; - Sint need; if (src == dst) { ASSERT(!erts_initialized); /* - * During start-up of system, the indices are the same. - * Handle this by faking a source area. + * During start-up of system, the indices are the same + * and erts_start_staging_ranges() has not been called. */ - src = (src+1) % ERTS_NUM_CODE_IX; - if (r[src].modules) { - erts_smp_atomic_add_nob(&mem_used, -r[src].allocated); - erts_free(ERTS_ALC_T_MODULE_REFS, r[src].modules); + if (r[dst].modules == NULL) { + Sint need = 128; + erts_smp_atomic_add_nob(&mem_used, need); + r[dst].modules = erts_alloc(ERTS_ALC_T_MODULE_REFS, + need * sizeof(Range)); + r[dst].allocated = need; + write_ptr = r[dst].modules; } - r[src] = r[dst]; - r[dst].modules = 0; } - CHECK(&r[src]); - - ASSERT(r[dst].modules == NULL); - need = r[dst].allocated = r[src].n + 1; - erts_smp_atomic_add_nob(&mem_used, need); - r[dst].modules = (Range *) erts_alloc(ERTS_ALC_T_MODULE_REFS, - need * sizeof(Range)); - n = 0; - for (i = 0; i < r[src].n; i++) { - Range* rp = r[src].modules+i; - if (code < rp->start) { - r[dst].modules[n].start = code; - erts_smp_atomic_init_nob(&r[dst].modules[n].end, - (erts_aint_t)(((byte *)code) + size)); - ASSERT(!n || RANGE_END(&r[dst].modules[n-1]) < code); - n++; - break; - } - if (rp->start < RANGE_END(rp)) { - /* Only insert a module that has not been purged. */ - r[dst].modules[n].start = rp->start; - erts_smp_atomic_init_nob(&r[dst].modules[n].end, - (erts_aint_t)(RANGE_END(rp))); - ASSERT(!n || RANGE_END(&r[dst].modules[n-1]) < rp->start); - n++; - } - } - - while (i < r[src].n) { - Range* rp = r[src].modules+i; - if (rp->start < RANGE_END(rp)) { - /* Only insert a module that has not been purged. */ - r[dst].modules[n].start = rp->start; - erts_smp_atomic_init_nob(&r[dst].modules[n].end, - (erts_aint_t)(RANGE_END(rp))); - ASSERT(!n || RANGE_END(&r[dst].modules[n-1]) < rp->start); - n++; - } - i++; - } - - if (n == 0 || code > r[dst].modules[n-1].start) { - r[dst].modules[n].start = code; - erts_smp_atomic_init_nob(&r[dst].modules[n].end, - (erts_aint_t)(((byte *)code) + size)); - ASSERT(!n || RANGE_END(&r[dst].modules[n-1]) < code); - n++; - } - - ASSERT(n <= r[src].n+1); - r[dst].n = n; - erts_smp_atomic_set_nob(&r[dst].mid, - (erts_aint_t) (r[dst].modules + n / 2)); - - CHECK(&r[dst]); - CHECK(&r[src]); + ASSERT(r[dst].modules); + write_ptr->start = code; + erts_smp_atomic_init_nob(&(write_ptr->end), + (erts_aint_t)(((byte *)code) + size)); + write_ptr++; } void @@ -240,6 +225,7 @@ erts_lookup_function_info(FunctionInfo* fi, BeamInstr* pc, int full_info) BeamInstr** high; BeamInstr** mid; Range* rp; + BeamCodeHeader* hdr; fi->current = NULL; fi->needed = 5; @@ -248,9 +234,10 @@ erts_lookup_function_info(FunctionInfo* fi, BeamInstr* pc, int full_info) if (rp == 0) { return; } + hdr = (BeamCodeHeader*) rp->start; - low = (BeamInstr **) (rp->start + MI_FUNCTIONS); - high = low + rp->start[MI_NUM_FUNCTIONS]; + low = hdr->functions; + high = low + hdr->num_functions; while (low < high) { mid = low + (high-low) / 2; if (pc < mid[0]) { @@ -258,10 +245,9 @@ erts_lookup_function_info(FunctionInfo* fi, BeamInstr* pc, int full_info) } else if (pc < mid[1]) { fi->current = mid[0]+2; if (full_info) { - BeamInstr** fp = (BeamInstr **) (rp->start + - MI_FUNCTIONS); + BeamInstr** fp = hdr->functions; int idx = mid - fp; - lookup_loc(fi, pc, rp->start, idx); + lookup_loc(fi, pc, hdr, idx); } return; } else { @@ -282,7 +268,7 @@ find_range(BeamInstr* pc) while (low < high) { if (pc < mid->start) { high = mid; - } else if (pc > RANGE_END(mid)) { + } else if (pc >= RANGE_END(mid)) { low = mid + 1; } else { erts_smp_atomic_set_nob(&r[active].mid, (erts_aint_t) mid); @@ -294,39 +280,34 @@ find_range(BeamInstr* pc) } static void -lookup_loc(FunctionInfo* fi, BeamInstr* orig_pc, BeamInstr* modp, int idx) +lookup_loc(FunctionInfo* fi, const BeamInstr* pc, + BeamCodeHeader* code_hdr, int idx) { - Eterm* line = (Eterm *) modp[MI_LINE_TABLE]; - Eterm* low; - Eterm* high; - Eterm* mid; - Eterm pc; + BeamCodeLineTab* lt = code_hdr->line_table; + const BeamInstr** low; + const BeamInstr** high; + const BeamInstr** mid; - if (line == 0) { + if (lt == NULL) { return; } - pc = (Eterm) (BeamInstr) orig_pc; - fi->fname_ptr = (Eterm *) (BeamInstr) line[MI_LINE_FNAME_PTR]; - low = (Eterm *) (BeamInstr) line[MI_LINE_FUNC_TAB+idx]; - high = (Eterm *) (BeamInstr) line[MI_LINE_FUNC_TAB+idx+1]; + fi->fname_ptr = lt->fname_ptr; + low = lt->func_tab[idx]; + high = lt->func_tab[idx+1]; while (high > low) { mid = low + (high-low) / 2; if (pc < mid[0]) { high = mid; } else if (pc < mid[1]) { int file; - int index = mid - (Eterm *) (BeamInstr) line[MI_LINE_FUNC_TAB]; + int index = mid - lt->func_tab[0]; - if (line[MI_LINE_LOC_SIZE] == 2) { - Uint16* loc_table = - (Uint16 *) (BeamInstr) line[MI_LINE_LOC_TAB]; - fi->loc = loc_table[index]; + if (lt->loc_size == 2) { + fi->loc = lt->loc_tab.p2[index]; } else { - Uint32* loc_table = - (Uint32 *) (BeamInstr) line[MI_LINE_LOC_TAB]; - ASSERT(line[MI_LINE_LOC_SIZE] == 4); - fi->loc = loc_table[index]; + ASSERT(lt->loc_size == 4); + fi->loc = lt->loc_tab.p4[index]; } if (fi->loc == LINE_INVALID_LOCATION) { return; diff --git a/erts/emulator/beam/benchmark.c b/erts/emulator/beam/benchmark.c index 8613131176..44f5c760c2 100644 --- a/erts/emulator/beam/benchmark.c +++ b/erts/emulator/beam/benchmark.c @@ -3,16 +3,17 @@ * * Copyright Ericsson AB 2002-2012. All Rights Reserved. * - * The contents of this file are subject to the Erlang Public License, - * Version 1.1, (the "License"); you may not use this file except in - * compliance with the License. You should have received a copy of the - * Erlang Public License along with this software. If not, it can be - * retrieved online at http://www.erlang.org/. - * - * Software distributed under the License is distributed on an "AS IS" - * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See - * the License for the specific language governing rights and limitations - * under the License. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. * * %CopyrightEnd% */ @@ -37,37 +38,9 @@ unsigned long long major_gc; #ifdef BM_TIMERS -#if (defined(__i386__) || defined(__x86_64__)) && USE_PERFCTR - -#include "libperfctr.h" -struct vperfctr *system_clock; -double cpu_khz; -BM_NEW_TIMER(start); - -static double get_hrvtime(void) -{ - unsigned long long ticks; - double milli_seconds; - - ticks = vperfctr_read_tsc(system_clock); - milli_seconds = (double)ticks / cpu_khz; - return milli_seconds; -} - -static void stop_hrvtime(void) -{ - if(system_clock) - { - vperfctr_stop(system_clock); - vperfctr_close(system_clock); - system_clock = NULL; - } -} - -#else /* not perfctr, asuming Solaris */ +/* assuming Solaris */ #include <time.h> BM_TIMER_T system_clock; -#endif unsigned long local_pause_times[MAX_PAUSE_TIME]; unsigned long pause_times[MAX_PAUSE_TIME]; @@ -117,40 +90,6 @@ unsigned long long message_sizes[1000]; void init_benchmarking() { #ifdef BM_TIMERS -#if (defined(__i386__) || defined(__x86_64__)) && USE_PERFCTR - /* pass `--with-perfctr=/path/to/perfctr' when configuring */ - struct perfctr_info info; - struct vperfctr_control control; - int i; - - system_clock = vperfctr_open(); - if (system_clock != NULL) - { - if (vperfctr_info(system_clock,&info) >= 0) - { - cpu_khz = (double)info.cpu_khz; - if (info.cpu_features & PERFCTR_FEATURE_RDTSC) - { - memset(&control,0,sizeof control); - control.cpu_control.tsc_on = 1; - } - } - if (vperfctr_control(system_clock,&control) < 0) - { - vperfctr_close(system_clock); - system_clock = NULL; - } - } - - for (i = 0; i < 1000; i++) - { - BM_START_TIMER(system); - BM_STOP_TIMER(system); - } - - timer_time = system_time / 1000; - start_time = 0; -#else int i; for (i = 0; i < 1000; i++) { @@ -158,7 +97,6 @@ void init_benchmarking() BM_STOP_TIMER(system); } timer_time = system_time / 1000; -#endif for (i = 0; i < MAX_PAUSE_TIME; i++) { local_pause_times[i] = 0; diff --git a/erts/emulator/beam/benchmark.h b/erts/emulator/beam/benchmark.h index 766edaac42..0fb0b93f12 100644 --- a/erts/emulator/beam/benchmark.h +++ b/erts/emulator/beam/benchmark.h @@ -3,16 +3,17 @@ * * Copyright Ericsson AB 2002-2012. All Rights Reserved. * - * The contents of this file are subject to the Erlang Public License, - * Version 1.1, (the "License"); you may not use this file except in - * compliance with the License. You should have received a copy of the - * Erlang Public License along with this software. If not, it can be - * retrieved online at http://www.erlang.org/. - * - * Software distributed under the License is distributed on an "AS IS" - * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See - * the License for the specific language governing rights and limitations - * under the License. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. * * %CopyrightEnd% */ @@ -37,10 +38,7 @@ /* BM_TIMERS keeps track of the time spent in diferent parts of the * system. It only measures accual active time, not time spent in idle - * mode. These timers requires hardware support. For Linux, use the - * package perfctr from user.it.uu.se/~mikpe/linux/perfctr. If this - * package is not specified when configuring the system - * (--with-perfctr=PATH), the Solaris hrtime_t will be used. + * mode. Currently, the Solaris hrtime_t will be used. * To add new timers look below. */ #define BM_TIMERS @@ -142,43 +140,12 @@ extern unsigned long long major_gc; * meassure (send time in shared heap for instance). */ -#if (defined(__i386__) || defined(__x86_64__)) && USE_PERFCTR -#include "libperfctr.h" - -#define BM_TIMER_T double - -extern struct vperfctr *system_clock; -extern double cpu_khz; -extern BM_TIMER_T start_time; - -#define BM_START_TIMER(t) start_time = \ - (BM_TIMER_T)vperfctr_read_tsc(system_clock) / \ - cpu_khz; - -#define BM_STOP_TIMER(t) do { \ - BM_TIMER_T tmp = ((BM_TIMER_T)vperfctr_read_tsc(system_clock) / cpu_khz); \ - tmp -= (start_time + timer_time); \ - t##_time += (tmp > 0 ? tmp : 0); \ -} while(0) - -#define BM_TIME_PRINTER(str,time) do { \ - int min,sec,milli,micro; \ - BM_TIMER_T tmp = (time) * 1000; \ - micro = (uint)(tmp - ((int)(tmp / 1000)) * 1000); \ - tmp /= 1000; \ - milli = (uint)(tmp - ((int)(tmp / 1000)) * 1000); \ - tmp /= 1000; \ - sec = (uint)(tmp - ((int)(tmp / 60)) * 60); \ - min = (uint)tmp / 60; \ - erts_fprintf(file,str": %d:%02d.%03d %03d\n",min,sec,milli,micro); \ -} while(0) - -#else /* !USE_PERFCTR (Assuming Solaris) */ +/* (Assuming Solaris) */ -#define BM_TIMER_T hrtime_t -#define BM_START_TIMER(t) system_clock = sys_gethrtime() +#define BM_TIMER_T ErtsMonotonicTime +#define BM_START_TIMER(t) system_clock = ERTS_MONOTONIC_TO_NSEC(erts_os_monotonic_time()) #define BM_STOP_TIMER(t) do { \ - BM_TIMER_T tmp = (sys_gethrtime() - system_clock) - timer_time; \ + BM_TIMER_T tmp = (ERTS_MONOTONIC_TO_NSEC(erts_os_monotonic_time()) - system_clock) - timer_time; \ t##_time += (tmp > 0 ? tmp : 0); \ } while(0) @@ -196,7 +163,6 @@ extern BM_TIMER_T start_time; } while(0) extern BM_TIMER_T system_clock; -#endif /* USE_PERFCTR */ extern BM_TIMER_T timer_time; extern BM_TIMER_T system_time; diff --git a/erts/emulator/beam/bif.c b/erts/emulator/beam/bif.c index 06a1230ca0..97d690db9f 100644 --- a/erts/emulator/beam/bif.c +++ b/erts/emulator/beam/bif.c @@ -1,18 +1,19 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 1996-2013. All Rights Reserved. + * Copyright Ericsson AB 1996-2014. All Rights Reserved. * - * The contents of this file are subject to the Erlang Public License, - * Version 1.1, (the "License"); you may not use this file except in - * compliance with the License. You should have received a copy of the - * Erlang Public License along with this software. If not, it can be - * retrieved online at http://www.erlang.org/. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Software distributed under the License is distributed on an "AS IS" - * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See - * the License for the specific language governing rights and limitations - * under the License. + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. * * %CopyrightEnd% */ @@ -28,7 +29,9 @@ #include "global.h" #include "erl_process.h" #include "error.h" +#define ERL_WANT_HIPE_BIF_WRAPPER__ #include "bif.h" +#undef ERL_WANT_HIPE_BIF_WRAPPER__ #include "big.h" #include "dist.h" #include "erl_version.h" @@ -40,16 +43,27 @@ #define ERTS_PTAB_WANT_BIF_IMPL__ #include "erl_ptab.h" #include "erl_bits.h" +#include "erl_bif_unique.h" +#include "erl_msacc.h" -static Export* flush_monitor_message_trap = NULL; +Export *erts_await_result; +static Export* flush_monitor_messages_trap = NULL; static Export* set_cpu_topology_trap = NULL; static Export* await_proc_exit_trap = NULL; static Export* await_port_send_result_trap = NULL; Export* erts_format_cpu_topology_trap = NULL; +static Export dsend_continue_trap_export; +Export *erts_convert_time_unit_trap = NULL; + +static Export *await_msacc_mod_trap = NULL; +static erts_smp_atomic32_t msacc; static Export *await_sched_wall_time_mod_trap; static erts_smp_atomic32_t sched_wall_time; +static erts_smp_mtx_t ports_snapshot_mtx; +erts_smp_atomic_t erts_dead_ports_ptr; /* To store dying ports during snapshot */ + #define DECL_AM(S) Eterm AM_ ## S = am_atom_put(#S, sizeof(#S) - 1) /* @@ -62,7 +76,7 @@ BIF_RETTYPE spawn_3(BIF_ALIST_3) ErlSpawnOpts so; Eterm pid; - so.flags = 0; + so.flags = erts_default_spo_flags; pid = erl_create_process(BIF_P, BIF_ARG_1, BIF_ARG_2, BIF_ARG_3, &so); if (is_non_value(pid)) { BIF_ERROR(BIF_P, so.error_code); @@ -391,7 +405,7 @@ remote_demonitor(Process *c_p, DistEntry *dep, Eterm ref, Eterm to) return res; } -static int demonitor(Process *c_p, Eterm ref) +static int demonitor(Process *c_p, Eterm ref, Eterm *multip) { ErtsMonitor *mon = NULL; /* The monitor entry to delete */ Process *rp; /* Local target process */ @@ -415,65 +429,73 @@ static int demonitor(Process *c_p, Eterm ref) goto done; } - if (mon->type != MON_ORIGIN) { - res = ERTS_DEMONITOR_BADARG; - goto done; - } - to = mon->pid; - - if (is_atom(to)) { - /* Monitoring a name at node to */ - ASSERT(is_node_name_atom(to)); - dep = erts_sysname_to_connected_dist_entry(to); - ASSERT(dep != erts_this_dist_entry); - if (dep) - deref_de = 1; - } else { - ASSERT(is_pid(to)); - dep = pid_dist_entry(to); - } - if (dep != erts_this_dist_entry) { - res = remote_demonitor(c_p, dep, ref, to); - /* remote_demonitor() unlocks link lock on c_p */ - unlock_link = 0; - } - else { /* Local monitor */ - if (deref_de) { - deref_de = 0; - erts_deref_dist_entry(dep); + switch (mon->type) { + case MON_TIME_OFFSET: + *multip = am_true; + erts_demonitor_time_offset(ref); + res = ERTS_DEMONITOR_TRUE; + break; + case MON_ORIGIN: + to = mon->pid; + *multip = am_false; + if (is_atom(to)) { + /* Monitoring a name at node to */ + ASSERT(is_node_name_atom(to)); + dep = erts_sysname_to_connected_dist_entry(to); + ASSERT(dep != erts_this_dist_entry); + if (dep) + deref_de = 1; + } else { + ASSERT(is_pid(to)); + dep = pid_dist_entry(to); } - dep = NULL; - rp = erts_pid2proc_opt(c_p, - ERTS_PROC_LOCK_MAIN|ERTS_PROC_LOCK_LINK, - to, - ERTS_PROC_LOCK_LINK, - ERTS_P2P_FLG_ALLOW_OTHER_X); - mon = erts_remove_monitor(&ERTS_P_MONITORS(c_p), ref); + if (dep != erts_this_dist_entry) { + res = remote_demonitor(c_p, dep, ref, to); + /* remote_demonitor() unlocks link lock on c_p */ + unlock_link = 0; + } + else { /* Local monitor */ + if (deref_de) { + deref_de = 0; + erts_deref_dist_entry(dep); + } + dep = NULL; + rp = erts_pid2proc_opt(c_p, + ERTS_PROC_LOCK_MAIN|ERTS_PROC_LOCK_LINK, + to, + ERTS_PROC_LOCK_LINK, + ERTS_P2P_FLG_ALLOW_OTHER_X); + mon = erts_remove_monitor(&ERTS_P_MONITORS(c_p), ref); #ifndef ERTS_SMP - ASSERT(mon); + ASSERT(mon); #else - if (!mon) - res = ERTS_DEMONITOR_FALSE; - else + if (!mon) + res = ERTS_DEMONITOR_FALSE; + else #endif - { - res = ERTS_DEMONITOR_TRUE; - erts_destroy_monitor(mon); - } - if (rp) { - ErtsMonitor *rmon; - rmon = erts_remove_monitor(&ERTS_P_MONITORS(rp), ref); - if (rp != c_p) - erts_smp_proc_unlock(rp, ERTS_PROC_LOCK_LINK); - if (rmon != NULL) - erts_destroy_monitor(rmon); - } - else { - ERTS_SMP_ASSERT_IS_NOT_EXITING(c_p); - } + { + res = ERTS_DEMONITOR_TRUE; + erts_destroy_monitor(mon); + } + if (rp) { + ErtsMonitor *rmon; + rmon = erts_remove_monitor(&ERTS_P_MONITORS(rp), ref); + if (rp != c_p) + erts_smp_proc_unlock(rp, ERTS_PROC_LOCK_LINK); + if (rmon != NULL) + erts_destroy_monitor(rmon); + } + else { + ERTS_SMP_ASSERT_IS_NOT_EXITING(c_p); + } + } + break; + default: + res = ERTS_DEMONITOR_BADARG; + *multip = am_false; + break; } - done: if (unlock_link) @@ -490,7 +512,8 @@ static int demonitor(Process *c_p, Eterm ref) BIF_RETTYPE demonitor_1(BIF_ALIST_1) { - switch (demonitor(BIF_P, BIF_ARG_1)) { + Eterm multi; + switch (demonitor(BIF_P, BIF_ARG_1, &multi)) { case ERTS_DEMONITOR_FALSE: case ERTS_DEMONITOR_TRUE: BIF_RET(am_true); @@ -508,6 +531,7 @@ BIF_RETTYPE demonitor_1(BIF_ALIST_1) BIF_RETTYPE demonitor_2(BIF_ALIST_2) { Eterm res = am_true; + Eterm multi = am_false; int info = 0; int flush = 0; Eterm list = BIF_ARG_2; @@ -530,13 +554,18 @@ BIF_RETTYPE demonitor_2(BIF_ALIST_2) if (is_not_nil(list)) goto badarg; - switch (demonitor(BIF_P, BIF_ARG_1)) { + switch (demonitor(BIF_P, BIF_ARG_1, &multi)) { case ERTS_DEMONITOR_FALSE: if (info) res = am_false; - if (flush) - BIF_TRAP2(flush_monitor_message_trap, BIF_P, BIF_ARG_1, res); + if (flush) { + flush_messages: + BIF_TRAP3(flush_monitor_messages_trap, BIF_P, + BIF_ARG_1, multi, res); + } case ERTS_DEMONITOR_TRUE: + if (multi == am_true && flush) + goto flush_messages; BIF_RET(res); case ERTS_DEMONITOR_YIELD_TRUE: ERTS_BIF_YIELD_RETURN(BIF_P, am_true); @@ -564,7 +593,7 @@ erts_queue_monitor_message(Process *p, Eterm reason_copy, ref_copy, item_copy; Uint reason_size, ref_size, item_size, heap_size; ErlOffHeap *ohp; - ErlHeapFragment *bp; + ErtsMessage *msgp; reason_size = IS_CONST(reason) ? 0 : size_object(reason); item_size = IS_CONST(item) ? 0 : size_object(item); @@ -572,11 +601,8 @@ erts_queue_monitor_message(Process *p, heap_size = 6+reason_size+ref_size+item_size; - hp = erts_alloc_message_heap(heap_size, - &bp, - &ohp, - p, - p_locksp); + msgp = erts_alloc_message_heap(p, p_locksp, heap_size, + &hp, &ohp); reason_copy = (IS_CONST(reason) ? reason @@ -587,22 +613,16 @@ erts_queue_monitor_message(Process *p, ref_copy = copy_struct(ref, ref_size, &hp, ohp); tup = TUPLE5(hp, am_DOWN, ref_copy, type, item_copy, reason_copy); - erts_queue_message(p, p_locksp, bp, tup, NIL -#ifdef USE_VM_PROBES - , NIL -#endif - ); + erts_queue_message(p, p_locksp, msgp, tup, NIL); } static BIF_RETTYPE -local_pid_monitor(Process *p, Eterm target) +local_pid_monitor(Process *p, Eterm target, Eterm mon_ref, int boolean) { BIF_RETTYPE ret; - Eterm mon_ref; Process *rp; ErtsProcLocks p_locks = ERTS_PROC_LOCK_MAIN|ERTS_PROC_LOCK_LINK; - mon_ref = erts_make_ref(p); ERTS_BIF_PREP_RET(ret, mon_ref); if (target == p->common.id) { return ret; @@ -615,12 +635,18 @@ local_pid_monitor(Process *p, Eterm target) if (!rp) { erts_smp_proc_unlock(p, ERTS_PROC_LOCK_LINK); p_locks &= ~ERTS_PROC_LOCK_LINK; - erts_queue_monitor_message(p, &p_locks, - mon_ref, am_process, target, am_noproc); + if (boolean) + ret = am_false; + else + erts_queue_monitor_message(p, &p_locks, + mon_ref, am_process, target, am_noproc); } else { ASSERT(rp != p); + if (boolean) + ret = am_true; + erts_add_monitor(&ERTS_P_MONITORS(p), MON_ORIGIN, mon_ref, target, NIL); erts_add_monitor(&ERTS_P_MONITORS(rp), MON_TARGET, mon_ref, p->common.id, NIL); @@ -744,13 +770,28 @@ BIF_RETTYPE monitor_2(BIF_ALIST_2) int deref_de = 0; /* Only process monitors are implemented */ - if (BIF_ARG_1 != am_process) { + switch (BIF_ARG_1) { + case am_time_offset: { + Eterm ref; + if (BIF_ARG_2 != am_clock_service) + goto error; + ref = erts_make_ref(BIF_P); + erts_smp_proc_lock(BIF_P, ERTS_PROC_LOCK_LINK); + erts_add_monitor(&ERTS_P_MONITORS(BIF_P), MON_TIME_OFFSET, + ref, am_clock_service, NIL); + erts_smp_proc_unlock(BIF_P, ERTS_PROC_LOCK_LINK); + erts_monitor_time_offset(BIF_P->common.id, ref); + BIF_RET(ref); + } + case am_process: + break; + default: goto error; } if (is_internal_pid(target)) { local_pid: - ret = local_pid_monitor(BIF_P, target); + ret = local_pid_monitor(BIF_P, target, erts_make_ref(BIF_P), 0); } else if (is_external_pid(target)) { dep = external_pid_dist_entry(target); if (dep == erts_this_dist_entry) @@ -793,7 +834,6 @@ BIF_RETTYPE monitor_2(BIF_ALIST_2) return ret; } - /**********************************************************************/ /* this is a combination of the spawn and link BIFs */ @@ -802,7 +842,7 @@ BIF_RETTYPE spawn_link_3(BIF_ALIST_3) ErlSpawnOpts so; Eterm pid; - so.flags = SPO_LINK; + so.flags = erts_default_spo_flags|SPO_LINK; pid = erl_create_process(BIF_P, BIF_ARG_1, BIF_ARG_2, BIF_ARG_3, &so); if (is_non_value(pid)) { BIF_ERROR(BIF_P, so.error_code); @@ -839,7 +879,7 @@ BIF_RETTYPE spawn_opt_1(BIF_ALIST_1) /* * Store default values for options. */ - so.flags = SPO_USE_ARGS; + so.flags = erts_default_spo_flags|SPO_USE_ARGS; so.min_heap_size = H_MIN_SIZE; so.min_vheap_size = BIN_VH_MIN_SIZE; so.priority = PRIORITY_NORMAL; @@ -874,6 +914,22 @@ BIF_RETTYPE spawn_opt_1(BIF_ALIST_1) so.priority = PRIORITY_LOW; else goto error; + } else if (arg == am_message_queue_data) { + switch (val) { + case am_mixed: + so.flags &= ~(SPO_OFF_HEAP_MSGQ|SPO_ON_HEAP_MSGQ); + break; + case am_on_heap: + so.flags &= ~SPO_OFF_HEAP_MSGQ; + so.flags |= SPO_ON_HEAP_MSGQ; + break; + case am_off_heap: + so.flags &= ~SPO_ON_HEAP_MSGQ; + so.flags |= SPO_OFF_HEAP_MSGQ; + break; + default: + goto error; + } } else if (arg == am_min_heap_size && is_small(val)) { Sint min_heap_size = signed_val(val); if (min_heap_size < 0) { @@ -1511,7 +1567,7 @@ static BIF_RETTYPE process_flag_aux(Process *BIF_P, scb->n = 0; } - scb = ERTS_PROC_SET_SAVED_CALLS_BUF(rp, ERTS_PROC_LOCK_MAIN, scb); + scb = ERTS_PROC_SET_SAVED_CALLS_BUF(rp, scb); if (!scb) old_value = make_small(0); @@ -1539,9 +1595,7 @@ BIF_RETTYPE process_flag_2(BIF_ALIST_2) if (is_not_atom(BIF_ARG_2)) { goto error; } - old_value = erts_proc_set_error_handler(BIF_P, - ERTS_PROC_LOCK_MAIN, - BIF_ARG_2); + old_value = erts_proc_set_error_handler(BIF_P, BIF_ARG_2); BIF_RET(old_value); } else if (BIF_ARG_1 == am_priority) { @@ -1652,6 +1706,12 @@ BIF_RETTYPE process_flag_2(BIF_ALIST_2) } BIF_RET(old_value); } + else if (BIF_ARG_1 == am_message_queue_data) { + old_value = erts_change_message_queue_management(BIF_P, BIF_ARG_2); + if (is_non_value(old_value)) + goto error; + BIF_RET(old_value); + } else if (BIF_ARG_1 == am_sensitive) { Uint is_sensitive; if (BIF_ARG_2 == am_true) { @@ -1777,6 +1837,8 @@ BIF_RETTYPE whereis_1(BIF_ALIST_1) * erlang:'!'/2 */ +HIPE_WRAPPER_BIF_DISABLE_GC(ebif_bang, 2) + BIF_RETTYPE ebif_bang_2(BIF_ALIST_2) { @@ -1795,34 +1857,36 @@ ebif_bang_2(BIF_ALIST_2) #define SEND_USER_ERROR (-5) #define SEND_INTERNAL_ERROR (-6) #define SEND_AWAIT_RESULT (-7) +#define SEND_YIELD_CONTINUE (-8) + -Sint do_send(Process *p, Eterm to, Eterm msg, int suspend, Eterm *refp); +Sint do_send(Process *p, Eterm to, Eterm msg, Eterm *refp, ErtsSendContext*); static Sint remote_send(Process *p, DistEntry *dep, - Eterm to, Eterm full_to, Eterm msg, int suspend) + Eterm to, Eterm full_to, Eterm msg, + ErtsSendContext* ctx) { Sint res; int code; - ErtsDSigData dsd; ASSERT(is_atom(to) || is_external_pid(to)); - code = erts_dsig_prepare(&dsd, dep, p, ERTS_DSP_NO_LOCK, !suspend); + code = erts_dsig_prepare(&ctx->dsd, dep, p, ERTS_DSP_NO_LOCK, !ctx->suspend); switch (code) { case ERTS_DSIG_PREP_NOT_ALIVE: case ERTS_DSIG_PREP_NOT_CONNECTED: res = SEND_TRAP; break; case ERTS_DSIG_PREP_WOULD_SUSPEND: - ASSERT(!suspend); + ASSERT(!ctx->suspend); res = SEND_YIELD; break; case ERTS_DSIG_PREP_CONNECTED: { if (is_atom(to)) - code = erts_dsig_send_reg_msg(&dsd, to, msg); + code = erts_dsig_send_reg_msg(to, msg, ctx); else - code = erts_dsig_send_msg(&dsd, to, msg); + code = erts_dsig_send_msg(to, msg, ctx); /* * Note that reductions have been bumped on calling * process by erts_dsig_send_reg_msg() or @@ -1830,6 +1894,8 @@ static Sint remote_send(Process *p, DistEntry *dep, */ if (code == ERTS_DSIG_SEND_YIELD) res = SEND_YIELD_RETURN; + else if (code == ERTS_DSIG_SEND_CONTINUE) + res = SEND_YIELD_CONTINUE; else res = 0; break; @@ -1850,7 +1916,8 @@ static Sint remote_send(Process *p, DistEntry *dep, } Sint -do_send(Process *p, Eterm to, Eterm msg, int suspend, Eterm *refp) { +do_send(Process *p, Eterm to, Eterm msg, Eterm *refp, ErtsSendContext* ctx) +{ Eterm portid; Port *pt; Process* rp; @@ -1881,13 +1948,18 @@ do_send(Process *p, Eterm to, Eterm msg, int suspend, Eterm *refp) { erts_send_error_to_logger(p->group_leader, dsbufp); return 0; } - return remote_send(p, dep, to, to, msg, suspend); + return remote_send(p, dep, to, to, msg, ctx); } else if (is_atom(to)) { Eterm id = erts_whereis_name_to_id(p, to); - rp = erts_proc_lookup(id); - if (rp) + rp = erts_proc_lookup_raw(id); + if (rp) { + if (IS_TRACED(p)) + trace_send(p, to, msg); + if (ERTS_PROC_GET_SAVED_CALLS_BUF(p)) + save_calls(p, &exp_send); goto send_message; + } pt = erts_port_lookup(id, (erts_port_synchronous_ops @@ -1931,7 +2003,7 @@ do_send(Process *p, Eterm to, Eterm msg, int suspend, Eterm *refp) { ret_val = 0; if (pt) { - int ps_flags = suspend ? 0 : ERTS_PORT_SIG_FLG_NOSUSPEND; + int ps_flags = ctx->suspend ? 0 : ERTS_PORT_SIG_FLG_NOSUSPEND; *refp = NIL; switch (erts_port_command(p, ps_flags, pt, msg, refp)) { @@ -1940,12 +2012,12 @@ do_send(Process *p, Eterm to, Eterm msg, int suspend, Eterm *refp) { return SEND_USER_ERROR; case ERTS_PORT_OP_BUSY: /* Nothing has been sent */ - if (suspend) + if (ctx->suspend) erts_suspend(p, ERTS_PROC_LOCK_MAIN, pt); return SEND_YIELD; case ERTS_PORT_OP_BUSY_SCHEDULED: /* Message was sent */ - if (suspend) { + if (ctx->suspend) { erts_suspend(p, ERTS_PROC_LOCK_MAIN, pt); ret_val = SEND_YIELD_RETURN; break; @@ -1972,11 +2044,7 @@ do_send(Process *p, Eterm to, Eterm msg, int suspend, Eterm *refp) { if (ERTS_PROC_GET_SAVED_CALLS_BUF(p)) save_calls(p, &exp_send); - if (SEQ_TRACE_TOKEN(p) != NIL -#ifdef USE_VM_PROBES - && SEQ_TRACE_TOKEN(p) != am_have_dt_utag -#endif - ) { + if (have_seqtrace(SEQ_TRACE_TOKEN(p))) { seq_trace_update_send(p); seq_trace_output(SEQ_TRACE_TOKEN(p), msg, SEQ_TRACE_SEND, portid, p); @@ -2025,9 +2093,14 @@ do_send(Process *p, Eterm to, Eterm msg, int suspend, Eterm *refp) { return 0; } - ret = remote_send(p, dep, tp[1], to, msg, suspend); - if (dep) - erts_deref_dist_entry(dep); + ret = remote_send(p, dep, tp[1], to, msg, ctx); + if (ret != SEND_YIELD_CONTINUE) { + if (dep) { + erts_deref_dist_entry(dep); + } + } else { + ctx->dep_to_deref = dep; + } return ret; } else { if (IS_TRACED(p)) /* XXX Is this really neccessary ??? */ @@ -2058,9 +2131,11 @@ do_send(Process *p, Eterm to, Eterm msg, int suspend, Eterm *refp) { } } +HIPE_WRAPPER_BIF_DISABLE_GC(send, 3) BIF_RETTYPE send_3(BIF_ALIST_3) { + BIF_RETTYPE retval; Eterm ref; Process *p = BIF_P; Eterm to = BIF_ARG_1; @@ -2068,34 +2143,51 @@ BIF_RETTYPE send_3(BIF_ALIST_3) Eterm opts = BIF_ARG_3; int connect = !0; - int suspend = !0; Eterm l = opts; Sint result; - + + DeclareTypedTmpHeap(ErtsSendContext, ctx, BIF_P); + + ERTS_MSACC_PUSH_STATE_M_X(); + + UseTmpHeap(sizeof(ErtsSendContext)/sizeof(Eterm), BIF_P); + + ctx->suspend = !0; + ctx->dep_to_deref = NULL; + ctx->return_term = am_ok; + ctx->dss.reds = (Sint) (ERTS_BIF_REDS_LEFT(p) * TERM_TO_BINARY_LOOP_FACTOR); + ctx->dss.phase = ERTS_DSIG_SEND_PHASE_INIT; + while (is_list(l)) { if (CAR(list_val(l)) == am_noconnect) { connect = 0; } else if (CAR(list_val(l)) == am_nosuspend) { - suspend = 0; + ctx->suspend = 0; } else { - BIF_ERROR(p, BADARG); + ERTS_BIF_PREP_ERROR(retval, p, BADARG); + goto done; } l = CDR(list_val(l)); } if(!is_nil(l)) { - BIF_ERROR(p, BADARG); + ERTS_BIF_PREP_ERROR(retval, p, BADARG); + goto done; } #ifdef DEBUG ref = NIL; #endif - result = do_send(p, to, msg, suspend, &ref); + ERTS_MSACC_SET_STATE_CACHED_M_X(ERTS_MSACC_STATE_SEND); + result = do_send(p, to, msg, &ref, ctx); + ERTS_MSACC_POP_STATE_M_X(); + if (result > 0) { ERTS_VBUMP_REDS(p, result); if (ERTS_IS_PROC_OUT_OF_REDS(p)) goto yield_return; - BIF_RET(am_ok); + ERTS_BIF_PREP_RET(retval, am_ok); + goto done; } switch (result) { @@ -2103,68 +2195,129 @@ BIF_RETTYPE send_3(BIF_ALIST_3) /* May need to yield even though we do not bump reds here... */ if (ERTS_IS_PROC_OUT_OF_REDS(p)) goto yield_return; - BIF_RET(am_ok); + ERTS_BIF_PREP_RET(retval, am_ok); break; case SEND_TRAP: if (connect) { - BIF_TRAP3(dsend3_trap, p, to, msg, opts); + ERTS_BIF_PREP_TRAP3(retval, dsend3_trap, p, to, msg, opts); } else { - BIF_RET(am_noconnect); + ERTS_BIF_PREP_RET(retval, am_noconnect); } break; case SEND_YIELD: - if (suspend) { - ERTS_BIF_YIELD3(bif_export[BIF_send_3], p, to, msg, opts); + if (ctx->suspend) { + ERTS_BIF_PREP_YIELD3(retval, + bif_export[BIF_send_3], p, to, msg, opts); } else { - BIF_RET(am_nosuspend); + ERTS_BIF_PREP_RET(retval, am_nosuspend); } break; case SEND_YIELD_RETURN: - if (!suspend) - BIF_RET(am_nosuspend); + if (!ctx->suspend) { + ERTS_BIF_PREP_RET(retval, am_nosuspend); + break; + } yield_return: - ERTS_BIF_YIELD_RETURN(p, am_ok); + ERTS_BIF_PREP_YIELD_RETURN(retval, p, am_ok); + break; case SEND_AWAIT_RESULT: ASSERT(is_internal_ref(ref)); - BIF_TRAP3(await_port_send_result_trap, p, ref, am_nosuspend, am_ok); + ERTS_BIF_PREP_TRAP3(retval, await_port_send_result_trap, p, ref, am_nosuspend, am_ok); + break; case SEND_BADARG: - BIF_ERROR(p, BADARG); + ERTS_BIF_PREP_ERROR(retval, p, BADARG); break; case SEND_USER_ERROR: - BIF_ERROR(p, EXC_ERROR); + ERTS_BIF_PREP_ERROR(retval, p, EXC_ERROR); break; case SEND_INTERNAL_ERROR: - BIF_ERROR(p, EXC_INTERNAL_ERROR); + ERTS_BIF_PREP_ERROR(retval, p, EXC_INTERNAL_ERROR); + break; + case SEND_YIELD_CONTINUE: + BUMP_ALL_REDS(p); + erts_set_gc_state(p, 0); + ERTS_BIF_PREP_TRAP1(retval, &dsend_continue_trap_export, p, + erts_dsend_export_trap_context(p, ctx)); break; default: - ASSERT(! "Illegal send result"); + erts_exit(ERTS_ABORT_EXIT, "send_3 invalid result %d\n", (int)result); break; } - ASSERT(! "Can not arrive here"); - BIF_ERROR(p, BADARG); + +done: + UnUseTmpHeap(sizeof(ErtsSendContext)/sizeof(Eterm), BIF_P); + return retval; } +HIPE_WRAPPER_BIF_DISABLE_GC(send, 2) + BIF_RETTYPE send_2(BIF_ALIST_2) { return erl_send(BIF_P, BIF_ARG_1, BIF_ARG_2); } +static BIF_RETTYPE dsend_continue_trap_1(BIF_ALIST_1) +{ + Binary* bin = ((ProcBin*) binary_val(BIF_ARG_1))->val; + ErtsSendContext* ctx = (ErtsSendContext*) ERTS_MAGIC_BIN_DATA(bin); + Sint initial_reds = (Sint) (ERTS_BIF_REDS_LEFT(BIF_P) * TERM_TO_BINARY_LOOP_FACTOR); + int result; + + ASSERT(ERTS_MAGIC_BIN_DESTRUCTOR(bin) == erts_dsend_context_dtor); + + ctx->dss.reds = initial_reds; + result = erts_dsig_send(&ctx->dsd, &ctx->dss); + + switch (result) { + case ERTS_DSIG_SEND_OK: + erts_set_gc_state(BIF_P, 1); + BIF_RET(ctx->return_term); + break; + case ERTS_DSIG_SEND_YIELD: /*SEND_YIELD_RETURN*/ + erts_set_gc_state(BIF_P, 1); + if (!ctx->suspend) + BIF_RET(am_nosuspend); + ERTS_BIF_YIELD_RETURN(BIF_P, ctx->return_term); + + case ERTS_DSIG_SEND_CONTINUE: { /*SEND_YIELD_CONTINUE*/ + BUMP_ALL_REDS(BIF_P); + BIF_TRAP1(&dsend_continue_trap_export, BIF_P, BIF_ARG_1); + } + default: + erts_exit(ERTS_ABORT_EXIT, "dsend_continue_trap invalid result %d\n", (int)result); + break; + } + ASSERT(! "Can not arrive here"); + BIF_ERROR(BIF_P, BADARG); +} + Eterm erl_send(Process *p, Eterm to, Eterm msg) { + Eterm retval; Eterm ref; Sint result; - + DeclareTypedTmpHeap(ErtsSendContext, ctx, p); + ERTS_MSACC_PUSH_AND_SET_STATE_M_X(ERTS_MSACC_STATE_SEND); + UseTmpHeap(sizeof(ErtsSendContext)/sizeof(Eterm), p); #ifdef DEBUG ref = NIL; #endif + ctx->suspend = !0; + ctx->dep_to_deref = NULL; + ctx->return_term = msg; + ctx->dss.reds = (Sint) (ERTS_BIF_REDS_LEFT(p) * TERM_TO_BINARY_LOOP_FACTOR); + ctx->dss.phase = ERTS_DSIG_SEND_PHASE_INIT; + + result = do_send(p, to, msg, &ref, ctx); + + ERTS_MSACC_POP_STATE_M_X(); - result = do_send(p, to, msg, !0, &ref); - if (result > 0) { ERTS_VBUMP_REDS(p, result); if (ERTS_IS_PROC_OUT_OF_REDS(p)) goto yield_return; - BIF_RET(msg); + ERTS_BIF_PREP_RET(retval, msg); + goto done; } switch (result) { @@ -2172,35 +2325,46 @@ Eterm erl_send(Process *p, Eterm to, Eterm msg) /* May need to yield even though we do not bump reds here... */ if (ERTS_IS_PROC_OUT_OF_REDS(p)) goto yield_return; - BIF_RET(msg); + ERTS_BIF_PREP_RET(retval, msg); break; case SEND_TRAP: - BIF_TRAP2(dsend2_trap, p, to, msg); + ERTS_BIF_PREP_TRAP2(retval, dsend2_trap, p, to, msg); break; case SEND_YIELD: - ERTS_BIF_YIELD2(bif_export[BIF_send_2], p, to, msg); + ERTS_BIF_PREP_YIELD2(retval, bif_export[BIF_send_2], p, to, msg); break; case SEND_YIELD_RETURN: yield_return: - ERTS_BIF_YIELD_RETURN(p, msg); + ERTS_BIF_PREP_YIELD_RETURN(retval, p, msg); + break; case SEND_AWAIT_RESULT: ASSERT(is_internal_ref(ref)); - BIF_TRAP3(await_port_send_result_trap, p, ref, msg, msg); + ERTS_BIF_PREP_TRAP3(retval, + await_port_send_result_trap, p, ref, msg, msg); + break; case SEND_BADARG: - BIF_ERROR(p, BADARG); + ERTS_BIF_PREP_ERROR(retval, p, BADARG); break; case SEND_USER_ERROR: - BIF_ERROR(p, EXC_ERROR); + ERTS_BIF_PREP_ERROR(retval, p, EXC_ERROR); break; case SEND_INTERNAL_ERROR: - BIF_ERROR(p, EXC_INTERNAL_ERROR); + ERTS_BIF_PREP_ERROR(retval, p, EXC_INTERNAL_ERROR); + break; + case SEND_YIELD_CONTINUE: + BUMP_ALL_REDS(p); + erts_set_gc_state(p, 0); + ERTS_BIF_PREP_TRAP1(retval, &dsend_continue_trap_export, p, + erts_dsend_export_trap_context(p, ctx)); break; default: - ASSERT(! "Illegal send result"); + erts_exit(ERTS_ABORT_EXIT, "invalid send result %d\n", (int)result); break; } - ASSERT(! "Can not arrive here"); - BIF_ERROR(p, BADARG); + +done: + UnUseTmpHeap(sizeof(ErtsSendContext)/sizeof(Eterm), p); + return retval; } /**********************************************************************/ @@ -2671,7 +2835,7 @@ BIF_RETTYPE list_to_atom_1(BIF_ALIST_1) { Eterm res; char *buf = (char *) erts_alloc(ERTS_ALC_T_TMP, MAX_ATOM_CHARACTERS); - int i = intlist_to_buf(BIF_ARG_1, buf, MAX_ATOM_CHARACTERS); + Sint i = intlist_to_buf(BIF_ARG_1, buf, MAX_ATOM_CHARACTERS); if (i < 0) { erts_free(ERTS_ALC_T_TMP, (void *) buf); @@ -2691,7 +2855,7 @@ BIF_RETTYPE list_to_atom_1(BIF_ALIST_1) BIF_RETTYPE list_to_existing_atom_1(BIF_ALIST_1) { - int i; + Sint i; char *buf = (char *) erts_alloc(ERTS_ALC_T_TMP, MAX_ATOM_CHARACTERS); if ((i = intlist_to_buf(BIF_ARG_1, buf, MAX_ATOM_CHARACTERS)) < 0) { @@ -2751,170 +2915,20 @@ BIF_RETTYPE integer_to_list_1(BIF_ALIST_1) /**********************************************************************/ -/* convert a list of ascii ascii integer value to an integer */ - - -#define LTI_BAD_STRUCTURE 0 -#define LTI_NO_INTEGER 1 -#define LTI_SOME_INTEGER 2 -#define LTI_ALL_INTEGER 3 - -static int do_list_to_integer(Process *p, Eterm orig_list, - Eterm *integer, Eterm *rest) -{ - Sint i = 0; - int skip = 0; - int neg = 0; - int n = 0; - int m; - int lg2; - Eterm res; - Eterm* hp; - Eterm *hp_end; - Eterm lst = orig_list; - Eterm tail = lst; - int error_res = LTI_BAD_STRUCTURE; - - if (is_nil(lst)) { - error_res = LTI_NO_INTEGER; - error: - *rest = tail; - *integer = make_small(0); - return error_res; - } - if (is_not_list(lst)) - goto error; - - /* if first char is a '-' then it is a negative integer */ - if (CAR(list_val(lst)) == make_small('-')) { - neg = 1; - skip = 1; - lst = CDR(list_val(lst)); - if (is_not_list(lst)) { - tail = lst; - error_res = LTI_NO_INTEGER; - goto error; - } - } else if (CAR(list_val(lst)) == make_small('+')) { - /* ignore plus */ - skip = 1; - lst = CDR(list_val(lst)); - if (is_not_list(lst)) { - tail = lst; - error_res = LTI_NO_INTEGER; - goto error; - } - } - - /* Calculate size and do type check */ - - while(1) { - if (is_not_small(CAR(list_val(lst)))) { - break; - } - if (unsigned_val(CAR(list_val(lst))) < '0' || - unsigned_val(CAR(list_val(lst))) > '9') { - break; - } - i = i * 10; - i = i + unsigned_val(CAR(list_val(lst))) - '0'; - n++; - lst = CDR(list_val(lst)); - if (is_nil(lst)) { - break; - } - if (is_not_list(lst)) { - break; - } - } - - tail = lst; - if (!n) { - error_res = LTI_NO_INTEGER; - goto error; - } - - - /* If n <= 8 then we know it's a small int - ** since 2^27 = 134217728. If n > 8 then we must - ** construct a bignum and let that routine do the checking - */ - - if (n <= SMALL_DIGITS) { /* It must be small */ - if (neg) i = -i; - res = make_small(i); - } else { - lg2 = (n+1)*230/69+1; - m = (lg2+D_EXP-1)/D_EXP; /* number of digits */ - m = BIG_NEED_SIZE(m); /* number of words + thing */ - - hp = HAlloc(p, m); - hp_end = hp + m; - - lst = orig_list; - if (skip) - lst = CDR(list_val(lst)); - - /* load first digits (at least one digit) */ - if ((i = (n % D_DECIMAL_EXP)) == 0) - i = D_DECIMAL_EXP; - n -= i; - m = 0; - while(i--) { - m = 10*m + (unsigned_val(CAR(list_val(lst))) - '0'); - lst = CDR(list_val(lst)); - } - res = small_to_big(m, hp); /* load first digits */ - - while(n) { - i = D_DECIMAL_EXP; - n -= D_DECIMAL_EXP; - m = 0; - while(i--) { - m = 10*m + (unsigned_val(CAR(list_val(lst))) - '0'); - lst = CDR(list_val(lst)); - } - if (is_small(res)) - res = small_to_big(signed_val(res), hp); - res = big_times_small(res, D_DECIMAL_BASE, hp); - if (is_small(res)) - res = small_to_big(signed_val(res), hp); - res = big_plus_small(res, m, hp); - } - - if (is_big(res)) /* check if small */ - res = big_plus_small(res, 0, hp); /* includes conversion to small */ - - if (neg) { - if (is_small(res)) - res = make_small(-signed_val(res)); - else { - Uint *big = big_val(res); /* point to thing */ - *big = bignum_header_neg(*big); - } - } +/* + * Converts a list of ascii base10 digits to an integer fully or partially. + * Returns result and the remaining tail. + * On error returns: {error,not_a_list}, or {error, no_integer} + */ - if (is_big(res)) { - hp += (big_arity(res)+1); - } - HRelease(p,hp_end,hp); - } - *integer = res; - *rest = tail; - if (tail != NIL) { - return LTI_SOME_INTEGER; - } - return LTI_ALL_INTEGER; -} BIF_RETTYPE string_to_integer_1(BIF_ALIST_1) { Eterm res; Eterm tail; Eterm *hp; /* must be a list */ - switch (do_list_to_integer(BIF_P,BIF_ARG_1,&res,&tail)) { - /* HAlloc after do_list_to_integer as it - might HAlloc itself (bignum) */ + switch (erts_list_to_integer(BIF_P, BIF_ARG_1, 10, &res, &tail)) { + /* HAlloc after erts_list_to_integer as it might HAlloc itself (bignum) */ case LTI_BAD_STRUCTURE: hp = HAlloc(BIF_P,3); BIF_RET(TUPLE2(hp, am_error, am_not_a_list)); @@ -2929,13 +2943,14 @@ BIF_RETTYPE string_to_integer_1(BIF_ALIST_1) BIF_RETTYPE list_to_integer_1(BIF_ALIST_1) { - /* Using do_list_to_integer is about twice as fast as using + /* Using erts_list_to_integer is about twice as fast as using erts_chars_to_integer because we do not have to copy the entire list */ Eterm res; Eterm dummy; /* must be a list */ - if (do_list_to_integer(BIF_P,BIF_ARG_1,&res,&dummy) != LTI_ALL_INTEGER) { + if (erts_list_to_integer(BIF_P, BIF_ARG_1, 10, + &res, &dummy) != LTI_ALL_INTEGER) { BIF_ERROR(BIF_P,BADARG); } BIF_RET(res); @@ -2943,14 +2958,12 @@ BIF_RETTYPE list_to_integer_1(BIF_ALIST_1) BIF_RETTYPE list_to_integer_2(BIF_ALIST_2) { - /* Bif implementation is about 50% faster than pure erlang, and since we have erts_chars_to_integer now it is simpler as well. This could be optmized further if we did not have to copy the list to buf. */ - int i; - Eterm res; - char *buf = NULL; + Sint i; + Eterm res, dummy; int base; i = erts_list_length(BIF_ARG_1); @@ -2958,31 +2971,16 @@ BIF_RETTYPE list_to_integer_2(BIF_ALIST_2) BIF_ERROR(BIF_P, BADARG); base = signed_val(BIF_ARG_2); - + if (base < 2 || base > 36) BIF_ERROR(BIF_P, BADARG); - /* Take fast path if base it 10 */ - if (base == 10) - return list_to_integer_1(BIF_P,&BIF_ARG_1); - - buf = (char *) erts_alloc(ERTS_ALC_T_TMP, i + 1); - - if (intlist_to_buf(BIF_ARG_1, buf, i) < 0) - goto list_to_integer_1_error; - buf[i] = '\0'; /* null terminal */ - - if ((res = erts_chars_to_integer(BIF_P,buf,i,base)) == THE_NON_VALUE) - goto list_to_integer_1_error; - - erts_free(ERTS_ALC_T_TMP, (void *) buf); + if (erts_list_to_integer(BIF_P, BIF_ARG_1, base, + &res, &dummy) != LTI_ALL_INTEGER) { + BIF_ERROR(BIF_P,BADARG); + } BIF_RET(res); - - list_to_integer_1_error: - erts_free(ERTS_ALC_T_TMP, (void *) buf); - BIF_ERROR(BIF_P, BADARG); - - } +} /**********************************************************************/ @@ -3288,7 +3286,7 @@ static BIF_RETTYPE do_charbuf_to_float(Process *BIF_P,char *buf) { BIF_RETTYPE list_to_float_1(BIF_ALIST_1) { - int i; + Sint i; Eterm res; char *buf = NULL; @@ -3405,7 +3403,7 @@ BIF_RETTYPE list_to_tuple_1(BIF_ALIST_1) Eterm* cons; Eterm res; Eterm* hp; - int len; + Sint len; if ((len = erts_list_length(list)) < 0 || len > ERTS_MAX_TUPLE_SIZE) { BIF_ERROR(BIF_P, BADARG); @@ -3434,91 +3432,6 @@ BIF_RETTYPE self_0(BIF_ALIST_0) /**********************************************************************/ -/* - New representation of refs in R9, see erl_term.h - - In the first data word, only the usual 18 bits are used. Ordinarily, - in "long refs" all words are used (in other words, practically never - wrap around), but for compatibility with older nodes, "short refs" - exist. Short refs come into being by being converted from the old - external format for refs (tag REFERENCE_EXT). Short refs are - converted back to the old external format. - - When converting a long ref to the external format in the case of - preparing for sending to an older node, the ref is truncated by only - using the first word (with 18 significant bits), and using the old tag - REFERENCE_EXT. - - When comparing refs or different size, only the parts up to the length - of the shorter operand are used. This has the desirable effect that a - long ref sent to an old node and back will be treated as equal to - the original, although some of the bits have been lost. - - The hash value for a ref always considers only the first word, since - in the above scenario, the original and the copy should have the same - hash value. -*/ - -static Uint32 reference0; /* Initialized in erts_init_bif */ -static Uint32 reference1; -static Uint32 reference2; -static erts_smp_spinlock_t make_ref_lock; -static erts_smp_mtx_t ports_snapshot_mtx; -erts_smp_atomic_t erts_dead_ports_ptr; /* To store dying ports during snapshot */ - -void -erts_make_ref_in_array(Uint32 ref[ERTS_MAX_REF_NUMBERS]) -{ - erts_smp_spin_lock(&make_ref_lock); - - reference0++; - if (reference0 >= MAX_REFERENCE) { - reference0 = 0; - reference1++; - if (reference1 == 0) { - reference2++; - } - } - - ref[0] = reference0; - ref[1] = reference1; - ref[2] = reference2; - - erts_smp_spin_unlock(&make_ref_lock); -} - -Eterm erts_make_ref_in_buffer(Eterm buffer[REF_THING_SIZE]) -{ - Eterm* hp = buffer; - Uint32 ref[ERTS_MAX_REF_NUMBERS]; - - erts_make_ref_in_array(ref); - write_ref_thing(hp, ref[0], ref[1], ref[2]); - return make_internal_ref(hp); -} - -Eterm erts_make_ref(Process *p) -{ - Eterm* hp; - Uint32 ref[ERTS_MAX_REF_NUMBERS]; - - ERTS_SMP_LC_ASSERT(ERTS_PROC_LOCK_MAIN & erts_proc_lc_my_proc_locks(p)); - - hp = HAlloc(p, REF_THING_SIZE); - - erts_make_ref_in_array(ref); - write_ref_thing(hp, ref[0], ref[1], ref[2]); - - return make_internal_ref(hp); -} - -BIF_RETTYPE make_ref_0(BIF_ALIST_0) -{ - return erts_make_ref(BIF_P); -} - -/**********************************************************************/ - /* return the time of day */ BIF_RETTYPE time_0(BIF_ALIST_0) @@ -3766,11 +3679,9 @@ BIF_RETTYPE now_0(BIF_ALIST_0) BIF_RETTYPE garbage_collect_0(BIF_ALIST_0) { - int reds; - FLAGS(BIF_P) |= F_NEED_FULLSWEEP; - reds = erts_garbage_collect(BIF_P, 0, NULL, 0); - BIF_RET2(am_true, reds); + erts_garbage_collect(BIF_P, 0, NULL, 0); + BIF_RET(am_true); } /**********************************************************************/ @@ -3825,7 +3736,7 @@ BIF_RETTYPE erts_debug_display_1(BIF_ALIST_1) erts_dsprintf_buf_t *dsbufp = erts_create_tmp_dsbuf(64); pres = erts_dsprintf(dsbufp, "%.*T\n", INT_MAX, BIF_ARG_1); if (pres < 0) - erl_exit(1, "Failed to convert term to string: %d (%s)\n", + erts_exit(ERTS_ERROR_EXIT, "Failed to convert term to string: %d (%s)\n", -pres, erl_errno_id(-pres)); hp = HAlloc(BIF_P, 2*dsbufp->str_len); /* we need length * 2 heap words */ res = buf_to_intlist(&hp, dsbufp->str, dsbufp->str_len, NIL); @@ -3839,7 +3750,7 @@ BIF_RETTYPE display_string_1(BIF_ALIST_1) { Process* p = BIF_P; Eterm string = BIF_ARG_1; - int len = is_string(string); + Sint len = is_string(string); char *str; if (len <= 0) { @@ -3847,7 +3758,7 @@ BIF_RETTYPE display_string_1(BIF_ALIST_1) } str = (char *) erts_alloc(ERTS_ALC_T_TMP, sizeof(char)*(len + 1)); if (intlist_to_buf(string, str, len) != len) - erl_exit(1, "%s:%d: Internal error\n", __FILE__, __LINE__); + erts_exit(ERTS_ERROR_EXIT, "%s:%d: Internal error\n", __FILE__, __LINE__); str[len] = '\0'; erts_fprintf(stderr, "%s", str); erts_free(ERTS_ALC_T_TMP, (void *) str); @@ -3867,7 +3778,7 @@ BIF_RETTYPE display_nl_0(BIF_ALIST_0) BIF_RETTYPE halt_0(BIF_ALIST_0) { VERBOSE(DEBUG_SYSTEM,("System halted by BIF halt()\n")); - erl_halt(0); + erts_halt(0); ERTS_BIF_YIELD1(bif_export[BIF_halt_1], BIF_P, am_undefined); } @@ -3880,20 +3791,21 @@ static char halt_msg[HALT_MSG_SIZE]; /* ARGSUSED */ BIF_RETTYPE halt_1(BIF_ALIST_1) { - Sint code; + Uint code; - if (is_small(BIF_ARG_1) && (code = signed_val(BIF_ARG_1)) >= 0) { + if (term_to_Uint_mask(BIF_ARG_1, &code)) { + int pos_int_code = (int) (code & INT_MAX); VERBOSE(DEBUG_SYSTEM,("System halted by BIF halt(%T)\n", BIF_ARG_1)); - erl_halt((int)(- code)); + erts_halt(pos_int_code); ERTS_BIF_YIELD1(bif_export[BIF_halt_1], BIF_P, am_undefined); } else if (ERTS_IS_ATOM_STR("abort", BIF_ARG_1)) { VERBOSE(DEBUG_SYSTEM,("System halted by BIF halt(%T)\n", BIF_ARG_1)); erts_smp_proc_unlock(BIF_P, ERTS_PROC_LOCK_MAIN); - erl_exit(ERTS_ABORT_EXIT, ""); + erts_exit(ERTS_ABORT_EXIT, ""); } else if (is_string(BIF_ARG_1) || BIF_ARG_1 == NIL) { - int i; + Sint i; if ((i = intlist_to_buf(BIF_ARG_1, halt_msg, HALT_MSG_SIZE-1)) < 0) { goto error; @@ -3901,11 +3813,11 @@ BIF_RETTYPE halt_1(BIF_ALIST_1) halt_msg[i] = '\0'; VERBOSE(DEBUG_SYSTEM,("System halted by BIF halt(%T)\n", BIF_ARG_1)); erts_smp_proc_unlock(BIF_P, ERTS_PROC_LOCK_MAIN); - erl_exit(ERTS_DUMP_EXIT, "%s\n", halt_msg); + erts_exit(ERTS_DUMP_EXIT, "%s\n", halt_msg); } else goto error; - return NIL; /* Pedantic (lint does not know about erl_exit) */ + return NIL; /* Pedantic (lint does not know about erts_exit) */ error: BIF_ERROR(BIF_P, BADARG); } @@ -3916,7 +3828,7 @@ BIF_RETTYPE halt_1(BIF_ALIST_1) /* ARGSUSED */ BIF_RETTYPE halt_2(BIF_ALIST_2) { - Sint code; + Uint code; Eterm optlist = BIF_ARG_2; int flush = 1; @@ -3943,26 +3855,27 @@ BIF_RETTYPE halt_2(BIF_ALIST_2) if (is_not_nil(optlist)) goto error; - if (is_small(BIF_ARG_1) && (code = signed_val(BIF_ARG_1)) >= 0) { + if (term_to_Uint_mask(BIF_ARG_1, &code)) { + int pos_int_code = (int) (code & INT_MAX); VERBOSE(DEBUG_SYSTEM, ("System halted by BIF halt(%T, %T)\n", BIF_ARG_1, BIF_ARG_2)); if (flush) { - erl_halt((int)(- code)); + erts_halt(pos_int_code); ERTS_BIF_YIELD1(bif_export[BIF_halt_1], BIF_P, am_undefined); } else { erts_smp_proc_unlock(BIF_P, ERTS_PROC_LOCK_MAIN); - erl_exit((int)(- code), ""); + erts_exit(pos_int_code, ""); } } else if (ERTS_IS_ATOM_STR("abort", BIF_ARG_1)) { VERBOSE(DEBUG_SYSTEM, ("System halted by BIF halt(%T, %T)\n", BIF_ARG_1, BIF_ARG_2)); erts_smp_proc_unlock(BIF_P, ERTS_PROC_LOCK_MAIN); - erl_exit(ERTS_ABORT_EXIT, ""); + erts_exit(ERTS_ABORT_EXIT, ""); } else if (is_string(BIF_ARG_1) || BIF_ARG_1 == NIL) { - int i; + Sint i; if ((i = intlist_to_buf(BIF_ARG_1, halt_msg, HALT_MSG_SIZE-1)) < 0) { goto error; @@ -3971,11 +3884,11 @@ BIF_RETTYPE halt_2(BIF_ALIST_2) VERBOSE(DEBUG_SYSTEM, ("System halted by BIF halt(%T, %T)\n", BIF_ARG_1, BIF_ARG_2)); erts_smp_proc_unlock(BIF_P, ERTS_PROC_LOCK_MAIN); - erl_exit(ERTS_DUMP_EXIT, "%s\n", halt_msg); + erts_exit(ERTS_DUMP_EXIT, "%s\n", halt_msg); } else goto error; - return NIL; /* Pedantic (lint does not know about erl_exit) */ + return NIL; /* Pedantic (lint does not know about erts_exit) */ error: BIF_ERROR(BIF_P, BADARG); } @@ -3984,16 +3897,19 @@ BIF_RETTYPE halt_2(BIF_ALIST_2) BIF_RETTYPE function_exported_3(BIF_ALIST_3) { + int arity; if (is_not_atom(BIF_ARG_1) || is_not_atom(BIF_ARG_2) || is_not_small(BIF_ARG_3)) { BIF_ERROR(BIF_P, BADARG); } - if (erts_find_function(BIF_ARG_1, BIF_ARG_2, signed_val(BIF_ARG_3), - erts_active_code_ix()) == NULL) { - BIF_RET(am_false); + arity = signed_val(BIF_ARG_3); + if (erts_find_function(BIF_ARG_1, BIF_ARG_2, arity, + erts_active_code_ix()) != NULL || + erts_is_builtin(BIF_ARG_1, BIF_ARG_2, arity)) { + BIF_RET(am_true); } - BIF_RET(am_true); + BIF_RET(am_false); } /**********************************************************************/ @@ -4028,7 +3944,7 @@ term2list_dsprintf(Process *p, Eterm term) erts_dsprintf_buf_t *dsbufp = erts_create_tmp_dsbuf(64); pres = erts_dsprintf(dsbufp, "%T", term); if (pres < 0) - erl_exit(1, "Failed to convert term to list: %d (%s)\n", + erts_exit(ERTS_ERROR_EXIT, "Failed to convert term to list: %d (%s)\n", -pres, erl_errno_id(-pres)); hp = HAlloc(p, 2*dsbufp->str_len); /* we need length * 2 heap words */ res = buf_to_intlist(&hp, dsbufp->str, dsbufp->str_len, NIL); @@ -4056,16 +3972,9 @@ BIF_RETTYPE make_fun_3(BIF_ALIST_3) if (arity < 0) { goto error; } -#if HALFWORD_HEAP - hp = HAlloc(BIF_P, 3); - hp[0] = HEADER_EXPORT; - /* Yes, May be misaligned, but X86_64 will fix it... */ - *((Export **) (hp+1)) = erts_export_get_or_make_stub(BIF_ARG_1, BIF_ARG_2, (Uint) arity); -#else hp = HAlloc(BIF_P, 2); hp[0] = HEADER_EXPORT; hp[1] = (Eterm) erts_export_get_or_make_stub(BIF_ARG_1, BIF_ARG_2, (Uint) arity); -#endif BIF_RET(make_export(hp)); } @@ -4109,7 +4018,7 @@ BIF_RETTYPE list_to_pid_1(BIF_ALIST_1) { Uint a = 0, b = 0, c = 0; char* cp; - int i; + Sint i; DistEntry *dep = NULL; char *buf = (char *) erts_alloc(ERTS_ALC_T_TMP, 65); /* @@ -4170,6 +4079,7 @@ BIF_RETTYPE list_to_pid_1(BIF_ALIST_1) goto bad; enp = erts_find_or_insert_node(dep->sysname, dep->creation); + ASSERT(enp != erts_this_node); etp = (ExternalThing *) HAlloc(BIF_P, EXTERNAL_THING_HEAD_SIZE + 1); etp->header = make_external_pid_header(1); @@ -4285,22 +4195,33 @@ BIF_RETTYPE system_flag_2(BIF_ALIST_2) Sint n; if (BIF_ARG_1 == am_multi_scheduling) { - if (BIF_ARG_2 == am_block || BIF_ARG_2 == am_unblock) { + if (BIF_ARG_2 == am_block || BIF_ARG_2 == am_unblock + || BIF_ARG_2 == am_block_normal || BIF_ARG_2 == am_unblock_normal) { #ifndef ERTS_SMP BIF_RET(am_disabled); #else + int block = (BIF_ARG_2 == am_block + || BIF_ARG_2 == am_block_normal); + int normal = (BIF_ARG_2 == am_block_normal + || BIF_ARG_2 == am_unblock_normal); if (erts_no_schedulers == 1) BIF_RET(am_disabled); else { switch (erts_block_multi_scheduling(BIF_P, ERTS_PROC_LOCK_MAIN, - BIF_ARG_2 == am_block, + block, + normal, 0)) { case ERTS_SCHDLR_SSPND_DONE_MSCHED_BLOCKED: BIF_RET(am_blocked); + case ERTS_SCHDLR_SSPND_DONE_NMSCHED_BLOCKED: + BIF_RET(am_blocked_normal); case ERTS_SCHDLR_SSPND_YIELD_DONE_MSCHED_BLOCKED: ERTS_BIF_YIELD_RETURN_X(BIF_P, am_blocked, am_multi_scheduling); + case ERTS_SCHDLR_SSPND_YIELD_DONE_NMSCHED_BLOCKED: + ERTS_BIF_YIELD_RETURN_X(BIF_P, am_blocked_normal, + am_multi_scheduling); case ERTS_SCHDLR_SSPND_DONE: BIF_RET(am_enabled); case ERTS_SCHDLR_SSPND_YIELD_RESTART: @@ -4333,11 +4254,7 @@ BIF_RETTYPE system_flag_2(BIF_ALIST_2) switch (erts_set_schedulers_online(BIF_P, ERTS_PROC_LOCK_MAIN, signed_val(BIF_ARG_2), - &old_no -#ifdef ERTS_DIRTY_SCHEDULERS - , 0 -#endif - )) { + &old_no, 0)) { case ERTS_SCHDLR_SSPND_DONE: BIF_RET(make_small(old_no)); case ERTS_SCHDLR_SSPND_YIELD_RESTART: @@ -4425,7 +4342,7 @@ BIF_RETTYPE system_flag_2(BIF_ALIST_2) } } else if (BIF_ARG_1 == make_small(1)) { int i, max; - ErlMessage* mp; + ErtsMessage* mp; erts_smp_proc_unlock(BIF_P, ERTS_PROC_LOCK_MAIN); erts_smp_thr_progress_block(); @@ -4496,6 +4413,53 @@ BIF_RETTYPE system_flag_2(BIF_ALIST_2) break; } #endif + } else if (BIF_ARG_1 == am_time_offset + && ERTS_IS_ATOM_STR("finalize", BIF_ARG_2)) { + ErtsTimeOffsetState res; + erts_smp_proc_unlock(BIF_P, ERTS_PROC_LOCK_MAIN); + res = erts_finalize_time_offset(); + erts_smp_proc_lock(BIF_P, ERTS_PROC_LOCK_MAIN); + switch (res) { + case ERTS_TIME_OFFSET_PRELIMINARY: { + DECL_AM(preliminary); + BIF_RET(AM_preliminary); + } + case ERTS_TIME_OFFSET_FINAL: { + DECL_AM(final); + BIF_RET(AM_final); + } + case ERTS_TIME_OFFSET_VOLATILE: { + DECL_AM(volatile); + BIF_RET(AM_volatile); + } + default: + ERTS_INTERNAL_ERROR("Unknown state"); + } +#ifdef ERTS_ENABLE_MSACC + } else if (BIF_ARG_1 == am_microstate_accounting) { + Eterm threads; + if (BIF_ARG_2 == am_true || BIF_ARG_2 == am_false) { + erts_aint32_t new = BIF_ARG_2 == am_true ? ERTS_MSACC_ENABLE : ERTS_MSACC_DISABLE; + erts_aint32_t old = erts_smp_atomic32_xchg_nob(&msacc, new); + Eterm ref = erts_msacc_request(BIF_P, new, &threads); + if (is_non_value(ref)) + BIF_RET(old ? am_true : am_false); + BIF_TRAP3(await_msacc_mod_trap, + BIF_P, + ref, + old ? am_true : am_false, + threads); + } else if (BIF_ARG_2 == am_reset) { + Eterm ref = erts_msacc_request(BIF_P, ERTS_MSACC_RESET, &threads); + erts_aint32_t old = erts_smp_atomic32_read_nob(&msacc); + ASSERT(is_value(ref)); + BIF_TRAP3(await_msacc_mod_trap, + BIF_P, + ref, + old ? am_true : am_false, + threads); + } +#endif } else if (ERTS_IS_ATOM_STR("scheduling_statistics", BIF_ARG_1)) { int what; if (ERTS_IS_ATOM_STR("disable", BIF_ARG_2)) @@ -4548,7 +4512,7 @@ BIF_RETTYPE hash_2(BIF_ALIST_2) if ((range = signed_val(BIF_ARG_2)) <= 0) { /* [1..MAX_SMALL] */ BIF_ERROR(BIF_P, BADARG); } -#if defined(ARCH_64) && !HALFWORD_HEAP +#if defined(ARCH_64) if (range > ((1L << 27) - 1)) BIF_ERROR(BIF_P, BADARG); #endif @@ -4620,7 +4584,7 @@ BIF_RETTYPE phash2_2(BIF_ALIST_2) /* * Return either a small or a big. Use the heap for bigs if there is room. */ -#if defined(ARCH_64) && !HALFWORD_HEAP +#if defined(ARCH_64) BIF_RET(make_small(final_hash)); #else if (IS_USMALL(0, final_hash)) { @@ -4647,7 +4611,7 @@ BIF_RETTYPE bump_reductions_1(BIF_ALIST_1) } BIF_RETTYPE erts_internal_cmp_term_2(BIF_ALIST_2) { - int res = CMP_TERM(BIF_ARG_1,BIF_ARG_2); + Sint res = CMP_TERM(BIF_ARG_1,BIF_ARG_2); /* ensure -1, 0, 1 result */ if (res < 0) { @@ -4660,29 +4624,27 @@ BIF_RETTYPE erts_internal_cmp_term_2(BIF_ALIST_2) { /* * Processes doing yield on return in a bif ends up in bif_return_trap(). */ -static BIF_RETTYPE bif_return_trap( -#ifdef DEBUG - BIF_ALIST_2 -#else - BIF_ALIST_1 -#endif - ) +static BIF_RETTYPE bif_return_trap(BIF_ALIST_2) { -#ifdef DEBUG + Eterm res = BIF_ARG_1; + switch (BIF_ARG_2) { - case am_multi_scheduling: #ifdef ERTS_SMP - erts_dbg_multi_scheduling_return_trap(BIF_P, BIF_ARG_1); -#endif - break; - case am_schedulers_online: + case am_multi_scheduling: { + int msb = erts_is_multi_scheduling_blocked(); + if (msb > 0) + res = am_blocked; + else if (msb < 0) + res = am_blocked_normal; + else + ERTS_INTERNAL_ERROR("Unexpected multi scheduling block state"); break; + } +#endif default: break; } -#endif - - BIF_RET(BIF_ARG_1); + BIF_RET(res); } /* @@ -4783,30 +4745,32 @@ void erts_init_trap_export(Export* ep, Eterm m, Eterm f, Uint a, void erts_init_bif(void) { - reference0 = 0; - reference1 = 0; - reference2 = 0; - - erts_smp_spinlock_init(&make_ref_lock, "make_ref"); erts_smp_mtx_init(&ports_snapshot_mtx, "ports_snapshot"); erts_smp_atomic_init_nob(&erts_dead_ports_ptr, (erts_aint_t) NULL); /* - * bif_return_trap/1 is a hidden BIF that bifs that need to - * yield the calling process traps to. The only thing it does: - * return the value passed as argument. + * bif_return_trap/2 is a hidden BIF that bifs that need to + * yield the calling process traps to. */ - erts_init_trap_export(&bif_return_trap_export, am_erlang, am_bif_return_trap, -#ifdef DEBUG - 2 -#else - 1 -#endif - , &bif_return_trap); + erts_init_trap_export(&bif_return_trap_export, + am_erlang, am_bif_return_trap, 2, + &bif_return_trap); + + erts_await_result = erts_export_put(am_erts_internal, + am_await_result, + 1); - flush_monitor_message_trap = erts_export_put(am_erlang, - am_flush_monitor_message, - 2); + erts_init_trap_export(&dsend_continue_trap_export, + am_erts_internal, am_dsend_continue_trap, 1, + dsend_continue_trap_1); + + flush_monitor_messages_trap = erts_export_put(am_erts_internal, + am_flush_monitor_messages, + 3); + + erts_convert_time_unit_trap = erts_export_put(am_erlang, + am_convert_time_unit, + 3); set_cpu_topology_trap = erts_export_put(am_erlang, am_set_cpu_topology, @@ -4818,8 +4782,12 @@ void erts_init_bif(void) await_port_send_result_trap = erts_export_put(am_erts_internal, am_await_port_send_result, 3); await_sched_wall_time_mod_trap - = erts_export_put(am_erlang, am_await_sched_wall_time_modifications, 2); + = erts_export_put(am_erlang, am_await_sched_wall_time_modifications, 2); + await_msacc_mod_trap + = erts_export_put(am_erts_internal, am_await_microstate_accounting_modifications, 3); + erts_smp_atomic32_init_nob(&sched_wall_time, 0); + erts_smp_atomic32_init_nob(&msacc, ERTS_MSACC_IS_ENABLED()); } #ifdef HARDDEBUG diff --git a/erts/emulator/beam/bif.h b/erts/emulator/beam/bif.h index 51b77a95ed..46af552b39 100644 --- a/erts/emulator/beam/bif.h +++ b/erts/emulator/beam/bif.h @@ -3,16 +3,17 @@ * * Copyright Ericsson AB 1996-2013. All Rights Reserved. * - * The contents of this file are subject to the Erlang Public License, - * Version 1.1, (the "License"); you may not use this file except in - * compliance with the License. You should have received a copy of the - * Erlang Public License along with this software. If not, it can be - * retrieved online at http://www.erlang.org/. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Software distributed under the License is distributed on an "AS IS" - * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See - * the License for the specific language governing rights and limitations - * under the License. + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. * * %CopyrightEnd% */ @@ -20,7 +21,9 @@ #ifndef __BIF_H__ #define __BIF_H__ +extern Export *erts_await_result; extern Export* erts_format_cpu_topology_trap; +extern Export *erts_convert_time_unit_trap; #define BIF_RETTYPE Eterm @@ -30,10 +33,12 @@ extern Export* erts_format_cpu_topology_trap; #define BIF_ALIST_1 Process* A__p, Eterm* BIF__ARGS #define BIF_ALIST_2 Process* A__p, Eterm* BIF__ARGS #define BIF_ALIST_3 Process* A__p, Eterm* BIF__ARGS +#define BIF_ALIST_4 Process* A__p, Eterm* BIF__ARGS #define BIF_ARG_1 (BIF__ARGS[0]) #define BIF_ARG_2 (BIF__ARGS[1]) #define BIF_ARG_3 (BIF__ARGS[2]) +#define BIF_ARG_4 (BIF__ARGS[3]) #define ERTS_IS_PROC_OUT_OF_REDS(p) \ ((p)->fcalls > 0 \ @@ -49,22 +54,24 @@ extern Export* erts_format_cpu_topology_trap; (p)->fcalls = -CONTEXT_REDS; \ } while(0) - -#define ERTS_VBUMP_ALL_REDS(p) \ +#define ERTS_VBUMP_ALL_REDS_INTERNAL(p, fcalls) \ do { \ if (!ERTS_PROC_GET_SAVED_CALLS_BUF((p))) { \ - if ((p)->fcalls > 0) \ - ERTS_PROC_GET_SCHDATA((p))->virtual_reds += (p)->fcalls; \ - (p)->fcalls = 0; \ + if ((fcalls) > 0) \ + ERTS_PROC_GET_SCHDATA((p))->virtual_reds += (fcalls); \ + (fcalls) = 0; \ } \ else { \ - if ((p)->fcalls > -CONTEXT_REDS) \ + if ((fcalls) > -CONTEXT_REDS) \ ERTS_PROC_GET_SCHDATA((p))->virtual_reds \ - += ((p)->fcalls - (-CONTEXT_REDS)); \ - (p)->fcalls = -CONTEXT_REDS; \ + += ((fcalls) - (-CONTEXT_REDS)); \ + (fcalls) = -CONTEXT_REDS; \ } \ } while(0) +#define ERTS_VBUMP_ALL_REDS(p) \ + ERTS_VBUMP_ALL_REDS_INTERNAL((p), (p)->fcalls) + #define BUMP_REDS(p, gc) do { \ ASSERT(p); \ ERTS_SMP_LC_ASSERT(ERTS_PROC_LOCK_MAIN & erts_proc_lc_my_proc_locks(p));\ @@ -105,10 +112,34 @@ do { \ } \ } while(0) -#define ERTS_BIF_REDS_LEFT(p) \ +#define ERTS_VBUMP_LEAVE_REDS_INTERNAL(P, Reds, FCalls) \ + do { \ + if (ERTS_PROC_GET_SAVED_CALLS_BUF((P))) { \ + int nreds__ = ((int)(Reds)) - CONTEXT_REDS; \ + if ((FCalls) > nreds__) { \ + ERTS_PROC_GET_SCHDATA((P))->virtual_reds \ + += (FCalls) - nreds__; \ + (FCalls) = nreds__; \ + } \ + } \ + else { \ + if ((FCalls) > (Reds)) { \ + ERTS_PROC_GET_SCHDATA((P))->virtual_reds \ + += (FCalls) - (Reds); \ + (FCalls) = (Reds); \ + } \ + } \ + } while (0) + +#define ERTS_VBUMP_LEAVE_REDS(P, Reds) \ + ERTS_VBUMP_LEAVE_REDS_INTERNAL(P, Reds, (P)->fcalls) + +#define ERTS_REDS_LEFT(p, FCalls) \ (ERTS_PROC_GET_SAVED_CALLS_BUF((p)) \ - ? ((p)->fcalls > -CONTEXT_REDS ? ((p)->fcalls - (-CONTEXT_REDS)) : 0)\ - : ((p)->fcalls > 0 ? (p)->fcalls : 0)) + ? ((FCalls) > -CONTEXT_REDS ? ((FCalls) - (-CONTEXT_REDS)) : 0) \ + : ((FCalls) > 0 ? (FCalls) : 0)) + +#define ERTS_BIF_REDS_LEFT(p) ERTS_REDS_LEFT(p, p->fcalls) #define BIF_RET2(x, gc) do { \ BUMP_REDS(BIF_P, (gc)); \ @@ -124,12 +155,85 @@ do { \ return THE_NON_VALUE; \ } while(0) +#define ERTS_BIF_ERROR_TRAPPED0(Proc, Reason, Bif) \ +do { \ + (Proc)->freason = (Reason); \ + (Proc)->current = (Bif)->code; \ + return THE_NON_VALUE; \ +} while (0) + +#define ERTS_BIF_ERROR_TRAPPED1(Proc, Reason, Bif, A0) \ +do { \ + Eterm* reg = ERTS_PROC_GET_SCHDATA((Proc))->x_reg_array; \ + (Proc)->freason = (Reason); \ + (Proc)->current = (Bif)->code; \ + reg[0] = (Eterm) (A0); \ + return THE_NON_VALUE; \ +} while (0) + +#define ERTS_BIF_ERROR_TRAPPED2(Proc, Reason, Bif, A0, A1) \ +do { \ + Eterm* reg = ERTS_PROC_GET_SCHDATA((Proc))->x_reg_array; \ + (Proc)->freason = (Reason); \ + (Proc)->current = (Bif)->code; \ + reg[0] = (Eterm) (A0); \ + reg[1] = (Eterm) (A1); \ + return THE_NON_VALUE; \ +} while (0) + +#define ERTS_BIF_ERROR_TRAPPED3(Proc, Reason, Bif, A0, A1, A2) \ +do { \ + Eterm* reg = ERTS_PROC_GET_SCHDATA((Proc))->x_reg_array; \ + (Proc)->freason = (Reason); \ + (Proc)->current = (Bif)->code; \ + reg[0] = (Eterm) (A0); \ + reg[1] = (Eterm) (A1); \ + reg[2] = (Eterm) (A2); \ + return THE_NON_VALUE; \ +} while (0) + #define ERTS_BIF_PREP_ERROR(Ret, Proc, Reason) \ do { \ (Proc)->freason = (Reason); \ (Ret) = THE_NON_VALUE; \ } while (0) +#define ERTS_BIF_PREP_ERROR_TRAPPED0(Ret, Proc, Reason, Bif) \ +do { \ + (Proc)->freason = (Reason); \ + (Proc)->current = (Bif)->code; \ + (Ret) = THE_NON_VALUE; \ +} while (0) + +#define ERTS_BIF_PREP_ERROR_TRAPPED1(Ret, Proc, Reason, Bif, A0) \ +do { \ + Eterm* reg = ERTS_PROC_GET_SCHDATA((Proc))->x_reg_array; \ + (Proc)->freason = (Reason); \ + (Proc)->current = (Bif)->code; \ + reg[0] = (Eterm) (A0); \ + (Ret) = THE_NON_VALUE; \ +} while (0) + +#define ERTS_BIF_PREP_ERROR_TRAPPED2(Ret, Proc, Reason, Bif, A0, A1) \ +do { \ + Eterm* reg = ERTS_PROC_GET_SCHDATA((Proc))->x_reg_array; \ + (Proc)->freason = (Reason); \ + (Proc)->current = (Bif)->code; \ + reg[0] = (Eterm) (A0); \ + reg[1] = (Eterm) (A1); \ + (Ret) = THE_NON_VALUE; \ +} while (0) + +#define ERTS_BIF_PREP_ERROR_TRAPPED3(Ret, Proc, Reason, Bif, A0, A1, A2) \ +do { \ + Eterm* reg = ERTS_PROC_GET_SCHDATA((Proc))->x_reg_array; \ + (Proc)->freason = (Reason); \ + (Proc)->current = (Bif)->code; \ + reg[0] = (Eterm) (A0); \ + reg[1] = (Eterm) (A1); \ + reg[2] = (Eterm) (A2); \ + (Ret) = THE_NON_VALUE; \ +} while (0) #define ERTS_BIF_PREP_TRAP0(Ret, Trap, Proc) \ do { \ @@ -234,37 +338,20 @@ do { \ } while(0) extern Export bif_return_trap_export; -#ifdef DEBUG -#define ERTS_BIF_PREP_YIELD_RETURN_X(RET, P, VAL, DEBUG_VAL) \ -do { \ - ERTS_VBUMP_ALL_REDS(P); \ - ERTS_BIF_PREP_TRAP2(RET, &bif_return_trap_export, (P), (VAL), \ - (DEBUG_VAL)); \ -} while (0) -#else -#define ERTS_BIF_PREP_YIELD_RETURN_X(RET, P, VAL, DEBUG_VAL) \ +#define ERTS_BIF_PREP_YIELD_RETURN_X(RET, P, VAL, OP) \ do { \ ERTS_VBUMP_ALL_REDS(P); \ - ERTS_BIF_PREP_TRAP1(RET, &bif_return_trap_export, (P), (VAL)); \ + ERTS_BIF_PREP_TRAP2(RET, &bif_return_trap_export, (P), (VAL), (OP));\ } while (0) -#endif #define ERTS_BIF_PREP_YIELD_RETURN(RET, P, VAL) \ ERTS_BIF_PREP_YIELD_RETURN_X(RET, (P), (VAL), am_undefined) -#ifdef DEBUG -#define ERTS_BIF_YIELD_RETURN_X(P, VAL, DEBUG_VAL) \ -do { \ - ERTS_VBUMP_ALL_REDS(P); \ - BIF_TRAP2(&bif_return_trap_export, (P), (VAL), (DEBUG_VAL)); \ -} while (0) -#else -#define ERTS_BIF_YIELD_RETURN_X(P, VAL, DEBUG_VAL) \ +#define ERTS_BIF_YIELD_RETURN_X(P, VAL, OP) \ do { \ ERTS_VBUMP_ALL_REDS(P); \ - BIF_TRAP1(&bif_return_trap_export, (P), (VAL)); \ + BIF_TRAP2(&bif_return_trap_export, (P), (VAL), (OP)); \ } while (0) -#endif #define ERTS_BIF_RETURN_YIELD(P) ERTS_VBUMP_ALL_REDS((P)) @@ -392,6 +479,54 @@ erts_bif_prep_await_proc_exit_apply_trap(Process *c_p, Eterm args[], int nargs); +#ifdef ERL_WANT_HIPE_BIF_WRAPPER__ + +#ifndef HIPE + +#define HIPE_WRAPPER_BIF_DISABLE_GC(BIF_NAME, ARITY) + +#else + +#include "erl_fun.h" +#include "hipe_mode_switch.h" + +/* + * Hipe wrappers used by native code for BIFs that disable GC while trapping. + * Also add usage of the wrapper in ../hipe/hipe_bif_list.m4 + * + * Problem: + * When native code calls a BIF that traps, hipe_mode_switch will push a + * "trap frame" on the Erlang stack in order to find its way back from beam_emu + * back to native caller when finally done. If GC is disabled and stack/heap + * is full there is no place to push the "trap frame". + * + * Solution: + * We reserve space on stack for the "trap frame" here before the BIF is called. + * If the BIF does not trap, the space is reclaimed here before returning. + * If the BIF traps, hipe_push_beam_trap_frame() will detect that a "trap frame" + * already is reserved and use it. + */ + + +#define HIPE_WRAPPER_BIF_DISABLE_GC(BIF_NAME, ARITY) \ +BIF_RETTYPE hipe_wrapper_ ## BIF_NAME ## _ ## ARITY (Process* c_p, \ + Eterm* args); \ +BIF_RETTYPE hipe_wrapper_ ## BIF_NAME ## _ ## ARITY (Process* c_p, \ + Eterm* args) \ +{ \ + BIF_RETTYPE res; \ + hipe_reserve_beam_trap_frame(c_p, args, ARITY); \ + res = BIF_NAME ## _ ## ARITY (c_p, args); \ + if (is_value(res) || c_p->freason != TRAP) { \ + hipe_unreserve_beam_trap_frame(c_p); \ + } \ + return res; \ +} + +#endif + +#endif /* ERL_WANT_HIPE_BIF_WRAPPER__ */ + #include "erl_bif_table.h" #endif diff --git a/erts/emulator/beam/bif.tab b/erts/emulator/beam/bif.tab index 2d888862bf..a6670c10e8 100644 --- a/erts/emulator/beam/bif.tab +++ b/erts/emulator/beam/bif.tab @@ -3,16 +3,17 @@ # # Copyright Ericsson AB 1996-2013. All Rights Reserved. # -# The contents of this file are subject to the Erlang Public License, -# Version 1.1, (the "License"); you may not use this file except in -# compliance with the License. You should have received a copy of the -# Erlang Public License along with this software. If not, it can be -# retrieved online at http://www.erlang.org/. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# Software distributed under the License is distributed on an "AS IS" -# basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See -# the License for the specific language governing rights and limitations -# under the License. +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. # # %CopyrightEnd% # @@ -45,6 +46,7 @@ bif erlang:apply/3 bif erlang:atom_to_list/1 bif erlang:binary_to_list/1 bif erlang:binary_to_list/3 +bif erlang:binary_to_term/1 bif erlang:crc32/1 bif erlang:crc32/2 bif erlang:crc32_combine/3 @@ -91,6 +93,8 @@ bif erlang:loaded/0 bif erlang:localtime/0 bif erlang:localtime_to_universaltime/2 bif erlang:make_ref/0 +bif erlang:unique_integer/0 +bif erlang:unique_integer/1 bif erlang:md5/1 bif erlang:md5_init/0 bif erlang:md5_update/2 @@ -103,8 +107,15 @@ ubif erlang:node/1 ubif erlang:node/0 bif erlang:nodes/1 bif erlang:now/0 +bif erlang:monotonic_time/0 +bif erlang:monotonic_time/1 +bif erlang:system_time/0 +bif erlang:system_time/1 +bif erlang:time_offset/0 +bif erlang:time_offset/1 +bif erlang:timestamp/0 -bif erlang:open_port/2 +bif erts_internal:open_port/2 bif erlang:pid_to_list/1 bif erlang:ports/0 @@ -114,7 +125,6 @@ bif erlang:process_flag/3 bif erlang:process_info/1 bif erlang:process_info/2 bif erlang:processes/0 -bif erlang:purge_module/1 bif erlang:put/2 bif erlang:register/2 bif erlang:registered/0 @@ -151,12 +161,20 @@ bif erts_internal:port_command/3 bif erts_internal:port_control/3 bif erts_internal:port_close/1 bif erts_internal:port_connect/2 -bif erts_internal:binary_to_term/1 -bif erts_internal:binary_to_term/2 bif erts_internal:request_system_task/3 bif erts_internal:check_process_code/2 +bif erts_internal:map_to_tuple_keys/1 +bif erts_internal:term_type/1 +bif erts_internal:map_hashmap_children/1 + +bif erts_internal:time_unit/0 +bif erts_internal:perf_counter_unit/0 + +bif erts_internal:is_system_process/1 + +bif erts_internal:system_check/1 # inet_db support bif erlang:port_set_data/2 @@ -194,15 +212,20 @@ bif math:erf/1 bif math:erfc/1 bif math:exp/1 bif math:log/1 +bif math:log2/1 bif math:log10/1 bif math:sqrt/1 bif math:atan2/2 bif math:pow/2 bif erlang:start_timer/3 +bif erlang:start_timer/4 bif erlang:send_after/3 +bif erlang:send_after/4 bif erlang:cancel_timer/1 +bif erlang:cancel_timer/2 bif erlang:read_timer/1 +bif erlang:read_timer/2 bif erlang:make_tuple/2 bif erlang:append_element/2 @@ -347,6 +370,9 @@ bif os:getenv/0 bif os:getenv/1 bif os:getpid/0 bif os:timestamp/0 +bif os:system_time/0 +bif os:system_time/1 +bif os:perf_counter/0 # # Bifs in the erl_ddll module (the module actually does not exist) @@ -480,6 +506,11 @@ bif erlang:call_on_load_function/1 bif erlang:finish_after_on_load/2 # +# New Bifs in R13B04 +# +bif erlang:binary_to_term/2 + +# # The binary match bifs (New in R14A - EEP9) # @@ -573,7 +604,7 @@ bif io:printable_range/0 bif os:unsetenv/1 # -# New in R17A +# New in 17.0 # bif re:inspect/2 @@ -595,6 +626,33 @@ bif maps:values/1 bif erts_internal:cmp_term/2 +bif ets:take/2 + +# +# New in 17.1 +# + +bif erlang:fun_info_mfa/1 + +# New in 18.0 +# + +bif erlang:get_keys/0 +bif ets:update_counter/4 +bif erts_debug:map_info/1 + +# +# New in 19.0 +# + +bif erts_internal:copy_literals/2 +bif erts_internal:purge_module/1 +bif binary:split/2 +bif binary:split/3 +bif erts_debug:size_shared/1 +bif erts_debug:copy_shared/1 +bif erlang:has_prepared_code_on_load/1 + # # Obsolete # diff --git a/erts/emulator/beam/big.c b/erts/emulator/beam/big.c index 41a041eba6..c738a9ecf5 100644 --- a/erts/emulator/beam/big.c +++ b/erts/emulator/beam/big.c @@ -3,16 +3,17 @@ * * Copyright Ericsson AB 1996-2014. All Rights Reserved. * - * The contents of this file are subject to the Erlang Public License, - * Version 1.1, (the "License"); you may not use this file except in - * compliance with the License. You should have received a copy of the - * Erlang Public License along with this software. If not, it can be - * retrieved online at http://www.erlang.org/. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Software distributed under the License is distributed on an "AS IS" - * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See - * the License for the specific language governing rights and limitations - * under the License. + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. * * %CopyrightEnd% */ @@ -47,7 +48,7 @@ _t_dst = (dst)+((sz)-1); \ _t_src = (src)+((sz)-1); \ while(_t_sz--) *_t_dst-- = *_t_src--; \ - } \ + } \ } while(0) /* add a and b with carry in + out */ @@ -274,7 +275,9 @@ _b = _b << _s; \ _vn1 = _b >> H_EXP; \ _vn0 = _b & LO_MASK; \ - _un32 = (_a1 << _s) | ((_a0>>(D_EXP-_s)) & (-_s >> (D_EXP-1))); \ + /* If needed to avoid undefined behaviour */ \ + if (_s) _un32 = (_a1 << _s) | ((_a0>>(D_EXP-_s)) & (-_s >> (D_EXP-1))); \ + else _un32 = _a1; \ _un10 = _a0 << _s; \ _un1 = _un10 >> H_EXP; \ _un0 = _un10 & LO_MASK; \ @@ -420,6 +423,25 @@ #endif +/* Forward declaration of lookup tables (See below in this file) used in list to + * integer conversions for different bases. Also used in bignum printing. + */ +static const byte digits_per_sint_lookup[36-1]; +static const byte digits_per_small_lookup[36-1]; +static const Sint largest_power_of_base_lookup[36-1]; + +static ERTS_INLINE byte get_digits_per_signed_int(Uint base) { + return digits_per_sint_lookup[base-2]; +} + +static ERTS_INLINE byte get_digits_per_small(Uint base) { + return digits_per_small_lookup[base-2]; +} + +static ERTS_INLINE Sint get_largest_power_of_base(Uint base) { + return largest_power_of_base_lookup[base-2]; +} + /* ** compare two number vectors */ @@ -1484,20 +1506,8 @@ Eterm uint_to_big(Uint x, Eterm *y) Eterm uword_to_big(UWord x, Eterm *y) { -#if HALFWORD_HEAP - Uint upper = x >> 32; - Uint lower = x & 0xFFFFFFFFUL; - if (upper == 0) { - *y = make_pos_bignum_header(1); - } else { - *y = make_pos_bignum_header(2); - BIG_DIGIT(y, 1) = upper; - } - BIG_DIGIT(y, 0) = lower; -#else *y = make_pos_bignum_header(1); BIG_DIGIT(y, 0) = x; -#endif return make_big(y); } @@ -1506,13 +1516,15 @@ Eterm uword_to_big(UWord x, Eterm *y) */ Eterm small_to_big(Sint x, Eterm *y) { + Uint xu; if (x >= 0) { + xu = x; *y = make_pos_bignum_header(1); } else { - x = -x; + xu = -(Uint)x; *y = make_neg_bignum_header(1); } - BIG_DIGIT(y, 0) = x; + BIG_DIGIT(y, 0) = xu; return make_big(y); } @@ -1520,7 +1532,7 @@ Eterm small_to_big(Sint x, Eterm *y) Eterm erts_uint64_to_big(Uint64 x, Eterm **hpp) { Eterm *hp = *hpp; -#if defined(ARCH_32) || HALFWORD_HEAP +#if defined(ARCH_32) if (x >= (((Uint64) 1) << 32)) { *hp = make_pos_bignum_header(2); BIG_DIGIT(hp, 0) = (Uint) (x & ((Uint) 0xffffffff)); @@ -1540,21 +1552,24 @@ Eterm erts_uint64_to_big(Uint64 x, Eterm **hpp) Eterm erts_sint64_to_big(Sint64 x, Eterm **hpp) { Eterm *hp = *hpp; + Uint64 ux; int neg; - if (x >= 0) + if (x >= 0) { neg = 0; + ux = x; + } else { neg = 1; - x = -x; + ux = -(Uint64)x; } -#if defined(ARCH_32) || HALFWORD_HEAP - if (x >= (((Uint64) 1) << 32)) { +#if defined(ARCH_32) + if (ux >= (((Uint64) 1) << 32)) { if (neg) *hp = make_neg_bignum_header(2); else *hp = make_pos_bignum_header(2); - BIG_DIGIT(hp, 0) = (Uint) (x & ((Uint) 0xffffffff)); - BIG_DIGIT(hp, 1) = (Uint) ((x >> 32) & ((Uint) 0xffffffff)); + BIG_DIGIT(hp, 0) = (Uint) (ux & ((Uint) 0xffffffff)); + BIG_DIGIT(hp, 1) = (Uint) ((ux >> 32) & ((Uint) 0xffffffff)); *hpp += 3; } else @@ -1564,12 +1579,52 @@ Eterm erts_sint64_to_big(Sint64 x, Eterm **hpp) *hp = make_neg_bignum_header(1); else *hp = make_pos_bignum_header(1); - BIG_DIGIT(hp, 0) = (Uint) x; + BIG_DIGIT(hp, 0) = (Uint) ux; *hpp += 2; } return make_big(hp); } +Eterm +erts_uint64_array_to_big(Uint **hpp, int neg, int len, Uint64 *array) +{ + Uint *headerp; + int i, pot_digits, digits; + + headerp = *hpp; + + pot_digits = digits = 0; + for (i = 0; i < len; i++) { +#if defined(ARCH_32) + Uint low_val = array[i] & ((Uint) 0xffffffff); + Uint high_val = (array[i] >> 32) & ((Uint) 0xffffffff); + BIG_DIGIT(headerp, pot_digits) = low_val; + pot_digits++; + if (low_val) + digits = pot_digits; + BIG_DIGIT(headerp, pot_digits) = high_val; + pot_digits++; + if (high_val) + digits = pot_digits; +#else + Uint val = array[i]; + BIG_DIGIT(headerp, pot_digits) = val; + pot_digits++; + if (val) + digits = pot_digits; +#endif + } + + if (neg) + *headerp = make_neg_bignum_header(digits); + else + *headerp = make_pos_bignum_header(digits); + + *hpp = headerp + 1 + digits; + + return make_big(headerp); +} + /* ** Convert a bignum to a double float */ @@ -1603,8 +1658,6 @@ big_to_double(Wterm x, double* resp) /* * Logic has been copied from erl_bif_guard.c and slightly * modified to use a static instead of dynamic heap - * - * HALFWORD: Return relative term with 'heap' as base. */ Eterm double_to_big(double x, Eterm *heap, Uint hsz) @@ -1635,7 +1688,7 @@ double_to_big(double x, Eterm *heap, Uint hsz) sz = BIG_NEED_SIZE(ds); /* number of words including arity */ hp = heap; - res = make_big_rel(hp, heap); + res = make_big(hp); xp = (ErtsDigit*) (hp + 1); ASSERT(ds < hsz); @@ -1685,8 +1738,10 @@ static Uint write_big(Wterm x, void (*write_func)(void *, char), void *arg) short sign = BIG_SIGN(xp); ErtsDigit rem; Uint n = 0; + const Uint digits_per_Sint = get_digits_per_signed_int(10); + const Sint largest_pow_of_base = get_largest_power_of_base(10); - if (xl == 1 && *dx < D_DECIMAL_BASE) { + if (xl == 1 && *dx < largest_pow_of_base) { rem = *dx; if (rem == 0) { (*write_func)(arg, '0'); n++; @@ -1704,7 +1759,7 @@ static Uint write_big(Wterm x, void (*write_func)(void *, char), void *arg) MOVE_DIGITS(tmp, dx, xl); while(1) { - tmpl = D_div(tmp, tmpl, D_DECIMAL_BASE, tmp, &rem); + tmpl = D_div(tmp, tmpl, largest_pow_of_base, tmp, &rem); if (tmpl == 1 && *tmp == 0) { while(rem) { (*write_func)(arg, (rem % 10)+'0'); n++; @@ -1712,7 +1767,7 @@ static Uint write_big(Wterm x, void (*write_func)(void *, char), void *arg) } break; } else { - int i = D_DECIMAL_EXP; + Uint i = digits_per_Sint; while(i--) { (*write_func)(arg, (rem % 10)+'0'); n++; rem /= 10; @@ -1893,6 +1948,8 @@ Eterm bytes_to_big(byte *xp, dsize_t xsz, int xsgn, Eterm *r) *rwp = d; rwp++; } + if (rsz > BIG_ARITY_MAX) + return NIL; if (xsgn) { *r = make_neg_bignum_header(rsz); } @@ -1978,6 +2035,32 @@ term_to_Uint(Eterm term, Uint *up) } } +/* same as term_to_Uint() + but also accept larger bignums by masking + */ +int +term_to_Uint_mask(Eterm term, Uint *up) +{ + if (is_small(term)) { + Sint i = signed_val(term); + if (i < 0) { + *up = BADARG; + return 0; + } + *up = (Uint) i; + return 1; + } else if (is_big(term) && !big_sign(term)) { + ErtsDigit* xr = big_v(term); + + ERTS_CT_ASSERT(sizeof(ErtsDigit) == sizeof(Uint)); + *up = (Uint)*xr; /* just pick first word */ + return 1; + } else { + *up = BADARG; + return 0; + } +} + int term_to_UWord(Eterm term, UWord *up) { @@ -2486,63 +2569,100 @@ int term_equals_2pow32(Eterm x) } } +static ERTS_INLINE int c2int_is_invalid_char(byte ch, int base) { + return (ch < '0' + || (ch > ('0' + base - 1) + && !(base > 10 + && ((ch >= 'a' && ch < ('a' + base - 10)) + || (ch >= 'A' && ch < ('A' + base - 10)))))); +} + +static ERTS_INLINE byte c2int_digit_from_base(byte ch) { + return ch <= '9' ? ch - '0' + : (10 + (ch <= 'Z' ? ch - 'A' : ch - 'a')); +} -#define IS_VALID_CHARACTER(CHAR,BASE) \ - (CHAR < '0' \ - || (CHAR > ('0' + BASE - 1) \ - && !(BASE > 10 \ - && ((CHAR >= 'a' && CHAR < ('a' + BASE - 10)) \ - || (CHAR >= 'A' && CHAR < ('A' + BASE - 10)))))) -#define CHARACTER_FROM_BASE(CHAR) \ - ((CHAR <= '9') ? CHAR - '0' : 10 + ((CHAR <= 'Z') ? CHAR - 'A' : CHAR - 'a')) -#define D_BASE_EXP(BASE) (d_base_exp_lookup[BASE-2]) -#define D_BASE_BASE(BASE) (d_base_base_lookup[BASE-2]) -#define LG2_LOOKUP(BASE) (lg2_lookup[base-2]) +/* + * How many bits are needed to store 1 digit of given base in binary + * Wo.Alpha formula: Table [log2[n], {n,2,36}] + */ +static const double lg2_lookup[36-1] = { + 1.0, 1.58496, 2.0, 2.32193, 2.58496, 2.80735, 3.0, 3.16993, 3.32193, + 3.45943, 3.58496, 3.70044, 3.80735, 3.90689, 4.0, 4.08746, 4.16993, 4.24793, + 4.32193, 4.39232, 4.45943, 4.52356, 4.58496, 4.64386, 4.70044, 4.75489, + 4.80735, 4.85798, 4.90689, 4.9542, 5.0, 5.04439, 5.08746, 5.12928, 5.16993 +}; +static ERTS_INLINE double lookup_log2(Uint base) { + return lg2_lookup[base - 2]; +} /* - * for i in 2..64 do - * lg2_lookup[i-2] = log2(i) - * end - * How many bits are needed to store string of size n + * How many digits can fit into a signed int (Sint) for given base, we take + * one digit away just to be on the safer side (some corner cases). */ -const double lg2_lookup[] = { 1.0, 1.58496, 2, 2.32193, 2.58496, 2.80735, 3.0, - 3.16993, 3.32193, 3.45943, 3.58496, 3.70044, 3.80735, 3.90689, 4.0, - 4.08746, 4.16993, 4.24793, 4.32193, 4.39232, 4.45943, 4.52356, 4.58496, - 4.64386, 4.70044, 4.75489, 4.80735, 4.85798, 4.90689, 4.9542, 5.0, - 5.04439, 5.08746, 5.12928, 5.16993, 5.20945, 5.24793, 5.2854, 5.32193, - 5.35755, 5.39232, 5.42626, 5.45943, 5.49185, 5.52356, 5.55459, 5.58496, - 5.61471, 5.64386, 5.67243, 5.70044, 5.72792, 5.75489, 5.78136, 5.80735, - 5.83289, 5.85798, 5.88264, 5.90689, 5.93074, 5.9542, 5.97728, 6.0 }; +static const byte digits_per_sint_lookup[36-1] = { +#if (SIZEOF_VOID_P == 4) + /* Wo.Alpha formula: Table [Trunc[31 / log[2,n]]-1, {n, 2, 36}] */ + 30, 18, 14, 12, 10, 10, 9, 8, 8, 7, 7, 7, 7, 6, 6, 6, 6, 6, 6, 6, 5, 5, 5, + 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 4 +#elif (SIZEOF_VOID_P == 8) + /* Wo.Alpha formula: Table [Trunc[63 / log[2,n]]-1, {n, 2, 36}] */ + 62, 38, 30, 26, 23, 21, 20, 18, 17, 17, 16, 16, 15, 15, 14, 14, 14, 13, 13, + 13, 13, 12, 12, 12, 12, 12, 12, 11, 11, 11, 11, 11, 11, 11, 11 +#else + #error "Please produce a lookup table for the new architecture" +#endif +}; /* - * for i in 2..64 do - * d_base_exp_lookup[i-2] = 31 / lg2_lookup[i-2]; - * end - * How many characters can fit in 31 bits + * How many digits can fit into Erlang Small (SMALL_BITS-1) counting sign bit */ -const byte d_base_exp_lookup[] = { 31, 19, 15, 13, 11, 11, 10, 9, 9, 8, 8, 8, 8, - 7, 7, 7, 7, 7, 7, 7, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 5, 5, 5, - 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, - 5, 5 }; +static const byte digits_per_small_lookup[36-1] = { +#if (SIZEOF_VOID_P == 4) + /* Wo.Alpha formula: Table [Trunc[27 / log[2,n]]-1, {n, 2, 36}] */ + 27, 17, 13, 11, 10, 9, 9, 8, 8, 7, 7, 7, 7, 6, 6, 6, 6, 6, 6, 6, 6, 5, 5, 5, + 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5 +#elif (SIZEOF_VOID_P == 8) + /* Wo.Alpha formula: Table [Trunc[59 / log[2,n]]-1, {n, 2, 36}] */ + 59, 37, 29, 25, 22, 21, 19, 18, 17, 17, 16, 15, 15, 15, 14, 14, 14, 13, 13, + 13, 13, 13, 12, 12, 12, 12, 12, 12, 12, 11, 11, 11, 11, 11, 11 +#else + #error "Please produce a lookup table for the new architecture" +#endif +}; /* - * for i in 2..64 do - * d_base_base_lookup[i-2] = pow(i,d_base_exp_lookup[i-2]); - * end - * How much can the characters which fit in 31 bit represent + * Largest power of base which can be represented in a signed int (Sint). + * Calculated by base 2..36 to the power of corresponding element from + * digits_per_sint_lookup. */ -const Uint d_base_base_lookup[] = { 2147483648u, 1162261467u, 1073741824u, - 1220703125u, 362797056u, 1977326743u, 1073741824u, 387420489u, - 1000000000u, 214358881u, 429981696u, 815730721u, 1475789056u, - 170859375u, 268435456u, 410338673u, 612220032u, 893871739u, 1280000000u, - 1801088541u, 113379904u, 148035889u, 191102976u, 244140625u, 308915776u, - 387420489u, 481890304u, 594823321u, 729000000u, 887503681u, 1073741824u, - 1291467969u, 1544804416u, 1838265625u, 60466176u, 69343957u, 79235168u, - 90224199u, 102400000u, 115856201u, 130691232u, 147008443u, 164916224u, - 184528125u, 205962976u, 229345007u, 254803968u, 282475249u, 312500000u, - 345025251u, 380204032u, 418195493u, 459165024u, 503284375u, 550731776u, - 601692057u, 656356768u, 714924299u, 777600000u, 844596301u, 916132832u, - 992436543u, 1073741824u }; +static const Sint largest_power_of_base_lookup[36-1] = { +#if (SIZEOF_VOID_P == 4) + /* Wo.Alpha formula: Table [Pow[n, Trunc[31 / log[2,n]]-1], {n, 2, 36}] */ + 1073741824, 387420489, 268435456, 244140625, 60466176, 282475249, 134217728, + 43046721, 100000000, 19487171, 35831808, 62748517, 105413504, 11390625, + 16777216, 24137569, 34012224, 47045881, 64000000, 85766121, 5153632, + 6436343,7962624, 9765625, 11881376, 14348907, 17210368, 20511149, 24300000, + 28629151, 33554432, 39135393, 45435424, 52521875, 1679616 +#elif (SIZEOF_VOID_P == 8) + /* Wo.Alpha formula: Table [Pow[n, Trunc[63 / log[2,n]]-1], {n, 2, 36}] + * with LL added after each element manually */ + 4611686018427387904LL, 1350851717672992089LL, 1152921504606846976LL, + 1490116119384765625LL, 789730223053602816LL, 558545864083284007LL, + 1152921504606846976LL, 150094635296999121LL, 100000000000000000LL, + 505447028499293771LL, 184884258895036416LL, 665416609183179841LL, + 155568095557812224LL, 437893890380859375LL, 72057594037927936LL, + 168377826559400929LL, 374813367582081024LL, 42052983462257059LL, + 81920000000000000LL, 154472377739119461LL, 282810057883082752LL, + 21914624432020321LL, 36520347436056576LL, 59604644775390625LL, + 95428956661682176LL, 150094635296999121LL, 232218265089212416LL, + 12200509765705829LL, 17714700000000000LL, 25408476896404831LL, + 36028797018963968LL, 50542106513726817LL, 70188843638032384LL, + 96549157373046875LL, 131621703842267136LL +#else + #error "Please produce a lookup table for the new architecture" +#endif +}; Eterm erts_chars_to_integer(Process *BIF_P, char *bytes, Uint size, const int base) { @@ -2552,8 +2672,11 @@ Eterm erts_chars_to_integer(Process *BIF_P, char *bytes, int neg = 0; byte b; Eterm *hp, *hp_end; - int m; + Sint m; int lg2; + const Uint digits_per_small = get_digits_per_small(base); + const Uint digits_per_Sint = get_digits_per_signed_int(base); + const Sint largest_pow_of_base = get_largest_power_of_base(base); if (size == 0) goto bytebuf_to_integer_1_error; @@ -2568,55 +2691,69 @@ Eterm erts_chars_to_integer(Process *BIF_P, char *bytes, size--; } - if (size < SMALL_DIGITS && base <= 10) { - /* * - * Take shortcut if we know that all chars are '0' < b < '9' and - * fit in a small. This improves speed by about 10% over the generic - * small case. - * */ - while (size--) { - b = *bytes++; - - if (b < '0' || b > ('0'+base-1)) - goto bytebuf_to_integer_1_error; + /* Trim leading zeroes */ + if (size) { + while (*bytes == '0') { + bytes++; + size--; + if (!size) { + /* All zero! */ + res = make_small(0); + goto bytebuf_to_integer_1_done; + } + } + } - i = i * base + b - '0'; - } + if (size == 0) + goto bytebuf_to_integer_1_error; - if (neg) - i = -i; - res = make_small(i); - goto bytebuf_to_integer_1_done; - } + if (size < digits_per_small) { + if (base <= 10) { + /* * + * Take shortcut if we know that all chars are '0' < b < '9' and + * fit in a small. This improves speed by about 10% over the generic + * small case. + * */ + while (size--) { + b = *bytes++; - /* - * Calculate the maximum number of bits which will - * be needed to represent the binary - */ - lg2 = ((size+2)*LG2_LOOKUP(base)+1); + if (b < '0' || b > ('0'+base-1)) + goto bytebuf_to_integer_1_error; - if (lg2 < SMALL_BITS) { - /* Take shortcut if we know it will fit in a small. - * This improves speed by about 30%. - */ - while (size) { - b = *bytes++; - size--; + i = i * base + b - '0'; + } - if (IS_VALID_CHARACTER(b,base)) - goto bytebuf_to_integer_1_error; + if (neg) + i = -i; + res = make_small(i); + goto bytebuf_to_integer_1_done; + } - i = i * base + CHARACTER_FROM_BASE(b); + /* Take shortcut if we know it will fit in a small. + * This improves speed by about 30%. + */ + while (size) { + b = *bytes++; + size--; - } + if (c2int_is_invalid_char(b, base)) + goto bytebuf_to_integer_1_error; - if (neg) - i = -i; - res = make_small(i); - goto bytebuf_to_integer_1_done; + i = i * base + c2int_digit_from_base(b); + } + if (neg) + i = -i; + res = make_small(i); + goto bytebuf_to_integer_1_done; } + /* + * Calculate the maximum number of bits which will + * be needed to represent the binary + */ + lg2 = ((size+2)*lookup_log2(base)+1); + /* Start calculating bignum */ m = (lg2 + D_EXP-1)/D_EXP; m = BIG_NEED_SIZE(m); @@ -2624,8 +2761,8 @@ Eterm erts_chars_to_integer(Process *BIF_P, char *bytes, hp = HAlloc(BIF_P, m); hp_end = hp + m; - if ((i = (size % D_BASE_EXP(base))) == 0) - i = D_BASE_EXP(base); + if ((i = (size % digits_per_Sint)) == 0) + i = digits_per_Sint; n = size - i; m = 0; @@ -2633,43 +2770,40 @@ Eterm erts_chars_to_integer(Process *BIF_P, char *bytes, while (i--) { b = *bytes++; - if (IS_VALID_CHARACTER(b,base)) { + if (c2int_is_invalid_char(b,base)) { HRelease(BIF_P, hp_end, hp); goto bytebuf_to_integer_1_error; } - m = base * m + CHARACTER_FROM_BASE(b); + m = base * m + c2int_digit_from_base(b); } res = small_to_big(m, hp); while (n) { - i = D_BASE_EXP(base); - n -= D_BASE_EXP(base); + i = digits_per_Sint; + n -= digits_per_Sint; m = 0; while (i--) { b = *bytes++; - if (IS_VALID_CHARACTER(b,base)) { + if (c2int_is_invalid_char(b,base)) { HRelease(BIF_P, hp_end, hp); goto bytebuf_to_integer_1_error; } - m = base * m + CHARACTER_FROM_BASE(b); + m = base * m + c2int_digit_from_base(b); } if (is_small(res)) { res = small_to_big(signed_val(res), hp); } - res = big_times_small(res, D_BASE_BASE(base), hp); + res = big_times_small(res, largest_pow_of_base, hp); if (is_small(res)) { res = small_to_big(signed_val(res), hp); } res = big_plus_small(res, m, hp); } - if (is_big(res)) /* check if small */ - res = big_plus_small(res, 0, hp); /* includes conversion to small */ - if (neg) { if (is_small(res)) res = make_small(-signed_val(res)); @@ -2679,8 +2813,12 @@ Eterm erts_chars_to_integer(Process *BIF_P, char *bytes, } } - if (is_big(res)) { - hp += (big_arity(res) + 1); + if (is_not_small(res)) { + res = big_plus_small(res, 0, hp); /* includes conversion to small */ + + if (is_not_small(res)) { + hp += (big_arity(res) + 1); + } } HRelease(BIF_P, hp_end, hp); goto bytebuf_to_integer_1_done; @@ -2690,5 +2828,166 @@ bytebuf_to_integer_1_error: bytebuf_to_integer_1_done: return res; +} +/* Converts list of digits with given 'base' to integer sequentially. Returns + * result in 'integer_out', remaining tail goes to 'tail_out' and returns result + * code if the list was consumed fully or partially or there was an error + */ +LTI_result_t erts_list_to_integer(Process *BIF_P, Eterm orig_list, + const Uint base, + Eterm *integer_out, Eterm *tail_out) +{ + Sint i = 0; + Uint ui = 0; + int skip = 0; + int neg = 0; + Sint n = 0; + Sint m; + int lg2; + Eterm res; + Eterm lst = orig_list; + Eterm tail = lst; + int error_res = LTI_BAD_STRUCTURE; + const Uint digits_per_small = get_digits_per_small(base); + const Uint digits_per_Sint = get_digits_per_signed_int(base); + + if (is_nil(lst)) { + error_res = LTI_NO_INTEGER; + error: + *tail_out = tail; + *integer_out = make_small(0); + return error_res; + } + if (is_not_list(lst)) + goto error; + + /* if first char is a '-' then it is a negative integer */ + if (CAR(list_val(lst)) == make_small('-')) { + neg = 1; + skip = 1; + lst = CDR(list_val(lst)); + if (is_not_list(lst)) { + tail = lst; + error_res = LTI_NO_INTEGER; + goto error; + } + } else if (CAR(list_val(lst)) == make_small('+')) { + /* ignore plus */ + skip = 1; + lst = CDR(list_val(lst)); + if (is_not_list(lst)) { + tail = lst; + error_res = LTI_NO_INTEGER; + goto error; + } + } + + /* Calculate size and do type check */ + + while(1) { + byte ch; + if (is_not_small(CAR(list_val(lst)))) { + break; + } + ch = unsigned_val(CAR(list_val(lst))); + if (c2int_is_invalid_char(ch, base)) { + break; + } + ui = ui * base; + ui = ui + c2int_digit_from_base(ch); + n++; + lst = CDR(list_val(lst)); + if (is_nil(lst)) { + break; + } + if (is_not_list(lst)) { + break; + } + } + + tail = lst; + if (!n) { + error_res = LTI_NO_INTEGER; + goto error; + } + + + /* If length fits inside Sint then we know it's a small int. Else we + * must construct a bignum and let that routine do the checking + */ + + if (n <= digits_per_small) { /* It must be small */ + i = neg ? -(Sint)ui : (Sint)ui; + res = make_small(i); + } else { + const Sint largest_pow_of_base = get_largest_power_of_base(base); + Eterm *hp; + Eterm *hp_end; + + /* Convert from log_base to log2 using lookup table */ + lg2 = ((n+2)*lookup_log2(base)+1); + m = (lg2+D_EXP-1)/D_EXP; /* number of digits */ + m = BIG_NEED_SIZE(m); /* number of words + thing */ + + hp = HAlloc(BIF_P, m); + hp_end = hp + m; + + lst = orig_list; + if (skip) + lst = CDR(list_val(lst)); + + /* load first digits (at least one digit) */ + if ((i = (n % digits_per_Sint)) == 0) + i = digits_per_Sint; + n -= i; + m = 0; + while(i--) { + m *= base; + m += c2int_digit_from_base(unsigned_val(CAR(list_val(lst)))); + lst = CDR(list_val(lst)); + } + res = small_to_big(m, hp); /* load first digits */ + + while(n) { + i = digits_per_Sint; + n -= digits_per_Sint; + m = 0; + while(i--) { + m *= base; + m += c2int_digit_from_base(unsigned_val(CAR(list_val(lst)))); + lst = CDR(list_val(lst)); + } + if (is_small(res)) + res = small_to_big(signed_val(res), hp); + res = big_times_small(res, largest_pow_of_base, hp); + if (is_small(res)) + res = small_to_big(signed_val(res), hp); + res = big_plus_small(res, m, hp); + } + + if (neg) { + if (is_small(res)) + res = make_small(-signed_val(res)); + else { + Uint *big = big_val(res); /* point to thing */ + *big = bignum_header_neg(*big); + } + } + + if (is_not_small(res)) { + res = big_plus_small(res, 0, hp); /* includes conversion to small */ + + if (is_not_small(res)) { + hp += (big_arity(res)+1); + } + } + HRelease(BIF_P, hp_end, hp); + } + *integer_out = res; + *tail_out = tail; + if (tail != NIL) { + return LTI_SOME_INTEGER; + } + return LTI_ALL_INTEGER; } diff --git a/erts/emulator/beam/big.h b/erts/emulator/beam/big.h index d80111822e..ee51e5d313 100644 --- a/erts/emulator/beam/big.h +++ b/erts/emulator/beam/big.h @@ -3,16 +3,17 @@ * * Copyright Ericsson AB 1996-2014. All Rights Reserved. * - * The contents of this file are subject to the Erlang Public License, - * Version 1.1, (the "License"); you may not use this file except in - * compliance with the License. You should have received a copy of the - * Erlang Public License along with this software. If not, it can be - * retrieved online at http://www.erlang.org/. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Software distributed under the License is distributed on an "AS IS" - * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See - * the License for the specific language governing rights and limitations - * under the License. + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. * * %CopyrightEnd% */ @@ -34,7 +35,7 @@ typedef Uint ErtsDigit; -#if ((SIZEOF_VOID_P == 4) || HALFWORD_HEAP) && defined(SIZEOF_LONG_LONG) && (SIZEOF_LONG_LONG == 8) +#if (SIZEOF_VOID_P == 4) && defined(SIZEOF_LONG_LONG) && (SIZEOF_LONG_LONG == 8) /* Assume 32-bit machine with long long support */ typedef Uint64 ErtsDoubleDigit; typedef Uint16 ErtsHalfDigit; @@ -53,9 +54,6 @@ typedef Uint32 ErtsHalfDigit; #error "can not determine machine size" #endif -#define D_DECIMAL_EXP 9 -#define D_DECIMAL_BASE 1000000000 - typedef Uint dsize_t; /* Vector size type */ #define D_EXP (ERTS_SIZEOF_ETERM*8) @@ -85,25 +83,25 @@ typedef Uint dsize_t; /* Vector size type */ /* The heap size needed for a bignum */ #define BIG_NEED_SIZE(x) ((x) + 1) +#define BIG_NEED_FOR_BITS(bits) BIG_NEED_SIZE(((bits)-1)/D_EXP + 1) #define BIG_UINT_HEAP_SIZE (1 + 1) /* always, since sizeof(Uint) <= sizeof(Eterm) */ -#if HALFWORD_HEAP -#define BIG_UWORD_HEAP_SIZE(UW) (((UW) >> (sizeof(Uint) * 8)) ? 3 : 2) -#else #define BIG_UWORD_HEAP_SIZE(UW) BIG_UINT_HEAP_SIZE -#endif -#if defined(ARCH_32) || HALFWORD_HEAP +#if defined(ARCH_32) #define ERTS_UINT64_BIG_HEAP_SIZE__(X) \ ((X) >= (((Uint64) 1) << 32) ? (1 + 2) : (1 + 1)) #define ERTS_SINT64_HEAP_SIZE(X) \ (IS_SSMALL((X)) \ ? 0 \ - : ERTS_UINT64_BIG_HEAP_SIZE__((X) >= 0 ? (X) : -(X))) + : ERTS_UINT64_BIG_HEAP_SIZE__((X) >= 0 ? (X) : -(Uint64)(X))) #define ERTS_UINT64_HEAP_SIZE(X) \ (IS_USMALL(0, (X)) ? 0 : ERTS_UINT64_BIG_HEAP_SIZE__((X))) +#define ERTS_MAX_SINT64_HEAP_SIZE (1 + 2) +#define ERTS_MAX_UINT64_HEAP_SIZE (1 + 2) +#define ERTS_UINT64_ARRAY_TO_BIG_MAX_HEAP_SZ(LEN) (2*(LEN)+1) #else @@ -111,6 +109,9 @@ typedef Uint dsize_t; /* Vector size type */ (IS_SSMALL((X)) ? 0 : (1 + 1)) #define ERTS_UINT64_HEAP_SIZE(X) \ (IS_USMALL(0, (X)) ? 0 : (1 + 1)) +#define ERTS_MAX_SINT64_HEAP_SIZE (1 + 1) +#define ERTS_MAX_UINT64_HEAP_SIZE (1 + 1) +#define ERTS_UINT64_ARRAY_TO_BIG_MAX_HEAP_SZ(LEN) ((LEN)+1) #endif @@ -153,9 +154,11 @@ Eterm bytes_to_big(byte*, dsize_t, int, Eterm*); byte* big_to_bytes(Eterm, byte*); int term_to_Uint(Eterm, Uint*); +int term_to_Uint_mask(Eterm, Uint*); int term_to_UWord(Eterm, UWord*); int term_to_Sint(Eterm, Sint*); #if HAVE_INT64 +Eterm erts_uint64_array_to_big(Uint **, int, int, Uint64 *); int term_to_Uint64(Eterm, Uint64*); int term_to_Sint64(Eterm, Sint64*); #endif @@ -168,5 +171,15 @@ Eterm erts_sint64_to_big(Sint64, Eterm **); Eterm erts_chars_to_integer(Process *, char*, Uint, const int); +/* How list_to_integer classifies the input, was it even a string? */ +typedef enum { + LTI_BAD_STRUCTURE = 0, + LTI_NO_INTEGER = 1, + LTI_SOME_INTEGER = 2, + LTI_ALL_INTEGER = 3 +} LTI_result_t; + +LTI_result_t erts_list_to_integer(Process *BIF_P, Eterm orig_list, + const Uint base, + Eterm *integer_out, Eterm *tail_out); #endif - diff --git a/erts/emulator/beam/binary.c b/erts/emulator/beam/binary.c index c7926f18af..3fd284b589 100644 --- a/erts/emulator/beam/binary.c +++ b/erts/emulator/beam/binary.c @@ -3,16 +3,17 @@ * * Copyright Ericsson AB 1996-2013. All Rights Reserved. * - * The contents of this file are subject to the Erlang Public License, - * Version 1.1, (the "License"); you may not use this file except in - * compliance with the License. You should have received a copy of the - * Erlang Public License along with this software. If not, it can be - * retrieved online at http://www.erlang.org/. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Software distributed under the License is distributed on an "AS IS" - * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See - * the License for the specific language governing rights and limitations - * under the License. + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. * * %CopyrightEnd% */ @@ -26,17 +27,21 @@ #include "global.h" #include "erl_process.h" #include "error.h" +#define ERL_WANT_HIPE_BIF_WRAPPER__ #include "bif.h" +#undef ERL_WANT_HIPE_BIF_WRAPPER__ #include "big.h" #include "erl_binary.h" #include "erl_bits.h" -#ifdef DEBUG -static int list_to_bitstr_buf(Eterm obj, char* buf, Uint len); -#else -static int list_to_bitstr_buf(Eterm obj, char* buf); -#endif -static int bitstr_list_len(Eterm obj, Uint* num_bytes); +#define L2B_B2L_MIN_EXEC_REDS (CONTEXT_REDS/4) +#define L2B_B2L_RESCHED_REDS (CONTEXT_REDS/40) + +static Export binary_to_list_continue_export; +static Export list_to_binary_continue_export; + +static BIF_RETTYPE binary_to_list_continue(BIF_ALIST_1); +static BIF_RETTYPE list_to_binary_continue(BIF_ALIST_1); void erts_init_binary(void) @@ -45,10 +50,19 @@ erts_init_binary(void) if ((((UWord) &((Binary *) 0)->orig_bytes[0]) % ((UWord) 8)) != 0) { /* I assume that any compiler should be able to optimize this away. If not, this test is not very expensive... */ - erl_exit(ERTS_ABORT_EXIT, + erts_exit(ERTS_ABORT_EXIT, "Internal error: Address of orig_bytes[0] of a Binary" " is *not* 8-byte aligned\n"); } + + erts_init_trap_export(&binary_to_list_continue_export, + am_erts_internal, am_binary_to_list_continue, 1, + &binary_to_list_continue); + + erts_init_trap_export(&list_to_binary_continue_export, + am_erts_internal, am_list_to_binary_continue, 1, + &list_to_binary_continue); + } /* @@ -75,8 +89,6 @@ new_binary(Process *p, byte *buf, Uint len) * Allocate the binary struct itself. */ bptr = erts_bin_nrml_alloc(len); - bptr->flags = 0; - bptr->orig_size = len; erts_refc_init(&bptr->refc, 1); if (buf != NULL) { sys_memcpy(bptr->orig_bytes, buf, len); @@ -114,8 +126,6 @@ Eterm erts_new_mso_binary(Process *p, byte *buf, int len) * Allocate the binary struct itself. */ bptr = erts_bin_nrml_alloc(len); - bptr->flags = 0; - bptr->orig_size = len; erts_refc_init(&bptr->refc, 1); if (buf != NULL) { sys_memcpy(bptr->orig_bytes, buf, len); @@ -169,7 +179,6 @@ erts_realloc_binary(Eterm bin, size_t size) } else { /* REFC */ ProcBin* pb = (ProcBin *) bval; Binary* newbin = erts_bin_realloc(pb->val, size); - newbin->orig_size = size; pb->val = newbin; pb->size = size; pb->bytes = (byte*) newbin->orig_bytes; @@ -333,6 +342,132 @@ BIF_RETTYPE integer_to_binary_1(BIF_ALIST_1) BIF_RET(res); } +#define ERTS_B2L_BYTES_PER_REDUCTION 256 + +typedef struct { + Eterm res; + Eterm *hp; +#ifdef DEBUG + Eterm *hp_end; +#endif + byte *bytes; + Uint size; + Uint bitoffs; +} ErtsB2LState; + +static void b2l_state_destructor(Binary *mbp) +{ + ASSERT(ERTS_MAGIC_BIN_DESTRUCTOR(mbp) == b2l_state_destructor); +} + +static BIF_RETTYPE +binary_to_list_chunk(Process *c_p, + Eterm mb_eterm, + ErtsB2LState* sp, + int reds_left, + int gc_disabled) +{ + BIF_RETTYPE ret; + int bump_reds; + Uint size; + byte *bytes; + + size = (reds_left + 1)*ERTS_B2L_BYTES_PER_REDUCTION; + if (size > sp->size) + size = sp->size; + bytes = sp->bytes + (sp->size - size); + + bump_reds = (size - 1)/ERTS_B2L_BYTES_PER_REDUCTION + 1; + BUMP_REDS(c_p, bump_reds); + + ASSERT(is_list(sp->res) || is_nil(sp->res)); + + sp->res = erts_bin_bytes_to_list(sp->res, + sp->hp, + bytes, + size, + sp->bitoffs); + sp->size -= size; + sp->hp += 2*size; + + if (sp->size > 0) { + + if (!gc_disabled) + erts_set_gc_state(c_p, 0); + + ASSERT(c_p->flags & F_DISABLE_GC); + ASSERT(is_value(mb_eterm)); + ERTS_BIF_PREP_TRAP1(ret, + &binary_to_list_continue_export, + c_p, + mb_eterm); + } + else { + + ASSERT(sp->hp == sp->hp_end); + ASSERT(sp->size == 0); + + if (!gc_disabled || !erts_set_gc_state(c_p, 1)) + ERTS_BIF_PREP_RET(ret, sp->res); + else + ERTS_BIF_PREP_YIELD_RETURN(ret, c_p, sp->res); + ASSERT(!(c_p->flags & F_DISABLE_GC)); + } + + return ret; +} + +static ERTS_INLINE BIF_RETTYPE +binary_to_list(Process *c_p, Eterm *hp, Eterm tail, byte *bytes, + Uint size, Uint bitoffs, int reds_left, int one_chunk) +{ + if (one_chunk) { + Eterm res; + BIF_RETTYPE ret; + int bump_reds = (size - 1)/ERTS_B2L_BYTES_PER_REDUCTION + 1; + BUMP_REDS(c_p, bump_reds); + res = erts_bin_bytes_to_list(tail, hp, bytes, size, bitoffs); + ERTS_BIF_PREP_RET(ret, res); + return ret; + } + else { + Binary *mbp = erts_create_magic_binary(sizeof(ErtsB2LState), + b2l_state_destructor); + ErtsB2LState *sp = ERTS_MAGIC_BIN_DATA(mbp); + Eterm mb; + + sp->res = tail; + sp->hp = hp; +#ifdef DEBUG + sp->hp_end = sp->hp + 2*size; +#endif + sp->bytes = bytes; + sp->size = size; + sp->bitoffs = bitoffs; + + hp = HAlloc(c_p, PROC_BIN_SIZE); + mb = erts_mk_magic_binary_term(&hp, &MSO(c_p), mbp); + return binary_to_list_chunk(c_p, mb, sp, reds_left, 0); + } +} + +static BIF_RETTYPE binary_to_list_continue(BIF_ALIST_1) +{ + Binary *mbp = ((ProcBin *) binary_val(BIF_ARG_1))->val; + + ASSERT(ERTS_MAGIC_BIN_DESTRUCTOR(mbp) == b2l_state_destructor); + + ASSERT(BIF_P->flags & F_DISABLE_GC); + + return binary_to_list_chunk(BIF_P, + BIF_ARG_1, + (ErtsB2LState*) ERTS_MAGIC_BIN_DATA(mbp), + ERTS_BIF_REDS_LEFT(BIF_P), + 1); +} + +HIPE_WRAPPER_BIF_DISABLE_GC(binary_to_list, 1) + BIF_RETTYPE binary_to_list_1(BIF_ALIST_1) { Eterm real_bin; @@ -340,11 +475,29 @@ BIF_RETTYPE binary_to_list_1(BIF_ALIST_1) Uint size; Uint bitsize; Uint bitoffs; + int reds_left; + int one_chunk; if (is_not_binary(BIF_ARG_1)) { goto error; } + size = binary_size(BIF_ARG_1); + reds_left = ERTS_BIF_REDS_LEFT(BIF_P); + one_chunk = size < reds_left*ERTS_B2L_BYTES_PER_REDUCTION; + if (!one_chunk) { + if (size < L2B_B2L_MIN_EXEC_REDS*ERTS_B2L_BYTES_PER_REDUCTION) { + if (reds_left <= L2B_B2L_RESCHED_REDS) { + /* Yield and do it with full context reds... */ + ERTS_BIF_YIELD1(bif_export[BIF_binary_to_list_1], + BIF_P, BIF_ARG_1); + } + /* Allow a bit more reductions... */ + one_chunk = 1; + reds_left = L2B_B2L_MIN_EXEC_REDS; + } + } + ERTS_GET_REAL_BIN(BIF_ARG_1, real_bin, offset, bitoffs, bitsize); if (bitsize != 0) { goto error; @@ -354,14 +507,16 @@ BIF_RETTYPE binary_to_list_1(BIF_ALIST_1) } else { Eterm* hp = HAlloc(BIF_P, 2 * size); byte* bytes = binary_bytes(real_bin)+offset; - - BIF_RET(erts_bin_bytes_to_list(NIL, hp, bytes, size, bitoffs)); + return binary_to_list(BIF_P, hp, NIL, bytes, size, + bitoffs, reds_left, one_chunk); } error: BIF_ERROR(BIF_P, BADARG); } +HIPE_WRAPPER_BIF_DISABLE_GC(binary_to_list, 3) + BIF_RETTYPE binary_to_list_3(BIF_ALIST_3) { byte* bytes; @@ -372,6 +527,8 @@ BIF_RETTYPE binary_to_list_3(BIF_ALIST_3) Uint start; Uint stop; Eterm* hp; + int reds_left; + int one_chunk; if (is_not_binary(BIF_ARG_1)) { goto error; @@ -380,6 +537,21 @@ BIF_RETTYPE binary_to_list_3(BIF_ALIST_3) goto error; } size = binary_size(BIF_ARG_1); + reds_left = ERTS_BIF_REDS_LEFT(BIF_P); + one_chunk = size < reds_left*ERTS_B2L_BYTES_PER_REDUCTION; + if (!one_chunk) { + if (size < L2B_B2L_MIN_EXEC_REDS*ERTS_B2L_BYTES_PER_REDUCTION) { + if (reds_left <= L2B_B2L_RESCHED_REDS) { + /* Yield and do it with full context reds... */ + ERTS_BIF_YIELD3(bif_export[BIF_binary_to_list_3], + BIF_P, BIF_ARG_1, BIF_ARG_2, BIF_ARG_3); + } + /* Allow a bit more reductions... */ + one_chunk = 1; + reds_left = L2B_B2L_MIN_EXEC_REDS; + } + } + ERTS_GET_BINARY_BYTES(BIF_ARG_1, bytes, bitoffs, bitsize); if (start < 1 || start > size || stop < 1 || stop > size || stop < start ) { @@ -387,12 +559,14 @@ BIF_RETTYPE binary_to_list_3(BIF_ALIST_3) } i = stop-start+1; hp = HAlloc(BIF_P, 2*i); - BIF_RET(erts_bin_bytes_to_list(NIL, hp, bytes+start-1, i, bitoffs)); - + return binary_to_list(BIF_P, hp, NIL, bytes+start-1, i, + bitoffs, reds_left, one_chunk); error: BIF_ERROR(BIF_P, BADARG); } +HIPE_WRAPPER_BIF_DISABLE_GC(bitstring_to_list, 1) + BIF_RETTYPE bitstring_to_list_1(BIF_ALIST_1) { Eterm real_bin; @@ -403,11 +577,27 @@ BIF_RETTYPE bitstring_to_list_1(BIF_ALIST_1) byte* bytes; Eterm previous = NIL; Eterm* hp; + int reds_left; + int one_chunk; if (is_not_binary(BIF_ARG_1)) { BIF_ERROR(BIF_P, BADARG); } size = binary_size(BIF_ARG_1); + reds_left = ERTS_BIF_REDS_LEFT(BIF_P); + one_chunk = size < reds_left*ERTS_B2L_BYTES_PER_REDUCTION; + if (!one_chunk) { + if (size < L2B_B2L_MIN_EXEC_REDS*ERTS_B2L_BYTES_PER_REDUCTION) { + if (reds_left <= L2B_B2L_RESCHED_REDS) { + /* Yield and do it with full context reds... */ + ERTS_BIF_YIELD1(bif_export[BIF_bitstring_to_list_1], + BIF_P, BIF_ARG_1); + } + /* Allow a bit more reductions... */ + one_chunk = 1; + reds_left = L2B_B2L_MIN_EXEC_REDS; + } + } ERTS_GET_REAL_BIN(BIF_ARG_1, real_bin, offset, bitoffs, bitsize); bytes = binary_bytes(real_bin)+offset; if (bitsize == 0) { @@ -431,124 +621,452 @@ BIF_RETTYPE bitstring_to_list_1(BIF_ALIST_1) previous = CONS(hp, make_binary(last), previous); hp += 2; } - BIF_RET(erts_bin_bytes_to_list(previous, hp, bytes, size, bitoffs)); + + return binary_to_list(BIF_P, hp, previous, bytes, size, + bitoffs, reds_left, one_chunk); } /* Turn a possibly deep list of ints (and binaries) into */ /* One large binary object */ -/* - * This bif also exists in the binary module, under the name - * binary:list_to_bin/1, why it's divided into interface and - * implementation. Also the backend for iolist_to_binary_1. - */ +typedef enum { + ERTS_L2B_OK, + ERTS_L2B_YIELD, + ERTS_L2B_TYPE_ERROR, + ERTS_L2B_OVERFLOW_ERROR +} ErtsL2BResult; -BIF_RETTYPE erts_list_to_binary_bif(Process *p, Eterm arg) -{ +#define ERTS_L2B_STATE_INITER(C_P, ARG, BIF, SZFunc, TBufFunc) \ + {ERTS_IOLIST2BUF_STATE_INITER((C_P), (ARG)), \ + (ARG), THE_NON_VALUE, (BIF), (SZFunc), (TBufFunc)} + +#define ERTS_L2B_STATE_MOVE(TO, FROM) \ + sys_memcpy((void *) (TO), (void *) (FROM), sizeof(ErtsL2BState)) + +typedef struct ErtsL2BState_ ErtsL2BState; + +struct ErtsL2BState_ { + ErtsIOList2BufState buf; + Eterm arg; Eterm bin; - Eterm h,t; - ErlDrvSizeT size; - byte* bytes; -#ifdef DEBUG - ErlDrvSizeT offset; -#endif + Export *bif; + int (*iolist_to_buf_size)(ErtsIOListState *); + ErlDrvSizeT (*iolist_to_buf)(ErtsIOList2BufState *); +}; + +static ERTS_INLINE ErtsL2BResult +list_to_binary_engine(ErtsL2BState *sp) +{ + ErlDrvSizeT res; + Process *c_p = sp->buf.iolist.c_p; + + /* + * have_size == 0 while sp->iolist_to_buf_size() + * has not finished the calculation. + */ + + if (!sp->buf.iolist.have_size) { + switch (sp->iolist_to_buf_size(&sp->buf.iolist)) { + case ERTS_IOLIST_YIELD: + return ERTS_L2B_YIELD; + case ERTS_IOLIST_OVERFLOW: + return ERTS_L2B_OVERFLOW_ERROR; + case ERTS_IOLIST_TYPE: + return ERTS_L2B_TYPE_ERROR; + case ERTS_IOLIST_OK: + break; + default: + ASSERT(0); + break; + } + + ASSERT(sp->buf.iolist.have_size); + + /* + * Size calculated... Setup state for + * sp->iolist_to_buf_*() + */ + + sp->bin = new_binary(c_p, + (byte *) NULL, + sp->buf.iolist.size); + + if (sp->buf.iolist.size == 0) + return ERTS_L2B_OK; - if (is_nil(arg)) { - BIF_RET(new_binary(p,(byte*)"",0)); + sp->buf.buf = (char *) binary_bytes(sp->bin); + sp->buf.len = sp->buf.iolist.size; + sp->buf.iolist.obj = sp->arg; + + if (sp->buf.iolist.reds_left <= 0) { + BUMP_ALL_REDS(c_p); + return ERTS_L2B_YIELD; + } } - if (is_not_list(arg)) { - goto error; + + ASSERT(sp->buf.iolist.size != 0); + ASSERT(is_value(sp->bin)); + ASSERT(sp->buf.buf); + + res = sp->iolist_to_buf(&sp->buf); + + if (!ERTS_IOLIST_TO_BUF_FAILED(res)) { + ASSERT(res == 0); + return ERTS_L2B_OK; } - /* check for [binary()] case */ - h = CAR(list_val(arg)); - t = CDR(list_val(arg)); - if (is_binary(h) && is_nil(t) && !( - HEADER_SUB_BIN == *(binary_val(h)) && ( - ((ErlSubBin *)binary_val(h))->bitoffs != 0 || - ((ErlSubBin *)binary_val(h))->bitsize != 0 - ))) { - return h; - } - switch (erts_iolist_size(arg, &size)) { - case ERTS_IOLIST_OVERFLOW: BIF_ERROR(p, SYSTEM_LIMIT); - case ERTS_IOLIST_TYPE: goto error; - default: ; - } - bin = new_binary(p, (byte *)NULL, size); - bytes = binary_bytes(bin); -#ifdef DEBUG - offset = -#endif - erts_iolist_to_buf(arg, (char*) bytes, size); - ASSERT(offset == 0); - BIF_RET(bin); + switch (res) { + case ERTS_IOLIST_TO_BUF_YIELD: + return ERTS_L2B_YIELD; + case ERTS_IOLIST_TO_BUF_OVERFLOW: + return ERTS_L2B_OVERFLOW_ERROR; + case ERTS_IOLIST_TO_BUF_TYPE_ERROR: + return ERTS_L2B_TYPE_ERROR; + default: + ERTS_INTERNAL_ERROR("Invalid return value from iolist_to_buf_yielding()"); + return ERTS_L2B_TYPE_ERROR; + } +} + +static void +l2b_state_destructor(Binary *mbp) +{ + ErtsL2BState *sp = ERTS_MAGIC_BIN_DATA(mbp); + ASSERT(ERTS_MAGIC_BIN_DESTRUCTOR(mbp) == l2b_state_destructor); + DESTROY_SAVED_ESTACK(&sp->buf.iolist.estack); +} + +static ERTS_INLINE Eterm +l2b_final_touch(Process *c_p, ErtsL2BState *sp) +{ + Eterm *hp; + ErlSubBin* sbin; + if (sp->buf.offset == 0) + return sp->bin; + + hp = HAlloc(c_p, ERL_SUB_BIN_SIZE); + ASSERT(sp->buf.offset > 0); + sbin = (ErlSubBin *) hp; + sbin->thing_word = HEADER_SUB_BIN; + sbin->size = sp->buf.iolist.size-1; + sbin->offs = 0; + sbin->orig = sp->bin; + sbin->bitoffs = 0; + sbin->bitsize = sp->buf.offset; + sbin->is_writable = 0; + return make_binary(sbin); +} + +static BIF_RETTYPE +list_to_binary_chunk(Eterm mb_eterm, + ErtsL2BState* sp, + int reds_left, + int gc_disabled) +{ + Eterm err = BADARG; + BIF_RETTYPE ret; + Process *c_p = sp->buf.iolist.c_p; + + sp->buf.iolist.reds_left = reds_left; - error: - BIF_ERROR(p, BADARG); + switch (list_to_binary_engine(sp)) { + + case ERTS_L2B_OK: { + Eterm result = l2b_final_touch(c_p, sp); + if (!gc_disabled || !erts_set_gc_state(c_p, 1)) + ERTS_BIF_PREP_RET(ret, result); + else + ERTS_BIF_PREP_YIELD_RETURN(ret, c_p, result); + ASSERT(!(c_p->flags & F_DISABLE_GC)); + break; + } + case ERTS_L2B_YIELD: + if (!gc_disabled) { + /* first yield... */ + Eterm *hp; + Binary *mbp = erts_create_magic_binary(sizeof(ErtsL2BState), + l2b_state_destructor); + ErtsL2BState *new_sp = ERTS_MAGIC_BIN_DATA(mbp); + + ERTS_L2B_STATE_MOVE(new_sp, sp); + sp = new_sp; + + hp = HAlloc(c_p, PROC_BIN_SIZE); + mb_eterm = erts_mk_magic_binary_term(&hp, &MSO(c_p), mbp); + + ASSERT(is_value(mb_eterm)); + + erts_set_gc_state(c_p, 0); + } + + ASSERT(c_p->flags & F_DISABLE_GC); + + ERTS_BIF_PREP_TRAP1(ret, + &list_to_binary_continue_export, + c_p, + mb_eterm); + break; + + case ERTS_L2B_OVERFLOW_ERROR: + err = SYSTEM_LIMIT; + /* fall through */ + + case ERTS_L2B_TYPE_ERROR: + if (!gc_disabled) + ERTS_BIF_PREP_ERROR(ret, c_p, err); + else { + if (erts_set_gc_state(c_p, 1)) + ERTS_VBUMP_ALL_REDS(c_p); + + ERTS_BIF_PREP_ERROR_TRAPPED1(ret, + c_p, + err, + sp->bif, + sp->arg); + } + + ASSERT(!(c_p->flags & F_DISABLE_GC)); + break; + + default: + ERTS_INTERNAL_ERROR("Invalid return value from list_to_binary_engine()"); + ERTS_BIF_PREP_ERROR(ret,c_p, EXC_INTERNAL_ERROR); + break; + } + return ret; +} + +static BIF_RETTYPE list_to_binary_continue(BIF_ALIST_1) +{ + Binary *mbp = ((ProcBin *) binary_val(BIF_ARG_1))->val; + ASSERT(ERTS_MAGIC_BIN_DESTRUCTOR(mbp) == l2b_state_destructor); + + ASSERT(BIF_P->flags & F_DISABLE_GC); + + return list_to_binary_chunk(BIF_ARG_1, + ERTS_MAGIC_BIN_DATA(mbp), + ERTS_BIF_REDS_LEFT(BIF_P), + 1); } +BIF_RETTYPE erts_list_to_binary_bif(Process *c_p, Eterm arg, Export *bif) +{ + int orig_reds_left = ERTS_BIF_REDS_LEFT(c_p); + BIF_RETTYPE ret; + + if (orig_reds_left < L2B_B2L_MIN_EXEC_REDS) { + if (orig_reds_left <= L2B_B2L_RESCHED_REDS) { + /* Yield and do it with full context reds... */ + ERTS_BIF_PREP_YIELD1(ret, bif, c_p, arg); + return ret; + } + /* Allow a bit more reductions... */ + orig_reds_left = L2B_B2L_MIN_EXEC_REDS; + } + + if (is_nil(arg)) + ERTS_BIF_PREP_RET(ret, new_binary(c_p, (byte *) "", 0)); + else if (is_not_list(arg)) + ERTS_BIF_PREP_ERROR(ret, c_p, BADARG); + else { + /* check for [binary()] case */ + Eterm h = CAR(list_val(arg)); + Eterm t = CDR(list_val(arg)); + if (is_binary(h) + && is_nil(t) + && !(HEADER_SUB_BIN == *(binary_val(h)) + && (((ErlSubBin *)binary_val(h))->bitoffs != 0 + || ((ErlSubBin *)binary_val(h))->bitsize != 0))) { + ERTS_BIF_PREP_RET(ret, h); + } + else { + ErtsL2BState state = ERTS_L2B_STATE_INITER(c_p, + arg, + bif, + erts_iolist_size_yielding, + erts_iolist_to_buf_yielding); + + /* + * First try to do it all at once without having to use + * yielding iolist_to_buf(). + */ + state.buf.iolist.reds_left = orig_reds_left; + switch (erts_iolist_size_yielding(&state.buf.iolist)) { + case ERTS_IOLIST_OK: { + ErlDrvSizeT size = state.buf.iolist.size; + Eterm bin; + char *buf; + + if (size == 0) { + ERTS_BIF_PREP_RET(ret, new_binary(c_p, (byte *) NULL, 0)); + break; /* done */ + } + + bin = new_binary(c_p, (byte *) NULL, size); + buf = (char *) binary_bytes(bin); + + if (size < ERTS_IOLIST_TO_BUF_BYTES_PER_RED*CONTEXT_REDS) { + /* An (over) estimation of reductions needed */ + int reds_left = state.buf.iolist.reds_left; + int to_buf_reds = orig_reds_left - reds_left; + to_buf_reds += size/ERTS_IOLIST_TO_BUF_BYTES_PER_RED; + if (to_buf_reds <= reds_left) { + ErlDrvSizeT res; + + res = erts_iolist_to_buf(arg, buf, size); + if (res == 0) { + BUMP_REDS(c_p, to_buf_reds); + ERTS_BIF_PREP_RET(ret, bin); + break; /* done */ + } + if (!ERTS_IOLIST_TO_BUF_FAILED(res)) + ERTS_INTERNAL_ERROR("iolist_size/iolist_to_buf missmatch"); + if (res == ERTS_IOLIST_TO_BUF_OVERFLOW) + goto overflow; + goto type_error; + } + } + /* + * Since size has been computed list_to_binary_chunk() expects + * state prepared for iolist_to_buf. + */ + state.bin = bin; + state.buf.buf = buf; + state.buf.len = size; + state.buf.iolist.obj = arg; + /* Fall through... */ + } + case ERTS_IOLIST_YIELD: + ret = list_to_binary_chunk(THE_NON_VALUE, + &state, + state.buf.iolist.reds_left, + 0); + break; + case ERTS_IOLIST_OVERFLOW: + overflow: + ERTS_BIF_PREP_ERROR(ret, c_p, SYSTEM_LIMIT); + break; + case ERTS_IOLIST_TYPE: + type_error: + default: + ERTS_BIF_PREP_ERROR(ret, c_p, BADARG); + break; + } + } + } + return ret; +} + +HIPE_WRAPPER_BIF_DISABLE_GC(list_to_binary, 1) + BIF_RETTYPE list_to_binary_1(BIF_ALIST_1) { - return erts_list_to_binary_bif(BIF_P, BIF_ARG_1); + return erts_list_to_binary_bif(BIF_P, BIF_ARG_1, bif_export[BIF_list_to_binary_1]); } -/* Turn a possibly deep list of ints (and binaries) into */ -/* One large binary object */ +HIPE_WRAPPER_BIF_DISABLE_GC(iolist_to_binary, 1) BIF_RETTYPE iolist_to_binary_1(BIF_ALIST_1) { if (is_binary(BIF_ARG_1)) { BIF_RET(BIF_ARG_1); } - return erts_list_to_binary_bif(BIF_P, BIF_ARG_1); + return erts_list_to_binary_bif(BIF_P, BIF_ARG_1, bif_export[BIF_iolist_to_binary_1]); } +static int bitstr_list_len(ErtsIOListState *); +static ErlDrvSizeT list_to_bitstr_buf_yielding(ErtsIOList2BufState *); +static ErlDrvSizeT list_to_bitstr_buf_not_yielding(ErtsIOList2BufState *); + +HIPE_WRAPPER_BIF_DISABLE_GC(list_to_bitstring, 1) + BIF_RETTYPE list_to_bitstring_1(BIF_ALIST_1) { - Eterm bin; - Uint sz; - int offset; - byte* bytes; - ErlSubBin* sb1; - Eterm* hp; - - if (is_nil(BIF_ARG_1)) { - BIF_RET(new_binary(BIF_P,(byte*)"",0)); - } - if (is_not_list(BIF_ARG_1)) { - error: - BIF_ERROR(BIF_P, BADARG); - } - switch (bitstr_list_len(BIF_ARG_1, &sz)) { - case ERTS_IOLIST_TYPE: - goto error; - case ERTS_IOLIST_OVERFLOW: - BIF_ERROR(BIF_P, SYSTEM_LIMIT); - } - bin = new_binary(BIF_P, (byte *)NULL, sz); - bytes = binary_bytes(bin); -#ifdef DEBUG - offset = list_to_bitstr_buf(BIF_ARG_1, (char*) bytes, sz); -#else - offset = list_to_bitstr_buf(BIF_ARG_1, (char*) bytes); -#endif - ASSERT(offset >= 0); - if (offset > 0) { - hp = HAlloc(BIF_P, ERL_SUB_BIN_SIZE); - sb1 = (ErlSubBin *) hp; - sb1->thing_word = HEADER_SUB_BIN; - sb1->size = sz-1; - sb1->offs = 0; - sb1->orig = bin; - sb1->bitoffs = 0; - sb1->bitsize = offset; - sb1->is_writable = 0; - bin = make_binary(sb1); + BIF_RETTYPE ret; + + if (is_nil(BIF_ARG_1)) + ERTS_BIF_PREP_RET(ret, new_binary(BIF_P, (byte *) "", 0)); + else if (is_not_list(BIF_ARG_1)) + ERTS_BIF_PREP_ERROR(ret, BIF_P, BADARG); + else { + /* check for [bitstring()] case */ + Eterm h = CAR(list_val(BIF_ARG_1)); + Eterm t = CDR(list_val(BIF_ARG_1)); + if (is_binary(h) && is_nil(t)) { + ERTS_BIF_PREP_RET(ret, h); + } + else { + ErtsL2BState state = ERTS_L2B_STATE_INITER(BIF_P, + BIF_ARG_1, + bif_export[BIF_list_to_bitstring_1], + bitstr_list_len, + list_to_bitstr_buf_yielding); + int orig_reds_left = ERTS_BIF_REDS_LEFT(BIF_P); + + /* + * First try to do it all at once without having to use + * yielding list_to_bitstr_buf(). + */ + state.buf.iolist.reds_left = orig_reds_left; + switch (bitstr_list_len(&state.buf.iolist)) { + case ERTS_IOLIST_OK: { + ErlDrvSizeT size = state.buf.iolist.size; + + state.bin = new_binary(BIF_P, (byte *) NULL, size); + state.buf.buf = (char *) binary_bytes(state.bin); + state.buf.len = size; + state.buf.iolist.obj = BIF_ARG_1; + + if (size < ERTS_IOLIST_TO_BUF_BYTES_PER_RED*CONTEXT_REDS) { + /* An (over) estimation of reductions needed */ + int reds_left = state.buf.iolist.reds_left; + int to_buf_reds = orig_reds_left - reds_left; + to_buf_reds += size/ERTS_IOLIST_TO_BUF_BYTES_PER_RED; + if (to_buf_reds <= reds_left) { + ErlDrvSizeT res; + + res = list_to_bitstr_buf_not_yielding(&state.buf); + if (res == 0) { + Eterm res_bin = l2b_final_touch(BIF_P, &state); + BUMP_REDS(BIF_P, to_buf_reds); + ERTS_BIF_PREP_RET(ret, res_bin); + break; /* done */ + } + if (!ERTS_IOLIST_TO_BUF_FAILED(res)) + ERTS_INTERNAL_ERROR("iolist_size/iolist_to_buf missmatch"); + if (res == ERTS_IOLIST_TO_BUF_OVERFLOW) + goto overflow; + goto type_error; + } + } + /* + * Since size has been computed list_to_binary_chunk() expects + * the state prepared for list_to_bitstr_buf. + */ + + /* Fall through... */ + } + case ERTS_IOLIST_YIELD: + ret = list_to_binary_chunk(THE_NON_VALUE, + &state, + state.buf.iolist.reds_left, + 0); + break; + case ERTS_IOLIST_OVERFLOW: + overflow: + ERTS_BIF_PREP_ERROR(ret, BIF_P, SYSTEM_LIMIT); + break; + case ERTS_IOLIST_TYPE: + type_error: + default: + ERTS_BIF_PREP_ERROR(ret, BIF_P, BADARG); + break; + } + } } - - BIF_RET(bin); + + return ret; } BIF_RETTYPE split_binary_2(BIF_ALIST_2) @@ -605,123 +1123,353 @@ BIF_RETTYPE split_binary_2(BIF_ALIST_2) * Local functions. */ +static int +list_to_bitstr_buf_bcopy(ErtsIOList2BufState *state, Eterm obj, int *yield_countp); + /* * The input list is assumed to be type-correct and the buffer is * assumed to be of sufficient size. Those assumptions are verified in * the DEBUG-built emulator. */ -static int +static ErlDrvSizeT +list_to_bitstr_buf(int yield_support, ErtsIOList2BufState *state) +{ + +#undef LIST_TO_BITSTR_BUF_BCOPY_DBG +#undef LIST_TO_BITSTR_BUF_BCOPY #ifdef DEBUG -list_to_bitstr_buf(Eterm obj, char* buf, Uint len) +#define LIST_TO_BITSTR_BUF_BCOPY_DBG \ + len -= size + (offset>7); #else -list_to_bitstr_buf(Eterm obj, char* buf) +#define LIST_TO_BITSTR_BUF_BCOPY_DBG #endif -{ - Eterm* objp; - int offset = 0; +#define LIST_TO_BITSTR_BUF_BCOPY(CONSP) \ + do { \ + byte* bptr; \ + Uint bitsize; \ + Uint bitoffs; \ + Uint num_bits; \ + size_t size = binary_size(obj); \ + if (yield_support) { \ + size_t max_size = ERTS_IOLIST_TO_BUF_BYTES_PER_YIELD_COUNT; \ + if (yield_count > 0) \ + max_size *= yield_count+1; \ + if (size > max_size) { \ + state->objp = CONSP; \ + goto L_bcopy_yield; \ + } \ + if (size >= ERTS_IOLIST_TO_BUF_BYTES_PER_YIELD_COUNT) { \ + int cost = (int) size; \ + cost /= ERTS_IOLIST_TO_BUF_BYTES_PER_YIELD_COUNT; \ + yield_count -= cost; \ + } \ + } \ + ASSERT(size <= len); \ + ERTS_GET_BINARY_BYTES(obj, bptr, bitoffs, bitsize); \ + num_bits = 8*size+bitsize; \ + copy_binary_to_buffer(buf, offset, bptr, bitoffs, num_bits); \ + offset += bitsize; \ + buf += size + (offset>7); \ + LIST_TO_BITSTR_BUF_BCOPY_DBG; \ + offset = offset & 7; \ + } while(0) + +#ifdef DEBUG + ErlDrvSizeT len; +#endif + Eterm obj; + char *buf; + Eterm *objp = NULL; + int offset; + int init_yield_count = 0, yield_count; DECLARE_ESTACK(s); - goto L_again; - - while (!ESTACK_ISEMPTY(s)) { - obj = ESTACK_POP(s); - L_again: - if (is_list(obj)) { - L_iter_list: - objp = list_val(obj); - obj = CAR(objp); - if (is_byte(obj)) { - ASSERT(len > 0); - if (offset == 0) { - *buf++ = unsigned_val(obj); - } else { - *buf = (char)((unsigned_val(obj) >> offset) | - ((*buf >> (8-offset)) << (8-offset))); - buf++; - *buf = (unsigned_val(obj) << (8-offset)); - } + + obj = state->iolist.obj; + buf = state->buf; + offset = state->offset; #ifdef DEBUG - len--; + len = state->len; #endif - } else if (is_binary(obj)) { - byte* bptr; - size_t size = binary_size(obj); - Uint bitsize; - Uint bitoffs; - Uint num_bits; - - ASSERT(size <= len); - ERTS_GET_BINARY_BYTES(obj, bptr, bitoffs, bitsize); - num_bits = 8*size+bitsize; - copy_binary_to_buffer(buf, offset, bptr, bitoffs, num_bits); - offset += bitsize; - buf += size + (offset>7); + + if (!yield_support) { + yield_count = init_yield_count = 0; /* Shut up faulty warning... >:-( */ + goto L_again; + } + else { + + if (state->iolist.reds_left <= 0) + return ERTS_IOLIST_TO_BUF_YIELD; + + ESTACK_CHANGE_ALLOCATOR(s, ERTS_ALC_T_SAVED_ESTACK); + init_yield_count = (ERTS_IOLIST_TO_BUF_YIELD_COUNT_PER_RED + * state->iolist.reds_left); + yield_count = init_yield_count; + + if (!state->iolist.estack.start) + goto L_again; + else { + int chk_stack; + /* Restart; restore state... */ + ESTACK_RESTORE(s, &state->iolist.estack); + + if (!state->bcopy.bptr) + chk_stack = 0; + else { + chk_stack = 1; + if (list_to_bitstr_buf_bcopy(state, THE_NON_VALUE, &yield_count)) { + /* Yield again... */ + BUMP_ALL_REDS(state->iolist.c_p); + state->iolist.reds_left = 0; + ESTACK_SAVE(s, &state->iolist.estack); + return ERTS_IOLIST_TO_BUF_YIELD; + } + buf = state->buf; + offset = state->offset; #ifdef DEBUG - len -= size + (offset>7); + len = state->len; #endif - offset = offset & 7; - } else if (is_list(obj)) { - ESTACK_PUSH(s, CDR(objp)); - goto L_iter_list; /* on head */ - } else { - ASSERT(is_nil(obj)); } - obj = CDR(objp); - if (is_list(obj)) { - goto L_iter_list; /* on tail */ - } else if (is_binary(obj)) { - byte* bptr; - size_t size = binary_size(obj); - Uint bitsize; - Uint bitoffs; - Uint num_bits; - - ASSERT(size <= len); - ERTS_GET_BINARY_BYTES(obj, bptr, bitoffs, bitsize); - num_bits = 8*size+bitsize; - copy_binary_to_buffer(buf, offset, bptr, bitoffs, num_bits); - offset += bitsize; - buf += size+(offset>7); + objp = state->objp; + state->objp = NULL; + + if (objp) + goto L_tail; + if (!chk_stack) + goto L_again; + /* check stack */ + } + } + + while (!ESTACK_ISEMPTY(s)) { + obj = ESTACK_POP(s); + L_again: + if (is_list(obj)) { + while (1) { /* Tail loop */ + while (1) { /* Head loop */ + if (yield_support && --yield_count <= 0) + goto L_yield; + objp = list_val(obj); + obj = CAR(objp); + if (is_byte(obj)) { + ASSERT(len > 0); + if (offset == 0) { + *buf++ = unsigned_val(obj); + } else { + *buf = (char)((unsigned_val(obj) >> offset) | + ((*buf >> (8-offset)) << (8-offset))); + buf++; + *buf = (unsigned_val(obj) << (8-offset)); + } #ifdef DEBUG - len -= size+(offset>7); + len--; #endif - offset = offset & 7; - } else { - ASSERT(is_nil(obj)); + } else if (is_binary(obj)) { + LIST_TO_BITSTR_BUF_BCOPY(objp); + } else if (is_list(obj)) { + ESTACK_PUSH(s, CDR(objp)); + continue; /* Head loop */ + } else { + ASSERT(is_nil(obj)); + } + break; + } + + L_tail: + + obj = CDR(objp); + if (is_list(obj)) { + continue; /* Tail loop */ + } else if (is_binary(obj)) { + LIST_TO_BITSTR_BUF_BCOPY(NULL); + } else { + ASSERT(is_nil(obj)); + } + break; } } else if (is_binary(obj)) { - byte* bptr; - size_t size = binary_size(obj); - Uint bitsize; - Uint bitoffs; - Uint num_bits; - - ASSERT(size <= len); - ERTS_GET_BINARY_BYTES(obj, bptr, bitoffs, bitsize); - num_bits = 8*size+bitsize; - copy_binary_to_buffer(buf, offset, bptr, bitoffs, num_bits); - offset += bitsize; - buf += size + (offset>7); -#ifdef DEBUG - len -= size + (offset>7); -#endif - offset = offset & 7; + LIST_TO_BITSTR_BUF_BCOPY(NULL); } else { + if (yield_support && --yield_count <= 0) + goto L_yield; ASSERT(is_nil(obj)); } } DESTROY_ESTACK(s); - return offset; + + if (yield_support) { + int reds; + CLEAR_SAVED_ESTACK(&state->iolist.estack); + reds = ((init_yield_count - yield_count - 1) + / ERTS_IOLIST_TO_BUF_YIELD_COUNT_PER_RED) + 1; + BUMP_REDS(state->iolist.c_p, reds); + state->iolist.reds_left -= reds; + if (state->iolist.reds_left < 0) + state->iolist.reds_left = 0; + } + state->buf = buf; + state->offset = offset; + return 0; + +L_bcopy_yield: + + state->buf = buf; + state->offset = offset; +#ifdef DEBUG + state->len = len; +#endif + + if (list_to_bitstr_buf_bcopy(state, obj, &yield_count) == 0) + ERTS_INTERNAL_ERROR("Missing yield"); + + BUMP_ALL_REDS(state->iolist.c_p); + state->iolist.reds_left = 0; + ESTACK_SAVE(s, &state->iolist.estack); + return ERTS_IOLIST_TO_BUF_YIELD; + +L_yield: + + BUMP_ALL_REDS(state->iolist.c_p); + state->iolist.reds_left = 0; + state->iolist.obj = obj; + state->buf = buf; + state->offset = offset; + ESTACK_SAVE(s, &state->iolist.estack); +#ifdef DEBUG + state->len = len; +#endif + return ERTS_IOLIST_TO_BUF_YIELD; + + +#undef LIST_TO_BITSTR_BUF_BCOPY_DBG +#undef LIST_TO_BITSTR_BUF_BCOPY + +} + +static ErlDrvSizeT +list_to_bitstr_buf_yielding(ErtsIOList2BufState *state) +{ + return list_to_bitstr_buf(1, state); +} + +static ErlDrvSizeT +list_to_bitstr_buf_not_yielding(ErtsIOList2BufState *state) +{ + return list_to_bitstr_buf(0, state); +} + +static int +list_to_bitstr_buf_bcopy(ErtsIOList2BufState *state, Eterm obj, int *yield_countp) +{ + int res; + char *buf = state->buf; + char *next_buf; + int offset = state->offset; + int next_offset; +#ifdef DEBUG + ErlDrvSizeT len = state->len; + ErlDrvSizeT next_len; +#endif + byte* bptr; + size_t size; + size_t max_size; + Uint bitoffs; + Uint num_bits; + Uint bitsize; + int yield_count = *yield_countp; + + if (state->bcopy.bptr) { + bptr = state->bcopy.bptr; + size = state->bcopy.size; + bitoffs = state->bcopy.bitoffs; + bitsize = state->bcopy.bitsize; + state->bcopy.bptr = NULL; + } + else { + + ASSERT(is_binary(obj)); + + size = binary_size(obj); + + ASSERT(size <= len); + + ERTS_GET_BINARY_BYTES(obj, bptr, bitoffs, bitsize); + } + + max_size = (size_t) ERTS_IOLIST_TO_BUF_BYTES_PER_YIELD_COUNT; + if (yield_count > 0) + max_size *= (size_t) (yield_count+1); + + if (size <= max_size) { + if (size >= ERTS_IOLIST_TO_BUF_BYTES_PER_YIELD_COUNT) { + int cost = (int) size; + cost /= ERTS_IOLIST_TO_BUF_BYTES_PER_YIELD_COUNT; + yield_count -= cost; + } + next_offset = offset + bitsize; + next_buf = buf + size+(next_offset>7); +#ifdef DEBUG + next_len = len - size+(next_offset>7); +#endif + next_offset &= 7; + num_bits = 8*size+bitsize; + res = 0; + } + else { + ASSERT(0 < max_size && max_size < size); + yield_count = 0; + state->bcopy.bptr = bptr + max_size; + state->bcopy.bitoffs = bitoffs; + state->bcopy.bitsize = bitsize; + state->bcopy.size = size - max_size; + next_buf = buf + max_size; +#ifdef DEBUG + next_len = len - max_size; +#endif + next_offset = offset; + num_bits = 8*max_size; + size = max_size; + res = 1; + } + + copy_binary_to_buffer(buf, offset, bptr, bitoffs, num_bits); + + state->offset = next_offset; + state->buf = next_buf; +#ifdef DEBUG + state->len = next_len; +#endif + *yield_countp = yield_count; + + return res; } static int -bitstr_list_len(Eterm obj, Uint* num_bytes) +bitstr_list_len(ErtsIOListState *state) { Eterm* objp; - Uint len = 0; - Uint offs = 0; + Eterm obj; + Uint len, offs; + int res, init_yield_count, yield_count; DECLARE_ESTACK(s); + + if (state->reds_left <= 0) + return ERTS_IOLIST_YIELD; + + len = (Uint) state->size; + offs = state->offs; + obj = state->obj; + + ESTACK_CHANGE_ALLOCATOR(s, ERTS_ALC_T_SAVED_ESTACK); + init_yield_count = ERTS_IOLIST_SIZE_YIELDS_COUNT_PER_RED; + init_yield_count *= state->reds_left; + yield_count = init_yield_count; + if (state->estack.start) { + /* Restart; restore estack... */ + ESTACK_RESTORE(s, &state->estack); + } + goto L_again; #define SAFE_ADD(Var, Val) \ @@ -748,46 +1496,55 @@ bitstr_list_len(Eterm obj, Uint* num_bytes) obj = ESTACK_POP(s); L_again: if (is_list(obj)) { - L_iter_list: - objp = list_val(obj); - /* Head */ - obj = CAR(objp); - if (is_byte(obj)) { - len++; - if (len == 0) { - goto L_overflow_error; + while (1) { /* Tail loop */ + while (1) { /* Head loop */ + if (--yield_count <= 0) + goto L_yield; + objp = list_val(obj); + /* Head */ + obj = CAR(objp); + if (is_byte(obj)) { + len++; + if (len == 0) { + goto L_overflow_error; + } + } else if (is_binary(obj)) { + SAFE_ADD(len, binary_size(obj)); + SAFE_ADD_BITSIZE(offs, obj); + } else if (is_list(obj)) { + ESTACK_PUSH(s, CDR(objp)); + continue; /* Head loop */ + } else if (is_not_nil(obj)) { + goto L_type_error; + } + break; } - } else if (is_binary(obj)) { - SAFE_ADD(len, binary_size(obj)); - SAFE_ADD_BITSIZE(offs, obj); - } else if (is_list(obj)) { - ESTACK_PUSH(s, CDR(objp)); - goto L_iter_list; /* on head */ - } else if (is_not_nil(obj)) { - goto L_type_error; + /* Tail */ + obj = CDR(objp); + if (is_list(obj)) + continue; /* Tail loop */ + else if (is_binary(obj)) { + SAFE_ADD(len, binary_size(obj)); + SAFE_ADD_BITSIZE(offs, obj); + } else if (is_not_nil(obj)) { + goto L_type_error; + } + break; } - /* Tail */ - obj = CDR(objp); - if (is_list(obj)) - goto L_iter_list; /* on tail */ - else if (is_binary(obj)) { + } else { + if (--yield_count <= 0) + goto L_yield; + if (is_binary(obj)) { SAFE_ADD(len, binary_size(obj)); SAFE_ADD_BITSIZE(offs, obj); } else if (is_not_nil(obj)) { goto L_type_error; } - } else if (is_binary(obj)) { - SAFE_ADD(len, binary_size(obj)); - SAFE_ADD_BITSIZE(offs, obj); - } else if (is_not_nil(obj)) { - goto L_type_error; } } #undef SAFE_ADD #undef SAFE_ADD_BITSIZE - DESTROY_ESTACK(s); - /* * Make sure that the number of bits in the bitstring will fit * in an Uint to ensure that the binary can be matched using @@ -800,15 +1557,42 @@ bitstr_list_len(Eterm obj, Uint* num_bytes) if (len << 3 < len) { goto L_overflow_error; } - *num_bytes = len; - return ERTS_IOLIST_OK; + state->size = len; - L_type_error: - DESTROY_ESTACK(s); - return ERTS_IOLIST_TYPE; + res = ERTS_IOLIST_OK; + + L_return: { + int yc = init_yield_count - yield_count; + int reds; + + DESTROY_ESTACK(s); + CLEAR_SAVED_ESTACK(&state->estack); + + reds = (yc - 1)/ERTS_IOLIST_SIZE_YIELDS_COUNT_PER_RED + 1; + BUMP_REDS(state->c_p, reds); + state->reds_left -= reds; + state->size = (ErlDrvSizeT) len; + state->have_size = 1; + return res; + } L_overflow_error: - DESTROY_ESTACK(s); - return ERTS_IOLIST_OVERFLOW; + res = ERTS_IOLIST_OVERFLOW; + len = 0; + goto L_return; + + L_type_error: + res = ERTS_IOLIST_TYPE; + len = 0; + goto L_return; + + L_yield: + BUMP_ALL_REDS(state->c_p); + state->reds_left = 0; + state->size = len; + state->offs = offs; + state->obj = obj; + ESTACK_SAVE(s, &state->estack); + return ERTS_IOLIST_YIELD; } diff --git a/erts/emulator/beam/break.c b/erts/emulator/beam/break.c index 7d4f52ee23..446598362c 100644 --- a/erts/emulator/beam/break.c +++ b/erts/emulator/beam/break.c @@ -3,16 +3,17 @@ * * Copyright Ericsson AB 1996-2013. All Rights Reserved. * - * The contents of this file are subject to the Erlang Public License, - * Version 1.1, (the "License"); you may not use this file except in - * compliance with the License. You should have received a copy of the - * Erlang Public License along with this software. If not, it can be - * retrieved online at http://www.erlang.org/. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Software distributed under the License is distributed on an "AS IS" - * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See - * the License for the specific language governing rights and limitations - * under the License. + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. * * %CopyrightEnd% */ @@ -36,7 +37,7 @@ #include "atom.h" #include "beam_load.h" #include "erl_instrument.h" -#include "erl_bif_timer.h" +#include "erl_hl_timer.h" #include "erl_thr_progress.h" /* Forward declarations -- should really appear somewhere else */ @@ -103,12 +104,12 @@ process_killer(void) erts_printf("(k)ill (n)ext (r)eturn:\n"); while(1) { if ((j = sys_get_key(0)) <= 0) - erl_exit(0, ""); + erts_exit(0, ""); switch(j) { case 'k': { ErtsProcLocks rp_locks = ERTS_PROC_LOCKS_XSIG_SEND; erts_aint32_t state; - erts_smp_proc_inc_refc(rp); + erts_proc_inc_refc(rp); erts_smp_proc_lock(rp, rp_locks); state = erts_smp_atomic32_read_acqb(&rp->state); if (state & (ERTS_PSFLG_FREE @@ -131,7 +132,7 @@ process_killer(void) 0); } erts_smp_proc_unlock(rp, rp_locks); - erts_smp_proc_dec_refc(rp); + erts_proc_dec_refc(rp); } case 'n': br = 1; break; case 'r': return; @@ -181,7 +182,6 @@ static void doit_print_monitor(ErtsMonitor *mon, void *vpcontext) ASSERT(is_node_name_atom(mon->pid)); erts_print(to, to_arg, "%s{to,{%T,%T},%T}", prefix, mon->name, mon->pid, mon->ref); - erts_print(to, to_arg,"}"); } else if (is_atom(mon->name)){ /* local by name */ erts_print(to, to_arg, "%s{to,{%T,%T},%T}", prefix, mon->name, erts_this_dist_entry->sysname, mon->ref); @@ -210,25 +210,12 @@ print_process_info(int to, void *to_arg, Process *p) erts_print(to, to_arg, "State: "); state = erts_smp_atomic32_read_acqb(&p->state); - if (state & ERTS_PSFLG_FREE) - erts_print(to, to_arg, "Non Existing\n"); /* Should never happen */ - else if (state & ERTS_PSFLG_EXITING) - erts_print(to, to_arg, "Exiting\n"); - else if (state & ERTS_PSFLG_GC) { - garbing = 1; - running = 1; - erts_print(to, to_arg, "Garbing\n"); - } - else if (state & ERTS_PSFLG_SUSPENDED) - erts_print(to, to_arg, "Suspended\n"); - else if (state & ERTS_PSFLG_RUNNING) { - running = 1; - erts_print(to, to_arg, "Running\n"); - } - else if (state & ERTS_PSFLG_ACTIVE) - erts_print(to, to_arg, "Scheduled\n"); - else - erts_print(to, to_arg, "Waiting\n"); + erts_dump_process_state(to, to_arg, state); + if (state & ERTS_PSFLG_GC) { + garbing = 1; + running = 1; + } else if (state & ERTS_PSFLG_RUNNING) + running = 1; /* * If the process is registered as a global process, display the @@ -241,9 +228,9 @@ print_process_info(int to, void *to_arg, Process *p) * Display the initial function name */ erts_print(to, to_arg, "Spawned as: %T:%T/%bpu\n", - p->initial[INITIAL_MOD], - p->initial[INITIAL_FUN], - p->initial[INITIAL_ARI]); + p->u.initial[INITIAL_MOD], + p->u.initial[INITIAL_FUN], + p->u.initial[INITIAL_ARI]); if (p->current != NULL) { if (running) { @@ -265,7 +252,7 @@ print_process_info(int to, void *to_arg, Process *p) /* display the message queue only if there is anything in it */ if (!ERTS_IS_CRASH_DUMPING && p->msg.first != NULL && !garbing) { - ErlMessage* mp; + ErtsMessage* mp; erts_print(to, to_arg, "Message queue: ["); for (mp = p->msg.first; mp; mp = mp->next) erts_print(to, to_arg, mp->next ? "%T," : "%T", ERL_MESSAGE_TERM(mp)); @@ -336,7 +323,7 @@ print_process_info(int to, void *to_arg, Process *p) erts_print(to, to_arg, "Heap unused: %bpu\n", (p->hend - p->htop)); erts_print(to, to_arg, "OldHeap unused: %bpu\n", (OLD_HEAP(p) == NULL) ? 0 : (OLD_HEND(p) - OLD_HTOP(p)) ); - erts_print(to, to_arg, "Memory: %beu\n", erts_process_memory(p)); + erts_print(to, to_arg, "Memory: %beu\n", erts_process_memory(p, !0)); if (garbing) { print_garb_info(to, to_arg, p); @@ -351,6 +338,10 @@ print_process_info(int to, void *to_arg, Process *p) #endif erts_stack_dump(to, to_arg, p); } + + /* Display all states */ + erts_print(to, to_arg, "Internal State: "); + erts_dump_extended_process_state(to, to_arg, state); } static void @@ -390,7 +381,7 @@ loaded(int to, void *to_arg) int i; int old = 0; int cur = 0; - BeamInstr* code; + BeamCodeHeader* code; Module* modp; ErtsCodeIndex code_ix; @@ -448,30 +439,30 @@ loaded(int to, void *to_arg) erts_print(to, to_arg, "\n"); erts_print(to, to_arg, "Current size: %d\n", modp->curr.code_length); - code = modp->curr.code; - if (code != NULL && code[MI_ATTR_PTR]) { + code = modp->curr.code_hdr; + if (code != NULL && code->attr_ptr) { erts_print(to, to_arg, "Current attributes: "); - dump_attributes(to, to_arg, (byte *) code[MI_ATTR_PTR], - code[MI_ATTR_SIZE]); + dump_attributes(to, to_arg, code->attr_ptr, + code->attr_size); } - if (code != NULL && code[MI_COMPILE_PTR]) { + if (code != NULL && code->compile_ptr) { erts_print(to, to_arg, "Current compilation info: "); - dump_attributes(to, to_arg, (byte *) code[MI_COMPILE_PTR], - code[MI_COMPILE_SIZE]); + dump_attributes(to, to_arg, code->compile_ptr, + code->compile_size); } if (modp->old.code_length != 0) { erts_print(to, to_arg, "Old size: %d\n", modp->old.code_length); - code = modp->old.code; - if (code[MI_ATTR_PTR]) { + code = modp->old.code_hdr; + if (code->attr_ptr) { erts_print(to, to_arg, "Old attributes: "); - dump_attributes(to, to_arg, (byte *) code[MI_ATTR_PTR], - code[MI_ATTR_SIZE]); + dump_attributes(to, to_arg, code->attr_ptr, + code->attr_size); } - if (code[MI_COMPILE_PTR]) { + if (code->compile_ptr) { erts_print(to, to_arg, "Old compilation info: "); - dump_attributes(to, to_arg, (byte *) code[MI_COMPILE_PTR], - code[MI_COMPILE_SIZE]); + dump_attributes(to, to_arg, code->compile_ptr, + code->compile_size); } } } @@ -502,7 +493,7 @@ do_break(void) halt immediately if break is called */ mode = erts_read_env("ERL_CONSOLE_MODE"); if (mode && strcmp(mode, "window") != 0) - erl_exit(0, ""); + erts_exit(0, ""); erts_free_read_env(mode); #endif /* __WIN32__ */ @@ -512,7 +503,7 @@ do_break(void) while (1) { if ((i = sys_get_key(0)) <= 0) - erl_exit(0, ""); + erts_exit(0, ""); switch (i) { case 'q': case 'a': @@ -522,9 +513,9 @@ do_break(void) * The usual reason for a read error is Ctrl-C. Treat this as * 'a' to avoid infinite loop. */ - erl_exit(0, ""); + erts_exit(0, ""); case 'A': /* Halt generating crash dump */ - erl_exit(1, "Crash dump requested by user"); + erts_exit(ERTS_ERROR_EXIT, "Crash dump requested by user"); case 'c': return; case 'p': @@ -545,7 +536,9 @@ do_break(void) erts_printf("Erlang (%s) emulator version " ERLANG_VERSION "\n", EMULATOR); +#if ERTS_SAVED_COMPILE_TIME erts_printf("Compiled on " ERLANG_COMPILE_DATE "\n"); +#endif return; case 'd': distribution_info(ERTS_PRINT_STDOUT, NULL); @@ -680,27 +673,39 @@ erl_crash_dump_v(char *file, int line, char* fmt, va_list args) char* dumpname; int secs; int env_erl_crash_dump_seconds_set = 1; + int i; if (ERTS_SOMEONE_IS_CRASH_DUMPING) return; #ifdef ERTS_SMP + /* Order all managed threads to block, this has to be done + first to guarantee that this is the only thread to generate + crash dump. */ + erts_thr_progress_fatal_error_block(&tpd_buf); + +#ifdef ERTS_SYS_SUSPEND_SIGNAL /* - * Wait for all managed threads to block. If all threads haven't blocked - * after a minute, we go anyway and hope for the best... - * - * We do not release system again. We expect an exit() or abort() after - * dump has been written. + * We suspend all scheduler threads so that we can dump some + * data about the currently running processes and scheduler data. + * We have to be very very careful when doing this as the schedulers + * could be anywhere. */ - erts_thr_progress_fatal_error_block(60000, &tpd_buf); - /* Either worked or not... */ + for (i = 0; i < erts_no_schedulers; i++) { + erts_tid_t tid = ERTS_SCHEDULER_IX(i)->tid; + if (!erts_equal_tids(tid,erts_thr_self())) + sys_thr_suspend(tid); + } + +#endif /* Allow us to pass certain places without locking... */ erts_smp_atomic32_set_mb(&erts_writing_erl_crash_dump, 1); erts_smp_tsd_set(erts_is_crash_dumping_key, (void *) 1); -#else + +#else /* !ERTS_SMP */ erts_writing_erl_crash_dump = 1; -#endif +#endif /* ERTS_SMP */ envsz = sizeof(env); /* ERL_CRASH_DUMP_SECONDS not set @@ -753,11 +758,12 @@ erl_crash_dump_v(char *file, int line, char* fmt, va_list args) dumpname = "erl_crash.dump"; else dumpname = &dumpnamebuf[0]; + + erts_fprintf(stderr,"\nCrash dump is being written to: %s...", dumpname); fd = open(dumpname,O_WRONLY | O_CREAT | O_TRUNC,0640); - if (fd < 0) + if (fd < 0) return; /* Can't create the crash dump, skip it */ - time(&now); erts_fdprintf(fd, "=erl_crash_dump:0.3\n%s", ctime(&now)); @@ -770,10 +776,77 @@ erl_crash_dump_v(char *file, int line, char* fmt, va_list args) } erts_fdprintf(fd, "System version: "); erts_print_system_version(fd, NULL, NULL); +#if ERTS_SAVED_COMPILE_TIME erts_fdprintf(fd, "%s\n", "Compiled: " ERLANG_COMPILE_DATE); +#endif + erts_fdprintf(fd, "Taints: "); erts_print_nif_taints(fd, NULL); erts_fdprintf(fd, "Atoms: %d\n", atom_table_size()); + +#ifdef USE_THREADS + /* We want to note which thread it was that called erts_exit */ + if (erts_get_scheduler_data()) { + erts_fdprintf(fd, "Calling Thread: scheduler:%d\n", + erts_get_scheduler_data()->no); + } else { + if (!erts_thr_getname(erts_thr_self(), dumpnamebuf, MAXPATHLEN)) + erts_fdprintf(fd, "Calling Thread: %s\n", dumpnamebuf); + else + erts_fdprintf(fd, "Calling Thread: %p\n", erts_thr_self()); + } +#else + erts_fdprintf(fd, "Calling Thread: scheduler:1\n"); +#endif + +#if defined(ERTS_HAVE_TRY_CATCH) + + /* + * erts_print_scheduler_info is not guaranteed to be safe to call + * here for all schedulers as we may have suspended a scheduler + * in the middle of updating the STACK_TOP and STACK_START + * variables and thus when scanning the stack we could get + * segmentation faults. We protect against this very unlikely + * scenario by using the ERTS_SYS_TRY_CATCH. + */ + for (i = 0; i < erts_no_schedulers; i++) { + ERTS_SYS_TRY_CATCH( + erts_print_scheduler_info(fd, NULL, ERTS_SCHEDULER_IX(i)), + erts_fdprintf(fd, "** crashed **\n")); + } +#endif + +#ifdef ERTS_SMP + +#ifdef ERTS_SYS_SUSPEND_SIGNAL + + /* We resume all schedulers so that we are in a known safe state + when we write the rest of the crash dump */ + for (i = 0; i < erts_no_schedulers; i++) { + erts_tid_t tid = ERTS_SCHEDULER_IX(i)->tid; + if (!erts_equal_tids(tid,erts_thr_self())) + sys_thr_resume(tid); + } +#endif + + /* + * Wait for all managed threads to block. If all threads haven't blocked + * after a minute, we go anyway and hope for the best... + * + * We do not release system again. We expect an exit() or abort() after + * dump has been written. + */ + erts_thr_progress_fatal_error_wait(60000); + /* Either worked or not... */ +#endif + +#ifndef ERTS_HAVE_TRY_CATCH + /* This is safe to call here, as all schedulers are blocked */ + for (i = 0; i < erts_no_schedulers; i++) { + erts_print_scheduler_info(fd, NULL, ERTS_SCHEDULER_IX(i)); + } +#endif + info(fd, NULL); /* General system info */ if (erts_ptab_initialized(&erts_proc)) process_info(fd, NULL); /* Info about each process and port */ @@ -803,7 +876,7 @@ erl_crash_dump_v(char *file, int line, char* fmt, va_list args) erts_fdprintf(fd, "=end\n"); close(fd); - erts_fprintf(stderr,"\nCrash dump was written to: %s\n", dumpname); + erts_fprintf(stderr,"done\n"); } void diff --git a/erts/emulator/beam/code_ix.c b/erts/emulator/beam/code_ix.c index 4344558348..1b0968c55c 100644 --- a/erts/emulator/beam/code_ix.c +++ b/erts/emulator/beam/code_ix.c @@ -3,16 +3,17 @@ * * Copyright Ericsson AB 2012. All Rights Reserved. * - * The contents of this file are subject to the Erlang Public License, - * Version 1.1, (the "License"); you may not use this file except in - * compliance with the License. You should have received a copy of the - * Erlang Public License along with this software. If not, it can be - * retrieved online at http://www.erlang.org/. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Software distributed under the License is distributed on an "AS IS" - * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See - * the License for the specific language governing rights and limitations - * under the License. + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. * * %CopyrightEnd% */ @@ -64,12 +65,12 @@ void erts_code_ix_init(void) CIX_TRACE("init"); } -void erts_start_staging_code_ix(void) +void erts_start_staging_code_ix(int num_new) { beam_catches_start_staging(); export_start_staging(); module_start_staging(); - erts_start_staging_ranges(); + erts_start_staging_ranges(num_new); CIX_TRACE("start"); } @@ -130,7 +131,7 @@ int erts_try_seize_code_write_permission(Process* c_p) ASSERT(code_writing_process != c_p); qitem = erts_alloc(ERTS_ALC_T_CODE_IX_LOCK_Q, sizeof(*qitem)); qitem->p = c_p; - erts_smp_proc_inc_refc(c_p); + erts_proc_inc_refc(c_p); qitem->next = code_write_queue; code_write_queue = qitem; erts_suspend(c_p, ERTS_PROC_LOCK_MAIN, NULL); @@ -151,7 +152,7 @@ void erts_release_code_write_permission(void) } erts_smp_proc_unlock(qitem->p, ERTS_PROC_LOCK_STATUS); code_write_queue = qitem->next; - erts_smp_proc_dec_refc(qitem->p); + erts_proc_dec_refc(qitem->p); erts_free(ERTS_ALC_T_CODE_IX_LOCK_Q, qitem); } code_writing_process = NULL; diff --git a/erts/emulator/beam/code_ix.h b/erts/emulator/beam/code_ix.h index 16ad900228..7a66211a5b 100644 --- a/erts/emulator/beam/code_ix.h +++ b/erts/emulator/beam/code_ix.h @@ -3,16 +3,17 @@ * * Copyright Ericsson AB 2012-2013. All Rights Reserved. * - * The contents of this file are subject to the Erlang Public License, - * Version 1.1, (the "License"); you may not use this file except in - * compliance with the License. You should have received a copy of the - * Erlang Public License along with this software. If not, it can be - * retrieved online at http://www.erlang.org/. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Software distributed under the License is distributed on an "AS IS" - * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See - * the License for the specific language governing rights and limitations - * under the License. + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. * * %CopyrightEnd% */ @@ -99,7 +100,7 @@ void erts_release_code_write_permission(void); * Must be followed by calls to either "end" and "commit" or "abort" before * code write permission can be released. */ -void erts_start_staging_code_ix(void); +void erts_start_staging_code_ix(int num_new); /* End the staging. * Preceded by "start" and must be followed by "commit". diff --git a/erts/emulator/beam/copy.c b/erts/emulator/beam/copy.c index 50548850eb..b38ebe066b 100644 --- a/erts/emulator/beam/copy.c +++ b/erts/emulator/beam/copy.c @@ -3,16 +3,17 @@ * * Copyright Ericsson AB 1996-2012. All Rights Reserved. * - * The contents of this file are subject to the Erlang Public License, - * Version 1.1, (the "License"); you may not use this file except in - * compliance with the License. You should have received a copy of the - * Erlang Public License along with this software. If not, it can be - * retrieved online at http://www.erlang.org/. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Software distributed under the License is distributed on an "AS IS" - * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See - * the License for the specific language governing rights and limitations - * under the License. + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. * * %CopyrightEnd% */ @@ -21,6 +22,8 @@ # include "config.h" #endif +#define ERL_WANT_GC_INTERNALS__ + #include "sys.h" #include "erl_vm.h" #include "global.h" @@ -32,56 +35,59 @@ #include "erl_bits.h" #include "dtrace-wrapper.h" -static void move_one_frag(Eterm** hpp, Eterm* src, Uint src_sz, ErlOffHeap*); +static void move_one_frag(Eterm** hpp, ErlHeapFragment*, ErlOffHeap*, int); /* * Copy object "obj" to process p. */ -Eterm -copy_object(Eterm obj, Process* to) -{ - Uint size = size_object(obj); - Eterm* hp = HAlloc(to, size); - Eterm res; +Eterm copy_object_x(Eterm obj, Process* to, Uint extra) { + if (!is_immed(obj)) { + Uint size = size_object(obj); + Eterm* hp = HAllocX(to, size, extra); + Eterm res; #ifdef USE_VM_PROBES - if (DTRACE_ENABLED(copy_object)) { - DTRACE_CHARBUF(proc_name, 64); + if (DTRACE_ENABLED(copy_object)) { + DTRACE_CHARBUF(proc_name, 64); - erts_snprintf(proc_name, sizeof(DTRACE_CHARBUF_NAME(proc_name)), - "%T", to->common.id); - DTRACE2(copy_object, proc_name, size); - } + erts_snprintf(proc_name, sizeof(DTRACE_CHARBUF_NAME(proc_name)), + "%T", to->common.id); + DTRACE2(copy_object, proc_name, size); + } #endif - res = copy_struct(obj, size, &hp, &to->off_heap); + res = copy_struct(obj, size, &hp, &to->off_heap); #ifdef DEBUG - if (eq(obj, res) == 0) { - erl_exit(ERTS_ABORT_EXIT, "copy not equal to source\n"); - } + if (eq(obj, res) == 0) { + erts_exit(ERTS_ABORT_EXIT, "copy not equal to source\n"); + } #endif - return res; + return res; + } + return obj; } /* * Return the "flat" size of the object. */ -#if HALFWORD_HEAP -Uint size_object_rel(Eterm obj, Eterm* base) -#else Uint size_object(Eterm obj) -#endif { Uint sum = 0; Eterm* ptr; int arity; +#ifdef DEBUG + Eterm mypid = erts_get_current_pid(); +#endif DECLARE_ESTACK(s); + + VERBOSE(DEBUG_SHCOPY, ("[pid=%T] size_object %p\n", mypid, obj)); + for (;;) { switch (primary_tag(obj)) { case TAG_PRIMARY_LIST: sum += 2; - ptr = list_val_rel(obj,base); + ptr = list_val(obj); obj = *ptr++; if (!IS_CONST(obj)) { ESTACK_PUSH(s, obj); @@ -90,11 +96,11 @@ Uint size_object(Eterm obj) break; case TAG_PRIMARY_BOXED: { - Eterm hdr = *boxed_val_rel(obj,base); + Eterm hdr = *boxed_val(obj); ASSERT(is_header(hdr)); switch (hdr & _TAG_HEADER_MASK) { case ARITYVAL_SUBTAG: - ptr = tuple_val_rel(obj,base); + ptr = tuple_val(obj); arity = header_arity(hdr); sum += arity + 1; if (arity == 0) { /* Empty tuple -- unusual. */ @@ -110,7 +116,7 @@ Uint size_object(Eterm obj) break; case FUN_SUBTAG: { - Eterm* bptr = fun_val_rel(obj,base); + Eterm* bptr = fun_val(obj); ErlFunThing* funp = (ErlFunThing *) bptr; unsigned eterms = 1 /* creator */ + funp->num_free; unsigned sz = thing_arityval(hdr); @@ -125,6 +131,52 @@ Uint size_object(Eterm obj) obj = *bptr; break; } + case MAP_SUBTAG: + switch (MAP_HEADER_TYPE(hdr)) { + case MAP_HEADER_TAG_FLATMAP_HEAD : + { + Uint n; + flatmap_t *mp; + mp = (flatmap_t*)flatmap_val(obj); + ptr = (Eterm *)mp; + n = flatmap_get_size(mp) + 1; + sum += n + 2; + ptr += 2; /* hdr + size words */ + while (n--) { + obj = *ptr++; + if (!IS_CONST(obj)) { + ESTACK_PUSH(s, obj); + } + } + goto pop_next; + } + case MAP_HEADER_TAG_HAMT_HEAD_BITMAP : + case MAP_HEADER_TAG_HAMT_HEAD_ARRAY : + case MAP_HEADER_TAG_HAMT_NODE_BITMAP : + { + Eterm *head; + Uint sz; + head = hashmap_val(obj); + sz = hashmap_bitcount(MAP_HEADER_VAL(hdr)); + sum += 1 + sz + header_arity(hdr); + head += 1 + header_arity(hdr); + + if (sz == 0) { + goto pop_next; + } + while(sz-- > 1) { + obj = head[sz]; + if (!IS_CONST(obj)) { + ESTACK_PUSH(s, obj); + } + } + obj = head[0]; + } + break; + default: + erts_exit(ERTS_ABORT_EXIT, "size_object: bad hashmap type %d\n", MAP_HEADER_TYPE(hdr)); + } + break; case SUB_BINARY_SUBTAG: { Eterm real_bin; @@ -133,7 +185,7 @@ Uint size_object(Eterm obj) Uint bitoffs; Uint extra_bytes; Eterm hdr; - ERTS_GET_REAL_BIN_REL(obj, real_bin, offset, bitoffs, bitsize, base); + ERTS_GET_REAL_BIN(obj, real_bin, offset, bitoffs, bitsize); if ((bitsize + bitoffs) > 8) { sum += ERL_SUB_BIN_SIZE; extra_bytes = 2; @@ -143,35 +195,17 @@ Uint size_object(Eterm obj) } else { extra_bytes = 0; } - hdr = *binary_val_rel(real_bin,base); + hdr = *binary_val(real_bin); if (thing_subtag(hdr) == REFC_BINARY_SUBTAG) { sum += PROC_BIN_SIZE; } else { - sum += heap_bin_size(binary_size_rel(obj,base)+extra_bytes); - } - goto pop_next; - } - break; - case MAP_SUBTAG: - { - Uint n; - map_t *mp; - mp = (map_t*)map_val_rel(obj,base); - ptr = (Eterm *)mp; - n = map_get_size(mp) + 1; - sum += n + 2; - ptr += 2; /* hdr + size words */ - while (n--) { - obj = *ptr++; - if (!IS_CONST(obj)) { - ESTACK_PUSH(s, obj); - } + sum += heap_bin_size(binary_size(obj)+extra_bytes); } goto pop_next; } break; - case BIN_MATCHSTATE_SUBTAG: - erl_exit(ERTS_ABORT_EXIT, + case BIN_MATCHSTATE_SUBTAG: + erts_exit(ERTS_ABORT_EXIT, "size_object: matchstate term not allowed"); default: sum += thing_arityval(hdr) + 1; @@ -183,25 +217,389 @@ Uint size_object(Eterm obj) pop_next: if (ESTACK_ISEMPTY(s)) { DESTROY_ESTACK(s); + VERBOSE(DEBUG_SHCOPY, ("[pid=%T] size was: %u\n", mypid, sum)); return sum; } obj = ESTACK_POP(s); break; default: - erl_exit(ERTS_ABORT_EXIT, "size_object: bad tag for %#x\n", obj); + erts_exit(ERTS_ABORT_EXIT, "size_object: bad tag for %#x\n", obj); } } } /* + * Machinery for sharing preserving information + * Using a WSTACK but not very transparently; consider refactoring + */ + +#define DECLARE_BITSTORE(s) \ + DECLARE_WSTACK(s); \ + int WSTK_CONCAT(s,_bitoffs) = 0; \ + int WSTK_CONCAT(s,_offset) = 0; \ + UWord WSTK_CONCAT(s,_buffer) = 0 + +#define DESTROY_BITSTORE(s) DESTROY_WSTACK(s) +#define BITSTORE_PUT(s,i) \ +do { \ + WSTK_CONCAT(s,_buffer) |= i << WSTK_CONCAT(s,_bitoffs); \ + WSTK_CONCAT(s,_bitoffs) += 2; \ + if (WSTK_CONCAT(s,_bitoffs) >= 8*sizeof(UWord)) { \ + WSTACK_PUSH(s, WSTK_CONCAT(s,_buffer)); \ + WSTK_CONCAT(s,_bitoffs) = 0; \ + WSTK_CONCAT(s,_buffer) = 0; \ + } \ +} while(0) +#define BITSTORE_CLOSE(s) \ +do { \ + if (WSTK_CONCAT(s,_bitoffs) > 0) { \ + WSTACK_PUSH(s, WSTK_CONCAT(s,_buffer)); \ + WSTK_CONCAT(s,_bitoffs) = 0; \ + } \ +} while(0) + +#define BITSTORE_FETCH(s,dst) \ +do { \ + UWord result; \ + if (WSTK_CONCAT(s,_bitoffs) <= 0) { \ + WSTK_CONCAT(s,_buffer) = s.wstart[WSTK_CONCAT(s,_offset)]; \ + WSTK_CONCAT(s,_offset)++; \ + WSTK_CONCAT(s,_bitoffs) = 8*sizeof(UWord); \ + } \ + WSTK_CONCAT(s,_bitoffs) -= 2; \ + result = WSTK_CONCAT(s,_buffer) & 3; \ + WSTK_CONCAT(s,_buffer) >>= 2; \ + (dst) = result; \ +} while(0) + +#define BOXED_VISITED_MASK ((Eterm) 3) +#define BOXED_VISITED ((Eterm) 1) +#define BOXED_SHARED_UNPROCESSED ((Eterm) 2) +#define BOXED_SHARED_PROCESSED ((Eterm) 3) + +#define COUNT_OFF_HEAP (0) + +#define IN_LITERAL_PURGE_AREA(info, ptr) \ + ((info)->range_ptr && ( \ + (info)->range_ptr <= (ptr) && \ + (ptr) < ((info)->range_ptr + (info)->range_sz))) +/* + * Return the real size of an object and find sharing information + * This currently returns the same as erts_debug:size/1. + * It is argued whether the size of subterms in constant pools + * should be counted or not. + */ + +Uint size_shared(Eterm obj) +{ + Eterm saved_obj = obj; + Uint sum = 0; + Eterm* ptr; + + DECLARE_EQUEUE(s); + DECLARE_BITSTORE(b); + + for (;;) { + switch (primary_tag(obj)) { + case TAG_PRIMARY_LIST: { + Eterm head, tail; + ptr = list_val(obj); + /* we're not counting anything that's outside our heap */ + if (!COUNT_OFF_HEAP && erts_is_literal(obj,ptr)) { + goto pop_next; + } + head = CAR(ptr); + tail = CDR(ptr); + /* if it's visited, don't count it */ + if (primary_tag(tail) == TAG_PRIMARY_HEADER || + primary_tag(head) == TAG_PRIMARY_HEADER) { + goto pop_next; + } + /* else make it visited now */ + switch (primary_tag(tail)) { + case TAG_PRIMARY_LIST: + ptr[1] = (tail - TAG_PRIMARY_LIST) | TAG_PRIMARY_HEADER; + break; + case TAG_PRIMARY_IMMED1: + CAR(ptr) = (head - primary_tag(head)) | TAG_PRIMARY_HEADER; + CDR(ptr) = (tail - TAG_PRIMARY_IMMED1) | primary_tag(head); + break; + case TAG_PRIMARY_BOXED: + BITSTORE_PUT(b, primary_tag(head)); + CAR(ptr) = (head - primary_tag(head)) | TAG_PRIMARY_HEADER; + CDR(ptr) = (tail - TAG_PRIMARY_BOXED) | TAG_PRIMARY_HEADER; + break; + } + /* and count it */ + sum += 2; + if (!IS_CONST(head)) { + EQUEUE_PUT(s, head); + } + obj = tail; + break; + } + case TAG_PRIMARY_BOXED: { + Eterm hdr; + ptr = boxed_val(obj); + /* we're not counting anything that's outside our heap */ + if (!COUNT_OFF_HEAP && erts_is_literal(obj,ptr)) { + goto pop_next; + } + hdr = *ptr; + /* if it's visited, don't count it */ + if (primary_tag(hdr) != TAG_PRIMARY_HEADER) { + goto pop_next; + } + /* else make it visited now */ + *ptr = (hdr - primary_tag(hdr)) + BOXED_VISITED; + /* and count it */ + ASSERT(is_header(hdr)); + switch (hdr & _TAG_HEADER_MASK) { + case ARITYVAL_SUBTAG: { + int arity = header_arity(hdr); + sum += arity + 1; + if (arity == 0) { /* Empty tuple -- unusual. */ + goto pop_next; + } + while (arity-- > 0) { + obj = *++ptr; + if (!IS_CONST(obj)) { + EQUEUE_PUT(s, obj); + } + } + goto pop_next; + } + case FUN_SUBTAG: { + ErlFunThing* funp = (ErlFunThing *) ptr; + unsigned eterms = 1 /* creator */ + funp->num_free; + unsigned sz = thing_arityval(hdr); + sum += 1 /* header */ + sz + eterms; + ptr += 1 /* header */ + sz; + while (eterms-- > 0) { + obj = *ptr++; + if (!IS_CONST(obj)) { + EQUEUE_PUT(s, obj); + } + } + goto pop_next; + } + case SUB_BINARY_SUBTAG: { + ErlSubBin* sb = (ErlSubBin *) ptr; + Uint extra_bytes; + Eterm hdr; + ASSERT((sb->thing_word & ~BOXED_VISITED_MASK) == HEADER_SUB_BIN); + if (sb->bitsize + sb->bitoffs > 8) { + sum += ERL_SUB_BIN_SIZE; + extra_bytes = 2; + } else if (sb->bitsize + sb->bitoffs > 0) { + sum += ERL_SUB_BIN_SIZE; + extra_bytes = 1; + } else { + extra_bytes = 0; + } + ptr = binary_val(sb->orig); + hdr = (*ptr) & ~BOXED_VISITED_MASK; + if (thing_subtag(hdr) == REFC_BINARY_SUBTAG) { + sum += PROC_BIN_SIZE; + } else { + ASSERT(thing_subtag(hdr) == HEAP_BINARY_SUBTAG); + sum += heap_bin_size(binary_size(obj) + extra_bytes); + } + goto pop_next; + } + case MAP_SUBTAG: + switch (MAP_HEADER_TYPE(hdr)) { + case MAP_HEADER_TAG_FLATMAP_HEAD : { + flatmap_t *mp = (flatmap_t*)flatmap_val(obj); + Uint n = flatmap_get_size(mp) + 1; + ptr = (Eterm *)mp; + sum += n + 2; + ptr += 2; /* hdr + size words */ + while (n--) { + obj = *ptr++; + if (!IS_CONST(obj)) { + EQUEUE_PUT(s, obj); + } + } + goto pop_next; + } + case MAP_HEADER_TAG_HAMT_HEAD_BITMAP : + case MAP_HEADER_TAG_HAMT_HEAD_ARRAY : + case MAP_HEADER_TAG_HAMT_NODE_BITMAP : { + Uint n = hashmap_bitcount(MAP_HEADER_VAL(hdr)); + sum += 1 + n + header_arity(hdr); + ptr += 1 + header_arity(hdr); + while (n--) { + obj = *ptr++; + if (!IS_CONST(obj)) { + EQUEUE_PUT(s, obj); + } + } + goto pop_next; + } + default: + erts_exit(ERTS_ABORT_EXIT, "size_shared: bad hashmap type %d\n", MAP_HEADER_TYPE(hdr)); + } + case BIN_MATCHSTATE_SUBTAG: + erts_exit(ERTS_ABORT_EXIT, + "size_shared: matchstate term not allowed"); + default: + sum += thing_arityval(hdr) + 1; + goto pop_next; + } + break; + } + case TAG_PRIMARY_IMMED1: + pop_next: + if (EQUEUE_ISEMPTY(s)) { + goto cleanup; + } + obj = EQUEUE_GET(s); + break; + default: + erts_exit(ERTS_ABORT_EXIT, "size_shared: bad tag for %#x\n", obj); + } + } + +cleanup: + obj = saved_obj; + BITSTORE_CLOSE(b); + for (;;) { + switch (primary_tag(obj)) { + case TAG_PRIMARY_LIST: { + Eterm head, tail; + ptr = list_val(obj); + if (!COUNT_OFF_HEAP && erts_is_literal(obj,ptr)) { + goto cleanup_next; + } + head = CAR(ptr); + tail = CDR(ptr); + /* if not already clean, clean it up */ + if (primary_tag(tail) == TAG_PRIMARY_HEADER) { + if (primary_tag(head) == TAG_PRIMARY_HEADER) { + Eterm saved; + BITSTORE_FETCH(b, saved); + CAR(ptr) = head = (head - TAG_PRIMARY_HEADER) | saved; + CDR(ptr) = tail = (tail - TAG_PRIMARY_HEADER) | TAG_PRIMARY_BOXED; + } else { + CDR(ptr) = tail = (tail - TAG_PRIMARY_HEADER) | TAG_PRIMARY_LIST; + } + } else if (primary_tag(head) == TAG_PRIMARY_HEADER) { + CAR(ptr) = head = (head - TAG_PRIMARY_HEADER) | primary_tag(tail); + CDR(ptr) = tail = (tail - primary_tag(tail)) | TAG_PRIMARY_IMMED1; + } else { + goto cleanup_next; + } + /* and its children too */ + if (!IS_CONST(head)) { + EQUEUE_PUT_UNCHECKED(s, head); + } + obj = tail; + break; + } + case TAG_PRIMARY_BOXED: { + Eterm hdr; + ptr = boxed_val(obj); + if (!COUNT_OFF_HEAP && erts_is_literal(obj,ptr)) { + goto cleanup_next; + } + hdr = *ptr; + /* if not already clean, clean it up */ + if (primary_tag(hdr) == TAG_PRIMARY_HEADER) { + goto cleanup_next; + } + else { + ASSERT(primary_tag(hdr) == BOXED_VISITED); + *ptr = hdr = (hdr - BOXED_VISITED) + TAG_PRIMARY_HEADER; + } + /* and its children too */ + switch (hdr & _TAG_HEADER_MASK) { + case ARITYVAL_SUBTAG: { + int arity = header_arity(hdr); + if (arity == 0) { /* Empty tuple -- unusual. */ + goto cleanup_next; + } + while (arity-- > 0) { + obj = *++ptr; + if (!IS_CONST(obj)) { + EQUEUE_PUT_UNCHECKED(s, obj); + } + } + goto cleanup_next; + } + case FUN_SUBTAG: { + ErlFunThing* funp = (ErlFunThing *) ptr; + unsigned eterms = 1 /* creator */ + funp->num_free; + unsigned sz = thing_arityval(hdr); + ptr += 1 /* header */ + sz; + while (eterms-- > 0) { + obj = *ptr++; + if (!IS_CONST(obj)) { + EQUEUE_PUT_UNCHECKED(s, obj); + } + } + goto cleanup_next; + } + case MAP_SUBTAG: + switch (MAP_HEADER_TYPE(hdr)) { + case MAP_HEADER_TAG_FLATMAP_HEAD : { + flatmap_t *mp = (flatmap_t *) ptr; + Uint n = flatmap_get_size(mp) + 1; + ptr += 2; /* hdr + size words */ + while (n--) { + obj = *ptr++; + if (!IS_CONST(obj)) { + EQUEUE_PUT_UNCHECKED(s, obj); + } + } + goto cleanup_next; + } + case MAP_HEADER_TAG_HAMT_HEAD_BITMAP : + case MAP_HEADER_TAG_HAMT_HEAD_ARRAY : + case MAP_HEADER_TAG_HAMT_NODE_BITMAP : { + Uint n = hashmap_bitcount(MAP_HEADER_VAL(hdr)); + sum += 1 + n + header_arity(hdr); + ptr += 1 + header_arity(hdr); + while (n--) { + obj = *ptr++; + if (!IS_CONST(obj)) { + EQUEUE_PUT_UNCHECKED(s, obj); + } + } + goto cleanup_next; + } + default: + erts_exit(ERTS_ABORT_EXIT, "size_shared: bad hashmap type %d\n", MAP_HEADER_TYPE(hdr)); + } + default: + goto cleanup_next; + } + break; + } + case TAG_PRIMARY_IMMED1: + cleanup_next: + if (EQUEUE_ISEMPTY(s)) { + goto all_clean; + } + obj = EQUEUE_GET(s); + break; + default: + erts_exit(ERTS_ABORT_EXIT, "size_shared: bad tag for %#x\n", obj); + } + } + + all_clean: + /* Return the result */ + DESTROY_EQUEUE(s); + DESTROY_BITSTORE(b); + return sum; +} + + +/* * Copy a structure to a heap. */ -#if HALFWORD_HEAP -Eterm copy_struct_rel(Eterm obj, Uint sz, Eterm** hpp, ErlOffHeap* off_heap, - Eterm* src_base, Eterm* dst_base) -#else -Eterm copy_struct(Eterm obj, Uint sz, Eterm** hpp, ErlOffHeap* off_heap) -#endif +Eterm copy_struct_x(Eterm obj, Uint sz, Eterm** hpp, ErlOffHeap* off_heap, Uint *bsz) { char* hstart; Uint hsize; @@ -216,19 +614,23 @@ Eterm copy_struct(Eterm obj, Uint sz, Eterm** hpp, ErlOffHeap* off_heap) Eterm* argp; Eterm* const_tuple; Eterm hdr; + Eterm *hend; int i; #ifdef DEBUG Eterm org_obj = obj; Uint org_sz = sz; + Eterm mypid = erts_get_current_pid(); #endif if (IS_CONST(obj)) return obj; + VERBOSE(DEBUG_SHCOPY, ("[pid=%T] copy_struct %p\n", mypid, obj)); + DTRACE1(copy_struct, (int32_t)sz); hp = htop = *hpp; - hbot = htop + sz; + hbot = hend = htop + sz; hstart = (char *)htop; hsize = (char*) hbot - hstart; const_tuple = 0; @@ -237,11 +639,11 @@ Eterm copy_struct(Eterm obj, Uint sz, Eterm** hpp, ErlOffHeap* off_heap) switch (primary_tag(obj)) { case TAG_PRIMARY_LIST: argp = &res; - objp = list_val_rel(obj,src_base); + objp = list_val(obj); goto L_copy_list; case TAG_PRIMARY_BOXED: argp = &res; goto L_copy_boxed; default: - erl_exit(ERTS_ABORT_EXIT, + erts_exit(ERTS_ABORT_EXIT, "%s, line %d: Internal error in copy_struct: 0x%08x\n", __FILE__, __LINE__,obj); } @@ -255,14 +657,11 @@ Eterm copy_struct(Eterm obj, Uint sz, Eterm** hpp, ErlOffHeap* off_heap) hp++; break; case TAG_PRIMARY_LIST: - objp = list_val_rel(obj,src_base); - #if !HALFWORD_HEAP || defined(DEBUG) - if (in_area(objp,hstart,hsize)) { - ASSERT(!HALFWORD_HEAP); + objp = list_val(obj); + if (ErtsInArea(objp,hstart,hsize)) { hp++; break; } - #endif argp = hp++; /* Fall through */ @@ -278,52 +677,41 @@ Eterm copy_struct(Eterm obj, Uint sz, Eterm** hpp, ErlOffHeap* off_heap) } else { CAR(htop) = elem; - #if HALFWORD_HEAP - CDR(htop) = CDR(objp); - *tailp = make_list_rel(htop,dst_base); - htop += 2; - goto L_copy; - #else tailp = &CDR(htop); htop += 2; - #endif } - ASSERT(!HALFWORD_HEAP || tp < hp || tp >= hbot); - *tp = make_list_rel(tailp - 1, dst_base); + *tp = make_list(tailp - 1); obj = CDR(objp); if (!is_list(obj)) { break; } - objp = list_val_rel(obj,src_base); + objp = list_val(obj); } switch (primary_tag(obj)) { case TAG_PRIMARY_IMMED1: *tailp = obj; goto L_copy; case TAG_PRIMARY_BOXED: argp = tailp; goto L_copy_boxed; default: - erl_exit(ERTS_ABORT_EXIT, + erts_exit(ERTS_ABORT_EXIT, "%s, line %d: Internal error in copy_struct: 0x%08x\n", __FILE__, __LINE__,obj); } case TAG_PRIMARY_BOXED: - #if !HALFWORD_HEAP || defined(DEBUG) - if (in_area(boxed_val_rel(obj,src_base),hstart,hsize)) { - ASSERT(!HALFWORD_HEAP); + if (ErtsInArea(boxed_val(obj),hstart,hsize)) { hp++; break; } - #endif argp = hp++; L_copy_boxed: - objp = boxed_val_rel(obj, src_base); + objp = boxed_val(obj); hdr = *objp; switch (hdr & _TAG_HEADER_MASK) { case ARITYVAL_SUBTAG: { int const_flag = 1; /* assume constant tuple */ i = arityval(hdr); - *argp = make_tuple_rel(htop, dst_base); + *argp = make_tuple(htop); tp = htop; /* tp is pointer to new arity value */ *htop++ = *objp++; /* copy arity value */ while (i--) { @@ -338,15 +726,6 @@ Eterm copy_struct(Eterm obj, Uint sz, Eterm** hpp, ErlOffHeap* off_heap) } } break; - case MAP_SUBTAG: - { - i = map_get_size(objp) + 3; - *argp = make_map_rel(htop, dst_base); - while (i--) { - *htop++ = *objp++; - } - } - break; case REFC_BINARY_SUBTAG: { ProcBin* pb; @@ -361,7 +740,7 @@ Eterm copy_struct(Eterm obj, Uint sz, Eterm** hpp, ErlOffHeap* off_heap) while (i--) { *tp++ = *objp++; } - *argp = make_binary_rel(hbot, dst_base); + *argp = make_binary(hbot); pb = (ProcBin*) hbot; erts_refc_inc(&pb->val->refc, 2); pb->next = off_heap->first; @@ -388,7 +767,7 @@ Eterm copy_struct(Eterm obj, Uint sz, Eterm** hpp, ErlOffHeap* off_heap) extra_bytes = 0; } real_size = size+extra_bytes; - objp = binary_val_rel(real_bin,src_base); + objp = binary_val(real_bin); if (thing_subtag(*objp) == HEAP_BINARY_SUBTAG) { ErlHeapBin* from = (ErlHeapBin *) objp; ErlHeapBin* to; @@ -418,7 +797,7 @@ Eterm copy_struct(Eterm obj, Uint sz, Eterm** hpp, ErlOffHeap* off_heap) off_heap->first = (struct erl_off_heap_header*) to; OH_OVERHEAD(off_heap, to->size / sizeof(Eterm)); } - *argp = make_binary_rel(hbot, dst_base); + *argp = make_binary(hbot); if (extra_bytes != 0) { ErlSubBin* res; hbot -= ERL_SUB_BIN_SIZE; @@ -430,7 +809,7 @@ Eterm copy_struct(Eterm obj, Uint sz, Eterm** hpp, ErlOffHeap* off_heap) res->offs = 0; res->is_writable = 0; res->orig = *argp; - *argp = make_binary_rel(hbot, dst_base); + *argp = make_binary(hbot); } break; } @@ -448,7 +827,7 @@ Eterm copy_struct(Eterm obj, Uint sz, Eterm** hpp, ErlOffHeap* off_heap) funp->next = off_heap->first; off_heap->first = (struct erl_off_heap_header*) funp; erts_refc_inc(&funp->fe->refc, 2); - *argp = make_fun_rel(tp, dst_base); + *argp = make_fun(tp); } break; case EXTERNAL_PID_SUBTAG: @@ -457,7 +836,7 @@ Eterm copy_struct(Eterm obj, Uint sz, Eterm** hpp, ErlOffHeap* off_heap) { ExternalThing *etp = (ExternalThing *) htop; - i = thing_arityval(hdr) + 1; + i = thing_arityval(hdr) + 1; tp = htop; while (i--) { @@ -468,17 +847,39 @@ Eterm copy_struct(Eterm obj, Uint sz, Eterm** hpp, ErlOffHeap* off_heap) off_heap->first = (struct erl_off_heap_header*)etp; erts_refc_inc(&etp->node->refc, 2); - *argp = make_external_rel(tp, dst_base); + *argp = make_external(tp); + } + break; + case MAP_SUBTAG: + tp = htop; + switch (MAP_HEADER_TYPE(hdr)) { + case MAP_HEADER_TAG_FLATMAP_HEAD : + i = flatmap_get_size(objp) + 3; + *argp = make_flatmap(htop); + while (i--) { + *htop++ = *objp++; + } + break; + case MAP_HEADER_TAG_HAMT_HEAD_BITMAP : + case MAP_HEADER_TAG_HAMT_HEAD_ARRAY : + *htop++ = *objp++; + case MAP_HEADER_TAG_HAMT_NODE_BITMAP : + i = 1 + hashmap_bitcount(MAP_HEADER_VAL(hdr)); + while (i--) { *htop++ = *objp++; } + *argp = make_hashmap(tp); + break; + default: + erts_exit(ERTS_ABORT_EXIT, "copy_struct: bad hashmap type %d\n", MAP_HEADER_TYPE(hdr)); } break; case BIN_MATCHSTATE_SUBTAG: - erl_exit(ERTS_ABORT_EXIT, + erts_exit(ERTS_ABORT_EXIT, "copy_struct: matchstate term not allowed"); default: i = thing_arityval(hdr)+1; hbot -= i; tp = hbot; - *argp = make_boxed_rel(hbot, dst_base); + *argp = make_boxed(hbot); while (i--) { *tp++ = *objp++; } @@ -494,22 +895,870 @@ Eterm copy_struct(Eterm obj, Uint sz, Eterm** hpp, ErlOffHeap* off_heap) } } + if (bsz) { + *hpp = htop; + *bsz = hend - hbot; + } else { #ifdef DEBUG - if (htop != hbot) - erl_exit(ERTS_ABORT_EXIT, - "Internal error in copy_struct() when copying %T:" - " htop=%p != hbot=%p (sz=%beu)\n", - org_obj, htop, hbot, org_sz); + if (htop != hbot) + erts_exit(ERTS_ABORT_EXIT, + "Internal error in copy_struct() when copying %T:" + " htop=%p != hbot=%p (sz=%beu)\n", + org_obj, htop, hbot, org_sz); #else - if (htop > hbot) { - erl_exit(ERTS_ABORT_EXIT, - "Internal error in copy_struct(): htop, hbot overrun\n"); - } + if (htop > hbot) { + erts_exit(ERTS_ABORT_EXIT, + "Internal error in copy_struct(): htop, hbot overrun\n"); + } #endif - *hpp = (Eterm *) (hstart+hsize); + *hpp = (Eterm *) (hstart+hsize); + } + VERBOSE(DEBUG_SHCOPY, ("[pid=%T] result is at %p\n", mypid, res)); return res; } + +/* + * Machinery for the table used by the sharing preserving copier + * Using an ESTACK but not very transparently; consider refactoring + */ + +#define DECLARE_SHTABLE(s) \ + DECLARE_ESTACK(s); \ + Uint ESTK_CONCAT(s,_offset) = 0 +#define DESTROY_SHTABLE(s) DESTROY_ESTACK(s) +#define SHTABLE_INCR 4 +#define SHTABLE_NEXT(s) ESTK_CONCAT(s,_offset) +#define SHTABLE_PUSH(s,x,y,b) \ +do { \ + if (s.sp > s.end - SHTABLE_INCR) { \ + erl_grow_estack(&(s), SHTABLE_INCR); \ + } \ + *s.sp++ = (x); \ + *s.sp++ = (y); \ + *s.sp++ = (Eterm) NULL; \ + *s.sp++ = (Eterm) (b); \ + ESTK_CONCAT(s,_offset) += SHTABLE_INCR; \ +} while(0) +#define SHTABLE_X(s,e) (s.start[e]) +#define SHTABLE_Y(s,e) (s.start[(e)+1]) +#define SHTABLE_FWD(s,e) ((Eterm *) (s.start[(e)+2])) +#define SHTABLE_FWD_UPD(s,e,p) (s.start[(e)+2] = (Eterm) (p)) +#define SHTABLE_REV(s,e) ((Eterm *) (s.start[(e)+3])) + +#define LIST_SHARED_UNPROCESSED ((Eterm) 0) +#define LIST_SHARED_PROCESSED ((Eterm) 1) + +#define HEAP_ELEM_TO_BE_FILLED _unchecked_make_list(NULL) + + +/* + * Specialized macros for using/reusing the persistent state + */ + +#define DECLARE_EQUEUE_INIT_INFO(q, info) \ + UWord* EQUE_DEF_QUEUE(q) = info->queue_default; \ + ErtsEQueue q = { \ + EQUE_DEF_QUEUE(q), /* start */ \ + EQUE_DEF_QUEUE(q), /* front */ \ + EQUE_DEF_QUEUE(q), /* back */ \ + 1, /* possibly_empty */ \ + EQUE_DEF_QUEUE(q) + DEF_EQUEUE_SIZE, /* end */ \ + ERTS_ALC_T_ESTACK /* alloc_type */ \ + } + +#define DECLARE_EQUEUE_FROM_INFO(q, info) \ + /* no EQUE_DEF_QUEUE(q), read-only */ \ + ErtsEQueue q = { \ + info->queue_start, /* start */ \ + info->queue_start, /* front */ \ + info->queue_start, /* back */ \ + 1, /* possibly_empty */ \ + info->queue_end, /* end */ \ + info->queue_alloc_type /* alloc_type */ \ + } + +#define DECLARE_BITSTORE_INIT_INFO(s, info) \ + UWord* WSTK_DEF_STACK(s) = info->bitstore_default; \ + ErtsWStack s = { \ + WSTK_DEF_STACK(s), /* wstart */ \ + WSTK_DEF_STACK(s), /* wsp */ \ + WSTK_DEF_STACK(s) + DEF_WSTACK_SIZE, /* wend */ \ + WSTK_DEF_STACK(s), /* wdflt */ \ + ERTS_ALC_T_ESTACK /* alloc_type */ \ + }; \ + int WSTK_CONCAT(s,_bitoffs) = 0; \ + /* no WSTK_CONCAT(s,_offset), write-only */ \ + UWord WSTK_CONCAT(s,_buffer) = 0 + +#define DECLARE_BITSTORE_FROM_INFO(s, info) \ + /* no WSTK_DEF_STACK(s), read-only */ \ + ErtsWStack s = { \ + info->bitstore_start, /* wstart */ \ + NULL, /* wsp, read-only */ \ + NULL, /* wend, read-only */ \ + NULL, /* wdef, read-only */ \ + info->bitstore_alloc_type /* alloc_type */ \ + }; \ + int WSTK_CONCAT(s,_bitoffs) = 0; \ + int WSTK_CONCAT(s,_offset) = 0; \ + UWord WSTK_CONCAT(s,_buffer) = 0 + +#define DECLARE_SHTABLE_INIT_INFO(s, info) \ + Eterm* ESTK_DEF_STACK(s) = info->shtable_default; \ + ErtsEStack s = { \ + ESTK_DEF_STACK(s), /* start */ \ + ESTK_DEF_STACK(s), /* sp */ \ + ESTK_DEF_STACK(s) + DEF_ESTACK_SIZE, /* end */ \ + ESTK_DEF_STACK(s), /* default */ \ + ERTS_ALC_T_ESTACK /* alloc_type */ \ + }; \ + Uint ESTK_CONCAT(s,_offset) = 0 + +#define DECLARE_SHTABLE_FROM_INFO(s, info) \ + /* no ESTK_DEF_STACK(s), read-only */ \ + ErtsEStack s = { \ + info->shtable_start, /* start */ \ + NULL, /* sp, read-only */ \ + NULL, /* end, read-only */ \ + NULL, /* def, read-only */ \ + info->shtable_alloc_type /* alloc_type */ \ + }; \ + /* no ESTK_CONCAT(s,_offset), read-only */ + +/* + * Copy object "obj" preserving sharing. + * First half: count size and calculate sharing. + */ +Uint copy_shared_calculate(Eterm obj, erts_shcopy_t *info) +{ + Uint sum; + Uint e; + unsigned sz; + Eterm* ptr; +#ifdef DEBUG + Eterm mypid = erts_get_current_pid(); +#endif + + DECLARE_EQUEUE_INIT_INFO(s, info); + DECLARE_BITSTORE_INIT_INFO(b, info); + DECLARE_SHTABLE_INIT_INFO(t, info); + + /* step #0: + ------------------------------------------------------- + get rid of the easy cases first: + - copying constants + - if not a proper process, do flat copy + */ + + if (IS_CONST(obj)) + return 0; + + VERBOSE(DEBUG_SHCOPY, ("[pid=%T] copy_shared_calculate %p\n", mypid, obj)); + VERBOSE(DEBUG_SHCOPY, ("[pid=%T] message is %T\n", mypid, obj)); + + /* step #1: + ------------------------------------------------------- + traverse the term and calculate the size; + when traversing, transform as you do in size_shared + but when you find shared objects: + + a. add entry in the table, indexed by i + b. mark them: + b1. boxed terms, set header to (i | 11) + store (old header, NONV, NULL, backptr) in the entry + b2. cons cells, set CDR to NONV, set CAR to i + store (old CAR, old CDR, NULL, backptr) in the entry + */ + + sum = 0; + + for (;;) { + switch (primary_tag(obj)) { + case TAG_PRIMARY_LIST: { + Eterm head, tail; + ptr = list_val(obj); + /* off heap list pointers are copied verbatim */ + if (erts_is_literal(obj,ptr)) { + VERBOSE(DEBUG_SHCOPY, ("[pid=%T] bypassed copying %p is %T\n", mypid, ptr, obj)); + if (IN_LITERAL_PURGE_AREA(info,ptr)) + info->literal_size += size_object(obj); + goto pop_next; + } + head = CAR(ptr); + tail = CDR(ptr); + /* if it's visited, don't count it; + if not already shared, make it shared and store it in the table */ + if (primary_tag(tail) == TAG_PRIMARY_HEADER || + primary_tag(head) == TAG_PRIMARY_HEADER) { + if (tail != THE_NON_VALUE) { + e = SHTABLE_NEXT(t); + VERBOSE(DEBUG_SHCOPY, ("[pid=%T] tabling L %p\n", mypid, ptr)); + SHTABLE_PUSH(t, head, tail, ptr); + CAR(ptr) = (e << _TAG_PRIMARY_SIZE) | LIST_SHARED_UNPROCESSED; + CDR(ptr) = THE_NON_VALUE; + } + goto pop_next; + } + /* else make it visited now */ + switch (primary_tag(tail)) { + case TAG_PRIMARY_LIST: + VERBOSE(DEBUG_SHCOPY, ("[pid=%T] mangling L/L %p\n", mypid, ptr)); + CDR(ptr) = (tail - TAG_PRIMARY_LIST) | TAG_PRIMARY_HEADER; + break; + case TAG_PRIMARY_IMMED1: + VERBOSE(DEBUG_SHCOPY, ("[pid=%T] mangling L/I %p\n", mypid, ptr)); + CAR(ptr) = (head - primary_tag(head)) | TAG_PRIMARY_HEADER; + CDR(ptr) = (tail - TAG_PRIMARY_IMMED1) | primary_tag(head); + break; + case TAG_PRIMARY_BOXED: + BITSTORE_PUT(b, primary_tag(head)); + VERBOSE(DEBUG_SHCOPY, ("[pid=%T] mangling L/B %p\n", mypid, ptr)); + CAR(ptr) = (head - primary_tag(head)) | TAG_PRIMARY_HEADER; + CDR(ptr) = (tail - TAG_PRIMARY_BOXED) | TAG_PRIMARY_HEADER; + break; + } + /* and count it */ + sum += 2; + if (!IS_CONST(head)) { + EQUEUE_PUT(s, head); + } + obj = tail; + break; + } + case TAG_PRIMARY_BOXED: { + Eterm hdr; + ptr = boxed_val(obj); + /* off heap pointers to boxes are copied verbatim */ + if (erts_is_literal(obj,ptr)) { + VERBOSE(DEBUG_SHCOPY, ("[pid=%T] bypassed copying %p is %T\n", mypid, ptr, obj)); + if (IN_LITERAL_PURGE_AREA(info,ptr)) + info->literal_size += size_object(obj); + goto pop_next; + } + hdr = *ptr; + /* if it's visited, don't count it; + if not already shared, make it shared and store it in the table */ + if (primary_tag(hdr) != TAG_PRIMARY_HEADER) { + if (primary_tag(hdr) == BOXED_VISITED) { + e = SHTABLE_NEXT(t); + VERBOSE(DEBUG_SHCOPY, ("[pid=%T] tabling B %p\n", mypid, ptr)); + SHTABLE_PUSH(t, hdr, THE_NON_VALUE, ptr); + *ptr = (e << _TAG_PRIMARY_SIZE) | BOXED_SHARED_UNPROCESSED; + } + goto pop_next; + } + /* else make it visited now */ + VERBOSE(DEBUG_SHCOPY, ("[pid=%T] mangling B %p\n", mypid, ptr)); + *ptr = (hdr - primary_tag(hdr)) + BOXED_VISITED; + /* and count it */ + ASSERT(is_header(hdr)); + switch (hdr & _TAG_HEADER_MASK) { + case ARITYVAL_SUBTAG: { + int arity = header_arity(hdr); + sum += arity + 1; + if (arity == 0) { /* Empty tuple -- unusual. */ + goto pop_next; + } + while (arity-- > 0) { + obj = *++ptr; + if (!IS_CONST(obj)) { + EQUEUE_PUT(s, obj); + } + } + goto pop_next; + } + case FUN_SUBTAG: { + ErlFunThing* funp = (ErlFunThing *) ptr; + unsigned eterms = 1 /* creator */ + funp->num_free; + sz = thing_arityval(hdr); + sum += 1 /* header */ + sz + eterms; + ptr += 1 /* header */ + sz; + while (eterms-- > 0) { + obj = *ptr++; + if (!IS_CONST(obj)) { + EQUEUE_PUT(s, obj); + } + } + goto pop_next; + } + case SUB_BINARY_SUBTAG: { + ErlSubBin* sb = (ErlSubBin *) ptr; + Eterm real_bin = sb->orig; + Uint bit_offset = sb->bitoffs; + Uint bit_size = sb->bitsize; + size_t size = sb->size; + Uint extra_bytes; + Eterm hdr; + if (bit_size + bit_offset > 8) { + sum += ERL_SUB_BIN_SIZE; + extra_bytes = 2; + } else if (bit_size + bit_offset > 0) { + sum += ERL_SUB_BIN_SIZE; + extra_bytes = 1; + } else { + extra_bytes = 0; + } + ASSERT(is_boxed(real_bin) && + (((*boxed_val(real_bin)) & + (_TAG_HEADER_MASK - _BINARY_XXX_MASK - BOXED_VISITED_MASK)) + == _TAG_HEADER_REFC_BIN)); + hdr = *_unchecked_binary_val(real_bin) & ~BOXED_VISITED_MASK; + if (thing_subtag(hdr) == HEAP_BINARY_SUBTAG) { + sum += heap_bin_size(size+extra_bytes); + } else { + ASSERT(thing_subtag(hdr) == REFC_BINARY_SUBTAG); + sum += PROC_BIN_SIZE; + } + goto pop_next; + } + case MAP_SUBTAG: + switch (MAP_HEADER_TYPE(hdr)) { + case MAP_HEADER_TAG_FLATMAP_HEAD : { + flatmap_t *mp = (flatmap_t *) ptr; + Uint n = flatmap_get_size(mp) + 1; + sum += n + 2; + ptr += 2; /* hdr + size words */ + while (n--) { + obj = *ptr++; + if (!IS_CONST(obj)) { + EQUEUE_PUT(s, obj); + } + } + goto pop_next; + } + case MAP_HEADER_TAG_HAMT_HEAD_BITMAP : + case MAP_HEADER_TAG_HAMT_HEAD_ARRAY : + case MAP_HEADER_TAG_HAMT_NODE_BITMAP : { + Uint n = hashmap_bitcount(MAP_HEADER_VAL(hdr)); + sum += 1 + n + header_arity(hdr); + ptr += 1 + header_arity(hdr); + + if (n == 0) { + goto pop_next; + } + while(n--) { + obj = *ptr++; + if (!IS_CONST(obj)) { + EQUEUE_PUT(s, obj); + } + } + goto pop_next; + } + default: + erts_exit(ERTS_ABORT_EXIT, "copy_shared_calculate: bad hashmap type %d\n", MAP_HEADER_TYPE(hdr)); + } + case BIN_MATCHSTATE_SUBTAG: + erts_exit(ERTS_ABORT_EXIT, + "size_shared: matchstate term not allowed"); + default: + sum += thing_arityval(hdr) + 1; + goto pop_next; + } + break; + } + case TAG_PRIMARY_IMMED1: + pop_next: + if (EQUEUE_ISEMPTY(s)) { + /* add sentinel to the table */ + SHTABLE_PUSH(t, THE_NON_VALUE, THE_NON_VALUE, NULL); + /* store persistent info */ + BITSTORE_CLOSE(b); + info->queue_start = s.start; + info->queue_end = s.end; + info->queue_alloc_type = s.alloc_type; + info->bitstore_start = b.wstart; + info->bitstore_alloc_type = b.alloc_type; + info->shtable_start = t.start; + info->shtable_alloc_type = t.alloc_type; + /* single point of return: the size of the object */ + VERBOSE(DEBUG_SHCOPY, ("[pid=%T] size was: %u\n", mypid, sum)); + return sum + info->literal_size; + } + obj = EQUEUE_GET(s); + break; + default: + erts_exit(ERTS_ABORT_EXIT, "[pid=%T] size_shared: bad tag for %#x\n", obj); + } + } +} + +/* + * Copy object "obj" preserving sharing. + * Second half: copy and restore the object. + */ +Uint copy_shared_perform(Eterm obj, Uint size, erts_shcopy_t *info, + Eterm** hpp, ErlOffHeap* off_heap) { + Uint e; + unsigned sz; + Eterm* ptr; + Eterm* hp; + Eterm* hscan; + Eterm result; + Eterm* resp; + Eterm *hbot, *hend; + unsigned remaining; +#ifdef DEBUG + Eterm mypid = erts_get_current_pid(); + Eterm saved_obj = obj; +#endif + + DECLARE_EQUEUE_FROM_INFO(s, info); + DECLARE_BITSTORE_FROM_INFO(b, info); + DECLARE_SHTABLE_FROM_INFO(t, info); + + /* step #0: + ------------------------------------------------------- + get rid of the easy cases first: + - copying constants + - if not a proper process, do flat copy + */ + + if (IS_CONST(obj)) + return obj; + + VERBOSE(DEBUG_SHCOPY, ("[pid=%T] copy_shared_perform %p\n", mypid, obj)); + + /* step #2: was performed before this function was called + ------------------------------------------------------- + allocate new space + */ + + hscan = hp = *hpp; + hbot = hend = hp + size; + + /* step #3: + ------------------------------------------------------- + traverse the term a second time and when traversing: + a. if the object is marked as shared + a1. if the entry contains a forwarding ptr, use that + a2. otherwise, copy it to the new space and store the + forwarding ptr to the entry + b. otherwise, reverse-transform as you do in size_shared + and copy to the new space + */ + + resp = &result; + remaining = 0; + for (;;) { + switch (primary_tag(obj)) { + case TAG_PRIMARY_LIST: { + Eterm head, tail; + ptr = list_val(obj); + /* off heap list pointers are copied verbatim */ + if (erts_is_literal(obj,ptr)) { + if (!IN_LITERAL_PURGE_AREA(info,ptr)) { + *resp = obj; + } else { + Uint bsz = 0; + *resp = copy_struct_x(obj, hbot - hp, &hp, off_heap, &bsz); + hbot -= bsz; + } + goto cleanup_next; + } + head = CAR(ptr); + tail = CDR(ptr); + /* if it is shared */ + if (tail == THE_NON_VALUE) { + e = head >> _TAG_PRIMARY_SIZE; + /* if it has been processed, just use the forwarding pointer */ + if (primary_tag(head) == LIST_SHARED_PROCESSED) { + *resp = make_list(SHTABLE_FWD(t, e)); + goto cleanup_next; + } + /* else, let's process it now, + copy it and keep the forwarding pointer */ + else { + CAR(ptr) = (head - primary_tag(head)) + LIST_SHARED_PROCESSED; + head = SHTABLE_X(t, e); + tail = SHTABLE_Y(t, e); + ptr = &(SHTABLE_X(t, e)); + VERBOSE(DEBUG_SHCOPY, ("[pid=%T] tabled L %p is %p\n", mypid, ptr, SHTABLE_REV(t, e))); + SHTABLE_FWD_UPD(t, e, hp); + } + } + /* if not already clean, clean it up and copy it */ + if (primary_tag(tail) == TAG_PRIMARY_HEADER) { + if (primary_tag(head) == TAG_PRIMARY_HEADER) { + Eterm saved; + BITSTORE_FETCH(b, saved); + VERBOSE(DEBUG_SHCOPY, ("[pid=%T] unmangling L/B %p\n", mypid, ptr)); + CAR(ptr) = head = (head - TAG_PRIMARY_HEADER) + saved; + CDR(ptr) = tail = (tail - TAG_PRIMARY_HEADER) + TAG_PRIMARY_BOXED; + } else { + VERBOSE(DEBUG_SHCOPY, ("[pid=%T] unmangling L/L %p\n", mypid, ptr)); + CDR(ptr) = tail = (tail - TAG_PRIMARY_HEADER) + TAG_PRIMARY_LIST; + } + } else if (primary_tag(head) == TAG_PRIMARY_HEADER) { + VERBOSE(DEBUG_SHCOPY, ("[pid=%T] unmangling L/I %p\n", mypid, ptr)); + CAR(ptr) = head = (head - TAG_PRIMARY_HEADER) | primary_tag(tail); + CDR(ptr) = tail = (tail - primary_tag(tail)) | TAG_PRIMARY_IMMED1; + } else { + ASSERT(0 && "cannot come here"); + goto cleanup_next; + } + /* and its children too */ + if (IS_CONST(head)) { + CAR(hp) = head; + } else { + EQUEUE_PUT_UNCHECKED(s, head); + CAR(hp) = HEAP_ELEM_TO_BE_FILLED; + } + *resp = make_list(hp); + resp = &(CDR(hp)); + hp += 2; + obj = tail; + break; + } + case TAG_PRIMARY_BOXED: { + Eterm hdr; + ptr = boxed_val(obj); + /* off heap pointers to boxes are copied verbatim */ + if (erts_is_literal(obj,ptr)) { + if (!IN_LITERAL_PURGE_AREA(info,ptr)) { + *resp = obj; + } else { + Uint bsz = 0; + *resp = copy_struct_x(obj, hbot - hp, &hp, off_heap, &bsz); + hbot -= bsz; + } + goto cleanup_next; + } + hdr = *ptr; + /* clean it up, unless it's already clean or shared and processed */ + switch (primary_tag(hdr)) { + case TAG_PRIMARY_HEADER: + ASSERT(0 && "cannot come here"); + /* if it is shared and has been processed, + just use the forwarding pointer */ + case BOXED_SHARED_PROCESSED: + e = hdr >> _TAG_PRIMARY_SIZE; + *resp = make_boxed(SHTABLE_FWD(t, e)); + goto cleanup_next; + /* if it is shared but has not been processed yet, let's process + it now: copy it and keep the forwarding pointer */ + case BOXED_SHARED_UNPROCESSED: + e = hdr >> _TAG_PRIMARY_SIZE; + *ptr = (hdr - primary_tag(hdr)) + BOXED_SHARED_PROCESSED; + hdr = SHTABLE_X(t, e); + ASSERT(primary_tag(hdr) == BOXED_VISITED); + VERBOSE(DEBUG_SHCOPY, ("[pid=%T] tabled B %p is %p\n", mypid, ptr, SHTABLE_REV(t, e))); + VERBOSE(DEBUG_SHCOPY, ("[pid=%T] unmangling B %p\n", mypid, ptr)); + SHTABLE_X(t, e) = hdr = (hdr - BOXED_VISITED) + TAG_PRIMARY_HEADER; + SHTABLE_FWD_UPD(t, e, hp); + break; + case BOXED_VISITED: + VERBOSE(DEBUG_SHCOPY, ("[pid=%T] unmangling B %p\n", mypid, ptr)); + *ptr = hdr = (hdr - BOXED_VISITED) + TAG_PRIMARY_HEADER; + break; + } + /* and its children too */ + switch (hdr & _TAG_HEADER_MASK) { + case ARITYVAL_SUBTAG: { + int arity = header_arity(hdr); + *resp = make_boxed(hp); + *hp++ = hdr; + while (arity-- > 0) { + obj = *++ptr; + if (IS_CONST(obj)) { + *hp++ = obj; + } else { + EQUEUE_PUT_UNCHECKED(s, obj); + *hp++ = HEAP_ELEM_TO_BE_FILLED; + } + } + goto cleanup_next; + } + case FUN_SUBTAG: { + ErlFunThing* funp = (ErlFunThing *) ptr; + unsigned eterms = 1 /* creator */ + funp->num_free; + sz = thing_arityval(hdr); + funp = (ErlFunThing *) hp; + *resp = make_fun(hp); + *hp++ = hdr; + ptr++; + while (sz-- > 0) { + *hp++ = *ptr++; + } + while (eterms-- > 0) { + obj = *ptr++; + if (IS_CONST(obj)) { + *hp++ = obj; + } else { + EQUEUE_PUT_UNCHECKED(s, obj); + *hp++ = HEAP_ELEM_TO_BE_FILLED; + } + } + funp->next = off_heap->first; + off_heap->first = (struct erl_off_heap_header*) funp; + erts_refc_inc(&funp->fe->refc, 2); + goto cleanup_next; + } + case MAP_SUBTAG: + *resp = make_flatmap(hp); + *hp++ = hdr; + switch (MAP_HEADER_TYPE(hdr)) { + case MAP_HEADER_TAG_FLATMAP_HEAD : { + flatmap_t *mp = (flatmap_t *) ptr; + Uint n = flatmap_get_size(mp) + 1; + *hp++ = *++ptr; /* keys */ + while (n--) { + obj = *++ptr; + if (IS_CONST(obj)) { + *hp++ = obj; + } else { + EQUEUE_PUT_UNCHECKED(s, obj); + *hp++ = HEAP_ELEM_TO_BE_FILLED; + } + } + goto cleanup_next; + } + case MAP_HEADER_TAG_HAMT_HEAD_BITMAP : + case MAP_HEADER_TAG_HAMT_HEAD_ARRAY : + *hp++ = *++ptr; /* total map size */ + case MAP_HEADER_TAG_HAMT_NODE_BITMAP : { + Uint n = hashmap_bitcount(MAP_HEADER_VAL(hdr)); + while (n--) { + obj = *++ptr; + if (IS_CONST(obj)) { + *hp++ = obj; + } else { + EQUEUE_PUT_UNCHECKED(s, obj); + *hp++ = HEAP_ELEM_TO_BE_FILLED; + } + } + goto cleanup_next; + } + default: + erts_exit(ERTS_ABORT_EXIT, "copy_shared_perform: bad hashmap type %d\n", MAP_HEADER_TYPE(hdr)); + } + case REFC_BINARY_SUBTAG: { + ProcBin* pb = (ProcBin *) ptr; + sz = thing_arityval(hdr); + if (pb->flags) { + erts_emasculate_writable_binary(pb); + } + pb = (ProcBin *) hp; + *resp = make_binary(hp); + *hp++ = hdr; + ptr++; + while (sz-- > 0) { + *hp++ = *ptr++; + } + erts_refc_inc(&pb->val->refc, 2); + pb->next = off_heap->first; + pb->flags = 0; + off_heap->first = (struct erl_off_heap_header*) pb; + OH_OVERHEAD(off_heap, pb->size / sizeof(Eterm)); + goto cleanup_next; + } + case SUB_BINARY_SUBTAG: { + ErlSubBin* sb = (ErlSubBin *) ptr; + Eterm real_bin = sb->orig; + Uint bit_offset = sb->bitoffs; + Uint bit_size = sb->bitsize; + Uint offset = sb->offs; + size_t size = sb->size; + Uint extra_bytes; + Uint real_size; + if ((bit_size + bit_offset) > 8) { + extra_bytes = 2; + } else if ((bit_size + bit_offset) > 0) { + extra_bytes = 1; + } else { + extra_bytes = 0; + } + real_size = size+extra_bytes; + ASSERT(is_boxed(real_bin) && + (((*boxed_val(real_bin)) & + (_TAG_HEADER_MASK - _BINARY_XXX_MASK - BOXED_VISITED_MASK)) + == _TAG_HEADER_REFC_BIN)); + ptr = _unchecked_binary_val(real_bin); + *resp = make_binary(hp); + if (extra_bytes != 0) { + ErlSubBin* res = (ErlSubBin *) hp; + hp += ERL_SUB_BIN_SIZE; + res->thing_word = HEADER_SUB_BIN; + res->size = size; + res->bitsize = bit_size; + res->bitoffs = bit_offset; + res->offs = 0; + res->is_writable = 0; + res->orig = make_binary(hp); + } + if (thing_subtag(*ptr & ~BOXED_VISITED_MASK) == HEAP_BINARY_SUBTAG) { + ErlHeapBin* from = (ErlHeapBin *) ptr; + ErlHeapBin* to = (ErlHeapBin *) hp; + hp += heap_bin_size(real_size); + to->thing_word = header_heap_bin(real_size); + to->size = real_size; + sys_memcpy(to->data, ((byte *)from->data)+offset, real_size); + } else { + ProcBin* from = (ProcBin *) ptr; + ProcBin* to = (ProcBin *) hp; + ASSERT(thing_subtag(*ptr & ~BOXED_VISITED_MASK) == REFC_BINARY_SUBTAG); + if (from->flags) { + erts_emasculate_writable_binary(from); + } + hp += PROC_BIN_SIZE; + to->thing_word = HEADER_PROC_BIN; + to->size = real_size; + to->val = from->val; + erts_refc_inc(&to->val->refc, 2); + to->bytes = from->bytes + offset; + to->next = off_heap->first; + to->flags = 0; + off_heap->first = (struct erl_off_heap_header*) to; + OH_OVERHEAD(off_heap, to->size / sizeof(Eterm)); + } + goto cleanup_next; + } + case EXTERNAL_PID_SUBTAG: + case EXTERNAL_PORT_SUBTAG: + case EXTERNAL_REF_SUBTAG: { + ExternalThing *etp = (ExternalThing *) hp; + sz = thing_arityval(hdr); + *resp = make_external(hp); + *hp++ = hdr; + ptr++; + while (sz-- > 0) { + *hp++ = *ptr++; + } + etp->next = off_heap->first; + off_heap->first = (struct erl_off_heap_header*) etp; + erts_refc_inc(&etp->node->refc, 2); + goto cleanup_next; + } + default: + sz = thing_arityval(hdr); + *resp = make_boxed(hp); + *hp++ = hdr; + ptr++; + while (sz-- > 0) { + *hp++ = *ptr++; + } + goto cleanup_next; + } + break; + } + case TAG_PRIMARY_IMMED1: + *resp = obj; + cleanup_next: + if (EQUEUE_ISEMPTY(s)) { + goto all_clean; + } + obj = EQUEUE_GET(s); + for (;;) { + ASSERT(hscan < hp); + if (remaining == 0) { + if (*hscan == HEAP_ELEM_TO_BE_FILLED) { + resp = hscan; + hscan += 2; + break; /* scanning loop */ + } else if (primary_tag(*hscan) == TAG_PRIMARY_HEADER) { + switch (*hscan & _TAG_HEADER_MASK) { + case ARITYVAL_SUBTAG: + remaining = header_arity(*hscan); + hscan++; + break; + case FUN_SUBTAG: { + ErlFunThing* funp = (ErlFunThing *) hscan; + hscan += 1 + thing_arityval(*hscan); + remaining = 1 + funp->num_free; + break; + } + case MAP_SUBTAG: + switch (MAP_HEADER_TYPE(*hscan)) { + case MAP_HEADER_TAG_FLATMAP_HEAD : { + flatmap_t *mp = (flatmap_t *) hscan; + remaining = flatmap_get_size(mp) + 1; + hscan += 2; + break; + } + case MAP_HEADER_TAG_HAMT_HEAD_BITMAP : + case MAP_HEADER_TAG_HAMT_HEAD_ARRAY : + case MAP_HEADER_TAG_HAMT_NODE_BITMAP : + remaining = hashmap_bitcount(MAP_HEADER_VAL(*hscan)); + hscan += MAP_HEADER_ARITY(*hscan) + 1; + break; + default: + erts_exit(ERTS_ABORT_EXIT, + "copy_shared_perform: bad hashmap type %d\n", + MAP_HEADER_TYPE(*hscan)); + } + break; + case SUB_BINARY_SUBTAG: + ASSERT(((ErlSubBin *) hscan)->bitoffs + + ((ErlSubBin *) hscan)->bitsize > 0); + hscan += ERL_SUB_BIN_SIZE; + break; + default: + hscan += 1 + thing_arityval(*hscan); + break; + } + } else { + hscan++; + } + } else if (*hscan == HEAP_ELEM_TO_BE_FILLED) { + resp = hscan++; + remaining--; + break; /* scanning loop */ + } else { + hscan++; + remaining--; + } + } + ASSERT(resp < hp); + break; + default: + erts_exit(ERTS_ABORT_EXIT, "size_shared: bad tag for %#x\n", obj); + } + } + + /* step #4: + ------------------------------------------------------- + traverse the table and reverse-transform all stored entries + */ + +all_clean: + for (e = 0; ; e += SHTABLE_INCR) { + ptr = SHTABLE_REV(t, e); + if (ptr == NULL) + break; + VERBOSE(DEBUG_SHCOPY, ("[copy] restoring shared: %x\n", ptr)); + /* entry was a list */ + if (SHTABLE_Y(t, e) != THE_NON_VALUE) { + VERBOSE(DEBUG_SHCOPY, ("[pid=%T] untabling L %p\n", mypid, ptr)); + CAR(ptr) = SHTABLE_X(t, e); + CDR(ptr) = SHTABLE_Y(t, e); + } + /* entry was boxed */ + else { + VERBOSE(DEBUG_SHCOPY, ("[pid=%T] untabling B %p\n", mypid, ptr)); + *ptr = SHTABLE_X(t, e); + ASSERT(primary_tag(*ptr) == TAG_PRIMARY_HEADER); + } + } + +#ifdef DEBUG + if (eq(saved_obj, result) == 0) { + erts_fprintf(stderr, "original = %T\n", saved_obj); + erts_fprintf(stderr, "copy = %T\n", result); + erts_exit(ERTS_ABORT_EXIT, "copy (shared) not equal to source\n"); + } +#endif + + VERBOSE(DEBUG_SHCOPY, ("[pid=%T] original was %T\n", mypid, saved_obj)); + VERBOSE(DEBUG_SHCOPY, ("[pid=%T] copy is %T\n", mypid, result)); + VERBOSE(DEBUG_SHCOPY, ("[pid=%T] result is at %p\n", mypid, result)); + + ASSERT(hbot == hp); + ASSERT(size == ((hp - *hpp) + (hend - hbot))); + *hpp = hend; + return result; +} + + /* * Copy a term that is guaranteed to be contained in a single * heap block. The heap block is copied word by word, and any @@ -519,21 +1768,12 @@ Eterm copy_struct(Eterm obj, Uint sz, Eterm** hpp, ErlOffHeap* off_heap) * * NOTE: Assumes that term is a tuple (ptr is an untagged tuple ptr). */ -#if HALFWORD_HEAP -Eterm copy_shallow_rel(Eterm* ptr, Uint sz, Eterm** hpp, ErlOffHeap* off_heap, - Eterm* src_base) -#else Eterm copy_shallow(Eterm* ptr, Uint sz, Eterm** hpp, ErlOffHeap* off_heap) -#endif { Eterm* tp = ptr; Eterm* hp = *hpp; const Eterm res = make_tuple(hp); -#if HALFWORD_HEAP - const Sint offs = COMPRESS_POINTER(hp - (tp - src_base)); -#else const Sint offs = (hp - tp) * sizeof(Eterm); -#endif while (sz--) { Eterm val = *tp++; @@ -565,11 +1805,6 @@ Eterm copy_shallow(Eterm* ptr, Uint sz, Eterm** hpp, ErlOffHeap* off_heap) erts_refc_inc(&funp->fe->refc, 2); } goto off_heap_common; - - case MAP_SUBTAG: - *hp++ = *tp++; - sz--; - break; case EXTERNAL_PID_SUBTAG: case EXTERNAL_PORT_SUBTAG: case EXTERNAL_REF_SUBTAG: @@ -605,7 +1840,6 @@ Eterm copy_shallow(Eterm* ptr, Uint sz, Eterm** hpp, ErlOffHeap* off_heap) } } *hpp = hp; - return res; } @@ -614,18 +1848,24 @@ Eterm copy_shallow(Eterm* ptr, Uint sz, Eterm** hpp, ErlOffHeap* off_heap) * move markers. * Typically used to copy a multi-fragmented message (from NIF). */ -void move_multi_frags(Eterm** hpp, ErlOffHeap* off_heap, ErlHeapFragment* first, - Eterm* refs, unsigned nrefs) +void erts_move_multi_frags(Eterm** hpp, ErlOffHeap* off_heap, ErlHeapFragment* first, + Eterm* refs, unsigned nrefs, int literals) { ErlHeapFragment* bp; Eterm* hp_start = *hpp; Eterm* hp_end; Eterm* hp; unsigned i; + Eterm literal_tag; + +#ifdef TAG_LITERAL_PTR + literal_tag = (Eterm) literals ? TAG_LITERAL_PTR : 0; +#else + literal_tag = (Eterm) 0; +#endif for (bp=first; bp!=NULL; bp=bp->next) { - move_one_frag(hpp, bp->mem, bp->used_size, off_heap); - OH_OVERHEAD(off_heap, bp->off_heap.overhead); + move_one_frag(hpp, bp, off_heap, literals); } hp_end = *hpp; for (hp=hp_start; hp<hp_end; ++hp) { @@ -638,6 +1878,9 @@ void move_multi_frags(Eterm** hpp, ErlOffHeap* off_heap, ErlHeapFragment* first, val = *ptr; if (IS_MOVED_BOXED(val)) { ASSERT(is_boxed(val)); +#ifdef TAG_LITERAL_PTR + val |= literal_tag; +#endif *hp = val; } break; @@ -645,7 +1888,11 @@ void move_multi_frags(Eterm** hpp, ErlOffHeap* off_heap, ErlHeapFragment* first, ptr = list_val(gval); val = *ptr; if (IS_MOVED_CONS(val)) { - *hp = ptr[1]; + val = ptr[1]; +#ifdef TAG_LITERAL_PTR + val |= literal_tag; +#endif + *hp = val; } break; case TAG_PRIMARY_HEADER: @@ -656,15 +1903,15 @@ void move_multi_frags(Eterm** hpp, ErlOffHeap* off_heap, ErlHeapFragment* first, } } for (i=0; i<nrefs; ++i) { - refs[i] = follow_moved(refs[i]); + refs[i] = follow_moved(refs[i], literal_tag); } } static void -move_one_frag(Eterm** hpp, Eterm* src, Uint src_sz, ErlOffHeap* off_heap) +move_one_frag(Eterm** hpp, ErlHeapFragment* frag, ErlOffHeap* off_heap, int literals) { - Eterm* ptr = src; - Eterm* end = ptr + src_sz; + Eterm* ptr = frag->mem; + Eterm* end = ptr + frag->used_size; Eterm dummy_ref; Eterm* hp = *hpp; @@ -695,5 +1942,6 @@ move_one_frag(Eterm** hpp, Eterm* src, Uint src_sz, ErlOffHeap* off_heap) } } *hpp = hp; + OH_OVERHEAD(off_heap, frag->off_heap.overhead); + frag->off_heap.first = NULL; } - diff --git a/erts/emulator/beam/dist.c b/erts/emulator/beam/dist.c index ec07ddcd9c..fa385f105d 100644 --- a/erts/emulator/beam/dist.c +++ b/erts/emulator/beam/dist.c @@ -1,18 +1,19 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 1996-2013. All Rights Reserved. + * Copyright Ericsson AB 1996-2014. All Rights Reserved. * - * The contents of this file are subject to the Erlang Public License, - * Version 1.1, (the "License"); you may not use this file except in - * compliance with the License. You should have received a copy of the - * Erlang Public License along with this software. If not, it can be - * retrieved online at http://www.erlang.org/. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Software distributed under the License is distributed on an "AS IS" - * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See - * the License for the specific language governing rights and limitations - * under the License. + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. * * %CopyrightEnd% */ @@ -44,6 +45,8 @@ #include "erl_thr_progress.h" #include "dtrace-wrapper.h" +#define DIST_CTL_DEFAULT_SIZE 64 + /* Turn this on to get printouts of all distribution messages * which go on the line */ @@ -65,9 +68,13 @@ static void bw(byte *buf, ErlDrvSizeT sz) static void dist_msg_dbg(ErtsDistExternal *edep, char *what, byte *buf, int sz) { + ErtsHeapFactory factory; + DeclareTmpHeapNoproc(ctl_default,DIST_CTL_DEFAULT_SIZE); + Eterm* ctl = ctl_default; byte *extp = edep->extp; Eterm msg; - Sint size = erts_decode_dist_ext_size(edep); + Sint ctl_len; + Sint size = ctl_len = erts_decode_dist_ext_size(edep); if (size < 0) { erts_fprintf(stderr, "DIST MSG DEBUG: erts_decode_dist_ext_size(%s) failed:\n", @@ -75,10 +82,9 @@ dist_msg_dbg(ErtsDistExternal *edep, char *what, byte *buf, int sz) bw(buf, sz); } else { - Eterm *hp; ErlHeapFragment *mbuf = new_message_buffer(size); - hp = mbuf->mem; - msg = erts_decode_dist_ext(&hp, &mbuf->off_heap, edep); + erts_factory_static_init(&factory, ctl, ctl_len, &mbuf->off_heap); + msg = erts_decode_dist_ext(&factory, edep); if (is_value(msg)) erts_fprintf(stderr, " %s: %T\n", what, msg); else { @@ -119,7 +125,7 @@ Export* dmonitor_p_trap = NULL; /* forward declarations */ static void clear_dist_entry(DistEntry*); -static int dsig_send(ErtsDSigData *, Eterm, Eterm, int); +static int dsig_send_ctl(ErtsDSigData* dsdp, Eterm ctl, int force_busy); static void send_nodes_mon_msgs(Process *, Eterm, Eterm, Eterm, Eterm); static void init_nodes_monitors(void); @@ -372,10 +378,11 @@ static void doit_node_link_net_exits(ErtsLink *lnk, void *vnecp) ASSERT(lnk->type == LINK_NODE); if (is_internal_pid(lnk->pid)) { ErtsProcLocks rp_locks = ERTS_PROC_LOCK_LINK; - rp = erts_pid2proc(NULL, 0, lnk->pid, rp_locks); - if (!rp) { + ErlOffHeap *ohp; + rp = erts_proc_lookup(lnk->pid); + if (!rp) goto done; - } + erts_smp_proc_lock(rp, rp_locks); rlnk = erts_remove_link(&ERTS_P_LINKS(rp), name); if (rlnk != NULL) { ASSERT(is_atom(rlnk->pid) && (rlnk->type == LINK_NODE)); @@ -383,16 +390,14 @@ static void doit_node_link_net_exits(ErtsLink *lnk, void *vnecp) } n = ERTS_LINK_REFC(lnk); for (i = 0; i < n; ++i) { - ErlHeapFragment* bp; - ErlOffHeap *ohp; Eterm tup; - Eterm *hp = erts_alloc_message_heap(3,&bp,&ohp,rp,&rp_locks); + Eterm *hp; + ErtsMessage *msgp; + + msgp = erts_alloc_message_heap(rp, &rp_locks, + 3, &hp, &ohp); tup = TUPLE2(hp, am_nodedown, name); - erts_queue_message(rp, &rp_locks, bp, tup, NIL -#ifdef USE_VM_PROBES - , NIL -#endif - ); + erts_queue_message(rp, &rp_locks, msgp, tup, NIL); } erts_smp_proc_unlock(rp, rp_locks); } @@ -622,9 +627,7 @@ alloc_dist_obuf(Uint size) ErtsDistOutputBuf *obuf; Uint obuf_size = sizeof(ErtsDistOutputBuf)+sizeof(byte)*(size-1); Binary *bin = erts_bin_drv_alloc(obuf_size); - bin->flags = BIN_FLAG_DRV; erts_refc_init(&bin->refc, 1); - bin->orig_size = (SWord) obuf_size; obuf = (ErtsDistOutputBuf *) &bin->orig_bytes[0]; #ifdef DEBUG obuf->dbg_pattern = ERTS_DIST_OUTPUT_BUF_DBG_PATTERN; @@ -709,6 +712,47 @@ static void clear_dist_entry(DistEntry *dep) } } +void erts_dsend_context_dtor(Binary* ctx_bin) +{ + ErtsSendContext* ctx = ERTS_MAGIC_BIN_DATA(ctx_bin); + switch (ctx->dss.phase) { + case ERTS_DSIG_SEND_PHASE_MSG_SIZE: + DESTROY_SAVED_WSTACK(&ctx->dss.u.sc.wstack); + break; + case ERTS_DSIG_SEND_PHASE_MSG_ENCODE: + DESTROY_SAVED_WSTACK(&ctx->dss.u.ec.wstack); + break; + default:; + } + if (ctx->dss.phase >= ERTS_DSIG_SEND_PHASE_ALLOC && ctx->dss.obuf) { + free_dist_obuf(ctx->dss.obuf); + } + if (ctx->dep_to_deref) + erts_deref_dist_entry(ctx->dep_to_deref); +} + +Eterm erts_dsend_export_trap_context(Process* p, ErtsSendContext* ctx) +{ + struct exported_ctx { + ErtsSendContext ctx; + ErtsAtomCacheMap acm; + }; + Binary* ctx_bin = erts_create_magic_binary(sizeof(struct exported_ctx), + erts_dsend_context_dtor); + struct exported_ctx* dst = ERTS_MAGIC_BIN_DATA(ctx_bin); + Eterm* hp = HAlloc(p, PROC_BIN_SIZE); + + sys_memcpy(&dst->ctx, ctx, sizeof(ErtsSendContext)); + ASSERT(ctx->dss.ctl == make_tuple(ctx->ctl_heap)); + dst->ctx.dss.ctl = make_tuple(dst->ctx.ctl_heap); + if (ctx->dss.acmp) { + sys_memcpy(&dst->acm, ctx->dss.acmp, sizeof(ErtsAtomCacheMap)); + dst->ctx.dss.acmp = &dst->acm; + } + return erts_mk_magic_binary_term(&hp, &MSO(p), ctx_bin); +} + + /* * The erts_dsig_send_*() functions implemented below, sends asynchronous * distributed signals to other Erlang nodes. Before sending a distributed @@ -731,7 +775,7 @@ erts_dsig_send_link(ErtsDSigData *dsdp, Eterm local, Eterm remote) int res; UseTmpHeapNoproc(4); - res = dsig_send(dsdp, ctl, THE_NON_VALUE, 0); + res = dsig_send_ctl(dsdp, ctl, 0); UnUseTmpHeapNoproc(4); return res; } @@ -744,7 +788,7 @@ erts_dsig_send_unlink(ErtsDSigData *dsdp, Eterm local, Eterm remote) int res; UseTmpHeapNoproc(4); - res = dsig_send(dsdp, ctl, THE_NON_VALUE, 0); + res = dsig_send_ctl(dsdp, ctl, 0); UnUseTmpHeapNoproc(4); return res; } @@ -772,7 +816,7 @@ erts_dsig_send_m_exit(ErtsDSigData *dsdp, Eterm watcher, Eterm watched, erts_smp_de_links_unlock(dsdp->dep); #endif - res = dsig_send(dsdp, ctl, THE_NON_VALUE, 1); + res = dsig_send_ctl(dsdp, ctl, 1); UnUseTmpHeapNoproc(6); return res; } @@ -793,7 +837,7 @@ erts_dsig_send_monitor(ErtsDSigData *dsdp, Eterm watcher, Eterm watched, make_small(DOP_MONITOR_P), watcher, watched, ref); - res = dsig_send(dsdp, ctl, THE_NON_VALUE, 0); + res = dsig_send_ctl(dsdp, ctl, 0); UnUseTmpHeapNoproc(5); return res; } @@ -815,18 +859,17 @@ erts_dsig_send_demonitor(ErtsDSigData *dsdp, Eterm watcher, make_small(DOP_DEMONITOR_P), watcher, watched, ref); - res = dsig_send(dsdp, ctl, THE_NON_VALUE, force); + res = dsig_send_ctl(dsdp, ctl, force); UnUseTmpHeapNoproc(5); return res; } int -erts_dsig_send_msg(ErtsDSigData *dsdp, Eterm remote, Eterm message) +erts_dsig_send_msg(Eterm remote, Eterm message, ErtsSendContext* ctx) { Eterm ctl; - DeclareTmpHeapNoproc(ctl_heap,5); Eterm token = NIL; - Process *sender = dsdp->proc; + Process *sender = ctx->dsd.proc; int res; #ifdef USE_VM_PROBES Sint tok_label = 0; @@ -838,12 +881,7 @@ erts_dsig_send_msg(ErtsDSigData *dsdp, Eterm remote, Eterm message) DTRACE_CHARBUF(receiver_name, 64); #endif - UseTmpHeapNoproc(5); - if (SEQ_TRACE_TOKEN(sender) != NIL -#ifdef USE_VM_PROBES - && SEQ_TRACE_TOKEN(sender) != am_have_dt_utag -#endif - ) { + if (have_seqtrace(SEQ_TRACE_TOKEN(sender))) { seq_trace_update_send(sender); token = SEQ_TRACE_TOKEN(sender); seq_trace_output(token, message, SEQ_TRACE_SEND, remote, sender); @@ -852,13 +890,13 @@ erts_dsig_send_msg(ErtsDSigData *dsdp, Eterm remote, Eterm message) *node_name = *sender_name = *receiver_name = '\0'; if (DTRACE_ENABLED(message_send) || DTRACE_ENABLED(message_send_remote)) { erts_snprintf(node_name, sizeof(DTRACE_CHARBUF_NAME(node_name)), - "%T", dsdp->dep->sysname); + "%T", ctx->dsd.dep->sysname); erts_snprintf(sender_name, sizeof(DTRACE_CHARBUF_NAME(sender_name)), "%T", sender->common.id); erts_snprintf(receiver_name, sizeof(DTRACE_CHARBUF_NAME(receiver_name)), "%T", remote); msize = size_object(message); - if (token != NIL && token != am_have_dt_utag) { + if (have_seqtrace(token)) { tok_label = signed_val(SEQ_TRACE_T_LABEL(token)); tok_lastcnt = signed_val(SEQ_TRACE_T_LASTCNT(token)); tok_serial = signed_val(SEQ_TRACE_T_SERIAL(token)); @@ -867,26 +905,28 @@ erts_dsig_send_msg(ErtsDSigData *dsdp, Eterm remote, Eterm message) #endif if (token != NIL) - ctl = TUPLE4(&ctl_heap[0], + ctl = TUPLE4(&ctx->ctl_heap[0], make_small(DOP_SEND_TT), am_Cookie, remote, token); else - ctl = TUPLE3(&ctl_heap[0], make_small(DOP_SEND), am_Cookie, remote); + ctl = TUPLE3(&ctx->ctl_heap[0], make_small(DOP_SEND), am_Cookie, remote); DTRACE6(message_send, sender_name, receiver_name, msize, tok_label, tok_lastcnt, tok_serial); DTRACE7(message_send_remote, sender_name, node_name, receiver_name, msize, tok_label, tok_lastcnt, tok_serial); - res = dsig_send(dsdp, ctl, message, 0); - UnUseTmpHeapNoproc(5); + ctx->dss.ctl = ctl; + ctx->dss.msg = message; + ctx->dss.force_busy = 0; + res = erts_dsig_send(&ctx->dsd, &ctx->dss); return res; } int -erts_dsig_send_reg_msg(ErtsDSigData *dsdp, Eterm remote_name, Eterm message) +erts_dsig_send_reg_msg(Eterm remote_name, Eterm message, + ErtsSendContext* ctx) { Eterm ctl; - DeclareTmpHeapNoproc(ctl_heap,6); Eterm token = NIL; - Process *sender = dsdp->proc; + Process *sender = ctx->dsd.proc; int res; #ifdef USE_VM_PROBES Sint tok_label = 0; @@ -898,12 +938,7 @@ erts_dsig_send_reg_msg(ErtsDSigData *dsdp, Eterm remote_name, Eterm message) DTRACE_CHARBUF(receiver_name, 128); #endif - UseTmpHeapNoproc(6); - if (SEQ_TRACE_TOKEN(sender) != NIL -#ifdef USE_VM_PROBES - && SEQ_TRACE_TOKEN(sender) != am_have_dt_utag -#endif - ) { + if (have_seqtrace(SEQ_TRACE_TOKEN(sender))) { seq_trace_update_send(sender); token = SEQ_TRACE_TOKEN(sender); seq_trace_output(token, message, SEQ_TRACE_SEND, remote_name, sender); @@ -912,13 +947,13 @@ erts_dsig_send_reg_msg(ErtsDSigData *dsdp, Eterm remote_name, Eterm message) *node_name = *sender_name = *receiver_name = '\0'; if (DTRACE_ENABLED(message_send) || DTRACE_ENABLED(message_send_remote)) { erts_snprintf(node_name, sizeof(DTRACE_CHARBUF_NAME(node_name)), - "%T", dsdp->dep->sysname); + "%T", ctx->dsd.dep->sysname); erts_snprintf(sender_name, sizeof(DTRACE_CHARBUF_NAME(sender_name)), "%T", sender->common.id); erts_snprintf(receiver_name, sizeof(DTRACE_CHARBUF_NAME(receiver_name)), "{%T,%s}", remote_name, node_name); msize = size_object(message); - if (token != NIL && token != am_have_dt_utag) { + if (have_seqtrace(token)) { tok_label = signed_val(SEQ_TRACE_T_LABEL(token)); tok_lastcnt = signed_val(SEQ_TRACE_T_LASTCNT(token)); tok_serial = signed_val(SEQ_TRACE_T_SERIAL(token)); @@ -927,17 +962,19 @@ erts_dsig_send_reg_msg(ErtsDSigData *dsdp, Eterm remote_name, Eterm message) #endif if (token != NIL) - ctl = TUPLE5(&ctl_heap[0], make_small(DOP_REG_SEND_TT), + ctl = TUPLE5(&ctx->ctl_heap[0], make_small(DOP_REG_SEND_TT), sender->common.id, am_Cookie, remote_name, token); else - ctl = TUPLE4(&ctl_heap[0], make_small(DOP_REG_SEND), + ctl = TUPLE4(&ctx->ctl_heap[0], make_small(DOP_REG_SEND), sender->common.id, am_Cookie, remote_name); DTRACE6(message_send, sender_name, receiver_name, msize, tok_label, tok_lastcnt, tok_serial); DTRACE7(message_send_remote, sender_name, node_name, receiver_name, msize, tok_label, tok_lastcnt, tok_serial); - res = dsig_send(dsdp, ctl, message, 0); - UnUseTmpHeapNoproc(6); + ctx->dss.ctl = ctl; + ctx->dss.msg = message; + ctx->dss.force_busy = 0; + res = erts_dsig_send(&ctx->dsd, &ctx->dss); return res; } @@ -961,11 +998,7 @@ erts_dsig_send_exit_tt(ErtsDSigData *dsdp, Eterm local, Eterm remote, #endif UseTmpHeapNoproc(6); - if (token != NIL -#ifdef USE_VM_PROBES - && token != am_have_dt_utag -#endif - ) { + if (have_seqtrace(token)) { seq_trace_update_send(dsdp->proc); seq_trace_output_exit(token, reason, SEQ_TRACE_SEND, remote, local); ctl = TUPLE5(&ctl_heap[0], @@ -984,7 +1017,7 @@ erts_dsig_send_exit_tt(ErtsDSigData *dsdp, Eterm local, Eterm remote, "{%T,%s}", remote, node_name); erts_snprintf(reason_str, sizeof(DTRACE_CHARBUF_NAME(reason_str)), "%T", reason); - if (token != NIL && token != am_have_dt_utag) { + if (have_seqtrace(token)) { tok_label = signed_val(SEQ_TRACE_T_LABEL(token)); tok_lastcnt = signed_val(SEQ_TRACE_T_LASTCNT(token)); tok_serial = signed_val(SEQ_TRACE_T_SERIAL(token)); @@ -994,7 +1027,7 @@ erts_dsig_send_exit_tt(ErtsDSigData *dsdp, Eterm local, Eterm remote, DTRACE7(process_exit_signal_remote, sender_name, node_name, remote_name, reason_str, tok_label, tok_lastcnt, tok_serial); /* forced, i.e ignore busy */ - res = dsig_send(dsdp, ctl, THE_NON_VALUE, 1); + res = dsig_send_ctl(dsdp, ctl, 1); UnUseTmpHeapNoproc(6); return res; } @@ -1010,7 +1043,7 @@ erts_dsig_send_exit(ErtsDSigData *dsdp, Eterm local, Eterm remote, Eterm reason) ctl = TUPLE4(&ctl_heap[0], make_small(DOP_EXIT), local, remote, reason); /* forced, i.e ignore busy */ - res = dsig_send(dsdp, ctl, THE_NON_VALUE, 1); + res = dsig_send_ctl(dsdp, ctl, 1); UnUseTmpHeapNoproc(5); return res; } @@ -1026,7 +1059,7 @@ erts_dsig_send_exit2(ErtsDSigData *dsdp, Eterm local, Eterm remote, Eterm reason ctl = TUPLE4(&ctl_heap[0], make_small(DOP_EXIT2), local, remote, reason); - res = dsig_send(dsdp, ctl, THE_NON_VALUE, 0); + res = dsig_send_ctl(dsdp, ctl, 0); UnUseTmpHeapNoproc(5); return res; } @@ -1043,7 +1076,7 @@ erts_dsig_send_group_leader(ErtsDSigData *dsdp, Eterm leader, Eterm remote) ctl = TUPLE3(&ctl_heap[0], make_small(DOP_GROUP_LEADER), leader, remote); - res = dsig_send(dsdp, ctl, THE_NON_VALUE, 0); + res = dsig_send_ctl(dsdp, ctl, 0); UnUseTmpHeapNoproc(4); return res; } @@ -1091,7 +1124,6 @@ int erts_net_message(Port *prt, byte *buf, ErlDrvSizeT len) { -#define DIST_CTL_DEFAULT_SIZE 64 ErtsDistExternal ede; byte *t; Sint ctl_len; @@ -1104,7 +1136,7 @@ int erts_net_message(Port *prt, Process* rp; DeclareTmpHeapNoproc(ctl_default,DIST_CTL_DEFAULT_SIZE); Eterm* ctl = ctl_default; - ErlOffHeap off_heap; + ErtsHeapFactory factory; Eterm* hp; Sint type; Eterm token; @@ -1118,9 +1150,6 @@ int erts_net_message(Port *prt, #endif UseTmpHeapNoproc(DIST_CTL_DEFAULT_SIZE); - /* Thanks to Luke Gorrie */ - off_heap.first = NULL; - off_heap.overhead = 0; ERTS_SMP_CHK_NO_PROC_LOCKS; @@ -1181,14 +1210,15 @@ int erts_net_message(Port *prt, } hp = ctl; - arg = erts_decode_dist_ext(&hp, &off_heap, &ede); + erts_factory_tmp_init(&factory, ctl, ctl_len, ERTS_ALC_T_DCTRL_BUF); + arg = erts_decode_dist_ext(&factory, &ede); if (is_non_value(arg)) { #ifdef ERTS_DIST_MSG_DBG - erts_fprintf(stderr, "DIST MSG DEBUG: erts_dist_ext_size(CTL) failed:\n"); + erts_fprintf(stderr, "DIST MSG DEBUG: erts_decode_dist_ext(CTL) failed:\n"); bw(buf, orig_len); #endif PURIFY_MSG("data error"); - goto data_error; + goto decode_error; } ctl_len = t - buf; @@ -1419,7 +1449,7 @@ int erts_net_message(Port *prt, ErlOffHeap *ohp; ASSERT(xsize); heap_frag = erts_dist_ext_trailer(ede_copy); - ERTS_INIT_HEAP_FRAG(heap_frag, token_size); + ERTS_INIT_HEAP_FRAG(heap_frag, token_size, token_size); hp = heap_frag->mem; ohp = &heap_frag->off_heap; token = tuple[5]; @@ -1468,7 +1498,7 @@ int erts_net_message(Port *prt, ErlOffHeap *ohp; ASSERT(xsize); heap_frag = erts_dist_ext_trailer(ede_copy); - ERTS_INIT_HEAP_FRAG(heap_frag, token_size); + ERTS_INIT_HEAP_FRAG(heap_frag, token_size, token_size); hp = heap_frag->mem; ohp = &heap_frag->off_heap; token = tuple[4]; @@ -1668,7 +1698,7 @@ int erts_net_message(Port *prt, goto invalid_message; } - erts_cleanup_offheap(&off_heap); + erts_factory_close(&factory); if (ctl != ctl_default) { erts_free(ERTS_ALC_T_DCTRL_BUF, (void *) ctl); } @@ -1681,206 +1711,248 @@ int erts_net_message(Port *prt, erts_dsprintf(dsbufp, "Invalid distribution message: %.200T", arg); erts_send_error_to_logger_nogl(dsbufp); } - data_error: +decode_error: PURIFY_MSG("data error"); - erts_cleanup_offheap(&off_heap); + erts_factory_close(&factory); if (ctl != ctl_default) { erts_free(ERTS_ALC_T_DCTRL_BUF, (void *) ctl); } +data_error: UnUseTmpHeapNoproc(DIST_CTL_DEFAULT_SIZE); erts_deliver_port_exit(prt, dep->cid, am_killed, 0); ERTS_SMP_CHK_NO_PROC_LOCKS; return -1; } -static int -dsig_send(ErtsDSigData *dsdp, Eterm ctl, Eterm msg, int force_busy) +static int dsig_send_ctl(ErtsDSigData* dsdp, Eterm ctl, int force_busy) +{ + struct erts_dsig_send_context ctx; + int ret; + ctx.ctl = ctl; + ctx.msg = THE_NON_VALUE; + ctx.force_busy = force_busy; + ctx.phase = ERTS_DSIG_SEND_PHASE_INIT; +#ifdef DEBUG + ctx.reds = 1; /* provoke assert below (no reduction count without msg) */ +#endif + ret = erts_dsig_send(dsdp, &ctx); + ASSERT(ret != ERTS_DSIG_SEND_CONTINUE); + return ret; +} + +int +erts_dsig_send(ErtsDSigData *dsdp, struct erts_dsig_send_context* ctx) { + int retval; + Sint initial_reds = ctx->reds; Eterm cid; - int suspended = 0; - int resume = 0; - Uint32 pass_through_size; - Uint data_size, dhdr_ext_size; - ErtsAtomCacheMap *acmp; - ErtsDistOutputBuf *obuf; - DistEntry *dep = dsdp->dep; - Uint32 flags = dep->flags; - Process *c_p = dsdp->proc; - if (!c_p || dsdp->no_suspend) - force_busy = 1; + while (1) { + switch (ctx->phase) { + case ERTS_DSIG_SEND_PHASE_INIT: + ctx->flags = dsdp->dep->flags; + ctx->c_p = dsdp->proc; - ERTS_SMP_LC_ASSERT(!c_p - || (ERTS_PROC_LOCK_MAIN - == erts_proc_lc_my_proc_locks(c_p))); + if (!ctx->c_p || dsdp->no_suspend) + ctx->force_busy = 1; - if (!erts_is_alive) - return ERTS_DSIG_SEND_OK; + ERTS_SMP_LC_ASSERT(!ctx->c_p + || (ERTS_PROC_LOCK_MAIN + == erts_proc_lc_my_proc_locks(ctx->c_p))); - if (flags & DFLAG_DIST_HDR_ATOM_CACHE) { - acmp = erts_get_atom_cache_map(c_p); - pass_through_size = 0; - } - else { - acmp = NULL; - pass_through_size = 1; - } + if (!erts_is_alive) + return ERTS_DSIG_SEND_OK; -#ifdef ERTS_DIST_MSG_DBG - erts_fprintf(stderr, ">>%s CTL: %T\n", pass_through_size ? "P" : " ", ctl); - if (is_value(msg)) - erts_fprintf(stderr, " MSG: %T\n", msg); -#endif + if (ctx->flags & DFLAG_DIST_HDR_ATOM_CACHE) { + ctx->acmp = erts_get_atom_cache_map(ctx->c_p); + ctx->pass_through_size = 0; + } + else { + ctx->acmp = NULL; + ctx->pass_through_size = 1; + } - data_size = pass_through_size; - erts_reset_atom_cache_map(acmp); - data_size += erts_encode_dist_ext_size(ctl, flags, acmp); - if (is_value(msg)) - data_size += erts_encode_dist_ext_size(msg, flags, acmp); - erts_finalize_atom_cache_map(acmp, flags); + #ifdef ERTS_DIST_MSG_DBG + erts_fprintf(stderr, ">>%s CTL: %T\n", ctx->pass_through_size ? "P" : " ", ctx->ctl); + if (is_value(ctx->msg)) + erts_fprintf(stderr, " MSG: %T\n", ctx->msg); + #endif + + ctx->data_size = ctx->pass_through_size; + erts_reset_atom_cache_map(ctx->acmp); + erts_encode_dist_ext_size(ctx->ctl, ctx->flags, ctx->acmp, &ctx->data_size); + + if (is_value(ctx->msg)) { + ctx->u.sc.wstack.wstart = NULL; + ctx->u.sc.flags = ctx->flags; + ctx->u.sc.level = 0; + ctx->phase = ERTS_DSIG_SEND_PHASE_MSG_SIZE; + } else { + ctx->phase = ERTS_DSIG_SEND_PHASE_ALLOC; + } + break; - dhdr_ext_size = erts_encode_ext_dist_header_size(acmp); - data_size += dhdr_ext_size; + case ERTS_DSIG_SEND_PHASE_MSG_SIZE: + if (erts_encode_dist_ext_size_int(ctx->msg, ctx, &ctx->data_size)) { + retval = ERTS_DSIG_SEND_CONTINUE; + goto done; + } - obuf = alloc_dist_obuf(data_size); - obuf->ext_endp = &obuf->data[0] + pass_through_size + dhdr_ext_size; + ctx->phase = ERTS_DSIG_SEND_PHASE_ALLOC; + case ERTS_DSIG_SEND_PHASE_ALLOC: + erts_finalize_atom_cache_map(ctx->acmp, ctx->flags); + + ctx->dhdr_ext_size = erts_encode_ext_dist_header_size(ctx->acmp); + ctx->data_size += ctx->dhdr_ext_size; + + ctx->obuf = alloc_dist_obuf(ctx->data_size); + ctx->obuf->ext_endp = &ctx->obuf->data[0] + ctx->pass_through_size + ctx->dhdr_ext_size; + + /* Encode internal version of dist header */ + ctx->obuf->extp = erts_encode_ext_dist_header_setup(ctx->obuf->ext_endp, ctx->acmp); + /* Encode control message */ + erts_encode_dist_ext(ctx->ctl, &ctx->obuf->ext_endp, ctx->flags, ctx->acmp, NULL, NULL); + if (is_value(ctx->msg)) { + ctx->u.ec.flags = ctx->flags; + ctx->u.ec.level = 0; + ctx->u.ec.wstack.wstart = NULL; + ctx->phase = ERTS_DSIG_SEND_PHASE_MSG_ENCODE; + } else { + ctx->phase = ERTS_DSIG_SEND_PHASE_FIN; + } + break; - /* Encode internal version of dist header */ - obuf->extp = erts_encode_ext_dist_header_setup(obuf->ext_endp, acmp); - /* Encode control message */ - erts_encode_dist_ext(ctl, &obuf->ext_endp, flags, acmp); - if (is_value(msg)) { - /* Encode message */ - erts_encode_dist_ext(msg, &obuf->ext_endp, flags, acmp); - } + case ERTS_DSIG_SEND_PHASE_MSG_ENCODE: + if (erts_encode_dist_ext(ctx->msg, &ctx->obuf->ext_endp, ctx->flags, ctx->acmp, &ctx->u.ec, &ctx->reds)) { + retval = ERTS_DSIG_SEND_CONTINUE; + goto done; + } - ASSERT(obuf->extp < obuf->ext_endp); - ASSERT(&obuf->data[0] <= obuf->extp - pass_through_size); - ASSERT(obuf->ext_endp <= &obuf->data[0] + data_size); + ctx->phase = ERTS_DSIG_SEND_PHASE_FIN; + case ERTS_DSIG_SEND_PHASE_FIN: { + DistEntry *dep = dsdp->dep; + int suspended = 0; + int resume = 0; - data_size = obuf->ext_endp - obuf->extp; + ASSERT(ctx->obuf->extp < ctx->obuf->ext_endp); + ASSERT(&ctx->obuf->data[0] <= ctx->obuf->extp - ctx->pass_through_size); + ASSERT(ctx->obuf->ext_endp <= &ctx->obuf->data[0] + ctx->data_size); - /* - * Signal encoded; now verify that the connection still exists, - * and if so enqueue the signal and schedule it for send. - */ - obuf->next = NULL; - erts_smp_de_rlock(dep); - cid = dep->cid; - if (cid != dsdp->cid - || dep->connection_id != dsdp->connection_id - || dep->status & ERTS_DE_SFLG_EXITING) { - /* Not the same connection as when we started; drop message... */ - erts_smp_de_runlock(dep); - free_dist_obuf(obuf); - } - else { - ErtsProcList *plp = NULL; - erts_smp_mtx_lock(&dep->qlock); - dep->qsize += size_obuf(obuf); - if (dep->qsize >= erts_dist_buf_busy_limit) - dep->qflgs |= ERTS_DE_QFLG_BUSY; - if (!force_busy && (dep->qflgs & ERTS_DE_QFLG_BUSY)) { - erts_smp_mtx_unlock(&dep->qlock); + ctx->data_size = ctx->obuf->ext_endp - ctx->obuf->extp; - plp = erts_proclist_create(c_p); - erts_suspend(c_p, ERTS_PROC_LOCK_MAIN, NULL); - suspended = 1; - erts_smp_mtx_lock(&dep->qlock); - } + /* + * Signal encoded; now verify that the connection still exists, + * and if so enqueue the signal and schedule it for send. + */ + ctx->obuf->next = NULL; + erts_smp_de_rlock(dep); + cid = dep->cid; + if (cid != dsdp->cid + || dep->connection_id != dsdp->connection_id + || dep->status & ERTS_DE_SFLG_EXITING) { + /* Not the same connection as when we started; drop message... */ + erts_smp_de_runlock(dep); + free_dist_obuf(ctx->obuf); + } + else { + ErtsProcList *plp = NULL; + erts_smp_mtx_lock(&dep->qlock); + dep->qsize += size_obuf(ctx->obuf); + if (dep->qsize >= erts_dist_buf_busy_limit) + dep->qflgs |= ERTS_DE_QFLG_BUSY; + if (!ctx->force_busy && (dep->qflgs & ERTS_DE_QFLG_BUSY)) { + erts_smp_mtx_unlock(&dep->qlock); + + plp = erts_proclist_create(ctx->c_p); + erts_suspend(ctx->c_p, ERTS_PROC_LOCK_MAIN, NULL); + suspended = 1; + erts_smp_mtx_lock(&dep->qlock); + } - /* Enqueue obuf on dist entry */ - if (dep->out_queue.last) - dep->out_queue.last->next = obuf; - else - dep->out_queue.first = obuf; - dep->out_queue.last = obuf; + /* Enqueue obuf on dist entry */ + if (dep->out_queue.last) + dep->out_queue.last->next = ctx->obuf; + else + dep->out_queue.first = ctx->obuf; + dep->out_queue.last = ctx->obuf; + + if (!ctx->force_busy) { + if (!(dep->qflgs & ERTS_DE_QFLG_BUSY)) { + if (suspended) + resume = 1; /* was busy when we started, but isn't now */ + #ifdef USE_VM_PROBES + if (resume && DTRACE_ENABLED(dist_port_not_busy)) { + DTRACE_CHARBUF(port_str, 64); + DTRACE_CHARBUF(remote_str, 64); + + erts_snprintf(port_str, sizeof(DTRACE_CHARBUF_NAME(port_str)), + "%T", cid); + erts_snprintf(remote_str, sizeof(DTRACE_CHARBUF_NAME(remote_str)), + "%T", dep->sysname); + DTRACE3(dist_port_not_busy, erts_this_node_sysname, + port_str, remote_str); + } + #endif + } + else { + /* Enqueue suspended process on dist entry */ + ASSERT(plp); + erts_proclist_store_last(&dep->suspended, plp); + } + } - if (!force_busy) { - if (!(dep->qflgs & ERTS_DE_QFLG_BUSY)) { - if (suspended) - resume = 1; /* was busy when we started, but isn't now */ -#ifdef USE_VM_PROBES - if (resume && DTRACE_ENABLED(dist_port_not_busy)) { - DTRACE_CHARBUF(port_str, 64); - DTRACE_CHARBUF(remote_str, 64); - - erts_snprintf(port_str, sizeof(DTRACE_CHARBUF_NAME(port_str)), - "%T", cid); - erts_snprintf(remote_str, sizeof(DTRACE_CHARBUF_NAME(remote_str)), - "%T", dep->sysname); - DTRACE3(dist_port_not_busy, erts_this_node_sysname, - port_str, remote_str); - } -#endif + erts_smp_mtx_unlock(&dep->qlock); + erts_schedule_dist_command(NULL, dep); + erts_smp_de_runlock(dep); + + if (resume) { + erts_resume(ctx->c_p, ERTS_PROC_LOCK_MAIN); + erts_proclist_destroy(plp); + /* + * Note that the calling process still have to yield as if it + * suspended. If not, the calling process could later be + * erroneously scheduled when it shouldn't be. + */ + } } - else { - /* Enqueue suspended process on dist entry */ - ASSERT(plp); - erts_proclist_store_last(&dep->suspended, plp); + ctx->obuf = NULL; + + if (suspended) { + #ifdef USE_VM_PROBES + if (!resume && DTRACE_ENABLED(dist_port_busy)) { + DTRACE_CHARBUF(port_str, 64); + DTRACE_CHARBUF(remote_str, 64); + DTRACE_CHARBUF(pid_str, 16); + + erts_snprintf(port_str, sizeof(DTRACE_CHARBUF_NAME(port_str)), "%T", cid); + erts_snprintf(remote_str, sizeof(DTRACE_CHARBUF_NAME(remote_str)), + "%T", dep->sysname); + erts_snprintf(pid_str, sizeof(DTRACE_CHARBUF_NAME(pid_str)), + "%T", ctx->c_p->common.id); + DTRACE4(dist_port_busy, erts_this_node_sysname, + port_str, remote_str, pid_str); + } + #endif + if (!resume && erts_system_monitor_flags.busy_dist_port) + monitor_generic(ctx->c_p, am_busy_dist_port, cid); + retval = ERTS_DSIG_SEND_YIELD; + } else { + retval = ERTS_DSIG_SEND_OK; } + goto done; } - - erts_smp_mtx_unlock(&dep->qlock); - erts_schedule_dist_command(NULL, dep); - erts_smp_de_runlock(dep); - - if (resume) { - erts_resume(c_p, ERTS_PROC_LOCK_MAIN); - erts_proclist_destroy(plp); - /* - * Note that the calling process still have to yield as if it - * suspended. If not, the calling process could later be - * erroneously scheduled when it shouldn't be. - */ + default: + erts_exit(ERTS_ABORT_EXIT, "dsig_send invalid phase (%d)\n", (int)ctx->phase); } } - if (c_p) { - int reds; - /* - * Bump reductions on calling process. - * - * This is the reduction cost: Always a base cost of 8 reductions - * plus 16 reductions per kilobyte generated external data. - */ - - data_size >>= (10-4); -#if defined(ARCH_64) && !HALFWORD_HEAP - data_size &= 0x003fffffffffffff; -#elif defined(ARCH_32) || HALFWORD_HEAP - data_size &= 0x003fffff; -#else -# error "Ohh come on ... !?!" -#endif - reds = 8 + ((int) data_size > 1000000 ? 1000000 : (int) data_size); - BUMP_REDS(c_p, reds); - } - - if (suspended) { -#ifdef USE_VM_PROBES - if (!resume && DTRACE_ENABLED(dist_port_busy)) { - DTRACE_CHARBUF(port_str, 64); - DTRACE_CHARBUF(remote_str, 64); - DTRACE_CHARBUF(pid_str, 16); - - erts_snprintf(port_str, sizeof(DTRACE_CHARBUF_NAME(port_str)), "%T", cid); - erts_snprintf(remote_str, sizeof(DTRACE_CHARBUF_NAME(remote_str)), - "%T", dep->sysname); - erts_snprintf(pid_str, sizeof(DTRACE_CHARBUF_NAME(pid_str)), - "%T", c_p->common.id); - DTRACE4(dist_port_busy, erts_this_node_sysname, - port_str, remote_str, pid_str); - } -#endif - if (!resume && erts_system_monitor_flags.busy_dist_port) - monitor_generic(c_p, am_busy_dist_port, cid); - return ERTS_DSIG_SEND_YIELD; +done: + if (ctx->msg && ctx->c_p) { + BUMP_REDS(ctx->c_p, (initial_reds - ctx->reds) / TERM_TO_BINARY_LOOP_FACTOR); } - return ERTS_DSIG_SEND_OK; + return retval; } - static Uint dist_port_command(Port *prt, ErtsDistOutputBuf *obuf) { @@ -1891,7 +1963,7 @@ dist_port_command(Port *prt, ErtsDistOutputBuf *obuf) ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(prt)); if (size > (Uint) INT_MAX) - erl_exit(ERTS_ABORT_EXIT, + erts_exit(ERTS_ABORT_EXIT, "Absurdly large distribution output data buffer " "(%beu bytes) passed.\n", size); @@ -1931,7 +2003,7 @@ dist_port_commandv(Port *prt, ErtsDistOutputBuf *obuf) ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(prt)); if (size > (Uint) INT_MAX) - erl_exit(ERTS_ABORT_EXIT, + erts_exit(ERTS_ABORT_EXIT, "Absurdly large distribution output data buffer " "(%beu bytes) passed.\n", size); @@ -1973,9 +2045,9 @@ dist_port_commandv(Port *prt, ErtsDistOutputBuf *obuf) } -#if defined(ARCH_64) && !HALFWORD_HEAP +#if defined(ARCH_64) #define ERTS_PORT_REDS_MASK__ 0x003fffffffffffffL -#elif defined(ARCH_32) || HALFWORD_HEAP +#elif defined(ARCH_32) #define ERTS_PORT_REDS_MASK__ 0x003fffff #else # error "Ohh come on ... !?!" @@ -2001,6 +2073,7 @@ erts_dist_command(Port *prt, int reds_limit) DistEntry *dep = prt->dist_entry; Uint (*send)(Port *prt, ErtsDistOutputBuf *obuf); erts_aint32_t sched_flags; + ErtsSchedulerData *esdp = erts_get_scheduler_data(); ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(prt)); @@ -2055,12 +2128,12 @@ erts_dist_command(Port *prt, int reds_limit) ErtsDistOutputBuf *fob; size = (*send)(prt, foq.first); + esdp->io.out += (Uint64) size; #ifdef ERTS_RAW_DIST_MSG_DBG erts_fprintf(stderr, ">> "); bw(foq.first->extp, size); #endif reds += ERTS_PORT_REDS_DIST_CMD_DATA(size); - erts_smp_atomic_add_nob(&erts_bytes_out, size); fob = foq.first; obufsize += size_obuf(fob); foq.first = foq.first->next; @@ -2140,12 +2213,12 @@ erts_dist_command(Port *prt, int reds_limit) ASSERT(&oq.first->data[0] <= oq.first->extp && oq.first->extp < oq.first->ext_endp); size = (*send)(prt, oq.first); + esdp->io.out += (Uint64) size; #ifdef ERTS_RAW_DIST_MSG_DBG erts_fprintf(stderr, ">> "); bw(oq.first->extp, size); #endif reds += ERTS_PORT_REDS_DIST_CMD_DATA(size); - erts_smp_atomic_add_nob(&erts_bytes_out, size); fob = oq.first; obufsize += size_obuf(fob); oq.first = oq.first->next; @@ -2437,7 +2510,7 @@ info_dist_entry(int to, void *arg, DistEntry *dep, int visible, int connected) erts_print(to, arg, "Name: %T", dep->sysname); #ifdef DEBUG - erts_print(to, arg, " (refc=%d)", erts_refc_read(&dep->refc, 1)); + erts_print(to, arg, " (refc=%d)", erts_refc_read(&dep->refc, 0)); #endif erts_print(to, arg, "\n"); if (!connected && is_nil(dep->cid)) { @@ -2483,7 +2556,9 @@ int distribution_info(int to, void *arg) /* Called by break handler */ } for (dep = erts_not_connected_dist_entries; dep; dep = dep->next) { - info_dist_entry(to, arg, dep, 0, 0); + if (dep != erts_this_dist_entry) { + info_dist_entry(to, arg, dep, 0, 0); + } } return(0); @@ -2561,13 +2636,8 @@ BIF_RETTYPE setnode_2(BIF_ALIST_2) if (!net_kernel) goto error; - /* By setting dist_entry==erts_this_dist_entry and DISTRIBUTION on - net_kernel do_net_exist will be called when net_kernel - is terminated !! */ - (void) ERTS_PROC_SET_DIST_ENTRY(net_kernel, - ERTS_PROC_LOCK_MAIN, - erts_this_dist_entry); - erts_refc_inc(&erts_this_dist_entry->refc, 2); + /* By setting F_DISTRIBUTION on net_kernel, + * do_net_exist will be called when net_kernel is terminated !! */ net_kernel->flags |= F_DISTRIBUTION; if (net_kernel != BIF_P) @@ -2928,11 +2998,11 @@ BIF_RETTYPE nodes_1(BIF_ALIST_1) erts_smp_rwmtx_rlock(&erts_dist_table_rwmtx); - ASSERT(erts_no_of_not_connected_dist_entries >= 0); + ASSERT(erts_no_of_not_connected_dist_entries > 0); ASSERT(erts_no_of_hidden_dist_entries >= 0); ASSERT(erts_no_of_visible_dist_entries >= 0); if(not_connected) - length += erts_no_of_not_connected_dist_entries; + length += (erts_no_of_not_connected_dist_entries - 1); if(hidden) length += erts_no_of_hidden_dist_entries; if(visible) @@ -2954,8 +3024,10 @@ BIF_RETTYPE nodes_1(BIF_ALIST_1) #endif if(not_connected) for(dep = erts_not_connected_dist_entries; dep; dep = dep->next) { - result = CONS(hp, dep->sysname, result); - hp += 2; + if (dep != erts_this_dist_entry) { + result = CONS(hp, dep->sysname, result); + hp += 2; + } } if(hidden) for(dep = erts_hidden_dist_entries; dep; dep = dep->next) { @@ -3186,11 +3258,16 @@ send_nodes_mon_msg(Process *rp, Uint sz) { Eterm msg; - ErlHeapFragment* bp; + Eterm *hp; + ErtsMessage *mp; ErlOffHeap *ohp; - Eterm *hp = erts_alloc_message_heap(sz, &bp, &ohp, rp, rp_locksp); #ifdef DEBUG - Eterm *hend = hp + sz; + Eterm *hend; +#endif + + mp = erts_alloc_message_heap(rp, rp_locksp, sz, &hp, &ohp); +#ifdef DEBUG + hend = hp + sz; #endif if (!nmp->opts) { @@ -3236,11 +3313,7 @@ send_nodes_mon_msg(Process *rp, } ASSERT(hend == hp); - erts_queue_message(rp, rp_locksp, bp, msg, NIL -#ifdef USE_VM_PROBES - , NIL -#endif - ); + erts_queue_message(rp, rp_locksp, mp, msg, NIL); } static void @@ -3297,7 +3370,7 @@ send_nodes_mon_msgs(Process *c_p, Eterm what, Eterm node, Eterm type, Eterm reas continue; break; default: - erl_exit(ERTS_ABORT_EXIT, "Bad node type found\n"); + erts_exit(ERTS_ABORT_EXIT, "Bad node type found\n"); } } @@ -3606,7 +3679,7 @@ erts_processes_monitoring_nodes(Process *c_p) case ERTS_NODES_MON_OPT_TYPES: type = am_all; break; case ERTS_NODES_MON_OPT_TYPE_VISIBLE: type = am_visible; break; case ERTS_NODES_MON_OPT_TYPE_HIDDEN: type = am_hidden; break; - default: erl_exit(ERTS_ABORT_EXIT, "Bad node type found\n"); + default: erts_exit(ERTS_ABORT_EXIT, "Bad node type found\n"); } olist = erts_bld_cons(hpp, szp, erts_bld_tuple(hpp, szp, 2, diff --git a/erts/emulator/beam/dist.h b/erts/emulator/beam/dist.h index f32b999198..fb777d9ac1 100644 --- a/erts/emulator/beam/dist.h +++ b/erts/emulator/beam/dist.h @@ -3,16 +3,17 @@ * * Copyright Ericsson AB 1996-2014. All Rights Reserved. * - * The contents of this file are subject to the Erlang Public License, - * Version 1.1, (the "License"); you may not use this file except in - * compliance with the License. You should have received a copy of the - * Erlang Public License along with this software. If not, it can be - * retrieved online at http://www.erlang.org/. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Software distributed under the License is distributed on an "AS IS" - * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See - * the License for the specific language governing rights and limitations - * under the License. + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. * * %CopyrightEnd% */ @@ -22,6 +23,7 @@ #include "erl_process.h" #include "erl_node_tables.h" +#include "zlib.h" #define DFLAG_PUBLISHED 0x01 #define DFLAG_ATOM_CACHE 0x02 @@ -264,17 +266,105 @@ erts_destroy_dist_link(ErtsDistLinkData *dldp) #endif + + +/* Define for testing */ +/* #define EXTREME_TTB_TRAPPING 1 */ + +#ifndef EXTREME_TTB_TRAPPING +#define TERM_TO_BINARY_LOOP_FACTOR 32 +#else +#define TERM_TO_BINARY_LOOP_FACTOR 1 +#endif + +typedef enum { TTBSize, TTBEncode, TTBCompress } TTBState; +typedef struct TTBSizeContext_ { + Uint flags; + int level; + Uint result; + Eterm obj; + ErtsWStack wstack; +} TTBSizeContext; + +typedef struct TTBEncodeContext_ { + Uint flags; + int level; + byte* ep; + Eterm obj; + ErtsWStack wstack; + Binary *result_bin; +} TTBEncodeContext; + +typedef struct { + Uint real_size; + Uint dest_len; + byte *dbytes; + Binary *result_bin; + Binary *destination_bin; + z_stream stream; +} TTBCompressContext; + +typedef struct { + int alive; + TTBState state; + union { + TTBSizeContext sc; + TTBEncodeContext ec; + TTBCompressContext cc; + } s; +} TTBContext; + +enum erts_dsig_send_phase { + ERTS_DSIG_SEND_PHASE_INIT, + ERTS_DSIG_SEND_PHASE_MSG_SIZE, + ERTS_DSIG_SEND_PHASE_ALLOC, + ERTS_DSIG_SEND_PHASE_MSG_ENCODE, + ERTS_DSIG_SEND_PHASE_FIN +}; + +struct erts_dsig_send_context { + enum erts_dsig_send_phase phase; + Sint reds; + + Eterm ctl; + Eterm msg; + int force_busy; + Uint32 pass_through_size; + Uint data_size, dhdr_ext_size; + ErtsAtomCacheMap *acmp; + ErtsDistOutputBuf *obuf; + Uint32 flags; + Process *c_p; + union { + TTBSizeContext sc; + TTBEncodeContext ec; + }u; +}; + +typedef struct { + int suspend; + + Eterm ctl_heap[6]; + ErtsDSigData dsd; + DistEntry* dep_to_deref; + struct erts_dsig_send_context dss; + + Eterm return_term; +}ErtsSendContext; + + /* * erts_dsig_send_* return values. */ #define ERTS_DSIG_SEND_OK 0 #define ERTS_DSIG_SEND_YIELD 1 +#define ERTS_DSIG_SEND_CONTINUE 2 extern int erts_dsig_send_link(ErtsDSigData *, Eterm, Eterm); -extern int erts_dsig_send_msg(ErtsDSigData *, Eterm, Eterm); +extern int erts_dsig_send_msg(Eterm, Eterm, ErtsSendContext*); extern int erts_dsig_send_exit_tt(ErtsDSigData *, Eterm, Eterm, Eterm, Eterm); extern int erts_dsig_send_unlink(ErtsDSigData *, Eterm, Eterm); -extern int erts_dsig_send_reg_msg(ErtsDSigData *, Eterm, Eterm); +extern int erts_dsig_send_reg_msg(Eterm, Eterm, ErtsSendContext*); extern int erts_dsig_send_group_leader(ErtsDSigData *, Eterm, Eterm); extern int erts_dsig_send_exit(ErtsDSigData *, Eterm, Eterm, Eterm); extern int erts_dsig_send_exit2(ErtsDSigData *, Eterm, Eterm, Eterm); @@ -282,6 +372,10 @@ extern int erts_dsig_send_demonitor(ErtsDSigData *, Eterm, Eterm, Eterm, int); extern int erts_dsig_send_monitor(ErtsDSigData *, Eterm, Eterm, Eterm); extern int erts_dsig_send_m_exit(ErtsDSigData *, Eterm, Eterm, Eterm, Eterm); +extern int erts_dsig_send(ErtsDSigData *dsdp, struct erts_dsig_send_context* ctx); +extern void erts_dsend_context_dtor(Binary*); +extern Eterm erts_dsend_export_trap_context(Process* p, ErtsSendContext* ctx); + extern int erts_dist_command(Port *prt, int reds); extern void erts_dist_port_not_busy(Port *prt); extern void erts_kill_dist_connection(DistEntry *dep, Uint32); diff --git a/erts/emulator/beam/dtrace-wrapper.h b/erts/emulator/beam/dtrace-wrapper.h index 6ec0c91e21..d343dc5ab0 100644 --- a/erts/emulator/beam/dtrace-wrapper.h +++ b/erts/emulator/beam/dtrace-wrapper.h @@ -4,16 +4,17 @@ * Copyright Dustin Sallings, Michal Ptaszek, Scott Lystig Fritchie 2011-2012. * All Rights Reserved. * - * The contents of this file are subject to the Erlang Public License, - * Version 1.1, (the "License"); you may not use this file except in - * compliance with the License. You should have received a copy of the - * Erlang Public License along with this software. If not, it can be - * retrieved online at http://www.erlang.org/. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Software distributed under the License is distributed on an "AS IS" - * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See - * the License for the specific language governing rights and limitations - * under the License. + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. * * %CopyrightEnd% */ diff --git a/erts/emulator/beam/elib_memmove.c b/erts/emulator/beam/elib_memmove.c index d2fe8649ed..d4ca30158e 100644 --- a/erts/emulator/beam/elib_memmove.c +++ b/erts/emulator/beam/elib_memmove.c @@ -3,16 +3,17 @@ * * Copyright Ericsson AB 1997-2009. All Rights Reserved. * - * The contents of this file are subject to the Erlang Public License, - * Version 1.1, (the "License"); you may not use this file except in - * compliance with the License. You should have received a copy of the - * Erlang Public License along with this software. If not, it can be - * retrieved online at http://www.erlang.org/. - * - * Software distributed under the License is distributed on an "AS IS" - * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See - * the License for the specific language governing rights and limitations - * under the License. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. * * %CopyrightEnd% */ diff --git a/erts/emulator/beam/erl_afit_alloc.c b/erts/emulator/beam/erl_afit_alloc.c index eca4e3b3bb..4b0541c10e 100644 --- a/erts/emulator/beam/erl_afit_alloc.c +++ b/erts/emulator/beam/erl_afit_alloc.c @@ -3,16 +3,17 @@ * * Copyright Ericsson AB 2003-2013. All Rights Reserved. * - * The contents of this file are subject to the Erlang Public License, - * Version 1.1, (the "License"); you may not use this file except in - * compliance with the License. You should have received a copy of the - * Erlang Public License along with this software. If not, it can be - * retrieved online at http://www.erlang.org/. - * - * Software distributed under the License is distributed on an "AS IS" - * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See - * the License for the specific language governing rights and limitations - * under the License. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. * * %CopyrightEnd% */ @@ -240,7 +241,7 @@ info_options(Allctr_t *allctr, if (hpp || szp) { if (!atoms_initialized) - erl_exit(1, "%s:%d: Internal error: Atoms not initialized", + erts_exit(ERTS_ERROR_EXIT, "%s:%d: Internal error: Atoms not initialized", __FILE__, __LINE__);; res = NIL; diff --git a/erts/emulator/beam/erl_afit_alloc.h b/erts/emulator/beam/erl_afit_alloc.h index b90ac8f7c5..ef050ff50e 100644 --- a/erts/emulator/beam/erl_afit_alloc.h +++ b/erts/emulator/beam/erl_afit_alloc.h @@ -3,16 +3,17 @@ * * Copyright Ericsson AB 2003-2013. All Rights Reserved. * - * The contents of this file are subject to the Erlang Public License, - * Version 1.1, (the "License"); you may not use this file except in - * compliance with the License. You should have received a copy of the - * Erlang Public License along with this software. If not, it can be - * retrieved online at http://www.erlang.org/. - * - * Software distributed under the License is distributed on an "AS IS" - * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See - * the License for the specific language governing rights and limitations - * under the License. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. * * %CopyrightEnd% */ diff --git a/erts/emulator/beam/erl_alloc.c b/erts/emulator/beam/erl_alloc.c index 05ac24e04d..490e0c0915 100644 --- a/erts/emulator/beam/erl_alloc.c +++ b/erts/emulator/beam/erl_alloc.c @@ -3,16 +3,17 @@ * * Copyright Ericsson AB 2002-2013. All Rights Reserved. * - * The contents of this file are subject to the Erlang Public License, - * Version 1.1, (the "License"); you may not use this file except in - * compliance with the License. You should have received a copy of the - * Erlang Public License along with this software. If not, it can be - * retrieved online at http://www.erlang.org/. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Software distributed under the License is distributed on an "AS IS" - * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See - * the License for the specific language governing rights and limitations - * under the License. + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. * * %CopyrightEnd% */ @@ -39,7 +40,7 @@ #include "erl_instrument.h" #include "erl_mseg.h" #include "erl_monitors.h" -#include "erl_bif_timer.h" +#include "erl_hl_timer.h" #include "erl_cpu_topology.h" #include "erl_thr_queue.h" #if defined(ERTS_ALC_T_DRV_SEL_D_STATE) || defined(ERTS_ALC_T_DRV_EV_D_STATE) @@ -102,9 +103,9 @@ static Uint install_debug_functions(void); static int lock_all_physical_memory = 0; -ErtsAllocatorFunctions_t erts_allctrs[ERTS_ALC_A_MAX+1]; +ErtsAllocatorFunctions_t ERTS_WRITE_UNLIKELY(erts_allctrs[ERTS_ALC_A_MAX+1]); ErtsAllocatorInfo_t erts_allctrs_info[ERTS_ALC_A_MAX+1]; -ErtsAllocatorThrSpec_t erts_allctr_thr_spec[ERTS_ALC_A_MAX+1]; +ErtsAllocatorThrSpec_t ERTS_WRITE_UNLIKELY(erts_allctr_thr_spec[ERTS_ALC_A_MAX+1]); #define ERTS_MIN(A, B) ((A) < (B) ? (A) : (B)) #define ERTS_MAX(A, B) ((A) > (B) ? (A) : (B)) @@ -122,10 +123,6 @@ typedef union { static ErtsAllocatorState_t std_alloc_state; static ErtsAllocatorState_t ll_alloc_state; -#if HALFWORD_HEAP -static ErtsAllocatorState_t std_low_alloc_state; -static ErtsAllocatorState_t ll_low_alloc_state; -#endif static ErtsAllocatorState_t sl_alloc_state; static ErtsAllocatorState_t temp_alloc_state; static ErtsAllocatorState_t eheap_alloc_state; @@ -133,6 +130,8 @@ static ErtsAllocatorState_t binary_alloc_state; static ErtsAllocatorState_t ets_alloc_state; static ErtsAllocatorState_t driver_alloc_state; static ErtsAllocatorState_t fix_alloc_state; +static ErtsAllocatorState_t literal_alloc_state; +static ErtsAllocatorState_t test_alloc_state; typedef struct { erts_smp_atomic32_t refc; @@ -149,24 +148,10 @@ typedef struct { #define ERTS_ALC_INFO_A_MSEG_ALLOC (ERTS_ALC_A_MAX + 2) #define ERTS_ALC_INFO_A_MAX ERTS_ALC_INFO_A_MSEG_ALLOC -#if !HALFWORD_HEAP ERTS_SCHED_PREF_QUICK_ALLOC_IMPL(aireq, - ErtsAllocInfoReq, - 5, - ERTS_ALC_T_AINFO_REQ) -#else -static ERTS_INLINE ErtsAllocInfoReq * -aireq_alloc(void) -{ - return erts_alloc(ERTS_ALC_T_AINFO_REQ, sizeof(ErtsAllocInfoReq)); -} - -static ERTS_INLINE void -aireq_free(ErtsAllocInfoReq *ptr) -{ - erts_free(ERTS_ALC_T_AINFO_REQ, ptr); -} -#endif + ErtsAllocInfoReq, + 5, + ERTS_ALC_T_AINFO_REQ) ErtsAlcType_t erts_fix_core_allocator_ix; @@ -180,6 +165,8 @@ enum allctr_type { struct au_init { int enable; int thr_spec; + int disable_allowed; + int thr_spec_allowed; int carrier_migration_allowed; enum allctr_type atype; struct { @@ -228,13 +215,11 @@ typedef struct { struct au_init ets_alloc; struct au_init driver_alloc; struct au_init fix_alloc; -#if HALFWORD_HEAP - struct au_init std_low_alloc; - struct au_init ll_low_alloc; -#endif + struct au_init literal_alloc; + struct au_init test_alloc; } erts_alc_hndl_args_init_t; -#define ERTS_AU_INIT__ {0, 0, 1, GOODFIT, DEFAULT_ALLCTR_INIT, {1,1,1,1}} +#define ERTS_AU_INIT__ {0, 0, 1, 1, 1, GOODFIT, DEFAULT_ALLCTR_INIT, {1,1,1,1}} #define SET_DEFAULT_ALLOC_OPTS(IP) \ do { \ @@ -258,10 +243,6 @@ set_default_sl_alloc_opts(struct au_init *ip) #endif ip->init.util.ts = ERTS_ALC_MTA_SHORT_LIVED; ip->init.util.rsbcst = 80; -#if HALFWORD_HEAP - ip->init.util.force = 1; - ip->init.util.low_mem = 1; -#endif ip->init.util.acul = ERTS_ALC_DEFAULT_ACUL; } @@ -297,9 +278,9 @@ set_default_ll_alloc_opts(struct au_init *ip) ip->init.util.name_prefix = "ll_"; ip->init.util.alloc_no = ERTS_ALC_A_LONG_LIVED; #ifndef SMALL_MEMORY - ip->init.util.mmbcs = 2*1024*1024 - 40; /* Main carrier size */ + ip->init.util.mmbcs = 2*1024*1024; /* Main carrier size */ #else - ip->init.util.mmbcs = 1*1024*1024 - 40; /* Main carrier size */ + ip->init.util.mmbcs = 1*1024*1024; /* Main carrier size */ #endif ip->init.util.ts = ERTS_ALC_MTA_LONG_LIVED; ip->init.util.asbcst = 0; @@ -310,6 +291,54 @@ set_default_ll_alloc_opts(struct au_init *ip) } static void +set_default_literal_alloc_opts(struct au_init *ip) +{ + SET_DEFAULT_ALLOC_OPTS(ip); + ip->enable = 1; + ip->thr_spec = 0; + ip->disable_allowed = 0; + ip->thr_spec_allowed = 0; + ip->carrier_migration_allowed = 0; + ip->atype = BESTFIT; + ip->init.bf.ao = 1; + ip->init.util.ramv = 0; + ip->init.util.mmsbc = 0; + ip->init.util.sbct = ~((UWord) 0); + ip->init.util.name_prefix = "literal_"; + ip->init.util.alloc_no = ERTS_ALC_A_LITERAL; +#ifndef SMALL_MEMORY + ip->init.util.mmbcs = 2*1024*1024; /* Main carrier size */ +#else + ip->init.util.mmbcs = 1*1024*1024; /* Main carrier size */ +#endif + ip->init.util.ts = ERTS_ALC_MTA_LITERAL; + ip->init.util.asbcst = 0; + ip->init.util.rsbcst = 0; + ip->init.util.rsbcmt = 0; + ip->init.util.rmbcmt = 0; + ip->init.util.acul = 0; + +#if defined(ARCH_32) +# if HAVE_ERTS_MSEG + ip->init.util.mseg_alloc = &erts_alcu_literal_32_mseg_alloc; + ip->init.util.mseg_realloc = &erts_alcu_literal_32_mseg_realloc; + ip->init.util.mseg_dealloc = &erts_alcu_literal_32_mseg_dealloc; +# endif + ip->init.util.sys_alloc = &erts_alcu_literal_32_sys_alloc; + ip->init.util.sys_realloc = &erts_alcu_literal_32_sys_realloc; + ip->init.util.sys_dealloc = &erts_alcu_literal_32_sys_dealloc; +#elif defined(ARCH_64) +# ifdef ERTS_HAVE_OS_PHYSICAL_MEMORY_RESERVATION + ip->init.util.mseg_alloc = &erts_alcu_literal_64_mseg_alloc; + ip->init.util.mseg_realloc = &erts_alcu_literal_64_mseg_realloc; + ip->init.util.mseg_dealloc = &erts_alcu_literal_64_mseg_dealloc; +# endif +#else +# error Unknown architecture +#endif +} + +static void set_default_temp_alloc_opts(struct au_init *ip) { SET_DEFAULT_ALLOC_OPTS(ip); @@ -327,10 +356,6 @@ set_default_temp_alloc_opts(struct au_init *ip) ip->init.util.ts = ERTS_ALC_MTA_TEMPORARY; ip->init.util.rsbcst = 90; ip->init.util.rmbcmt = 100; -#if HALFWORD_HEAP - ip->init.util.force = 1; - ip->init.util.low_mem = 1; -#endif } static void @@ -349,10 +374,6 @@ set_default_eheap_alloc_opts(struct au_init *ip) #endif ip->init.util.ts = ERTS_ALC_MTA_EHEAP; ip->init.util.rsbcst = 50; -#if HALFWORD_HEAP - ip->init.util.force = 1; - ip->init.util.low_mem = 1; -#endif ip->init.util.acul = ERTS_ALC_DEFAULT_ACUL_EHEAP_ALLOC; } @@ -431,6 +452,33 @@ set_default_fix_alloc_opts(struct au_init *ip, ip->init.util.acul = ERTS_ALC_DEFAULT_ACUL; } +static void +set_default_test_alloc_opts(struct au_init *ip) +{ + SET_DEFAULT_ALLOC_OPTS(ip); + ip->enable = 0; /* Disabled by default */ + ip->thr_spec = -1 * erts_no_schedulers; + ip->atype = AOFIRSTFIT; + ip->init.aoff.flavor = AOFF_BF; + ip->init.util.name_prefix = "test_"; + ip->init.util.alloc_no = ERTS_ALC_A_TEST; + ip->init.util.mmbcs = 0; /* Main carrier size */ + ip->init.util.ts = ERTS_ALC_MTA_TEST; + ip->init.util.acul = ERTS_ALC_DEFAULT_ACUL; + + /* Use a constant minimal MBC size */ +#if ERTS_SA_MB_CARRIERS + ip->init.util.smbcs = ERTS_SACRR_UNIT_SZ; + ip->init.util.lmbcs = ERTS_SACRR_UNIT_SZ; + ip->init.util.sbct = ERTS_SACRR_UNIT_SZ; +#else + ip->init.util.smbcs = 1 << 12; + ip->init.util.lmbcs = 1 << 12; + ip->init.util.sbct = 1 << 12; +#endif +} + + #ifdef ERTS_SMP static void @@ -559,22 +607,31 @@ erts_alloc_init(int *argc, char **argv, ErtsAllocInitOpts *eaiop) fix_type_sizes[ERTS_ALC_FIX_TYPE_IX(ERTS_ALC_T_PROC)] = sizeof(Process); -#if !HALFWORD_HEAP fix_type_sizes[ERTS_ALC_FIX_TYPE_IX(ERTS_ALC_T_MONITOR_SH)] = ERTS_MONITOR_SH_SIZE * sizeof(Uint); fix_type_sizes[ERTS_ALC_FIX_TYPE_IX(ERTS_ALC_T_NLINK_SH)] = ERTS_LINK_SH_SIZE * sizeof(Uint); -#endif fix_type_sizes[ERTS_ALC_FIX_TYPE_IX(ERTS_ALC_T_DRV_EV_D_STATE)] = sizeof(ErtsDrvEventDataState); fix_type_sizes[ERTS_ALC_FIX_TYPE_IX(ERTS_ALC_T_DRV_SEL_D_STATE)] = sizeof(ErtsDrvSelectDataState); fix_type_sizes[ERTS_ALC_FIX_TYPE_IX(ERTS_ALC_T_MSG_REF)] - = sizeof(ErlMessage); + = sizeof(ErtsMessageRef); #ifdef ERTS_SMP fix_type_sizes[ERTS_ALC_FIX_TYPE_IX(ERTS_ALC_T_THR_Q_EL_SL)] = sizeof(ErtsThrQElement_t); #endif + fix_type_sizes[ERTS_ALC_FIX_TYPE_IX(ERTS_ALC_T_LL_PTIMER)] + = erts_timer_type_size(ERTS_ALC_T_LL_PTIMER); + fix_type_sizes[ERTS_ALC_FIX_TYPE_IX(ERTS_ALC_T_HL_PTIMER)] + = erts_timer_type_size(ERTS_ALC_T_HL_PTIMER); + fix_type_sizes[ERTS_ALC_FIX_TYPE_IX(ERTS_ALC_T_BIF_TIMER)] + = erts_timer_type_size(ERTS_ALC_T_BIF_TIMER); +#ifdef ERTS_BTM_ACCESSOR_SUPPORT + fix_type_sizes[ERTS_ALC_FIX_TYPE_IX(ERTS_ALC_T_ABIF_TIMER)] + = erts_timer_type_size(ERTS_ALC_T_ABIF_TIMER); +#endif + #ifdef HARD_DEBUG hdbg_init(); #endif @@ -601,6 +658,8 @@ erts_alloc_init(int *argc, char **argv, ErtsAllocInitOpts *eaiop) set_default_driver_alloc_opts(&init.driver_alloc); set_default_fix_alloc_opts(&init.fix_alloc, fix_type_sizes); + set_default_literal_alloc_opts(&init.literal_alloc); + set_default_test_alloc_opts(&init.test_alloc); if (argc && argv) handle_args(argc, argv, &init); @@ -611,11 +670,11 @@ erts_alloc_init(int *argc, char **argv, ErtsAllocInitOpts *eaiop) if (mlockall(MCL_CURRENT|MCL_FUTURE) != 0) { int err = errno; char *errstr = err ? strerror(err) : "unknown"; - erl_exit(-1, "Failed to lock physical memory: %s (%d)\n", + erts_exit(1, "Failed to lock physical memory: %s (%d)\n", errstr, err); } #else - erl_exit(-1, "Failed to lock physical memory: Not supported\n"); + erts_exit(1, "Failed to lock physical memory: Not supported\n"); #endif } @@ -628,6 +687,7 @@ erts_alloc_init(int *argc, char **argv, ErtsAllocInitOpts *eaiop) init.ets_alloc.thr_spec = 0; init.driver_alloc.thr_spec = 0; init.fix_alloc.thr_spec = 0; + init.literal_alloc.thr_spec = 0; #endif /* Make adjustments for carrier migration support */ @@ -640,6 +700,7 @@ erts_alloc_init(int *argc, char **argv, ErtsAllocInitOpts *eaiop) adjust_carrier_migration_support(&init.ets_alloc); adjust_carrier_migration_support(&init.driver_alloc); adjust_carrier_migration_support(&init.fix_alloc); + adjust_carrier_migration_support(&init.literal_alloc); if (init.erts_alloc_config) { /* Adjust flags that erts_alloc_config won't like */ @@ -654,6 +715,7 @@ erts_alloc_init(int *argc, char **argv, ErtsAllocInitOpts *eaiop) init.ets_alloc.thr_spec = 0; init.driver_alloc.thr_spec = 0; init.fix_alloc.thr_spec = 0; + init.literal_alloc.thr_spec = 0; /* No carrier migration */ init.temp_alloc.init.util.acul = 0; @@ -665,6 +727,7 @@ erts_alloc_init(int *argc, char **argv, ErtsAllocInitOpts *eaiop) init.ets_alloc.init.util.acul = 0; init.driver_alloc.init.util.acul = 0; init.fix_alloc.init.util.acul = 0; + init.literal_alloc.init.util.acul = 0; } #ifdef ERTS_SMP @@ -681,6 +744,7 @@ erts_alloc_init(int *argc, char **argv, ErtsAllocInitOpts *eaiop) adjust_tpref(&init.ets_alloc, erts_no_schedulers); adjust_tpref(&init.driver_alloc, erts_no_schedulers); adjust_tpref(&init.fix_alloc, erts_no_schedulers); + adjust_tpref(&init.literal_alloc, erts_no_schedulers); #else /* No thread specific if not smp */ @@ -699,6 +763,7 @@ erts_alloc_init(int *argc, char **argv, ErtsAllocInitOpts *eaiop) refuse_af_strategy(&init.ets_alloc); refuse_af_strategy(&init.driver_alloc); refuse_af_strategy(&init.fix_alloc); + refuse_af_strategy(&init.literal_alloc); #ifdef ERTS_SMP if (!init.temp_alloc.thr_spec) @@ -733,24 +798,6 @@ erts_alloc_init(int *argc, char **argv, ErtsAllocInitOpts *eaiop) erts_allctrs[ERTS_ALC_A_SYSTEM].free = erts_sys_free; erts_allctrs_info[ERTS_ALC_A_SYSTEM].enabled = 1; -#if HALFWORD_HEAP - /* Init low memory variants by cloning */ - init.std_low_alloc = init.std_alloc; - init.std_low_alloc.init.util.name_prefix = "std_low_"; - init.std_low_alloc.init.util.alloc_no = ERTS_ALC_A_STANDARD_LOW; - init.std_low_alloc.init.util.force = 1; - init.std_low_alloc.init.util.low_mem = 1; - - init.ll_low_alloc = init.ll_alloc; - init.ll_low_alloc.init.util.name_prefix = "ll_low_"; - init.ll_low_alloc.init.util.alloc_no = ERTS_ALC_A_LONG_LIVED_LOW; - init.ll_low_alloc.init.util.force = 1; - init.ll_low_alloc.init.util.low_mem = 1; - - set_au_allocator(ERTS_ALC_A_STANDARD_LOW, &init.std_low_alloc, ncpu); - set_au_allocator(ERTS_ALC_A_LONG_LIVED_LOW, &init.ll_low_alloc, ncpu); -#endif /* HALFWORD */ - set_au_allocator(ERTS_ALC_A_TEMPORARY, &init.temp_alloc, ncpu); set_au_allocator(ERTS_ALC_A_SHORT_LIVED, &init.sl_alloc, ncpu); set_au_allocator(ERTS_ALC_A_STANDARD, &init.std_alloc, ncpu); @@ -760,16 +807,18 @@ erts_alloc_init(int *argc, char **argv, ErtsAllocInitOpts *eaiop) set_au_allocator(ERTS_ALC_A_ETS, &init.ets_alloc, ncpu); set_au_allocator(ERTS_ALC_A_DRIVER, &init.driver_alloc, ncpu); set_au_allocator(ERTS_ALC_A_FIXED_SIZE, &init.fix_alloc, ncpu); + set_au_allocator(ERTS_ALC_A_LITERAL, &init.literal_alloc, ncpu); + set_au_allocator(ERTS_ALC_A_TEST, &init.test_alloc, ncpu); for (i = ERTS_ALC_A_MIN; i <= ERTS_ALC_A_MAX; i++) { if (!erts_allctrs[i].alloc) - erl_exit(ERTS_ABORT_EXIT, + erts_exit(ERTS_ABORT_EXIT, "Missing alloc function for %s\n", ERTS_ALC_A2AD(i)); if (!erts_allctrs[i].realloc) - erl_exit(ERTS_ABORT_EXIT, + erts_exit(ERTS_ABORT_EXIT, "Missing realloc function for %s\n", ERTS_ALC_A2AD(i)); if (!erts_allctrs[i].free) - erl_exit(ERTS_ABORT_EXIT, + erts_exit(ERTS_ABORT_EXIT, "Missing free function for %s\n", ERTS_ALC_A2AD(i)); } @@ -793,14 +842,6 @@ erts_alloc_init(int *argc, char **argv, ErtsAllocInitOpts *eaiop) start_au_allocator(ERTS_ALC_A_LONG_LIVED, &init.ll_alloc, &ll_alloc_state); -#if HALFWORD_HEAP - start_au_allocator(ERTS_ALC_A_LONG_LIVED_LOW, - &init.ll_low_alloc, - &ll_low_alloc_state); - start_au_allocator(ERTS_ALC_A_STANDARD_LOW, - &init.std_low_alloc, - &std_low_alloc_state); -#endif start_au_allocator(ERTS_ALC_A_EHEAP, &init.eheap_alloc, &eheap_alloc_state); @@ -820,13 +861,18 @@ erts_alloc_init(int *argc, char **argv, ErtsAllocInitOpts *eaiop) start_au_allocator(ERTS_ALC_A_FIXED_SIZE, &init.fix_alloc, &fix_alloc_state); + start_au_allocator(ERTS_ALC_A_LITERAL, + &init.literal_alloc, + &literal_alloc_state); + + start_au_allocator(ERTS_ALC_A_TEST, + &init.test_alloc, + &test_alloc_state); erts_mtrace_install_wrapper_functions(); extra_block_size += erts_instr_init(init.instr.stat, init.instr.map); -#if !HALFWORD_HEAP init_aireq_alloc(); -#endif #ifdef DEBUG extra_block_size += install_debug_functions(); @@ -843,7 +889,7 @@ erts_alloc_late_init(void) static void * erts_realloc_fixed_size(ErtsAlcType_t type, void *extra, void *p, Uint size) { - erl_exit(ERTS_ABORT_EXIT, + erts_exit(ERTS_ABORT_EXIT, "Attempt to reallocate a block of the fixed size type %s\n", ERTS_ALC_T2TD(type)); } @@ -924,6 +970,10 @@ set_au_allocator(ErtsAlcType_t alctr_n, struct au_init *init, int ncpu) else #endif { +#ifdef ERTS_SMP + erts_exit(ERTS_ABORT_EXIT, "%salloc is not thread safe\n", + init->init.util.name_prefix); +#else af->alloc = erts_alcu_alloc; if (init->init.util.fix_type_size) af->realloc = erts_realloc_fixed_size; @@ -932,6 +982,7 @@ set_au_allocator(ErtsAlcType_t alctr_n, struct au_init *init, int ncpu) else af->realloc = erts_alcu_realloc; af->free = erts_alcu_free; +#endif } af->extra = NULL; ai->alloc_util = 1; @@ -965,7 +1016,7 @@ start_au_allocator(ErtsAlcType_t alctr_n, * tspec->size) + ERTS_CACHE_LINE_SIZE - 1)); if (!states) - erl_exit(ERTS_ABORT_EXIT, + erts_exit(ERTS_ABORT_EXIT, "Failed to allocate allocator states for %salloc\n", init->init.util.name_prefix); tspec->allctr = (Allctr_t **) states; @@ -993,7 +1044,7 @@ start_au_allocator(ErtsAlcType_t alctr_n, (tot_fix_list_size + ERTS_CACHE_LINE_SIZE - 1)); if (!fix_lists) - erl_exit(ERTS_ABORT_EXIT, + erts_exit(ERTS_ABORT_EXIT, "Failed to allocate fix lists for %salloc\n", init->init.util.name_prefix); @@ -1004,7 +1055,7 @@ start_au_allocator(ErtsAlcType_t alctr_n, } for (i = 0; i < size; i++) { - void *as; + Allctr_t *as; atype = init->atype; if (!init->thr_spec) @@ -1041,22 +1092,22 @@ start_au_allocator(ErtsAlcType_t alctr_n, switch (atype) { case GOODFIT: - as = (void *) erts_gfalc_start((GFAllctr_t *) as0, + as = erts_gfalc_start((GFAllctr_t *) as0, &init->init.gf, &init->init.util); break; case BESTFIT: - as = (void *) erts_bfalc_start((BFAllctr_t *) as0, + as = erts_bfalc_start((BFAllctr_t *) as0, &init->init.bf, &init->init.util); break; case AFIT: - as = (void *) erts_afalc_start((AFAllctr_t *) as0, + as = erts_afalc_start((AFAllctr_t *) as0, &init->init.af, &init->init.util); break; case AOFIRSTFIT: - as = (void *) erts_aoffalc_start((AOFFAllctr_t *) as0, + as = erts_aoffalc_start((AOFFAllctr_t *) as0, &init->init.aoff, &init->init.util); break; @@ -1067,7 +1118,7 @@ start_au_allocator(ErtsAlcType_t alctr_n, } if (!as) - erl_exit(ERTS_ABORT_EXIT, + erts_exit(ERTS_ABORT_EXIT, "Failed to start %salloc\n", init->init.util.name_prefix); ASSERT(as == (void *) as0); @@ -1226,9 +1277,6 @@ get_acul_value(struct au_init *auip, char *param_end, char** argv, int* ip) if (sys_strcmp(value, "de") == 0) { switch (auip->init.util.alloc_no) { case ERTS_ALC_A_LONG_LIVED: -#if HALFWORD_HEAP - case ERTS_ALC_A_LONG_LIVED_LOW: -#endif return ERTS_ALC_DEFAULT_ENABLED_ACUL_LL_ALLOC; case ERTS_ALC_A_EHEAP: return ERTS_ALC_DEFAULT_ENABLED_ACUL_EHEAP_ALLOC; @@ -1306,9 +1354,17 @@ handle_au_arg(struct au_init *auip, else goto bad_switch; break; - case 'e': - auip->enable = get_bool_value(sub_param+1, argv, ip); + case 'e': { + int e = get_bool_value(sub_param + 1, argv, ip); + if (!auip->disable_allowed && !e) { + if (!u_switch) + bad_value(param, sub_param + 1, "false"); + else + ASSERT(auip->enable); /* ignore */ + } + else auip->enable = e; break; + } case 'l': if (has_prefix("lmbcs", sub_param)) { auip->default_.lmbcs = 0; @@ -1377,7 +1433,14 @@ handle_au_arg(struct au_init *auip, case 't': { int res = get_bool_value(sub_param+1, argv, ip); if (res > 0) { - auip->thr_spec = 1; + if (!auip->thr_spec_allowed) { + if (!u_switch) + bad_value(param, sub_param + 1, "true"); + else + ASSERT(!auip->thr_spec); /* ignore */ + } + else + auip->thr_spec = 1; break; } else if (res == 0) { @@ -1406,6 +1469,7 @@ handle_args(int *argc, char **argv, erts_alc_hndl_args_init_t *init) &init->fix_alloc, &init->sl_alloc, &init->temp_alloc + /* test_alloc not affected by +Mea??? or +Mu??? */ }; int aui_sz = (int) sizeof(aui)/sizeof(aui[0]); char *arg; @@ -1425,6 +1489,16 @@ handle_args(int *argc, char **argv, erts_alc_hndl_args_init_t *init) case 'B': handle_au_arg(&init->binary_alloc, &argv[i][3], argv, &i, 0); break; + case 'I': + if (has_prefix("scs", argv[i]+3)) { +#if HAVE_ERTS_MSEG + init->mseg.literal_mmap.scs = +#endif + get_mb_value(argv[i]+6, argv, &i); + } + else + handle_au_arg(&init->literal_alloc, &argv[i][3], argv, &i, 0); + break; case 'D': handle_au_arg(&init->std_alloc, &argv[i][3], argv, &i, 0); break; @@ -1461,25 +1535,25 @@ handle_args(int *argc, char **argv, erts_alc_hndl_args_init_t *init) } else if (has_prefix("scs", argv[i]+3)) { #if HAVE_ERTS_MSEG - init->mseg.mmap.scs = + init->mseg.dflt_mmap.scs = #endif get_mb_value(argv[i]+6, argv, &i); } else if (has_prefix("sco", argv[i]+3)) { #if HAVE_ERTS_MSEG - init->mseg.mmap.sco = + init->mseg.dflt_mmap.sco = #endif get_bool_value(argv[i]+6, argv, &i); } else if (has_prefix("scrpm", argv[i]+3)) { #if HAVE_ERTS_MSEG - init->mseg.mmap.scrpm = + init->mseg.dflt_mmap.scrpm = #endif get_bool_value(argv[i]+8, argv, &i); } else if (has_prefix("scrfsd", argv[i]+3)) { #if HAVE_ERTS_MSEG - init->mseg.mmap.scrfsd = + init->mseg.dflt_mmap.scrfsd = #endif get_amount_value(argv[i]+9, argv, &i); } @@ -1496,6 +1570,9 @@ handle_args(int *argc, char **argv, erts_alc_hndl_args_init_t *init) case 'T': handle_au_arg(&init->temp_alloc, &argv[i][3], argv, &i, 0); break; + case 'Z': + handle_au_arg(&init->test_alloc, &argv[i][3], argv, &i, 0); + break; case 'Y': { /* sys_alloc */ if (has_prefix("tt", param+2)) { /* set trim threshold */ @@ -1858,7 +1935,7 @@ erts_alc_fatal_error(int error, int func, ErtsAlcType_t n, ...) case ERTS_ALC_O_FREE: op_str = "free"; break; default: op_str = "UNKNOWN"; break; } - erl_exit(ERTS_ABORT_EXIT, + erts_exit(ERTS_ABORT_EXIT, "%s: %s operation not supported (memory type: \"%s\")\n", allctr_str, op_str, t_str); break; @@ -1872,18 +1949,18 @@ erts_alc_fatal_error(int error, int func, ErtsAlcType_t n, ...) va_start(argp, n); size = va_arg(argp, Uint); va_end(argp); - erl_exit(1, + erts_exit(1, "%s: Cannot %s %lu bytes of memory (of type \"%s\").\n", allctr_str, op, size, t_str); break; } case ERTS_ALC_E_NOALLCTR: - erl_exit(ERTS_ABORT_EXIT, + erts_exit(ERTS_ABORT_EXIT, "erts_alloc: Unknown allocator type: %d\n", ERTS_ALC_T2A(ERTS_ALC_N2T(n))); break; default: - erl_exit(ERTS_ABORT_EXIT, "erts_alloc: Unknown error: %d\n", error); + erts_exit(ERTS_ABORT_EXIT, "erts_alloc: Unknown error: %d\n", error); break; } } @@ -1945,48 +2022,6 @@ alcu_size(ErtsAlcType_t ai, ErtsAlcUFixInfo_t *fi, int fisz) return res; } -#if HALFWORD_HEAP -static ERTS_INLINE int -alcu_is_low(ErtsAlcType_t ai) -{ - int is_low = 0; - ASSERT(erts_allctrs_info[ai].enabled); - ASSERT(erts_allctrs_info[ai].alloc_util); - - if (!erts_allctrs_info[ai].thr_spec) { - Allctr_t *allctr = erts_allctrs_info[ai].extra; - is_low = allctr->mseg_opt.low_mem; - } - else { - ErtsAllocatorThrSpec_t *tspec = &erts_allctr_thr_spec[ai]; - int i; -# ifdef DEBUG - int found_one = 0; -# endif - - ASSERT(tspec->enabled); - - for (i = tspec->size - 1; i >= 0; i--) { - Allctr_t *allctr = tspec->allctr[i]; - if (allctr) { -# ifdef DEBUG - if (!found_one) { - is_low = allctr->mseg_opt.low_mem; - found_one = 1; - } - else ASSERT(is_low == allctr->mseg_opt.low_mem); -# else - is_low = allctr->mseg_opt.low_mem; - break; -# endif - } - } - ASSERT(found_one); - } - return is_low; -} -#endif /* HALFWORD */ - static ERTS_INLINE void add_fix_values(UWord *ap, UWord *up, ErtsAlcUFixInfo_t *fi, ErtsAlcType_t type) { @@ -2016,9 +2051,6 @@ erts_memory(int *print_to_p, void *print_to_arg, void *proc, Eterm earg) int code; int ets; int maximum; -#if HALFWORD_HEAP - int low; -#endif } want = {0}; struct { UWord total; @@ -2031,9 +2063,6 @@ erts_memory(int *print_to_p, void *print_to_arg, void *proc, Eterm earg) UWord code; UWord ets; UWord maximum; -#if HALFWORD_HEAP - UWord low; -#endif } size = {0}; Eterm atoms[sizeof(size)/sizeof(UWord)]; UWord *uintps[sizeof(size)/sizeof(UWord)]; @@ -2092,11 +2121,6 @@ erts_memory(int *print_to_p, void *print_to_arg, void *proc, Eterm earg) atoms[length] = am_maximum; uintps[length++] = &size.maximum; } -#if HALFWORD_HEAP - want.low = 1; - atoms[length] = am_low; - uintps[length++] = &size.low; -#endif } else { DeclareTmpHeapNoproc(tmp_heap,2); @@ -2190,15 +2214,6 @@ erts_memory(int *print_to_p, void *print_to_arg, void *proc, Eterm earg) return am_badarg; } break; -#if HALFWORD_HEAP - case am_low: - if (!want.low) { - want.low = 1; - atoms[length] = am_low; - uintps[length++] = &size.low; - } - break; -#endif default: UnUseTmpHeapNoproc(2); return am_badarg; @@ -2210,11 +2225,12 @@ erts_memory(int *print_to_p, void *print_to_arg, void *proc, Eterm earg) return am_badarg; } - /* All alloc_util allocators *have* to be enabled */ + /* All alloc_util allocators *have* to be enabled, except test_alloc */ for (ai = ERTS_ALC_A_MIN; ai <= ERTS_ALC_A_MAX; ai++) { switch (ai) { case ERTS_ALC_A_SYSTEM: + case ERTS_ALC_A_TEST: break; default: if (!erts_allctrs_info[ai].enabled @@ -2255,6 +2271,8 @@ erts_memory(int *print_to_p, void *print_to_arg, void *proc, Eterm earg) * contain any allocated memory. */ continue; + case ERTS_ALC_A_TEST: + continue; case ERTS_ALC_A_EHEAP: save = &size.processes; break; @@ -2276,11 +2294,6 @@ erts_memory(int *print_to_p, void *print_to_arg, void *proc, Eterm earg) if (save) *save = asz; size.total += asz; -#if HALFWORD_HEAP - if (alcu_is_low(ai)) { - size.low += asz; - } -#endif } } } @@ -2307,7 +2320,6 @@ erts_memory(int *print_to_p, void *print_to_arg, void *proc, Eterm earg) &size.processes_used, fi, ERTS_ALC_T_PROC); -#if !HALFWORD_HEAP add_fix_values(&size.processes, &size.processes_used, fi, @@ -2317,11 +2329,28 @@ erts_memory(int *print_to_p, void *print_to_arg, void *proc, Eterm earg) &size.processes_used, fi, ERTS_ALC_T_NLINK_SH); -#endif add_fix_values(&size.processes, &size.processes_used, fi, ERTS_ALC_T_MSG_REF); + add_fix_values(&size.processes, + &size.processes_used, + fi, + ERTS_ALC_T_LL_PTIMER); + add_fix_values(&size.processes, + &size.processes_used, + fi, + ERTS_ALC_T_HL_PTIMER); + add_fix_values(&size.processes, + &size.processes_used, + fi, + ERTS_ALC_T_BIF_TIMER); +#ifdef ERTS_BTM_ACCESSOR_SUPPORT + add_fix_values(&size.processes, + &size.processes_used, + fi, + ERTS_ALC_T_ABIF_TIMER); +#endif } if (want.atom || want.atom_used) { @@ -2645,14 +2674,17 @@ erts_alloc_util_allocators(void *proc) /* * Currently all allocators except sys_alloc are * alloc_util allocators. + * Also hide test_alloc which is disabled by default + * and only intended for our own testing. */ - sz = ((ERTS_ALC_A_MAX + 1 - ERTS_ALC_A_MIN) - 1)*2; + sz = ((ERTS_ALC_A_MAX + 1 - ERTS_ALC_A_MIN) - 2)*2; ASSERT(sz > 0); hp = HAlloc((Process *) proc, sz); res = NIL; for (i = ERTS_ALC_A_MAX; i >= ERTS_ALC_A_MIN; i--) { switch (i) { case ERTS_ALC_A_SYSTEM: + case ERTS_ALC_A_TEST: break; default: { char *alc_str = (char *) ERTS_ALC_A2AD(i); @@ -2735,7 +2767,7 @@ erts_allocator_info(int to, void *arg) erts_mseg_info(i, &to, arg, 0, NULL, NULL); } erts_print(to, arg, "=allocator:mseg_alloc.erts_mmap\n"); - erts_mmap_info(&to, arg, NULL, NULL, &emis); + erts_mmap_info(&erts_dflt_mmapper, &to, arg, NULL, NULL, &emis); } #endif @@ -2961,12 +2993,12 @@ reply_alloc_info(void *vair) int global_instances = air->req_sched == sched_id; ErtsProcLocks rp_locks; Process *rp = air->proc; - Eterm ref_copy = NIL, ai_list, msg; - Eterm *hp = NULL, *hp_end = NULL, *hp_start = NULL; + Eterm ref_copy = NIL, ai_list, msg = NIL; + Eterm *hp = NULL, *hp_start = NULL, *hp_end = NULL; Eterm **hpp; Uint sz, *szp; ErlOffHeap *ohp = NULL; - ErlHeapFragment *bp = NULL; + ErtsMessage *mp = NULL; struct erts_mmap_info_struct emis; int i; Eterm (*info_func)(Allctr_t *, @@ -3087,7 +3119,8 @@ reply_alloc_info(void *vair) ai_list = erts_bld_cons(hpp, szp, ainfo, ai_list); - ainfo = (air->only_sz ? NIL : erts_mmap_info(NULL, NULL, hpp, szp, &emis)); + ainfo = (air->only_sz ? NIL : + erts_mmap_info(&erts_dflt_mmapper, NULL, NULL, hpp, szp, &emis)); ainfo = erts_bld_tuple3(hpp, szp, alloc_atom, erts_bld_atom(hpp,szp,"erts_mmap"), @@ -3114,7 +3147,7 @@ reply_alloc_info(void *vair) make_small(0), ainfo); } else { - erl_exit(ERTS_ABORT_EXIT, "%s:%d: internal error\n", + erts_exit(ERTS_ABORT_EXIT, "%s:%d: internal error\n", __FILE__, __LINE__); } } @@ -3167,30 +3200,23 @@ reply_alloc_info(void *vair) if (hpp) break; - hp = erts_alloc_message_heap(sz, &bp, &ohp, rp, &rp_locks); + mp = erts_alloc_message_heap(rp, &rp_locks, sz, &hp, &ohp); hp_start = hp; hp_end = hp + sz; szp = NULL; hpp = &hp; } - if (bp) - bp = erts_resize_message_buffer(bp, hp - hp_start, &msg, 1); - else { - ASSERT(hp); - HRelease(rp, hp_end, hp); - } - erts_queue_message(rp, &rp_locks, bp, msg, NIL -#ifdef USE_VM_PROBES - , NIL -#endif - ); + if (hp != hp_end) + erts_shrink_message_heap(&mp, rp, hp_start, hp, hp_end, &msg, 1); + + erts_queue_message(rp, &rp_locks, mp, msg, NIL); if (air->req_sched == sched_id) rp_locks &= ~ERTS_PROC_LOCK_MAIN; erts_smp_proc_unlock(rp, rp_locks); - erts_smp_proc_dec_refc(rp); + erts_proc_dec_refc(rp); if (erts_smp_atomic32_dec_read_nob(&air->refc) == 0) aireq_free(air); @@ -3264,7 +3290,7 @@ erts_request_alloc_info(struct process *c_p, erts_smp_atomic32_init_nob(&air->refc, (erts_aint32_t) erts_no_schedulers); - erts_smp_proc_add_refc(c_p, (Sint32) erts_no_schedulers); + erts_proc_add_refc(c_p, (Sint) erts_no_schedulers); #ifdef ERTS_SMP if (erts_no_schedulers > 1) @@ -3355,7 +3381,7 @@ void *safe_realloc(void *ptr, Uint sz) * Keep alloc_SUITE_data/allocator_test.h updated if changes are made * * to erts_alc_test() * \* */ -#define ERTS_ALC_TEST_ABORT erl_exit(ERTS_ABORT_EXIT, "%s:%d: Internal error\n") +#define ERTS_ALC_TEST_ABORT erts_exit(ERTS_ABORT_EXIT, "%s:%d: Internal error\n") UWord erts_alc_test(UWord op, UWord a1, UWord a2, UWord a3) { @@ -3408,8 +3434,11 @@ UWord erts_alc_test(UWord op, UWord a1, UWord a2, UWord a3) init.enable = 1; init.atype = GOODFIT; init.init.util.name_prefix = (char *) a1; - init.init.util.ts = a2 ? 1 : 0; - +#ifdef ERTS_SMP + init.init.util.ts = 1; +#else + init.init.util.ts = a2 ? 1 : 0; +#endif if ((char **) a3) { char **argv = (char **) a3; int i = 0; @@ -3540,6 +3569,41 @@ UWord erts_alc_test(UWord op, UWord a1, UWord a2, UWord a3) #else case 0xf13: return (UWord) 0; #endif + case 0xf14: return (UWord) erts_alloc(ERTS_ALC_T_TEST, (Uint)a1); + + case 0xf15: erts_free(ERTS_ALC_T_TEST, (void*)a1); return 0; + + case 0xf16: { + Uint extra_hdr_sz = UNIT_CEILING((Uint)a1); + ErtsAllocatorThrSpec_t* ts = &erts_allctr_thr_spec[ERTS_ALC_A_TEST]; + Uint offset = ts->allctr[0]->mbc_header_size; + void* orig_creating_mbc = ts->allctr[0]->creating_mbc; + void* orig_destroying_mbc = ts->allctr[0]->destroying_mbc; + void* new_creating_mbc = *(void**)a2; /* inout arg */ + void* new_destroying_mbc = *(void**)a3; /* inout arg */ + int i; + + for (i=0; i < ts->size; i++) { + Allctr_t* ap = ts->allctr[i]; + if (ap->mbc_header_size != offset + || ap->creating_mbc != orig_creating_mbc + || ap->destroying_mbc != orig_destroying_mbc + || ap->mbc_list.first != NULL) + return -1; + } + for (i=0; i < ts->size; i++) { + ts->allctr[i]->mbc_header_size += extra_hdr_sz; + ts->allctr[i]->creating_mbc = new_creating_mbc; + ts->allctr[i]->destroying_mbc = new_destroying_mbc; + } + *(void**)a2 = orig_creating_mbc; + *(void**)a3 = orig_destroying_mbc; + return offset; + } + case 0xf17: { + ErtsAllocatorThrSpec_t* ts = &erts_allctr_thr_spec[ERTS_ALC_A_TEST]; + return ts->allctr[0]->largest_mbc_size; + } default: break; } @@ -3782,7 +3846,7 @@ check_memory_fence(void *ptr, Uint *size, ErtsAlcType_t n, int func) found_type = GET_TYPE_OF_PATTERN(pre_pattern); if (pre_pattern != MK_PATTERN(n)) { if ((FIXED_FENCE_PATTERN_MASK & pre_pattern) != FIXED_FENCE_PATTERN) - erl_exit(ERTS_ABORT_EXIT, + erts_exit(ERTS_ABORT_EXIT, "ERROR: Fence at beginning of memory block (p=0x%u) " "clobbered.\n", (UWord) ptr); @@ -3799,12 +3863,12 @@ check_memory_fence(void *ptr, Uint *size, ErtsAlcType_t n, int func) char *op_str; if ((FIXED_FENCE_PATTERN_MASK & post_pattern) != FIXED_FENCE_PATTERN) - erl_exit(ERTS_ABORT_EXIT, + erts_exit(ERTS_ABORT_EXIT, "ERROR: Fence at end of memory block (p=0x%u, sz=%u) " "clobbered.\n", (UWord) ptr, (UWord) sz); if (found_type != GET_TYPE_OF_PATTERN(post_pattern)) - erl_exit(ERTS_ABORT_EXIT, + erts_exit(ERTS_ABORT_EXIT, "ERROR: Fence around memory block (p=0x%u, sz=%u) " "clobbered.\n", (UWord) ptr, (UWord) sz); @@ -3827,7 +3891,7 @@ check_memory_fence(void *ptr, Uint *size, ErtsAlcType_t n, int func) default: op_str = "???"; break; } - erl_exit(ERTS_ABORT_EXIT, + erts_exit(ERTS_ABORT_EXIT, "ERROR: Memory block (p=0x%u, sz=%u) allocated as type \"%s\"," " but %s as type \"%s\".\n", (UWord) ptr, (UWord) sz, ftype, op_str, otype); @@ -3939,7 +4003,7 @@ static Uint install_debug_functions(void) { int i; - ASSERT(sizeof(erts_allctrs) == sizeof(real_allctrs)); + ERTS_CT_ASSERT(sizeof(erts_allctrs) == sizeof(real_allctrs)); sys_memcpy((void *)real_allctrs,(void *)erts_allctrs,sizeof(erts_allctrs)); diff --git a/erts/emulator/beam/erl_alloc.h b/erts/emulator/beam/erl_alloc.h index d3109b9432..71e4713624 100644 --- a/erts/emulator/beam/erl_alloc.h +++ b/erts/emulator/beam/erl_alloc.h @@ -3,16 +3,17 @@ * * Copyright Ericsson AB 2002-2013. All Rights Reserved. * - * The contents of this file are subject to the Erlang Public License, - * Version 1.1, (the "License"); you may not use this file except in - * compliance with the License. You should have received a copy of the - * Erlang Public License along with this software. If not, it can be - * retrieved online at http://www.erlang.org/. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Software distributed under the License is distributed on an "AS IS" - * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See - * the License for the specific language governing rights and limitations - * under the License. + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. * * %CopyrightEnd% */ @@ -29,6 +30,7 @@ #ifdef USE_THREADS #include "erl_threads.h" #endif +#include "erl_mmap.h" #ifdef DEBUG # undef ERTS_ALC_WANT_INLINE @@ -42,9 +44,11 @@ #if ERTS_CAN_INLINE && ERTS_ALC_WANT_INLINE # define ERTS_ALC_DO_INLINE 1 # define ERTS_ALC_INLINE static ERTS_INLINE +# define ERTS_ALC_FORCE_INLINE static ERTS_FORCE_INLINE #else # define ERTS_ALC_DO_INLINE 0 # define ERTS_ALC_INLINE +# define ERTS_ALC_FORCE_INLINE #endif #define ERTS_ALC_NO_FIXED_SIZES \ @@ -176,6 +180,12 @@ void sys_free(void *) __deprecated; /* erts_free() */ void *sys_alloc(Uint ) __deprecated; /* erts_alloc_fnf() */ void *sys_realloc(void *, Uint) __deprecated; /* erts_realloc_fnf() */ +#undef ERTS_HAVE_IS_IN_LITERAL_RANGE +#if defined(ARCH_32) || defined(ERTS_HAVE_OS_PHYSICAL_MEMORY_RESERVATION) +# define ERTS_HAVE_IS_IN_LITERAL_RANGE +#endif + + /* * erts_alloc[_fnf](), erts_realloc[_fnf](), erts_free() works as * malloc(), realloc(), and free() with the following exceptions: @@ -203,6 +213,9 @@ void erts_free(ErtsAlcType_t type, void *ptr); void *erts_alloc_fnf(ErtsAlcType_t type, Uint size); void *erts_realloc_fnf(ErtsAlcType_t type, void *ptr, Uint size); int erts_is_allctr_wrapper_prelocked(void); +#ifdef ERTS_HAVE_IS_IN_LITERAL_RANGE +int erts_is_in_literal_range(void* ptr); +#endif #endif /* #if !ERTS_ALC_DO_INLINE */ @@ -220,12 +233,14 @@ ERTS_ALC_INLINE void *erts_alloc(ErtsAlcType_t type, Uint size) { void *res; + ERTS_MSACC_PUSH_AND_SET_STATE_X(ERTS_MSACC_STATE_ALLOC); res = (*erts_allctrs[ERTS_ALC_T2A(type)].alloc)( ERTS_ALC_T2N(type), erts_allctrs[ERTS_ALC_T2A(type)].extra, size); if (!res) erts_alloc_n_enomem(ERTS_ALC_T2N(type), size); + ERTS_MSACC_POP_STATE_X(); return res; } @@ -233,6 +248,7 @@ ERTS_ALC_INLINE void *erts_realloc(ErtsAlcType_t type, void *ptr, Uint size) { void *res; + ERTS_MSACC_PUSH_AND_SET_STATE_X(ERTS_MSACC_STATE_ALLOC); res = (*erts_allctrs[ERTS_ALC_T2A(type)].realloc)( ERTS_ALC_T2N(type), erts_allctrs[ERTS_ALC_T2A(type)].extra, @@ -240,37 +256,48 @@ void *erts_realloc(ErtsAlcType_t type, void *ptr, Uint size) size); if (!res) erts_realloc_n_enomem(ERTS_ALC_T2N(type), ptr, size); + ERTS_MSACC_POP_STATE_X(); return res; } ERTS_ALC_INLINE void erts_free(ErtsAlcType_t type, void *ptr) { + ERTS_MSACC_PUSH_AND_SET_STATE_X(ERTS_MSACC_STATE_ALLOC); (*erts_allctrs[ERTS_ALC_T2A(type)].free)( ERTS_ALC_T2N(type), erts_allctrs[ERTS_ALC_T2A(type)].extra, ptr); + ERTS_MSACC_POP_STATE_X(); } ERTS_ALC_INLINE void *erts_alloc_fnf(ErtsAlcType_t type, Uint size) { - return (*erts_allctrs[ERTS_ALC_T2A(type)].alloc)( + void *res; + ERTS_MSACC_PUSH_AND_SET_STATE_X(ERTS_MSACC_STATE_ALLOC); + res = (*erts_allctrs[ERTS_ALC_T2A(type)].alloc)( ERTS_ALC_T2N(type), erts_allctrs[ERTS_ALC_T2A(type)].extra, size); + ERTS_MSACC_POP_STATE_X(); + return res; } ERTS_ALC_INLINE void *erts_realloc_fnf(ErtsAlcType_t type, void *ptr, Uint size) { - return (*erts_allctrs[ERTS_ALC_T2A(type)].realloc)( + void *res; + ERTS_MSACC_PUSH_AND_SET_STATE_X(ERTS_MSACC_STATE_ALLOC); + res = (*erts_allctrs[ERTS_ALC_T2A(type)].realloc)( ERTS_ALC_T2N(type), erts_allctrs[ERTS_ALC_T2A(type)].extra, ptr, size); + ERTS_MSACC_POP_STATE_X(); + return res; } ERTS_ALC_INLINE @@ -280,6 +307,28 @@ int erts_is_allctr_wrapper_prelocked(void) && !!erts_tsd_get(erts_allctr_prelock_tsd_key); /* by me */ } +#ifdef ERTS_HAVE_IS_IN_LITERAL_RANGE + +ERTS_ALC_FORCE_INLINE +int erts_is_in_literal_range(void* ptr) +{ +#if defined(ARCH_32) + Uint ix = (UWord)ptr >> ERTS_MMAP_SUPERALIGNED_BITS; + + return erts_literal_vspace_map[ix / ERTS_VSPACE_WORD_BITS] + & ((UWord)1 << (ix % ERTS_VSPACE_WORD_BITS)); + +#elif defined(ARCH_64) + extern char* erts_literals_start; + extern UWord erts_literals_size; + return ErtsInArea(ptr, erts_literals_start, erts_literals_size); +#else +# error No ARCH_xx +#endif +} + +#endif /* ERTS_HAVE_IS_IN_LITERAL_RANGE */ + #endif /* #if ERTS_ALC_DO_INLINE || defined(ERTS_ALC_INTERNAL__) */ #define ERTS_ALC_GET_THR_IX() ((int) erts_get_scheduler_id()) diff --git a/erts/emulator/beam/erl_alloc.types b/erts/emulator/beam/erl_alloc.types index 17ac6316b7..14067283bd 100644 --- a/erts/emulator/beam/erl_alloc.types +++ b/erts/emulator/beam/erl_alloc.types @@ -3,16 +3,17 @@ # # Copyright Ericsson AB 2003-2014. All Rights Reserved. # -# The contents of this file are subject to the Erlang Public License, -# Version 1.1, (the "License"); you may not use this file except in -# compliance with the License. You should have received a copy of the -# Erlang Public License along with this software. If not, it can be -# retrieved online at http://www.erlang.org/. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# Software distributed under the License is distributed on an "AS IS" -# basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See -# the License for the specific language governing rights and limitations -# under the License. +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. # # %CopyrightEnd% # @@ -85,11 +86,7 @@ allocator LONG_LIVED true ll_alloc allocator EHEAP true eheap_alloc allocator ETS true ets_alloc allocator FIXED_SIZE true fix_alloc - -+if halfword -allocator LONG_LIVED_LOW true ll_low_alloc -allocator STANDARD_LOW true std_low_alloc -+endif +allocator LITERAL true literal_alloc +else # Non smp build @@ -100,18 +97,14 @@ allocator LONG_LIVED false ll_alloc allocator EHEAP false eheap_alloc allocator ETS false ets_alloc allocator FIXED_SIZE false fix_alloc - -+if halfword -allocator LONG_LIVED_LOW false ll_low_alloc -allocator STANDARD_LOW false std_low_alloc -+endif +allocator LITERAL false literal_alloc +endif allocator BINARY true binary_alloc allocator DRIVER true driver_alloc - +allocator TEST true test_alloc # --- Class declarations ----------------------------------------------------- # @@ -159,13 +152,19 @@ type OLD_HEAP EHEAP PROCESSES old_heap type HEAP_FRAG EHEAP PROCESSES heap_frag type TMP_HEAP TEMPORARY PROCESSES tmp_heap type MSG_REF FIXED_SIZE PROCESSES msg_ref +type MSG EHEAP PROCESSES message +type MSGQ_CHNG SHORT_LIVED PROCESSES messages_queue_change type MSG_ROOTS TEMPORARY PROCESSES msg_roots type ROOTSET TEMPORARY PROCESSES root_set type LOADER_TMP TEMPORARY CODE loader_tmp type PREPARED_CODE SHORT_LIVED CODE prepared_code -type BIF_TIMER_TABLE LONG_LIVED SYSTEM bif_timer_table -type SL_BIF_TIMER SHORT_LIVED PROCESSES bif_timer_sl -type LL_BIF_TIMER STANDARD PROCESSES bif_timer_ll +type TIMER_SERVICE LONG_LIVED SYSTEM timer_service +type LL_PTIMER FIXED_SIZE PROCESSES ll_ptimer +type HL_PTIMER FIXED_SIZE PROCESSES hl_ptimer +type BIF_TIMER FIXED_SIZE PROCESSES bif_timer +# type ABIF_TIMER FIXED_SIZE PROCESSES accessor_bif_timer +type TIMER_REQUEST SHORT_LIVED PROCESSES timer_request +type BTM_YIELD_STATE SHORT_LIVED PROCESSES btm_yield_state type REG_TABLE STANDARD SYSTEM reg_tab type FUN_TABLE STANDARD CODE fun_tab type DIST_TABLE STANDARD SYSTEM dist_tab @@ -267,9 +266,10 @@ type CODE_IX_LOCK_Q SHORT_LIVED SYSTEM code_ix_lock_q type PROC_INTERVAL LONG_LIVED SYSTEM process_interval type BUSY_CALLER_TAB SHORT_LIVED SYSTEM busy_caller_table type BUSY_CALLER SHORT_LIVED SYSTEM busy_caller -type PORT_DATA_HEAP STANDARD SYSTEM port_data_heap type PROC_SYS_TSK SHORT_LIVED PROCESSES proc_sys_task type PROC_SYS_TSK_QS SHORT_LIVED PROCESSES proc_sys_task_queues +type NEW_TIME_OFFSET SHORT_LIVED SYSTEM new_time_offset +type IOB_REQ SHORT_LIVED SYSTEM io_bytes_request +if threads_no_smp # Need thread safe allocs, but std_alloc and fix_alloc are not; @@ -324,8 +324,6 @@ type ACTIVE_PROCS STANDARD PROCESSES active_procs +endif +if smp -type SL_PTIMER SHORT_LIVED SYSTEM ptimer_sl -type LL_PTIMER STANDARD SYSTEM ptimer_ll type SYS_MSG_Q SHORT_LIVED PROCESSES system_messages_queue type FP_EXCEPTION LONG_LIVED SYSTEM fp_exception type LL_MPATHS LONG_LIVED SYSTEM ll_migration_paths @@ -345,45 +343,28 @@ type SSB SHORT_LIVED PROCESSES ssb +endif - -+if halfword - -type DDLL_PROCESS STANDARD_LOW SYSTEM ddll_processes -type MONITOR_LH STANDARD_LOW PROCESSES monitor_lh -type NLINK_LH STANDARD_LOW PROCESSES nlink_lh -type CODE LONG_LIVED_LOW CODE code -type DB_HEIR_DATA STANDARD_LOW ETS db_heir_data -type DB_MS_PSDO_PROC LONG_LIVED_LOW ETS db_match_pseudo_proc -type SCHDLR_DATA LONG_LIVED_LOW SYSTEM scheduler_data -type LL_TEMP_TERM LONG_LIVED_LOW SYSTEM ll_temp_term - -type EXPORT LONG_LIVED_LOW CODE export_entry -type MONITOR_SH STANDARD_LOW PROCESSES monitor_sh -type NLINK_SH STANDARD_LOW PROCESSES nlink_sh -type AINFO_REQ STANDARD_LOW SYSTEM alloc_info_request -type SCHED_WTIME_REQ STANDARD_LOW SYSTEM sched_wall_time_request -type GC_INFO_REQ STANDARD_LOW SYSTEM gc_info_request - -+else # "fullword" +type DEBUG SHORT_LIVED SYSTEM debugging type DDLL_PROCESS STANDARD SYSTEM ddll_processes type MONITOR_LH STANDARD PROCESSES monitor_lh type NLINK_LH STANDARD PROCESSES nlink_lh type CODE LONG_LIVED CODE code +type LITERAL LITERAL CODE literal type DB_HEIR_DATA STANDARD ETS db_heir_data type DB_MS_PSDO_PROC LONG_LIVED ETS db_match_pseudo_proc type SCHDLR_DATA LONG_LIVED SYSTEM scheduler_data type LL_TEMP_TERM LONG_LIVED SYSTEM ll_temp_term +type NIF_TRAP_EXPORT STANDARD CODE nif_trap_export_entry type EXPORT LONG_LIVED CODE export_entry type MONITOR_SH FIXED_SIZE PROCESSES monitor_sh type NLINK_SH FIXED_SIZE PROCESSES nlink_sh type AINFO_REQ SHORT_LIVED SYSTEM alloc_info_request type SCHED_WTIME_REQ SHORT_LIVED SYSTEM sched_wall_time_request type GC_INFO_REQ SHORT_LIVED SYSTEM gc_info_request - -+endif - +type PORT_DATA_HEAP STANDARD SYSTEM port_data_heap +type MSACC DRIVER SYSTEM microstate_accounting +type SYS_CHECK_REQ SHORT_LIVED SYSTEM system_check_request # # Types used by system specific code @@ -395,6 +376,7 @@ type DRV_EV_STATE LONG_LIVED SYSTEM driver_event_state type DRV_EV_D_STATE FIXED_SIZE SYSTEM driver_event_data_state type DRV_SEL_D_STATE FIXED_SIZE SYSTEM driver_select_data_state type FD_LIST SHORT_LIVED SYSTEM fd_list +type ACTIVE_FD_ARR SHORT_LIVED SYSTEM active_fd_array type POLLSET LONG_LIVED SYSTEM pollset type POLLSET_UPDREQ SHORT_LIVED SYSTEM pollset_update_req type POLL_FDS LONG_LIVED SYSTEM poll_fds @@ -411,24 +393,16 @@ type CS_PROG_PATH LONG_LIVED SYSTEM cs_prog_path type ENVIRONMENT TEMPORARY SYSTEM environment type PUTENV_STR SYSTEM SYSTEM putenv_string type PRT_REP_EXIT STANDARD SYSTEM port_report_exit +type SYS_BLOCKING STANDARD SYSTEM sys_blocking ++if smp +type SYS_WRITE_BUF TEMPORARY SYSTEM sys_write_buf ++else +type SYS_WRITE_BUF BINARY SYSTEM sys_write_buf +endif -+if ose - -type SYS_READ_BUF TEMPORARY SYSTEM sys_read_buf -type FD_TAB LONG_LIVED SYSTEM fd_tab -type FD_ENTRY_BUF STANDARD SYSTEM fd_entry_buf -type FD_SIG_LIST SHORT_LIVED SYSTEM fd_sig_list -type DRV_EV STANDARD SYSTEM driver_event -type CS_PROG_PATH LONG_LIVED SYSTEM cs_prog_path -type ENVIRONMENT TEMPORARY SYSTEM environment -type PUTENV_STR SYSTEM SYSTEM putenv_string -type PRT_REP_EXIT STANDARD SYSTEM port_report_exit - +endif - +if win32 type DRV_DATA_BUF SYSTEM SYSTEM drv_data_buf @@ -440,4 +414,7 @@ type CON_VPRINTF_BUF TEMPORARY SYSTEM con_vprintf_buf +endif +# This type should only be used for test +type TEST TEST SYSTEM testing + # ---------------------------------------------------------------------------- diff --git a/erts/emulator/beam/erl_alloc_util.c b/erts/emulator/beam/erl_alloc_util.c index 45f0cc4312..5e7dd7cce8 100644 --- a/erts/emulator/beam/erl_alloc_util.c +++ b/erts/emulator/beam/erl_alloc_util.c @@ -3,16 +3,17 @@ * * Copyright Ericsson AB 2002-2013. All Rights Reserved. * - * The contents of this file are subject to the Erlang Public License, - * Version 1.1, (the "License"); you may not use this file except in - * compliance with the License. You should have received a copy of the - * Erlang Public License along with this software. If not, it can be - * retrieved online at http://www.erlang.org/. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Software distributed under the License is distributed on an "AS IS" - * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See - * the License for the specific language governing rights and limitations - * under the License. + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. * * %CopyrightEnd% */ @@ -205,7 +206,7 @@ MBC after deallocating first block: ASSERT(((UWord)(F) & (~FLG_MASK|THIS_FREE_BLK_HDR_FLG|PREV_FREE_BLK_HDR_FLG)) == THIS_FREE_BLK_HDR_FLG), \ (B)->bhdr = ((Sz) | (F)), \ (B)->u.carrier = (C)) - + # define IS_MBC_FIRST_ABLK(AP,B) \ ((((UWord)(B) & ~ERTS_SACRR_UNIT_MASK) == MBC_HEADER_SIZE(AP)) \ && ((B)->bhdr & MBC_ABLK_OFFSET_MASK) == 0) @@ -378,9 +379,8 @@ do { \ #ifdef ERTS_SMP #define SBC_HEADER_SIZE \ - (UNIT_CEILING(sizeof(Carrier_t) \ - - sizeof(ErtsAlcCPoolData_t) \ - + ABLK_HDR_SZ) \ + (UNIT_CEILING(offsetof(Carrier_t, cpool) \ + + ABLK_HDR_SZ) \ - ABLK_HDR_SZ) #else #define SBC_HEADER_SIZE \ @@ -719,7 +719,7 @@ static void make_name_atoms(Allctr_t *allctr); static Block_t *create_carrier(Allctr_t *, Uint, UWord); static void destroy_carrier(Allctr_t *, Block_t *, Carrier_t **); static void mbc_free(Allctr_t *allctr, void *p, Carrier_t **busy_pcrr_pp); -static void dealloc_block(Allctr_t *, void *, int); +static void dealloc_block(Allctr_t *, void *, ErtsAlcFixList_t *, int); /* internal data... */ @@ -751,12 +751,77 @@ internal_free(void *ptr) #endif +#ifdef ARCH_32 + +/* + * Bit vector for the entire 32-bit virtual address space + * with one bit for each super aligned memory segment. + */ + +#define VSPACE_MAP_BITS (1 << (32 - ERTS_MMAP_SUPERALIGNED_BITS)) +#define VSPACE_MAP_SZ (VSPACE_MAP_BITS / ERTS_VSPACE_WORD_BITS) + +static ERTS_INLINE void set_bit(UWord* map, Uint ix) +{ + ASSERT(ix / ERTS_VSPACE_WORD_BITS < VSPACE_MAP_SZ); + map[ix / ERTS_VSPACE_WORD_BITS] + |= ((UWord)1 << (ix % ERTS_VSPACE_WORD_BITS)); +} + +static ERTS_INLINE void clr_bit(UWord* map, Uint ix) +{ + ASSERT(ix / ERTS_VSPACE_WORD_BITS < VSPACE_MAP_SZ); + map[ix / ERTS_VSPACE_WORD_BITS] + &= ~((UWord)1 << (ix % ERTS_VSPACE_WORD_BITS)); +} + +static ERTS_INLINE int is_bit_set(UWord* map, Uint ix) +{ + ASSERT(ix / ERTS_VSPACE_WORD_BITS < VSPACE_MAP_SZ); + return map[ix / ERTS_VSPACE_WORD_BITS] + & ((UWord)1 << (ix % ERTS_VSPACE_WORD_BITS)); +} + +UWord erts_literal_vspace_map[VSPACE_MAP_SZ]; + +static void set_literal_range(void* start, Uint size) +{ + Uint ix = (UWord)start >> ERTS_MMAP_SUPERALIGNED_BITS; + Uint n = size >> ERTS_MMAP_SUPERALIGNED_BITS; + + ASSERT(!((UWord)start & ERTS_INV_SUPERALIGNED_MASK)); + ASSERT(!((UWord)size & ERTS_INV_SUPERALIGNED_MASK)); + ASSERT(n); + while (n--) { + ASSERT(!is_bit_set(erts_literal_vspace_map, ix)); + set_bit(erts_literal_vspace_map, ix); + ix++; + } +} + +static void clear_literal_range(void* start, Uint size) +{ + Uint ix = (UWord)start >> ERTS_MMAP_SUPERALIGNED_BITS; + Uint n = size >> ERTS_MMAP_SUPERALIGNED_BITS; + + ASSERT(!((UWord)start & ERTS_INV_SUPERALIGNED_MASK)); + ASSERT(!((UWord)size & ERTS_INV_SUPERALIGNED_MASK)); + ASSERT(n); + while (n--) { + ASSERT(is_bit_set(erts_literal_vspace_map, ix)); + clr_bit(erts_literal_vspace_map, ix); + ix++; + } +} + +#endif /* ARCH_32 */ + /* mseg ... */ #if HAVE_ERTS_MSEG -static ERTS_INLINE void * -alcu_mseg_alloc(Allctr_t *allctr, Uint *size_p, Uint flags) +void* +erts_alcu_mseg_alloc(Allctr_t *allctr, Uint *size_p, Uint flags) { void *res; UWord size = (UWord) *size_p; @@ -766,8 +831,9 @@ alcu_mseg_alloc(Allctr_t *allctr, Uint *size_p, Uint flags) return res; } -static ERTS_INLINE void * -alcu_mseg_realloc(Allctr_t *allctr, void *seg, Uint old_size, Uint *new_size_p) +void* +erts_alcu_mseg_realloc(Allctr_t *allctr, void *seg, + Uint old_size, Uint *new_size_p) { void *res; UWord new_size = (UWord) *new_size_p; @@ -778,17 +844,103 @@ alcu_mseg_realloc(Allctr_t *allctr, void *seg, Uint old_size, Uint *new_size_p) return res; } -static ERTS_INLINE void -alcu_mseg_dealloc(Allctr_t *allctr, void *seg, Uint size, Uint flags) +void +erts_alcu_mseg_dealloc(Allctr_t *allctr, void *seg, Uint size, Uint flags) { erts_mseg_dealloc_opt(allctr->alloc_no, seg, (UWord) size, flags, &allctr->mseg_opt); INC_CC(allctr->calls.mseg_dealloc); } -#endif -static ERTS_INLINE void * -alcu_sys_alloc(Allctr_t *allctr, Uint size, int superalign) +#if defined(ARCH_32) + +void* +erts_alcu_literal_32_mseg_alloc(Allctr_t *allctr, Uint *size_p, Uint flags) +{ + void* res; + ERTS_LC_ASSERT(allctr->alloc_no == ERTS_ALC_A_LITERAL + && !allctr->t && allctr->thread_safe); + + res = erts_alcu_mseg_alloc(allctr, size_p, flags); + if (res) + set_literal_range(res, *size_p); + return res; +} + +void* +erts_alcu_literal_32_mseg_realloc(Allctr_t *allctr, void *seg, + Uint old_size, Uint *new_size_p) +{ + void* res; + ERTS_LC_ASSERT(allctr->alloc_no == ERTS_ALC_A_LITERAL + && !allctr->t && allctr->thread_safe); + + if (seg && old_size) + clear_literal_range(seg, old_size); + res = erts_alcu_mseg_realloc(allctr, seg, old_size, new_size_p); + if (res) + set_literal_range(res, *new_size_p); + return res; +} + +void +erts_alcu_literal_32_mseg_dealloc(Allctr_t *allctr, void *seg, Uint size, + Uint flags) +{ + ERTS_LC_ASSERT(allctr->alloc_no == ERTS_ALC_A_LITERAL + && !allctr->t && allctr->thread_safe); + + erts_alcu_mseg_dealloc(allctr, seg, size, flags); + + clear_literal_range(seg, size); +} + +#elif defined(ARCH_64) && defined(ERTS_HAVE_OS_PHYSICAL_MEMORY_RESERVATION) + +void* +erts_alcu_literal_64_mseg_alloc(Allctr_t *allctr, Uint *size_p, Uint flags) +{ + void* res; + UWord size = (UWord) *size_p; + Uint32 mmap_flags = ERTS_MMAPFLG_SUPERCARRIER_ONLY; + if (flags & ERTS_MSEG_FLG_2POW) + mmap_flags |= ERTS_MMAPFLG_SUPERALIGNED; + + res = erts_mmap(&erts_literal_mmapper, mmap_flags, &size); + *size_p = (Uint)size; + INC_CC(allctr->calls.mseg_alloc); + return res; +} + +void* +erts_alcu_literal_64_mseg_realloc(Allctr_t *allctr, void *seg, + Uint old_size, Uint *new_size_p) +{ + void *res; + UWord new_size = (UWord) *new_size_p; + res = erts_mremap(&erts_literal_mmapper, ERTS_MSEG_FLG_NONE, seg, old_size, &new_size); + *new_size_p = (Uint) new_size; + INC_CC(allctr->calls.mseg_realloc); + return res; +} + +void +erts_alcu_literal_64_mseg_dealloc(Allctr_t *allctr, void *seg, Uint size, + Uint flags) +{ + Uint32 mmap_flags = ERTS_MMAPFLG_SUPERCARRIER_ONLY; + if (flags & ERTS_MSEG_FLG_2POW) + mmap_flags |= ERTS_MMAPFLG_SUPERALIGNED; + + erts_munmap(&erts_literal_mmapper, mmap_flags, seg, (UWord)size); + INC_CC(allctr->calls.mseg_dealloc); +} +#endif /* ARCH_64 && ERTS_HAVE_OS_PHYSICAL_MEMORY_RESERVATION */ + +#endif /* HAVE_ERTS_MSEG */ + +void* +erts_alcu_sys_alloc(Allctr_t *allctr, Uint size, int superalign) { void *res; #if ERTS_SA_MB_CARRIERS && ERTS_HAVE_ERTS_SYS_ALIGNED_ALLOC @@ -803,8 +955,8 @@ alcu_sys_alloc(Allctr_t *allctr, Uint size, int superalign) return res; } -static ERTS_INLINE void * -alcu_sys_realloc(Allctr_t *allctr, void *ptr, Uint size, Uint old_size, int superalign) +void* +erts_alcu_sys_realloc(Allctr_t *allctr, void *ptr, Uint size, Uint old_size, int superalign) { void *res; @@ -824,8 +976,8 @@ alcu_sys_realloc(Allctr_t *allctr, void *ptr, Uint size, Uint old_size, int supe return res; } -static ERTS_INLINE void -alcu_sys_free(Allctr_t *allctr, void *ptr, int superalign) +void +erts_alcu_sys_dealloc(Allctr_t *allctr, void *ptr, Uint size, int superalign) { #if ERTS_SA_MB_CARRIERS && ERTS_HAVE_ERTS_SYS_ALIGNED_ALLOC if (superalign) @@ -838,6 +990,49 @@ alcu_sys_free(Allctr_t *allctr, void *ptr, int superalign) erts_mtrace_crr_free(allctr->alloc_no, ERTS_ALC_A_SYSTEM, ptr); } +#ifdef ARCH_32 + +void* +erts_alcu_literal_32_sys_alloc(Allctr_t *allctr, Uint size, int superalign) +{ + void* res; + ERTS_LC_ASSERT(allctr->alloc_no == ERTS_ALC_A_LITERAL + && !allctr->t && allctr->thread_safe); + + res = erts_alcu_sys_alloc(allctr, size, 1); + if (res) + set_literal_range(res, size); + return res; +} + +void* +erts_alcu_literal_32_sys_realloc(Allctr_t *allctr, void *ptr, Uint size, Uint old_size, int superalign) +{ + void* res; + ERTS_LC_ASSERT(allctr->alloc_no == ERTS_ALC_A_LITERAL + && !allctr->t && allctr->thread_safe); + + if (ptr && old_size) + clear_literal_range(ptr, old_size); + res = erts_alcu_sys_realloc(allctr, ptr, size, old_size, 1); + if (res) + set_literal_range(res, size); + return res; +} + +void +erts_alcu_literal_32_sys_dealloc(Allctr_t *allctr, void *ptr, Uint size, int superalign) +{ + ERTS_LC_ASSERT(allctr->alloc_no == ERTS_ALC_A_LITERAL + && !allctr->t && allctr->thread_safe); + + erts_alcu_sys_dealloc(allctr, ptr, size, 1); + + clear_literal_range(ptr, size); +} + +#endif /* ARCH_32 */ + static Uint get_next_mbc_size(Allctr_t *allctr) { @@ -929,6 +1124,88 @@ unlink_carrier(CarrierList_t *cl, Carrier_t *crr) #ifdef ERTS_SMP +#ifdef DEBUG +static int is_in_list(ErtsDoubleLink_t* sentinel, ErtsDoubleLink_t* node) +{ + ErtsDoubleLink_t* p; + + ASSERT(node != sentinel); + for (p = sentinel->next; p != sentinel; p = p->next) { + if (p == node) + return 1; + } + return 0; +} +#endif /* DEBUG */ + +static ERTS_INLINE void +link_edl_after(ErtsDoubleLink_t* after_me, ErtsDoubleLink_t* node) +{ + ErtsDoubleLink_t* before_me = after_me->next; + ASSERT(node != after_me && node != before_me); + node->next = before_me; + node->prev = after_me; + before_me->prev = node; + after_me->next = node; +} + +static ERTS_INLINE void +link_edl_before(ErtsDoubleLink_t* before_me, ErtsDoubleLink_t* node) +{ + ErtsDoubleLink_t* after_me = before_me->prev; + ASSERT(node != before_me && node != after_me); + node->next = before_me; + node->prev = after_me; + before_me->prev = node; + after_me->next = node; +} + +static ERTS_INLINE void +unlink_edl(ErtsDoubleLink_t* node) +{ + node->next->prev = node->prev; + node->prev->next = node->next; +} + +static ERTS_INLINE void +relink_edl_before(ErtsDoubleLink_t* before_me, ErtsDoubleLink_t* node) +{ + if (node != before_me && node != before_me->prev) { + unlink_edl(node); + link_edl_before(before_me, node); + } +} + +static ERTS_INLINE int is_abandoned(Carrier_t *crr) +{ + return crr->cpool.abandoned.next != NULL; +} + +static ERTS_INLINE void +link_abandoned_carrier(ErtsDoubleLink_t* list, Carrier_t *crr) +{ + ASSERT(!is_abandoned(crr)); + + link_edl_after(list, &crr->cpool.abandoned); + + ASSERT(crr->cpool.abandoned.next != &crr->cpool.abandoned); + ASSERT(crr->cpool.abandoned.prev != &crr->cpool.abandoned); +} + +static ERTS_INLINE void +unlink_abandoned_carrier(Carrier_t *crr) +{ + ASSERT(is_in_list(&crr->cpool.orig_allctr->cpool.pooled_list, + &crr->cpool.abandoned) || + is_in_list(&crr->cpool.orig_allctr->cpool.traitor_list, + &crr->cpool.abandoned)); + + unlink_edl(&crr->cpool.abandoned); + + crr->cpool.abandoned.next = NULL; + crr->cpool.abandoned.prev = NULL; +} + static ERTS_INLINE void clear_busy_pool_carrier(Allctr_t *allctr, Carrier_t *crr) { @@ -955,7 +1232,7 @@ clear_busy_pool_carrier(Allctr_t *allctr, Carrier_t *crr) } } -#endif +#endif /* ERTS_SMP */ #if 0 #define ERTS_DBG_CHK_FIX_LIST(A, FIX, IX, B) \ @@ -986,17 +1263,21 @@ typedef struct { } ErtsAllctrFixDDBlock_t; #endif +#define ERTS_ALC_FIX_NO_UNUSE (((ErtsAlcType_t) 1) << ERTS_ALC_N_BITS) + static ERTS_INLINE void dealloc_fix_block(Allctr_t *allctr, ErtsAlcType_t type, void *ptr, + ErtsAlcFixList_t *fix, int dec_cc_on_redirect) { #ifdef ERTS_SMP /* May be redirected... */ - ((ErtsAllctrFixDDBlock_t *) ptr)->fix_type = type; + ASSERT((type & ERTS_ALC_FIX_NO_UNUSE) == 0); + ((ErtsAllctrFixDDBlock_t *) ptr)->fix_type = type | ERTS_ALC_FIX_NO_UNUSE; #endif - dealloc_block(allctr, ptr, dec_cc_on_redirect); + dealloc_block(allctr, ptr, fix, dec_cc_on_redirect); } static ERTS_INLINE void @@ -1042,8 +1323,7 @@ fix_cpool_check_shrink(Allctr_t *allctr, if (fix->u.cpool.min_list_size > fix->list_size) fix->u.cpool.min_list_size = fix->list_size; - fix->u.cpool.allocated--; - dealloc_fix_block(allctr, type, p, 0); + dealloc_fix_block(allctr, type, p, fix, 0); } } } @@ -1089,7 +1369,8 @@ static ERTS_INLINE void fix_cpool_free(Allctr_t *allctr, ErtsAlcType_t type, void *p, - Carrier_t **busy_pcrr_pp) + Carrier_t **busy_pcrr_pp, + int unuse) { ErtsAlcFixList_t *fix; @@ -1097,8 +1378,9 @@ fix_cpool_free(Allctr_t *allctr, && type <= ERTS_ALC_N_MAX_A_FIXED_SIZE); fix = &allctr->fix[type - ERTS_ALC_N_MIN_A_FIXED_SIZE]; - - fix->u.cpool.used--; + + if (unuse) + fix->u.cpool.used--; if ((!busy_pcrr_pp || !*busy_pcrr_pp) && !fix->u.cpool.shrink_list @@ -1156,8 +1438,7 @@ fix_cpool_alloc_shrink(Allctr_t *allctr, erts_aint32_t flgs) fix->list = *((void **) ptr); fix->list_size--; fix->u.cpool.shrink_list--; - fix->u.cpool.allocated--; - dealloc_fix_block(allctr, type, ptr, 0); + dealloc_fix_block(allctr, type, ptr, fix, 0); } if (fix->u.cpool.min_list_size > fix->list_size) fix->u.cpool.min_list_size = fix->list_size; @@ -1318,7 +1599,7 @@ fix_nocpool_alloc_shrink(Allctr_t *allctr, erts_aint32_t flgs) ptr = fix->list; fix->list = *((void **) ptr); fix->list_size--; - dealloc_block(allctr, ptr, 0); + dealloc_block(allctr, ptr, NULL, 0); fix->u.nocpool.allocated--; } if (fix->list_size != 0) { @@ -1351,6 +1632,16 @@ erts_alcu_fix_alloc_shrink(Allctr_t *allctr, erts_aint32_t flgs) static void dealloc_carrier(Allctr_t *allctr, Carrier_t *crr, int superaligned); +static ERTS_INLINE void +dealloc_mbc(Allctr_t *allctr, Carrier_t *crr) +{ + ASSERT(IS_MB_CARRIER(crr)); + if (allctr->destroying_mbc) + allctr->destroying_mbc(allctr, crr); + + dealloc_carrier(allctr, crr, 1); +} + #ifdef ERTS_SMP static ERTS_INLINE Allctr_t* @@ -1361,7 +1652,7 @@ get_pref_allctr(void *extra) pref_ix = ERTS_ALC_GET_THR_IX(); - ASSERT(sizeof(UWord) == sizeof(Allctr_t *)); + ERTS_CT_ASSERT(sizeof(UWord) == sizeof(Allctr_t *)); ASSERT(0 <= pref_ix && pref_ix < tspec->size); return tspec->allctr[pref_ix]; @@ -1665,11 +1956,13 @@ handle_delayed_fix_dealloc(Allctr_t *allctr, void *ptr) type = ((ErtsAllctrFixDDBlock_t *) ptr)->fix_type; - ASSERT(ERTS_ALC_N_MIN_A_FIXED_SIZE <= type - && type <= ERTS_ALC_N_MAX_A_FIXED_SIZE); + ASSERT(ERTS_ALC_N_MIN_A_FIXED_SIZE + <= (type & ~ERTS_ALC_FIX_NO_UNUSE)); + ASSERT((type & ~ERTS_ALC_FIX_NO_UNUSE) + <= ERTS_ALC_N_MAX_A_FIXED_SIZE); if (!ERTS_ALC_IS_CPOOL_ENABLED(allctr)) - fix_nocpool_free(allctr, type, ptr); + fix_nocpool_free(allctr, (type & ~ERTS_ALC_FIX_NO_UNUSE), ptr); else { Block_t *blk = UMEM2BLK(ptr); Carrier_t *busy_pcrr_p; @@ -1684,7 +1977,9 @@ handle_delayed_fix_dealloc(Allctr_t *allctr, void *ptr) NULL, &busy_pcrr_p); if (used_allctr == allctr) { doit: - fix_cpool_free(allctr, type, ptr, &busy_pcrr_p); + fix_cpool_free(allctr, (type & ~ERTS_ALC_FIX_NO_UNUSE), + ptr, &busy_pcrr_p, + !(type & ERTS_ALC_FIX_NO_UNUSE)); clear_busy_pool_carrier(allctr, busy_pcrr_p); } else { @@ -1775,6 +2070,18 @@ handle_delayed_dealloc(Allctr_t *allctr, * data has been overwritten by the queue. */ Carrier_t *crr = FIRST_BLK_TO_MBC(allctr, blk); + + /* Restore word overwritten by the dd-queue as it will be read + * if this carrier is pulled from dc_list by cpool_fetch() + */ + ERTS_ALC_CPOOL_ASSERT(FBLK_TO_MBC(blk) != crr); + ERTS_CT_ASSERT(sizeof(ErtsAllctrDDBlock_t) == sizeof(void*)); +#ifdef MBC_ABLK_OFFSET_BITS + blk->u.carrier = crr; +#else + blk->carrier = crr; +#endif + ERTS_ALC_CPOOL_ASSERT(ERTS_ALC_IS_CPOOL_ENABLED(allctr)); ERTS_ALC_CPOOL_ASSERT(allctr == crr->cpool.orig_allctr); ERTS_ALC_CPOOL_ASSERT(((erts_aint_t) allctr) @@ -1792,7 +2099,7 @@ handle_delayed_dealloc(Allctr_t *allctr, if (fix) handle_delayed_fix_dealloc(allctr, ptr); else - dealloc_block(allctr, ptr, 1); + dealloc_block(allctr, ptr, NULL, 1); } } @@ -1898,15 +2205,24 @@ erts_alcu_check_delayed_dealloc(Allctr_t *allctr, ERTS_ALCU_DD_OPS_LIM_LOW, NULL, NULL, NULL) static void -dealloc_block(Allctr_t *allctr, void *ptr, int dec_cc_on_redirect) +dealloc_block(Allctr_t *allctr, void *ptr, ErtsAlcFixList_t *fix, int dec_cc_on_redirect) { Block_t *blk = UMEM2BLK(ptr); ERTS_SMP_LC_ASSERT(!allctr->thread_safe || erts_lc_mtx_is_locked(&allctr->mutex)); - if (IS_SBC_BLK(blk)) + if (IS_SBC_BLK(blk)) { destroy_carrier(allctr, blk, NULL); +#ifdef ERTS_SMP + if (fix && ERTS_ALC_IS_CPOOL_ENABLED(allctr)) { + ErtsAlcType_t type = ((ErtsAllctrFixDDBlock_t *) ptr)->fix_type; + if (!(type & ERTS_ALC_FIX_NO_UNUSE)) + fix->u.cpool.used--; + fix->u.cpool.allocated--; + } +#endif + } #ifndef ERTS_SMP else mbc_free(allctr, ptr, NULL); @@ -1919,6 +2235,12 @@ dealloc_block(Allctr_t *allctr, void *ptr, int dec_cc_on_redirect) used_allctr = get_used_allctr(allctr, ERTS_ALC_TS_PREF_LOCK_NO, ptr, NULL, &busy_pcrr_p); if (used_allctr == allctr) { + if (fix) { + ErtsAlcType_t type = ((ErtsAllctrFixDDBlock_t *) ptr)->fix_type; + if (!(type & ERTS_ALC_FIX_NO_UNUSE)) + fix->u.cpool.used--; + fix->u.cpool.allocated--; + } mbc_free(allctr, ptr, &busy_pcrr_p); clear_busy_pool_carrier(allctr, busy_pcrr_p); } @@ -1957,7 +2279,7 @@ mbc_alloc_block(Allctr_t *allctr, Uint size, Uint *blk_szp) if (!blk) { blk = create_carrier(allctr, get_blk_sz, CFLG_MBC); -#if !HALFWORD_HEAP && !ERTS_SUPER_ALIGNED_MSEG_ONLY +#if !ERTS_SUPER_ALIGNED_MSEG_ONLY if (!blk) { /* Emergency! We couldn't create the carrier as we wanted. Try to place it in a sys_alloced sbc. */ @@ -2563,10 +2885,9 @@ mbc_realloc(Allctr_t *allctr, void *p, Uint size, Uint32 alcu_flgs, #ifdef ERTS_SMP #define ERTS_ALC_MAX_DEALLOC_CARRIER 10 -#define ERTS_ALC_CPOOL_MAX_FETCH_INSPECT 10 +#define ERTS_ALC_CPOOL_MAX_FETCH_INSPECT 20 +#define ERTS_ALC_CPOOL_MAX_TRAITOR_INSPECT 10 #define ERTS_ALC_CPOOL_CHECK_LIMIT_COUNT 100 -#define ERTS_ALC_CPOOL_MAX_NO_CARRIERS 5 -#define ERTS_ALC_CPOOL_INSERT_ALLOWED_OFFSET 100 #define ERTS_ALC_CPOOL_MAX_FAILED_STAT_READS 3 #define ERTS_ALC_CPOOL_PTR_MOD_MRK (((erts_aint_t) 1) << 0) @@ -2743,9 +3064,6 @@ cpool_insert(Allctr_t *allctr, Carrier_t *crr) (erts_aint_t) CARRIER_SZ(crr)); erts_atomic_inc_nob(&allctr->cpool.stat.no_carriers); - erts_smp_atomic_set_nob(&crr->allctr, - ((erts_aint_t) allctr)|ERTS_CRR_ALCTR_FLG_IN_POOL); - /* * We search in 'next' direction and begin by passing * one element before trying to insert. This in order to @@ -2804,6 +3122,9 @@ cpool_insert(Allctr_t *allctr, Carrier_t *crr) cpool_set_mod_marked(&cpd2p->prev, (erts_aint_t) &crr->cpool, (erts_aint_t) cpd1p); + + erts_smp_atomic_set_wb(&crr->allctr, + ((erts_aint_t) allctr)|ERTS_CRR_ALCTR_FLG_IN_POOL); } static void @@ -2904,59 +3225,163 @@ cpool_delete(Allctr_t *allctr, Allctr_t *prev_allctr, Carrier_t *crr) static Carrier_t * cpool_fetch(Allctr_t *allctr, UWord size) { - int i; + int i, i_stop, has_passed_sentinel; Carrier_t *crr; ErtsAlcCPoolData_t *cpdp; - ErtsAlcCPoolData_t *sentinel = &carrier_pool[allctr->alloc_no].sentinel; + ErtsAlcCPoolData_t *cpool_entrance; + ErtsAlcCPoolData_t *sentinel; + ErtsDoubleLink_t* dl; + ErtsDoubleLink_t* first_old_traitor; ERTS_ALC_CPOOL_ASSERT(allctr->alloc_no == ERTS_ALC_A_INVALID /* testcase */ || erts_thr_progress_is_managed_thread()); - i = 0; + i = ERTS_ALC_CPOOL_MAX_FETCH_INSPECT; + first_old_traitor = allctr->cpool.traitor_list.next; + cpool_entrance = NULL; - /* First; check our own pending dealloc carrier list... */ - crr = allctr->cpool.dc_list.last; - while (crr && i < ERTS_ALC_CPOOL_MAX_FETCH_INSPECT) { - if (erts_atomic_read_nob(&crr->cpool.max_size) >= size) { - unlink_carrier(&allctr->cpool.dc_list, crr); -#ifdef ERTS_ALC_CPOOL_DEBUG - ERTS_ALC_CPOOL_ASSERT(erts_smp_atomic_xchg_nob(&crr->allctr, - ((erts_aint_t) allctr)) - == (((erts_aint_t) allctr) & ~ERTS_CRR_ALCTR_FLG_MASK)); -#else - erts_smp_atomic_set_nob(&crr->allctr, ((erts_aint_t) allctr)); -#endif - return crr; + /* + * Search my own pooled_list, + * i.e my abandoned carriers that were in the pool last time I checked. + */ + + dl = allctr->cpool.pooled_list.next; + while(dl != &allctr->cpool.pooled_list) { + erts_aint_t exp, act; + crr = (Carrier_t *) (((char *) dl) - offsetof(Carrier_t, cpool.abandoned)); + + ASSERT(!is_in_list(&allctr->cpool.traitor_list, dl)); + ASSERT(crr->cpool.orig_allctr == allctr); + dl = dl->next; + exp = erts_smp_atomic_read_rb(&crr->allctr); + if ((exp & ERTS_CRR_ALCTR_FLG_MASK) == ERTS_CRR_ALCTR_FLG_IN_POOL + && erts_atomic_read_nob(&crr->cpool.max_size) >= size) { + /* Try to fetch it... */ + act = erts_smp_atomic_cmpxchg_mb(&crr->allctr, + (erts_aint_t) allctr, + exp); + if (act == exp) { + cpool_delete(allctr, ((Allctr_t *) (act & ~ERTS_CRR_ALCTR_FLG_MASK)), crr); + unlink_abandoned_carrier(crr); + + /* Move sentinel to continue next search from here */ + relink_edl_before(dl, &allctr->cpool.pooled_list); + return crr; + } + exp = act; + } + if (exp & ERTS_CRR_ALCTR_FLG_IN_POOL) { + if (!cpool_entrance) + cpool_entrance = &crr->cpool; + } + else { /* Not in pool, move to traitor_list */ + unlink_abandoned_carrier(crr); + link_abandoned_carrier(&allctr->cpool.traitor_list, crr); + } + if (--i <= 0) { + /* Move sentinel to continue next search from here */ + relink_edl_before(dl, &allctr->cpool.pooled_list); + return NULL; } - crr = crr->prev; - i++; } - /* ... then the pool ... */ + /* Now search traitor_list. + * i.e carriers employed by other allocators last time I checked. + * They might have been abandoned since then. + */ + + i_stop = (i < ERTS_ALC_CPOOL_MAX_TRAITOR_INSPECT ? + 0 : i - ERTS_ALC_CPOOL_MAX_TRAITOR_INSPECT); + dl = first_old_traitor; + while(dl != &allctr->cpool.traitor_list) { + erts_aint_t exp, act; + crr = (Carrier_t *) (((char *) dl) - offsetof(Carrier_t, cpool.abandoned)); + ASSERT(dl != &allctr->cpool.pooled_list); + ASSERT(crr->cpool.orig_allctr == allctr); + dl = dl->next; + exp = erts_smp_atomic_read_rb(&crr->allctr); + if (exp & ERTS_CRR_ALCTR_FLG_IN_POOL) { + if (!(exp & ERTS_CRR_ALCTR_FLG_BUSY) + && erts_atomic_read_nob(&crr->cpool.max_size) >= size) { + /* Try to fetch it... */ + act = erts_smp_atomic_cmpxchg_mb(&crr->allctr, + (erts_aint_t) allctr, + exp); + if (act == exp) { + cpool_delete(allctr, ((Allctr_t *) (act & ~ERTS_CRR_ALCTR_FLG_MASK)), crr); + unlink_abandoned_carrier(crr); + + /* Move sentinel to continue next search from here */ + relink_edl_before(dl, &allctr->cpool.traitor_list); + return crr; + } + exp = act; + } + if (exp & ERTS_CRR_ALCTR_FLG_IN_POOL) { + if (!cpool_entrance) + cpool_entrance = &crr->cpool; + + /* Move to pooled_list */ + unlink_abandoned_carrier(crr); + link_abandoned_carrier(&allctr->cpool.pooled_list, crr); + } + } + if (--i <= i_stop) { + /* Move sentinel to continue next search from here */ + relink_edl_before(dl, &allctr->cpool.traitor_list); + if (i > 0) + break; + else + return NULL; + } + } /* - * We search in 'prev' direction and begin by passing - * one element before trying to fetch. This in order to - * avoid contention with threads inserting elements. + * Finally search the shared pool and try employ foreign carriers */ - cpdp = cpool_aint2cpd(cpool_read(&sentinel->prev)); - if (cpdp == sentinel) - return NULL; + sentinel = &carrier_pool[allctr->alloc_no].sentinel; + if (cpool_entrance) { + /* We saw a pooled carried above, use it as entrance into the pool + */ + cpdp = cpool_entrance; + } + else { + /* No pooled carried seen above. Start search at cpool sentinel, + * but begin by passing one element before trying to fetch. + * This in order to avoid contention with threads inserting elements. + */ + cpool_entrance = sentinel; + cpdp = cpool_aint2cpd(cpool_read(&cpool_entrance->prev)); + if (cpdp == sentinel) + goto check_dc_list; + } - while (i < ERTS_ALC_CPOOL_MAX_FETCH_INSPECT) { + has_passed_sentinel = 0; + while (1) { erts_aint_t exp; cpdp = cpool_aint2cpd(cpool_read(&cpdp->prev)); - if (cpdp == sentinel) { + if (cpdp == cpool_entrance) { + if (cpool_entrance == sentinel) { + cpdp = cpool_aint2cpd(cpool_read(&cpdp->prev)); + if (cpdp == sentinel) + break; + } + i = 0; /* Last one to inspect */ + } + else if (cpdp == sentinel) { + if (has_passed_sentinel) { + /* We been here before. cpool_entrance must have been removed */ + break; + } cpdp = cpool_aint2cpd(cpool_read(&cpdp->prev)); if (cpdp == sentinel) - return NULL; - i = ERTS_ALC_CPOOL_MAX_FETCH_INSPECT; /* Last one to inspect */ + break; + has_passed_sentinel = 1; } - crr = (Carrier_t *) (((char *) cpdp) - offsetof(Carrier_t, cpool)); + crr = (Carrier_t *)(((char *)cpdp) - offsetof(Carrier_t, cpool)); exp = erts_smp_atomic_read_rb(&crr->allctr); - if (((exp & (ERTS_CRR_ALCTR_FLG_IN_POOL|ERTS_CRR_ALCTR_FLG_BUSY)) - == ERTS_CRR_ALCTR_FLG_IN_POOL) + if (((exp & (ERTS_CRR_ALCTR_FLG_MASK)) == ERTS_CRR_ALCTR_FLG_IN_POOL) && (erts_atomic_read_nob(&cpdp->max_size) >= size)) { erts_aint_t act; /* Try to fetch it... */ @@ -2965,11 +3390,40 @@ cpool_fetch(Allctr_t *allctr, UWord size) exp); if (act == exp) { cpool_delete(allctr, ((Allctr_t *) (act & ~ERTS_CRR_ALCTR_FLG_MASK)), crr); + if (crr->cpool.orig_allctr == allctr) { + unlink_abandoned_carrier(crr); + } return crr; } } - i++; + if (--i <= 0) + return NULL; } + +check_dc_list: + /* Last; check our own pending dealloc carrier list... */ + crr = allctr->cpool.dc_list.last; + while (crr) { + if (erts_atomic_read_nob(&crr->cpool.max_size) >= size) { + Block_t* blk; + unlink_carrier(&allctr->cpool.dc_list, crr); +#ifdef ERTS_ALC_CPOOL_DEBUG + ERTS_ALC_CPOOL_ASSERT(erts_smp_atomic_xchg_nob(&crr->allctr, + ((erts_aint_t) allctr)) + == (((erts_aint_t) allctr) & ~ERTS_CRR_ALCTR_FLG_MASK)); +#else + erts_smp_atomic_set_nob(&crr->allctr, ((erts_aint_t) allctr)); +#endif + blk = MBC_TO_FIRST_BLK(allctr, crr); + ASSERT(FBLK_TO_MBC(blk) == crr); + allctr->link_free_block(allctr, blk); + return crr; + } + crr = crr->prev; + if (--i <= 0) + return NULL; + } + return NULL; } @@ -2993,7 +3447,7 @@ check_pending_dealloc_carrier(Allctr_t *allctr, dcrr = crr; crr = crr->next; - dealloc_carrier(allctr, dcrr, 1); + dealloc_mbc(allctr, dcrr); i++; } while (crr && i < ERTS_ALC_MAX_DEALLOC_CARRIER); @@ -3024,18 +3478,20 @@ static void schedule_dealloc_carrier(Allctr_t *allctr, Carrier_t *crr) { Allctr_t *orig_allctr; + Block_t *blk; int check_pending_dealloc; erts_aint_t max_size; + ASSERT(IS_MB_CARRIER(crr)); + if (!ERTS_ALC_IS_CPOOL_ENABLED(allctr)) { - dealloc_carrier(allctr, crr, 1); + dealloc_mbc(allctr, crr); return; } orig_allctr = crr->cpool.orig_allctr; if (allctr != orig_allctr) { - Block_t *blk = MBC_TO_FIRST_BLK(allctr, crr); int cinit = orig_allctr->dd.ix - allctr->dd.ix; /* @@ -3052,6 +3508,7 @@ schedule_dealloc_carrier(Allctr_t *allctr, Carrier_t *crr) * since the block is an mbc block that is free and last * in the carrier. */ + blk = MBC_TO_FIRST_BLK(allctr, crr); ERTS_ALC_CPOOL_ASSERT(IS_FREE_LAST_MBC_BLK(blk)); ERTS_ALC_CPOOL_ASSERT(IS_MBC_FIRST_ABLK(allctr, blk)); @@ -3066,13 +3523,18 @@ schedule_dealloc_carrier(Allctr_t *allctr, Carrier_t *crr) return; } + if (is_abandoned(crr)) + unlink_abandoned_carrier(crr); + if (crr->cpool.thr_prgr == ERTS_THR_PRGR_INVALID || erts_thr_progress_has_reached(crr->cpool.thr_prgr)) { - dealloc_carrier(allctr, crr, 1); + dealloc_mbc(allctr, crr); return; } - max_size = (erts_aint_t) allctr->largest_fblk_in_mbc(allctr, crr); + blk = MBC_TO_FIRST_BLK(allctr, crr); + ASSERT(IS_FREE_LAST_MBC_BLK(blk)); + max_size = (erts_aint_t) MBC_FBLK_SZ(blk); erts_atomic_set_nob(&crr->cpool.max_size, max_size); crr->next = NULL; @@ -3112,6 +3574,8 @@ cpool_init_carrier_data(Allctr_t *allctr, Carrier_t *crr) limit = (csz/100)*allctr->cpool.util_limit; crr->cpool.abandon_limit = limit; } + crr->cpool.abandoned.next = NULL; + crr->cpool.abandoned.prev = NULL; } static void @@ -3142,6 +3606,9 @@ abandon_carrier(Allctr_t *allctr, Carrier_t *crr) STAT_MBC_CPOOL_INSERT(allctr, crr); unlink_carrier(&allctr->mbc_list, crr); + if (crr->cpool.orig_allctr == allctr) { + link_abandoned_carrier(&allctr->cpool.pooled_list, crr); + } allctr->remove_mbc(allctr, crr); @@ -3259,8 +3726,7 @@ create_carrier(Allctr_t *allctr, Uint umem_sz, UWord flags) int is_mseg = 0; #endif - if (HALFWORD_HEAP - || (ERTS_SUPER_ALIGNED_MSEG_ONLY && (flags & CFLG_MBC)) + if ((ERTS_SUPER_ALIGNED_MSEG_ONLY && (flags & CFLG_MBC)) || !allow_sys_alloc_carriers) { flags |= CFLG_FORCE_MSEG; flags &= ~CFLG_FORCE_SYS_ALLOC; @@ -3268,13 +3734,38 @@ create_carrier(Allctr_t *allctr, Uint umem_sz, UWord flags) return NULL; #endif } + flags |= allctr->crr_set_flgs; + flags &= ~allctr->crr_clr_flgs; ASSERT((flags & CFLG_SBC && !(flags & CFLG_MBC)) || (flags & CFLG_MBC && !(flags & CFLG_SBC))); ASSERT(!(flags & CFLG_FORCE_MSEG && flags & CFLG_FORCE_SYS_ALLOC)); - blk_sz = UMEMSZ2BLKSZ(allctr, umem_sz); + if (umem_sz > (ERTS_UINT_MAX - ERTS_UINT_MAX/100)) { + /* Do an overly conservative _overflow_ check here so we don't + * have to deal with it from here on. I guess we could be more accurate + * but I don't think the need to allocate over 99% of the address space + * will ever arise on any machine, neither 32 nor 64 bit. + */ + return NULL; + } + + if (flags & CFLG_MAIN_CARRIER) { + ASSERT(flags & CFLG_MBC); + ASSERT(flags & CFLG_NO_CPOOL); + ASSERT(umem_sz == allctr->main_carrier_size); + ERTS_UNDEF(blk_sz, 0); + + if (allctr->main_carrier_size < allctr->min_mbc_size) + allctr->main_carrier_size = allctr->min_mbc_size; + crr_sz = bcrr_sz = allctr->main_carrier_size; + } + else { + ERTS_UNDEF(bcrr_sz, 0); + ERTS_UNDEF(crr_sz, 0); + blk_sz = UMEMSZ2BLKSZ(allctr, umem_sz); + } #ifdef ERTS_SMP allctr->cpool.disable_abandon = ERTS_ALC_CPOOL_MAX_DISABLE_ABANDON; @@ -3320,13 +3811,15 @@ create_carrier(Allctr_t *allctr, Uint umem_sz, UWord flags) mseg_flags = ERTS_MSEG_FLG_NONE; } else { - crr_sz = (*allctr->get_next_mbc_size)(allctr); - if (crr_sz < MBC_HEADER_SIZE(allctr) + blk_sz) - crr_sz = MBC_HEADER_SIZE(allctr) + blk_sz; - mseg_flags = ERTS_MSEG_FLG_2POW; + if (!(flags & CFLG_MAIN_CARRIER)) { + crr_sz = (*allctr->get_next_mbc_size)(allctr); + if (crr_sz < MBC_HEADER_SIZE(allctr) + blk_sz) + crr_sz = MBC_HEADER_SIZE(allctr) + blk_sz; + } + mseg_flags = ERTS_MSEG_FLG_2POW; } - crr = (Carrier_t *) alcu_mseg_alloc(allctr, &crr_sz, mseg_flags); + crr = (Carrier_t *) allctr->mseg_alloc(allctr, &crr_sz, mseg_flags); if (!crr) { have_tried_mseg = 1; if (!(have_tried_sys_alloc || flags & CFLG_FORCE_MSEG)) @@ -3358,23 +3851,22 @@ create_carrier(Allctr_t *allctr, Uint umem_sz, UWord flags) if (flags & CFLG_SBC) { bcrr_sz = blk_sz + SBC_HEADER_SIZE; } - else { + else if (!(flags & CFLG_MAIN_CARRIER)) { bcrr_sz = MBC_HEADER_SIZE(allctr) + blk_sz; - if (!(flags & CFLG_MAIN_CARRIER) - && bcrr_sz < allctr->smallest_mbc_size) - bcrr_sz = allctr->smallest_mbc_size; + if (bcrr_sz < allctr->smallest_mbc_size) + bcrr_sz = allctr->smallest_mbc_size; } crr_sz = (flags & CFLG_FORCE_SIZE ? UNIT_CEILING(bcrr_sz) : SYS_ALLOC_CARRIER_CEILING(bcrr_sz)); - crr = (Carrier_t *) alcu_sys_alloc(allctr, crr_sz, flags & CFLG_MBC); + crr = (Carrier_t *) allctr->sys_alloc(allctr, crr_sz, flags & CFLG_MBC); if (!crr) { if (crr_sz > UNIT_CEILING(bcrr_sz)) { crr_sz = UNIT_CEILING(bcrr_sz); - crr = (Carrier_t *) alcu_sys_alloc(allctr, crr_sz, flags & CFLG_MBC); + crr = (Carrier_t *) allctr->sys_alloc(allctr, crr_sz, flags & CFLG_MBC); } if (!crr) { #if HAVE_ERTS_MSEG @@ -3473,7 +3965,7 @@ resize_carrier(Allctr_t *allctr, Block_t *old_blk, Uint umem_sz, UWord flags) new_crr_sz = new_blk_sz + SBC_HEADER_SIZE; new_crr_sz = ERTS_SACRR_UNIT_CEILING(new_crr_sz); - new_crr = (Carrier_t *) alcu_mseg_realloc(allctr, + new_crr = (Carrier_t *) allctr->mseg_realloc(allctr, old_crr, old_crr_sz, &new_crr_sz); @@ -3488,11 +3980,6 @@ resize_carrier(Allctr_t *allctr, Block_t *old_blk, Uint umem_sz, UWord flags) DEBUG_SAVE_ALIGNMENT(new_crr); return new_blk; } -#if HALFWORD_HEAP - /* Old carrier unchanged; restore stat */ - STAT_MSEG_SBC_ALLOC(allctr, old_crr_sz, old_blk_sz); - return NULL; -#endif create_flags |= CFLG_FORCE_SYS_ALLOC; /* since mseg_realloc() failed */ } @@ -3503,7 +3990,7 @@ resize_carrier(Allctr_t *allctr, Block_t *old_blk, Uint umem_sz, UWord flags) (void *) BLK2UMEM(old_blk), MIN(new_blk_sz, old_blk_sz) - ABLK_HDR_SZ); unlink_carrier(&allctr->sbc_list, old_crr); - alcu_mseg_dealloc(allctr, old_crr, old_crr_sz, ERTS_MSEG_FLG_NONE); + allctr->mseg_dealloc(allctr, old_crr, old_crr_sz, ERTS_MSEG_FLG_NONE); } else { /* Old carrier unchanged; restore stat */ @@ -3520,7 +4007,7 @@ resize_carrier(Allctr_t *allctr, Block_t *old_blk, Uint umem_sz, UWord flags) ? UNIT_CEILING(new_bcrr_sz) : SYS_ALLOC_CARRIER_CEILING(new_bcrr_sz)); - new_crr = (Carrier_t *) alcu_sys_realloc(allctr, + new_crr = (Carrier_t *) allctr->sys_realloc(allctr, (void *) old_crr, new_crr_sz, old_crr_sz, @@ -3541,7 +4028,7 @@ resize_carrier(Allctr_t *allctr, Block_t *old_blk, Uint umem_sz, UWord flags) else if (new_crr_sz > UNIT_CEILING(new_bcrr_sz)) { new_crr_sz = new_blk_sz + SBC_HEADER_SIZE; new_crr_sz = UNIT_CEILING(new_crr_sz); - new_crr = (Carrier_t *) alcu_sys_realloc(allctr, + new_crr = (Carrier_t *) allctr->sys_realloc(allctr, (void *) old_crr, new_crr_sz, old_crr_sz, @@ -3564,7 +4051,7 @@ resize_carrier(Allctr_t *allctr, Block_t *old_blk, Uint umem_sz, UWord flags) (void *) BLK2UMEM(old_blk), MIN(new_blk_sz, old_blk_sz) - ABLK_HDR_SZ); unlink_carrier(&allctr->sbc_list, old_crr); - alcu_sys_free(allctr, old_crr, 0); + allctr->sys_dealloc(allctr, old_crr, CARRIER_SZ(old_crr), 0); } else { /* Old carrier unchanged; restore... */ @@ -3580,13 +4067,13 @@ dealloc_carrier(Allctr_t *allctr, Carrier_t *crr, int superaligned) { #if HAVE_ERTS_MSEG if (IS_MSEG_CARRIER(crr)) - alcu_mseg_dealloc(allctr, crr, CARRIER_SZ(crr), + allctr->mseg_dealloc(allctr, crr, CARRIER_SZ(crr), (superaligned ? ERTS_MSEG_FLG_2POW : ERTS_MSEG_FLG_NONE)); else #endif - alcu_sys_free(allctr, crr, superaligned); + allctr->sys_dealloc(allctr, crr, CARRIER_SZ(crr), superaligned); } static void @@ -3633,13 +4120,15 @@ destroy_carrier(Allctr_t *allctr, Block_t *blk, Carrier_t **busy_pcrr_pp) } #endif - if (allctr->destroying_mbc) - (*allctr->destroying_mbc)(allctr, crr); - #ifdef ERTS_SMP if (busy_pcrr_pp && *busy_pcrr_pp) { ERTS_ALC_CPOOL_ASSERT(*busy_pcrr_pp == crr); *busy_pcrr_pp = NULL; + ERTS_ALC_CPOOL_ASSERT(erts_smp_atomic_read_nob(&crr->allctr) + == (((erts_aint_t) allctr) + | ERTS_CRR_ALCTR_FLG_IN_POOL + | ERTS_CRR_ALCTR_FLG_BUSY)); + erts_smp_atomic_set_nob(&crr->allctr, ((erts_aint_t) allctr)); cpool_delete(allctr, allctr, crr); } else @@ -3654,12 +4143,15 @@ destroy_carrier(Allctr_t *allctr, Block_t *blk, Carrier_t **busy_pcrr_pp) else #endif STAT_SYS_ALLOC_MBC_FREE(allctr, crr_sz); + + if (allctr->remove_mbc) + allctr->remove_mbc(allctr, crr); } #ifdef ERTS_SMP schedule_dealloc_carrier(allctr, crr); #else - dealloc_carrier(allctr, crr, 1); + dealloc_mbc(allctr, crr); #endif } } @@ -3676,9 +4168,6 @@ static struct { Eterm e; Eterm t; Eterm ramv; -#if HALFWORD_HEAP - Eterm low; -#endif Eterm sbct; #if HAVE_ERTS_MSEG Eterm asbcst; @@ -3769,9 +4258,6 @@ init_atoms(Allctr_t *allctr) AM_INIT(e); AM_INIT(t); AM_INIT(ramv); -#if HALFWORD_HEAP - AM_INIT(low); -#endif AM_INIT(sbct); #if HAVE_ERTS_MSEG AM_INIT(asbcst); @@ -4232,7 +4718,7 @@ make_name_atoms(Allctr_t *allctr) size_t prefix_len = strlen(allctr->name_prefix); if (prefix_len > MAX_ATOM_CHARACTERS + sizeof(realloc) - 1) - erl_exit(1,"Too long allocator name: %salloc\n",allctr->name_prefix); + erts_exit(ERTS_ERROR_EXIT,"Too long allocator name: %salloc\n",allctr->name_prefix); memcpy((void *) buf, (void *) allctr->name_prefix, prefix_len); @@ -4377,9 +4863,6 @@ info_options(Allctr_t *allctr, "option e: true\n" "option t: %s\n" "option ramv: %s\n" -#if HALFWORD_HEAP - "option low: %s\n" -#endif "option sbct: %beu\n" #if HAVE_ERTS_MSEG "option asbcst: %bpu\n" @@ -4398,9 +4881,6 @@ info_options(Allctr_t *allctr, "option acul: %d\n", topt, allctr->ramv ? "true" : "false", -#if HALFWORD_HEAP - allctr->mseg_opt.low_mem ? "true" : "false", -#endif allctr->sbc_threshold, #if HAVE_ERTS_MSEG allctr->mseg_opt.abs_shrink_th, @@ -4463,9 +4943,6 @@ info_options(Allctr_t *allctr, add_2tup(hpp, szp, &res, am.sbct, bld_uint(hpp, szp, allctr->sbc_threshold)); -#if HALFWORD_HEAP - add_2tup(hpp, szp, &res, am.low, allctr->mseg_opt.low_mem ? am_true : am_false); -#endif add_2tup(hpp, szp, &res, am.ramv, allctr->ramv ? am_true : am_false); add_2tup(hpp, szp, &res, am.t, (allctr->t ? am_true : am_false)); add_2tup(hpp, szp, &res, am.e, am_true); @@ -4860,6 +5337,11 @@ do_erts_alcu_alloc(ErtsAlcType_t type, void *extra, Uint size) void *erts_alcu_alloc(ErtsAlcType_t type, void *extra, Uint size) { void *res; +#ifdef ERTS_SMP + ASSERT(!"This is not thread safe"); +#elif defined(USE_THREADS) + ASSERT(erts_equal_tids(erts_main_thread, erts_thr_self())); +#endif res = do_erts_alcu_alloc(type, extra, size); DEBUG_CHECK_ALIGNMENT(res); return res; @@ -4973,7 +5455,7 @@ do_erts_alcu_free(ErtsAlcType_t type, void *extra, void *p, if (allctr->fix) { if (ERTS_ALC_IS_CPOOL_ENABLED(allctr)) - fix_cpool_free(allctr, type, p, busy_pcrr_pp); + fix_cpool_free(allctr, type, p, busy_pcrr_pp, 1); else fix_nocpool_free(allctr, type, p); } @@ -5150,11 +5632,7 @@ do_erts_alcu_realloc(ErtsAlcType_t type, Block_t *new_blk; if(IS_SBC_BLK(blk)) { do_carrier_resize: -#if HALFWORD_HEAP - new_blk = resize_carrier(allctr, blk, size, CFLG_SBC | CFLG_FORCE_MSEG); -#else new_blk = resize_carrier(allctr, blk, size, CFLG_SBC); -#endif res = new_blk ? BLK2UMEM(new_blk) : NULL; } else if (alcu_flgs & ERTS_ALCU_FLG_FAIL_REALLOC_MOVE) @@ -5434,7 +5912,7 @@ erts_alcu_start(Allctr_t *allctr, AllctrInit_t *init) /* erts_alcu_start assumes that allctr has been zeroed */ if (((UWord)allctr & ERTS_CRR_ALCTR_FLG_MASK) != 0) { - erl_exit(ERTS_ABORT_EXIT, "%s:%d:erts_alcu_start: Alignment error\n", + erts_exit(ERTS_ABORT_EXIT, "%s:%d:erts_alcu_start: Alignment error\n", __FILE__, __LINE__); } @@ -5448,11 +5926,8 @@ erts_alcu_start(Allctr_t *allctr, AllctrInit_t *init) #ifdef ERTS_SMP if (init->tspec || init->tpref) allctr->mseg_opt.sched_spec = 1; -#endif -# if HALFWORD_HEAP - allctr->mseg_opt.low_mem = init->low_mem; -# endif -#endif +#endif /* ERTS_SMP */ +#endif /* HAVE_ERTS_MSEG */ allctr->name_prefix = init->name_prefix; if (!allctr->name_prefix) @@ -5519,6 +5994,10 @@ erts_alcu_start(Allctr_t *allctr, AllctrInit_t *init) allctr->min_block_size = sz; } + allctr->cpool.pooled_list.next = &allctr->cpool.pooled_list; + allctr->cpool.pooled_list.prev = &allctr->cpool.pooled_list; + allctr->cpool.traitor_list.next = &allctr->cpool.traitor_list; + allctr->cpool.traitor_list.prev = &allctr->cpool.traitor_list; allctr->cpool.dc_list.first = NULL; allctr->cpool.dc_list.last = NULL; allctr->cpool.abandon_limit = 0; @@ -5597,24 +6076,59 @@ erts_alcu_start(Allctr_t *allctr, AllctrInit_t *init) + ABLK_HDR_SZ) - ABLK_HDR_SZ); + if (init->sys_alloc) { + ASSERT(init->sys_realloc && init->sys_dealloc); + allctr->sys_alloc = init->sys_alloc; + allctr->sys_realloc = init->sys_realloc; + allctr->sys_dealloc = init->sys_dealloc; + } + else { + ASSERT(!init->sys_realloc && !init->sys_dealloc); + allctr->sys_alloc = &erts_alcu_sys_alloc; + allctr->sys_realloc = &erts_alcu_sys_realloc; + allctr->sys_dealloc = &erts_alcu_sys_dealloc; + } +#if HAVE_ERTS_MSEG + if (init->mseg_alloc) { + ASSERT(init->mseg_realloc && init->mseg_dealloc); + allctr->mseg_alloc = init->mseg_alloc; + allctr->mseg_realloc = init->mseg_realloc; + allctr->mseg_dealloc = init->mseg_dealloc; + } + else { + ASSERT(!init->mseg_realloc && !init->mseg_dealloc); + allctr->mseg_alloc = &erts_alcu_mseg_alloc; + allctr->mseg_realloc = &erts_alcu_mseg_realloc; + allctr->mseg_dealloc = &erts_alcu_mseg_dealloc; + } + /* If a custom carrier alloc function is specified, make sure it's used */ + if (init->mseg_alloc && !init->sys_alloc) { + allctr->crr_set_flgs = CFLG_FORCE_MSEG; + allctr->crr_clr_flgs = CFLG_FORCE_SYS_ALLOC; + } + else if (!init->mseg_alloc && init->sys_alloc) { + allctr->crr_set_flgs = CFLG_FORCE_SYS_ALLOC; + allctr->crr_clr_flgs = CFLG_FORCE_MSEG; + } +#endif + if (allctr->main_carrier_size) { Block_t *blk; blk = create_carrier(allctr, allctr->main_carrier_size, - CFLG_MBC + (ERTS_SUPER_ALIGNED_MSEG_ONLY + ? CFLG_FORCE_MSEG : CFLG_FORCE_SYS_ALLOC) + | CFLG_MBC | CFLG_FORCE_SIZE | CFLG_NO_CPOOL -#if !HALFWORD_HEAP && !ERTS_SUPER_ALIGNED_MSEG_ONLY - | CFLG_FORCE_SYS_ALLOC -#endif | CFLG_MAIN_CARRIER); if (!blk) { #ifdef USE_THREADS if (allctr->thread_safe) erts_mtx_destroy(&allctr->mutex); #endif - erl_exit(ERTS_ABORT_EXIT, + erts_exit(ERTS_ABORT_EXIT, "Failed to create main carrier for %salloc\n", init->name_prefix); } @@ -5696,7 +6210,7 @@ erts_alcu_init(AlcUInit_t *init) erts_atomic_init_nob(&sentinel->prev, (erts_aint_t) sentinel); } #endif - ASSERT(SBC_BLK_SZ_MASK == MBC_FBLK_SZ_MASK); /* see BLK_SZ */ + ERTS_CT_ASSERT(SBC_BLK_SZ_MASK == MBC_FBLK_SZ_MASK); /* see BLK_SZ */ #if HAVE_ERTS_MSEG ASSERT(erts_mseg_unit_size() == ERTS_SACRR_UNIT_SZ); max_mseg_carriers = init->mmc; @@ -5759,7 +6273,7 @@ erts_alcu_test(UWord op, UWord a1, UWord a2) case 0x019: return (UWord) PREV_BLK((Block_t *) a1); case 0x01a: return (UWord) IS_MBC_FIRST_BLK((Allctr_t*)a1, (Block_t *) a2); case 0x01b: return (UWord) sizeof(Unit_t); - case 0x01c: return (unsigned long) BLK_TO_MBC((Block_t*) a1); + case 0x01c: return (UWord) BLK_TO_MBC((Block_t*) a1); case 0x01d: ((Allctr_t*) a1)->add_mbc((Allctr_t*)a1, (Carrier_t*)a2); break; case 0x01e: ((Allctr_t*) a1)->remove_mbc((Allctr_t*)a1, (Carrier_t*)a2); break; #ifdef ERTS_SMP @@ -5784,6 +6298,16 @@ erts_alcu_test(UWord op, UWord a1, UWord a2) case 0x023: return (UWord) 0; case 0x024: return (UWord) 0; #endif + case 0x025: /* UMEM2BLK_TEST*/ +#ifdef DEBUG +# ifdef HARD_DEBUG + return (UWord)UMEM2BLK(a1-3*sizeof(UWord)); +# else + return (UWord)UMEM2BLK(a1-2*sizeof(UWord)); +# endif +#else + return (UWord)UMEM2BLK(a1); +#endif default: ASSERT(0); return ~((UWord) 0); } @@ -5820,7 +6344,7 @@ erts_alcu_verify_unused(Allctr_t *allctr) if (no) { UWord sz = allctr->sbcs.blocks.curr.size; sz += allctr->mbcs.blocks.curr.size; - erl_exit(ERTS_ABORT_EXIT, + erts_exit(ERTS_ABORT_EXIT, "%salloc() used when expected to be unused!\n" "Total amount of blocks allocated: %bpu\n" "Total amount of bytes allocated: %bpu\n", diff --git a/erts/emulator/beam/erl_alloc_util.h b/erts/emulator/beam/erl_alloc_util.h index 7be6b1ed9d..b7d717ed23 100644 --- a/erts/emulator/beam/erl_alloc_util.h +++ b/erts/emulator/beam/erl_alloc_util.h @@ -3,16 +3,17 @@ * * Copyright Ericsson AB 2002-2013. All Rights Reserved. * - * The contents of this file are subject to the Erlang Public License, - * Version 1.1, (the "License"); you may not use this file except in - * compliance with the License. You should have received a copy of the - * Erlang Public License along with this software. If not, it can be - * retrieved online at http://www.erlang.org/. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Software distributed under the License is distributed on an "AS IS" - * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See - * the License for the specific language governing rights and limitations - * under the License. + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. * * %CopyrightEnd% */ @@ -23,6 +24,12 @@ #define ERTS_ALCU_VSN_STR "3.0" #include "erl_alloc_types.h" +#ifdef USE_THREADS +#define ERL_THREADS_EMU_INTERNAL__ +#include "erl_threads.h" +#endif + +#include "erl_mseg.h" #define ERTS_AU_PREF_ALLOC_BITS 11 #define ERTS_AU_MAX_PREF_ALLOC_INSTANCES (1 << ERTS_AU_PREF_ALLOC_BITS) @@ -44,7 +51,6 @@ typedef struct { int tspec; int tpref; int ramv; - int low_mem; /* HALFWORD only */ UWord sbct; UWord asbcst; UWord rsbcst; @@ -60,6 +66,15 @@ typedef struct { void *fix; size_t *fix_type_size; + +#if HAVE_ERTS_MSEG + void* (*mseg_alloc)(Allctr_t*, Uint *size_p, Uint flags); + void* (*mseg_realloc)(Allctr_t*, void *seg, Uint old_size, Uint *new_size_p); + void (*mseg_dealloc)(Allctr_t*, void *seg, Uint size, Uint flags); +#endif + void* (*sys_alloc)(Allctr_t *allctr, Uint size, int superalign); + void* (*sys_realloc)(Allctr_t *allctr, void *ptr, Uint size, Uint old_size, int superalign); + void (*sys_dealloc)(Allctr_t *allctr, void *ptr, Uint size, int superalign); } AllctrInit_t; typedef struct { @@ -89,7 +104,6 @@ typedef struct { 0, /* (bool) tspec: thread specific */\ 0, /* (bool) tpref: thread preferred */\ 0, /* (bool) ramv: realloc always moves */\ - 0, /* (bool) low_mem: HALFWORD only */\ 512*1024, /* (bytes) sbct: sbc threshold */\ 2*1024*2024, /* (amount) asbcst: abs sbc shrink threshold */\ 20, /* (%) rsbcst: rel sbc shrink threshold */\ @@ -124,7 +138,6 @@ typedef struct { 0, /* (bool) tspec: thread specific */\ 0, /* (bool) tpref: thread preferred */\ 0, /* (bool) ramv: realloc always moves */\ - 0, /* (bool) low_mem: HALFWORD only */\ 64*1024, /* (bytes) sbct: sbc threshold */\ 2*1024*2024, /* (amount) asbcst: abs sbc shrink threshold */\ 20, /* (%) rsbcst: rel sbc shrink threshold */\ @@ -175,20 +188,44 @@ void erts_alcu_check_delayed_dealloc(Allctr_t *, int, int *, ErtsThrPrgrVal * #endif erts_aint32_t erts_alcu_fix_alloc_shrink(Allctr_t *, erts_aint32_t); +#ifdef ARCH_32 +extern UWord erts_literal_vspace_map[]; +# define ERTS_VSPACE_WORD_BITS (sizeof(UWord)*8) +#endif + +void* erts_alcu_mseg_alloc(Allctr_t*, Uint *size_p, Uint flags); +void* erts_alcu_mseg_realloc(Allctr_t*, void *seg, Uint old_size, Uint *new_size_p); +void erts_alcu_mseg_dealloc(Allctr_t*, void *seg, Uint size, Uint flags); + +#if HAVE_ERTS_MSEG +# if defined(ARCH_32) +void* erts_alcu_literal_32_mseg_alloc(Allctr_t*, Uint *size_p, Uint flags); +void* erts_alcu_literal_32_mseg_realloc(Allctr_t*, void *seg, Uint old_size, Uint *new_size_p); +void erts_alcu_literal_32_mseg_dealloc(Allctr_t*, void *seg, Uint size, Uint flags); + +# elif defined(ARCH_64) && defined(ERTS_HAVE_OS_PHYSICAL_MEMORY_RESERVATION) +void* erts_alcu_literal_64_mseg_alloc(Allctr_t*, Uint *size_p, Uint flags); +void* erts_alcu_literal_64_mseg_realloc(Allctr_t*, void *seg, Uint old_size, Uint *new_size_p); +void erts_alcu_literal_64_mseg_dealloc(Allctr_t*, void *seg, Uint size, Uint flags); +# endif +#endif /* HAVE_ERTS_MSEG */ + +void* erts_alcu_sys_alloc(Allctr_t*, Uint size, int superalign); +void* erts_alcu_sys_realloc(Allctr_t*, void *ptr, Uint size, Uint old_size, int superalign); +void erts_alcu_sys_dealloc(Allctr_t*, void *ptr, Uint size, int superalign); +#ifdef ARCH_32 +void* erts_alcu_literal_32_sys_alloc(Allctr_t*, Uint size, int superalign); +void* erts_alcu_literal_32_sys_realloc(Allctr_t*, void *ptr, Uint size, Uint old_size, int superalign); +void erts_alcu_literal_32_sys_dealloc(Allctr_t*, void *ptr, Uint size, int superalign); #endif +#endif /* !ERL_ALLOC_UTIL__ */ + #if defined(GET_ERL_ALLOC_UTIL_IMPL) && !defined(ERL_ALLOC_UTIL_IMPL__) #define ERL_ALLOC_UTIL_IMPL__ #define ERTS_ALCU_FLG_FAIL_REALLOC_MOVE (((Uint32) 1) << 0) -#ifdef USE_THREADS -#define ERL_THREADS_EMU_INTERNAL__ -#include "erl_threads.h" -#endif - -#include "erl_mseg.h" - #undef ERTS_ALLOC_UTIL_HARD_DEBUG #ifdef DEBUG # if 0 @@ -222,7 +259,7 @@ erts_aint32_t erts_alcu_fix_alloc_shrink(Allctr_t *, erts_aint32_t); #if ERTS_HAVE_MSEG_SUPER_ALIGNED \ || (!HAVE_ERTS_MSEG && ERTS_HAVE_ERTS_SYS_ALIGNED_ALLOC) -# ifndef MSEG_ALIGN_BITS +# ifdef MSEG_ALIGN_BITS # define ERTS_SUPER_ALIGN_BITS MSEG_ALIGN_BITS # else # define ERTS_SUPER_ALIGN_BITS 18 @@ -268,15 +305,21 @@ typedef union {char c[ERTS_ALLOC_ALIGN_BYTES]; long l; double d;} Unit_t; #ifdef ERTS_SMP +typedef struct ErtsDoubleLink_t_ { + struct ErtsDoubleLink_t_ *next; + struct ErtsDoubleLink_t_ *prev; +}ErtsDoubleLink_t; + typedef struct { erts_atomic_t next; erts_atomic_t prev; - Allctr_t *orig_allctr; + Allctr_t *orig_allctr; /* read-only while carrier is alive */ ErtsThrPrgrVal thr_prgr; erts_atomic_t max_size; UWord abandon_limit; UWord blocks; UWord blocks_size; + ErtsDoubleLink_t abandoned; /* node in pooled_list or traitor_list */ } ErtsAlcCPoolData_t; #endif @@ -494,13 +537,20 @@ struct Allctr_t_ { Uint min_mbc_size; Uint min_mbc_first_free_size; Uint min_block_size; + UWord crr_set_flgs; + UWord crr_clr_flgs; /* Carriers */ CarrierList_t mbc_list; CarrierList_t sbc_list; #ifdef ERTS_SMP struct { - CarrierList_t dc_list; + /* pooled_list, traitor list and dc_list contain only + carriers _created_ by this allocator */ + ErtsDoubleLink_t pooled_list; + ErtsDoubleLink_t traitor_list; + CarrierList_t dc_list; + UWord abandon_limit; int disable_abandon; int check_limit_count; @@ -534,6 +584,15 @@ struct Allctr_t_ { void (*remove_mbc) (Allctr_t *, Carrier_t *); UWord (*largest_fblk_in_mbc) (Allctr_t *, Carrier_t *); +#if HAVE_ERTS_MSEG + void* (*mseg_alloc)(Allctr_t*, Uint *size_p, Uint flags); + void* (*mseg_realloc)(Allctr_t*, void *seg, Uint old_size, Uint *new_size_p); + void (*mseg_dealloc)(Allctr_t*, void *seg, Uint size, Uint flags); +#endif + void* (*sys_alloc)(Allctr_t *allctr, Uint size, int superalign); + void* (*sys_realloc)(Allctr_t *allctr, void *ptr, Uint size, Uint old_size, int superalign); + void (*sys_dealloc)(Allctr_t *allctr, void *ptr, Uint size, int superalign); + void (*init_atoms) (void); #ifdef ERTS_ALLOC_UTIL_HARD_DEBUG diff --git a/erts/emulator/beam/erl_ao_firstfit_alloc.c b/erts/emulator/beam/erl_ao_firstfit_alloc.c index 396aa88e0b..5cd067ff54 100644 --- a/erts/emulator/beam/erl_ao_firstfit_alloc.c +++ b/erts/emulator/beam/erl_ao_firstfit_alloc.c @@ -3,16 +3,17 @@ * * Copyright Ericsson AB 2003-2013. All Rights Reserved. * - * The contents of this file are subject to the Erlang Public License, - * Version 1.1, (the "License"); you may not use this file except in - * compliance with the License. You should have received a copy of the - * Erlang Public License along with this software. If not, it can be - * retrieved online at http://www.erlang.org/. - * - * Software distributed under the License is distributed on an "AS IS" - * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See - * the License for the specific language governing rights and limitations - * under the License. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. * * %CopyrightEnd% */ @@ -208,7 +209,9 @@ static Block_t* aoff_get_free_block(Allctr_t *, Uint, Block_t *, Uint); static void aoff_link_free_block(Allctr_t *, Block_t*); static void aoff_unlink_free_block(Allctr_t *allctr, Block_t *del); static void aoff_creating_mbc(Allctr_t*, Carrier_t*); +#ifdef DEBUG static void aoff_destroying_mbc(Allctr_t*, Carrier_t*); +#endif static void aoff_add_mbc(Allctr_t*, Carrier_t*); static void aoff_remove_mbc(Allctr_t*, Carrier_t*); static UWord aoff_largest_fblk_in_mbc(Allctr_t*, Carrier_t*); @@ -270,7 +273,11 @@ erts_aoffalc_start(AOFFAllctr_t *alc, allctr->get_next_mbc_size = NULL; allctr->creating_mbc = aoff_creating_mbc; +#ifdef DEBUG allctr->destroying_mbc = aoff_destroying_mbc; +#else + allctr->destroying_mbc = NULL; +#endif allctr->add_mbc = aoff_add_mbc; allctr->remove_mbc = aoff_remove_mbc; allctr->largest_fblk_in_mbc = aoff_largest_fblk_in_mbc; @@ -884,17 +891,18 @@ static void aoff_creating_mbc(Allctr_t *allctr, Carrier_t *carrier) HARD_CHECK_TREE(NULL, 0, *root, 0); } +#define IS_CRR_IN_TREE(CRR,ROOT) \ + ((CRR)->rbt_node.parent || (ROOT) == &(CRR)->rbt_node) + +#ifdef DEBUG static void aoff_destroying_mbc(Allctr_t *allctr, Carrier_t *carrier) { AOFFAllctr_t *alc = (AOFFAllctr_t *) allctr; AOFF_Carrier_t *crr = (AOFF_Carrier_t*) carrier; - AOFF_RBTree_t *root = alc->mbc_root; - if (crr->rbt_node.parent || &crr->rbt_node == root) { - aoff_remove_mbc(allctr, carrier); - } - /*else already removed */ + ASSERT(!IS_CRR_IN_TREE(crr, alc->mbc_root)); } +#endif static void aoff_add_mbc(Allctr_t *allctr, Carrier_t *carrier) { @@ -902,6 +910,7 @@ static void aoff_add_mbc(Allctr_t *allctr, Carrier_t *carrier) AOFF_Carrier_t *crr = (AOFF_Carrier_t*) carrier; AOFF_RBTree_t **root = &alc->mbc_root; + ASSERT(!IS_CRR_IN_TREE(crr, *root)); HARD_CHECK_TREE(NULL, 0, *root, 0); /* Link carrier in address order tree @@ -918,6 +927,10 @@ static void aoff_remove_mbc(Allctr_t *allctr, Carrier_t *carrier) AOFF_RBTree_t **root = &alc->mbc_root; ASSERT(allctr == ERTS_ALC_CARRIER_TO_ALLCTR(carrier)); + + if (!IS_CRR_IN_TREE(crr,*root)) + return; + HARD_CHECK_TREE(NULL, 0, *root, 0); rbt_delete(root, &crr->rbt_node); @@ -1022,7 +1035,7 @@ info_options(Allctr_t *allctr, if (hpp || szp) { if (!atoms_initialized) - erl_exit(1, "%s:%d: Internal error: Atoms not initialized", + erts_exit(ERTS_ERROR_EXIT, "%s:%d: Internal error: Atoms not initialized", __FILE__, __LINE__);; res = NIL; diff --git a/erts/emulator/beam/erl_ao_firstfit_alloc.h b/erts/emulator/beam/erl_ao_firstfit_alloc.h index 25b344c6a8..4200f20622 100644 --- a/erts/emulator/beam/erl_ao_firstfit_alloc.h +++ b/erts/emulator/beam/erl_ao_firstfit_alloc.h @@ -3,16 +3,17 @@ * * Copyright Ericsson AB 2003-2013. All Rights Reserved. * - * The contents of this file are subject to the Erlang Public License, - * Version 1.1, (the "License"); you may not use this file except in - * compliance with the License. You should have received a copy of the - * Erlang Public License along with this software. If not, it can be - * retrieved online at http://www.erlang.org/. - * - * Software distributed under the License is distributed on an "AS IS" - * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See - * the License for the specific language governing rights and limitations - * under the License. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. * * %CopyrightEnd% */ diff --git a/erts/emulator/beam/erl_arith.c b/erts/emulator/beam/erl_arith.c index 5150a8a507..3671025d22 100644 --- a/erts/emulator/beam/erl_arith.c +++ b/erts/emulator/beam/erl_arith.c @@ -3,16 +3,17 @@ * * Copyright Ericsson AB 1999-2010. All Rights Reserved. * - * The contents of this file are subject to the Erlang Public License, - * Version 1.1, (the "License"); you may not use this file except in - * compliance with the License. You should have received a copy of the - * Erlang Public License along with this software. If not, it can be - * retrieved online at http://www.erlang.org/. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Software distributed under the License is distributed on an "AS IS" - * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See - * the License for the specific language governing rights and limitations - * under the License. + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. * * %CopyrightEnd% */ @@ -41,15 +42,8 @@ # define MAX(x, y) (((x) > (y)) ? (x) : (y)) #endif -#if !HEAP_ON_C_STACK -# define DECLARE_TMP(VariableName,N,P) \ - Eterm *VariableName = ((ERTS_PROC_GET_SCHDATA(P)->erl_arith_tmp_heap) + (2 * N)) -#else -# define DECLARE_TMP(VariableName,N,P) \ - Eterm VariableName[2] -#endif -# define ARG_IS_NOT_TMP(Arg,Tmp) ((Arg) != make_big((Tmp))) - +#define DECLARE_TMP(VariableName,N,P) Eterm VariableName[2] +#define ARG_IS_NOT_TMP(Arg,Tmp) ((Arg) != make_big((Tmp))) static Eterm shift(Process* p, Eterm arg1, Eterm arg2, int right); @@ -2048,3 +2042,8 @@ Eterm erts_gc_bnot(Process* p, Eterm* reg, Uint live) } return result; } + +/* Needed to remove compiler optimization */ +double erts_get_positive_zero_float() { + return 0.0f; +} diff --git a/erts/emulator/beam/erl_async.c b/erts/emulator/beam/erl_async.c index decae6b2ca..cdeeb5281b 100644 --- a/erts/emulator/beam/erl_async.c +++ b/erts/emulator/beam/erl_async.c @@ -3,16 +3,17 @@ * * Copyright Ericsson AB 2000-2013. All Rights Reserved. * - * The contents of this file are subject to the Erlang Public License, - * Version 1.1, (the "License"); you may not use this file except in - * compliance with the License. You should have received a copy of the - * Erlang Public License along with this software. If not, it can be - * retrieved online at http://www.erlang.org/. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Software distributed under the License is distributed on an "AS IS" - * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See - * the License for the specific language governing rights and limitations - * under the License. + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. * * %CopyrightEnd% */ @@ -166,7 +167,6 @@ async_ready_q(Uint sched_id) #endif - void erts_init_async(void) { @@ -176,7 +176,7 @@ erts_init_async(void) ErtsThrQInit_t qinit = ERTS_THR_Q_INIT_DEFAULT; #endif erts_thr_opts_t thr_opts = ERTS_THR_OPTS_DEFAULT_INITER; - char *ptr; + char *ptr, thr_name[16]; size_t tot_size = 0; int i; @@ -227,23 +227,16 @@ erts_init_async(void) thr_opts.suggested_stack_size = erts_async_thread_suggested_stack_size; -#ifdef ETHR_HAVE_THREAD_NAMES - thr_opts.name = malloc(sizeof(char)*(strlen("async_XXXX")+1)); -#endif + thr_opts.name = thr_name; for (i = 0; i < erts_async_max_threads; i++) { ErtsAsyncQ *aq = async_q(i); -#ifdef ETHR_HAVE_THREAD_NAMES - sprintf(thr_opts.name, "async_%d", i+1); -#endif + erts_snprintf(thr_opts.name, 16, "async_%d", i+1); erts_thr_create(&aq->thr_id, async_main, (void*) aq, &thr_opts); } -#ifdef ETHR_HAVE_THREAD_NAMES - free(thr_opts.name); -#endif /* Wait for async threads to initialize... */ erts_mtx_lock(&async->init.data.mtx); @@ -407,13 +400,19 @@ static ERTS_INLINE void call_async_ready(ErtsAsync *a) ERTS_PORT_SFLGS_INVALID_DRIVER_LOOKUP); #endif if (!p) { - if (a->async_free) + if (a->async_free) { + ERTS_MSACC_PUSH_AND_SET_STATE(ERTS_MSACC_STATE_PORT); a->async_free(a->async_data); + ERTS_MSACC_POP_STATE(); + } } else { if (async_ready(p, a->async_data)) { - if (a->async_free) + if (a->async_free) { + ERTS_MSACC_PUSH_AND_SET_STATE(ERTS_MSACC_STATE_PORT); a->async_free(a->async_data); + ERTS_MSACC_POP_STATE(); + } } #if ERTS_USE_ASYNC_READY_Q erts_port_release(p); @@ -467,6 +466,8 @@ static erts_tse_t *async_thread_init(ErtsAsyncQ *aq) { ErtsThrQInit_t qinit = ERTS_THR_Q_INIT_DEFAULT; erts_tse_t *tse = erts_tse_fetch(); + ERTS_DECLARE_DUMMY(Uint no); + #ifdef ERTS_SMP ErtsThrPrgrCallbacks callbacks; @@ -490,10 +491,12 @@ static erts_tse_t *async_thread_init(ErtsAsyncQ *aq) /* Inform main thread that we are done initializing... */ erts_mtx_lock(&async->init.data.mtx); - async->init.data.no_initialized++; + no = async->init.data.no_initialized++; erts_cnd_signal(&async->init.data.cnd); erts_mtx_unlock(&async->init.data.mtx); + erts_msacc_init_thread("async", no, 0); + return tse; } @@ -501,6 +504,7 @@ static void *async_main(void* arg) { ErtsAsyncQ *aq = (ErtsAsyncQ *) arg; erts_tse_t *tse = async_thread_init(aq); + ERTS_MSACC_DECLARE_CACHE(); while (1) { ErtsThrQPrepEnQ_t *prep_enq; @@ -508,11 +512,14 @@ static void *async_main(void* arg) if (is_nil(a->port)) break; /* Time to die */ + ERTS_MSACC_UPDATE_CACHE(); + #if ERTS_ASYNC_PRINT_JOB erts_fprintf(stderr, "<- %ld\n", a->async_id); #endif - + ERTS_MSACC_SET_STATE_CACHED(ERTS_MSACC_STATE_PORT); a->async_invoke(a->async_data); + ERTS_MSACC_SET_STATE_CACHED(ERTS_MSACC_STATE_OTHER); async_reply(a, prep_enq); } @@ -635,10 +642,13 @@ long driver_async(ErlDrvPort ix, unsigned int* key, unsigned int qix; #if ERTS_USE_ASYNC_READY_Q Uint sched_id; + ERTS_MSACC_PUSH_STATE(); sched_id = erts_get_scheduler_id(); if (!sched_id) sched_id = 1; +#else + ERTS_MSACC_PUSH_STATE(); #endif prt = erts_drvport2port(ix); @@ -691,12 +701,17 @@ long driver_async(ErlDrvPort ix, unsigned int* key, return id; } #endif - + + ERTS_MSACC_SET_STATE_CACHED(ERTS_MSACC_STATE_PORT); (*a->async_invoke)(a->async_data); + ERTS_MSACC_POP_STATE(); if (async_ready(prt, a->async_data)) { - if (a->async_free != NULL) + if (a->async_free != NULL) { + ERTS_MSACC_SET_STATE_CACHED(ERTS_MSACC_STATE_PORT); (*a->async_free)(a->async_data); + ERTS_MSACC_POP_STATE(); + } } erts_free(ERTS_ALC_T_ASYNC, (void *) a); diff --git a/erts/emulator/beam/erl_async.h b/erts/emulator/beam/erl_async.h index 95374a8fc9..65538bcef0 100644 --- a/erts/emulator/beam/erl_async.h +++ b/erts/emulator/beam/erl_async.h @@ -3,16 +3,17 @@ * * Copyright Ericsson AB 2011. All Rights Reserved. * - * The contents of this file are subject to the Erlang Public License, - * Version 1.1, (the "License"); you may not use this file except in - * compliance with the License. You should have received a copy of the - * Erlang Public License along with this software. If not, it can be - * retrieved online at http://www.erlang.org/. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Software distributed under the License is distributed on an "AS IS" - * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See - * the License for the specific language governing rights and limitations - * under the License. + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. * * %CopyrightEnd% */ diff --git a/erts/emulator/beam/erl_bestfit_alloc.c b/erts/emulator/beam/erl_bestfit_alloc.c index 59c14899a2..f39a18ac88 100644 --- a/erts/emulator/beam/erl_bestfit_alloc.c +++ b/erts/emulator/beam/erl_bestfit_alloc.c @@ -3,16 +3,17 @@ * * Copyright Ericsson AB 2003-2013. All Rights Reserved. * - * The contents of this file are subject to the Erlang Public License, - * Version 1.1, (the "License"); you may not use this file except in - * compliance with the License. You should have received a copy of the - * Erlang Public License along with this software. If not, it can be - * retrieved online at http://www.erlang.org/. - * - * Software distributed under the License is distributed on an "AS IS" - * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See - * the License for the specific language governing rights and limitations - * under the License. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. * * %CopyrightEnd% */ @@ -939,7 +940,7 @@ info_options(Allctr_t *allctr, if (hpp || szp) { if (!atoms_initialized) - erl_exit(1, "%s:%d: Internal error: Atoms not initialized", + erts_exit(ERTS_ERROR_EXIT, "%s:%d: Internal error: Atoms not initialized", __FILE__, __LINE__);; res = NIL; diff --git a/erts/emulator/beam/erl_bestfit_alloc.h b/erts/emulator/beam/erl_bestfit_alloc.h index 870439e886..b315518b88 100644 --- a/erts/emulator/beam/erl_bestfit_alloc.h +++ b/erts/emulator/beam/erl_bestfit_alloc.h @@ -3,16 +3,17 @@ * * Copyright Ericsson AB 2003-2013. All Rights Reserved. * - * The contents of this file are subject to the Erlang Public License, - * Version 1.1, (the "License"); you may not use this file except in - * compliance with the License. You should have received a copy of the - * Erlang Public License along with this software. If not, it can be - * retrieved online at http://www.erlang.org/. - * - * Software distributed under the License is distributed on an "AS IS" - * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See - * the License for the specific language governing rights and limitations - * under the License. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. * * %CopyrightEnd% */ diff --git a/erts/emulator/beam/erl_bif_binary.c b/erts/emulator/beam/erl_bif_binary.c index ff775691b3..aec72bd61a 100644 --- a/erts/emulator/beam/erl_bif_binary.c +++ b/erts/emulator/beam/erl_bif_binary.c @@ -3,16 +3,17 @@ * * Copyright Ericsson AB 2010-2013. All Rights Reserved. * - * The contents of this file are subject to the Erlang Public License, - * Version 1.1, (the "License"); you may not use this file except in - * compliance with the License. You should have received a copy of the - * Erlang Public License along with this software. If not, it can be - * retrieved online at http://www.erlang.org/. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Software distributed under the License is distributed on an "AS IS" - * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See - * the License for the specific language governing rights and limitations - * under the License. + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. * * %CopyrightEnd% */ @@ -32,10 +33,13 @@ #include "global.h" #include "erl_process.h" #include "error.h" +#define ERL_WANT_HIPE_BIF_WRAPPER__ #include "bif.h" +#undef ERL_WANT_HIPE_BIF_WRAPPER__ #include "big.h" #include "erl_binary.h" #include "erl_bits.h" +#include "erl_bif_unique.h" /* @@ -51,10 +55,8 @@ /* Init and local variables */ -static Export binary_match_trap_export; -static BIF_RETTYPE binary_match_trap(BIF_ALIST_3); -static Export binary_matches_trap_export; -static BIF_RETTYPE binary_matches_trap(BIF_ALIST_3); +static Export binary_find_trap_export; +static BIF_RETTYPE binary_find_trap(BIF_ALIST_3); static Export binary_longest_prefix_trap_export; static BIF_RETTYPE binary_longest_prefix_trap(BIF_ALIST_3); static Export binary_longest_suffix_trap_export; @@ -66,19 +68,15 @@ static BIF_RETTYPE binary_copy_trap(BIF_ALIST_2); static Uint max_loop_limit; static BIF_RETTYPE -binary_match(Process *p, Eterm arg1, Eterm arg2, Eterm arg3); +binary_match(Process *p, Eterm arg1, Eterm arg2, Eterm arg3, Uint flags); static BIF_RETTYPE -binary_matches(Process *p, Eterm arg1, Eterm arg2, Eterm arg3); +binary_split(Process *p, Eterm arg1, Eterm arg2, Eterm arg3); void erts_init_bif_binary(void) { - erts_init_trap_export(&binary_match_trap_export, - am_erlang, am_binary_match_trap, 3, - &binary_match_trap); - - erts_init_trap_export(&binary_matches_trap_export, - am_erlang, am_binary_matches_trap, 3, - &binary_matches_trap); + erts_init_trap_export(&binary_find_trap_export, + am_erlang, am_binary_find_trap, 3, + &binary_find_trap); erts_init_trap_export(&binary_longest_prefix_trap_export, am_erlang, am_binary_longest_prefix_trap, 3, @@ -310,8 +308,8 @@ static BMData *create_bmdata(MyAllocator *my, byte *x, Uint len, /* * Aho Corasick - Build a Trie and fill in the failure functions * when all strings are added. - * The algorithm is nicely described by Dieter B�hler of University of - * T�bingen: + * The algorithm is nicely described by Dieter Bühler of University of + * Tübingen: * http://www-sr.informatik.uni-tuebingen.de/~buehler/AC/AC.html */ @@ -533,21 +531,23 @@ static void ac_init_find_all(ACFindAllState *state, ACTrie *act, Sint startpos, state->out = NULL; } -static void ac_restore_find_all(ACFindAllState *state, char *buff) +static void ac_restore_find_all(ACFindAllState *state, + const ACFindAllState *src) { - memcpy(state,buff,sizeof(ACFindAllState)); + memcpy(state, src, sizeof(ACFindAllState)); if (state->allocated > 0) { state->out = erts_alloc(ERTS_ALC_T_TMP, sizeof(FindallData) * (state->allocated)); - memcpy(state->out,buff+sizeof(ACFindAllState),sizeof(FindallData)*state->m); + memcpy(state->out, src+1, sizeof(FindallData)*state->m); } else { state->out = NULL; } } -static void ac_serialize_find_all(ACFindAllState *state, char *buff) +static void ac_serialize_find_all(const ACFindAllState *state, + ACFindAllState *dst) { - memcpy(buff,state,sizeof(ACFindAllState)); - memcpy(buff+sizeof(ACFindAllState),state->out,sizeof(FindallData)*state->m); + memcpy(dst, state, sizeof(ACFindAllState)); + memcpy(dst+1, state->out, sizeof(FindallData)*state->m); } static void ac_clean_find_all(ACFindAllState *state) @@ -561,9 +561,6 @@ static void ac_clean_find_all(ACFindAllState *state) #endif } -#define SIZEOF_AC_SERIALIZED_FIND_ALL_STATE(S) \ - (sizeof(ACFindAllState)+(sizeof(FindallData)*(S).m)) - /* * Differs to the find_first function in that it stores all matches and the values * arte returned only in the state. @@ -810,24 +807,24 @@ static void bm_init_find_all(BMFindAllState *state, Sint startpos, Uint len) state->out = NULL; } -static void bm_restore_find_all(BMFindAllState *state, char *buff) +static void bm_restore_find_all(BMFindAllState *state, + const BMFindAllState *src) { - memcpy(state,buff,sizeof(BMFindAllState)); + memcpy(state, src, sizeof(BMFindAllState)); if (state->allocated > 0) { state->out = erts_alloc(ERTS_ALC_T_TMP, sizeof(FindallData) * (state->allocated)); - memcpy(state->out,buff+sizeof(BMFindAllState), - sizeof(FindallData)*state->m); + memcpy(state->out, src+1, sizeof(FindallData)*state->m); } else { state->out = NULL; } } -static void bm_serialize_find_all(BMFindAllState *state, char *buff) +static void bm_serialize_find_all(const BMFindAllState *state, + BMFindAllState *dst) { - memcpy(buff,state,sizeof(BMFindAllState)); - memcpy(buff+sizeof(BMFindAllState),state->out, - sizeof(FindallData)*state->m); + memcpy(dst, state, sizeof(BMFindAllState)); + memcpy(dst+1, state->out, sizeof(FindallData)*state->m); } static void bm_clean_find_all(BMFindAllState *state) @@ -841,9 +838,6 @@ static void bm_clean_find_all(BMFindAllState *state) #endif } -#define SIZEOF_BM_SERIALIZED_FIND_ALL_STATE(S) \ - (sizeof(BMFindAllState)+(sizeof(FindallData)*(S).m)) - /* * Differs to the find_first function in that it stores all matches and the * values are returned only in the state. @@ -1026,283 +1020,142 @@ BIF_RETTYPE binary_compile_pattern_1(BIF_ALIST_1) #define DO_BIN_MATCH_BADARG -1 #define DO_BIN_MATCH_RESTART -2 -static int do_binary_match(Process *p, Eterm subject, Uint hsstart, Uint hsend, - Eterm type, Binary *bin, Eterm state_term, - Eterm *res_term) -{ - byte *bytes; - Uint bitoffs, bitsize; - byte *temp_alloc = NULL; - - ERTS_GET_BINARY_BYTES(subject, bytes, bitoffs, bitsize); - if (bitsize != 0) { - goto badarg; - } - if (bitoffs != 0) { - bytes = erts_get_aligned_binary_bytes(subject, &temp_alloc); - } - if (state_term != NIL) { - Eterm *ptr = big_val(state_term); - type = ptr[1]; - } - - if (type == am_bm) { - BMData *bm; - Sint pos; - Eterm ret; - Eterm *hp; - BMFindFirstState state; - Uint reds = get_reds(p, BM_LOOP_FACTOR); - Uint save_reds = reds; - - bm = (BMData *) ERTS_MAGIC_BIN_DATA(bin); -#ifdef HARDDEBUG - dump_bm_data(bm); -#endif - if (state_term == NIL) { - bm_init_find_first_match(&state, hsstart, hsend); - } else { - Eterm *ptr = big_val(state_term); - memcpy(&state,ptr+2,sizeof(state)); - } -#ifdef HARDDEBUG - erts_printf("(bm) state->pos = %ld, state->len = %lu\n",state.pos, - state.len); -#endif - pos = bm_find_first_match(&state, bm, bytes, &reds); - if (pos == BM_NOT_FOUND) { - ret = am_nomatch; - } else if (pos == BM_RESTART) { - int x = (sizeof(BMFindFirstState) / sizeof(Eterm)) + - !!(sizeof(BMFindFirstState) % sizeof(Eterm)); -#ifdef HARDDEBUG - erts_printf("Trap bm!\n"); -#endif - hp = HAlloc(p,x+2); - hp[0] = make_pos_bignum_header(x+1); - hp[1] = type; - memcpy(hp+2,&state,sizeof(state)); - *res_term = make_big(hp); - erts_free_aligned_binary_bytes(temp_alloc); - return DO_BIN_MATCH_RESTART; - } else { - Eterm erlen = erts_make_integer((Uint) bm->len, p); - ret = erts_make_integer(pos,p); - hp = HAlloc(p,3); - ret = TUPLE2(hp, ret, erlen); - } - erts_free_aligned_binary_bytes(temp_alloc); - BUMP_REDS(p, (save_reds - reds) / BM_LOOP_FACTOR); - *res_term = ret; - return DO_BIN_MATCH_OK; - } else if (type == am_ac) { - ACTrie *act; - Uint pos, rlen; - int acr; - ACFindFirstState state; - Eterm ret; - Eterm *hp; - Uint reds = get_reds(p, AC_LOOP_FACTOR); - Uint save_reds = reds; +#define BINARY_FIND_ALL 0x01 +#define BINARY_SPLIT_TRIM 0x02 +#define BINARY_SPLIT_TRIM_ALL 0x04 - act = (ACTrie *) ERTS_MAGIC_BIN_DATA(bin); -#ifdef HARDDEBUG - dump_ac_trie(act); -#endif - if (state_term == NIL) { - ac_init_find_first_match(&state, act, hsstart, hsend); - } else { - Eterm *ptr = big_val(state_term); - memcpy(&state,ptr+2,sizeof(state)); - } - acr = ac_find_first_match(&state, bytes, &pos, &rlen, &reds); - if (acr == AC_NOT_FOUND) { - ret = am_nomatch; - } else if (acr == AC_RESTART) { - int x = (sizeof(state) / sizeof(Eterm)) + - !!(sizeof(ACFindFirstState) % sizeof(Eterm)); -#ifdef HARDDEBUG - erts_printf("Trap ac!\n"); -#endif - hp = HAlloc(p,x+2); - hp[0] = make_pos_bignum_header(x+1); - hp[1] = type; - memcpy(hp+2,&state,sizeof(state)); - *res_term = make_big(hp); - erts_free_aligned_binary_bytes(temp_alloc); - return DO_BIN_MATCH_RESTART; - } else { - Eterm epos = erts_make_integer(pos,p); - Eterm erlen = erts_make_integer(rlen,p); - hp = HAlloc(p,3); - ret = TUPLE2(hp, epos, erlen); - } - erts_free_aligned_binary_bytes(temp_alloc); - BUMP_REDS(p, (save_reds - reds) / AC_LOOP_FACTOR); - *res_term = ret; - return DO_BIN_MATCH_OK; - } - badarg: - return DO_BIN_MATCH_BADARG; -} +typedef struct BinaryFindState { + Eterm type; + Uint flags; + Uint hsstart; + Uint hsend; + Eterm (*not_found_result) (Process *, Eterm, struct BinaryFindState *); + Eterm (*single_result) (Process *, Eterm, struct BinaryFindState *, Sint, Sint); + Eterm (*global_result) (Process *, Eterm, struct BinaryFindState *, FindallData *, Uint); +} BinaryFindState; + +typedef struct BinaryFindState_bignum { + Eterm bignum_hdr; + BinaryFindState bfs; + union { + BMFindFirstState bmffs; + BMFindAllState bmfas; + ACFindFirstState acffs; + ACFindAllState acfas; + } data; +} BinaryFindState_bignum; + +#define SIZEOF_BINARY_FIND_STATE(S) \ + (sizeof(BinaryFindState)+sizeof(S)) + +#define SIZEOF_BINARY_FIND_ALL_STATE(S) \ + (sizeof(BinaryFindState)+sizeof(S)+(sizeof(FindallData)*(S).m)) + +static Eterm do_match_not_found_result(Process *p, Eterm subject, BinaryFindState *bfs); +static Eterm do_match_single_result(Process *p, Eterm subject, BinaryFindState *bfs, + Sint pos, Sint len); +static Eterm do_match_global_result(Process *p, Eterm subject, BinaryFindState *bfs, + FindallData *fad, Uint fad_sz); +static Eterm do_split_not_found_result(Process *p, Eterm subject, BinaryFindState *bfs); +static Eterm do_split_single_result(Process *p, Eterm subject, BinaryFindState *bfs, + Sint pos, Sint len); +static Eterm do_split_global_result(Process *p, Eterm subject, BinaryFindState *bfs, + FindallData *fad, Uint fad_sz); -static int do_binary_matches(Process *p, Eterm subject, Uint hsstart, - Uint hsend, Eterm type, Binary *bin, - Eterm state_term, Eterm *res_term) +static int parse_match_opts_list(Eterm l, Eterm bin, Uint *posp, Uint *endp) { - byte *bytes; - Uint bitoffs, bitsize; - byte *temp_alloc = NULL; - - ERTS_GET_BINARY_BYTES(subject, bytes, bitoffs, bitsize); - if (bitsize != 0) { - goto badarg; - } - if (bitoffs != 0) { - bytes = erts_get_aligned_binary_bytes(subject, &temp_alloc); - } - if (state_term != NIL) { - Eterm *ptr = big_val(state_term); - type = ptr[1]; - } - - if (type == am_bm) { - BMData *bm; - Sint pos; - Eterm ret,tpl; - Eterm *hp; - BMFindAllState state; - Uint reds = get_reds(p, BM_LOOP_FACTOR); - Uint save_reds = reds; - - bm = (BMData *) ERTS_MAGIC_BIN_DATA(bin); -#ifdef HARDDEBUG - dump_bm_data(bm); -#endif - if (state_term == NIL) { - bm_init_find_all(&state, hsstart, hsend); - } else { - Eterm *ptr = big_val(state_term); - bm_restore_find_all(&state,(char *) (ptr+2)); - } - - pos = bm_find_all_non_overlapping(&state, bm, bytes, &reds); - if (pos == BM_NOT_FOUND) { - ret = NIL; - } else if (pos == BM_RESTART) { - int x = - (SIZEOF_BM_SERIALIZED_FIND_ALL_STATE(state) / sizeof(Eterm)) + - !!(SIZEOF_BM_SERIALIZED_FIND_ALL_STATE(state) % sizeof(Eterm)); -#ifdef HARDDEBUG - erts_printf("Trap bm!\n"); -#endif - hp = HAlloc(p,x+2); - hp[0] = make_pos_bignum_header(x+1); - hp[1] = type; - bm_serialize_find_all(&state, (char *) (hp+2)); - *res_term = make_big(hp); - erts_free_aligned_binary_bytes(temp_alloc); - bm_clean_find_all(&state); - return DO_BIN_MATCH_RESTART; - } else { - FindallData *fad = state.out; - int i; - for (i = 0; i < state.m; ++i) { - fad[i].epos = erts_make_integer(fad[i].pos,p); - fad[i].elen = erts_make_integer(fad[i].len,p); + Eterm *tp; + Uint pos; + Sint len; + if (l == THE_NON_VALUE || l == NIL) { + /* Invalid term or NIL, we're called from binary_match(es)_2 or + have no options*/ + *posp = 0; + *endp = binary_size(bin); + return 0; + } else if (is_list(l)) { + while(is_list(l)) { + Eterm t = CAR(list_val(l)); + Uint orig_size; + if (!is_tuple(t)) { + goto badarg; } - hp = HAlloc(p,state.m * (3 + 2)); - ret = NIL; - for (i = state.m - 1; i >= 0; --i) { - tpl = TUPLE2(hp, fad[i].epos, fad[i].elen); - hp +=3; - ret = CONS(hp,tpl,ret); - hp += 2; + tp = tuple_val(t); + if (arityval(*tp) != 2) { + goto badarg; } - } - erts_free_aligned_binary_bytes(temp_alloc); - bm_clean_find_all(&state); - BUMP_REDS(p, (save_reds - reds) / BM_LOOP_FACTOR); - *res_term = ret; - return DO_BIN_MATCH_OK; - } else if (type == am_ac) { - ACTrie *act; - int acr; - ACFindAllState state; - Eterm ret,tpl; - Eterm *hp; - Uint reds = get_reds(p, AC_LOOP_FACTOR); - Uint save_reds = reds; - - act = (ACTrie *) ERTS_MAGIC_BIN_DATA(bin); -#ifdef HARDDEBUG - dump_ac_trie(act); -#endif - if (state_term == NIL) { - ac_init_find_all(&state, act, hsstart, hsend); - } else { - Eterm *ptr = big_val(state_term); - ac_restore_find_all(&state,(char *) (ptr+2)); - } - acr = ac_find_all_non_overlapping(&state, bytes, &reds); - if (acr == AC_NOT_FOUND) { - ret = NIL; - } else if (acr == AC_RESTART) { - int x = - (SIZEOF_AC_SERIALIZED_FIND_ALL_STATE(state) / sizeof(Eterm)) + - !!(SIZEOF_AC_SERIALIZED_FIND_ALL_STATE(state) % sizeof(Eterm)); -#ifdef HARDDEBUG - erts_printf("Trap ac!\n"); -#endif - hp = HAlloc(p,x+2); - hp[0] = make_pos_bignum_header(x+1); - hp[1] = type; - ac_serialize_find_all(&state, (char *) (hp+2)); - *res_term = make_big(hp); - erts_free_aligned_binary_bytes(temp_alloc); - ac_clean_find_all(&state); - return DO_BIN_MATCH_RESTART; - } else { - FindallData *fad = state.out; - int i; - for (i = 0; i < state.m; ++i) { - fad[i].epos = erts_make_integer(fad[i].pos,p); - fad[i].elen = erts_make_integer(fad[i].len,p); + if (tp[1] != am_scope || is_not_tuple(tp[2])) { + goto badarg; } - hp = HAlloc(p,state.m * (3 + 2)); - ret = NIL; - for (i = state.m - 1; i >= 0; --i) { - tpl = TUPLE2(hp, fad[i].epos, fad[i].elen); - hp +=3; - ret = CONS(hp,tpl,ret); - hp += 2; + tp = tuple_val(tp[2]); + if (arityval(*tp) != 2) { + goto badarg; + } + if (!term_to_Uint(tp[1], &pos)) { + goto badarg; } + if (!term_to_Sint(tp[2], &len)) { + goto badarg; + } + if (len < 0) { + Uint lentmp = -(Uint)len; + /* overflow */ + if ((Sint)lentmp < 0) { + goto badarg; + } + len = lentmp; + pos -= len; + } + /* overflow */ + if ((pos + len) < pos || (len > 0 && (pos + len) == pos)) { + goto badarg; + } + *endp = len + pos; + *posp = pos; + if ((orig_size = binary_size(bin)) < pos || + orig_size < (*endp)) { + goto badarg; + } + l = CDR(list_val(l)); } - erts_free_aligned_binary_bytes(temp_alloc); - ac_clean_find_all(&state); - BUMP_REDS(p, (save_reds - reds) / AC_LOOP_FACTOR); - *res_term = ret; - return DO_BIN_MATCH_OK; + return 0; + } else { + badarg: + return 1; } - badarg: - return DO_BIN_MATCH_BADARG; } -static int parse_match_opts_list(Eterm l, Eterm bin, Uint *posp, Uint *endp) +static int parse_split_opts_list(Eterm l, Eterm bin, Uint *posp, Uint *endp, Uint *optp) { Eterm *tp; Uint pos; Sint len; - if (l == ((Eterm) 0) || l == NIL) { - /* Invalid term or NIL, we're called from binary_match(es)_2 or - have no options*/ - *posp = 0; - *endp = binary_size(bin); + *optp = 0; + *posp = 0; + *endp = binary_size(bin); + if (l == THE_NON_VALUE || l == NIL) { return 0; } else if (is_list(l)) { while(is_list(l)) { Eterm t = CAR(list_val(l)); Uint orig_size; + if (is_atom(t)) { + if (t == am_global) { + *optp |= BINARY_FIND_ALL; + l = CDR(list_val(l)); + continue; + } + if (t == am_trim) { + *optp |= BINARY_SPLIT_TRIM; + l = CDR(list_val(l)); + continue; + } + if (t == am_trim_all) { + *optp |= BINARY_SPLIT_TRIM_ALL; + l = CDR(list_val(l)); + continue; + } + } if (!is_tuple(t)) { goto badarg; } @@ -1324,9 +1177,9 @@ static int parse_match_opts_list(Eterm l, Eterm bin, Uint *posp, Uint *endp) goto badarg; } if (len < 0) { - Sint lentmp = -len; + Uint lentmp = -(Uint)len; /* overflow */ - if (lentmp == len || lentmp < 0 || -lentmp != len) { + if ((Sint)lentmp < 0) { goto badarg; } len = lentmp; @@ -1351,48 +1204,207 @@ static int parse_match_opts_list(Eterm l, Eterm bin, Uint *posp, Uint *endp) } } -static BIF_RETTYPE binary_match_trap(BIF_ALIST_3) +static int do_binary_find(Process *p, Eterm subject, BinaryFindState *bfs, Binary *bin, + Eterm state_term, Eterm *res_term) { - int runres; - Eterm result; - Binary *bin = ((ProcBin *) binary_val(BIF_ARG_3))->val; - runres = do_binary_match(BIF_P,BIF_ARG_1,0,0,NIL,bin,BIF_ARG_2,&result); - if (runres == DO_BIN_MATCH_OK) { - BIF_RET(result); - } else { - BUMP_ALL_REDS(BIF_P); - BIF_TRAP3(&binary_match_trap_export, BIF_P, BIF_ARG_1, result, - BIF_ARG_3); + byte *bytes; + Uint bitoffs, bitsize; + byte *temp_alloc = NULL; + BinaryFindState_bignum *state_ptr = NULL; + + ERTS_GET_BINARY_BYTES(subject, bytes, bitoffs, bitsize); + if (bitsize != 0) { + goto badarg; + } + if (bitoffs != 0) { + bytes = erts_get_aligned_binary_bytes(subject, &temp_alloc); + } + if (state_term != NIL) { + state_ptr = (BinaryFindState_bignum *)(big_val(state_term)); + bfs = &(state_ptr->bfs); } -} -static BIF_RETTYPE binary_matches_trap(BIF_ALIST_3) -{ - int runres; - Eterm result; - Binary *bin = ((ProcBin *) binary_val(BIF_ARG_3))->val; - runres = do_binary_matches(BIF_P,BIF_ARG_1,0,0,NIL,bin,BIF_ARG_2,&result); - if (runres == DO_BIN_MATCH_OK) { - BIF_RET(result); + if (bfs->flags & BINARY_FIND_ALL) { + if (bfs->type == am_bm) { + BMData *bm; + Sint pos; + BMFindAllState state; + Uint reds = get_reds(p, BM_LOOP_FACTOR); + Uint save_reds = reds; + + bm = (BMData *) ERTS_MAGIC_BIN_DATA(bin); +#ifdef HARDDEBUG + dump_bm_data(bm); +#endif + if (state_term == NIL) { + bm_init_find_all(&state, bfs->hsstart, bfs->hsend); + } else { + bm_restore_find_all(&state, &(state_ptr->data.bmfas)); + } + + pos = bm_find_all_non_overlapping(&state, bm, bytes, &reds); + if (pos == BM_NOT_FOUND) { + *res_term = bfs->not_found_result(p, subject, bfs); + } else if (pos == BM_RESTART) { + int x = + (SIZEOF_BINARY_FIND_ALL_STATE(state) / sizeof(Eterm)) + + !!(SIZEOF_BINARY_FIND_ALL_STATE(state) % sizeof(Eterm)); +#ifdef HARDDEBUG + erts_printf("Trap bm!\n"); +#endif + state_ptr = (BinaryFindState_bignum*) HAlloc(p, x+1); + state_ptr->bignum_hdr = make_pos_bignum_header(x); + memcpy(&state_ptr->bfs, bfs, sizeof(BinaryFindState)); + bm_serialize_find_all(&state, &state_ptr->data.bmfas); + *res_term = make_big(&state_ptr->bignum_hdr); + erts_free_aligned_binary_bytes(temp_alloc); + bm_clean_find_all(&state); + return DO_BIN_MATCH_RESTART; + } else { + *res_term = bfs->global_result(p, subject, bfs, state.out, state.m); + } + erts_free_aligned_binary_bytes(temp_alloc); + bm_clean_find_all(&state); + BUMP_REDS(p, (save_reds - reds) / BM_LOOP_FACTOR); + return DO_BIN_MATCH_OK; + } else if (bfs->type == am_ac) { + ACTrie *act; + int acr; + ACFindAllState state; + Uint reds = get_reds(p, AC_LOOP_FACTOR); + Uint save_reds = reds; + + act = (ACTrie *) ERTS_MAGIC_BIN_DATA(bin); +#ifdef HARDDEBUG + dump_ac_trie(act); +#endif + if (state_term == NIL) { + ac_init_find_all(&state, act, bfs->hsstart, bfs->hsend); + } else { + ac_restore_find_all(&state, &(state_ptr->data.acfas)); + } + acr = ac_find_all_non_overlapping(&state, bytes, &reds); + if (acr == AC_NOT_FOUND) { + *res_term = bfs->not_found_result(p, subject, bfs); + } else if (acr == AC_RESTART) { + int x = + (SIZEOF_BINARY_FIND_ALL_STATE(state) / sizeof(Eterm)) + + !!(SIZEOF_BINARY_FIND_ALL_STATE(state) % sizeof(Eterm)); +#ifdef HARDDEBUG + erts_printf("Trap ac!\n"); +#endif + state_ptr = (BinaryFindState_bignum*) HAlloc(p, x+1); + state_ptr->bignum_hdr = make_pos_bignum_header(x); + memcpy(&state_ptr->bfs, bfs, sizeof(BinaryFindState)); + ac_serialize_find_all(&state, &state_ptr->data.acfas); + *res_term = make_big(&state_ptr->bignum_hdr); + erts_free_aligned_binary_bytes(temp_alloc); + ac_clean_find_all(&state); + return DO_BIN_MATCH_RESTART; + } else { + *res_term = bfs->global_result(p, subject, bfs, state.out, state.m); + } + erts_free_aligned_binary_bytes(temp_alloc); + ac_clean_find_all(&state); + BUMP_REDS(p, (save_reds - reds) / AC_LOOP_FACTOR); + return DO_BIN_MATCH_OK; + } } else { - BUMP_ALL_REDS(BIF_P); - BIF_TRAP3(&binary_matches_trap_export, BIF_P, BIF_ARG_1, result, - BIF_ARG_3); - } -} + if (bfs->type == am_bm) { + BMData *bm; + Sint pos; + BMFindFirstState state; + Uint reds = get_reds(p, BM_LOOP_FACTOR); + Uint save_reds = reds; + + bm = (BMData *) ERTS_MAGIC_BIN_DATA(bin); +#ifdef HARDDEBUG + dump_bm_data(bm); +#endif + if (state_term == NIL) { + bm_init_find_first_match(&state, bfs->hsstart, bfs->hsend); + } else { + memcpy(&state, &state_ptr->data.bmffs, sizeof(BMFindFirstState)); + } -BIF_RETTYPE binary_match_3(BIF_ALIST_3) -{ - return binary_match(BIF_P, BIF_ARG_1, BIF_ARG_2, BIF_ARG_3); +#ifdef HARDDEBUG + erts_printf("(bm) state->pos = %ld, state->len = %lu\n",state.pos, + state.len); +#endif + pos = bm_find_first_match(&state, bm, bytes, &reds); + if (pos == BM_NOT_FOUND) { + *res_term = bfs->not_found_result(p, subject, bfs); + } else if (pos == BM_RESTART) { + int x = + (SIZEOF_BINARY_FIND_STATE(state) / sizeof(Eterm)) + + !!(SIZEOF_BINARY_FIND_STATE(state) % sizeof(Eterm)); +#ifdef HARDDEBUG + erts_printf("Trap bm!\n"); +#endif + state_ptr = (BinaryFindState_bignum*) HAlloc(p, x+1); + state_ptr->bignum_hdr = make_pos_bignum_header(x); + memcpy(&state_ptr->bfs, bfs, sizeof(BinaryFindState)); + memcpy(&state_ptr->data.acffs, &state, sizeof(BMFindFirstState)); + *res_term = make_big(&state_ptr->bignum_hdr); + erts_free_aligned_binary_bytes(temp_alloc); + return DO_BIN_MATCH_RESTART; + } else { + *res_term = bfs->single_result(p, subject, bfs, pos, bm->len); + } + erts_free_aligned_binary_bytes(temp_alloc); + BUMP_REDS(p, (save_reds - reds) / BM_LOOP_FACTOR); + return DO_BIN_MATCH_OK; + } else if (bfs->type == am_ac) { + ACTrie *act; + Uint pos, rlen; + int acr; + ACFindFirstState state; + Uint reds = get_reds(p, AC_LOOP_FACTOR); + Uint save_reds = reds; + + act = (ACTrie *) ERTS_MAGIC_BIN_DATA(bin); +#ifdef HARDDEBUG + dump_ac_trie(act); +#endif + if (state_term == NIL) { + ac_init_find_first_match(&state, act, bfs->hsstart, bfs->hsend); + } else { + memcpy(&state, &state_ptr->data.acffs, sizeof(ACFindFirstState)); + } + acr = ac_find_first_match(&state, bytes, &pos, &rlen, &reds); + if (acr == AC_NOT_FOUND) { + *res_term = bfs->not_found_result(p, subject, bfs); + } else if (acr == AC_RESTART) { + int x = + (SIZEOF_BINARY_FIND_STATE(state) / sizeof(Eterm)) + + !!(SIZEOF_BINARY_FIND_STATE(state) % sizeof(Eterm)); +#ifdef HARDDEBUG + erts_printf("Trap ac!\n"); +#endif + state_ptr = (BinaryFindState_bignum*) HAlloc(p, x+1); + state_ptr->bignum_hdr = make_pos_bignum_header(x); + memcpy(&state_ptr->bfs, bfs, sizeof(BinaryFindState)); + memcpy(&state_ptr->data.acffs, &state, sizeof(ACFindFirstState)); + *res_term = make_big(&state_ptr->bignum_hdr); + erts_free_aligned_binary_bytes(temp_alloc); + return DO_BIN_MATCH_RESTART; + } else { + *res_term = bfs->single_result(p, subject, bfs, pos, rlen); + } + erts_free_aligned_binary_bytes(temp_alloc); + BUMP_REDS(p, (save_reds - reds) / AC_LOOP_FACTOR); + return DO_BIN_MATCH_OK; + } + } + badarg: + return DO_BIN_MATCH_BADARG; } static BIF_RETTYPE -binary_match(Process *p, Eterm arg1, Eterm arg2, Eterm arg3) +binary_match(Process *p, Eterm arg1, Eterm arg2, Eterm arg3, Uint flags) { - Uint hsstart; - Uint hsend; + BinaryFindState bfs; Eterm *tp; - Eterm type; Binary *bin; Eterm bin_term = NIL; int runres; @@ -1401,11 +1413,12 @@ binary_match(Process *p, Eterm arg1, Eterm arg2, Eterm arg3) if (is_not_binary(arg1)) { goto badarg; } - if (parse_match_opts_list(arg3,arg1,&hsstart,&hsend)) { + bfs.flags = flags; + if (parse_match_opts_list(arg3, arg1, &(bfs.hsstart), &(bfs.hsend))) { goto badarg; } - if (hsend == 0) { - BIF_RET(am_nomatch); + if (bfs.hsend == 0) { + BIF_RET(do_match_not_found_result(p, arg1, &bfs)); } if (is_tuple(arg2)) { tp = tuple_val(arg2); @@ -1416,21 +1429,24 @@ binary_match(Process *p, Eterm arg1, Eterm arg2, Eterm arg3) !ERTS_TERM_IS_MAGIC_BINARY(tp[2])) { goto badarg; } - type = tp[1]; + bfs.type = tp[1]; bin = ((ProcBin *) binary_val(tp[2]))->val; - if (type == am_bm && + if (bfs.type == am_bm && ERTS_MAGIC_BIN_DESTRUCTOR(bin) != cleanup_my_data_bm) { goto badarg; } - if (type == am_ac && + if (bfs.type == am_ac && ERTS_MAGIC_BIN_DESTRUCTOR(bin) != cleanup_my_data_ac) { goto badarg; } bin_term = tp[2]; - } else if (do_binary_match_compile(arg2,&type,&bin)) { + } else if (do_binary_match_compile(arg2, &(bfs.type), &bin)) { goto badarg; } - runres = do_binary_match(p,arg1,hsstart,hsend,type,bin,NIL,&result); + bfs.not_found_result = &do_match_not_found_result; + bfs.single_result = &do_match_single_result; + bfs.global_result = &do_match_global_result; + runres = do_binary_find(p, arg1, &bfs, bin, NIL, &result); if (runres == DO_BIN_MATCH_RESTART && bin_term == NIL) { Eterm *hp = HAlloc(p, PROC_BIN_SIZE); bin_term = erts_mk_magic_binary_term(&hp, &MSO(p), bin); @@ -1442,7 +1458,7 @@ binary_match(Process *p, Eterm arg1, Eterm arg2, Eterm arg3) BIF_RET(result); case DO_BIN_MATCH_RESTART: BUMP_ALL_REDS(p); - BIF_TRAP3(&binary_match_trap_export, p, arg1, result, bin_term); + BIF_TRAP3(&binary_find_trap_export, p, arg1, result, bin_term); default: goto badarg; } @@ -1450,17 +1466,31 @@ binary_match(Process *p, Eterm arg1, Eterm arg2, Eterm arg3) BIF_ERROR(p,BADARG); } +BIF_RETTYPE binary_match_2(BIF_ALIST_2) +{ + return binary_match(BIF_P, BIF_ARG_1, BIF_ARG_2, THE_NON_VALUE, 0); +} + +BIF_RETTYPE binary_match_3(BIF_ALIST_3) +{ + return binary_match(BIF_P, BIF_ARG_1, BIF_ARG_2, BIF_ARG_3, 0); +} + +BIF_RETTYPE binary_matches_2(BIF_ALIST_2) +{ + return binary_match(BIF_P, BIF_ARG_1, BIF_ARG_2, THE_NON_VALUE, BINARY_FIND_ALL); +} + BIF_RETTYPE binary_matches_3(BIF_ALIST_3) { - return binary_matches(BIF_P, BIF_ARG_1, BIF_ARG_2, BIF_ARG_3); + return binary_match(BIF_P, BIF_ARG_1, BIF_ARG_2, BIF_ARG_3, BINARY_FIND_ALL); } static BIF_RETTYPE -binary_matches(Process *p, Eterm arg1, Eterm arg2, Eterm arg3) +binary_split(Process *p, Eterm arg1, Eterm arg2, Eterm arg3) { - Uint hsstart, hsend; + BinaryFindState bfs; Eterm *tp; - Eterm type; Binary *bin; Eterm bin_term = NIL; int runres; @@ -1469,11 +1499,12 @@ binary_matches(Process *p, Eterm arg1, Eterm arg2, Eterm arg3) if (is_not_binary(arg1)) { goto badarg; } - if (parse_match_opts_list(arg3,arg1,&hsstart,&hsend)) { + if (parse_split_opts_list(arg3, arg1, &(bfs.hsstart), &(bfs.hsend), &(bfs.flags))) { goto badarg; } - if (hsend == 0) { - BIF_RET(NIL); + if (bfs.hsend == 0) { + result = do_split_not_found_result(p, arg1, &bfs); + BIF_RET(result); } if (is_tuple(arg2)) { tp = tuple_val(arg2); @@ -1484,54 +1515,267 @@ binary_matches(Process *p, Eterm arg1, Eterm arg2, Eterm arg3) !ERTS_TERM_IS_MAGIC_BINARY(tp[2])) { goto badarg; } - type = tp[1]; + bfs.type = tp[1]; bin = ((ProcBin *) binary_val(tp[2]))->val; - if (type == am_bm && + if (bfs.type == am_bm && ERTS_MAGIC_BIN_DESTRUCTOR(bin) != cleanup_my_data_bm) { goto badarg; } - if (type == am_ac && + if (bfs.type == am_ac && ERTS_MAGIC_BIN_DESTRUCTOR(bin) != cleanup_my_data_ac) { goto badarg; } bin_term = tp[2]; - } else if (do_binary_match_compile(arg2,&type,&bin)) { + } else if (do_binary_match_compile(arg2, &(bfs.type), &bin)) { goto badarg; } - runres = do_binary_matches(p,arg1,hsstart,hsend,type,bin, - NIL,&result); + bfs.not_found_result = &do_split_not_found_result; + bfs.single_result = &do_split_single_result; + bfs.global_result = &do_split_global_result; + runres = do_binary_find(p, arg1, &bfs, bin, NIL, &result); if (runres == DO_BIN_MATCH_RESTART && bin_term == NIL) { Eterm *hp = HAlloc(p, PROC_BIN_SIZE); bin_term = erts_mk_magic_binary_term(&hp, &MSO(p), bin); } else if (bin_term == NIL) { erts_bin_free(bin); } - switch (runres) { + switch(runres) { case DO_BIN_MATCH_OK: BIF_RET(result); case DO_BIN_MATCH_RESTART: - BUMP_ALL_REDS(p); - BIF_TRAP3(&binary_matches_trap_export, p, arg1, result, - bin_term); + BIF_TRAP3(&binary_find_trap_export, p, arg1, result, bin_term); default: goto badarg; } badarg: - BIF_ERROR(p,BADARG); + BIF_ERROR(p, BADARG); } +BIF_RETTYPE binary_split_2(BIF_ALIST_2) +{ + return binary_split(BIF_P, BIF_ARG_1, BIF_ARG_2, THE_NON_VALUE); +} -BIF_RETTYPE binary_match_2(BIF_ALIST_2) +BIF_RETTYPE binary_split_3(BIF_ALIST_3) { - return binary_match(BIF_P,BIF_ARG_1,BIF_ARG_2,((Eterm) 0)); + return binary_split(BIF_P, BIF_ARG_1, BIF_ARG_2, BIF_ARG_3); } +static Eterm do_match_not_found_result(Process *p, Eterm subject, BinaryFindState *bfs) +{ + if (bfs->flags & BINARY_FIND_ALL) { + return NIL; + } else { + return am_nomatch; + } +} -BIF_RETTYPE binary_matches_2(BIF_ALIST_2) +static Eterm do_match_single_result(Process *p, Eterm subject, BinaryFindState *bfs, + Sint pos, Sint len) +{ + Eterm erlen; + Eterm *hp; + Eterm ret; + + erlen = erts_make_integer((Uint)(len), p); + ret = erts_make_integer(pos, p); + hp = HAlloc(p, 3); + ret = TUPLE2(hp, ret, erlen); + + return ret; +} + +static Eterm do_match_global_result(Process *p, Eterm subject, BinaryFindState *bfs, + FindallData *fad, Uint fad_sz) +{ + Sint i; + Eterm tpl; + Eterm *hp; + Eterm ret; + + for (i = 0; i < fad_sz; ++i) { + fad[i].epos = erts_make_integer(fad[i].pos, p); + fad[i].elen = erts_make_integer(fad[i].len, p); + } + hp = HAlloc(p, fad_sz * (3 + 2)); + ret = NIL; + for (i = fad_sz - 1; i >= 0; --i) { + tpl = TUPLE2(hp, fad[i].epos, fad[i].elen); + hp += 3; + ret = CONS(hp, tpl, ret); + hp += 2; + } + + return ret; +} + +static Eterm do_split_not_found_result(Process *p, Eterm subject, BinaryFindState *bfs) { - return binary_matches(BIF_P,BIF_ARG_1,BIF_ARG_2,((Eterm) 0)); + Eterm *hp; + Eterm ret; + + if (bfs->flags & (BINARY_SPLIT_TRIM | BINARY_SPLIT_TRIM_ALL) + && binary_size(subject) == 0) { + return NIL; + } + hp = HAlloc(p, 2); + ret = CONS(hp, subject, NIL); + + return ret; } +static Eterm do_split_single_result(Process *p, Eterm subject, BinaryFindState *bfs, + Sint pos, Sint len) +{ + size_t orig_size; + Eterm orig; + Uint offset; + Uint bit_offset; + Uint bit_size; + ErlSubBin *sb1; + ErlSubBin *sb2; + Eterm *hp; + Eterm ret; + + orig_size = binary_size(subject); + + if ((bfs->flags & (BINARY_SPLIT_TRIM | BINARY_SPLIT_TRIM_ALL)) && + (orig_size - pos - len) == 0) { + if (pos == 0) { + ret = NIL; + } else { + hp = HAlloc(p, (ERL_SUB_BIN_SIZE + 2)); + ERTS_GET_REAL_BIN(subject, orig, offset, bit_offset, bit_size); + sb1 = (ErlSubBin *) hp; + sb1->thing_word = HEADER_SUB_BIN; + sb1->size = pos; + sb1->offs = offset; + sb1->orig = orig; + sb1->bitoffs = bit_offset; + sb1->bitsize = bit_size; + sb1->is_writable = 0; + hp += ERL_SUB_BIN_SIZE; + + ret = CONS(hp, make_binary(sb1), NIL); + hp += 2; + } + } else { + if ((bfs->flags & BINARY_SPLIT_TRIM_ALL) && (pos == 0)) { + hp = HAlloc(p, 1 * (ERL_SUB_BIN_SIZE + 2)); + ERTS_GET_REAL_BIN(subject, orig, offset, bit_offset, bit_size); + sb1 = NULL; + } else { + hp = HAlloc(p, 2 * (ERL_SUB_BIN_SIZE + 2)); + ERTS_GET_REAL_BIN(subject, orig, offset, bit_offset, bit_size); + sb1 = (ErlSubBin *) hp; + sb1->thing_word = HEADER_SUB_BIN; + sb1->size = pos; + sb1->offs = offset; + sb1->orig = orig; + sb1->bitoffs = bit_offset; + sb1->bitsize = 0; + sb1->is_writable = 0; + hp += ERL_SUB_BIN_SIZE; + } + + sb2 = (ErlSubBin *) hp; + sb2->thing_word = HEADER_SUB_BIN; + sb2->size = orig_size - pos - len; + sb2->offs = offset + pos + len; + sb2->orig = orig; + sb2->bitoffs = bit_offset; + sb2->bitsize = bit_size; + sb2->is_writable = 0; + hp += ERL_SUB_BIN_SIZE; + + ret = CONS(hp, make_binary(sb2), NIL); + hp += 2; + if (sb1 != NULL) { + ret = CONS(hp, make_binary(sb1), ret); + hp += 2; + } + } + return ret; +} + +static Eterm do_split_global_result(Process *p, Eterm subject, BinaryFindState *bfs, + FindallData *fad, Uint fad_sz) +{ + size_t orig_size; + Eterm orig; + Uint offset; + Uint bit_offset; + Uint bit_size; + ErlSubBin *sb; + Sint i; + Sint tail; + Uint list_size; + Uint end_pos; + Uint do_trim = bfs->flags & (BINARY_SPLIT_TRIM | BINARY_SPLIT_TRIM_ALL); + Eterm *hp; + Eterm *hendp; + Eterm ret; + + tail = fad_sz - 1; + list_size = fad_sz + 1; + orig_size = binary_size(subject); + end_pos = (Uint)(orig_size); + + hp = HAlloc(p, list_size * (ERL_SUB_BIN_SIZE + 2)); + hendp = hp + list_size * (ERL_SUB_BIN_SIZE + 2); + ERTS_GET_REAL_BIN(subject, orig, offset, bit_offset, bit_size); + ASSERT(bit_size == 0); + + ret = NIL; + + for (i = tail; i >= 0; --i) { + sb = (ErlSubBin *)(hp); + sb->size = end_pos - (fad[i].pos + fad[i].len); + if (!(sb->size == 0 && do_trim)) { + sb->thing_word = HEADER_SUB_BIN; + sb->offs = offset + fad[i].pos + fad[i].len; + sb->orig = orig; + sb->bitoffs = bit_offset; + sb->bitsize = 0; + sb->is_writable = 0; + hp += ERL_SUB_BIN_SIZE; + ret = CONS(hp, make_binary(sb), ret); + hp += 2; + do_trim &= ~BINARY_SPLIT_TRIM; + } + end_pos = fad[i].pos; + } + + sb = (ErlSubBin *)(hp); + sb->size = fad[0].pos; + if (!(sb->size == 0 && do_trim)) { + sb->thing_word = HEADER_SUB_BIN; + sb->offs = offset; + sb->orig = orig; + sb->bitoffs = bit_offset; + sb->bitsize = 0; + sb->is_writable = 0; + hp += ERL_SUB_BIN_SIZE; + ret = CONS(hp, make_binary(sb), ret); + hp += 2; + } + HRelease(p, hendp, hp); + return ret; +} + +static BIF_RETTYPE binary_find_trap(BIF_ALIST_3) +{ + int runres; + Eterm result; + Binary *bin = ((ProcBin *) binary_val(BIF_ARG_3))->val; + runres = do_binary_find(BIF_P, BIF_ARG_1, NULL, bin, BIF_ARG_2, &result); + if (runres == DO_BIN_MATCH_OK) { + BIF_RET(result); + } else { + BUMP_ALL_REDS(BIF_P); + BIF_TRAP3(&binary_find_trap_export, BIF_P, BIF_ARG_1, result, BIF_ARG_3); + } +} BIF_RETTYPE erts_binary_part(Process *p, Eterm binary, Eterm epos, Eterm elen) { @@ -1555,9 +1799,9 @@ BIF_RETTYPE erts_binary_part(Process *p, Eterm binary, Eterm epos, Eterm elen) goto badarg; } if (len < 0) { - Sint lentmp = -len; + Uint lentmp = -(Uint)len; /* overflow */ - if (lentmp == len || lentmp < 0 || -lentmp != len) { + if ((Sint)lentmp < 0) { goto badarg; } len = lentmp; @@ -1644,9 +1888,9 @@ BIF_RETTYPE erts_gc_binary_part(Process *p, Eterm *reg, Eterm live, int range_is goto badarg; } if (len < 0) { - Sint lentmp = -len; + Uint lentmp = -(Uint)len; /* overflow */ - if (lentmp == len || lentmp < 0 || -lentmp != len) { + if ((Sint)lentmp < 0) { goto badarg; } len = lentmp; @@ -2213,9 +2457,9 @@ static BIF_RETTYPE binary_bin_to_list_common(Process *p, goto badarg; } if (len < 0) { - Sint lentmp = -len; + Uint lentmp = -(Uint)len; /* overflow */ - if (lentmp == len || lentmp < 0 || -lentmp != len) { + if ((Sint)lentmp < 0) { goto badarg; } len = lentmp; @@ -2294,18 +2538,11 @@ BIF_RETTYPE binary_bin_to_list_1(BIF_ALIST_1) BIF_ERROR(BIF_P,BADARG); } -/* - * Ok, erlang:list_to_binary does not interrupt, and we really don't want - * an alternative implementation for the exact same thing, why we - * have descided to use the old non-restarting implementation for now. - * In reality, there are seldom many iterations involved in doing this, so the - * problem of long-running bifs is not really that big in this case. - * So, for now we use the old implementation also in the module binary. - */ +HIPE_WRAPPER_BIF_DISABLE_GC(binary_list_to_bin, 1) BIF_RETTYPE binary_list_to_bin_1(BIF_ALIST_1) { - return erts_list_to_binary_bif(BIF_P, BIF_ARG_1); + return erts_list_to_binary_bif(BIF_P, BIF_ARG_1, bif_export[BIF_binary_list_to_bin_1]); } typedef struct { @@ -2431,8 +2668,6 @@ static BIF_RETTYPE do_binary_copy(Process *p, Eterm bin, Eterm en) } cbs->result = erts_bin_nrml_alloc(target_size); /* Always offheap if trapping */ - cbs->result->flags = 0; - cbs->result->orig_size = target_size; erts_refc_init(&(cbs->result->refc), 1); t = (byte *) cbs->result->orig_bytes; /* No offset or anything */ pos = 0; @@ -2555,7 +2790,6 @@ BIF_RETTYPE binary_referenced_byte_size_1(BIF_ALIST_1) } pb = (ProcBin *) binary_val(bin); if (pb->thing_word == HEADER_PROC_BIN) { - /* XXX:PaN - Halfword - orig_size is a long, we should handle that */ res = erts_make_integer((Uint) pb->val->orig_size, BIF_P); } else { /* heap binary */ res = erts_make_integer((Uint) ((ErlHeapBin *) pb)->size, BIF_P); @@ -2573,7 +2807,7 @@ BIF_RETTYPE binary_referenced_byte_size_1(BIF_ALIST_1) #endif static int get_need(Uint u) { -#if defined(ARCH_64) && !HALFWORD_HEAP +#if defined(ARCH_64) if (u > 0xFFFFFFFFUL) { if (u > 0xFFFFFFFFFFFFUL) { if (u > 0xFFFFFFFFFFFFFFUL) { diff --git a/erts/emulator/beam/erl_bif_chksum.c b/erts/emulator/beam/erl_bif_chksum.c index 4302fe8f79..e3074d6309 100644 --- a/erts/emulator/beam/erl_bif_chksum.c +++ b/erts/emulator/beam/erl_bif_chksum.c @@ -3,16 +3,17 @@ * * Copyright Ericsson AB 2008-2013. All Rights Reserved. * - * The contents of this file are subject to the Erlang Public License, - * Version 1.1, (the "License"); you may not use this file except in - * compliance with the License. You should have received a copy of the - * Erlang Public License along with this software. If not, it can be - * retrieved online at http://www.erlang.org/. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Software distributed under the License is distributed on an "AS IS" - * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See - * the License for the specific language governing rights and limitations - * under the License. + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. * * %CopyrightEnd% */ diff --git a/erts/emulator/beam/erl_bif_ddll.c b/erts/emulator/beam/erl_bif_ddll.c index 56cd2ba04f..f006856b6a 100644 --- a/erts/emulator/beam/erl_bif_ddll.c +++ b/erts/emulator/beam/erl_bif_ddll.c @@ -3,16 +3,17 @@ * * Copyright Ericsson AB 2006-2013. All Rights Reserved. * - * The contents of this file are subject to the Erlang Public License, - * Version 1.1, (the "License"); you may not use this file except in - * compliance with the License. You should have received a copy of the - * Erlang Public License along with this software. If not, it can be - * retrieved online at http://www.erlang.org/. - * - * Software distributed under the License is distributed on an "AS IS" - * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See - * the License for the specific language governing rights and limitations - * under the License. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. * * %CopyrightEnd% */ @@ -45,6 +46,7 @@ #include "big.h" #include "dist.h" #include "erl_version.h" +#include "erl_bif_unique.h" #include "dtrace-wrapper.h" #ifdef ERTS_SMP @@ -1307,7 +1309,7 @@ static Eterm notify_when_loaded(Process *p, Eterm name_term, char *name, ErtsPro case ERL_DE_FORCE_RELOAD: break; default: - erl_exit(1,"Internal error, unknown state %u in dynamic driver.", drv->handle->status); + erts_exit(ERTS_ERROR_EXIT,"Internal error, unknown state %u in dynamic driver.", drv->handle->status); } p->flags |= F_USING_DDLL; r = add_monitor(p, drv->handle, ERL_DE_PROC_AWAIT_LOAD); @@ -1705,18 +1707,19 @@ static void notify_proc(Process *proc, Eterm ref, Eterm driver_name, Eterm type, Eterm mess; Eterm r; Eterm *hp; - ErlHeapFragment *bp; - ErlOffHeap *ohp; + ErtsMessage *mp; ErtsProcLocks rp_locks = 0; + ErlOffHeap *ohp; ERTS_SMP_CHK_NO_PROC_LOCKS; assert_drv_list_rwlocked(); if (errcode != 0) { int need = load_error_need(errcode); Eterm e; - hp = erts_alloc_message_heap(6 /* tuple */ + 3 /* Error tuple */ + - REF_THING_SIZE + need, &bp, &ohp, - proc, &rp_locks); + mp = erts_alloc_message_heap(proc, &rp_locks, + (6 /* tuple */ + 3 /* Error tuple */ + + REF_THING_SIZE + need), + &hp, &ohp); r = copy_ref(ref,hp); hp += REF_THING_SIZE; e = build_load_error_hp(hp, errcode); @@ -1725,16 +1728,14 @@ static void notify_proc(Process *proc, Eterm ref, Eterm driver_name, Eterm type, hp += 3; mess = TUPLE5(hp,type,r,am_driver,driver_name,mess); } else { - hp = erts_alloc_message_heap(6 /* tuple */ + REF_THING_SIZE, &bp, &ohp, proc, &rp_locks); + mp = erts_alloc_message_heap(proc, &rp_locks, + 6 /* tuple */ + REF_THING_SIZE, + &hp, &ohp); r = copy_ref(ref,hp); hp += REF_THING_SIZE; mess = TUPLE5(hp,type,r,am_driver,driver_name,tag); } - erts_queue_message(proc, &rp_locks, bp, mess, am_undefined -#ifdef USE_VM_PROBES - , NIL -#endif - ); + erts_queue_message(proc, &rp_locks, mp, mess, am_undefined); erts_smp_proc_unlock(proc, rp_locks); ERTS_SMP_CHK_NO_PROC_LOCKS; } diff --git a/erts/emulator/beam/erl_bif_guard.c b/erts/emulator/beam/erl_bif_guard.c index bbd8aa31d9..4a9a6a5fcd 100644 --- a/erts/emulator/beam/erl_bif_guard.c +++ b/erts/emulator/beam/erl_bif_guard.c @@ -3,16 +3,17 @@ * * Copyright Ericsson AB 2006-2011. All Rights Reserved. * - * The contents of this file are subject to the Erlang Public License, - * Version 1.1, (the "License"); you may not use this file except in - * compliance with the License. You should have received a copy of the - * Erlang Public License along with this software. If not, it can be - * retrieved online at http://www.erlang.org/. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Software distributed under the License is distributed on an "AS IS" - * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See - * the License for the specific language governing rights and limitations - * under the License. + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. * * %CopyrightEnd% */ @@ -459,23 +460,25 @@ Eterm erts_gc_byte_size_1(Process* p, Eterm* reg, Uint live) Eterm erts_gc_map_size_1(Process* p, Eterm* reg, Uint live) { Eterm arg = reg[live]; - if (is_map(arg)) { - map_t *mp = (map_t*)map_val(arg); - Uint size = map_get_size(mp); - if (IS_USMALL(0, size)) { - return make_small(size); - } else { - Eterm* hp; - if (ERTS_NEED_GC(p, BIG_UINT_HEAP_SIZE)) { - erts_garbage_collect(p, BIG_UINT_HEAP_SIZE, reg, live); - } - hp = p->htop; - p->htop += BIG_UINT_HEAP_SIZE; - return uint_to_big(size, hp); - } - } else { - BIF_ERROR(p, BADARG); - } + if (is_flatmap(arg)) { + flatmap_t *mp = (flatmap_t*)flatmap_val(arg); + return make_small(flatmap_get_size(mp)); + } else if (is_hashmap(arg)) { + Eterm* hp; + Uint size; + size = hashmap_size(arg); + if (IS_USMALL(0, size)) { + return make_small(size); + } + if (ERTS_NEED_GC(p, BIG_UINT_HEAP_SIZE)) { + erts_garbage_collect(p, BIG_UINT_HEAP_SIZE, reg, live); + } + hp = p->htop; + p->htop += BIG_UINT_HEAP_SIZE; + return uint_to_big(size, hp); + } + p->fvalue = arg; + BIF_ERROR(p, BADMAP); } Eterm erts_gc_abs_1(Process* p, Eterm* reg, Uint live) diff --git a/erts/emulator/beam/erl_bif_info.c b/erts/emulator/beam/erl_bif_info.c index 2adba9b240..8c748c9bf7 100755..100644 --- a/erts/emulator/beam/erl_bif_info.c +++ b/erts/emulator/beam/erl_bif_info.c @@ -3,16 +3,17 @@ * * Copyright Ericsson AB 1999-2013. All Rights Reserved. * - * The contents of this file are subject to the Erlang Public License, - * Version 1.1, (the "License"); you may not use this file except in - * compliance with the License. You should have received a copy of the - * Erlang Public License along with this software. If not, it can be - * retrieved online at http://www.erlang.org/. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Software distributed under the License is distributed on an "AS IS" - * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See - * the License for the specific language governing rights and limitations - * under the License. + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. * * %CopyrightEnd% */ @@ -27,6 +28,7 @@ #include "erl_process.h" #include "error.h" #include "erl_driver.h" +#include "erl_nif.h" #include "bif.h" #include "big.h" #include "erl_version.h" @@ -41,6 +43,7 @@ #include "erl_cpu_topology.h" #include "erl_async.h" #include "erl_thr_progress.h" +#include "erl_bif_unique.h" #define ERTS_PTAB_WANT_DEBUG_FUNCS__ #include "erl_ptab.h" #ifdef HIPE @@ -58,9 +61,12 @@ static Export* alloc_info_trap = NULL; static Export* alloc_sizes_trap = NULL; +static Export* gather_io_bytes_trap = NULL; static Export *gather_sched_wall_time_res_trap; +static Export *gather_msacc_res_trap; static Export *gather_gc_info_res_trap; +static Export *gather_system_check_res_trap; #define DECL_AM(S) Eterm AM_ ## S = am_atom_put(#S, sizeof(#S) - 1) @@ -69,9 +75,6 @@ static char otp_version[] = ERLANG_OTP_VERSION; static char erts_system_version[] = ("Erlang/OTP " ERLANG_OTP_RELEASE "%s" " [erts-" ERLANG_VERSION "]" -#if !HEAP_ON_C_STACK && !HALFWORD_HEAP - " [no-c-stack-objects]" -#endif #ifndef OTP_RELEASE #ifdef ERLANG_GIT_VERSION " [source-" ERLANG_GIT_VERSION "]" @@ -80,17 +83,13 @@ static char erts_system_version[] = ("Erlang/OTP " ERLANG_OTP_RELEASE #endif #endif #ifdef ARCH_64 -#if HALFWORD_HEAP - " [64-bit halfword]" -#else " [64-bit]" #endif -#endif #ifdef ERTS_SMP " [smp:%beu:%beu]" #endif #ifdef USE_THREADS -#ifdef ERTS_DIRTY_SCHEDULERS +#if defined(ERTS_DIRTY_SCHEDULERS) && defined(ERTS_SMP) " [ds:%beu:%beu:%beu]" #endif " [async-threads:%d]" @@ -115,6 +114,9 @@ static char erts_system_version[] = ("Erlang/OTP " ERLANG_OTP_RELEASE #ifdef ERTS_ENABLE_LOCK_COUNT " [lock-counting]" #endif +#ifdef ERTS_OPCODE_COUNTER_SUPPORT + " [instruction-counting]" +#endif #ifdef PURIFY " [purify-compiled]" #endif @@ -130,6 +132,9 @@ static char erts_system_version[] = ("Erlang/OTP " ERLANG_OTP_RELEASE #ifdef USE_SYSTEMTAP " [systemtap]" #endif +#ifdef SHCOPY + " [sharing-preserving]" +#endif "\n"); #define ASIZE(a) (sizeof(a)/sizeof(a[0])) @@ -315,13 +320,11 @@ erts_print_system_version(int to, void *arg, Process *c_p) char *ov = otp_version; #ifdef ERTS_SMP Uint total, online, active; -#ifdef ERTS_DIRTY_SCHEDULERS Uint dirty_cpu, dirty_cpu_onln, dirty_io; - (void) erts_schedulers_state(&total, &online, &active, &dirty_cpu, &dirty_cpu_onln, &dirty_io, 0); -#else - (void) erts_schedulers_state(&total, &online, &active, NULL, NULL, NULL, 0); -#endif + erts_schedulers_state(&total, &online, &active, + &dirty_cpu, &dirty_cpu_onln, NULL, + &dirty_io, NULL); #endif for (i = 0; i < sizeof(otp_version)-4; i++) { if (ov[i] == '-' && ov[i+1] == 'r' && ov[i+2] == 'c') @@ -536,6 +539,7 @@ pi_locks(Eterm info) switch (info) { case am_status: case am_priority: + case am_trap_exit: return ERTS_PROC_LOCK_STATUS; case am_links: case am_monitors: @@ -588,7 +592,9 @@ static Eterm pi_args[] = { am_min_bin_vheap_size, am_current_location, am_current_stacktrace, -}; + am_message_queue_data, + am_garbage_collection_info +}; #define ERTS_PI_ARGS ((int) (sizeof(pi_args)/sizeof(Eterm))) @@ -635,6 +641,8 @@ pi_arg2ix(Eterm arg) case am_min_bin_vheap_size: return 28; case am_current_location: return 29; case am_current_stacktrace: return 30; + case am_message_queue_data: return 31; + case am_garbage_collection_info: return 32; default: return -1; } } @@ -663,18 +671,12 @@ static Eterm pi_1_keys[] = { #define ERTS_PI_1_NO_OF_KEYS (sizeof(pi_1_keys)/sizeof(Eterm)) static Eterm pi_1_keys_list; -#if HEAP_ON_C_STACK static Eterm pi_1_keys_list_heap[2*ERTS_PI_1_NO_OF_KEYS]; -#endif static void process_info_init(void) { -#if HEAP_ON_C_STACK Eterm *hp = &pi_1_keys_list_heap[0]; -#else - Eterm *hp = erts_alloc(ERTS_ALC_T_LL_TEMP_TERM,sizeof(Eterm)*2*ERTS_PI_1_NO_OF_KEYS); -#endif int i; pi_1_keys_list = NIL; @@ -723,9 +725,10 @@ pi_pid2proc(Process *c_p, Eterm pid, ErtsProcLocks info_locks) -BIF_RETTYPE +static BIF_RETTYPE process_info_aux(Process *BIF_P, Process *rp, + ErtsProcLocks rp_locks, Eterm rpid, Eterm item, int always_wrap); @@ -816,10 +819,31 @@ process_info_list(Process *c_p, Eterm pid, Eterm list, int always_wrap, *fail_type = ERTS_PI_FAIL_TYPE_AWAIT_EXIT; goto done; } - else if (!(locks & ERTS_PROC_LOCK_STATUS)) { - erts_smp_proc_unlock(rp, ERTS_PROC_LOCK_STATUS); + else { + ErtsProcLocks unlock_locks = 0; + + if (c_p == rp) + locks |= ERTS_PROC_LOCK_MAIN; + + if (!(locks & ERTS_PROC_LOCK_STATUS)) + unlock_locks |= ERTS_PROC_LOCK_STATUS; + + if (locks & ERTS_PROC_LOCK_MSGQ) { + /* + * Move in queue into private queue and + * release msgq lock, enabling others to + * send messages to the process while it + * is being inspected... + */ + ASSERT(locks & ERTS_PROC_LOCK_MAIN); + ERTS_SMP_MSGQ_MV_INQ2PRIVQ(rp); + locks &= ~ERTS_PROC_LOCK_MSGQ; + unlock_locks |= ERTS_PROC_LOCK_MSGQ; + } + + if (unlock_locks) + erts_smp_proc_unlock(rp, unlock_locks); } - /* * We always handle 'messages' first if it should be part @@ -831,7 +855,7 @@ process_info_list(Process *c_p, Eterm pid, Eterm list, int always_wrap, if (want_messages) { ix = pi_arg2ix(am_messages); ASSERT(part_res[ix] == THE_NON_VALUE); - part_res[ix] = process_info_aux(c_p, rp, pid, am_messages, always_wrap); + part_res[ix] = process_info_aux(c_p, rp, locks, pid, am_messages, always_wrap); ASSERT(part_res[ix] != THE_NON_VALUE); } @@ -839,7 +863,7 @@ process_info_list(Process *c_p, Eterm pid, Eterm list, int always_wrap, ix = res_elem_ix[res_elem_ix_ix]; if (part_res[ix] == THE_NON_VALUE) { arg = pi_ix2arg(ix); - part_res[ix] = process_info_aux(c_p, rp, pid, arg, always_wrap); + part_res[ix] = process_info_aux(c_p, rp, locks, pid, arg, always_wrap); ASSERT(part_res[ix] != THE_NON_VALUE); } } @@ -906,7 +930,7 @@ BIF_RETTYPE process_info_1(BIF_ALIST_1) case ERTS_PI_FAIL_TYPE_AWAIT_EXIT: ERTS_BIF_AWAIT_X_DATA_TRAP(BIF_P, BIF_ARG_1, am_undefined); default: - erl_exit(ERTS_ABORT_EXIT, "%s:%d: Internal error", __FILE__, __LINE__); + erts_exit(ERTS_ABORT_EXIT, "%s:%d: Internal error", __FILE__, __LINE__); } } @@ -946,7 +970,7 @@ BIF_RETTYPE process_info_2(BIF_ALIST_2) case ERTS_PI_FAIL_TYPE_AWAIT_EXIT: ERTS_BIF_AWAIT_X_DATA_TRAP(BIF_P, BIF_ARG_1, am_undefined); default: - erl_exit(ERTS_ABORT_EXIT, "%s:%d: Internal error", + erts_exit(ERTS_ABORT_EXIT, "%s:%d: Internal error", __FILE__, __LINE__); } } @@ -970,9 +994,31 @@ BIF_RETTYPE process_info_2(BIF_ALIST_2) ERTS_BIF_AWAIT_X_DATA_TRAP(BIF_P, BIF_ARG_1, am_undefined); } else { + ErtsProcLocks unlock_locks = 0; + + if (BIF_P == rp) + info_locks |= ERTS_PROC_LOCK_MAIN; + if (!(info_locks & ERTS_PROC_LOCK_STATUS)) - erts_smp_proc_unlock(rp, ERTS_PROC_LOCK_STATUS); - res = process_info_aux(BIF_P, rp, pid, BIF_ARG_2, 0); + unlock_locks |= ERTS_PROC_LOCK_STATUS; + + if (info_locks & ERTS_PROC_LOCK_MSGQ) { + /* + * Move in queue into private queue and + * release msgq lock, enabling others to + * send messages to the process while it + * is being inspected... + */ + ASSERT(info_locks & ERTS_PROC_LOCK_MAIN); + ERTS_SMP_MSGQ_MV_INQ2PRIVQ(rp); + info_locks &= ~ERTS_PROC_LOCK_MSGQ; + unlock_locks |= ERTS_PROC_LOCK_MSGQ; + } + + if (unlock_locks) + erts_smp_proc_unlock(rp, unlock_locks); + + res = process_info_aux(BIF_P, rp, info_locks, pid, BIF_ARG_2, 0); } ASSERT(is_value(res)); @@ -990,6 +1036,7 @@ BIF_RETTYPE process_info_2(BIF_ALIST_2) Eterm process_info_aux(Process *BIF_P, Process *rp, + ErtsProcLocks rp_locks, Eterm rpid, Eterm item, int always_wrap) @@ -1048,9 +1095,9 @@ process_info_aux(Process *BIF_P, case am_initial_call: hp = HAlloc(BIF_P, 3+4); res = TUPLE3(hp, - rp->initial[INITIAL_MOD], - rp->initial[INITIAL_FUN], - make_small(rp->initial[INITIAL_ARI])); + rp->u.initial[INITIAL_MOD], + rp->u.initial[INITIAL_FUN], + make_small(rp->u.initial[INITIAL_ARI])); hp += 4; break; @@ -1061,161 +1108,55 @@ process_info_aux(Process *BIF_P, break; case am_messages: { - ErlMessage* mp; - int n; - ERTS_SMP_MSGQ_MV_INQ2PRIVQ(rp); - n = rp->msg.len; - - if (n == 0 || ERTS_TRACE_FLAGS(rp) & F_SENSITIVE) { + if (rp->msg.len == 0 || ERTS_TRACE_FLAGS(rp) & F_SENSITIVE) { hp = HAlloc(BIF_P, 3); } else { - int remove_bad_messages = 0; - struct { - Uint copy_struct_size; - ErlMessage* msgp; - } *mq = erts_alloc(ERTS_ALC_T_TMP, n*sizeof(*mq)); - Sint i = 0; - Uint heap_need = 3; + ErtsMessageInfo *mip; + Sint i; + Uint heap_need; +#ifdef DEBUG Eterm *hp_end; +#endif - for (mp = rp->msg.first; mp; mp = mp->next) { - heap_need += 2; - mq[i].msgp = mp; - if (rp != BIF_P) { - Eterm msg = ERL_MESSAGE_TERM(mq[i].msgp); - if (is_value(msg)) { - mq[i].copy_struct_size = (is_immed(msg)? 0 : - size_object(msg)); - } - else if (mq[i].msgp->data.attached) { - mq[i].copy_struct_size - = erts_msg_attached_data_size(mq[i].msgp); - } - else { - /* Bad distribution message; ignore */ - remove_bad_messages = 1; - mq[i].copy_struct_size = 0; - } - heap_need += mq[i].copy_struct_size; - } - else { - mq[i].copy_struct_size = 0; - if (mp->data.attached) - heap_need += erts_msg_attached_data_size(mp); - } - i++; - } + mip = erts_alloc(ERTS_ALC_T_TMP, + rp->msg.len*sizeof(ErtsMessageInfo)); - hp = HAlloc(BIF_P, heap_need); + /* + * Note that message queue may shrink when calling + * erts_prep_msgq_for_inspection() since it removes + * corrupt distribution messages. + */ + heap_need = erts_prep_msgq_for_inspection(BIF_P, rp, rp_locks, mip); + heap_need += 3; /* top 2-tuple */ + heap_need += rp->msg.len*2; /* Cons cells */ + + hp = HAlloc(BIF_P, heap_need); /* heap_need is exact */ +#ifdef DEBUG hp_end = hp + heap_need; - ASSERT(i == n); - for (i--; i >= 0; i--) { - Eterm msg = ERL_MESSAGE_TERM(mq[i].msgp); - if (rp != BIF_P) { - if (is_value(msg)) { - if (mq[i].copy_struct_size) - msg = copy_struct(msg, - mq[i].copy_struct_size, - &hp, - &MSO(BIF_P)); - } - else if (mq[i].msgp->data.attached) { - ErlHeapFragment *hfp; - /* - * Decode it into a message buffer and attach it - * to the message instead of the attached external - * term. - * - * Note that we may not pass a process pointer - * to erts_msg_distext2heap(), since it would then - * try to alter locks on that process. - */ - msg = erts_msg_distext2heap( - NULL, NULL, &hfp, &ERL_MESSAGE_TOKEN(mq[i].msgp), - mq[i].msgp->data.dist_ext); - - ERL_MESSAGE_TERM(mq[i].msgp) = msg; - mq[i].msgp->data.heap_frag = hfp; - - if (is_non_value(msg)) { - ASSERT(!mq[i].msgp->data.heap_frag); - /* Bad distribution message; ignore */ - remove_bad_messages = 1; - continue; - } - else { - /* Make our copy of the message */ - ASSERT(size_object(msg) == hfp->used_size); - msg = copy_struct(msg, - hfp->used_size, - &hp, - &MSO(BIF_P)); - } - } - else { - /* Bad distribution message; ignore */ - remove_bad_messages = 1; - continue; - } - } - else { - if (mq[i].msgp->data.attached) { - /* Decode it on the heap */ - erts_move_msg_attached_data_to_heap(&hp, - &MSO(BIF_P), - mq[i].msgp); - msg = ERL_MESSAGE_TERM(mq[i].msgp); - ASSERT(!mq[i].msgp->data.attached); - if (is_non_value(msg)) { - /* Bad distribution message; ignore */ - remove_bad_messages = 1; - continue; - } - } - } - +#endif + + /* Build list of messages... */ + for (i = rp->msg.len - 1, res = NIL; i >= 0; i--) { + Eterm msg = ERL_MESSAGE_TERM(mip[i].msgp); + Uint sz = mip[i].size; + + if (sz != 0) + msg = copy_struct(msg, sz, &hp, &BIF_P->off_heap); + res = CONS(hp, msg, res); hp += 2; } - HRelease(BIF_P, hp_end, hp+3); - erts_free(ERTS_ALC_T_TMP, mq); - if (remove_bad_messages) { - ErlMessage **mpp; - /* - * We need to remove bad distribution messages from - * the queue, so that the value returned for - * 'message_queue_len' is consistent with the value - * returned for 'messages'. - */ - mpp = &rp->msg.first; - mp = rp->msg.first; - while (mp) { - if (is_value(ERL_MESSAGE_TERM(mp))) { - mpp = &mp->next; - mp = mp->next; - } - else { - ErlMessage* bad_mp = mp; - ASSERT(!mp->data.attached); - if (rp->msg.save == &mp->next) - rp->msg.save = mpp; - if (rp->msg.last == &mp->next) - rp->msg.last = mpp; - *mpp = mp->next; - mp = mp->next; - rp->msg.len--; - free_message(bad_mp); - } - } - } + + ASSERT(hp_end == hp + 3); + + erts_free(ERTS_ALC_T_TMP, mip); } break; } case am_message_queue_len: hp = HAlloc(BIF_P, 3); - ERTS_SMP_MSGQ_MV_INQ2PRIVQ(rp); res = make_small(rp->msg.len); break; @@ -1403,7 +1344,7 @@ process_info_aux(Process *BIF_P, } case am_total_heap_size: { - ErlMessage *mp; + ErtsMessage *mp; Uint total_heap_size; Uint hsz = 3; @@ -1413,8 +1354,6 @@ process_info_aux(Process *BIF_P, total_heap_size += rp->mbuf_sz; - ERTS_SMP_MSGQ_MV_INQ2PRIVQ(rp); - for (mp = rp->msg.first; mp; mp = mp->next) if (mp->data.attached) total_heap_size += erts_msg_attached_data_size(mp); @@ -1436,7 +1375,7 @@ process_info_aux(Process *BIF_P, case am_memory: { /* Memory consumed in bytes */ Uint hsz = 3; - Uint size = erts_process_memory(rp); + Uint size = erts_process_memory(rp, 0); (void) erts_bld_uint(NULL, &hsz, size); hp = HAlloc(BIF_P, hsz); res = erts_bld_uint(&hp, NULL, size); @@ -1461,6 +1400,32 @@ process_info_aux(Process *BIF_P, break; } + case am_garbage_collection_info: { + Uint sz = 0, actual_sz = 0; + + if (rp == BIF_P) { + sz += ERTS_PROCESS_GC_INFO_MAX_SIZE; + } else { + erts_process_gc_info(rp, &sz, NULL); + sz += 3; + } + + hp = HAlloc(BIF_P, sz); + res = erts_process_gc_info(rp, &actual_sz, &hp); + + /* We may have some extra space, fill with 0 tuples */ + if (actual_sz <= sz - 3) { + for (; actual_sz < sz - 3; hp++, actual_sz++) + hp[0] = make_arityval(0); + } else { + for (; actual_sz < sz; hp++, actual_sz++) + hp[0] = make_arityval(0); + hp = HAlloc(BIF_P, 3); + } + + break; + } + case am_group_leader: { int sz = NC_HEAP_SIZE(rp->group_leader); hp = HAlloc(BIF_P, 3 + sz); @@ -1562,6 +1527,25 @@ process_info_aux(Process *BIF_P, break; } + case am_message_queue_data: + switch (rp->flags & (F_OFF_HEAP_MSGQ|F_ON_HEAP_MSGQ)) { + case F_OFF_HEAP_MSGQ: + res = am_off_heap; + break; + case F_ON_HEAP_MSGQ: + res = am_on_heap; + break; + case 0: + res = am_mixed; + break; + default: + res = am_error; + ERTS_INTERNAL_ERROR("Inconsistent message queue management state"); + break; + } + hp = HAlloc(BIF_P, 3); + break; + default: return THE_NON_VALUE; /* will produce badarg */ @@ -1736,12 +1720,12 @@ info_1_tuple(Process* BIF_P, /* Pointer to current process. */ if (arity == 2) { Eterm res = THE_NON_VALUE; char *buf; - int len = is_string(*tp); + Sint len = is_string(*tp); if (len <= 0) return res; buf = (char *) erts_alloc(ERTS_ALC_T_TMP, len+1); if (intlist_to_buf(*tp, buf, len) != len) - erl_exit(1, "%s:%d: Internal error\n", __FILE__, __LINE__); + erts_exit(ERTS_ERROR_EXIT, "%s:%d: Internal error\n", __FILE__, __LINE__); buf[len] = '\0'; res = erts_instr_dump_memory_map(buf) ? am_true : am_false; erts_free(ERTS_ALC_T_TMP, (void *) buf); @@ -1755,12 +1739,12 @@ info_1_tuple(Process* BIF_P, /* Pointer to current process. */ else { Eterm res = THE_NON_VALUE; char *buf; - int len = is_string(tp[1]); + Sint len = is_string(tp[1]); if (len <= 0) return res; buf = (char *) erts_alloc(ERTS_ALC_T_TMP, len+1); if (intlist_to_buf(tp[1], buf, len) != len) - erl_exit(1, "%s:%d: Internal error\n", __FILE__, __LINE__); + erts_exit(ERTS_ERROR_EXIT, "%s:%d: Internal error\n", __FILE__, __LINE__); buf[len] = '\0'; res = erts_instr_dump_stat(buf, 1) ? am_true : am_false; erts_free(ERTS_ALC_T_TMP, (void *) buf); @@ -2058,12 +2042,18 @@ BIF_RETTYPE system_info_1(BIF_ALIST_1) #ifndef ERTS_SMP BIF_RET(am_disabled); #else +#ifndef ERTS_DIRTY_SCHEDULERS if (erts_no_schedulers == 1) BIF_RET(am_disabled); - else { - BIF_RET(erts_is_multi_scheduling_blocked() - ? am_blocked - : am_enabled); + else +#endif + { + int msb = erts_is_multi_scheduling_blocked(); + BIF_RET(!msb + ? am_enabled + : (msb > 0 + ? am_blocked + : am_blocked_normal)); } #endif } else if (BIF_ARG_1 == am_build_type) { @@ -2098,6 +2088,50 @@ BIF_RETTYPE system_info_1(BIF_ALIST_1) BIF_RET(am_opt); #endif BIF_RET(res); + } else if (BIF_ARG_1 == am_time_offset) { + switch (erts_time_offset_state()) { + case ERTS_TIME_OFFSET_PRELIMINARY: { + ERTS_DECL_AM(preliminary); + BIF_RET(AM_preliminary); + } + case ERTS_TIME_OFFSET_FINAL: { + ERTS_DECL_AM(final); + BIF_RET(AM_final); + } + case ERTS_TIME_OFFSET_VOLATILE: { + ERTS_DECL_AM(volatile); + BIF_RET(AM_volatile); + } + default: + ERTS_INTERNAL_ERROR("Invalid time offset state"); + } + } else if (ERTS_IS_ATOM_STR("os_monotonic_time_source", BIF_ARG_1)) { + BIF_RET(erts_monotonic_time_source(BIF_P)); + } else if (ERTS_IS_ATOM_STR("os_system_time_source", BIF_ARG_1)) { + BIF_RET(erts_system_time_source(BIF_P)); + } else if (ERTS_IS_ATOM_STR("time_correction", BIF_ARG_1)) { + BIF_RET(erts_has_time_correction() ? am_true : am_false); + } else if (ERTS_IS_ATOM_STR("start_time", BIF_ARG_1)) { + BIF_RET(erts_get_monotonic_start_time(BIF_P)); + } else if (ERTS_IS_ATOM_STR("end_time", BIF_ARG_1)) { + BIF_RET(erts_get_monotonic_end_time(BIF_P)); + } else if (ERTS_IS_ATOM_STR("time_warp_mode", BIF_ARG_1)) { + switch (erts_time_warp_mode()) { + case ERTS_NO_TIME_WARP_MODE: { + ERTS_DECL_AM(no_time_warp); + BIF_RET(AM_no_time_warp); + } + case ERTS_SINGLE_TIME_WARP_MODE: { + ERTS_DECL_AM(single_time_warp); + BIF_RET(AM_single_time_warp); + } + case ERTS_MULTI_TIME_WARP_MODE: { + ERTS_DECL_AM(multi_time_warp); + BIF_RET(AM_multi_time_warp); + } + default: + ERTS_INTERNAL_ERROR("Invalid time warp mode"); + } } else if (BIF_ARG_1 == am_allocated_areas) { res = erts_allocated_areas(NULL, NULL, BIF_P); BIF_RET(res); @@ -2300,7 +2334,7 @@ BIF_RETTYPE system_info_1(BIF_ALIST_1) for (i = num_instructions-1; i >= 0; i--) { res = erts_bld_cons(hpp, hszp, erts_bld_tuple(hpp, hszp, 2, - erts_atom_put(opc[i].name, + erts_atom_put((byte *)opc[i].name, strlen(opc[i].name), ERTS_ATOM_ENC_LATIN1, 1), @@ -2459,6 +2493,13 @@ BIF_RETTYPE system_info_1(BIF_ALIST_1) ERL_DRV_EXTENDED_MINOR_VERSION); hp = HAlloc(BIF_P, 2*n); BIF_RET(buf_to_intlist(&hp, buf, n, NIL)); + } else if (ERTS_IS_ATOM_STR("nif_version", BIF_ARG_1)) { + char buf[42]; + int n = erts_snprintf(buf, 42, "%d.%d", + ERL_NIF_MAJOR_VERSION, + ERL_NIF_MINOR_VERSION); + hp = HAlloc(BIF_P, 2*n); + BIF_RET(buf_to_intlist(&hp, buf, n, NIL)); } else if (ERTS_IS_ATOM_STR("smp_support", BIF_ARG_1)) { #ifdef ERTS_SMP BIF_RET(am_true); @@ -2481,77 +2522,120 @@ BIF_RETTYPE system_info_1(BIF_ALIST_1) res = TUPLE3(hp, make_small(1), make_small(1), make_small(1)); BIF_RET(res); #else + Eterm *hp; Uint total, online, active; - switch (erts_schedulers_state(&total, - &online, - &active, - NULL, - NULL, - NULL, - 1)) { - case ERTS_SCHDLR_SSPND_DONE: { - Eterm *hp = HAlloc(BIF_P, 4); - res = TUPLE3(hp, - make_small(total), - make_small(online), - make_small(active)); - BIF_RET(res); + erts_schedulers_state(&total, &online, &active, + NULL, NULL, NULL, NULL, NULL); + hp = HAlloc(BIF_P, 4); + res = TUPLE3(hp, + make_small(total), + make_small(online), + make_small(active)); + BIF_RET(res); +#endif + } else if (ERTS_IS_ATOM_STR("schedulers_state", BIF_ARG_1)) { +#ifndef ERTS_SMP + Eterm *hp = HAlloc(BIF_P, 4); + res = TUPLE3(hp, make_small(1), make_small(1), make_small(1)); + BIF_RET(res); +#else + Eterm *hp; + Uint total, online, active; + erts_schedulers_state(&total, &online, &active, + NULL, NULL, NULL, NULL, NULL); + hp = HAlloc(BIF_P, 4); + res = TUPLE3(hp, + make_small(total), + make_small(online), + make_small(active)); + BIF_RET(res); +#endif + } else if (ERTS_IS_ATOM_STR("all_schedulers_state", BIF_ARG_1)) { +#ifndef ERTS_SMP + Eterm *hp = HAlloc(BIF_P, 2+5); + res = CONS(hp+5, + TUPLE4(hp, + am_normal, + make_small(1), + make_small(1), + make_small(1)), + NIL); + BIF_RET(res); +#else + Eterm *hp, tpl; + Uint sz, total, online, active, + dirty_cpu_total, dirty_cpu_online, dirty_cpu_active, + dirty_io_total, dirty_io_active; + erts_schedulers_state(&total, &online, &active, + &dirty_cpu_total, &dirty_cpu_online, &dirty_cpu_active, + &dirty_io_total, &dirty_io_active); + + sz = 2+5; + if (dirty_cpu_total) + sz += 2+5; + if (dirty_io_total) + sz += 2+5; + + hp = HAlloc(BIF_P, sz); + + res = NIL; + if (dirty_io_total) { + tpl = TUPLE4(hp, + am_dirty_io, + make_small(dirty_io_total), + make_small(dirty_io_total), + make_small(dirty_io_active)); + hp += 5; + res = CONS(hp, tpl, res); + hp += 2; } - case ERTS_SCHDLR_SSPND_YIELD_RESTART: - ERTS_VBUMP_ALL_REDS(BIF_P); - BIF_TRAP1(bif_export[BIF_system_info_1], - BIF_P, BIF_ARG_1); - default: - ASSERT(0); - BIF_ERROR(BIF_P, EXC_INTERNAL_ERROR); + if (dirty_cpu_total) { + tpl = TUPLE4(hp, + am_dirty_cpu, + make_small(dirty_cpu_total), + make_small(dirty_cpu_online), + make_small(dirty_cpu_active)); + hp += 5; + res = CONS(hp, tpl, res); + hp += 2; } + tpl = TUPLE4(hp, + am_normal, + make_small(total), + make_small(online), + make_small(active)); + hp += 5; + res = CONS(hp, tpl, res); + BIF_RET(res); #endif } else if (ERTS_IS_ATOM_STR("schedulers_online", BIF_ARG_1)) { #ifndef ERTS_SMP BIF_RET(make_small(1)); #else - Uint total, online, active; - switch (erts_schedulers_state(&total, &online, &active, NULL, NULL, NULL, 1)) { - case ERTS_SCHDLR_SSPND_DONE: - BIF_RET(make_small(online)); - case ERTS_SCHDLR_SSPND_YIELD_RESTART: - ERTS_VBUMP_ALL_REDS(BIF_P); - BIF_TRAP1(bif_export[BIF_system_info_1], - BIF_P, BIF_ARG_1); - default: - ASSERT(0); - BIF_ERROR(BIF_P, EXC_INTERNAL_ERROR); - } + Uint online; + erts_schedulers_state(NULL, &online, NULL, NULL, NULL, NULL, NULL, NULL); + BIF_RET(make_small(online)); #endif } else if (ERTS_IS_ATOM_STR("schedulers_active", BIF_ARG_1)) { #ifndef ERTS_SMP BIF_RET(make_small(1)); #else - Uint total, online, active; - switch (erts_schedulers_state(&total, &online, &active, NULL, NULL, NULL, 1)) { - case ERTS_SCHDLR_SSPND_DONE: - BIF_RET(make_small(active)); - case ERTS_SCHDLR_SSPND_YIELD_RESTART: - ERTS_VBUMP_ALL_REDS(BIF_P); - BIF_TRAP1(bif_export[BIF_system_info_1], - BIF_P, BIF_ARG_1); - default: - ASSERT(0); - BIF_ERROR(BIF_P, EXC_INTERNAL_ERROR); - } + Uint active; + erts_schedulers_state(NULL, NULL, &active, NULL, NULL, NULL, NULL, NULL); + BIF_RET(make_small(active)); #endif #if defined(ERTS_SMP) && defined(ERTS_DIRTY_SCHEDULERS) } else if (ERTS_IS_ATOM_STR("dirty_cpu_schedulers", BIF_ARG_1)) { Uint dirty_cpu; - erts_schedulers_state(NULL, NULL, NULL, &dirty_cpu, NULL, NULL, 1); + erts_schedulers_state(NULL, NULL, NULL, &dirty_cpu, NULL, NULL, NULL, NULL); BIF_RET(make_small(dirty_cpu)); } else if (ERTS_IS_ATOM_STR("dirty_cpu_schedulers_online", BIF_ARG_1)) { Uint dirty_cpu_onln; - erts_schedulers_state(NULL, NULL, NULL, NULL, &dirty_cpu_onln, NULL, 1); + erts_schedulers_state(NULL, NULL, NULL, NULL, &dirty_cpu_onln, NULL, NULL, NULL); BIF_RET(make_small(dirty_cpu_onln)); } else if (ERTS_IS_ATOM_STR("dirty_io_schedulers", BIF_ARG_1)) { Uint dirty_io; - erts_schedulers_state(NULL, NULL, NULL, NULL, NULL, &dirty_io, 1); + erts_schedulers_state(NULL, NULL, NULL, NULL, NULL, NULL, &dirty_io, NULL); BIF_RET(make_small(dirty_io)); #endif } else if (ERTS_IS_ATOM_STR("run_queues", BIF_ARG_1)) { @@ -2605,7 +2689,16 @@ BIF_RETTYPE system_info_1(BIF_ALIST_1) if (erts_no_schedulers == 1) BIF_RET(NIL); else - BIF_RET(erts_multi_scheduling_blockers(BIF_P)); + BIF_RET(erts_multi_scheduling_blockers(BIF_P, 0)); +#endif + } else if (ERTS_IS_ATOM_STR("normal_multi_scheduling_blockers", BIF_ARG_1)) { +#ifndef ERTS_SMP + BIF_RET(NIL); +#else + if (erts_no_schedulers == 1) + BIF_RET(NIL); + else + BIF_RET(erts_multi_scheduling_blockers(BIF_P, 1)); #endif } else if (ERTS_IS_ATOM_STR("modified_timing_level", BIF_ARG_1)) { BIF_RET(ERTS_USE_MODIFIED_TIMING() @@ -2630,6 +2723,15 @@ BIF_RETTYPE system_info_1(BIF_ALIST_1) hp = hsz ? HAlloc(BIF_P, hsz) : NULL; res = erts_bld_uint(&hp, NULL, erts_dist_buf_busy_limit); BIF_RET(res); + } else if (ERTS_IS_ATOM_STR("delayed_node_table_gc", BIF_ARG_1)) { + Uint hsz = 0; + Uint dntgc = erts_delayed_node_table_gc(); + if (dntgc == ERTS_NODE_TAB_DELAY_GC_INFINITY) + BIF_RET(am_infinity); + (void) erts_bld_uint(NULL, &hsz, dntgc); + hp = hsz ? HAlloc(BIF_P, hsz) : NULL; + res = erts_bld_uint(&hp, NULL, dntgc); + BIF_RET(res); } else if (ERTS_IS_ATOM_STR("ethread_info", BIF_ARG_1)) { BIF_RET(erts_get_ethread_info(BIF_P)); } @@ -2663,6 +2765,19 @@ BIF_RETTYPE system_info_1(BIF_ALIST_1) BIF_RET(am_true); } #endif + else if (BIF_ARG_1 == am_message_queue_data) { + switch (erts_default_spo_flags & (SPO_ON_HEAP_MSGQ|SPO_OFF_HEAP_MSGQ)) { + case SPO_OFF_HEAP_MSGQ: + BIF_RET(am_off_heap); + case SPO_ON_HEAP_MSGQ: + BIF_RET(am_on_heap); + case 0: + BIF_RET(am_mixed); + default: + ERTS_INTERNAL_ERROR("Inconsistent message queue management state"); + BIF_RET(am_error); + } + } else if (ERTS_IS_ATOM_STR("compile_info",BIF_ARG_1)) { Uint sz; Eterm res = NIL, tup, text; @@ -2691,6 +2806,30 @@ BIF_RETTYPE system_info_1(BIF_ALIST_1) else if (ERTS_IS_ATOM_STR("ets_limit",BIF_ARG_1)) { BIF_RET(make_small(erts_db_get_max_tabs())); } + else if (ERTS_IS_ATOM_STR("tolerant_timeofday",BIF_ARG_1)) { + if (erts_has_time_correction() + && erts_time_offset_state() == ERTS_TIME_OFFSET_FINAL) { + BIF_RET(am_enabled); + } + BIF_RET(am_disabled); + } + else if (ERTS_IS_ATOM_STR("eager_check_io",BIF_ARG_1)) { + BIF_RET(erts_eager_check_io ? am_true : am_false); + } + else if (ERTS_IS_ATOM_STR("literal_test",BIF_ARG_1)) { +#ifdef ERTS_HAVE_IS_IN_LITERAL_RANGE +#ifdef ARCH_64 + DECL_AM(range); + BIF_RET(AM_range); +#else /* ARCH_32 */ + DECL_AM(range_bitmask); + BIF_RET(AM_range_bitmask); +#endif /* ARCH_32 */ +#else /* ! ERTS_HAVE_IS_IN_LITERAL_RANGE */ + DECL_AM(tag); + BIF_RET(AM_tag); +#endif + } BIF_ERROR(BIF_P, BADARG); } @@ -3050,6 +3189,25 @@ fun_info_2(BIF_ALIST_2) return TUPLE2(hp, what, val); } +BIF_RETTYPE +fun_info_mfa_1(BIF_ALIST_1) +{ + Process* p = BIF_P; + Eterm fun = BIF_ARG_1; + Eterm* hp; + + if (is_fun(fun)) { + ErlFunThing* funp = (ErlFunThing *) fun_val(fun); + hp = HAlloc(p, 4); + BIF_RET(TUPLE3(hp,funp->fe->module,funp->fe->address[-2],make_small(funp->arity))); + } else if (is_export(fun)) { + Export* exp = (Export *) ((UWord) (export_val(fun))[1]); + hp = HAlloc(p, 4); + BIF_RET(TUPLE3(hp,exp->code[0],exp->code[1],make_small(exp->code[2]))); + } + BIF_ERROR(p, BADARG); +} + BIF_RETTYPE is_process_alive_1(BIF_ALIST_1) { if(is_internal_pid(BIF_ARG_1)) { @@ -3115,7 +3273,6 @@ BIF_RETTYPE process_display_2(BIF_ALIST_2) BIF_RET(am_true); } - /* this is a general call which return some possibly useful information */ BIF_RETTYPE statistics_1(BIF_ALIST_1) @@ -3128,6 +3285,47 @@ BIF_RETTYPE statistics_1(BIF_ALIST_1) if (is_non_value(res)) BIF_RET(am_undefined); BIF_TRAP1(gather_sched_wall_time_res_trap, BIF_P, res); + } else if (BIF_ARG_1 == am_total_active_tasks + || BIF_ARG_1 == am_total_run_queue_lengths) { + Uint no = erts_run_queues_len(NULL, 0, BIF_ARG_1 == am_total_active_tasks); + if (IS_USMALL(0, no)) + res = make_small(no); + else { + Eterm *hp = HAlloc(BIF_P, BIG_UINT_HEAP_SIZE); + res = uint_to_big(no, hp); + } + BIF_RET(res); + } else if (BIF_ARG_1 == am_active_tasks + || BIF_ARG_1 == am_run_queue_lengths) { + Eterm res, *hp, **hpp; + Uint sz, *szp; + int no_qs = erts_no_run_queues; + Uint *qszs = erts_alloc(ERTS_ALC_T_TMP,sizeof(Uint)*no_qs*2); + (void) erts_run_queues_len(qszs, 0, BIF_ARG_1 == am_active_tasks); + sz = 0; + szp = &sz; + hpp = NULL; + while (1) { + int i; + for (i = 0; i < no_qs; i++) + qszs[no_qs+i] = erts_bld_uint(hpp, szp, qszs[i]); + res = erts_bld_list(hpp, szp, no_qs, &qszs[no_qs]); + if (hpp) { + erts_free(ERTS_ALC_T_TMP, qszs); + BIF_RET(res); + } + hp = HAlloc(BIF_P, sz); + szp = NULL; + hpp = &hp; + } +#ifdef ERTS_ENABLE_MSACC + } else if (BIF_ARG_1 == am_microstate_accounting) { + Eterm threads; + res = erts_msacc_request(BIF_P, ERTS_MSACC_GATHER, &threads); + if (is_non_value(res)) + BIF_RET(am_undefined); + BIF_TRAP2(gather_msacc_res_trap, BIF_P, res, threads); +#endif } else if (BIF_ARG_1 == am_context_switches) { Eterm cs = erts_make_integer(erts_get_total_context_switches(), BIF_P); hp = HAlloc(BIF_P, 3); @@ -3176,7 +3374,7 @@ BIF_RETTYPE statistics_1(BIF_ALIST_1) res = TUPLE2(hp, b1, b2); BIF_RET(res); } else if (BIF_ARG_1 == am_run_queue) { - res = erts_run_queues_len(NULL); + res = erts_run_queues_len(NULL, 1, 0); BIF_RET(make_small(res)); } else if (BIF_ARG_1 == am_wall_clock) { UWord w1, w2; @@ -3188,30 +3386,15 @@ BIF_RETTYPE statistics_1(BIF_ALIST_1) res = TUPLE2(hp, b1, b2); BIF_RET(res); } else if (BIF_ARG_1 == am_io) { - Eterm r1, r2; - Eterm in, out; - Uint hsz = 9; - Uint bytes_in = (Uint) erts_smp_atomic_read_nob(&erts_bytes_in); - Uint bytes_out = (Uint) erts_smp_atomic_read_nob(&erts_bytes_out); - - (void) erts_bld_uint(NULL, &hsz, bytes_in); - (void) erts_bld_uint(NULL, &hsz, bytes_out); - hp = HAlloc(BIF_P, hsz); - in = erts_bld_uint(&hp, NULL, bytes_in); - out = erts_bld_uint(&hp, NULL, bytes_out); - - r1 = TUPLE2(hp, am_input, in); - hp += 3; - r2 = TUPLE2(hp, am_output, out); - hp += 3; - BIF_RET(TUPLE2(hp, r1, r2)); + Eterm ref = erts_request_io_bytes(BIF_P); + BIF_TRAP2(gather_io_bytes_trap, BIF_P, ref, make_small(erts_no_schedulers)); } else if (ERTS_IS_ATOM_STR("run_queues", BIF_ARG_1)) { Eterm res, *hp, **hpp; Uint sz, *szp; int no_qs = erts_no_run_queues; Uint *qszs = erts_alloc(ERTS_ALC_T_TMP,sizeof(Uint)*no_qs*2); - (void) erts_run_queues_len(qszs); + (void) erts_run_queues_len(qszs, 0, 0); sz = 0; szp = &sz; hpp = NULL; @@ -3280,17 +3463,38 @@ BIF_RETTYPE erts_debug_get_internal_state_1(BIF_ALIST_1) BIF_RET(make_small((Uint) words)); } else if (ERTS_IS_ATOM_STR("check_io_debug", BIF_ARG_1)) { - /* Used by (emulator) */ - int res; + /* Used by driver_SUITE (emulator) */ + Uint sz, *szp; + Eterm res, *hp, **hpp; + int no_errors; + ErtsCheckIoDebugInfo ciodi = {0}; #ifdef HAVE_ERTS_CHECK_IO_DEBUG erts_smp_proc_unlock(BIF_P,ERTS_PROC_LOCK_MAIN); - res = erts_check_io_debug(); + no_errors = erts_check_io_debug(&ciodi); erts_smp_proc_lock(BIF_P,ERTS_PROC_LOCK_MAIN); #else - res = 0; + no_errors = 0; #endif - ASSERT(res >= 0); - BIF_RET(erts_make_integer((Uint) res, BIF_P)); + sz = 0; + szp = &sz; + hpp = NULL; + while (1) { + res = erts_bld_tuple(hpp, szp, 4, + erts_bld_uint(hpp, szp, + (Uint) no_errors), + erts_bld_uint(hpp, szp, + (Uint) ciodi.no_used_fds), + erts_bld_uint(hpp, szp, + (Uint) ciodi.no_driver_select_structs), + erts_bld_uint(hpp, szp, + (Uint) ciodi.no_driver_event_structs)); + if (hpp) + break; + hp = HAlloc(BIF_P, sz); + szp = NULL; + hpp = &hp; + } + BIF_RET(res); } else if (ERTS_IS_ATOM_STR("process_info_args", BIF_ARG_1)) { /* Used by process_SUITE (emulator) */ @@ -3342,8 +3546,31 @@ BIF_RETTYPE erts_debug_get_internal_state_1(BIF_ALIST_1) BIF_RET(res); } else if (ERTS_IS_ATOM_STR("mmap", BIF_ARG_1)) { - BIF_RET(erts_mmap_debug_info(BIF_P)); + BIF_RET(erts_mmap_debug_info(&erts_dflt_mmapper, BIF_P)); } + else if (ERTS_IS_ATOM_STR("unique_monotonic_integer_state", BIF_ARG_1)) { + BIF_RET(erts_debug_get_unique_monotonic_integer_state(BIF_P)); + } + else if (ERTS_IS_ATOM_STR("min_unique_monotonic_integer", BIF_ARG_1)) { + Sint64 value = erts_get_min_unique_monotonic_integer(); + if (IS_SSMALL(value)) + BIF_RET(make_small(value)); + else { + Uint hsz = ERTS_SINT64_HEAP_SIZE(value); + Eterm *hp = HAlloc(BIF_P, hsz); + BIF_RET(erts_sint64_to_big(value, &hp)); + } + } + else if (ERTS_IS_ATOM_STR("min_unique_integer", BIF_ARG_1)) { + Sint64 value = erts_get_min_unique_integer(); + if (IS_SSMALL(value)) + BIF_RET(make_small(value)); + else { + Uint hsz = ERTS_SINT64_HEAP_SIZE(value); + Eterm *hp = HAlloc(BIF_P, hsz); + BIF_RET(erts_sint64_to_big(value, &hp)); + } + } } else if (is_tuple(BIF_ARG_1)) { Eterm* tp = tuple_val(BIF_ARG_1); @@ -3538,6 +3765,58 @@ BIF_RETTYPE erts_debug_get_internal_state_1(BIF_ALIST_1) BIF_RET(erts_debug_reader_groups_map(BIF_P, (int) groups)); } + else if (ERTS_IS_ATOM_STR("internal_hash", tp[1])) { + Uint hash = (Uint) make_internal_hash(tp[2]); + Uint hsz = 0; + Eterm* hp; + erts_bld_uint(NULL, &hsz, hash); + hp = HAlloc(BIF_P,hsz); + return erts_bld_uint(&hp, NULL, hash); + } + else if (ERTS_IS_ATOM_STR("atom", tp[1])) { + Uint ix; + if (!term_to_Uint(tp[2], &ix)) + BIF_ERROR(BIF_P, BADARG); + while (ix >= atom_table_size()) { + char tmp[20]; + erts_snprintf(tmp, sizeof(tmp), "am%x", atom_table_size()); + erts_atom_put((byte *) tmp, strlen(tmp), ERTS_ATOM_ENC_LATIN1, 1); + } + return make_atom(ix); + } + + break; + } + case 3: { + if (ERTS_IS_ATOM_STR("check_time_config", tp[1])) { + int res, time_correction; + ErtsTimeWarpMode time_warp_mode; + if (tp[2] == am_true) + time_correction = !0; + else if (tp[2] == am_false) + time_correction = 0; + else + break; + if (ERTS_IS_ATOM_STR("no_time_warp", tp[3])) + time_warp_mode = ERTS_NO_TIME_WARP_MODE; + else if (ERTS_IS_ATOM_STR("single_time_warp", tp[3])) + time_warp_mode = ERTS_SINGLE_TIME_WARP_MODE; + else if (ERTS_IS_ATOM_STR("multi_time_warp", tp[3])) + time_warp_mode = ERTS_MULTI_TIME_WARP_MODE; + else + break; + res = erts_check_time_adj_support(time_correction, + time_warp_mode); + BIF_RET(res ? am_true : am_false); + } + else if (ERTS_IS_ATOM_STR("make_unique_integer", tp[1])) { + Eterm res = erts_debug_make_unique_integer(BIF_P, + tp[2], + tp[3]); + if (is_non_value(res)) + break; + BIF_RET(res); + } break; } default: @@ -3547,8 +3826,52 @@ BIF_RETTYPE erts_debug_get_internal_state_1(BIF_ALIST_1) BIF_ERROR(BIF_P, BADARG); } +BIF_RETTYPE erts_internal_is_system_process_1(BIF_ALIST_1) +{ + if (is_internal_pid(BIF_ARG_1)) { + Process *rp = erts_proc_lookup(BIF_ARG_1); + if (rp && (rp->static_flags & ERTS_STC_FLG_SYSTEM_PROC)) + BIF_RET(am_true); + BIF_RET(am_false); + } + + if (is_external_pid(BIF_ARG_1) + && external_pid_dist_entry(BIF_ARG_1) == erts_this_dist_entry) { + BIF_RET(am_false); + } + + BIF_ERROR(BIF_P, BADARG); +} + +BIF_RETTYPE erts_internal_system_check_1(BIF_ALIST_1) +{ + Eterm res; + if (ERTS_IS_ATOM_STR("schedulers", BIF_ARG_1)) { + res = erts_system_check_request(BIF_P); + if (is_non_value(res)) + BIF_RET(am_undefined); + BIF_TRAP1(gather_system_check_res_trap, BIF_P, res); + } + + BIF_ERROR(BIF_P, BADARG); +} + static erts_smp_atomic_t hipe_test_reschedule_flag; +#if defined(VALGRIND) && defined(__GNUC__) +/* Force noinline for valgrind suppression */ +static void broken_halt_test(Eterm bif_arg_2) __attribute__((noinline)); +#endif + +static void broken_halt_test(Eterm bif_arg_2) +{ + /* Ugly ugly code used by bif_SUITE:erlang_halt/1 */ +#if defined(ERTS_HAVE_TRY_CATCH) + erts_get_scheduler_data()->run_queue = NULL; +#endif + erts_exit(ERTS_DUMP_EXIT, "%T", bif_arg_2); +} + BIF_RETTYPE erts_debug_set_internal_state_2(BIF_ALIST_2) { @@ -3645,9 +3968,7 @@ BIF_RETTYPE erts_debug_set_internal_state_2(BIF_ALIST_2) BIF_RET(am_false); } else { - FLAGS(rp) |= F_FORCE_GC; - if (BIF_P != rp) - erts_smp_proc_unlock(rp, ERTS_PROC_LOCK_MAIN); + ERTS_FORCE_GC(BIF_P); BIF_RET(am_true); } } @@ -3793,7 +4114,7 @@ BIF_RETTYPE erts_debug_set_internal_state_2(BIF_ALIST_2) BIF_RET(am_true); } else if (ERTS_IS_ATOM_STR("abort", BIF_ARG_1)) { - erl_exit(ERTS_ABORT_EXIT, "%T\n", BIF_ARG_2); + erts_exit(ERTS_ABORT_EXIT, "%T\n", BIF_ARG_2); } else if (ERTS_IS_ATOM_STR("kill_dist_connection", BIF_ARG_1)) { DistEntry *dep = erts_sysname_to_connected_dist_entry(BIF_ARG_2); @@ -3836,11 +4157,47 @@ BIF_RETTYPE erts_debug_set_internal_state_2(BIF_ALIST_2) } else if (ERTS_IS_ATOM_STR("wait", BIF_ARG_1)) { if (ERTS_IS_ATOM_STR("deallocations", BIF_ARG_2)) { - if (erts_debug_wait_deallocations(BIF_P)) { + int flag = ERTS_DEBUG_WAIT_COMPLETED_DEALLOCATIONS; + if (erts_debug_wait_completed(BIF_P, flag)) { ERTS_BIF_YIELD_RETURN(BIF_P, am_ok); } } + if (ERTS_IS_ATOM_STR("timer_cancellations", BIF_ARG_2)) { + int flag = ERTS_DEBUG_WAIT_COMPLETED_TIMER_CANCELLATIONS; + if (erts_debug_wait_completed(BIF_P, flag)) { + ERTS_BIF_YIELD_RETURN(BIF_P, am_ok); + } + } + } + else if (ERTS_IS_ATOM_STR("broken_halt", BIF_ARG_1)) { + broken_halt_test(BIF_ARG_2); + } + else if (ERTS_IS_ATOM_STR("unique_monotonic_integer_state", BIF_ARG_1)) { + int res = erts_debug_set_unique_monotonic_integer_state(BIF_ARG_2); + BIF_RET(res ? am_true : am_false); + } + else if (ERTS_IS_ATOM_STR("node_tab_delayed_delete", BIF_ARG_1)) { + /* node_container_SUITE */ + Sint64 msecs; + if (term_to_Sint64(BIF_ARG_2, &msecs)) { + /* Negative value restore original value... */ + erts_smp_proc_unlock(BIF_P, ERTS_PROC_LOCK_MAIN); + erts_debug_test_node_tab_delayed_delete(msecs); + erts_smp_proc_lock(BIF_P, ERTS_PROC_LOCK_MAIN); + BIF_RET(am_ok); + } } + else if (ERTS_IS_ATOM_STR("fill_heap", BIF_ARG_1)) { + UWord left = HeapWordsLeft(BIF_P); + if (left > 1) { + Eterm* hp = HAlloc(BIF_P, left); + *hp = make_pos_bignum_header(left - 1); + } + if (BIF_ARG_2 == am_true) { + FLAGS(BIF_P) |= F_NEED_FULLSWEEP; + } + BIF_RET(am_ok); + } } BIF_ERROR(BIF_P, BADARG); @@ -3851,16 +4208,19 @@ static Eterm lcnt_build_lock_stats_term(Eterm **hpp, Uint *szp, erts_lcnt_lock_s Uint tries = 0, colls = 0; unsigned long timer_s = 0, timer_ns = 0, timer_n = 0; unsigned int line = 0; + unsigned int i; Eterm af, uil; Eterm uit, uic; Eterm uits, uitns, uitn; Eterm tt, tstat, tloc, t; + Eterm thist, vhist[ERTS_LCNT_HISTOGRAM_SLOT_SIZE]; /* term: - * [{{file, line}, {tries, colls, {seconds, nanoseconds, n_blocks}}}] + * [{{file, line}, {tries, colls, {seconds, nanoseconds, n_blocks}}, + * { .. histogram .. }] */ - + tries = (Uint) ethr_atomic_read(&stats->tries); colls = (Uint) ethr_atomic_read(&stats->colls); @@ -3869,23 +4229,27 @@ static Eterm lcnt_build_lock_stats_term(Eterm **hpp, Uint *szp, erts_lcnt_lock_s timer_ns = stats->timer.ns; timer_n = stats->timer_n; - af = erts_atom_put(stats->file, strlen(stats->file), ERTS_ATOM_ENC_LATIN1, 1); + af = erts_atom_put((byte *)stats->file, strlen(stats->file), ERTS_ATOM_ENC_LATIN1, 1); uil = erts_bld_uint( hpp, szp, line); tloc = erts_bld_tuple(hpp, szp, 2, af, uil); - uit = erts_bld_uint( hpp, szp, tries); - uic = erts_bld_uint( hpp, szp, colls); - + uit = erts_bld_uint( hpp, szp, tries); + uic = erts_bld_uint( hpp, szp, colls); + uits = erts_bld_uint( hpp, szp, timer_s); uitns = erts_bld_uint( hpp, szp, timer_ns); uitn = erts_bld_uint( hpp, szp, timer_n); tt = erts_bld_tuple(hpp, szp, 3, uits, uitns, uitn); tstat = erts_bld_tuple(hpp, szp, 3, uit, uic, tt); - - t = erts_bld_tuple(hpp, szp, 2, tloc, tstat); - - res = erts_bld_cons( hpp, szp, t, res); + + for(i = 0; i < ERTS_LCNT_HISTOGRAM_SLOT_SIZE; i++) { + vhist[i] = erts_bld_uint(hpp, szp, stats->hist.ns[i]); + } + thist = erts_bld_tuplev(hpp, szp, ERTS_LCNT_HISTOGRAM_SLOT_SIZE, vhist); + + t = erts_bld_tuple(hpp, szp, 3, tloc, tstat, thist); + res = erts_bld_cons( hpp, szp, t, res); return res; } @@ -3906,13 +4270,13 @@ static Eterm lcnt_build_lock_term(Eterm **hpp, Uint *szp, erts_lcnt_lock_t *lock ASSERT(ltype); - type = erts_atom_put(ltype, strlen(ltype), ERTS_ATOM_ENC_LATIN1, 1); - name = erts_atom_put(lock->name, strlen(lock->name), ERTS_ATOM_ENC_LATIN1, 1); + type = erts_atom_put((byte *)ltype, strlen(ltype), ERTS_ATOM_ENC_LATIN1, 1); + name = erts_atom_put((byte *)lock->name, strlen(lock->name), ERTS_ATOM_ENC_LATIN1, 1); if (lock->flag & ERTS_LCNT_LT_ALLOC) { /* use allocator types names as id's for allocator locks */ ltype = (char *) ERTS_ALC_A2AD(signed_val(lock->id)); - id = erts_atom_put(ltype, strlen(ltype), ERTS_ATOM_ENC_LATIN1, 1); + id = erts_atom_put((byte *)ltype, strlen(ltype), ERTS_ATOM_ENC_LATIN1, 1); } else if (lock->flag & ERTS_LCNT_LT_PROCLOCK) { /* use registered names as id's for process locks if available */ proc = erts_proc_lookup(lock->id); @@ -3923,16 +4287,15 @@ static Eterm lcnt_build_lock_term(Eterm **hpp, Uint *szp, erts_lcnt_lock_t *lock id = lock->id; } } else { - id = lock->id; + id = lock->id; } - + for (i = 0; i < lock->n_stats; i++) { stats = lcnt_build_lock_stats_term(hpp, szp, &(lock->stats[i]), stats); } - - t = erts_bld_tuple(hpp, szp, 4, name, id, type, stats); - - res = erts_bld_cons( hpp, szp, t, res); + + t = erts_bld_tuple(hpp, szp, 4, name, id, type, stats); + res = erts_bld_cons( hpp, szp, t, res); return res; } @@ -3952,12 +4315,12 @@ static Eterm lcnt_build_result_term(Eterm **hpp, Uint *szp, erts_lcnt_data_t *da dtns = erts_bld_uint( hpp, szp, data->duration.ns); tdt = erts_bld_tuple(hpp, szp, 2, dts, dtns); - adur = erts_atom_put(str_duration, strlen(str_duration), ERTS_ATOM_ENC_LATIN1, 1); + adur = erts_atom_put((byte *)str_duration, strlen(str_duration), ERTS_ATOM_ENC_LATIN1, 1); tdur = erts_bld_tuple(hpp, szp, 2, adur, tdt); /* lock tuple */ - aloc = erts_atom_put(str_locks, strlen(str_locks), ERTS_ATOM_ENC_LATIN1, 1); + aloc = erts_atom_put((byte *)str_locks, strlen(str_locks), ERTS_ATOM_ENC_LATIN1, 1); for (lock = data->current_locks->head; lock != NULL ; lock = lock->next ) { lloc = lcnt_build_lock_term(hpp, szp, lock, lloc); @@ -4032,59 +4395,41 @@ BIF_RETTYPE erts_debug_lock_counters_1(BIF_ALIST_1) BIF_RET(am_ok); } else if (is_tuple(BIF_ARG_1)) { - Eterm* tp = tuple_val(BIF_ARG_1); - - switch (arityval(tp[0])) { - case 2: { - int opt = 0; - int val = 0; - if (ERTS_IS_ATOM_STR("copy_save", tp[1])) { - opt = ERTS_LCNT_OPT_COPYSAVE; - } else if (ERTS_IS_ATOM_STR("process_locks", tp[1])) { - opt = ERTS_LCNT_OPT_PROCLOCK; - } else if (ERTS_IS_ATOM_STR("port_locks", tp[1])) { - opt = ERTS_LCNT_OPT_PORTLOCK; - } else if (ERTS_IS_ATOM_STR("suspend", tp[1])) { - opt = ERTS_LCNT_OPT_SUSPEND; - } else if (ERTS_IS_ATOM_STR("location", tp[1])) { - opt = ERTS_LCNT_OPT_LOCATION; - } else { - BIF_ERROR(BIF_P, BADARG); - } - if (tp[2] == am_true) { - val = 1; - } else if (tp[2] == am_false) { - val = 0; - } else { - BIF_ERROR(BIF_P, BADARG); - } - - erts_smp_proc_unlock(BIF_P, ERTS_PROC_LOCK_MAIN); - erts_smp_thr_progress_block(); - - if (val) { - res = erts_lcnt_set_rt_opt(opt) ? am_true : am_false; - } else { - res = erts_lcnt_clear_rt_opt(opt) ? am_true : am_false; - } + Eterm* ptr = tuple_val(BIF_ARG_1); + + if ((arityval(ptr[0]) == 2) && (ptr[2] == am_false || ptr[2] == am_true)) { + int lock_opt = 0, enable = (ptr[2] == am_true) ? 1 : 0; + if (ERTS_IS_ATOM_STR("copy_save", ptr[1])) { + lock_opt = ERTS_LCNT_OPT_COPYSAVE; + } else if (ERTS_IS_ATOM_STR("process_locks", ptr[1])) { + lock_opt = ERTS_LCNT_OPT_PROCLOCK; + } else if (ERTS_IS_ATOM_STR("port_locks", ptr[1])) { + lock_opt = ERTS_LCNT_OPT_PORTLOCK; + } else if (ERTS_IS_ATOM_STR("suspend", ptr[1])) { + lock_opt = ERTS_LCNT_OPT_SUSPEND; + } else if (ERTS_IS_ATOM_STR("location", ptr[1])) { + lock_opt = ERTS_LCNT_OPT_LOCATION; + } else { + BIF_ERROR(BIF_P, BADARG); + } + + erts_smp_proc_unlock(BIF_P, ERTS_PROC_LOCK_MAIN); + erts_smp_thr_progress_block(); + + if (enable) res = erts_lcnt_set_rt_opt(lock_opt) ? am_true : am_false; + else res = erts_lcnt_clear_rt_opt(lock_opt) ? am_true : am_false; + #ifdef ERTS_SMP - if (res != tp[2]) { - if (opt == ERTS_LCNT_OPT_PORTLOCK) { - erts_lcnt_enable_io_lock_count(val); - } else if (opt == ERTS_LCNT_OPT_PROCLOCK) { - erts_lcnt_enable_proc_lock_count(val); - } - } + if (res != ptr[2] && lock_opt == ERTS_LCNT_OPT_PORTLOCK) { + erts_lcnt_enable_io_lock_count(enable); + } else if (res != ptr[2] && lock_opt == ERTS_LCNT_OPT_PROCLOCK) { + erts_lcnt_enable_proc_lock_count(enable); + } #endif - erts_smp_thr_progress_unblock(); - erts_smp_proc_lock(BIF_P, ERTS_PROC_LOCK_MAIN); - BIF_RET(res); - break; - } - - default: - break; - } + erts_smp_thr_progress_unblock(); + erts_smp_proc_lock(BIF_P, ERTS_PROC_LOCK_MAIN); + BIF_RET(res); + } } #endif @@ -4102,14 +4447,17 @@ static void os_info_init(void) os_flavor(buf, 1024); flav = erts_atom_put((byte *) buf, strlen(buf), ERTS_ATOM_ENC_LATIN1, 1); erts_free(ERTS_ALC_T_TMP, (void *) buf); - hp = erts_alloc(ERTS_ALC_T_LL_TEMP_TERM, (3+4)*sizeof(Eterm)); + hp = erts_alloc(ERTS_ALC_T_LITERAL, (3+4)*sizeof(Eterm)); os_type_tuple = TUPLE2(hp, type, flav); + erts_set_literal_tag(&os_type_tuple, hp, 3); + hp += 3; os_version(&major, &minor, &build); os_version_tuple = TUPLE3(hp, make_small(major), make_small(minor), make_small(build)); + erts_set_literal_tag(&os_version_tuple, hp, 4); } void @@ -4124,6 +4472,12 @@ erts_bif_info_init(void) = erts_export_put(am_erlang, am_gather_sched_wall_time_result, 1); gather_gc_info_res_trap = erts_export_put(am_erlang, am_gather_gc_info_result, 1); + gather_io_bytes_trap + = erts_export_put(am_erts_internal, am_gather_io_bytes, 2); + gather_msacc_res_trap + = erts_export_put(am_erts_internal, am_gather_microstate_accounting_result, 2); + gather_system_check_res_trap + = erts_export_put(am_erts_internal, am_gather_system_check_result, 1); process_info_init(); os_info_init(); } diff --git a/erts/emulator/beam/erl_bif_lists.c b/erts/emulator/beam/erl_bif_lists.c index 820ed2385d..fe64e76575 100644 --- a/erts/emulator/beam/erl_bif_lists.c +++ b/erts/emulator/beam/erl_bif_lists.c @@ -3,16 +3,17 @@ * * Copyright Ericsson AB 1999-2011. All Rights Reserved. * - * The contents of this file are subject to the Erlang Public License, - * Version 1.1, (the "License"); you may not use this file except in - * compliance with the License. You should have received a copy of the - * Erlang Public License along with this software. If not, it can be - * retrieved online at http://www.erlang.org/. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Software distributed under the License is distributed on an "AS IS" - * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See - * the License for the specific language governing rights and limitations - * under the License. + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. * * %CopyrightEnd% */ @@ -41,7 +42,7 @@ static BIF_RETTYPE append(Process* p, Eterm A, Eterm B) Eterm last; size_t need; Eterm* hp; - int i; + Sint i; if ((i = erts_list_length(A)) < 0) { BIF_ERROR(p, BADARG); @@ -98,9 +99,9 @@ static Eterm subtract(Process* p, Eterm A, Eterm B) Eterm small_vec[SMALL_VEC_SIZE]; /* Preallocated memory for small lists */ Eterm* vec_p; Eterm* vp; - int i; - int n; - int m; + Sint i; + Sint n; + Sint m; if ((n = erts_list_length(A)) < 0) { BIF_ERROR(p, BADARG); @@ -390,7 +391,7 @@ keyfind(int Bif, Process* p, Eterm Key, Eterm Pos, Eterm List) Eterm *tuple_ptr = tuple_val(term); if (pos <= arityval(*tuple_ptr)) { Eterm element = tuple_ptr[pos]; - if (CMP(Key, element) == 0) { + if (CMP_EQ(Key, element)) { return term; } } diff --git a/erts/emulator/beam/erl_bif_op.c b/erts/emulator/beam/erl_bif_op.c index 37dd6457db..0f20ded1d6 100644 --- a/erts/emulator/beam/erl_bif_op.c +++ b/erts/emulator/beam/erl_bif_op.c @@ -3,16 +3,17 @@ * * Copyright Ericsson AB 1999-2013. All Rights Reserved. * - * The contents of this file are subject to the Erlang Public License, - * Version 1.1, (the "License"); you may not use this file except in - * compliance with the License. You should have received a copy of the - * Erlang Public License along with this software. If not, it can be - * retrieved online at http://www.erlang.org/. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Software distributed under the License is distributed on an "AS IS" - * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See - * the License for the specific language governing rights and limitations - * under the License. + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. * * %CopyrightEnd% */ @@ -88,22 +89,22 @@ BIF_RETTYPE not_1(BIF_ALIST_1) BIF_RETTYPE sgt_2(BIF_ALIST_2) { - BIF_RET(cmp_gt(BIF_ARG_1, BIF_ARG_2) ? am_true : am_false); + BIF_RET(CMP_GT(BIF_ARG_1, BIF_ARG_2) ? am_true : am_false); } BIF_RETTYPE sge_2(BIF_ALIST_2) { - BIF_RET(cmp_ge(BIF_ARG_1, BIF_ARG_2) ? am_true : am_false); + BIF_RET(CMP_GE(BIF_ARG_1, BIF_ARG_2) ? am_true : am_false); } BIF_RETTYPE slt_2(BIF_ALIST_2) { - BIF_RET(cmp_lt(BIF_ARG_1, BIF_ARG_2) ? am_true : am_false); + BIF_RET(CMP_LT(BIF_ARG_1, BIF_ARG_2) ? am_true : am_false); } BIF_RETTYPE sle_2(BIF_ALIST_2) { - BIF_RET(cmp_le(BIF_ARG_1, BIF_ARG_2) ? am_true : am_false); + BIF_RET(CMP_LE(BIF_ARG_1, BIF_ARG_2) ? am_true : am_false); } BIF_RETTYPE seq_2(BIF_ALIST_2) @@ -113,7 +114,7 @@ BIF_RETTYPE seq_2(BIF_ALIST_2) BIF_RETTYPE seqeq_2(BIF_ALIST_2) { - BIF_RET(cmp_eq(BIF_ARG_1, BIF_ARG_2) ? am_true : am_false); + BIF_RET(CMP_EQ(BIF_ARG_1, BIF_ARG_2) ? am_true : am_false); } BIF_RETTYPE sneq_2(BIF_ALIST_2) @@ -123,7 +124,7 @@ BIF_RETTYPE sneq_2(BIF_ALIST_2) BIF_RETTYPE sneqeq_2(BIF_ALIST_2) { - BIF_RET(cmp_ne(BIF_ARG_1, BIF_ARG_2) ? am_true : am_false); + BIF_RET(CMP_NE(BIF_ARG_1, BIF_ARG_2) ? am_true : am_false); } BIF_RETTYPE is_atom_1(BIF_ALIST_1) @@ -257,7 +258,7 @@ Eterm erl_is_function(Process* p, Eterm arg1, Eterm arg2) BIF_RET(am_true); } } else if (is_export(arg1)) { - Export* exp = (Export *) EXPAND_POINTER((export_val(arg1))[1]); + Export* exp = (Export *) (export_val(arg1)[1]); if (exp->code[2] == (Uint) arity) { BIF_RET(am_true); diff --git a/erts/emulator/beam/erl_bif_os.c b/erts/emulator/beam/erl_bif_os.c index e07c622928..2333ca0851 100644 --- a/erts/emulator/beam/erl_bif_os.c +++ b/erts/emulator/beam/erl_bif_os.c @@ -3,16 +3,17 @@ * * Copyright Ericsson AB 1999-2012. All Rights Reserved. * - * The contents of this file are subject to the Erlang Public License, - * Version 1.1, (the "License"); you may not use this file except in - * compliance with the License. You should have received a copy of the - * Erlang Public License along with this software. If not, it can be - * retrieved online at http://www.erlang.org/. - * - * Software distributed under the License is distributed on an "AS IS" - * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See - * the License for the specific language governing rights and limitations - * under the License. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. * * %CopyrightEnd% */ diff --git a/erts/emulator/beam/erl_bif_port.c b/erts/emulator/beam/erl_bif_port.c index afb33c1cdb..86bcf43678 100644 --- a/erts/emulator/beam/erl_bif_port.c +++ b/erts/emulator/beam/erl_bif_port.c @@ -3,16 +3,17 @@ * * Copyright Ericsson AB 2001-2013. All Rights Reserved. * - * The contents of this file are subject to the Erlang Public License, - * Version 1.1, (the "License"); you may not use this file except in - * compliance with the License. You should have received a copy of the - * Erlang Public License along with this software. If not, it can be - * retrieved online at http://www.erlang.org/. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Software distributed under the License is distributed on an "AS IS" - * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See - * the License for the specific language governing rights and limitations - * under the License. + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. * * %CopyrightEnd% */ @@ -40,6 +41,7 @@ #include "external.h" #include "packet_parser.h" #include "erl_bits.h" +#include "erl_bif_unique.h" #include "dtrace-wrapper.h" static Port *open_port(Process* p, Eterm name, Eterm settings, int *err_typep, int *err_nump); @@ -49,10 +51,10 @@ static void free_args(char **); char *erts_default_arg0 = "default"; -BIF_RETTYPE open_port_2(BIF_ALIST_2) +BIF_RETTYPE erts_internal_open_port_2(BIF_ALIST_2) { Port *port; - Eterm port_id; + Eterm res; char *str; int err_type, err_num; @@ -60,27 +62,58 @@ BIF_RETTYPE open_port_2(BIF_ALIST_2) if (!port) { if (err_type == -3) { ASSERT(err_num == BADARG || err_num == SYSTEM_LIMIT); - BIF_ERROR(BIF_P, err_num); + if (err_num == BADARG) + res = am_badarg; + else if (err_num == SYSTEM_LIMIT) + res = am_system_limit; + else + /* this is only here to silence gcc, it should not happen */ + BIF_ERROR(BIF_P, EXC_INTERNAL_ERROR); } else if (err_type == -2) { str = erl_errno_id(err_num); + res = erts_atom_put((byte *) str, strlen(str), ERTS_ATOM_ENC_LATIN1, 1); } else { - str = "einval"; + res = am_einval; } - BIF_P->fvalue = erts_atom_put((byte *) str, strlen(str), ERTS_ATOM_ENC_LATIN1, 1); - BIF_ERROR(BIF_P, EXC_ERROR); - } + BIF_RET(res); + } + + if (port->drv_ptr->flags & ERL_DRV_FLAG_USE_INIT_ACK) { + /* Copied from erl_port_task.c */ + port->async_open_port = erts_alloc(ERTS_ALC_T_PRTSD, + sizeof(*port->async_open_port)); + erts_make_ref_in_array(port->async_open_port->ref); + port->async_open_port->to = BIF_P->common.id; + + erts_smp_proc_lock(BIF_P, ERTS_PROC_LOCKS_MSG_RECEIVE | ERTS_PROC_LOCK_LINK); + if (ERTS_PROC_PENDING_EXIT(BIF_P)) { + /* need to exit caller instead */ + erts_smp_proc_unlock(BIF_P, ERTS_PROC_LOCKS_MSG_RECEIVE | ERTS_PROC_LOCK_LINK); + KILL_CATCHES(BIF_P); + BIF_P->freason = EXC_EXIT; + erts_port_release(port); + BIF_RET(am_badarg); + } + + ERTS_SMP_MSGQ_MV_INQ2PRIVQ(BIF_P); + BIF_P->msg.save = BIF_P->msg.last; - erts_smp_proc_lock(BIF_P, ERTS_PROC_LOCK_LINK); + erts_smp_proc_unlock(BIF_P, ERTS_PROC_LOCKS_MSG_RECEIVE); + + res = erts_proc_store_ref(BIF_P, port->async_open_port->ref); + } else { + res = port->common.id; + erts_smp_proc_lock(BIF_P, ERTS_PROC_LOCK_LINK); + } - port_id = port->common.id; erts_add_link(&ERTS_P_LINKS(port), LINK_PID, BIF_P->common.id); - erts_add_link(&ERTS_P_LINKS(BIF_P), LINK_PID, port_id); + erts_add_link(&ERTS_P_LINKS(BIF_P), LINK_PID, port->common.id); erts_smp_proc_unlock(BIF_P, ERTS_PROC_LOCK_LINK); erts_port_release(port); - BIF_RET(port_id); + BIF_RET(res); } static ERTS_INLINE Port * @@ -472,7 +505,7 @@ cleanup_old_port_data(erts_aint_t data) ErtsPortDataHeap *pdhp = (ErtsPortDataHeap *) data; size_t size; ERTS_SMP_DATA_DEPENDENCY_READ_MEMORY_BARRIER; - size = sizeof(ErtsPortDataHeap) + pdhp->hsize*(sizeof(Eterm) - 1); + size = sizeof(ErtsPortDataHeap) + (pdhp->hsize-1)*sizeof(Eterm); erts_schedule_thr_prgr_later_cleanup_op(free_port_data_heap, (void *) pdhp, &pdhp->later_op, @@ -493,8 +526,8 @@ void erts_cleanup_port_data(Port *prt) { ASSERT(erts_atomic32_read_nob(&prt->state) & ERTS_PORT_SFLGS_INVALID_LOOKUP); - cleanup_old_port_data(erts_smp_atomic_read_nob(&prt->data)); - erts_smp_atomic_set_nob(&prt->data, (erts_aint_t) THE_NON_VALUE); + cleanup_old_port_data(erts_smp_atomic_xchg_nob(&prt->data, + (erts_aint_t) NULL)); } Uint @@ -508,7 +541,7 @@ erts_port_data_size(Port *prt) } else { ErtsPortDataHeap *pdhp = (ErtsPortDataHeap *) data; - return (Uint) sizeof(ErtsPortDataHeap) + pdhp->hsize*(sizeof(Eterm)-1); + return (Uint) sizeof(ErtsPortDataHeap) + (pdhp->hsize-1)*sizeof(Eterm); } } @@ -550,10 +583,11 @@ BIF_RETTYPE port_set_data_2(BIF_ALIST_2) hsize = size_object(BIF_ARG_2); pdhp = erts_alloc(ERTS_ALC_T_PORT_DATA_HEAP, - sizeof(ErtsPortDataHeap) + hsize*(sizeof(Eterm)-1)); + sizeof(ErtsPortDataHeap) + (hsize-1)*sizeof(Eterm)); hp = &pdhp->heap[0]; pdhp->off_heap.first = NULL; pdhp->off_heap.overhead = 0; + pdhp->hsize = hsize; pdhp->data = copy_struct(BIF_ARG_2, hsize, &hp, &pdhp->off_heap); data = (erts_aint_t) pdhp; ASSERT((data & 0x3) == 0); @@ -561,8 +595,14 @@ BIF_RETTYPE port_set_data_2(BIF_ALIST_2) data = erts_smp_atomic_xchg_wb(&prt->data, data); + if (data == (erts_aint_t)NULL) { + /* Port terminated by racing thread */ + data = erts_smp_atomic_xchg_wb(&prt->data, data); + ASSERT(data != (erts_aint_t)NULL); + cleanup_old_port_data(data); + BIF_ERROR(BIF_P, BADARG); + } cleanup_old_port_data(data); - BIF_RET(am_true); } @@ -581,6 +621,8 @@ BIF_RETTYPE port_get_data_1(BIF_ALIST_1) BIF_ERROR(BIF_P, BADARG); data = erts_smp_atomic_read_ddrb(&prt->data); + if (data == (erts_aint_t)NULL) + BIF_ERROR(BIF_P, BADARG); /* Port terminated by racing thread */ if ((data & 0x3) != 0) { res = (Eterm) (UWord) data; @@ -607,7 +649,7 @@ BIF_RETTYPE port_get_data_1(BIF_ALIST_1) static Port * open_port(Process* p, Eterm name, Eterm settings, int *err_typep, int *err_nump) { - int i; + Sint i; Eterm option; Uint arity; Eterm* tp; @@ -781,7 +823,7 @@ open_port(Process* p, Eterm name, Eterm settings, int *err_typep, int *err_nump) } else { name_buf = (char *) erts_alloc(ERTS_ALC_T_TMP, i + 1); if (intlist_to_buf(name, name_buf, i) != i) - erl_exit(1, "%s:%d: Internal error\n", __FILE__, __LINE__); + erts_exit(ERTS_ERROR_EXIT, "%s:%d: Internal error\n", __FILE__, __LINE__); name_buf[i] = '\0'; } driver = &vanilla_driver; @@ -935,8 +977,8 @@ static char **convert_args(Eterm l) { char **pp; char *b; - int n; - int i = 0; + Sint n; + Sint i = 0; Eterm str; if (is_not_list(l) && is_not_nil(l)) { return NULL; @@ -982,7 +1024,7 @@ static byte* convert_environment(Process* p, Eterm env) Eterm* temp_heap; Eterm* hp; Uint heap_size; - int n; + Sint n; Sint size; byte* bytes; int encoding = erts_get_native_filename_encoding(); @@ -1159,7 +1201,7 @@ static Eterm http_bld_uri(struct packet_callback_args* pca, return erts_bld_tuple(hpp, szp, 3, am_scheme, s1, s2); default: - erl_exit(1, "%s, line %d: type=%u\n", __FILE__, __LINE__, uri->type); + erts_exit(ERTS_ERROR_EXIT, "%s, line %d: type=%u\n", __FILE__, __LINE__, uri->type); } } @@ -1319,7 +1361,8 @@ BIF_RETTYPE decode_packet_3(BIF_ALIST_3) ErlSubBin* rest; Eterm res; Eterm options; - int code; + int code; + char delimiter = '\n'; if (!is_binary(BIF_ARG_2) || (!is_list(BIF_ARG_3) && !is_nil(BIF_ARG_3))) { @@ -1360,6 +1403,11 @@ BIF_RETTYPE decode_packet_3(BIF_ALIST_3) case am_line_length: trunc_len = val; goto next_option; + case am_line_delimiter: + if (type == TCP_PB_LINE_LF && val >= 0 && val <= 255) { + delimiter = (char)val; + goto next_option; + } } } } @@ -1380,7 +1428,7 @@ BIF_RETTYPE decode_packet_3(BIF_ALIST_3) pca.aligned_ptr = bin_ptr; } packet_sz = packet_get_length(type, (char*)pca.aligned_ptr, pca.bin_sz, - max_plen, trunc_len, &http_state); + max_plen, trunc_len, delimiter, &http_state); if (!(packet_sz > 0 && packet_sz <= pca.bin_sz)) { if (packet_sz < 0) { goto error; diff --git a/erts/emulator/beam/erl_bif_re.c b/erts/emulator/beam/erl_bif_re.c index 448c6f6f6d..7f7cd376ac 100644 --- a/erts/emulator/beam/erl_bif_re.c +++ b/erts/emulator/beam/erl_bif_re.c @@ -3,16 +3,17 @@ * * Copyright Ericsson AB 2008-2013. All Rights Reserved. * - * The contents of this file are subject to the Erlang Public License, - * Version 1.1, (the "License"); you may not use this file except in - * compliance with the License. You should have received a copy of the - * Erlang Public License along with this software. If not, it can be - * retrieved online at http://www.erlang.org/. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Software distributed under the License is distributed on an "AS IS" - * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See - * the License for the specific language governing rights and limitations - * under the License. + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. * * %CopyrightEnd% */ @@ -99,7 +100,7 @@ Sint erts_re_set_loop_limit(Sint limit) static int term_to_int(Eterm term, int *sp) { -#if defined(ARCH_64) && !HALFWORD_HEAP +#if defined(ARCH_64) if (is_small(term)) { Uint x = signed_val(term); @@ -150,7 +151,7 @@ static int term_to_int(Eterm term, int *sp) static Eterm make_signed_integer(int x, Process *p) { -#if defined(ARCH_64) && !HALFWORD_HEAP +#if defined(ARCH_64) return make_small(x); #else Eterm* hp; diff --git a/erts/emulator/beam/erl_bif_timer.c b/erts/emulator/beam/erl_bif_timer.c deleted file mode 100644 index 03ac97283c..0000000000 --- a/erts/emulator/beam/erl_bif_timer.c +++ /dev/null @@ -1,705 +0,0 @@ -/* - * %CopyrightBegin% - * - * Copyright Ericsson AB 2005-2012. All Rights Reserved. - * - * The contents of this file are subject to the Erlang Public License, - * Version 1.1, (the "License"); you may not use this file except in - * compliance with the License. You should have received a copy of the - * Erlang Public License along with this software. If not, it can be - * retrieved online at http://www.erlang.org/. - * - * Software distributed under the License is distributed on an "AS IS" - * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See - * the License for the specific language governing rights and limitations - * under the License. - * - * %CopyrightEnd% - */ - -#ifdef HAVE_CONFIG_H -# include "config.h" -#endif - -#include "erl_bif_timer.h" -#include "global.h" -#include "bif.h" -#include "error.h" -#include "big.h" -#include "erl_thr_progress.h" - -/**************************************************************************** -** BIF Timer support -****************************************************************************/ - -#define BTM_FLG_SL_TIMER (((Uint32) 1) << 0) -#define BTM_FLG_CANCELED (((Uint32) 1) << 1) -#define BTM_FLG_HEAD (((Uint32) 1) << 2) -#define BTM_FLG_BYNAME (((Uint32) 1) << 3) -#define BTM_FLG_WRAP (((Uint32) 1) << 4) - -struct ErtsBifTimer_ { - struct { - union { - ErtsBifTimer **head; - ErtsBifTimer *prev; - } u; - ErtsBifTimer *next; - } tab; - union { - Eterm name; - struct { - ErtsBifTimer *prev; - ErtsBifTimer *next; - Process *ess; - } proc; - } receiver; - ErlTimer tm; - ErlHeapFragment* bp; - Uint32 flags; - Eterm message; - Uint32 ref_numbers[ERTS_REF_NUMBERS]; -}; - -#ifdef SMALL_MEMORY -#define TIMER_HASH_VEC_SZ 3331 -#define BTM_PREALC_SZ 10 -#else -#define TIMER_HASH_VEC_SZ 10007 -#define BTM_PREALC_SZ 100 -#endif -static ErtsBifTimer **bif_timer_tab; -static Uint no_bif_timers; - - -static erts_smp_rwmtx_t bif_timer_lock; - -#define erts_smp_safe_btm_rwlock(P, L) \ - safe_btm_lock((P), (L), 1) -#define erts_smp_safe_btm_rlock(P, L) \ - safe_btm_lock((P), (L), 0) -#define erts_smp_btm_rwlock() \ - erts_smp_rwmtx_rwlock(&bif_timer_lock) -#define erts_smp_btm_tryrwlock() \ - erts_smp_rwmtx_tryrwlock(&bif_timer_lock) -#define erts_smp_btm_rwunlock() \ - erts_smp_rwmtx_rwunlock(&bif_timer_lock) -#define erts_smp_btm_rlock() \ - erts_smp_rwmtx_rlock(&bif_timer_lock) -#define erts_smp_btm_tryrlock() \ - erts_smp_rwmtx_tryrlock(&bif_timer_lock) -#define erts_smp_btm_runlock() \ - erts_smp_rwmtx_runlock(&bif_timer_lock) -#define erts_smp_btm_lock_init() \ - erts_smp_rwmtx_init(&bif_timer_lock, "bif_timers") - - -static ERTS_INLINE int -safe_btm_lock(Process *c_p, ErtsProcLocks c_p_locks, int rw_lock) -{ - ASSERT(c_p && c_p_locks); -#ifdef ERTS_SMP - if ((rw_lock ? erts_smp_btm_tryrwlock() : erts_smp_btm_tryrlock()) != EBUSY) - return 0; - erts_smp_proc_unlock(c_p, c_p_locks); - if (rw_lock) - erts_smp_btm_rwlock(); - else - erts_smp_btm_rlock(); - erts_smp_proc_lock(c_p, c_p_locks); - if (ERTS_PROC_IS_EXITING(c_p)) { - if (rw_lock) - erts_smp_btm_rwunlock(); - else - erts_smp_btm_runlock(); - return 1; - } -#endif - return 0; -} - -ERTS_SCHED_PREF_PALLOC_IMPL(btm_pre, ErtsBifTimer, BTM_PREALC_SZ) - -static ERTS_INLINE int -get_index(Uint32 *ref_numbers, Uint32 len) -{ - Uint32 hash; - /* len can potentially be larger than ERTS_REF_NUMBERS - if it has visited another node... */ - if (len > ERTS_REF_NUMBERS) - len = ERTS_REF_NUMBERS; - -#if ERTS_REF_NUMBERS != 3 -#error "ERTS_REF_NUMBERS changed. Update me..." -#endif - switch (len) { - case 3: if (!ref_numbers[2]) len = 2; - case 2: if (!ref_numbers[1]) len = 1; - default: break; - } - - ASSERT(1 <= len && len <= ERTS_REF_NUMBERS); - - hash = block_hash((byte *) ref_numbers, len * sizeof(Uint32), 0x08d12e65); - return (int) (hash % ((Uint32) TIMER_HASH_VEC_SZ)); -} - -static Eterm -create_ref(Uint *hp, Uint32 *ref_numbers, Uint32 len) -{ - Uint32 *datap; - int i; - - - if (len > ERTS_MAX_REF_NUMBERS) { - /* Such large refs should no be able to appear in the emulator */ - erl_exit(1, "%s:%d: Internal error\n", __FILE__, __LINE__); - } - -#if defined(ARCH_64) && !HALFWORD_HEAP - hp[0] = make_ref_thing_header(len/2 + 1); - datap = (Uint32 *) &hp[1]; - *(datap++) = len; -#else - hp[0] = make_ref_thing_header(len); - datap = (Uint32 *) &hp[1]; -#endif - - for (i = 0; i < len; i++) - datap[i] = ref_numbers[i]; - - return make_internal_ref(hp); -} - -static int -eq_non_standard_ref_numbers(Uint32 *rn1, Uint32 len1, Uint32 *rn2, Uint32 len2) -{ -#if defined(ARCH_64) && !HALFWORD_HEAP -#define MAX_REF_HEAP_SZ (1+(ERTS_MAX_REF_NUMBERS/2+1)) -#else -#define MAX_REF_HEAP_SZ (1+ERTS_MAX_REF_NUMBERS) -#endif - DeclareTmpHeapNoproc(r1_hp,(MAX_REF_HEAP_SZ * 2)); - Eterm *r2_hp = r1_hp +MAX_REF_HEAP_SZ; - - return eq(create_ref(r1_hp, rn1, len1), create_ref(r2_hp, rn2, len2)); -#undef MAX_REF_HEAP_SZ -} - -static ERTS_INLINE int -eq_ref_numbers(Uint32 *rn1, Uint32 len1, Uint32 *rn2, Uint32 len2) -{ - int res; - if (len1 != ERTS_REF_NUMBERS || len2 != ERTS_REF_NUMBERS) { - /* Can potentially happen, but will never... */ - return eq_non_standard_ref_numbers(rn1, len1, rn2, len2); - } - -#if ERTS_REF_NUMBERS != 3 -#error "ERTS_REF_NUMBERS changed. Update me..." -#endif - res = rn1[0] == rn2[0] && rn1[1] == rn2[1] && rn1[2] == rn2[2]; - - ASSERT(res - ? eq_non_standard_ref_numbers(rn1, len1, rn2, len2) - : !eq_non_standard_ref_numbers(rn1, len1, rn2, len2)); - - return res; -} - -static ERTS_INLINE ErtsBifTimer * -tab_find(Eterm ref) -{ - Uint32 *ref_numbers = internal_ref_numbers(ref); - Uint32 ref_numbers_len = internal_ref_no_of_numbers(ref); - int ix = get_index(ref_numbers, ref_numbers_len); - ErtsBifTimer* btm; - - for (btm = bif_timer_tab[ix]; btm; btm = btm->tab.next) - if (eq_ref_numbers(ref_numbers, ref_numbers_len, - btm->ref_numbers, ERTS_REF_NUMBERS)) - return btm; - return NULL; -} - -static ERTS_INLINE void -tab_remove(ErtsBifTimer* btm) -{ - if (btm->flags & BTM_FLG_HEAD) { - *btm->tab.u.head = btm->tab.next; - if (btm->tab.next) { - btm->tab.next->flags |= BTM_FLG_HEAD; - btm->tab.next->tab.u.head = btm->tab.u.head; - } - } - else { - btm->tab.u.prev->tab.next = btm->tab.next; - if (btm->tab.next) - btm->tab.next->tab.u.prev = btm->tab.u.prev; - } - btm->flags |= BTM_FLG_CANCELED; - ASSERT(no_bif_timers > 0); - no_bif_timers--; -} - -static ERTS_INLINE void -tab_insert(ErtsBifTimer* btm) -{ - int ix = get_index(btm->ref_numbers, ERTS_REF_NUMBERS); - ErtsBifTimer* btm_list = bif_timer_tab[ix]; - - if (btm_list) { - btm_list->flags &= ~BTM_FLG_HEAD; - btm_list->tab.u.prev = btm; - } - - btm->flags |= BTM_FLG_HEAD; - btm->tab.u.head = &bif_timer_tab[ix]; - btm->tab.next = btm_list; - bif_timer_tab[ix] = btm; - no_bif_timers++; -} - -static ERTS_INLINE void -link_proc(Process *p, ErtsBifTimer* btm) -{ - btm->receiver.proc.ess = p; - btm->receiver.proc.prev = NULL; - btm->receiver.proc.next = p->u.bif_timers; - if (p->u.bif_timers) - p->u.bif_timers->receiver.proc.prev = btm; - p->u.bif_timers = btm; -} - -static ERTS_INLINE void -unlink_proc(ErtsBifTimer* btm) -{ - if (btm->receiver.proc.prev) - btm->receiver.proc.prev->receiver.proc.next = btm->receiver.proc.next; - else - btm->receiver.proc.ess->u.bif_timers = btm->receiver.proc.next; - if (btm->receiver.proc.next) - btm->receiver.proc.next->receiver.proc.prev = btm->receiver.proc.prev; -} - -static void -bif_timer_cleanup(ErtsBifTimer* btm) -{ - ASSERT(btm); - - if (btm->bp) - free_message_buffer(btm->bp); - - if (!btm_pre_free(btm)) { - if (btm->flags & BTM_FLG_SL_TIMER) - erts_free(ERTS_ALC_T_SL_BIF_TIMER, (void *) btm); - else - erts_free(ERTS_ALC_T_LL_BIF_TIMER, (void *) btm); - } -} - -static void -bif_timer_timeout(ErtsBifTimer* btm) -{ - ASSERT(btm); - - - erts_smp_btm_rwlock(); - - if (btm->flags & BTM_FLG_CANCELED) { - /* - * A concurrent cancel is ongoing. Do not send the timeout message, - * but cleanup here since the cancel call-back won't be called. - */ -#ifndef ERTS_SMP - ASSERT(0); -#endif - } - else { - ErtsProcLocks rp_locks = 0; - Process* rp; - - tab_remove(btm); - - ASSERT(!erts_get_current_process()); - - if (btm->flags & BTM_FLG_BYNAME) - rp = erts_whereis_process(NULL, 0, btm->receiver.name, 0, 0); - else { - rp = btm->receiver.proc.ess; - unlink_proc(btm); - } - - if (rp) { - Eterm message; - ErlHeapFragment *bp; - - bp = btm->bp; - btm->bp = NULL; /* Prevent cleanup of message buffer... */ - - if (!(btm->flags & BTM_FLG_WRAP)) - message = btm->message; - else { -#if ERTS_REF_NUMBERS != 3 -#error "ERTS_REF_NUMBERS changed. Update me..." -#endif - Eterm ref; - Uint *hp; - Uint wrap_size = REF_THING_SIZE + 4; - message = btm->message; - - if (!bp) { - ErlOffHeap *ohp; - ASSERT(is_immed(message)); - hp = erts_alloc_message_heap(wrap_size, - &bp, - &ohp, - rp, - &rp_locks); - } else { - Eterm old_size = bp->used_size; - bp = erts_resize_message_buffer(bp, old_size + wrap_size, - &message, 1); - hp = &bp->mem[0] + old_size; - } - - write_ref_thing(hp, - btm->ref_numbers[0], - btm->ref_numbers[1], - btm->ref_numbers[2]); - ref = make_internal_ref(hp); - hp += REF_THING_SIZE; - message = TUPLE3(hp, am_timeout, ref, message); - } - - erts_queue_message(rp, &rp_locks, bp, message, NIL -#ifdef USE_VM_PROBES - , NIL -#endif - ); - erts_smp_proc_unlock(rp, rp_locks); - } - } - - erts_smp_btm_rwunlock(); - - bif_timer_cleanup(btm); -} - -static Eterm -setup_bif_timer(Uint32 xflags, - Process *c_p, - Eterm time, - Eterm receiver, - Eterm message) -{ - Process *rp; - ErtsBifTimer* btm; - Uint timeout; - Eterm ref; - Uint32 *ref_numbers; - - if (!term_to_Uint(time, &timeout)) - return THE_NON_VALUE; -#if defined(ARCH_64) && !HALFWORD_HEAP - if ((timeout >> 32) != 0) - return THE_NON_VALUE; -#endif - if (is_not_internal_pid(receiver) && is_not_atom(receiver)) - return THE_NON_VALUE; - - ref = erts_make_ref(c_p); - - if (is_atom(receiver)) - rp = NULL; - else { - rp = erts_pid2proc(c_p, ERTS_PROC_LOCK_MAIN, - receiver, ERTS_PROC_LOCK_MSGQ); - if (!rp) - return ref; - } - - if (timeout < ERTS_ALC_MIN_LONG_LIVED_TIME) { - if (timeout < 1000) { - btm = btm_pre_alloc(); - if (!btm) - goto sl_timer_alloc; - btm->flags = 0; - } - else { - sl_timer_alloc: - btm = (ErtsBifTimer *) erts_alloc(ERTS_ALC_T_SL_BIF_TIMER, - sizeof(ErtsBifTimer)); - btm->flags = BTM_FLG_SL_TIMER; - } - } - else { - btm = (ErtsBifTimer *) erts_alloc(ERTS_ALC_T_LL_BIF_TIMER, - sizeof(ErtsBifTimer)); - btm->flags = 0; - } - - if (rp) { - link_proc(rp, btm); - erts_smp_proc_unlock(rp, ERTS_PROC_LOCK_MSGQ); - } - else { - ASSERT(is_atom(receiver)); - btm->receiver.name = receiver; - btm->flags |= BTM_FLG_BYNAME; - } - - btm->flags |= xflags; - - ref_numbers = internal_ref_numbers(ref); - ASSERT(internal_ref_no_of_numbers(ref) == 3); -#if ERTS_REF_NUMBERS != 3 -#error "ERTS_REF_NUMBERS changed. Update me..." -#endif - btm->ref_numbers[0] = ref_numbers[0]; - btm->ref_numbers[1] = ref_numbers[1]; - btm->ref_numbers[2] = ref_numbers[2]; - - ASSERT(eq_ref_numbers(btm->ref_numbers, ERTS_REF_NUMBERS, - ref_numbers, ERTS_REF_NUMBERS)); - - if (is_immed(message)) { - btm->bp = NULL; - btm->message = message; - } - else { - ErlHeapFragment* bp; - Eterm* hp; - Uint size; - - size = size_object(message); - btm->bp = bp = new_message_buffer(size); - hp = bp->mem; - btm->message = copy_struct(message, size, &hp, &bp->off_heap); - } - - tab_insert(btm); - ASSERT(btm == tab_find(ref)); - btm->tm.active = 0; /* MUST be initalized */ - erts_set_timer(&btm->tm, - (ErlTimeoutProc) bif_timer_timeout, - (ErlCancelProc) bif_timer_cleanup, - (void *) btm, - timeout); - return ref; -} - -/* send_after(Time, Pid, Message) -> Ref */ -BIF_RETTYPE send_after_3(BIF_ALIST_3) -{ - Eterm res; - - if (erts_smp_safe_btm_rwlock(BIF_P, ERTS_PROC_LOCK_MAIN)) - ERTS_BIF_EXITED(BIF_P); - - res = setup_bif_timer(0, BIF_P, BIF_ARG_1, BIF_ARG_2, BIF_ARG_3); - - erts_smp_btm_rwunlock(); - - if (is_non_value(res)) { - BIF_ERROR(BIF_P, BADARG); - } - else { - ASSERT(is_internal_ref(res)); - BIF_RET(res); - } -} - -/* start_timer(Time, Pid, Message) -> Ref */ -BIF_RETTYPE start_timer_3(BIF_ALIST_3) -{ - Eterm res; - - if (erts_smp_safe_btm_rwlock(BIF_P, ERTS_PROC_LOCK_MAIN)) - ERTS_BIF_EXITED(BIF_P); - - res = setup_bif_timer(BTM_FLG_WRAP, BIF_P, BIF_ARG_1, BIF_ARG_2, BIF_ARG_3); - - erts_smp_btm_rwunlock(); - - if (is_non_value(res)) { - BIF_ERROR(BIF_P, BADARG); - } - else { - ASSERT(is_internal_ref(res)); - BIF_RET(res); - } -} - -/* cancel_timer(Ref) -> false | RemainingTime */ -BIF_RETTYPE cancel_timer_1(BIF_ALIST_1) -{ - Eterm res; - ErtsBifTimer *btm; - - if (is_not_internal_ref(BIF_ARG_1)) { - if (is_ref(BIF_ARG_1)) { - BIF_RET(am_false); - } - BIF_ERROR(BIF_P, BADARG); - } - - if (erts_smp_safe_btm_rwlock(BIF_P, ERTS_PROC_LOCK_MAIN)) - ERTS_BIF_EXITED(BIF_P); - - btm = tab_find(BIF_ARG_1); - if (!btm || btm->flags & BTM_FLG_CANCELED) { - erts_smp_btm_rwunlock(); - res = am_false; - } - else { - Uint left = erts_time_left(&btm->tm); - if (!(btm->flags & BTM_FLG_BYNAME)) { - erts_smp_proc_lock(btm->receiver.proc.ess, ERTS_PROC_LOCK_MSGQ); - unlink_proc(btm); - erts_smp_proc_unlock(btm->receiver.proc.ess, ERTS_PROC_LOCK_MSGQ); - } - tab_remove(btm); - ASSERT(!tab_find(BIF_ARG_1)); - erts_cancel_timer(&btm->tm); - erts_smp_btm_rwunlock(); - res = erts_make_integer(left, BIF_P); - } - - BIF_RET(res); -} - -/* read_timer(Ref) -> false | RemainingTime */ -BIF_RETTYPE read_timer_1(BIF_ALIST_1) -{ - Eterm res; - ErtsBifTimer *btm; - - if (is_not_internal_ref(BIF_ARG_1)) { - if (is_ref(BIF_ARG_1)) { - BIF_RET(am_false); - } - BIF_ERROR(BIF_P, BADARG); - } - - if (erts_smp_safe_btm_rlock(BIF_P, ERTS_PROC_LOCK_MAIN)) - ERTS_BIF_EXITED(BIF_P); - - btm = tab_find(BIF_ARG_1); - if (!btm || btm->flags & BTM_FLG_CANCELED) { - res = am_false; - } - else { - Uint left = erts_time_left(&btm->tm); - res = erts_make_integer(left, BIF_P); - } - - erts_smp_btm_runlock(); - - BIF_RET(res); -} - -void -erts_print_bif_timer_info(int to, void *to_arg) -{ - int i; - int lock = !ERTS_IS_CRASH_DUMPING; - - if (lock) - erts_smp_btm_rlock(); - - for (i = 0; i < TIMER_HASH_VEC_SZ; i++) { - ErtsBifTimer *btm; - for (btm = bif_timer_tab[i]; btm; btm = btm->tab.next) { - Eterm receiver = (btm->flags & BTM_FLG_BYNAME - ? btm->receiver.name - : btm->receiver.proc.ess->common.id); - erts_print(to, to_arg, "=timer:%T\n", receiver); - erts_print(to, to_arg, "Message: %T\n", btm->message); - erts_print(to, to_arg, "Time left: %u\n", - erts_time_left(&btm->tm)); - } - } - - if (lock) - erts_smp_btm_runlock(); -} - - -void -erts_cancel_bif_timers(Process *p, ErtsProcLocks plocks) -{ - ErtsBifTimer *btm; - - if (erts_smp_btm_tryrwlock() == EBUSY) { - erts_smp_proc_unlock(p, plocks); - erts_smp_btm_rwlock(); - erts_smp_proc_lock(p, plocks); - } - - btm = p->u.bif_timers; - while (btm) { - ErtsBifTimer *tmp_btm; - ASSERT(!(btm->flags & BTM_FLG_CANCELED)); - tab_remove(btm); - tmp_btm = btm; - btm = btm->receiver.proc.next; - erts_cancel_timer(&tmp_btm->tm); - } - - p->u.bif_timers = NULL; - - erts_smp_btm_rwunlock(); -} - -void erts_bif_timer_init(void) -{ - int i; - no_bif_timers = 0; - init_btm_pre_alloc(); - erts_smp_btm_lock_init(); - bif_timer_tab = erts_alloc(ERTS_ALC_T_BIF_TIMER_TABLE, - sizeof(ErtsBifTimer *)*TIMER_HASH_VEC_SZ); - for (i = 0; i < TIMER_HASH_VEC_SZ; ++i) - bif_timer_tab[i] = NULL; -} - -Uint -erts_bif_timer_memory_size(void) -{ - Uint res; - int lock = !ERTS_IS_CRASH_DUMPING; - - if (lock) - erts_smp_btm_rlock(); - - res = (sizeof(ErtsBifTimer *)*TIMER_HASH_VEC_SZ - + no_bif_timers*sizeof(ErtsBifTimer)); - - if (lock) - erts_smp_btm_runlock(); - - return res; -} - - -void -erts_bif_timer_foreach(void (*func)(Eterm, Eterm, ErlHeapFragment *, void *), - void *arg) -{ - int i; - - ERTS_SMP_LC_ASSERT(erts_smp_thr_progress_is_blocking()); - - for (i = 0; i < TIMER_HASH_VEC_SZ; i++) { - ErtsBifTimer *btm; - for (btm = bif_timer_tab[i]; btm; btm = btm->tab.next) { - (*func)((btm->flags & BTM_FLG_BYNAME - ? btm->receiver.name - : btm->receiver.proc.ess->common.id), - btm->message, - btm->bp, - arg); - } - } -} diff --git a/erts/emulator/beam/erl_bif_timer.h b/erts/emulator/beam/erl_bif_timer.h deleted file mode 100644 index 1197c176f5..0000000000 --- a/erts/emulator/beam/erl_bif_timer.h +++ /dev/null @@ -1,36 +0,0 @@ -/* - * %CopyrightBegin% - * - * Copyright Ericsson AB 2005-2009. All Rights Reserved. - * - * The contents of this file are subject to the Erlang Public License, - * Version 1.1, (the "License"); you may not use this file except in - * compliance with the License. You should have received a copy of the - * Erlang Public License along with this software. If not, it can be - * retrieved online at http://www.erlang.org/. - * - * Software distributed under the License is distributed on an "AS IS" - * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See - * the License for the specific language governing rights and limitations - * under the License. - * - * %CopyrightEnd% - */ - - -#ifndef ERL_BIF_TIMER_H__ -#define ERL_BIF_TIMER_H__ - -typedef struct ErtsBifTimer_ ErtsBifTimer; - -#include "sys.h" -#include "erl_process.h" -#include "erl_message.h" - -Uint erts_bif_timer_memory_size(void); -void erts_print_bif_timer_info(int to, void *to_arg); -void erts_cancel_bif_timers(Process *p, ErtsProcLocks plocks); -void erts_bif_timer_init(void); -void erts_bif_timer_foreach(void (*func)(Eterm,Eterm,ErlHeapFragment *,void *), - void *arg); -#endif diff --git a/erts/emulator/beam/erl_bif_trace.c b/erts/emulator/beam/erl_bif_trace.c index 06fbbea123..71a7079b09 100644 --- a/erts/emulator/beam/erl_bif_trace.c +++ b/erts/emulator/beam/erl_bif_trace.c @@ -3,16 +3,17 @@ * * Copyright Ericsson AB 1999-2013. All Rights Reserved. * - * The contents of this file are subject to the Erlang Public License, - * Version 1.1, (the "License"); you may not use this file except in - * compliance with the License. You should have received a copy of the - * Erlang Public License along with this software. If not, it can be - * retrieved online at http://www.erlang.org/. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Software distributed under the License is distributed on an "AS IS" - * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See - * the License for the specific language governing rights and limitations - * under the License. + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. * * %CopyrightEnd% */ @@ -38,6 +39,7 @@ #include "beam_bp.h" #include "erl_binary.h" #include "erl_thr_progress.h" +#include "erl_bif_unique.h" #define DECL_AM(S) Eterm AM_ ## S = am_atom_put(#S, sizeof(#S) - 1) @@ -358,7 +360,7 @@ trace_pattern(Process* p, Eterm MFA, Eterm Pattern, Eterm flaglist) ASSERT(finish_bp.stager == NULL); finish_bp.stager = p; erts_schedule_thr_prgr_later_op(smp_bp_finisher, NULL, &finish_bp.lop); - erts_smp_proc_inc_refc(p); + erts_proc_inc_refc(p); erts_suspend(p, ERTS_PROC_LOCK_MAIN, NULL); ERTS_BIF_YIELD_RETURN(p, make_small(matches)); } @@ -392,7 +394,7 @@ static void smp_bp_finisher(void* null) erts_resume(p, ERTS_PROC_LOCK_STATUS); } erts_smp_proc_unlock(p, ERTS_PROC_LOCK_STATUS); - erts_smp_proc_dec_refc(p); + erts_proc_dec_refc(p); } } #endif /* ERTS_SMP */ @@ -429,6 +431,9 @@ Uint erts_trace_flag2bit(Eterm flag) { switch (flag) { + case am_timestamp: return F_NOW_TS; + case am_strict_monotonic_timestamp: return F_STRICT_MON_TS; + case am_monotonic_timestamp: return F_MON_TS; case am_all: return TRACEE_FLAGS; case am_send: return F_TRACE_SEND; case am_receive: return F_TRACE_RECEIVE; @@ -437,7 +442,6 @@ erts_trace_flag2bit(Eterm flag) case am_set_on_first_spawn: return F_TRACE_SOS1; case am_set_on_link: return F_TRACE_SOL; case am_set_on_first_link: return F_TRACE_SOL1; - case am_timestamp: return F_TIMESTAMP; case am_running: return F_TRACE_SCHED; case am_exiting: return F_TRACE_SCHED_EXIT; case am_garbage_collection: return F_TRACE_GC; @@ -590,7 +594,7 @@ Eterm trace_3(BIF_ALIST_3) ERTS_TRACE_FLAGS(tracee_port) |= mask; else ERTS_TRACE_FLAGS(tracee_port) &= ~mask; - + if (!ERTS_TRACE_FLAGS(tracee_port)) ERTS_TRACER_PROC(tracee_port) = NIL; else if (tracer != NIL) @@ -651,7 +655,7 @@ Eterm trace_3(BIF_ALIST_3) if (pid_spec == am_all) { if (on) { if (!erts_cpu_timestamp) { -#ifdef HAVE_CLOCK_GETTIME +#ifdef HAVE_CLOCK_GETTIME_CPU_TIME /* Perhaps clock_gettime was found during config on a different machine than this. We check @@ -678,7 +682,7 @@ Eterm trace_3(BIF_ALIST_3) if (erts_start_now_cpu() < 0) { goto error; } -#endif /* HAVE_CLOCK_GETTIME */ +#endif /* HAVE_CLOCK_GETTIME_CPU_TIME */ erts_cpu_timestamp = !0; } } @@ -976,7 +980,7 @@ trace_info_pid(Process* p, Eterm pid_spec, Eterm key) } if (key == am_flags) { - int num_flags = 19; /* MAXIMUM number of flags. */ + int num_flags = 21; /* MAXIMUM number of flags. */ Uint needed = 3+2*num_flags; Eterm flag_list = NIL; Eterm* limit; @@ -994,6 +998,9 @@ trace_info_pid(Process* p, Eterm pid_spec, Eterm key) #endif hp = HAlloc(p, needed); limit = hp+needed; + FLAG(F_NOW_TS, am_timestamp); + FLAG(F_STRICT_MON_TS, am_strict_monotonic_timestamp); + FLAG(F_MON_TS, am_monotonic_timestamp); FLAG(F_TRACE_SEND, am_send); FLAG(F_TRACE_RECEIVE, am_receive); FLAG(F_TRACE_SOS, am_set_on_spawn); @@ -1005,7 +1012,6 @@ trace_info_pid(Process* p, Eterm pid_spec, Eterm key) FLAG(F_TRACE_SCHED, am_running); FLAG(F_TRACE_SCHED_EXIT, am_exiting); FLAG(F_TRACE_GC, am_garbage_collection); - FLAG(F_TIMESTAMP, am_timestamp); FLAG(F_TRACE_ARITY_ONLY, am_arity); FLAG(F_TRACE_RETURN_TO, am_return_to); FLAG(F_TRACE_SILENT, am_silent); @@ -1796,7 +1802,11 @@ Eterm erts_seq_trace(Process *p, Eterm arg1, Eterm arg2, } else if (arg1 == am_print) { current_flag = SEQ_TRACE_PRINT; } else if (arg1 == am_timestamp) { - current_flag = SEQ_TRACE_TIMESTAMP; + current_flag = SEQ_TRACE_NOW_TS; + } else if (arg1 == am_strict_monotonic_timestamp) { + current_flag = SEQ_TRACE_STRICT_MON_TS; + } else if (arg1 == am_monotonic_timestamp) { + current_flag = SEQ_TRACE_MON_TS; } else current_flag = 0; @@ -1877,11 +1887,7 @@ new_seq_trace_token(Process* p) { Eterm* hp; - if (SEQ_TRACE_TOKEN(p) == NIL -#ifdef USE_VM_PROBES - || SEQ_TRACE_TOKEN(p) == am_have_dt_utag -#endif - ) { + if (have_no_seqtrace(SEQ_TRACE_TOKEN(p))) { hp = HAlloc(p, 6); SEQ_TRACE_TOKEN(p) = TUPLE5(hp, make_small(0), /* Flags */ make_small(0), /* Label */ @@ -1901,13 +1907,11 @@ BIF_RETTYPE erl_seq_trace_info(Process *p, Eterm item) BIF_ERROR(p, BADARG); } - if (SEQ_TRACE_TOKEN(p) == NIL -#ifdef USE_VM_PROBES - || SEQ_TRACE_TOKEN(p) == am_have_dt_utag -#endif - ) { - if ((item == am_send) || (item == am_receive) || - (item == am_print) || (item == am_timestamp)) { + if (have_no_seqtrace(SEQ_TRACE_TOKEN(p))) { + if ((item == am_send) || (item == am_receive) || + (item == am_print) || (item == am_timestamp) + || (item == am_monotonic_timestamp) + || (item == am_strict_monotonic_timestamp)) { hp = HAlloc(p,3); res = TUPLE2(hp, item, am_false); BIF_RET(res); @@ -1925,7 +1929,11 @@ BIF_RETTYPE erl_seq_trace_info(Process *p, Eterm item) } else if (item == am_print) { current_flag = SEQ_TRACE_PRINT; } else if (item == am_timestamp) { - current_flag = SEQ_TRACE_TIMESTAMP; + current_flag = SEQ_TRACE_NOW_TS; + } else if (item == am_strict_monotonic_timestamp) { + current_flag = SEQ_TRACE_STRICT_MON_TS; + } else if (item == am_monotonic_timestamp) { + current_flag = SEQ_TRACE_MON_TS; } else { current_flag = 0; } @@ -1962,11 +1970,7 @@ BIF_RETTYPE seq_trace_info_1(BIF_ALIST_1) */ BIF_RETTYPE seq_trace_print_1(BIF_ALIST_1) { - if (SEQ_TRACE_TOKEN(BIF_P) == NIL -#ifdef USE_VM_PROBES - || SEQ_TRACE_TOKEN(BIF_P) == am_have_dt_utag -#endif - ) { + if (have_no_seqtrace(SEQ_TRACE_TOKEN(BIF_P))) { BIF_RET(am_false); } seq_trace_update_send(BIF_P); @@ -1985,11 +1989,7 @@ BIF_RETTYPE seq_trace_print_1(BIF_ALIST_1) */ BIF_RETTYPE seq_trace_print_2(BIF_ALIST_2) { - if (SEQ_TRACE_TOKEN(BIF_P) == NIL -#ifdef USE_VM_PROBES - || SEQ_TRACE_TOKEN(BIF_P) == am_have_dt_utag -#endif - ) { + if (have_no_seqtrace(SEQ_TRACE_TOKEN(BIF_P))) { BIF_RET(am_false); } if (!(is_atom(BIF_ARG_1) || is_small(BIF_ARG_1))) { @@ -2235,6 +2235,7 @@ static Eterm system_profile_get(Process *p) { if (erts_system_profile_flags.exclusive) { res = CONS(hp, am_exclusive, res); hp += 2; } + return TUPLE2(hp, system_profile, res); } } @@ -2253,6 +2254,7 @@ BIF_RETTYPE system_profile_2(BIF_ALIST_2) int system_blocked = 0; Process *profiler_p = NULL; Port *profiler_port = NULL; + int ts; if (profiler == am_undefined || list == NIL) { prev = system_profile_get(p); @@ -2284,7 +2286,8 @@ BIF_RETTYPE system_profile_2(BIF_ALIST_2) goto error; } - for (scheduler = 0, runnable_ports = 0, runnable_procs = 0, exclusive = 0; + for (ts = ERTS_TRACE_FLG_NOW_TIMESTAMP, scheduler = 0, + runnable_ports = 0, runnable_procs = 0, exclusive = 0; is_list(list); list = CDR(list_val(list))) { @@ -2297,6 +2300,12 @@ BIF_RETTYPE system_profile_2(BIF_ALIST_2) exclusive = !0; } else if (t == am_scheduler) { scheduler = !0; + } else if (t == am_timestamp) { + ts = ERTS_TRACE_FLG_NOW_TIMESTAMP; + } else if (t == am_strict_monotonic_timestamp) { + ts = ERTS_TRACE_FLG_STRICT_MONOTONIC_TIMESTAMP; + } else if (t == am_monotonic_timestamp) { + ts = ERTS_TRACE_FLG_MONOTONIC_TIMESTAMP; } else goto error; } if (is_not_nil(list)) goto error; @@ -2309,7 +2318,7 @@ BIF_RETTYPE system_profile_2(BIF_ALIST_2) erts_system_profile_flags.runnable_ports = !!runnable_ports; erts_system_profile_flags.runnable_procs = !!runnable_procs; erts_system_profile_flags.exclusive = !!exclusive; - + erts_system_profile_ts_type = ts; erts_smp_thr_progress_unblock(); erts_smp_proc_lock(p, ERTS_PROC_LOCK_MAIN); diff --git a/erts/emulator/beam/erl_bif_unique.c b/erts/emulator/beam/erl_bif_unique.c new file mode 100644 index 0000000000..c9c8561f64 --- /dev/null +++ b/erts/emulator/beam/erl_bif_unique.c @@ -0,0 +1,559 @@ +/* + * %CopyrightBegin% + * + * Copyright Ericsson AB 2014. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * %CopyrightEnd% + */ + +#ifdef HAVE_CONFIG_H +# include "config.h" +#endif + +#include "sys.h" +#include "erl_vm.h" +#include "erl_alloc.h" +#include "export.h" +#include "bif.h" +#include "erl_bif_unique.h" + +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *\ + * Reference * +\* */ + +static union { + erts_atomic64_t count; + char align__[ERTS_CACHE_LINE_SIZE]; +} global_reference erts_align_attribute(ERTS_CACHE_LINE_SIZE); + + +/* + * ref[0] indicate thread creating reference as follows: + * + * - ref[0] == 0 => Non-scheduler thread; + * - else; ref[0] <= erts_no_schedulers => + * ordinary scheduler with id == ref[0]; + * - else; ref[0] <= erts_no_schedulers + * + erts_no_dirty_cpu_schedulers => + * dirty cpu scheduler with id == 'ref[0] - erts_no_schedulers'; + * - else => + * dirty io scheduler with id == 'ref[0] + * - erts_no_schedulers + * - erts_no_dirty_cpu_schedulers' + */ + +#ifdef DEBUG +static Uint32 max_thr_id; +#endif + +static void +init_reference(void) +{ +#ifdef DEBUG + max_thr_id = (Uint32) erts_no_schedulers; +#ifdef ERTS_DIRTY_SCHEDULERS + max_thr_id += (Uint32) erts_no_dirty_cpu_schedulers; + max_thr_id += (Uint32) erts_no_dirty_io_schedulers; +#endif +#endif + erts_atomic64_init_nob(&global_reference.count, 0); +} + +static ERTS_INLINE void +global_make_ref_in_array(Uint32 thr_id, Uint32 ref[ERTS_MAX_REF_NUMBERS]) +{ + Uint64 value; + + value = (Uint64) erts_atomic64_inc_read_mb(&global_reference.count); + + erts_set_ref_numbers(ref, thr_id, value); +} + +static ERTS_INLINE void +make_ref_in_array(Uint32 ref[ERTS_MAX_REF_NUMBERS]) +{ + ErtsSchedulerData *esdp = erts_get_scheduler_data(); + if (esdp) + erts_sched_make_ref_in_array(esdp, ref); + else + global_make_ref_in_array(0, ref); +} + +void +erts_make_ref_in_array(Uint32 ref[ERTS_MAX_REF_NUMBERS]) +{ + make_ref_in_array(ref); +} + +Eterm erts_make_ref_in_buffer(Eterm buffer[REF_THING_SIZE]) +{ + Eterm* hp = buffer; + Uint32 ref[ERTS_MAX_REF_NUMBERS]; + + make_ref_in_array(ref); + write_ref_thing(hp, ref[0], ref[1], ref[2]); + return make_internal_ref(hp); +} + +Eterm erts_make_ref(Process *c_p) +{ + Eterm* hp; + Uint32 ref[ERTS_MAX_REF_NUMBERS]; + + ERTS_SMP_LC_ASSERT(ERTS_PROC_LOCK_MAIN & erts_proc_lc_my_proc_locks(c_p)); + + hp = HAlloc(c_p, REF_THING_SIZE); + + make_ref_in_array(ref); + write_ref_thing(hp, ref[0], ref[1], ref[2]); + + return make_internal_ref(hp); +} + +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *\ + * Unique Integer * +\* */ + +static struct { + union { + struct { + int left_shift; + int right_shift; + Uint64 mask; + Uint64 val0_max; + } o; + char align__[ERTS_CACHE_LINE_SIZE]; + } r; + union { + erts_atomic64_t val1; + char align__[ERTS_CACHE_LINE_SIZE]; + } w; +} unique_data erts_align_attribute(ERTS_CACHE_LINE_SIZE); + +static void +init_unique_integer(void) +{ + int bits; + unique_data.r.o.val0_max = (Uint64) erts_no_schedulers; +#ifdef ERTS_DIRTY_SCHEDULERS + unique_data.r.o.val0_max += (Uint64) erts_no_dirty_cpu_schedulers; + unique_data.r.o.val0_max += (Uint64) erts_no_dirty_io_schedulers; +#endif + bits = erts_fit_in_bits_int64(unique_data.r.o.val0_max); + unique_data.r.o.left_shift = bits; + unique_data.r.o.right_shift = 64 - bits; + unique_data.r.o.mask = (((Uint64) 1) << bits) - 1; + erts_atomic64_init_nob(&unique_data.w.val1, -1); +} + +#define ERTS_MAX_UNIQUE_INT_HEAP_SIZE ERTS_UINT64_ARRAY_TO_BIG_MAX_HEAP_SZ(2) + +static ERTS_INLINE Eterm +bld_unique_integer_term(Eterm **hpp, Uint *szp, + Uint64 val0, Uint64 val1, + int positive) +{ + Uint hsz; + Uint64 unique_val[2]; + + unique_val[0] = ((Uint64) val0); + unique_val[0] |= ((Uint64) val1) << unique_data.r.o.left_shift; + unique_val[1] = ((Uint64) val1) >> unique_data.r.o.right_shift; + unique_val[1] &= unique_data.r.o.mask; + + if (positive) { + unique_val[0]++; + if (unique_val[0] == 0) + unique_val[1]++; + } + else { + ASSERT(MIN_SMALL < 0); + if (unique_val[1] == 0 + && unique_val[0] < ((Uint64) -1*((Sint64) MIN_SMALL))) { + Sint64 s_unique_val = (Sint64) unique_val[0]; + s_unique_val += MIN_SMALL; + ASSERT(MIN_SMALL <= s_unique_val && s_unique_val < 0); + if (szp) + *szp = 0; + if (!hpp) + return THE_NON_VALUE; + return make_small((Sint) s_unique_val); + } + if (unique_val[0] < ((Uint64) -1*((Sint64) MIN_SMALL))) { + ASSERT(unique_val[1] != 0); + unique_val[1] -= 1; + } + unique_val[0] += MIN_SMALL; + } + + if (!unique_val[1]) { + if (unique_val[0] <= MAX_SMALL) { + if (szp) + *szp = 0; + if (!hpp) + return THE_NON_VALUE; + return make_small((Uint) unique_val[0]); + } + + if (szp) + *szp = ERTS_UINT64_HEAP_SIZE(unique_val[0]); + if (!hpp) + return THE_NON_VALUE; + return erts_uint64_to_big(unique_val[0], hpp); + } + else { + Eterm tmp, *tmp_hp, res; + DeclareTmpHeapNoproc(local_heap, 2*ERTS_MAX_UNIQUE_INT_HEAP_SIZE); + + UseTmpHeapNoproc(2*ERTS_MAX_UNIQUE_INT_HEAP_SIZE); + + tmp_hp = local_heap; + + tmp = erts_uint64_array_to_big(&tmp_hp, 0, 2, unique_val); + ASSERT(is_big(tmp)); + + hsz = big_arity(tmp) + 1; + + ASSERT(hsz <= ERTS_MAX_UNIQUE_INT_HEAP_SIZE); + + if (szp) + *szp = hsz; + + if (!hpp) + res = THE_NON_VALUE; + else { + int hix; + Eterm *hp = *hpp; + tmp_hp = big_val(tmp); + for (hix = 0; hix < hsz; hix++) + hp[hix] = tmp_hp[hix]; + + *hpp = hp + hsz; + res = make_big(hp); + } + + UnUseTmpHeapNoproc(2*ERTS_MAX_UNIQUE_INT_HEAP_SIZE); + + return res; + } +} + +static ERTS_INLINE Eterm unique_integer_bif(Process *c_p, int positive) +{ + ErtsSchedulerData *esdp; + Uint64 thr_id, unique; + Uint hsz; + Eterm *hp; + + esdp = ERTS_PROC_GET_SCHDATA(c_p); + thr_id = (Uint64) esdp->thr_id; + unique = esdp->unique++; + bld_unique_integer_term(NULL, &hsz, thr_id, unique, positive); + hp = hsz ? HAlloc(c_p, hsz) : NULL; + return bld_unique_integer_term(&hp, NULL, thr_id, unique, positive); +} + +Uint +erts_raw_unique_integer_heap_size(Uint64 val[ERTS_UNIQUE_INT_RAW_VALUES], + int positive) +{ + Uint sz; + bld_unique_integer_term(NULL, &sz, val[0], val[1], positive); + return sz; +} + +Eterm +erts_raw_make_unique_integer(Eterm **hpp, Uint64 val[ERTS_UNIQUE_INT_RAW_VALUES], + int positive) +{ + return bld_unique_integer_term(hpp, NULL, val[0], val[1], positive); +} + +void +erts_raw_get_unique_integer(Uint64 val[ERTS_UNIQUE_INT_RAW_VALUES]) +{ + ErtsSchedulerData *esdp = erts_get_scheduler_data(); + if (esdp) { + val[0] = (Uint64) esdp->thr_id; + val[1] = esdp->unique++; + } + else { + val[0] = (Uint64) 0; + val[1] = (Uint64) erts_atomic64_inc_read_nob(&unique_data.w.val1); + } +} + + +Sint64 +erts_get_min_unique_integer(void) +{ + return (Sint64) MIN_SMALL; +} + +/* --- Debug --- */ + +Eterm +erts_debug_make_unique_integer(Process *c_p, Eterm etval0, Eterm etval1) +{ + Uint64 val0, val1; + Uint hsz; + Eterm res, *hp, *end_hp; + + if (!term_to_Uint64(etval0, &val0)) + return THE_NON_VALUE; + + if (!term_to_Uint64(etval1, &val1)) + return THE_NON_VALUE; + + bld_unique_integer_term(NULL, &hsz, val0, val1, 0); + + hp = HAlloc(c_p, hsz); + end_hp = hp + hsz; + + res = bld_unique_integer_term(&hp, NULL, val0, val1, 0); + if (hp != end_hp) + ERTS_INTERNAL_ERROR("Heap allocation error"); + + return res; +} + +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *\ + * Strict Monotonic Counter * +\* */ + +static struct { + union { + erts_atomic64_t value; + char align__[ERTS_CACHE_LINE_SIZE]; + } w; +} raw_unique_monotonic_integer erts_align_attribute(ERTS_CACHE_LINE_SIZE); + +#if defined(ARCH_32) +# define ERTS_UNIQUE_MONOTONIC_OFFSET ERTS_SINT64_MIN +#else +# define ERTS_UNIQUE_MONOTONIC_OFFSET MIN_SMALL +#endif + +static void +init_unique_monotonic_integer(void) +{ + erts_atomic64_init_nob(&raw_unique_monotonic_integer.w.value, + (erts_aint64_t) -1); +} + +static ERTS_INLINE Uint64 +get_raw_unique_monotonic_integer(void) +{ + return (Uint64) erts_atomic64_inc_read_mb(&raw_unique_monotonic_integer.w.value); +} + +static ERTS_INLINE Uint +get_unique_monotonic_integer_heap_size(Uint64 raw, int positive) +{ + if (positive) { + Uint64 value = raw+1; + return ERTS_UINT64_HEAP_SIZE(value); + } + else { + Sint64 value = ((Sint64) raw) + ERTS_UNIQUE_MONOTONIC_OFFSET; + if (IS_SSMALL(value)) + return 0; +#if defined(ARCH_32) + return ERTS_SINT64_HEAP_SIZE(value); +#else + return ERTS_UINT64_HEAP_SIZE((Uint64) value); +#endif + } +} + +static ERTS_INLINE Eterm +make_unique_monotonic_integer_value(Eterm *hp, Uint hsz, Uint64 raw, int positive) +{ + Eterm res; +#ifdef DEBUG + Eterm *end_hp = hp + hsz; +#endif + + if (positive) { + Uint64 value = raw+1; + res = hsz ? erts_uint64_to_big(value, &hp) : make_small(value); + } + else { + Sint64 value = ((Sint64) raw) + ERTS_UNIQUE_MONOTONIC_OFFSET; + if (hsz == 0) + res = make_small(value); + else { +#if defined(ARCH_32) + res = erts_sint64_to_big(value, &hp); +#else + res = erts_uint64_to_big((Uint64) value, &hp); +#endif + } + } + + ASSERT(end_hp == hp); + + return res; +} + +static ERTS_INLINE Eterm +unique_monotonic_integer_bif(Process *c_p, int positive) +{ + Uint64 raw; + Uint hsz; + Eterm *hp; + + raw = get_raw_unique_monotonic_integer(); + hsz = get_unique_monotonic_integer_heap_size(raw, positive); + hp = hsz ? HAlloc(c_p, hsz) : NULL; + return make_unique_monotonic_integer_value(hp, hsz, raw, positive); +} + +Sint64 +erts_raw_get_unique_monotonic_integer(void) +{ + return get_raw_unique_monotonic_integer(); +} + +Uint +erts_raw_unique_monotonic_integer_heap_size(Sint64 raw, int positive) +{ + return get_unique_monotonic_integer_heap_size(raw, positive); +} + +Eterm +erts_raw_make_unique_monotonic_integer_value(Eterm **hpp, Sint64 raw, int positive) +{ + Uint hsz = get_unique_monotonic_integer_heap_size(raw, positive); + Eterm res = make_unique_monotonic_integer_value(*hpp, hsz, raw, positive); + *hpp += hsz; + return res; +} + +Sint64 +erts_get_min_unique_monotonic_integer(void) +{ + return ERTS_UNIQUE_MONOTONIC_OFFSET; +} + +/* --- Debug --- */ + +int +erts_debug_set_unique_monotonic_integer_state(Eterm et_value) +{ + Sint64 value; + + if (!term_to_Sint64(et_value, &value)) { + Uint64 uvalue; + if (!term_to_Uint64(et_value, &uvalue)) + return 0; + value = (Sint64) uvalue; + } + + erts_atomic64_set_mb(&raw_unique_monotonic_integer.w.value, + (erts_aint64_t) value); + return 1; +} + +Eterm +erts_debug_get_unique_monotonic_integer_state(Process *c_p) +{ + Uint64 value; + Eterm hsz, *hp; + + value = (Uint64) erts_atomic64_read_mb(&raw_unique_monotonic_integer.w.value); + + if (IS_USMALL(0, value)) + return make_small(value); + hsz = ERTS_UINT64_HEAP_SIZE(value); + hp = HAlloc(c_p, hsz); + return erts_uint64_to_big(value, &hp); +} + +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *\ + * Initilazation * +\* */ + +void +erts_bif_unique_init(void) +{ + init_reference(); + init_unique_monotonic_integer(); + init_unique_integer(); +} + +void +erts_sched_bif_unique_init(ErtsSchedulerData *esdp) +{ + esdp->unique = (Uint64) 0; + esdp->ref = (Uint64) 0; +} + +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *\ + * The BIFs * +\* */ + + +BIF_RETTYPE make_ref_0(BIF_ALIST_0) +{ + BIF_RETTYPE res; + Eterm* hp; + + ERTS_SMP_LC_ASSERT(ERTS_PROC_LOCK_MAIN & erts_proc_lc_my_proc_locks(BIF_P)); + + hp = HAlloc(BIF_P, REF_THING_SIZE); + + res = erts_sched_make_ref_in_buffer(ERTS_PROC_GET_SCHDATA(BIF_P), hp); + + BIF_RET(res); +} + +BIF_RETTYPE unique_integer_0(BIF_ALIST_0) +{ + BIF_RET(unique_integer_bif(BIF_P, 0)); +} + +BIF_RETTYPE unique_integer_1(BIF_ALIST_1) +{ + Eterm modlist = BIF_ARG_1; + int monotonic = 0; + int positive = 0; + BIF_RETTYPE res; + + while (is_list(modlist)) { + Eterm *consp = list_val(modlist); + switch (CAR(consp)) { + case am_monotonic: + monotonic = 1; + break; + case am_positive: + positive = 1; + break; + default: + BIF_ERROR(BIF_P, BADARG); + } + modlist = CDR(consp); + } + + if (is_not_nil(modlist)) + BIF_ERROR(BIF_P, BADARG); + + if (monotonic) + res = unique_monotonic_integer_bif(BIF_P, positive); + else + res = unique_integer_bif(BIF_P, positive); + + BIF_RET(res); +} diff --git a/erts/emulator/beam/erl_bif_unique.h b/erts/emulator/beam/erl_bif_unique.h new file mode 100644 index 0000000000..94fd6163f2 --- /dev/null +++ b/erts/emulator/beam/erl_bif_unique.h @@ -0,0 +1,136 @@ +/* + * %CopyrightBegin% + * + * Copyright Ericsson AB 2014. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * %CopyrightEnd% + */ + +#ifndef ERTS_BIF_UNIQUE_H__ +#define ERTS_BIF_UNIQUE_H__ + +#include "erl_process.h" +#include "big.h" + +void erts_bif_unique_init(void); +void erts_sched_bif_unique_init(ErtsSchedulerData *esdp); + +/* reference */ +Eterm erts_make_ref(Process *); +Eterm erts_make_ref_in_buffer(Eterm buffer[REF_THING_SIZE]); +void erts_make_ref_in_array(Uint32 ref[ERTS_MAX_REF_NUMBERS]); + +/* strict monotonic counter */ + +#define ERTS_MAX_UNIQUE_MONOTONIC_INTEGER_HEAP_SIZE ERTS_MAX_UINT64_HEAP_SIZE + +/* + * Note that a raw value is an intermediate value that + * not necessarily correspond to the end result. + */ +Sint64 erts_raw_get_unique_monotonic_integer(void); +Uint erts_raw_unique_monotonic_integer_heap_size(Sint64 raw, int positive); +Eterm erts_raw_make_unique_monotonic_integer_value(Eterm **hpp, Sint64 raw, + int positive); + +Sint64 erts_get_min_unique_monotonic_integer(void); + +int erts_debug_set_unique_monotonic_integer_state(Eterm et_value); +Eterm erts_debug_get_unique_monotonic_integer_state(Process *c_p); + +/* unique integer */ +#define ERTS_UNIQUE_INT_RAW_VALUES 2 +#define ERTS_MAX_UNIQUE_INT_HEAP_SIZE ERTS_UINT64_ARRAY_TO_BIG_MAX_HEAP_SZ(2) + +Uint erts_raw_unique_integer_heap_size(Uint64 val[ERTS_UNIQUE_INT_RAW_VALUES], + int positive); +Eterm erts_raw_make_unique_integer(Eterm **hpp, + Uint64 val[ERTS_UNIQUE_INT_RAW_VALUES], + int postive); +void erts_raw_get_unique_integer(Uint64 val[ERTS_UNIQUE_INT_RAW_VALUES]); +Sint64 erts_get_min_unique_integer(void); + +Eterm erts_debug_make_unique_integer(Process *c_p, + Eterm etval0, + Eterm etval1); + + +ERTS_GLB_INLINE void erts_set_ref_numbers(Uint32 ref[ERTS_MAX_REF_NUMBERS], + Uint32 thr_id, Uint64 value); +ERTS_GLB_INLINE Uint32 erts_get_ref_numbers_thr_id(Uint32 ref[ERTS_MAX_REF_NUMBERS]); +ERTS_GLB_INLINE Uint64 erts_get_ref_numbers_value(Uint32 ref[ERTS_MAX_REF_NUMBERS]); +ERTS_GLB_INLINE void erts_sched_make_ref_in_array(ErtsSchedulerData *esdp, + Uint32 ref[ERTS_MAX_REF_NUMBERS]); +ERTS_GLB_INLINE Eterm erts_sched_make_ref_in_buffer(ErtsSchedulerData *esdp, + Eterm buffer[REF_THING_SIZE]); + +#if ERTS_GLB_INLINE_INCL_FUNC_DEF + +ERTS_GLB_INLINE void +erts_set_ref_numbers(Uint32 ref[ERTS_MAX_REF_NUMBERS], Uint32 thr_id, Uint64 value) +{ + /* + * We cannot use thread id in the first 18-bit word since + * the hash/phash/phash2 BIFs only hash on this word. If + * we did, we would get really poor hash values. Instead + * we have to shuffle the bits a bit. + */ + ASSERT(thr_id == (thr_id & ((Uint32) 0x3ffff))); + ref[0] = (Uint32) (value & ((Uint64) 0x3ffff)); + ref[1] = (((Uint32) (value & ((Uint64) 0xfffc0000))) + | (thr_id & ((Uint32) 0x3ffff))); + ref[2] = (Uint32) ((value >> 32) & ((Uint64) 0xffffffff)); +} + +ERTS_GLB_INLINE Uint32 +erts_get_ref_numbers_thr_id(Uint32 ref[ERTS_MAX_REF_NUMBERS]) +{ + return ref[1] & ((Uint32) 0x3ffff); +} + +ERTS_GLB_INLINE Uint64 +erts_get_ref_numbers_value(Uint32 ref[ERTS_MAX_REF_NUMBERS]) +{ + return (((((Uint64) ref[2]) & ((Uint64) 0xffffffff)) << 32) + | (((Uint64) ref[1]) & ((Uint64) 0xfffc0000)) + | (((Uint64) ref[0]) & ((Uint64) 0x3ffff))); +} + +ERTS_GLB_INLINE void +erts_sched_make_ref_in_array(ErtsSchedulerData *esdp, + Uint32 ref[ERTS_MAX_REF_NUMBERS]) +{ + Uint64 value; + + ASSERT(esdp); + value = esdp->ref++; + erts_set_ref_numbers(ref, (Uint32) esdp->thr_id, value); +} + +ERTS_GLB_INLINE Eterm +erts_sched_make_ref_in_buffer(ErtsSchedulerData *esdp, + Eterm buffer[REF_THING_SIZE]) +{ + Eterm* hp = buffer; + Uint32 ref[ERTS_MAX_REF_NUMBERS]; + + erts_sched_make_ref_in_array(esdp, ref); + write_ref_thing(hp, ref[0], ref[1], ref[2]); + return make_internal_ref(hp); +} + +#endif /* ERTS_GLB_INLINE_INCL_FUNC_DEF */ + +#endif /* ERTS_BIF_UNIQUE_H__ */ diff --git a/erts/emulator/beam/erl_binary.h b/erts/emulator/beam/erl_binary.h index 819b19e566..e181b5555d 100644 --- a/erts/emulator/beam/erl_binary.h +++ b/erts/emulator/beam/erl_binary.h @@ -3,16 +3,17 @@ * * Copyright Ericsson AB 2000-2013. All Rights Reserved. * - * The contents of this file are subject to the Erlang Public License, - * Version 1.1, (the "License"); you may not use this file except in - * compliance with the License. You should have received a copy of the - * Erlang Public License along with this software. If not, it can be - * retrieved online at http://www.erlang.org/. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Software distributed under the License is distributed on an "AS IS" - * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See - * the License for the specific language governing rights and limitations - * under the License. + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. * * %CopyrightEnd% */ @@ -71,7 +72,6 @@ typedef struct erl_heap_bin { */ #define binary_size(Bin) (binary_val(Bin)[1]) -#define binary_size_rel(Bin,BasePtr) (binary_val_rel(Bin,BasePtr)[1]) #define binary_bitsize(Bin) \ ((*binary_val(Bin) == HEADER_SUB_BIN) ? \ @@ -94,12 +94,9 @@ typedef struct erl_heap_bin { * Bitsize: output variable (Uint) */ -#define ERTS_GET_BINARY_BYTES(Bin,Bytep,Bitoffs,Bitsize) \ - ERTS_GET_BINARY_BYTES_REL(Bin,Bytep,Bitoffs,Bitsize,NULL) - -#define ERTS_GET_BINARY_BYTES_REL(Bin,Bytep,Bitoffs,Bitsize,BasePtr) \ +#define ERTS_GET_BINARY_BYTES(Bin,Bytep,Bitoffs,Bitsize) \ do { \ - Eterm* _real_bin = binary_val_rel(Bin,BasePtr); \ + Eterm* _real_bin = binary_val(Bin); \ Uint _offs = 0; \ Bitoffs = Bitsize = 0; \ if (*_real_bin == HEADER_SUB_BIN) { \ @@ -107,7 +104,7 @@ do { \ _offs = _sb->offs; \ Bitoffs = _sb->bitoffs; \ Bitsize = _sb->bitsize; \ - _real_bin = binary_val_rel(_sb->orig,BasePtr); \ + _real_bin = binary_val(_sb->orig); \ } \ if (*_real_bin == HEADER_PROC_BIN) { \ Bytep = ((ProcBin *) _real_bin)->bytes + _offs; \ @@ -130,11 +127,8 @@ do { \ */ #define ERTS_GET_REAL_BIN(Bin, RealBin, ByteOffset, BitOffset, BitSize) \ - ERTS_GET_REAL_BIN_REL(Bin, RealBin, ByteOffset, BitOffset, BitSize, NULL) - -#define ERTS_GET_REAL_BIN_REL(Bin, RealBin, ByteOffset, BitOffset, BitSize, BasePtr) \ do { \ - ErlSubBin* _sb = (ErlSubBin *) binary_val_rel(Bin,BasePtr); \ + ErlSubBin* _sb = (ErlSubBin *) binary_val(Bin); \ if (_sb->thing_word == HEADER_SUB_BIN) { \ RealBin = _sb->orig; \ ByteOffset = _sb->offs; \ @@ -166,7 +160,7 @@ Eterm erts_bin_bytes_to_list(Eterm previous, Eterm* hp, byte* bytes, Uint size, * Common implementation for erlang:list_to_binary/1 and binary:list_to_bin/1 */ -BIF_RETTYPE erts_list_to_binary_bif(Process *p, Eterm arg); +BIF_RETTYPE erts_list_to_binary_bif(Process *p, Eterm arg, Export *bif); BIF_RETTYPE erts_gc_binary_part(Process *p, Eterm *reg, Eterm live, int range_is_tuple); BIF_RETTYPE erts_binary_part(Process *p, Eterm binary, Eterm epos, Eterm elen); @@ -194,6 +188,9 @@ ERTS_GLB_INLINE Binary *erts_bin_nrml_alloc(Uint size); ERTS_GLB_INLINE Binary *erts_bin_realloc_fnf(Binary *bp, Uint size); ERTS_GLB_INLINE Binary *erts_bin_realloc(Binary *bp, Uint size); ERTS_GLB_INLINE void erts_bin_free(Binary *bp); +ERTS_GLB_INLINE Binary *erts_create_magic_binary_x(Uint size, + void (*destructor)(Binary *), + int unaligned); ERTS_GLB_INLINE Binary *erts_create_magic_binary(Uint size, void (*destructor)(Binary *)); @@ -231,35 +228,58 @@ erts_free_aligned_binary_bytes(byte* buf) # define CHICKEN_PAD (sizeof(void*) - 1) #endif +/* Caller must initialize 'refc' +*/ ERTS_GLB_INLINE Binary * erts_bin_drv_alloc_fnf(Uint size) { Uint bsize = ERTS_SIZEOF_Binary(size) + CHICKEN_PAD; - void *res; + Binary *res; + + if (bsize < size) /* overflow */ + return NULL; res = erts_alloc_fnf(ERTS_ALC_T_DRV_BINARY, bsize); ERTS_CHK_BIN_ALIGNMENT(res); - return (Binary *) res; + if (res) { + res->orig_size = size; + res->flags = BIN_FLAG_DRV; + } + return res; } +/* Caller must initialize 'refc' +*/ ERTS_GLB_INLINE Binary * erts_bin_drv_alloc(Uint size) { Uint bsize = ERTS_SIZEOF_Binary(size) + CHICKEN_PAD; - void *res; + Binary *res; + + if (bsize < size) /* overflow */ + erts_alloc_enomem(ERTS_ALC_T_DRV_BINARY, size); res = erts_alloc(ERTS_ALC_T_DRV_BINARY, bsize); ERTS_CHK_BIN_ALIGNMENT(res); - return (Binary *) res; + res->orig_size = size; + res->flags = BIN_FLAG_DRV; + return res; } +/* Caller must initialize 'refc' +*/ ERTS_GLB_INLINE Binary * erts_bin_nrml_alloc(Uint size) { Uint bsize = ERTS_SIZEOF_Binary(size) + CHICKEN_PAD; - void *res; + Binary *res; + + if (bsize < size) /* overflow */ + erts_alloc_enomem(ERTS_ALC_T_BINARY, size); res = erts_alloc(ERTS_ALC_T_BINARY, bsize); ERTS_CHK_BIN_ALIGNMENT(res); - return (Binary *) res; + res->orig_size = size; + res->flags = 0; + return res; } ERTS_GLB_INLINE Binary * @@ -267,12 +287,15 @@ erts_bin_realloc_fnf(Binary *bp, Uint size) { Binary *nbp; Uint bsize = ERTS_SIZEOF_Binary(size) + CHICKEN_PAD; + ErtsAlcType_t type = (bp->flags & BIN_FLAG_DRV) ? ERTS_ALC_T_DRV_BINARY + : ERTS_ALC_T_BINARY; ASSERT((bp->flags & BIN_FLAG_MAGIC) == 0); - if (bp->flags & BIN_FLAG_DRV) - nbp = erts_realloc_fnf(ERTS_ALC_T_DRV_BINARY, (void *) bp, bsize); - else - nbp = erts_realloc_fnf(ERTS_ALC_T_BINARY, (void *) bp, bsize); + if (bsize < size) /* overflow */ + return NULL; + nbp = erts_realloc_fnf(type, (void *) bp, bsize); ERTS_CHK_BIN_ALIGNMENT(nbp); + if (nbp) + nbp->orig_size = size; return nbp; } @@ -281,18 +304,16 @@ erts_bin_realloc(Binary *bp, Uint size) { Binary *nbp; Uint bsize = ERTS_SIZEOF_Binary(size) + CHICKEN_PAD; + ErtsAlcType_t type = (bp->flags & BIN_FLAG_DRV) ? ERTS_ALC_T_DRV_BINARY + : ERTS_ALC_T_BINARY; ASSERT((bp->flags & BIN_FLAG_MAGIC) == 0); - if (bp->flags & BIN_FLAG_DRV) - nbp = erts_realloc_fnf(ERTS_ALC_T_DRV_BINARY, (void *) bp, bsize); - else - nbp = erts_realloc_fnf(ERTS_ALC_T_BINARY, (void *) bp, bsize); + if (bsize < size) /* overflow */ + erts_realloc_enomem(type, bp, size); + nbp = erts_realloc_fnf(type, (void *) bp, bsize); if (!nbp) - erts_realloc_n_enomem(ERTS_ALC_T2N(bp->flags & BIN_FLAG_DRV - ? ERTS_ALC_T_DRV_BINARY - : ERTS_ALC_T_BINARY), - bp, - bsize); + erts_realloc_enomem(type, bp, bsize); ERTS_CHK_BIN_ALIGNMENT(nbp); + nbp->orig_size = size; return nbp; } @@ -308,20 +329,30 @@ erts_bin_free(Binary *bp) } ERTS_GLB_INLINE Binary * -erts_create_magic_binary(Uint size, void (*destructor)(Binary *)) +erts_create_magic_binary_x(Uint size, void (*destructor)(Binary *), + int unaligned) { - Uint bsize = ERTS_MAGIC_BIN_SIZE(size); + Uint bsize = unaligned ? ERTS_MAGIC_BIN_UNALIGNED_SIZE(size) + : ERTS_MAGIC_BIN_SIZE(size); Binary* bptr = erts_alloc_fnf(ERTS_ALC_T_BINARY, bsize); + ASSERT(bsize > size); if (!bptr) erts_alloc_n_enomem(ERTS_ALC_T2N(ERTS_ALC_T_BINARY), bsize); ERTS_CHK_BIN_ALIGNMENT(bptr); bptr->flags = BIN_FLAG_MAGIC; - bptr->orig_size = ERTS_MAGIC_BIN_ORIG_SIZE(size); + bptr->orig_size = unaligned ? ERTS_MAGIC_BIN_UNALIGNED_ORIG_SIZE(size) + : ERTS_MAGIC_BIN_ORIG_SIZE(size); erts_refc_init(&bptr->refc, 0); ERTS_MAGIC_BIN_DESTRUCTOR(bptr) = destructor; return bptr; } +ERTS_GLB_INLINE Binary * +erts_create_magic_binary(Uint size, void (*destructor)(Binary *)) +{ + return erts_create_magic_binary_x(size, destructor, 0); +} + #endif /* #if ERTS_GLB_INLINE_INCL_FUNC_DEF */ -#endif +#endif /* !__ERL_BINARY_H */ diff --git a/erts/emulator/beam/erl_bits.c b/erts/emulator/beam/erl_bits.c index 73765772c8..11d83686a3 100644 --- a/erts/emulator/beam/erl_bits.c +++ b/erts/emulator/beam/erl_bits.c @@ -3,16 +3,17 @@ * * Copyright Ericsson AB 1999-2013. All Rights Reserved. * - * The contents of this file are subject to the Erlang Public License, - * Version 1.1, (the "License"); you may not use this file except in - * compliance with the License. You should have received a copy of the - * Erlang Public License along with this software. If not, it can be - * retrieved online at http://www.erlang.org/. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Software distributed under the License is distributed on an "AS IS" - * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See - * the License for the specific language governing rights and limitations - * under the License. + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. * * %CopyrightEnd% */ @@ -107,6 +108,14 @@ erts_bits_destroy_state(ERL_BITS_PROTO_0) void erts_init_bits(void) { + ERTS_CT_ASSERT(offsetof(Binary,orig_bytes) % 8 == 0); + ERTS_CT_ASSERT(offsetof(ErtsMagicBinary,u.aligned.data) % 8 == 0); + ERTS_CT_ASSERT(ERTS_MAGIC_BIN_BYTES_TO_ALIGN == + (offsetof(ErtsMagicBinary,u.aligned.data) + - offsetof(ErtsMagicBinary,u.unaligned.data))); + ERTS_CT_ASSERT(offsetof(ErtsBinary,driver.binary.orig_bytes) + == offsetof(Binary,orig_bytes)); + erts_smp_atomic_init_nob(&bits_bufs_size, 0); #if defined(ERTS_SMP) /* erl_process.c calls erts_bits_init_state() on all state instances */ @@ -165,6 +174,26 @@ erts_bs_start_match_2(Process *p, Eterm Binary, Uint Max) return make_matchstate(ms); } +#ifdef DEBUG +# define CHECK_MATCH_BUFFER(MB) check_match_buffer(MB) + +static void check_match_buffer(ErlBinMatchBuffer* mb) +{ + Eterm realbin; + Uint byteoffs; + byte* bytes, bitoffs, bitsz; + ProcBin* pb; + ERTS_GET_REAL_BIN(mb->orig, realbin, byteoffs, bitoffs, bitsz); + bytes = binary_bytes(realbin) + byteoffs; + ERTS_ASSERT(mb->base >= bytes && mb->base <= (bytes + binary_size(mb->orig))); + pb = (ProcBin *) boxed_val(realbin); + if (pb->thing_word == HEADER_PROC_BIN) + ERTS_ASSERT(pb->flags == 0); +} +#else +# define CHECK_MATCH_BUFFER(MB) +#endif + Eterm erts_bs_get_integer_2(Process *p, Uint num_bits, unsigned flags, ErlBinMatchBuffer* mb) { @@ -185,6 +214,7 @@ erts_bs_get_integer_2(Process *p, Uint num_bits, unsigned flags, ErlBinMatchBuff return SMALL_ZERO; } + CHECK_MATCH_BUFFER(mb); if (mb->size - mb->offset < num_bits) { /* Asked for too many bits. */ return THE_NON_VALUE; } @@ -252,7 +282,7 @@ erts_bs_get_integer_2(Process *p, Uint num_bits, unsigned flags, ErlBinMatchBuff * Simply shift whole bytes into the result. */ switch (BYTE_OFFSET(n)) { -#if defined(ARCH_64) && !HALFWORD_HEAP +#if defined(ARCH_64) case 7: w = (w << 8) | *bp++; case 6: w = (w << 8) | *bp++; case 5: w = (w << 8) | *bp++; @@ -357,7 +387,7 @@ erts_bs_get_integer_2(Process *p, Uint num_bits, unsigned flags, ErlBinMatchBuff case 3: v32 = LSB[0] + (LSB[1]<<8) + (LSB[2]<<16); goto big_small; -#if !defined(ARCH_64) || HALFWORD_HEAP +#if !defined(ARCH_64) case 4: v32 = (LSB[0] + (LSB[1]<<8) + (LSB[2]<<16) + (LSB[3]<<24)); if (!IS_USMALL(sgn, v32)) { @@ -403,7 +433,10 @@ erts_bs_get_integer_2(Process *p, Uint num_bits, unsigned flags, ErlBinMatchBuff words_needed = 1+WSIZE(bytes); hp = HeapOnlyAlloc(p, words_needed); res = bytes_to_big(LSB, bytes, sgn, hp); - if (is_small(res)) { + if (is_nil(res)) { + p->htop = hp; + res = THE_NON_VALUE; + } else if (is_small(res)) { p->htop = hp; } else if ((actual = bignum_header_arity(*hp)+1) < words_needed) { p->htop = hp + actual; @@ -422,6 +455,7 @@ erts_bs_get_binary_2(Process *p, Uint num_bits, unsigned flags, ErlBinMatchBuffe { ErlSubBin* sb; + CHECK_MATCH_BUFFER(mb); if (mb->size - mb->offset < num_bits) { /* Asked for too many bits. */ return THE_NON_VALUE; } @@ -453,6 +487,7 @@ erts_bs_get_float_2(Process *p, Uint num_bits, unsigned flags, ErlBinMatchBuffer byte* fptr; FloatDef f; + CHECK_MATCH_BUFFER(mb); if (num_bits == 0) { f.fd = 0.0; hp = HeapOnlyAlloc(p, FLOAT_SIZE_OBJECT); @@ -506,6 +541,8 @@ erts_bs_get_binary_all_2(Process *p, ErlBinMatchBuffer* mb) { ErlSubBin* sb; Uint size; + + CHECK_MATCH_BUFFER(mb); size = mb->size-mb->offset; sb = (ErlSubBin *) HeapOnlyAlloc(p, ERL_SUB_BIN_SIZE); sb->thing_word = HEADER_SUB_BIN; @@ -1299,7 +1336,6 @@ erts_bs_append(Process* c_p, Eterm* reg, Uint live, Eterm build_size_term, if (binp->orig_size < pb->size) { Uint new_size = 2*pb->size; binp = erts_bin_realloc(binp, new_size); - binp->orig_size = new_size; pb->val = binp; pb->bytes = (byte *) binp->orig_bytes; } @@ -1371,8 +1407,6 @@ erts_bs_append(Process* c_p, Eterm* reg, Uint live, Eterm build_size_term, * Allocate the binary data struct itself. */ bptr = erts_bin_nrml_alloc(bin_size); - bptr->flags = 0; - bptr->orig_size = bin_size; erts_refc_init(&bptr->refc, 1); erts_current_bin = (byte *) bptr->orig_bytes; @@ -1475,7 +1509,6 @@ erts_bs_private_append(Process* p, Eterm bin, Eterm build_size_term, Uint unit) * is safe to reallocate it. */ binp = erts_bin_realloc(binp, new_size); - binp->orig_size = new_size; pb->val = binp; pb->bytes = (byte *) binp->orig_bytes; } else { @@ -1488,8 +1521,6 @@ erts_bs_private_append(Process* p, Eterm bin, Eterm build_size_term, Uint unit) * binary and copy the contents of the old binary into it. */ Binary* bptr = erts_bin_nrml_alloc(new_size); - bptr->flags = 0; - bptr->orig_size = new_size; erts_refc_init(&bptr->refc, 1); sys_memcpy(bptr->orig_bytes, binp->orig_bytes, binp->orig_size); pb->flags |= PB_IS_WRITABLE | PB_ACTIVE_WRITER; @@ -1537,8 +1568,6 @@ erts_bs_init_writable(Process* p, Eterm sz) * Allocate the binary data struct itself. */ bptr = erts_bin_nrml_alloc(bin_size); - bptr->flags = 0; - bptr->orig_size = bin_size; erts_refc_init(&bptr->refc, 1); /* @@ -1585,9 +1614,7 @@ erts_emasculate_writable_binary(ProcBin* pb) /* Our allocators are 8 byte aligned, i.e., shrinking with less than 8 bytes will have no real effect */ if (unused >= 8) { - Uint new_size = pb->size; binp = erts_bin_realloc(binp, pb->size); - binp->orig_size = new_size; pb->val = binp; pb->bytes = (byte *) binp->orig_bytes; } @@ -1602,6 +1629,7 @@ erts_bs_get_unaligned_uint32(ErlBinMatchBuffer* mb) byte* LSB; byte* MSB; + CHECK_MATCH_BUFFER(mb); ASSERT((mb->offset & 7) != 0); ASSERT(mb->size - mb->offset >= 32); @@ -1661,6 +1689,8 @@ erts_bs_get_utf8(ErlBinMatchBuffer* mb) 2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2, 3,3,3,3,3,3,3,3,9,9,9,9,9,9,9,9 }; + CHECK_MATCH_BUFFER(mb); + if ((remaining_bits = mb->size - mb->offset) < 8) { return THE_NON_VALUE; } @@ -1745,6 +1775,7 @@ erts_bs_get_utf16(ErlBinMatchBuffer* mb, Uint flags) return THE_NON_VALUE; } + CHECK_MATCH_BUFFER(mb); /* * Set up the pointer to the source bytes. */ diff --git a/erts/emulator/beam/erl_bits.h b/erts/emulator/beam/erl_bits.h index 388d943755..8b7807fbd9 100644 --- a/erts/emulator/beam/erl_bits.h +++ b/erts/emulator/beam/erl_bits.h @@ -3,16 +3,17 @@ * * Copyright Ericsson AB 1999-2011. All Rights Reserved. * - * The contents of this file are subject to the Erlang Public License, - * Version 1.1, (the "License"); you may not use this file except in - * compliance with the License. You should have received a copy of the - * Erlang Public License along with this software. If not, it can be - * retrieved online at http://www.erlang.org/. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Software distributed under the License is distributed on an "AS IS" - * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See - * the License for the specific language governing rights and limitations - * under the License. + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. * * %CopyrightEnd% */ diff --git a/erts/emulator/beam/erl_cpu_topology.c b/erts/emulator/beam/erl_cpu_topology.c index f594cb9392..c263eebfcc 100644 --- a/erts/emulator/beam/erl_cpu_topology.c +++ b/erts/emulator/beam/erl_cpu_topology.c @@ -3,16 +3,17 @@ * * Copyright Ericsson AB 2010-2013. All Rights Reserved. * - * The contents of this file are subject to the Erlang Public License, - * Version 1.1, (the "License"); you may not use this file except in - * compliance with the License. You should have received a copy of the - * Erlang Public License along with this software. If not, it can be - * retrieved online at http://www.erlang.org/. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Software distributed under the License is distributed on an "AS IS" - * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See - * the License for the specific language governing rights and limitations - * under the License. + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. * * %CopyrightEnd% */ @@ -401,7 +402,7 @@ cpu_bind_order_sort(erts_cpu_topology_t *cpudata, break; default: cmp_func = NULL; - erl_exit(ERTS_ABORT_EXIT, + erts_exit(ERTS_ABORT_EXIT, "Bad cpu bind type: %d\n", (int) cpu_bind_order); break; @@ -1589,7 +1590,7 @@ get_cpu_topology_term(Process *c_p, int type) } break; default: - erl_exit(ERTS_ABORT_EXIT, "Bad cpu topology type: %d\n", type); + erts_exit(ERTS_ABORT_EXIT, "Bad cpu topology type: %d\n", type); break; } @@ -1966,7 +1967,7 @@ cpu_group_insert(erts_cpu_groups_map_t *map, ix = 0; } while (ix != start); - erl_exit(ERTS_ABORT_EXIT, "Reader groups map full\n"); + erts_exit(ERTS_ABORT_EXIT, "Reader groups map full\n"); } @@ -2289,7 +2290,7 @@ remove_cpu_groups(erts_cpu_groups_callback_t callback, void *arg) prev_cgm = cgm; } - erl_exit(ERTS_ABORT_EXIT, "Cpu groups not found\n"); + erts_exit(ERTS_ABORT_EXIT, "Cpu groups not found\n"); } static int @@ -2319,7 +2320,7 @@ cpu_groups_lookup(erts_cpu_groups_map_t *map, ix = 0; } while (ix != start); - erl_exit(ERTS_ABORT_EXIT, "Logical cpu id %d not found\n", logical); + erts_exit(ERTS_ABORT_EXIT, "Logical cpu id %d not found\n", logical); } static void diff --git a/erts/emulator/beam/erl_cpu_topology.h b/erts/emulator/beam/erl_cpu_topology.h index b502258dae..f3fbfc6da0 100644 --- a/erts/emulator/beam/erl_cpu_topology.h +++ b/erts/emulator/beam/erl_cpu_topology.h @@ -3,16 +3,17 @@ * * Copyright Ericsson AB 2010-2013. All Rights Reserved. * - * The contents of this file are subject to the Erlang Public License, - * Version 1.1, (the "License"); you may not use this file except in - * compliance with the License. You should have received a copy of the - * Erlang Public License along with this software. If not, it can be - * retrieved online at http://www.erlang.org/. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Software distributed under the License is distributed on an "AS IS" - * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See - * the License for the specific language governing rights and limitations - * under the License. + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. * * %CopyrightEnd% */ diff --git a/erts/emulator/beam/erl_db.c b/erts/emulator/beam/erl_db.c index a5d67571e2..441dbdde4f 100644 --- a/erts/emulator/beam/erl_db.c +++ b/erts/emulator/beam/erl_db.c @@ -3,16 +3,17 @@ * * Copyright Ericsson AB 1996-2014. All Rights Reserved. * - * The contents of this file are subject to the Erlang Public License, - * Version 1.1, (the "License"); you may not use this file except in - * compliance with the License. You should have received a copy of the - * Erlang Public License along with this software. If not, it can be - * retrieved online at http://www.erlang.org/. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Software distributed under the License is distributed on an "AS IS" - * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See - * the License for the specific language governing rights and limitations - * under the License. + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. * * %CopyrightEnd% */ @@ -259,10 +260,11 @@ static void schedule_free_dbtable(DbTable* tb) /* * NON-SMP case: Caller is *not* allowed to access the *tb * structure after this function has returned! - * SMP case: Caller is allowed to access the *tb structure - * until the bif has returned (we typically - * need to unlock the table lock after this - * function has returned). + * SMP case: Caller is allowed to access the *common* part of the *tb + * structure until the bif has returned (we typically need to + * unlock the table lock after this function has returned). + * Caller is *not* allowed to access the specialized part + * (hash or tree) of *tb after this function has returned. */ ASSERT(erts_refc_read(&tb->common.ref, 0) == 0); erts_schedule_thr_prgr_later_cleanup_op(free_dbtable, @@ -278,6 +280,8 @@ static ERTS_INLINE void db_init_lock(DbTable* tb, int use_frequent_read_lock, erts_smp_rwmtx_opt_t rwmtx_opt = ERTS_SMP_RWMTX_OPT_DEFAULT_INITER; if (use_frequent_read_lock) rwmtx_opt.type = ERTS_SMP_RWMTX_TYPE_FREQUENT_READ; + if (erts_ets_rwmtx_spin_count >= 0) + rwmtx_opt.main_spincount = erts_ets_rwmtx_spin_count; #endif #ifdef ERTS_SMP erts_smp_rwmtx_init_opt_x(&tb->common.rwlock, &rwmtx_opt, @@ -619,7 +623,7 @@ BIF_RETTYPE ets_safe_fixtable_2(BIF_ALIST_2) erts_fprintf(stderr, "ets:safe_fixtable(%T,%T); Process: %T, initial: %T:%T/%bpu\n", BIF_ARG_1, BIF_ARG_2, BIF_P->common.id, - BIF_P->initial[0], BIF_P->initial[1], BIF_P->initial[2]); + BIF_P->u.initial[0], BIF_P->u.initial[1], BIF_P->u.initial[2]); #endif kind = (BIF_ARG_2 == am_true) ? LCK_READ : LCK_WRITE_REC; @@ -752,6 +756,31 @@ BIF_RETTYPE ets_prev_2(BIF_ALIST_2) BIF_RET(ret); } +/* +** take(Tab, Key) +*/ +BIF_RETTYPE ets_take_2(BIF_ALIST_2) +{ + DbTable* tb; +#ifdef DEBUG + int cret; +#endif + Eterm ret; + CHECK_TABLES(); + + tb = db_get_table(BIF_P, BIF_ARG_1, DB_WRITE, LCK_WRITE_REC); + if (!tb) { + BIF_ERROR(BIF_P, BADARG); + } +#ifdef DEBUG + cret = +#endif + tb->common.meth->db_take(BIF_P, tb, BIF_ARG_2, &ret); + ASSERT(cret == DB_ERROR_NONE); + db_unlock(tb, LCK_WRITE_REC); + BIF_RET(ret); +} + /* ** update_element(Tab, Key, {Pos, Value}) ** update_element(Tab, Key, [{Pos, Value}]) @@ -779,7 +808,7 @@ BIF_RETTYPE ets_update_element_3(BIF_ALIST_3) list = BIF_ARG_3; } - if (!tb->common.meth->db_lookup_dbterm(tb, BIF_ARG_2, &handle)) { + if (!tb->common.meth->db_lookup_dbterm(BIF_P, tb, BIF_ARG_2, THE_NON_VALUE, &handle)) { cret = DB_ERROR_BADKEY; goto bail_out; } @@ -818,7 +847,7 @@ BIF_RETTYPE ets_update_element_3(BIF_ALIST_3) } finalize: - tb->common.meth->db_finalize_dbterm(&handle); + tb->common.meth->db_finalize_dbterm(cret, &handle); bail_out: UnUseTmpHeap(2,BIF_P); @@ -837,14 +866,8 @@ bail_out: } } -/* -** update_counter(Tab, Key, Incr) -** update_counter(Tab, Key, {Upop}) -** update_counter(Tab, Key, [{Upop}]) -** Upop = {Pos,Incr} | {Pos,Incr,Threshold,WarpTo} -** Returns new value(s) (integer or [integer]) -*/ -BIF_RETTYPE ets_update_counter_3(BIF_ALIST_3) +static BIF_RETTYPE +do_update_counter(Process *p, Eterm arg1, Eterm arg2, Eterm arg3, Eterm arg4) { DbTable* tb; int cret = DB_ERROR_BADITEM; @@ -854,7 +877,7 @@ BIF_RETTYPE ets_update_counter_3(BIF_ALIST_3) Eterm* ret_list_currp = NULL; Eterm* ret_list_prevp = NULL; Eterm iter; - DeclareTmpHeap(cell,5,BIF_P); + DeclareTmpHeap(cell, 5, p); Eterm *tuple = cell+2; DbUpdateHandle handle; Uint halloc_size = 0; /* overestimated heap usage */ @@ -862,28 +885,29 @@ BIF_RETTYPE ets_update_counter_3(BIF_ALIST_3) Eterm* hstart; Eterm* hend; - if ((tb = db_get_table(BIF_P, BIF_ARG_1, DB_WRITE, LCK_WRITE_REC)) == NULL) { - BIF_ERROR(BIF_P, BADARG); + if ((tb = db_get_table(p, arg1, DB_WRITE, LCK_WRITE_REC)) == NULL) { + BIF_ERROR(p, BADARG); } - UseTmpHeap(5,BIF_P); + UseTmpHeap(5, p); if (!(tb->common.status & (DB_SET | DB_ORDERED_SET))) { goto bail_out; } - if (is_integer(BIF_ARG_3)) { /* Incr */ - upop_list = CONS(cell, TUPLE2(tuple, make_small(tb->common.keypos+1), - BIF_ARG_3), NIL); + if (is_integer(arg3)) { /* Incr */ + upop_list = CONS(cell, + TUPLE2(tuple, make_small(tb->common.keypos+1), arg3), + NIL); } - else if (is_tuple(BIF_ARG_3)) { /* {Upop} */ - upop_list = CONS(cell, BIF_ARG_3, NIL); + else if (is_tuple(arg3)) { /* {Upop} */ + upop_list = CONS(cell, arg3, NIL); } else { /* [{Upop}] (probably) */ - upop_list = BIF_ARG_3; + upop_list = arg3; ret_list_prevp = &ret; } - if (!tb->common.meth->db_lookup_dbterm(tb, BIF_ARG_2, &handle)) { + if (!tb->common.meth->db_lookup_dbterm(p, tb, arg2, arg4, &handle)) { goto bail_out; /* key not found */ } @@ -956,13 +980,13 @@ BIF_RETTYPE ets_update_counter_3(BIF_ALIST_3) if (ret_list_prevp) { /* Prepare to return a list */ ret = NIL; halloc_size += list_size; - hstart = HAlloc(BIF_P, halloc_size); + hstart = HAlloc(p, halloc_size); ret_list_currp = hstart; htop = hstart + list_size; hend = hstart + halloc_size; } else { - hstart = htop = HAlloc(BIF_P, halloc_size); + hstart = htop = HAlloc(p, halloc_size); } hend = hstart + halloc_size; @@ -1009,26 +1033,54 @@ BIF_RETTYPE ets_update_counter_3(BIF_ALIST_3) (is_list(ret) && (list_val(ret)+list_size)==ret_list_currp)); ASSERT(htop <= hend); - HRelease(BIF_P,hend,htop); + HRelease(p, hend, htop); finalize: - tb->common.meth->db_finalize_dbterm(&handle); + tb->common.meth->db_finalize_dbterm(cret, &handle); bail_out: - UnUseTmpHeap(5,BIF_P); + UnUseTmpHeap(5, p); db_unlock(tb, LCK_WRITE_REC); switch (cret) { case DB_ERROR_NONE: BIF_RET(ret); case DB_ERROR_SYSRES: - BIF_ERROR(BIF_P, SYSTEM_LIMIT); + BIF_ERROR(p, SYSTEM_LIMIT); default: - BIF_ERROR(BIF_P, BADARG); + BIF_ERROR(p, BADARG); break; } } +/* +** update_counter(Tab, Key, Incr) +** update_counter(Tab, Key, Upop) +** update_counter(Tab, Key, [{Upop}]) +** Upop = {Pos,Incr} | {Pos,Incr,Threshold,WarpTo} +** Returns new value(s) (integer or [integer]) +*/ +BIF_RETTYPE ets_update_counter_3(BIF_ALIST_3) +{ + return do_update_counter(BIF_P, BIF_ARG_1, BIF_ARG_2, BIF_ARG_3, THE_NON_VALUE); +} + +/* +** update_counter(Tab, Key, Incr, Default) +** update_counter(Tab, Key, Upop, Default) +** update_counter(Tab, Key, [{Upop}], Default) +** Upop = {Pos,Incr} | {Pos,Incr,Threshold,WarpTo} +** Returns new value(s) (integer or [integer]) +*/ +BIF_RETTYPE ets_update_counter_4(BIF_ALIST_4) +{ + if (is_not_tuple(BIF_ARG_4)) { + BIF_ERROR(BIF_P, BADARG); + } + return do_update_counter(BIF_P, BIF_ARG_1, BIF_ARG_2, BIF_ARG_3, BIF_ARG_4); +} + + /* ** The put BIF */ @@ -1198,7 +1250,7 @@ BIF_RETTYPE ets_rename_2(BIF_ALIST_2) erts_fprintf(stderr, "ets:rename(%T,%T); Process: %T, initial: %T:%T/%bpu\n", BIF_ARG_1, BIF_ARG_2, BIF_P->common.id, - BIF_P->initial[0], BIF_P->initial[1], BIF_P->initial[2]); + BIF_P->u.initial[0], BIF_P->u.initial[1], BIF_P->u.initial[2]); #endif @@ -1243,7 +1295,7 @@ BIF_RETTYPE ets_rename_2(BIF_ALIST_2) goto badarg; if (!remove_named_tab(tb, 1)) - erl_exit(1,"Could not find named tab %s", tb->common.id); + erts_exit(ERTS_ERROR_EXIT,"Could not find named tab %s", tb->common.id); tb->common.id = tb->common.the_name = BIF_ARG_2; @@ -1324,7 +1376,6 @@ BIF_RETTYPE ets_new_2(BIF_ALIST_2) status |= DB_ORDERED_SET; status &= ~(DB_SET | DB_BAG | DB_DUPLICATE_BAG); } - /*TT*/ else if (is_tuple(val)) { Eterm *tp = tuple_val(val); if (arityval(tp[0]) == 2) { @@ -1514,7 +1565,7 @@ BIF_RETTYPE ets_new_2(BIF_ALIST_2) erts_fprintf(stderr, "ets:new(%T,%T)=%T; Process: %T, initial: %T:%T/%bpu\n", BIF_ARG_1, BIF_ARG_2, ret, BIF_P->common.id, - BIF_P->initial[0], BIF_P->initial[1], BIF_P->initial[2]); + BIF_P->u.initial[0], BIF_P->u.initial[1], BIF_P->u.initial[2]); erts_fprintf(stderr, "ets: new: meta_pid_to_tab common.memory_size = %ld\n", erts_smp_atomic_read_nob(&meta_pid_to_tab->common.memory_size)); erts_fprintf(stderr, "ets: new: meta_pid_to_fixed_tab common.memory_size = %ld\n", @@ -1529,7 +1580,7 @@ BIF_RETTYPE ets_new_2(BIF_ALIST_2) BIF_P->common.id, make_small(slot)), 0) != DB_ERROR_NONE) { - erl_exit(1,"Could not update ets metadata."); + erts_exit(ERTS_ERROR_EXIT,"Could not update ets metadata."); } db_meta_unlock(meta_pid_to_tab, LCK_WRITE_REC); @@ -1647,7 +1698,7 @@ BIF_RETTYPE ets_delete_1(BIF_ALIST_1) erts_fprintf(stderr, "ets:delete(%T); Process: %T, initial: %T:%T/%bpu\n", BIF_ARG_1, BIF_P->common.id, - BIF_P->initial[0], BIF_P->initial[1], BIF_P->initial[2]); + BIF_P->u.initial[0], BIF_P->u.initial[1], BIF_P->u.initial[2]); #endif CHECK_TABLES(); @@ -1722,15 +1773,9 @@ BIF_RETTYPE ets_delete_1(BIF_ALIST_1) * (it looks like an continuation pointer), but that is will crash the * emulator if this BIF is call traced. */ -#if HALFWORD_HEAP - Eterm *hp = HAlloc(BIF_P, 3); - hp[0] = make_pos_bignum_header(2); - *((UWord *) (UWord) (hp+1)) = (UWord) tb; -#else Eterm *hp = HAlloc(BIF_P, 2); hp[0] = make_pos_bignum_header(1); hp[1] = (Eterm) tb; -#endif BIF_TRAP1(&ets_delete_continue_exp, BIF_P, make_big(hp)); } else { @@ -1786,7 +1831,7 @@ BIF_RETTYPE ets_give_away_3(BIF_ALIST_3) tb->common.id, from_pid, BIF_ARG_3), - 0); + 0); erts_smp_proc_unlock(to_proc, to_locks); UnUseTmpHeap(5,BIF_P); BIF_RET(am_true); @@ -2642,7 +2687,9 @@ BIF_RETTYPE ets_match_object_3(BIF_ALIST_3) BIF_RETTYPE ets_info_1(BIF_ALIST_1) { static Eterm fields[] = {am_protection, am_keypos, am_type, am_named_table, - am_node, am_size, am_name, am_heir, am_owner, am_memory, am_compressed}; + am_node, am_size, am_name, am_heir, am_owner, am_memory, am_compressed, + am_write_concurrency, + am_read_concurrency}; Eterm results[sizeof(fields)/sizeof(Eterm)]; DbTable* tb; Eterm res; @@ -2786,7 +2833,7 @@ BIF_RETTYPE ets_match_spec_run_r_3(BIF_ALIST_3) BIF_TRAP3(bif_export[BIF_ets_match_spec_run_r_3], BIF_P,lst,BIF_ARG_2,ret); } - res = db_prog_match(BIF_P, mp, CAR(list_val(lst)), NULL, NULL, 0, + res = db_prog_match(BIF_P, mp, CAR(list_val(lst)), NULL, 0, ERTS_PAM_COPY_RESULT, &dummy); if (is_value(res)) { hp = HAlloc(BIF_P, 2); @@ -2805,10 +2852,11 @@ BIF_RETTYPE ets_match_spec_run_r_3(BIF_ALIST_3) ** External interface (NOT BIF's) */ +int erts_ets_rwmtx_spin_count = -1; /* Init the db */ -void init_db(void) +void init_db(ErtsDbSpinCount db_spin_count) { DbTable init_tb; int i; @@ -2817,10 +2865,48 @@ void init_db(void) size_t size; #ifdef ERTS_SMP + int max_spin_count = (1 << 15) - 1; /* internal limit */ erts_smp_rwmtx_opt_t rwmtx_opt = ERTS_SMP_RWMTX_OPT_DEFAULT_INITER; rwmtx_opt.type = ERTS_SMP_RWMTX_TYPE_FREQUENT_READ; rwmtx_opt.lived = ERTS_SMP_RWMTX_LONG_LIVED; + switch (db_spin_count) { + case ERTS_DB_SPNCNT_NONE: + erts_ets_rwmtx_spin_count = 0; + break; + case ERTS_DB_SPNCNT_VERY_LOW: + erts_ets_rwmtx_spin_count = 100; + break; + case ERTS_DB_SPNCNT_LOW: + erts_ets_rwmtx_spin_count = 200; + erts_ets_rwmtx_spin_count += erts_no_schedulers * 50; + if (erts_ets_rwmtx_spin_count > 1000) + erts_ets_rwmtx_spin_count = 1000; + break; + case ERTS_DB_SPNCNT_HIGH: + erts_ets_rwmtx_spin_count = 2000; + erts_ets_rwmtx_spin_count += erts_no_schedulers * 100; + if (erts_ets_rwmtx_spin_count > 15000) + erts_ets_rwmtx_spin_count = 15000; + break; + case ERTS_DB_SPNCNT_VERY_HIGH: + erts_ets_rwmtx_spin_count = 15000; + erts_ets_rwmtx_spin_count += erts_no_schedulers * 500; + if (erts_ets_rwmtx_spin_count > max_spin_count) + erts_ets_rwmtx_spin_count = max_spin_count; + break; + case ERTS_DB_SPNCNT_EXTREMELY_HIGH: + erts_ets_rwmtx_spin_count = max_spin_count; + break; + case ERTS_DB_SPNCNT_NORMAL: + default: + erts_ets_rwmtx_spin_count = -1; + break; + } + + if (erts_ets_rwmtx_spin_count >= 0) + rwmtx_opt.main_spincount = erts_ets_rwmtx_spin_count; + meta_main_tab_locks = erts_alloc_permanent_cache_aligned(ERTS_ALC_T_DB_TABLES, sizeof(erts_meta_main_tab_lock_t) @@ -2847,7 +2933,7 @@ void init_db(void) bits = erts_fit_in_bits_int32(db_max_tabs-1); if (bits > SMALL_BITS) { - erl_exit(1,"Max limit for ets tabled too high %u (max %u).", + erts_exit(ERTS_ERROR_EXIT,"Max limit for ets tabled too high %u (max %u).", db_max_tabs, ((Uint)1)<<SMALL_BITS); } meta_main_tab_slot_mask = (((Uint)1)<<bits) - 1; @@ -2908,7 +2994,7 @@ void init_db(void) db_init_lock(meta_pid_to_tab, "meta_pid_to_tab", "meta_pid_to_tab_FIX");*/ if (db_create_hash(NULL, meta_pid_to_tab) != DB_ERROR_NONE) { - erl_exit(1,"Unable to create ets metadata tables."); + erts_exit(ERTS_ERROR_EXIT,"Unable to create ets metadata tables."); } erts_smp_atomic_set_nob(&init_tb.common.memory_size, 0); @@ -2939,7 +3025,7 @@ void init_db(void) db_init_lock(meta_pid_to_fixed_tab, "meta_pid_to_fixed_tab", "meta_pid_to_fixed_tab_FIX");*/ if (db_create_hash(NULL, meta_pid_to_fixed_tab) != DB_ERROR_NONE) { - erl_exit(1,"Unable to create ets metadata tables."); + erts_exit(ERTS_ERROR_EXIT,"Unable to create ets metadata tables."); } /* Non visual BIF to trap to. */ @@ -3124,7 +3210,7 @@ retry: tb->common.id, p->common.id, heir_data), - 0); + 0); erts_smp_proc_unlock(to_proc, to_locks); return !0; } @@ -3136,7 +3222,7 @@ retry: * yielding. */ #define ERTS_DB_INTERNAL_ERROR(LSTR) \ - erl_exit(ERTS_ABORT_EXIT, "%s:%d:erts_db_process_exiting(): " LSTR "\n", \ + erts_exit(ERTS_ABORT_EXIT, "%s:%d:erts_db_process_exiting(): " LSTR "\n", \ __FILE__, __LINE__) int @@ -3279,34 +3365,37 @@ erts_db_process_exiting(Process *c_p, ErtsProcLocks c_p_locks) } erts_smp_rwmtx_runlock(mmtl); if (tb) { - int reds; - DbFixation** pp; + int reds = 0; db_lock(tb, LCK_WRITE_REC); - #ifdef ERTS_SMP - erts_smp_mtx_lock(&tb->common.fixlock); - #endif - reds = 10; - - for (pp = &tb->common.fixations; *pp != NULL; - pp = &(*pp)->next) { - if ((*pp)->pid == pid) { - DbFixation* fix = *pp; - erts_aint_t diff = -((erts_aint_t) fix->counter); - erts_refc_add(&tb->common.ref,diff,0); - *pp = fix->next; - erts_db_free(ERTS_ALC_T_DB_FIXATION, - tb, fix, sizeof(DbFixation)); - ERTS_ETS_MISC_MEM_ADD(-sizeof(DbFixation)); - break; + if (!(tb->common.status & DB_DELETE)) { + DbFixation** pp; + + #ifdef ERTS_SMP + erts_smp_mtx_lock(&tb->common.fixlock); + #endif + reds = 10; + + for (pp = &tb->common.fixations; *pp != NULL; + pp = &(*pp)->next) { + if ((*pp)->pid == pid) { + DbFixation* fix = *pp; + erts_aint_t diff = -((erts_aint_t) fix->counter); + erts_refc_add(&tb->common.ref,diff,0); + *pp = fix->next; + erts_db_free(ERTS_ALC_T_DB_FIXATION, + tb, fix, sizeof(DbFixation)); + ERTS_ETS_MISC_MEM_ADD(-sizeof(DbFixation)); + break; + } + } + #ifdef ERTS_SMP + erts_smp_mtx_unlock(&tb->common.fixlock); + #endif + if (!IS_FIXED(tb) && IS_HASH_TABLE(tb->common.status)) { + db_unfix_table_hash(&(tb->hash)); + reds += 40; } - } - #ifdef ERTS_SMP - erts_smp_mtx_unlock(&tb->common.fixlock); - #endif - if (!IS_FIXED(tb) && IS_HASH_TABLE(tb->common.status)) { - db_unfix_table_hash(&(tb->hash)); - reds += 40; } db_unlock(tb, LCK_WRITE_REC); BUMP_REDS(c_p, reds); @@ -3370,10 +3459,10 @@ static void fix_table_locked(Process* p, DbTable* tb) #endif erts_refc_inc(&tb->common.ref,1); fix = tb->common.fixations; - if (fix == NULL) { - get_now(&(tb->common.megasec), - &(tb->common.sec), - &(tb->common.microsec)); + if (fix == NULL) { + tb->common.time.monotonic + = erts_get_monotonic_time(ERTS_PROC_GET_SCHDATA(p)); + tb->common.time.offset = erts_get_time_offset(); } else { for (; fix != NULL; fix = fix->next) { @@ -3405,7 +3494,7 @@ static void fix_table_locked(Process* p, DbTable* tb) make_small(tb->common.slot)), 0) != DB_ERROR_NONE) { UnUseTmpHeap(3,p); - erl_exit(1,"Could not insert ets metadata in safe_fixtable."); + erts_exit(ERTS_ERROR_EXIT,"Could not insert ets metadata in safe_fixtable."); } UnUseTmpHeap(3,p); db_meta_unlock(meta_pid_to_fixed_tab, LCK_WRITE_REC); @@ -3556,11 +3645,8 @@ static BIF_RETTYPE ets_delete_trap(BIF_ALIST_1) Eterm* ptr = big_val(cont); DbTable *tb = *((DbTable **) (UWord) (ptr + 1)); -#if HALFWORD_HEAP - ASSERT(*ptr == make_pos_bignum_header(2)); -#else ASSERT(*ptr == make_pos_bignum_header(1)); -#endif + db_lock(tb, LCK_WRITE); trap = free_table_cont(p, tb, 0, 1); db_unlock(tb, LCK_WRITE); @@ -3635,6 +3721,7 @@ static int free_table_cont(Process *p, static Eterm table_info(Process* p, DbTable* tb, Eterm What) { Eterm ret = THE_NON_VALUE; + int use_monotonic; if (What == am_size) { ret = make_small(erts_smp_atomic_read_nob(&tb->common.nitems)); @@ -3666,6 +3753,10 @@ static Eterm table_info(Process* p, DbTable* tb, Eterm What) ret = am_protected; else if (tb->common.status & DB_PUBLIC) ret = am_public; + } else if (What == am_write_concurrency) { + ret = tb->common.status & DB_FINE_LOCKED ? am_true : am_false; + } else if (What == am_read_concurrency) { + ret = tb->common.status & DB_FREQ_READ ? am_true : am_false; } else if (What == am_name) { ret = tb->common.the_name; } else if (What == am_keypos) { @@ -3688,7 +3779,10 @@ static Eterm table_info(Process* p, DbTable* tb, Eterm What) ret = am_true; else ret = am_false; - } else if (What == am_atom_put("safe_fixed",10)) { + } else if ((use_monotonic + = ERTS_IS_ATOM_STR("safe_fixed_monotonic_time", + What)) + || ERTS_IS_ATOM_STR("safe_fixed", What)) { #ifdef ERTS_SMP erts_smp_mtx_lock(&tb->common.fixlock); #endif @@ -3697,7 +3791,19 @@ static Eterm table_info(Process* p, DbTable* tb, Eterm What) Eterm *hp; Eterm tpl, lst; DbFixation *fix; - need = 7; + Sint64 mtime; + + need = 3; + if (use_monotonic) { + mtime = (Sint64) tb->common.time.monotonic; + mtime += ERTS_MONOTONIC_OFFSET_NATIVE; + if (!IS_SSMALL(mtime)) + need += ERTS_SINT64_HEAP_SIZE(mtime); + } + else { + mtime = 0; + need += 4; + } for (fix = tb->common.fixations; fix != NULL; fix = fix->next) { need += 5; } @@ -3709,11 +3815,18 @@ static Eterm table_info(Process* p, DbTable* tb, Eterm What) lst = CONS(hp,tpl,lst); hp += 2; } - tpl = TUPLE3(hp, - make_small(tb->common.megasec), - make_small(tb->common.sec), - make_small(tb->common.microsec)); - hp += 4; + if (use_monotonic) + tpl = (IS_SSMALL(mtime) + ? make_small(mtime) + : erts_sint64_to_big(mtime, &hp)); + else { + Uint ms, s, us; + erts_make_timestamp_value(&ms, &s, &us, + tb->common.time.monotonic, + tb->common.time.offset); + tpl = TUPLE3(hp, make_small(ms), make_small(s), make_small(us)); + hp += 4; + } ret = TUPLE2(hp, tpl, lst); } else { ret = am_false; @@ -3748,7 +3861,7 @@ static Eterm table_info(Process* p, DbTable* tb, Eterm What) avg, std_dev_real, std_dev_exp, make_small(stats.min_chain_len), make_small(stats.max_chain_len), - make_small(db_kept_items_hash(&tb->hash))); + make_small(stats.kept_items)); } else { ret = am_false; @@ -3770,6 +3883,11 @@ static void print_table(int to, void *to_arg, int show, DbTable* tb) + sizeof(Uint) - 1) / sizeof(Uint))); + erts_print(to, to_arg, "Type: %T\n", table_info(NULL, tb, am_type)); + erts_print(to, to_arg, "Protection: %T\n", table_info(NULL, tb, am_protection)); + erts_print(to, to_arg, "Compressed: %T\n", table_info(NULL, tb, am_compressed)); + erts_print(to, to_arg, "Write Concurrency: %T\n", table_info(NULL, tb, am_write_concurrency)); + erts_print(to, to_arg, "Read Concurrency: %T\n", table_info(NULL, tb, am_read_concurrency)); } void db_info(int to, void *to_arg, int show) /* Called by break handler */ diff --git a/erts/emulator/beam/erl_db.h b/erts/emulator/beam/erl_db.h index 5b4681fc90..a589af784c 100644 --- a/erts/emulator/beam/erl_db.h +++ b/erts/emulator/beam/erl_db.h @@ -3,16 +3,17 @@ * * Copyright Ericsson AB 1996-2013. All Rights Reserved. * - * The contents of this file are subject to the Erlang Public License, - * Version 1.1, (the "License"); you may not use this file except in - * compliance with the License. You should have received a copy of the - * Erlang Public License along with this software. If not, it can be - * retrieved online at http://www.erlang.org/. - * - * Software distributed under the License is distributed on an "AS IS" - * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See - * the License for the specific language governing rights and limitations - * under the License. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. * * %CopyrightEnd% */ @@ -61,7 +62,17 @@ union db_table { "ERL_MAX_ETS_TABLES" */ #define ERL_MAX_ETS_TABLES_ENV "ERL_MAX_ETS_TABLES" -void init_db(void); +typedef enum { + ERTS_DB_SPNCNT_NONE, + ERTS_DB_SPNCNT_VERY_LOW, + ERTS_DB_SPNCNT_LOW, + ERTS_DB_SPNCNT_NORMAL, + ERTS_DB_SPNCNT_HIGH, + ERTS_DB_SPNCNT_VERY_HIGH, + ERTS_DB_SPNCNT_EXTREMELY_HIGH +} ErtsDbSpinCount; + +void init_db(ErtsDbSpinCount); int erts_db_process_exiting(Process *, ErtsProcLocks); void db_info(int, void *, int); void erts_db_foreach_table(void (*)(DbTable *, void *), void *); @@ -69,6 +80,7 @@ void erts_db_foreach_offheap(DbTable *, void (*func)(ErlOffHeap *, void *), void *); +extern int erts_ets_rwmtx_spin_count; extern int user_requested_db_max_tabs; /* set in erl_init */ extern int erts_ets_realloc_always_moves; /* set in erl_init */ extern int erts_ets_always_compress; /* set in erl_init */ diff --git a/erts/emulator/beam/erl_db_hash.c b/erts/emulator/beam/erl_db_hash.c index 06dac8f161..240b79fc60 100644 --- a/erts/emulator/beam/erl_db_hash.c +++ b/erts/emulator/beam/erl_db_hash.c @@ -3,16 +3,17 @@ * * Copyright Ericsson AB 1998-2012. All Rights Reserved. * - * The contents of this file are subject to the Erlang Public License, - * Version 1.1, (the "License"); you may not use this file except in - * compliance with the License. You should have received a copy of the - * Erlang Public License along with this software. If not, it can be - * retrieved online at http://www.erlang.org/. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Software distributed under the License is distributed on an "AS IS" - * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See - * the License for the specific language governing rights and limitations - * under the License. + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. * * %CopyrightEnd% */ @@ -147,8 +148,11 @@ static ERTS_INLINE Uint hash_to_ix(DbTableHash* tb, HashValue hval) } /* Remember a slot containing a pseudo-deleted item (INVALID_HASH) -*/ -static ERTS_INLINE void add_fixed_deletion(DbTableHash* tb, int ix) + * Return false if we got raced by unfixing thread + * and the object should be deleted for real. + */ +static ERTS_INLINE int add_fixed_deletion(DbTableHash* tb, int ix, + erts_aint_t fixated_by_me) { erts_aint_t was_next; erts_aint_t exp_next; @@ -159,12 +163,18 @@ static ERTS_INLINE void add_fixed_deletion(DbTableHash* tb, int ix) fixd->slot = ix; was_next = erts_smp_atomic_read_acqb(&tb->fixdel); do { /* Lockless atomic insertion in linked list: */ - exp_next = was_next; + if (NFIXED(tb) <= fixated_by_me) { + erts_db_free(ERTS_ALC_T_DB_FIX_DEL, (DbTable*)tb, + fixd, sizeof(FixedDeletion)); + return 0; /* raced by unfixer */ + } + exp_next = was_next; fixd->next = (FixedDeletion*) exp_next; - was_next = erts_smp_atomic_cmpxchg_relb(&tb->fixdel, - (erts_aint_t) fixd, - exp_next); + was_next = erts_smp_atomic_cmpxchg_mb(&tb->fixdel, + (erts_aint_t) fixd, + exp_next); }while (was_next != exp_next); + return 1; } @@ -174,7 +184,7 @@ static ERTS_INLINE void add_fixed_deletion(DbTableHash* tb, int ix) /* optimised version of make_hash (normal case? atomic key) */ #define MAKE_HASH(term) \ ((is_atom(term) ? (atom_tab(atom_val(term))->slot.bucket.hvalue) : \ - make_hash2(term)) % MAX_HASH) + make_internal_hash(term)) % MAX_HASH) #ifdef ERTS_SMP # define DB_HASH_LOCK_MASK (DB_HASH_LOCK_CNT-1) @@ -382,7 +392,7 @@ static HashDbTerm* search_list(DbTableHash* tb, Eterm key, static void shrink(DbTableHash* tb, int nactive); static void grow(DbTableHash* tb, int nactive); static Eterm build_term_list(Process* p, HashDbTerm* ptr1, HashDbTerm* ptr2, - DbTableHash*); + Uint sz, DbTableHash*); static int analyze_pattern(DbTableHash *tb, Eterm pattern, struct mp_info *mpi); @@ -426,6 +436,7 @@ static int db_select_count_continue_hash(Process *p, DbTable *tbl, static int db_select_delete_continue_hash(Process *p, DbTable *tbl, Eterm continuation, Eterm *ret); +static int db_take_hash(Process *, DbTable *, Eterm, Eterm *); static void db_print_hash(int to, void *to_arg, int show, @@ -443,8 +454,11 @@ static int db_delete_all_objects_hash(Process* p, DbTable* tbl); #ifdef HARDDEBUG static void db_check_table_hash(DbTableHash *tb); #endif -static int db_lookup_dbterm_hash(DbTable *tbl, Eterm key, DbUpdateHandle* handle); -static void db_finalize_dbterm_hash(DbUpdateHandle* handle); +static int +db_lookup_dbterm_hash(Process *p, DbTable *tbl, Eterm key, Eterm obj, + DbUpdateHandle* handle); +static void +db_finalize_dbterm_hash(int cret, DbUpdateHandle* handle); static ERTS_INLINE void try_shrink(DbTableHash* tb) { @@ -455,9 +469,6 @@ static ERTS_INLINE void try_shrink(DbTableHash* tb) } } -#define EQ_REL(x,y,y_base) \ - (is_same(x,NULL,y,y_base) || (is_not_both_immed((x),(y)) && eq_rel((x),NULL,(y),y_base))) - /* Is this a live object (not pseodo-deleted) with the specified key? */ static ERTS_INLINE int has_live_key(DbTableHash* tb, HashDbTerm* b, @@ -467,7 +478,7 @@ static ERTS_INLINE int has_live_key(DbTableHash* tb, HashDbTerm* b, else { Eterm itemKey = GETKEY(tb, b->dbterm.tpl); ASSERT(!is_header(itemKey)); - return EQ_REL(key, itemKey, b->dbterm.tpl); + return EQ(key, itemKey); } } @@ -480,7 +491,7 @@ static ERTS_INLINE int has_key(DbTableHash* tb, HashDbTerm* b, else { Eterm itemKey = GETKEY(tb, b->dbterm.tpl); ASSERT(!is_header(itemKey)); - return EQ_REL(key, itemKey, b->dbterm.tpl); + return EQ(key, itemKey); } } @@ -536,6 +547,7 @@ DbTableMethod db_hash = db_select_delete_continue_hash, db_select_count_hash, db_select_count_continue_hash, + db_take_hash, db_delete_all_objects_hash, db_free_table_hash, db_free_table_continue_hash, @@ -601,8 +613,8 @@ void db_unfix_table_hash(DbTableHash *tb) || (erts_smp_lc_rwmtx_is_rlocked(&tb->common.rwlock) && !tb->common.is_thread_safe)); restart: - fixdel = (FixedDeletion*) erts_smp_atomic_xchg_acqb(&tb->fixdel, - (erts_aint_t) NULL); + fixdel = (FixedDeletion*) erts_smp_atomic_xchg_mb(&tb->fixdel, + (erts_aint_t) NULL); while (fixdel != NULL) { FixedDeletion *fx = fixdel; int ix = fx->slot; @@ -646,25 +658,6 @@ restart: /* ToDo: Maybe try grow/shrink the table as well */ } -/* Only used by tests -*/ -Uint db_kept_items_hash(DbTableHash *tb) -{ - Uint kept_items = 0; - Uint ix = 0; - erts_smp_rwmtx_t* lck = RLOCK_HASH(tb,ix); - HashDbTerm* b; - do { - for (b = BUCKET(tb, ix); b != NULL; b = b->next) { - if (b->hvalue == INVALID_HASH) { - ++kept_items; - } - } - ix = next_slot(tb, ix, &lck); - }while (ix); - return kept_items; -} - int db_create_hash(Process *p, DbTable *tbl) { DbTableHash *tb = &tbl->hash; @@ -684,6 +677,8 @@ int db_create_hash(Process *p, DbTable *tbl) int i; if (tb->common.type & DB_FREQ_READ) rwmtx_opt.type = ERTS_SMP_RWMTX_TYPE_FREQUENT_READ; + if (erts_ets_rwmtx_spin_count >= 0) + rwmtx_opt.main_spincount = erts_ets_rwmtx_spin_count; tb->locks = (DbTableHashFineLocks*) erts_db_alloc_fnf(ERTS_ALC_T_DB_SEG, /* Other type maybe? */ (DbTable *) tb, sizeof(DbTableHashFineLocks)); @@ -879,34 +874,49 @@ Ldone: return ret; } +static Eterm +get_term_list(Process *p, DbTableHash *tb, Eterm key, HashValue hval, + HashDbTerm *b1, HashDbTerm **bend) +{ + HashDbTerm* b2 = b1->next; + Eterm copy; + Uint sz = b1->dbterm.size + 2; + + if (tb->common.status & (DB_BAG | DB_DUPLICATE_BAG)) { + while (b2 && has_key(tb, b2, key, hval)) { + if (b2->hvalue != INVALID_HASH) + sz += b2->dbterm.size + 2; + + b2 = b2->next; + } + } + copy = build_term_list(p, b1, b2, sz, tb); + CHECK_TABLES(); + if (bend) { + *bend = b2; + } + return copy; +} + int db_get_hash(Process *p, DbTable *tbl, Eterm key, Eterm *ret) { DbTableHash *tb = &tbl->hash; HashValue hval; int ix; - HashDbTerm* b1; + HashDbTerm* b; erts_smp_rwmtx_t* lck; hval = MAKE_HASH(key); lck = RLOCK_HASH(tb,hval); ix = hash_to_ix(tb, hval); - b1 = BUCKET(tb, ix); - - while(b1 != 0) { - if (has_live_key(tb,b1,key,hval)) { - HashDbTerm* b2 = b1->next; - Eterm copy; + b = BUCKET(tb, ix); - if (tb->common.status & (DB_BAG | DB_DUPLICATE_BAG)) { - while(b2 != NULL && has_key(tb,b2,key,hval)) - b2 = b2->next; - } - copy = build_term_list(p, b1, b2, tb); - CHECK_TABLES(); - *ret = copy; + while(b != 0) { + if (has_live_key(tb, b, key, hval)) { + *ret = get_term_list(p, tb, key, hval, b, NULL); goto done; } - b1 = b1->next; + b = b->next; } *ret = NIL; done: @@ -1048,7 +1058,6 @@ static int db_get_element_hash(Process *p, DbTable *tbl, Eterm copy = db_copy_element_from_ets(&tb->common, p, &b->dbterm, ndex, &hp, 2); elem_list = CONS(hp, copy, elem_list); - hp += 2; } b = b->next; } @@ -1139,9 +1148,9 @@ int db_erase_hash(DbTable *tbl, Eterm key, Eterm *ret) while(b != 0) { if (has_live_key(tb,b,key,hval)) { --nitems_diff; - if (nitems_diff == -1 && IS_FIXED(tb)) { + if (nitems_diff == -1 && IS_FIXED(tb) + && add_fixed_deletion(tb, ix, 0)) { /* Pseudo remove (no need to keep several of same key) */ - add_fixed_deletion(tb, ix); b->hvalue = INVALID_HASH; } else { *bp = b->next; @@ -1193,9 +1202,8 @@ static int db_erase_object_hash(DbTable *tbl, Eterm object, Eterm *ret) ++nkeys; if (db_eq(&tb->common,object, &b->dbterm)) { --nitems_diff; - if (nkeys==1 && IS_FIXED(tb)) { /* Pseudo remove */ - add_fixed_deletion(tb,ix); - b->hvalue = INVALID_HASH; + if (nkeys==1 && IS_FIXED(tb) && add_fixed_deletion(tb,ix,0)) { + b->hvalue = INVALID_HASH; /* Pseudo remove */ bp = &b->next; b = b->next; } else { @@ -1240,7 +1248,7 @@ static int db_slot_hash(Process *p, DbTable *tbl, Eterm slot_term, Eterm *ret) lck = RLOCK_HASH(tb, slot); nactive = NACTIVE(tb); if (slot < nactive) { - *ret = build_term_list(p, BUCKET(tb, slot), 0, tb); + *ret = build_term_list(p, BUCKET(tb, slot), NULL, 0, tb); retval = DB_ERROR_NONE; } else if (slot == nactive) { @@ -1817,14 +1825,17 @@ static int db_select_delete_hash(Process *p, int did_erase = 0; if (db_match_dbterm(&tb->common, p, mpi.mp, 0, &(*current)->dbterm, NULL, 0) == am_true) { + HashDbTerm *del; if (NFIXED(tb) > fixated_by_me) { /* fixated by others? */ if (slot_ix != last_pseudo_delete) { - add_fixed_deletion(tb, slot_ix); - last_pseudo_delete = slot_ix; + if (!add_fixed_deletion(tb, slot_ix, fixated_by_me)) + goto do_erase; + last_pseudo_delete = slot_ix; } (*current)->hvalue = INVALID_HASH; } else { - HashDbTerm *del = *current; + do_erase: + del = *current; *current = (*current)->next; free_term(tb, del); did_erase = 1; @@ -1928,14 +1939,17 @@ static int db_select_delete_continue_hash(Process *p, int did_erase = 0; if (db_match_dbterm(&tb->common, p, mp, 0, &(*current)->dbterm, NULL, 0) == am_true) { + HashDbTerm *del; if (NFIXED(tb) > fixated_by_me) { /* fixated by others? */ if (slot_ix != last_pseudo_delete) { - add_fixed_deletion(tb, slot_ix); + if (!add_fixed_deletion(tb, slot_ix, fixated_by_me)) + goto do_erase; last_pseudo_delete = slot_ix; } (*current)->hvalue = INVALID_HASH; } else { - HashDbTerm *del = *current; + do_erase: + del = *current; *current = (*current)->next; free_term(tb, del); did_erase = 1; @@ -2069,6 +2083,46 @@ trap: } +static int db_take_hash(Process *p, DbTable *tbl, Eterm key, Eterm *ret) +{ + DbTableHash *tb = &tbl->hash; + HashDbTerm **bp, *b; + HashValue hval = MAKE_HASH(key); + erts_smp_rwmtx_t *lck = WLOCK_HASH(tb, hval); + int ix = hash_to_ix(tb, hval); + int nitems_diff = 0; + + *ret = NIL; + for (bp = &BUCKET(tb, ix), b = *bp; b; bp = &b->next, b = b->next) { + if (has_live_key(tb, b, key, hval)) { + HashDbTerm *bend; + + *ret = get_term_list(p, tb, key, hval, b, &bend); + while (b != bend) { + --nitems_diff; + if (nitems_diff == -1 && IS_FIXED(tb) + && add_fixed_deletion(tb, ix, 0)) { + /* Pseudo remove (no need to keep several of same key) */ + bp = &b->next; + b->hvalue = INVALID_HASH; + b = b->next; + } else { + *bp = b->next; + free_term(tb, b); + b = *bp; + } + } + break; + } + } + WUNLOCK_HASH(lck); + if (nitems_diff) { + erts_smp_atomic_add_nob(&tb->common.nitems, nitems_diff); + try_shrink(tb); + } + return DB_ERROR_NONE; +} + /* ** Other interface routines (not directly coupled to one bif) */ @@ -2088,7 +2142,7 @@ int db_mark_all_deleted_hash(DbTable *tbl) for (i = 0; i < NACTIVE(tb); i++) { if ((list = BUCKET(tb,i)) != NULL) { - add_fixed_deletion(tb, i); + add_fixed_deletion(tb, i, 0); do { list->hvalue = INVALID_HASH; list = list->next; @@ -2104,10 +2158,38 @@ int db_mark_all_deleted_hash(DbTable *tbl) static void db_print_hash(int to, void *to_arg, int show, DbTable *tbl) { DbTableHash *tb = &tbl->hash; + DbHashStats stats; int i; erts_print(to, to_arg, "Buckets: %d\n", NACTIVE(tb)); - + +#ifdef ERTS_SMP + i = tbl->common.is_thread_safe; + /* If crash dumping we set table to thread safe in order to + avoid taking any locks */ + if (ERTS_IS_CRASH_DUMPING) + tbl->common.is_thread_safe = 1; + + db_calc_stats_hash(&tbl->hash, &stats); + + tbl->common.is_thread_safe = i; +#else + db_calc_stats_hash(&tbl->hash, &stats); +#endif + + erts_print(to, to_arg, "Chain Length Avg: %f\n", stats.avg_chain_len); + erts_print(to, to_arg, "Chain Length Max: %d\n", stats.max_chain_len); + erts_print(to, to_arg, "Chain Length Min: %d\n", stats.min_chain_len); + erts_print(to, to_arg, "Chain Length Std Dev: %f\n", + stats.std_dev_chain_len); + erts_print(to, to_arg, "Chain Length Expected Std Dev: %f\n", + stats.std_dev_expected); + + if (IS_FIXED(tb)) + erts_print(to, to_arg, "Fixed: %d\n", stats.kept_items); + else + erts_print(to, to_arg, "Fixed: false\n"); + if (show) { for (i = 0; i < NACTIVE(tb); i++) { HashDbTerm* list = BUCKET(tb,i); @@ -2119,11 +2201,11 @@ static void db_print_hash(int to, void *to_arg, int show, DbTable *tbl) erts_print(to, to_arg, "*"); if (tb->common.compress) { Eterm key = GETKEY(tb, list->dbterm.tpl); - erts_print(to, to_arg, "key=%R", key, list->dbterm.tpl); + erts_print(to, to_arg, "key=%T", key); } else { - Eterm obj = make_tuple_rel(list->dbterm.tpl,list->dbterm.tpl); - erts_print(to, to_arg, "%R", obj, list->dbterm.tpl); + Eterm obj = make_tuple(list->dbterm.tpl); + erts_print(to, to_arg, "%T", obj); } if (list->next != 0) erts_print(to, to_arg, ","); @@ -2402,10 +2484,10 @@ static int alloc_seg(DbTableHash *tb) */ static int free_seg(DbTableHash *tb, int free_records) { - int seg_ix = (tb->nslots >> SEGSZ_EXP) - 1; + const int seg_ix = (tb->nslots >> SEGSZ_EXP) - 1; + struct segment** const segtab = SEGTAB(tb); + struct ext_segment* const top = (struct ext_segment*) segtab[seg_ix]; int bytes; - struct segment** segtab = SEGTAB(tb); - struct ext_segment* top = (struct ext_segment*) segtab[seg_ix]; int nrecords = 0; ASSERT(top != NULL); @@ -2468,7 +2550,7 @@ static int free_seg(DbTableHash *tb, int free_records) (void*)top, bytes); #ifdef DEBUG if (seg_ix > 0) { - if (seg_ix < tb->nsegs) SEGTAB(tb)[seg_ix] = NULL; + segtab[seg_ix] = NULL; } else { SET_SEGTAB(tb, NULL); } @@ -2483,23 +2565,23 @@ static int free_seg(DbTableHash *tb, int free_records) ** Copy terms from ptr1 until ptr2 ** works for ptr1 == ptr2 == 0 => [] ** or ptr2 == 0 +** sz is either precalculated heap size or 0 if not known */ static Eterm build_term_list(Process* p, HashDbTerm* ptr1, HashDbTerm* ptr2, - DbTableHash* tb) + Uint sz, DbTableHash* tb) { - int sz = 0; HashDbTerm* ptr; Eterm list = NIL; Eterm copy; Eterm *hp, *hend; - ptr = ptr1; - while(ptr != ptr2) { - - if (ptr->hvalue != INVALID_HASH) - sz += ptr->dbterm.size + 2; - - ptr = ptr->next; + if (!sz) { + ptr = ptr1; + while(ptr != ptr2) { + if (ptr->hvalue != INVALID_HASH) + sz += ptr->dbterm.size + 2; + ptr = ptr->next; + } } hp = HAlloc(p, sz); @@ -2730,59 +2812,126 @@ static HashDbTerm* next(DbTableHash *tb, Uint *iptr, erts_smp_rwmtx_t** lck_ptr, return NULL; } -static int db_lookup_dbterm_hash(DbTable *tbl, Eterm key, DbUpdateHandle* handle) +static int +db_lookup_dbterm_hash(Process *p, DbTable *tbl, Eterm key, Eterm obj, + DbUpdateHandle* handle) { DbTableHash *tb = &tbl->hash; - HashDbTerm* b; - HashDbTerm** prevp; - int ix; HashValue hval; + HashDbTerm **bp, *b; erts_smp_rwmtx_t* lck; + int flags = 0; + + ASSERT(tb->common.status & DB_SET); hval = MAKE_HASH(key); - lck = WLOCK_HASH(tb,hval); - ix = hash_to_ix(tb, hval); - prevp = &BUCKET(tb, ix); - b = *prevp; + lck = WLOCK_HASH(tb, hval); + bp = &BUCKET(tb, hash_to_ix(tb, hval)); + b = *bp; - while (b != 0) { - if (has_live_key(tb,b,key,hval)) { - handle->tb = tbl; - handle->bp = (void**) prevp; - handle->dbterm = &b->dbterm; - handle->mustResize = 0; - handle->new_size = b->dbterm.size; - #if HALFWORD_HEAP - handle->abs_vec = NULL; - #endif - handle->lck = lck; - /* KEEP hval WLOCKED, db_finalize_dbterm_hash will WUNLOCK */ - return 1; - } - prevp = &b->next; - b = *prevp; + for (;;) { + if (b == NULL) { + break; + } + if (has_key(tb, b, key, hval)) { + if (b->hvalue != INVALID_HASH) { + goto Ldone; + } + break; + } + bp = &b->next; + b = *bp; } - WUNLOCK_HASH(lck); - return 0; + + if (obj == THE_NON_VALUE) { + WUNLOCK_HASH(lck); + return 0; + } + + { + Eterm *objp = tuple_val(obj); + int arity = arityval(*objp); + Eterm *htop, *hend; + + ASSERT(arity >= tb->common.keypos); + htop = HAlloc(p, arity + 1); + hend = htop + arity + 1; + sys_memcpy(htop, objp, sizeof(Eterm) * (arity + 1)); + htop[tb->common.keypos] = key; + obj = make_tuple(htop); + + if (b == NULL) { + HashDbTerm *q = new_dbterm(tb, obj); + + q->hvalue = hval; + q->next = NULL; + *bp = b = q; + + { + int nitems = erts_smp_atomic_inc_read_nob(&tb->common.nitems); + int nactive = NACTIVE(tb); + + if (nitems > nactive * (CHAIN_LEN + 1) && !IS_FIXED(tb)) { + grow(tb, nactive); + } + } + } else { + HashDbTerm *q, *next = b->next; + + ASSERT(b->hvalue == INVALID_HASH); + q = replace_dbterm(tb, b, obj); + q->next = next; + q->hvalue = hval; + *bp = b = q; + erts_smp_atomic_inc_nob(&tb->common.nitems); + } + + HRelease(p, hend, htop); + flags |= DB_NEW_OBJECT; + } + +Ldone: + handle->tb = tbl; + handle->bp = (void **)bp; + handle->dbterm = &b->dbterm; + handle->flags = flags; + handle->new_size = b->dbterm.size; + handle->lck = lck; + return 1; } /* Must be called after call to db_lookup_dbterm */ -static void db_finalize_dbterm_hash(DbUpdateHandle* handle) +static void +db_finalize_dbterm_hash(int cret, DbUpdateHandle* handle) { DbTable* tbl = handle->tb; - HashDbTerm* oldp = (HashDbTerm*) *(handle->bp); + DbTableHash *tb = &tbl->hash; + HashDbTerm **bp = (HashDbTerm **) handle->bp; + HashDbTerm *b = *bp; erts_smp_rwmtx_t* lck = (erts_smp_rwmtx_t*) handle->lck; - ERTS_SMP_LC_ASSERT(IS_HASH_WLOCKED(&tbl->hash,lck)); /* locked by db_lookup_dbterm_hash */ + ERTS_SMP_LC_ASSERT(IS_HASH_WLOCKED(tb, lck)); /* locked by db_lookup_dbterm_hash */ + + ASSERT((&b->dbterm == handle->dbterm) == !(tb->common.compress && handle->flags & DB_MUST_RESIZE)); - ASSERT((&oldp->dbterm == handle->dbterm) == !(tbl->common.compress && handle->mustResize)); + if (handle->flags & DB_NEW_OBJECT && cret != DB_ERROR_NONE) { + if (IS_FIXED(tb) && add_fixed_deletion(tb, hash_to_ix(tb, b->hvalue), + 0)) { + b->hvalue = INVALID_HASH; + } else { + *bp = b->next; + free_term(tb, b); + } - if (handle->mustResize) { + WUNLOCK_HASH(lck); + erts_smp_atomic_dec_nob(&tb->common.nitems); + try_shrink(tb); + } else if (handle->flags & DB_MUST_RESIZE) { db_finalize_resize(handle, offsetof(HashDbTerm,dbterm)); WUNLOCK_HASH(lck); - free_term(&tbl->hash, oldp); + free_term(tb, b); } else { WUNLOCK_HASH(lck); @@ -2833,6 +2982,7 @@ void db_calc_stats_hash(DbTableHash* tb, DbHashStats* stats) erts_smp_rwmtx_t* lck; int sum = 0; int sq_sum = 0; + int kept_items = 0; int ix; int len; @@ -2844,6 +2994,8 @@ void db_calc_stats_hash(DbTableHash* tb, DbHashStats* stats) len = 0; for (b = BUCKET(tb,ix); b!=NULL; b=b->next) { len++; + if (b->hvalue == INVALID_HASH) + ++kept_items; } sum += len; sq_sum += len*len; @@ -2855,7 +3007,8 @@ void db_calc_stats_hash(DbTableHash* tb, DbHashStats* stats) stats->std_dev_chain_len = sqrt((sq_sum - stats->avg_chain_len*sum) / NACTIVE(tb)); /* Expected standard deviation from a good uniform hash function, ie binomial distribution (not taking the linear hashing into acount) */ - stats->std_dev_expected = sqrt(stats->avg_chain_len * (1 - 1.0/NACTIVE(tb))); + stats->std_dev_expected = sqrt(stats->avg_chain_len * (1 - 1.0/NACTIVE(tb))); + stats->kept_items = kept_items; } #ifdef HARDDEBUG @@ -2869,7 +3022,7 @@ void db_check_table_hash(DbTable *tbl) if ((list = BUCKET(tb,j)) != 0) { while (list != 0) { if (!is_tuple(make_tuple(list->dbterm.tpl))) { - erl_exit(1, "Bad term in slot %d of ets table", j); + erts_exit(ERTS_ERROR_EXIT, "Bad term in slot %d of ets table", j); } list = list->next; } diff --git a/erts/emulator/beam/erl_db_hash.h b/erts/emulator/beam/erl_db_hash.h index 908cec11d4..66d8ec71d9 100644 --- a/erts/emulator/beam/erl_db_hash.h +++ b/erts/emulator/beam/erl_db_hash.h @@ -1,18 +1,19 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 1998-2013. All Rights Reserved. + * Copyright Ericsson AB 1998-2014. All Rights Reserved. * - * The contents of this file are subject to the Erlang Public License, - * Version 1.1, (the "License"); you may not use this file except in - * compliance with the License. You should have received a copy of the - * Erlang Public License along with this software. If not, it can be - * retrieved online at http://www.erlang.org/. - * - * Software distributed under the License is distributed on an "AS IS" - * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See - * the License for the specific language governing rights and limitations - * under the License. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. * * %CopyrightEnd% */ @@ -42,7 +43,7 @@ typedef struct hash_db_term { typedef struct db_table_hash_fine_locks { union { erts_smp_rwmtx_t lck; - byte _cache_line_alignment[64]; + byte _cache_line_alignment[ERTS_ALC_CACHE_LINE_ALIGN_SIZE(sizeof(erts_smp_rwmtx_t))]; }lck_vec[DB_HASH_LOCK_CNT]; } DbTableHashFineLocks; @@ -104,6 +105,7 @@ typedef struct { float std_dev_expected; int max_chain_len; int min_chain_len; + int kept_items; }DbHashStats; void db_calc_stats_hash(DbTableHash* tb, DbHashStats*); diff --git a/erts/emulator/beam/erl_db_tree.c b/erts/emulator/beam/erl_db_tree.c index a62a83a928..0f642f0867 100644 --- a/erts/emulator/beam/erl_db_tree.c +++ b/erts/emulator/beam/erl_db_tree.c @@ -3,16 +3,17 @@ * * Copyright Ericsson AB 1998-2013. All Rights Reserved. * - * The contents of this file are subject to the Erlang Public License, - * Version 1.1, (the "License"); you may not use this file except in - * compliance with the License. You should have received a copy of the - * Erlang Public License along with this software. If not, it can be - * retrieved online at http://www.erlang.org/. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Software distributed under the License is distributed on an "AS IS" - * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See - * the License for the specific language governing rights and limitations - * under the License. + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. * * %CopyrightEnd% */ @@ -279,7 +280,7 @@ struct select_delete_context { /* ** Forward declarations */ -static TreeDbTerm *linkout_tree(DbTableTree *tb, Eterm key, Eterm* key_base); +static TreeDbTerm *linkout_tree(DbTableTree *tb, Eterm key); static TreeDbTerm *linkout_object_tree(DbTableTree *tb, Eterm object); static int do_free_tree_cont(DbTableTree *tb, int num_left); @@ -290,15 +291,15 @@ static int delsub(TreeDbTerm **this); static TreeDbTerm *slot_search(Process *p, DbTableTree *tb, Sint slot); static TreeDbTerm *find_node(DbTableTree *tb, Eterm key); static TreeDbTerm **find_node2(DbTableTree *tb, Eterm key); -static TreeDbTerm *find_next(DbTableTree *tb, DbTreeStack*, Eterm key, Eterm* kbase); -static TreeDbTerm *find_prev(DbTableTree *tb, DbTreeStack*, Eterm key, Eterm* kbase); +static TreeDbTerm *find_next(DbTableTree *tb, DbTreeStack*, Eterm key); +static TreeDbTerm *find_prev(DbTableTree *tb, DbTreeStack*, Eterm key); static TreeDbTerm *find_next_from_pb_key(DbTableTree *tb, DbTreeStack*, Eterm key); static TreeDbTerm *find_prev_from_pb_key(DbTableTree *tb, DbTreeStack*, Eterm key); static void traverse_backwards(DbTableTree *tb, DbTreeStack*, - Eterm lastkey, Eterm* lk_base, + Eterm lastkey, int (*doit)(DbTableTree *tb, TreeDbTerm *, void *, @@ -306,7 +307,7 @@ static void traverse_backwards(DbTableTree *tb, void *context); static void traverse_forward(DbTableTree *tb, DbTreeStack*, - Eterm lastkey, Eterm* lk_base, + Eterm lastkey, int (*doit)(DbTableTree *tb, TreeDbTerm *, void *, @@ -314,8 +315,8 @@ static void traverse_forward(DbTableTree *tb, void *context); static int key_given(DbTableTree *tb, Eterm pattern, TreeDbTerm **ret, Eterm *partly_bound_key); -static Sint cmp_partly_bound(Eterm partly_bound_key, Eterm bound_key, Eterm* bk_base); -static Sint do_cmp_partly_bound(Eterm a, Eterm b, Eterm* b_base, int *done); +static Sint cmp_partly_bound(Eterm partly_bound_key, Eterm bound_key); +static Sint do_cmp_partly_bound(Eterm a, Eterm b, int *done); static int analyze_pattern(DbTableTree *tb, Eterm pattern, struct mp_info *mpi); @@ -383,6 +384,7 @@ static int db_select_delete_tree(Process *p, DbTable *tbl, Eterm pattern, Eterm *ret); static int db_select_delete_continue_tree(Process *p, DbTable *tbl, Eterm continuation, Eterm *ret); +static int db_take_tree(Process *, DbTable *, Eterm, Eterm *); static void db_print_tree(int to, void *to_arg, int show, DbTable *tbl); static int db_free_table_tree(DbTable *tbl); @@ -398,8 +400,11 @@ static int db_delete_all_objects_tree(Process* p, DbTable* tbl); #ifdef HARDDEBUG static void db_check_table_tree(DbTable *tbl); #endif -static int db_lookup_dbterm_tree(DbTable *, Eterm key, DbUpdateHandle*); -static void db_finalize_dbterm_tree(DbUpdateHandle*); +static int +db_lookup_dbterm_tree(Process *, DbTable *, Eterm key, Eterm obj, + DbUpdateHandle*); +static void +db_finalize_dbterm_tree(int cret, DbUpdateHandle *); /* ** Static variables @@ -431,6 +436,7 @@ DbTableMethod db_tree = db_select_delete_continue_tree, db_select_count_tree, db_select_count_continue_tree, + db_take_tree, db_delete_all_objects_tree, db_free_table_tree, db_free_table_continue_tree, @@ -511,7 +517,7 @@ static int db_next_tree(Process *p, DbTable *tbl, Eterm key, Eterm *ret) if (is_atom(key) && key == am_EOT) return DB_ERROR_BADKEY; stack = get_any_stack(tb); - this = find_next(tb, stack, key, NULL); + this = find_next(tb, stack, key); release_stack(tb,stack); if (this == NULL) { *ret = am_EOT; @@ -557,7 +563,7 @@ static int db_prev_tree(Process *p, DbTable *tbl, Eterm key, Eterm *ret) if (is_atom(key) && key == am_EOT) return DB_ERROR_BADKEY; stack = get_any_stack(tb); - this = find_prev(tb, stack, key, NULL); + this = find_prev(tb, stack, key); release_stack(tb,stack); if (this == NULL) { *ret = am_EOT; @@ -567,19 +573,13 @@ static int db_prev_tree(Process *p, DbTable *tbl, Eterm key, Eterm *ret) return DB_ERROR_NONE; } -static ERTS_INLINE Sint cmp_key(DbTableTree* tb, Eterm key, Eterm* key_base, - TreeDbTerm* obj) -{ - return cmp_rel(key, key_base, - GETKEY(tb,obj->dbterm.tpl), obj->dbterm.tpl); +static ERTS_INLINE Sint cmp_key(DbTableTree* tb, Eterm key, TreeDbTerm* obj) { + return CMP(key, GETKEY(tb,obj->dbterm.tpl)); } -static ERTS_INLINE int cmp_key_eq(DbTableTree* tb, Eterm key, Eterm* key_base, - TreeDbTerm* obj) -{ +static ERTS_INLINE int cmp_key_eq(DbTableTree* tb, Eterm key, TreeDbTerm* obj) { Eterm obj_key = GETKEY(tb,obj->dbterm.tpl); - return is_same(key, key_base, obj_key, obj->dbterm.tpl) - || cmp_rel(key, key_base, obj_key, obj->dbterm.tpl) == 0; + return is_same(key, obj_key) || CMP(key, obj_key) == 0; } static int db_put_tree(DbTable *tbl, Eterm obj, int key_clash_fail) @@ -613,7 +613,7 @@ static int db_put_tree(DbTable *tbl, Eterm obj, int key_clash_fail) (*this)->balance = 0; (*this)->left = (*this)->right = NULL; break; - } else if ((c = cmp_key(tb, key, NULL, *this)) < 0) { + } else if ((c = cmp_key(tb, key, *this)) < 0) { /* go lefts */ dstack[dpos++] = DIR_LEFT; tstack[tpos++] = this; @@ -768,7 +768,7 @@ static int db_erase_tree(DbTable *tbl, Eterm key, Eterm *ret) *ret = am_true; - if ((res = linkout_tree(tb, key, NULL)) != NULL) { + if ((res = linkout_tree(tb, key)) != NULL) { free_term(tb, res); } return DB_ERROR_NONE; @@ -964,15 +964,15 @@ static int db_select_continue_tree(Process *p, stack = get_any_stack(tb); if (chunk_size) { if (reverse) { - traverse_backwards(tb, stack, lastkey, NULL, &doit_select_chunk, &sc); + traverse_backwards(tb, stack, lastkey, &doit_select_chunk, &sc); } else { - traverse_forward(tb, stack, lastkey, NULL, &doit_select_chunk, &sc); + traverse_forward(tb, stack, lastkey, &doit_select_chunk, &sc); } } else { if (reverse) { - traverse_forward(tb, stack, lastkey, NULL, &doit_select, &sc); + traverse_forward(tb, stack, lastkey, &doit_select, &sc); } else { - traverse_backwards(tb, stack, lastkey, NULL, &doit_select, &sc); + traverse_backwards(tb, stack, lastkey, &doit_select, &sc); } } release_stack(tb,stack); @@ -997,9 +997,9 @@ static int db_select_continue_tree(Process *p, } key = GETKEY(tb, sc.lastobj); - sz = size_object_rel(key,sc.lastobj); + sz = size_object(key); hp = HAlloc(p, 9 + sz); - key = copy_struct_rel(key, sz, &hp, &MSO(p), sc.lastobj, NULL); + key = copy_struct(key, sz, &hp, &MSO(p)); continuation = TUPLE8 (hp, tptr[1], @@ -1020,8 +1020,8 @@ static int db_select_continue_tree(Process *p, key = GETKEY(tb, sc.lastobj); if (chunk_size) { if (end_condition != NIL && - ((!reverse && cmp_partly_bound(end_condition,key,sc.lastobj) < 0) || - (reverse && cmp_partly_bound(end_condition,key,sc.lastobj) > 0))) { + ((!reverse && cmp_partly_bound(end_condition,key) < 0) || + (reverse && cmp_partly_bound(end_condition,key) > 0))) { /* done anyway */ if (!sc.got) { RET_TO_BIF(am_EOT, DB_ERROR_NONE); @@ -1033,16 +1033,16 @@ static int db_select_continue_tree(Process *p, } } else { if (end_condition != NIL && - ((!reverse && cmp_partly_bound(end_condition,key,sc.lastobj) > 0) || - (reverse && cmp_partly_bound(end_condition,key,sc.lastobj) < 0))) { + ((!reverse && cmp_partly_bound(end_condition,key) > 0) || + (reverse && cmp_partly_bound(end_condition,key) < 0))) { /* done anyway */ RET_TO_BIF(sc.accum,DB_ERROR_NONE); } } /* Not done yet, let's trap. */ - sz = size_object_rel(key,sc.lastobj); + sz = size_object(key); hp = HAlloc(p, 9 + sz); - key = copy_struct_rel(key, sz, &hp, &MSO(p), sc.lastobj, NULL); + key = copy_struct(key, sz, &hp, &MSO(p)); continuation = TUPLE8 (hp, tptr[1], @@ -1069,7 +1069,6 @@ static int db_select_tree(Process *p, DbTable *tbl, struct select_context sc; struct mp_info mpi; Eterm lastkey = THE_NON_VALUE; - Eterm* lk_base = NULL; Eterm key; Eterm continuation; unsigned sz; @@ -1111,7 +1110,7 @@ static int db_select_tree(Process *p, DbTable *tbl, sc.all_objects = mpi.all_objects; if (!mpi.got_partial && mpi.some_limitation && - CMP(mpi.least,mpi.most) == 0) { + CMP_EQ(mpi.least,mpi.most)) { doit_select(tb,mpi.save_term,&sc,0 /* direction doesn't matter */); RET_TO_BIF(sc.accum,DB_ERROR_NONE); } @@ -1121,20 +1120,18 @@ static int db_select_tree(Process *p, DbTable *tbl, if (mpi.some_limitation) { if ((this = find_prev_from_pb_key(tb, stack, mpi.least)) != NULL) { lastkey = GETKEY(tb, this->dbterm.tpl); - lk_base = this->dbterm.tpl; } sc.end_condition = mpi.most; } - traverse_forward(tb, stack, lastkey, lk_base, &doit_select, &sc); + traverse_forward(tb, stack, lastkey, &doit_select, &sc); } else { if (mpi.some_limitation) { if ((this = find_next_from_pb_key(tb, stack, mpi.most)) != NULL) { lastkey = GETKEY(tb, this->dbterm.tpl); - lk_base = this->dbterm.tpl; } sc.end_condition = mpi.least; } - traverse_backwards(tb, stack, lastkey, lk_base, &doit_select, &sc); + traverse_backwards(tb, stack, lastkey, &doit_select, &sc); } release_stack(tb,stack); #ifdef HARDDEBUG @@ -1147,9 +1144,9 @@ static int db_select_tree(Process *p, DbTable *tbl, } key = GETKEY(tb, sc.lastobj); - sz = size_object_rel(key, sc.lastobj); + sz = size_object(key); hp = HAlloc(p, 9 + sz + PROC_BIN_SIZE); - key = copy_struct_rel(key, sz, &hp, &MSO(p), sc.lastobj, NULL); + key = copy_struct(key, sz, &hp, &MSO(p)); if (mpi.all_objects) (mpi.mp)->flags |= BIN_FLAG_ALL_OBJECTS; mpb=db_make_mp_binary(p,mpi.mp,&hp); @@ -1207,7 +1204,7 @@ static int db_select_count_continue_tree(Process *p, tptr = tuple_val(continuation); if (arityval(*tptr) != 5) - erl_exit(1,"Internal error in ets:select_count/1"); + erts_exit(ERTS_ERROR_EXIT,"Internal error in ets:select_count/1"); lastkey = tptr[2]; end_condition = tptr[3]; @@ -1230,7 +1227,7 @@ static int db_select_count_continue_tree(Process *p, } stack = get_any_stack(tb); - traverse_backwards(tb, stack, lastkey, NULL, &doit_select_count, &sc); + traverse_backwards(tb, stack, lastkey, &doit_select_count, &sc); release_stack(tb,stack); BUMP_REDS(p, 1000 - sc.max); @@ -1240,12 +1237,12 @@ static int db_select_count_continue_tree(Process *p, } key = GETKEY(tb, sc.lastobj); if (end_condition != NIL && - (cmp_partly_bound(end_condition,key,sc.lastobj) > 0)) { + (cmp_partly_bound(end_condition,key) > 0)) { /* done anyway */ RET_TO_BIF(make_small(sc.got),DB_ERROR_NONE); } /* Not done yet, let's trap. */ - sz = size_object_rel(key, sc.lastobj); + sz = size_object(key); if (IS_USMALL(0, sc.got)) { hp = HAlloc(p, sz + 6); egot = make_small(sc.got); @@ -1255,7 +1252,7 @@ static int db_select_count_continue_tree(Process *p, egot = uint_to_big(sc.got, hp); hp += BIG_UINT_HEAP_SIZE; } - key = copy_struct_rel(key, sz, &hp, &MSO(p), sc.lastobj, NULL); + key = copy_struct(key, sz, &hp, &MSO(p)); continuation = TUPLE5 (hp, tptr[1], @@ -1278,7 +1275,6 @@ static int db_select_count_tree(Process *p, DbTable *tbl, struct select_count_context sc; struct mp_info mpi; Eterm lastkey = THE_NON_VALUE; - Eterm* lk_base = NULL; Eterm key; Eterm continuation; unsigned sz; @@ -1319,7 +1315,7 @@ static int db_select_count_tree(Process *p, DbTable *tbl, sc.all_objects = mpi.all_objects; if (!mpi.got_partial && mpi.some_limitation && - CMP(mpi.least,mpi.most) == 0) { + CMP_EQ(mpi.least,mpi.most)) { doit_select_count(tb,mpi.save_term,&sc,0 /* dummy */); RET_TO_BIF(erts_make_integer(sc.got,p),DB_ERROR_NONE); } @@ -1328,12 +1324,11 @@ static int db_select_count_tree(Process *p, DbTable *tbl, if (mpi.some_limitation) { if ((this = find_next_from_pb_key(tb, stack, mpi.most)) != NULL) { lastkey = GETKEY(tb, this->dbterm.tpl); - lk_base = this->dbterm.tpl; } sc.end_condition = mpi.least; } - traverse_backwards(tb, stack, lastkey, lk_base, &doit_select_count, &sc); + traverse_backwards(tb, stack, lastkey, &doit_select_count, &sc); release_stack(tb,stack); BUMP_REDS(p, 1000 - sc.max); if (sc.max > 0) { @@ -1341,7 +1336,7 @@ static int db_select_count_tree(Process *p, DbTable *tbl, } key = GETKEY(tb, sc.lastobj); - sz = size_object_rel(key, sc.lastobj); + sz = size_object(key); if (IS_USMALL(0, sc.got)) { hp = HAlloc(p, sz + PROC_BIN_SIZE + 6); egot = make_small(sc.got); @@ -1351,7 +1346,7 @@ static int db_select_count_tree(Process *p, DbTable *tbl, egot = uint_to_big(sc.got, hp); hp += BIG_UINT_HEAP_SIZE; } - key = copy_struct_rel(key, sz, &hp, &MSO(p), sc.lastobj, NULL); + key = copy_struct(key, sz, &hp, &MSO(p)); if (mpi.all_objects) (mpi.mp)->flags |= BIN_FLAG_ALL_OBJECTS; mpb = db_make_mp_binary(p,mpi.mp,&hp); @@ -1382,7 +1377,6 @@ static int db_select_chunk_tree(Process *p, DbTable *tbl, struct select_context sc; struct mp_info mpi; Eterm lastkey = THE_NON_VALUE; - Eterm* lk_base = NULL; Eterm key; Eterm continuation; unsigned sz; @@ -1424,7 +1418,7 @@ static int db_select_chunk_tree(Process *p, DbTable *tbl, sc.all_objects = mpi.all_objects; if (!mpi.got_partial && mpi.some_limitation && - CMP(mpi.least,mpi.most) == 0) { + CMP_EQ(mpi.least,mpi.most)) { doit_select(tb,mpi.save_term,&sc, 0 /* direction doesn't matter */); if (sc.accum != NIL) { hp=HAlloc(p, 3); @@ -1439,20 +1433,18 @@ static int db_select_chunk_tree(Process *p, DbTable *tbl, if (mpi.some_limitation) { if ((this = find_next_from_pb_key(tb, stack, mpi.most)) != NULL) { lastkey = GETKEY(tb, this->dbterm.tpl); - lk_base = this->dbterm.tpl; } sc.end_condition = mpi.least; } - traverse_backwards(tb, stack, lastkey, lk_base, &doit_select_chunk, &sc); + traverse_backwards(tb, stack, lastkey, &doit_select_chunk, &sc); } else { if (mpi.some_limitation) { if ((this = find_prev_from_pb_key(tb, stack, mpi.least)) != NULL) { lastkey = GETKEY(tb, this->dbterm.tpl); - lk_base = this->dbterm.tpl; } sc.end_condition = mpi.most; } - traverse_forward(tb, stack, lastkey, lk_base, &doit_select_chunk, &sc); + traverse_forward(tb, stack, lastkey, &doit_select_chunk, &sc); } release_stack(tb,stack); @@ -1477,9 +1469,9 @@ static int db_select_chunk_tree(Process *p, DbTable *tbl, } key = GETKEY(tb, sc.lastobj); - sz = size_object_rel(key, sc.lastobj); + sz = size_object(key); hp = HAlloc(p, 9 + sz + PROC_BIN_SIZE); - key = copy_struct_rel(key, sz, &hp, &MSO(p), sc.lastobj, NULL); + key = copy_struct(key, sz, &hp, &MSO(p)); if (mpi.all_objects) (mpi.mp)->flags |= BIN_FLAG_ALL_OBJECTS; mpb = db_make_mp_binary(p,mpi.mp,&hp); @@ -1502,9 +1494,9 @@ static int db_select_chunk_tree(Process *p, DbTable *tbl, } key = GETKEY(tb, sc.lastobj); - sz = size_object_rel(key, sc.lastobj); + sz = size_object(key); hp = HAlloc(p, 9 + sz + PROC_BIN_SIZE); - key = copy_struct_rel(key, sz, &hp, &MSO(p), sc.lastobj, NULL); + key = copy_struct(key, sz, &hp, &MSO(p)); if (mpi.all_objects) (mpi.mp)->flags |= BIN_FLAG_ALL_OBJECTS; @@ -1580,7 +1572,7 @@ static int db_select_delete_continue_tree(Process *p, sc.keypos = tb->common.keypos; ASSERT(!erts_smp_atomic_read_nob(&tb->is_stack_busy)); - traverse_backwards(tb, &tb->static_stack, lastkey, NULL, &doit_select_delete, &sc); + traverse_backwards(tb, &tb->static_stack, lastkey, &doit_select_delete, &sc); BUMP_REDS(p, 1000 - sc.max); @@ -1589,11 +1581,11 @@ static int db_select_delete_continue_tree(Process *p, } key = GETKEY(tb, (sc.lastterm)->dbterm.tpl); if (end_condition != NIL && - cmp_partly_bound(end_condition,key,sc.lastterm->dbterm.tpl) > 0) { /* done anyway */ + cmp_partly_bound(end_condition,key) > 0) { /* done anyway */ RET_TO_BIF(erts_make_integer(sc.accum,p),DB_ERROR_NONE); } /* Not done yet, let's trap. */ - sz = size_object_rel(key, sc.lastterm->dbterm.tpl); + sz = size_object(key); if (IS_USMALL(0, sc.accum)) { hp = HAlloc(p, sz + 6); eaccsum = make_small(sc.accum); @@ -1603,7 +1595,7 @@ static int db_select_delete_continue_tree(Process *p, eaccsum = uint_to_big(sc.accum, hp); hp += BIG_UINT_HEAP_SIZE; } - key = copy_struct_rel(key, sz, &hp, &MSO(p), sc.lastterm->dbterm.tpl, NULL); + key = copy_struct(key, sz, &hp, &MSO(p)); continuation = TUPLE5 (hp, tptr[1], @@ -1624,7 +1616,6 @@ static int db_select_delete_tree(Process *p, DbTable *tbl, struct select_delete_context sc; struct mp_info mpi; Eterm lastkey = THE_NON_VALUE; - Eterm* lk_base = NULL; Eterm key; Eterm continuation; unsigned sz; @@ -1668,7 +1659,7 @@ static int db_select_delete_tree(Process *p, DbTable *tbl, sc.mp = mpi.mp; if (!mpi.got_partial && mpi.some_limitation && - CMP(mpi.least,mpi.most) == 0) { + CMP_EQ(mpi.least,mpi.most)) { doit_select_delete(tb,mpi.save_term,&sc, 0 /* direction doesn't matter */); RET_TO_BIF(erts_make_integer(sc.accum,p),DB_ERROR_NONE); @@ -1677,12 +1668,11 @@ static int db_select_delete_tree(Process *p, DbTable *tbl, if (mpi.some_limitation) { if ((this = find_next_from_pb_key(tb, &tb->static_stack, mpi.most)) != NULL) { lastkey = GETKEY(tb, this->dbterm.tpl); - lk_base = this->dbterm.tpl; } sc.end_condition = mpi.least; } - traverse_backwards(tb, &tb->static_stack, lastkey, lk_base, &doit_select_delete, &sc); + traverse_backwards(tb, &tb->static_stack, lastkey, &doit_select_delete, &sc); BUMP_REDS(p, 1000 - sc.max); if (sc.max > 0) { @@ -1690,7 +1680,7 @@ static int db_select_delete_tree(Process *p, DbTable *tbl, } key = GETKEY(tb, (sc.lastterm)->dbterm.tpl); - sz = size_object_rel(key, sc.lastterm->dbterm.tpl); + sz = size_object(key); if (IS_USMALL(0, sc.accum)) { hp = HAlloc(p, sz + PROC_BIN_SIZE + 6); eaccsum = make_small(sc.accum); @@ -1700,7 +1690,7 @@ static int db_select_delete_tree(Process *p, DbTable *tbl, eaccsum = uint_to_big(sc.accum, hp); hp += BIG_UINT_HEAP_SIZE; } - key = copy_struct_rel(key, sz, &hp, &MSO(p), sc.lastterm->dbterm.tpl, NULL); + key = copy_struct(key, sz, &hp, &MSO(p)); mpb = db_make_mp_binary(p,mpi.mp,&hp); continuation = TUPLE5 @@ -1722,6 +1712,28 @@ static int db_select_delete_tree(Process *p, DbTable *tbl, } +static int db_take_tree(Process *p, DbTable *tbl, Eterm key, Eterm *ret) +{ + DbTableTree *tb = &tbl->tree; + TreeDbTerm *this; + + *ret = NIL; + this = linkout_tree(tb, key); + if (this) { + Eterm copy, *hp, *hend; + + hp = HAlloc(p, this->dbterm.size + 2); + hend = hp + this->dbterm.size + 2; + copy = db_copy_object_from_ets(&tb->common, + &this->dbterm, &hp, &MSO(p)); + *ret = CONS(hp, copy, NIL); + hp += 2; + HRelease(p, hend, hp); + free_term(tb, this); + } + return DB_ERROR_NONE; +} + /* ** Other interface routines (not directly coupled to one bif) */ @@ -1817,9 +1829,7 @@ do_db_tree_foreach_offheap(TreeDbTerm *tdbt, do_db_tree_foreach_offheap(tdbt->right, func, arg); } -static TreeDbTerm *linkout_tree(DbTableTree *tb, - Eterm key, Eterm* key_base) -{ +static TreeDbTerm *linkout_tree(DbTableTree *tb, Eterm key) { TreeDbTerm **tstack[STACK_NEED]; int tpos = 0; int dstack[STACK_NEED+1]; @@ -1841,7 +1851,7 @@ static TreeDbTerm *linkout_tree(DbTableTree *tb, for (;;) { if (!*this) { /* Failure */ return NULL; - } else if ((c = cmp_key(tb, key, key_base, *this)) < 0) { + } else if ((c = cmp_key(tb, key, *this)) < 0) { dstack[dpos++] = DIR_LEFT; tstack[tpos++] = this; this = &((*this)->left); @@ -1905,7 +1915,7 @@ static TreeDbTerm *linkout_object_tree(DbTableTree *tb, for (;;) { if (!*this) { /* Failure */ return NULL; - } else if ((c = cmp_key(tb,key,NULL,*this)) < 0) { + } else if ((c = cmp_key(tb,key,*this)) < 0) { dstack[dpos++] = DIR_LEFT; tstack[tpos++] = this; this = &((*this)->left); @@ -2291,15 +2301,13 @@ done: * Find next and previous in sort order */ -static TreeDbTerm *find_next(DbTableTree *tb, DbTreeStack* stack, - Eterm key, Eterm* key_base) -{ +static TreeDbTerm *find_next(DbTableTree *tb, DbTreeStack* stack, Eterm key) { TreeDbTerm *this; TreeDbTerm *tmp; Sint c; if(( this = TOP_NODE(stack)) != NULL) { - if (!cmp_key_eq(tb,key,key_base,this)) { + if (!cmp_key_eq(tb,key,this)) { /* Start from the beginning */ stack->pos = stack->slot = 0; } @@ -2309,7 +2317,7 @@ static TreeDbTerm *find_next(DbTableTree *tb, DbTreeStack* stack, return NULL; for (;;) { PUSH_NODE(stack, this); - if (( c = cmp_key(tb,key,key_base,this) ) > 0) { + if (( c = cmp_key(tb,key,this) ) > 0) { if (this->right == NULL) /* We are at the previos and the element does not exist */ @@ -2349,15 +2357,13 @@ static TreeDbTerm *find_next(DbTableTree *tb, DbTreeStack* stack, return this; } -static TreeDbTerm *find_prev(DbTableTree *tb, DbTreeStack* stack, - Eterm key, Eterm* key_base) -{ +static TreeDbTerm *find_prev(DbTableTree *tb, DbTreeStack* stack, Eterm key) { TreeDbTerm *this; TreeDbTerm *tmp; Sint c; if(( this = TOP_NODE(stack)) != NULL) { - if (!cmp_key_eq(tb,key,key_base,this)) { + if (!cmp_key_eq(tb,key,this)) { /* Start from the beginning */ stack->pos = stack->slot = 0; } @@ -2367,7 +2373,7 @@ static TreeDbTerm *find_prev(DbTableTree *tb, DbTreeStack* stack, return NULL; for (;;) { PUSH_NODE(stack, this); - if (( c = cmp_key(tb,key,key_base,this) ) < 0) { + if (( c = cmp_key(tb,key,this) ) < 0) { if (this->left == NULL) /* We are at the next and the element does not exist */ @@ -2420,8 +2426,7 @@ static TreeDbTerm *find_next_from_pb_key(DbTableTree *tb, DbTreeStack* stack, return NULL; for (;;) { PUSH_NODE(stack, this); - if (( c = cmp_partly_bound(key,GETKEY(tb, this->dbterm.tpl), - this->dbterm.tpl) ) >= 0) { + if (( c = cmp_partly_bound(key,GETKEY(tb, this->dbterm.tpl))) >= 0) { if (this->right == NULL) { do { tmp = POP_NODE(stack); @@ -2454,8 +2459,7 @@ static TreeDbTerm *find_prev_from_pb_key(DbTableTree *tb, DbTreeStack* stack, return NULL; for (;;) { PUSH_NODE(stack, this); - if (( c = cmp_partly_bound(key,GETKEY(tb, this->dbterm.tpl), - this->dbterm.tpl) ) <= 0) { + if (( c = cmp_partly_bound(key,GETKEY(tb, this->dbterm.tpl))) <= 0) { if (this->left == NULL) { do { tmp = POP_NODE(stack); @@ -2486,10 +2490,10 @@ static TreeDbTerm *find_node(DbTableTree *tb, Eterm key) DbTreeStack* stack = get_static_stack(tb); if(!stack || EMPTY_NODE(stack) - || !cmp_key_eq(tb, key, NULL, (this=TOP_NODE(stack)))) { + || !cmp_key_eq(tb, key, (this=TOP_NODE(stack)))) { this = tb->root; - while (this != NULL && (res = cmp_key(tb,key,NULL,this)) != 0) { + while (this != NULL && (res = cmp_key(tb,key,this)) != 0) { if (res < 0) this = this->left; else @@ -2511,7 +2515,7 @@ static TreeDbTerm **find_node2(DbTableTree *tb, Eterm key) Sint res; this = &tb->root; - while ((*this) != NULL && (res = cmp_key(tb, key, NULL, *this)) != 0) { + while ((*this) != NULL && (res = cmp_key(tb, key, *this)) != 0) { if (res < 0) this = &((*this)->left); else @@ -2522,33 +2526,63 @@ static TreeDbTerm **find_node2(DbTableTree *tb, Eterm key) return this; } -static int db_lookup_dbterm_tree(DbTable *tbl, Eterm key, DbUpdateHandle* handle) +static int +db_lookup_dbterm_tree(Process *p, DbTable *tbl, Eterm key, Eterm obj, + DbUpdateHandle* handle) { DbTableTree *tb = &tbl->tree; TreeDbTerm **pp = find_node2(tb, key); - - if (pp == NULL) return 0; + int flags = 0; + + if (pp == NULL) { + if (obj == THE_NON_VALUE) { + return 0; + } else { + Eterm *objp = tuple_val(obj); + int arity = arityval(*objp); + Eterm *htop, *hend; + + ASSERT(arity >= tb->common.keypos); + htop = HAlloc(p, arity + 1); + hend = htop + arity + 1; + sys_memcpy(htop, objp, sizeof(Eterm) * (arity + 1)); + htop[tb->common.keypos] = key; + obj = make_tuple(htop); + + if (db_put_tree(tbl, obj, 1) != DB_ERROR_NONE) { + return 0; + } + + pp = find_node2(tb, key); + ASSERT(pp != NULL); + HRelease(p, hend, htop); + flags |= DB_NEW_OBJECT; + } + } handle->tb = tbl; handle->dbterm = &(*pp)->dbterm; - handle->mustResize = 0; + handle->flags = flags; handle->bp = (void**) pp; handle->new_size = (*pp)->dbterm.size; -#if HALFWORD_HEAP - handle->abs_vec = NULL; -#endif return 1; } -static void db_finalize_dbterm_tree(DbUpdateHandle* handle) +static void +db_finalize_dbterm_tree(int cret, DbUpdateHandle *handle) { - if (handle->mustResize) { - TreeDbTerm* oldp = (TreeDbTerm*) *handle->bp; + DbTable *tbl = handle->tb; + DbTableTree *tb = &tbl->tree; + TreeDbTerm *bp = (TreeDbTerm *) *handle->bp; + if (handle->flags & DB_NEW_OBJECT && cret != DB_ERROR_NONE) { + Eterm ret; + db_erase_tree(tbl, GETKEY(tb, bp->dbterm.tpl), &ret); + } else if (handle->flags & DB_MUST_RESIZE) { db_finalize_resize(handle, offsetof(TreeDbTerm,dbterm)); - reset_static_stack(&handle->tb->tree); + reset_static_stack(tb); - free_term(&handle->tb->tree, oldp); + free_term(tb, bp); } #ifdef DEBUG handle->dbterm = 0; @@ -2561,7 +2595,7 @@ static void db_finalize_dbterm_tree(DbUpdateHandle* handle) */ static void traverse_backwards(DbTableTree *tb, DbTreeStack* stack, - Eterm lastkey, Eterm* lk_base, + Eterm lastkey, int (*doit)(DbTableTree *, TreeDbTerm *, void *, @@ -2580,16 +2614,15 @@ static void traverse_backwards(DbTableTree *tb, this = this->right; } this = TOP_NODE(stack); - next = find_prev(tb, stack, GETKEY(tb, this->dbterm.tpl), - this->dbterm.tpl); + next = find_prev(tb, stack, GETKEY(tb, this->dbterm.tpl)); if (!((*doit)(tb, this, context, 0))) return; } else { - next = find_prev(tb, stack, lastkey, lk_base); + next = find_prev(tb, stack, lastkey); } while ((this = next) != NULL) { - next = find_prev(tb, stack, GETKEY(tb, this->dbterm.tpl), this->dbterm.tpl); + next = find_prev(tb, stack, GETKEY(tb, this->dbterm.tpl)); if (!((*doit)(tb, this, context, 0))) return; } @@ -2600,7 +2633,7 @@ static void traverse_backwards(DbTableTree *tb, */ static void traverse_forward(DbTableTree *tb, DbTreeStack* stack, - Eterm lastkey, Eterm* lk_base, + Eterm lastkey, int (*doit)(DbTableTree *, TreeDbTerm *, void *, @@ -2619,15 +2652,15 @@ static void traverse_forward(DbTableTree *tb, this = this->left; } this = TOP_NODE(stack); - next = find_next(tb, stack, GETKEY(tb, this->dbterm.tpl), this->dbterm.tpl); + next = find_next(tb, stack, GETKEY(tb, this->dbterm.tpl)); if (!((*doit)(tb, this, context, 1))) return; } else { - next = find_next(tb, stack, lastkey, lk_base); + next = find_next(tb, stack, lastkey); } while ((this = next) != NULL) { - next = find_next(tb, stack, GETKEY(tb, this->dbterm.tpl), this->dbterm.tpl); + next = find_next(tb, stack, GETKEY(tb, this->dbterm.tpl)); if (!((*doit)(tb, this, context, 1))) return; } @@ -2656,7 +2689,7 @@ static int key_given(DbTableTree *tb, Eterm pattern, TreeDbTerm **ret, *ret = this; return 1; } else if (partly_bound != NULL && key != am_Underscore && - db_is_variable(key) < 0) + db_is_variable(key) < 0 && !db_has_map(key)) *partly_bound = key; return 0; @@ -2664,7 +2697,7 @@ static int key_given(DbTableTree *tb, Eterm pattern, TreeDbTerm **ret, -static Sint do_cmp_partly_bound(Eterm a, Eterm b, Eterm* b_base, int *done) +static Sint do_cmp_partly_bound(Eterm a, Eterm b, int *done) { Eterm* aa; Eterm* bb; @@ -2678,44 +2711,44 @@ static Sint do_cmp_partly_bound(Eterm a, Eterm b, Eterm* b_base, int *done) *done = 1; return 0; } - if (is_same(a,NULL,b,b_base)) + if (is_same(a,b)) return 0; switch (a & _TAG_PRIMARY_MASK) { case TAG_PRIMARY_LIST: if (!is_list(b)) { - return cmp_rel(a,NULL,b,b_base); + return CMP(a,b); } aa = list_val(a); - bb = list_val_rel(b,b_base); + bb = list_val(b); while (1) { - if ((j = do_cmp_partly_bound(*aa++, *bb++, b_base, done)) != 0 || *done) + if ((j = do_cmp_partly_bound(*aa++, *bb++, done)) != 0 || *done) return j; - if (is_same(*aa, NULL, *bb, b_base)) + if (is_same(*aa, *bb)) return 0; if (is_not_list(*aa) || is_not_list(*bb)) - return do_cmp_partly_bound(*aa, *bb, b_base, done); + return do_cmp_partly_bound(*aa, *bb, done); aa = list_val(*aa); - bb = list_val_rel(*bb,b_base); + bb = list_val(*bb); } case TAG_PRIMARY_BOXED: if ((b & _TAG_PRIMARY_MASK) != TAG_PRIMARY_BOXED) { - return cmp_rel(a,NULL,b,b_base); + return CMP(a,b); } a_hdr = ((*boxed_val(a)) & _TAG_HEADER_MASK) >> _TAG_PRIMARY_SIZE; - b_hdr = ((*boxed_val_rel(b,b_base)) & _TAG_HEADER_MASK) >> _TAG_PRIMARY_SIZE; + b_hdr = ((*boxed_val(b)) & _TAG_HEADER_MASK) >> _TAG_PRIMARY_SIZE; if (a_hdr != b_hdr) { - return cmp_rel(a, NULL, b, b_base); + return CMP(a,b); } if (a_hdr == (_TAG_HEADER_ARITYVAL >> _TAG_PRIMARY_SIZE)) { aa = tuple_val(a); - bb = tuple_val_rel(b, b_base); + bb = tuple_val(b); /* compare the arities */ i = arityval(*aa); /* get the arity*/ if (i < arityval(*bb)) return(-1); if (i > arityval(*bb)) return(1); while (i--) { - if ((j = do_cmp_partly_bound(*++aa, *++bb, b_base, done)) != 0 + if ((j = do_cmp_partly_bound(*++aa, *++bb, done)) != 0 || *done) return j; } @@ -2723,14 +2756,13 @@ static Sint do_cmp_partly_bound(Eterm a, Eterm b, Eterm* b_base, int *done) } /* Drop through */ default: - return cmp_rel(a, NULL, b, b_base); + return CMP(a,b); } } -static Sint cmp_partly_bound(Eterm partly_bound_key, Eterm bound_key, Eterm* bk_base) -{ +static Sint cmp_partly_bound(Eterm partly_bound_key, Eterm bound_key) { int done = 0; - Sint ret = do_cmp_partly_bound(partly_bound_key, bound_key, bk_base, &done); + Sint ret = do_cmp_partly_bound(partly_bound_key, bound_key, &done); #ifdef HARDDEBUG erts_fprintf(stderr,"\ncmp_partly_bound: %T", partly_bound_key); if (ret < 0) @@ -2739,7 +2771,7 @@ static Sint cmp_partly_bound(Eterm partly_bound_key, Eterm bound_key, Eterm* bk_ erts_fprintf(stderr," > "); else erts_fprintf(stderr," == "); - erts_fprintf(stderr,"%R\n", bound_key, bk_base); + erts_fprintf(stderr,"%T\n", bound_key); #endif return ret; } @@ -2956,12 +2988,10 @@ static int doit_select(DbTableTree *tb, TreeDbTerm *this, void *ptr, if (sc->end_condition != NIL && ((forward && cmp_partly_bound(sc->end_condition, - GETKEY_WITH_POS(sc->keypos, this->dbterm.tpl), - this->dbterm.tpl) < 0) || + GETKEY_WITH_POS(sc->keypos, this->dbterm.tpl)) < 0) || (!forward && cmp_partly_bound(sc->end_condition, - GETKEY_WITH_POS(sc->keypos, this->dbterm.tpl), - this->dbterm.tpl) > 0))) { + GETKEY_WITH_POS(sc->keypos, this->dbterm.tpl)) > 0))) { return 0; } ret = db_match_dbterm(&tb->common,sc->p,sc->mp,sc->all_objects, @@ -2993,8 +3023,7 @@ static int doit_select_count(DbTableTree *tb, TreeDbTerm *this, void *ptr, /* Always backwards traversing */ if (sc->end_condition != NIL && (cmp_partly_bound(sc->end_condition, - GETKEY_WITH_POS(sc->keypos, this->dbterm.tpl), - this->dbterm.tpl) > 0)) { + GETKEY_WITH_POS(sc->keypos, this->dbterm.tpl)) > 0)) { return 0; } ret = db_match_dbterm(&tb->common, sc->p, sc->mp, 0, @@ -3020,12 +3049,10 @@ static int doit_select_chunk(DbTableTree *tb, TreeDbTerm *this, void *ptr, if (sc->end_condition != NIL && ((forward && cmp_partly_bound(sc->end_condition, - GETKEY_WITH_POS(sc->keypos, this->dbterm.tpl), - this->dbterm.tpl) < 0) || + GETKEY_WITH_POS(sc->keypos, this->dbterm.tpl)) < 0) || (!forward && cmp_partly_bound(sc->end_condition, - GETKEY_WITH_POS(sc->keypos, this->dbterm.tpl), - this->dbterm.tpl) > 0))) { + GETKEY_WITH_POS(sc->keypos, this->dbterm.tpl)) > 0))) { return 0; } @@ -3063,14 +3090,13 @@ static int doit_select_delete(DbTableTree *tb, TreeDbTerm *this, void *ptr, if (sc->end_condition != NIL && cmp_partly_bound(sc->end_condition, - GETKEY_WITH_POS(sc->keypos, this->dbterm.tpl), - this->dbterm.tpl) > 0) + GETKEY_WITH_POS(sc->keypos, this->dbterm.tpl)) > 0) return 0; ret = db_match_dbterm(&tb->common, sc->p, sc->mp, 0, &this->dbterm, NULL, 0); if (ret == am_true) { key = GETKEY(sc->tb, this->dbterm.tpl); - linkout_tree(sc->tb, key, this->dbterm.tpl); + linkout_tree(sc->tb, key); sc->erase_lastterm = 1; ++sc->accum; } @@ -3096,11 +3122,10 @@ static void do_dump_tree2(DbTableTree* tb, int to, void *to_arg, int show, } else { prefix = ""; - term = make_tuple_rel(t->dbterm.tpl,t->dbterm.tpl); + term = make_tuple(t->dbterm.tpl); } - erts_print(to, to_arg, "%*s%s%R (addr = %p, bal = %d)\n", - offset, "", prefix, term, t->dbterm.tpl, - t, t->balance); + erts_print(to, to_arg, "%*s%s%T (addr = %p, bal = %d)\n", + offset, "", prefix, term, t, t->balance); } do_dump_tree2(tb, to, to_arg, show, t->left, offset + 4); } diff --git a/erts/emulator/beam/erl_db_tree.h b/erts/emulator/beam/erl_db_tree.h index 7bc235e135..6098387f5d 100644 --- a/erts/emulator/beam/erl_db_tree.h +++ b/erts/emulator/beam/erl_db_tree.h @@ -3,16 +3,17 @@ * * Copyright Ericsson AB 1998-2009. All Rights Reserved. * - * The contents of this file are subject to the Erlang Public License, - * Version 1.1, (the "License"); you may not use this file except in - * compliance with the License. You should have received a copy of the - * Erlang Public License along with this software. If not, it can be - * retrieved online at http://www.erlang.org/. - * - * Software distributed under the License is distributed on an "AS IS" - * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See - * the License for the specific language governing rights and limitations - * under the License. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. * * %CopyrightEnd% */ diff --git a/erts/emulator/beam/erl_db_util.c b/erts/emulator/beam/erl_db_util.c index 3927615e04..973be58420 100644 --- a/erts/emulator/beam/erl_db_util.c +++ b/erts/emulator/beam/erl_db_util.c @@ -3,16 +3,17 @@ * * Copyright Ericsson AB 1998-2014. All Rights Reserved. * - * The contents of this file are subject to the Erlang Public License, - * Version 1.1, (the "License"); you may not use this file except in - * compliance with the License. You should have received a copy of the - * Erlang Public License along with this software. If not, it can be - * retrieved online at http://www.erlang.org/. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Software distributed under the License is distributed on an "AS IS" - * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See - * the License for the specific language governing rights and limitations - * under the License. + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. * * %CopyrightEnd% */ @@ -198,11 +199,6 @@ set_match_trace(Process *tracee_p, Eterm fail_term, Eterm tracer, return ret; } - -/* Type checking... */ - -#define BOXED_IS_TUPLE(Boxed) is_arity_value(*boxed_val((Boxed))) - /* ** ** Types and enum's (compiled matches) @@ -218,7 +214,9 @@ typedef enum { matchTuple, matchPushT, matchPushL, + matchPushM, matchPop, + matchSwap, matchBind, matchCmp, matchEqBin, @@ -227,21 +225,21 @@ typedef enum { matchEqRef, matchEq, matchList, + matchMap, + matchKey, matchSkip, matchPushC, matchConsA, /* Car is below Cdr */ matchConsB, /* Cdr is below Car (unusual) */ matchMkTuple, + matchMkFlatMap, + matchMkHashMap, matchCall0, matchCall1, matchCall2, matchCall3, matchPushV, -#if HALFWORD_HEAP - matchPushVGuard, /* First guard-only variable reference */ -#endif - matchPushVResult, /* First variable reference in result, or (if HALFWORD) - in guard if also referenced in result */ + matchPushVResult, /* First variable reference in result */ matchPushExpr, /* Push the whole expression we're matching ('$_') */ matchPushArrayAsList, /* Only when parameter is an Array and not an erlang term (DCOMP_TRACE) */ @@ -308,9 +306,6 @@ DMC_DECLARE_STACK_TYPE(unsigned); typedef struct DMCVariable { int is_bound; int is_in_body; -#if HALFWORD_HEAP - int first_guard_label; /* to maybe change from PushVGuard to PushVResult */ -#endif } DMCVariable; typedef struct DMCHeap { @@ -372,7 +367,6 @@ typedef struct MatchVariable { Eterm term; #ifdef DEBUG Process* proc; - Eterm* base; #endif } MatchVariable; @@ -413,7 +407,7 @@ cleanup_match_pseudo_process(ErtsMatchPseudoProcess *mpsp, int keep_heap) else { int i; for (i = 0; i < ERTS_DEFAULT_MS_HEAP_SIZE; i++) { -#if defined(ARCH_64) && !HALFWORD_HEAP +#if defined(ARCH_64) mpsp->default_heap[i] = (Eterm) 0xdeadbeefdeadbeef; #else mpsp->default_heap[i] = (Eterm) 0xdeadbeef; @@ -856,6 +850,13 @@ static int match_compact(ErlHeapFragment *expr, DMCErrInfo *err_info); static Uint my_size_object(Eterm t); static Eterm my_copy_struct(Eterm t, Eterm **hp, ErlOffHeap* off_heap); +/* Guard subroutines */ +static void +dmc_rearrange_constants(DMCContext *context, DMC_STACK_TYPE(UWord) *text, + int textpos, Eterm *p, Uint nelems); +static DMCRet +dmc_array(DMCContext *context, DMCHeap *heap, DMC_STACK_TYPE(UWord) *text, + Eterm *p, Uint nelems, int *constant); /* Guard compilation */ static void do_emit_constant(DMCContext *context, DMC_STACK_TYPE(UWord) *text, Eterm t); @@ -869,6 +870,9 @@ static DMCRet dmc_tuple(DMCContext *context, DMC_STACK_TYPE(UWord) *text, Eterm t, int *constant); +static DMCRet +dmc_map(DMCContext *context, DMCHeap *heap, DMC_STACK_TYPE(UWord) *text, + Eterm t, int *constant); static DMCRet dmc_variable(DMCContext *context, DMCHeap *heap, DMC_STACK_TYPE(UWord) *text, @@ -888,12 +892,14 @@ static DMCRet compile_guard_expr(DMCContext *context, DMCHeap *heap, DMC_STACK_TYPE(UWord) *text, Eterm t); -/* match expression subroutine */ +/* match expression subroutines */ static DMCRet dmc_one_term(DMCContext *context, DMCHeap *heap, DMC_STACK_TYPE(Eterm) *stack, DMC_STACK_TYPE(UWord) *text, Eterm c); +static Eterm +dmc_private_copy(DMCContext *context, Eterm c); #ifdef DMC_DEBUG @@ -1226,7 +1232,7 @@ Eterm erts_match_set_run(Process *p, Binary *mpsp, { Eterm ret; - ret = db_prog_match(p, mpsp, NIL, NULL, args, num_args, + ret = db_prog_match(p, mpsp, NIL, args, num_args, in_flags, return_flags); #if defined(HARDDEBUG) if (is_non_value(ret)) { @@ -1251,7 +1257,7 @@ static Eterm erts_match_set_run_ets(Process *p, Binary *mpsp, { Eterm ret; - ret = db_prog_match(p, mpsp, args, NULL, NULL, num_args, + ret = db_prog_match(p, mpsp, args, NULL, num_args, ERTS_PAM_COPY_RESULT, return_flags); #if defined(HARDDEBUG) @@ -1364,7 +1370,112 @@ restart: for (;;) { switch (t & _TAG_PRIMARY_MASK) { case TAG_PRIMARY_BOXED: - if (!BOXED_IS_TUPLE(t)) { + if (is_flatmap(t)) { + num_iters = flatmap_get_size(flatmap_val(t)); + if (!structure_checked) { + DMC_PUSH(text, matchMap); + DMC_PUSH(text, num_iters); + } + structure_checked = 0; + for (i = 0; i < num_iters; ++i) { + Eterm key = flatmap_get_keys(flatmap_val(t))[i]; + if (db_is_variable(key) >= 0) { + if (context.err_info) { + add_dmc_err(context.err_info, + "Variable found in map key.", + -1, 0UL, dmcError); + } + goto error; + } else if (key == am_Underscore) { + if (context.err_info) { + add_dmc_err(context.err_info, + "Underscore found in map key.", + -1, 0UL, dmcError); + } + goto error; + } + DMC_PUSH(text, matchKey); + DMC_PUSH(text, dmc_private_copy(&context, key)); + { + int old_stack = ++(context.stack_used); + Eterm value = flatmap_get_values(flatmap_val(t))[i]; + res = dmc_one_term(&context, &heap, &stack, &text, + value); + ASSERT(res != retFail); + if (res == retRestart) { + goto restart; + } + if (old_stack != context.stack_used) { + ASSERT(old_stack + 1 == context.stack_used); + DMC_PUSH(text, matchSwap); + } + if (context.stack_used > context.stack_need) { + context.stack_need = context.stack_used; + } + DMC_PUSH(text, matchPop); + --(context.stack_used); + } + } + break; + } + if (is_hashmap(t)) { + DECLARE_WSTACK(wstack); + Eterm *kv; + num_iters = hashmap_size(t); + if (!structure_checked) { + DMC_PUSH(text, matchMap); + DMC_PUSH(text, num_iters); + } + structure_checked = 0; + + hashmap_iterator_init(&wstack, t, 0); + + while ((kv=hashmap_iterator_next(&wstack)) != NULL) { + Eterm key = CAR(kv); + Eterm value = CDR(kv); + if (db_is_variable(key) >= 0) { + if (context.err_info) { + add_dmc_err(context.err_info, + "Variable found in map key.", + -1, 0UL, dmcError); + } + DESTROY_WSTACK(wstack); + goto error; + } else if (key == am_Underscore) { + if (context.err_info) { + add_dmc_err(context.err_info, + "Underscore found in map key.", + -1, 0UL, dmcError); + } + DESTROY_WSTACK(wstack); + goto error; + } + DMC_PUSH(text, matchKey); + DMC_PUSH(text, dmc_private_copy(&context, key)); + { + int old_stack = ++(context.stack_used); + res = dmc_one_term(&context, &heap, &stack, &text, + value); + ASSERT(res != retFail); + if (res == retRestart) { + DESTROY_WSTACK(wstack); + goto restart; + } + if (old_stack != context.stack_used) { + ASSERT(old_stack + 1 == context.stack_used); + DMC_PUSH(text, matchSwap); + } + if (context.stack_used > context.stack_need) { + context.stack_need = context.stack_used; + } + DMC_PUSH(text, matchPop); + --(context.stack_used); + } + } + DESTROY_WSTACK(wstack); + break; + } + if (!is_tuple(t)) { goto simple_term; } num_iters = arityval(*tuple_val(t)); @@ -1636,71 +1747,6 @@ static Eterm dpm_array_to_list(Process *psp, Eterm *arr, int arity) return ret; } - -#if HALFWORD_HEAP -struct heap_checkpoint_t -{ - Process *p; - Eterm* htop; - ErlHeapFragment* mbuf; - unsigned used_size; - ErlOffHeap off_heap; -}; - -static void heap_checkpoint_init(Process* p, struct heap_checkpoint_t* hcp) -{ - hcp->p = p; - hcp->htop = HEAP_TOP(p); - hcp->mbuf = MBUF(p); - hcp->used_size = hcp->mbuf ? hcp->mbuf->used_size : 0; - hcp->off_heap = MSO(p); -} - -static void heap_checkpoint_revert(struct heap_checkpoint_t* hcp) -{ - struct erl_off_heap_header* oh = MSO(hcp->p).first; - - if (oh != hcp->off_heap.first) { - ASSERT(oh != NULL); - if (hcp->off_heap.first) { - while (oh->next != hcp->off_heap.first) { - oh = oh->next; - } - oh->next = NULL; - } - erts_cleanup_offheap(&MSO(hcp->p)); - MSO(hcp->p) = hcp->off_heap; - } - if (MBUF(hcp->p) != hcp->mbuf) { - ErlHeapFragment* hf = MBUF(hcp->p); - ASSERT(hf != NULL); - if (hcp->mbuf) { - while (hf->next != hcp->mbuf) { - hf = hf->next; - } - hf->next = NULL; - } - free_message_buffer(MBUF(hcp->p)); - MBUF(hcp->p) = hcp->mbuf; - } - if (hcp->mbuf != NULL && hcp->mbuf->used_size != hcp->used_size) { - hcp->mbuf->used_size = hcp->used_size; - } - HEAP_TOP(hcp->p) = hcp->htop; -} -#endif /* HALFWORD_HEAP */ - -static ERTS_INLINE Eterm copy_object_rel(Process* p, Eterm term, Eterm* base) -{ - if (!is_immed(term)) { - Uint sz = size_object_rel(term, base); - Eterm* top = HAllocX(p, sz, HEAP_XTRA); - return copy_struct_rel(term, sz, &top, &MSO(p), base, NULL); - } - return term; -} - - /* ** Execution of the match program, this is Pam. ** May return THE_NON_VALUE, which is a bailout. @@ -1708,17 +1754,15 @@ static ERTS_INLINE Eterm copy_object_rel(Process* p, Eterm term, Eterm* base) ** i.e. 'DCOMP_TRACE' was specified */ Eterm db_prog_match(Process *c_p, Binary *bprog, - Eterm term, Eterm* base, + Eterm term, Eterm *termp, int arity, enum erts_pam_run_flags in_flags, Uint32 *return_flags) { MatchProg *prog = Binary2MatchProg(bprog); - Eterm *ep; - Eterm *tp; + const Eterm *ep, *tp, **sp; Eterm t; - Eterm **sp; Eterm *esp; MatchVariable* variables; BeamInstr *cp; @@ -1738,17 +1782,12 @@ Eterm db_prog_match(Process *c_p, Binary *bprog, Eterm bif_args[3]; int fail_label; int atomic_trace; -#if HALFWORD_HEAP - struct heap_checkpoint_t c_p_checkpoint = {}; -#endif #ifdef DMC_DEBUG Uint *heap_fence; Uint *stack_fence; Uint save_op; #endif /* DMC_DEBUG */ - ASSERT(base==NULL || HALFWORD_HEAP); - mpsp = get_match_pseudo_process(c_p, prog->heap_size); psp = &mpsp->process; @@ -1799,29 +1838,23 @@ Eterm db_prog_match(Process *c_p, Binary *bprog, do_catch != 0 */ *return_flags = 0U; - variables = mpsp->u.variables; -#if HALFWORD_HEAP - c_p_checkpoint.p = NULL; -#endif restart: ep = &term; esp = (Eterm*)((char*)mpsp->u.heap + prog->stack_offset); - sp = (Eterm **) esp; + sp = (const Eterm **)esp; ret = am_true; do_catch = 0; fail_label = -1; build_proc = psp; esdp->current_process = psp; - ASSERT_HALFWORD(!c_p_checkpoint.p); #ifdef DEBUG ASSERT(variables == mpsp->u.variables); for (i=0; i<prog->num_bindings; i++) { variables[i].term = THE_NON_VALUE; variables[i].proc = NULL; - variables[i].base = base; } #endif @@ -1829,11 +1862,11 @@ restart: #ifdef DMC_DEBUG if (*heap_fence != FENCE_PATTERN) { - erl_exit(1, "Heap fence overwritten in db_prog_match after op " + erts_exit(ERTS_ERROR_EXIT, "Heap fence overwritten in db_prog_match after op " "0x%08x, overwritten with 0x%08x.", save_op, *heap_fence); } if (*stack_fence != FENCE_PATTERN) { - erl_exit(1, "Stack fence overwritten in db_prog_match after op " + erts_exit(ERTS_ERROR_EXIT, "Stack fence overwritten in db_prog_match after op " "0x%08x, overwritten with 0x%08x.", save_op, *stack_fence); } @@ -1857,9 +1890,9 @@ restart: variables[n].term = dpm_array_to_list(psp, termp, arity); break; case matchTuple: /* *ep is a tuple of arity n */ - if (!is_tuple_rel(*ep,base)) + if (!is_tuple(*ep)) FAIL(); - ep = tuple_val_rel(*ep,base); + ep = tuple_val(*ep); n = *pc++; if (arityval(*ep) != n) FAIL(); @@ -1867,9 +1900,9 @@ restart: break; case matchPushT: /* *ep is a tuple of arity n, push ptr to first element */ - if (!is_tuple_rel(*ep,base)) + if (!is_tuple(*ep)) FAIL(); - tp = tuple_val_rel(*ep,base); + tp = tuple_val(*ep); n = *pc++; if (arityval(*tp) != n) FAIL(); @@ -1879,46 +1912,94 @@ restart: case matchList: if (!is_list(*ep)) FAIL(); - ep = list_val_rel(*ep,base); + ep = list_val(*ep); break; case matchPushL: if (!is_list(*ep)) FAIL(); - *sp++ = list_val_rel(*ep,base); + *sp++ = list_val(*ep); ++ep; break; + case matchMap: + if (!is_map(*ep)) { + FAIL(); + } + n = *pc++; + if (is_flatmap(*ep)) { + if (flatmap_get_size(flatmap_val(*ep)) < n) { + FAIL(); + } + } else { + ASSERT(is_hashmap(*ep)); + if (hashmap_size(*ep) < n) { + FAIL(); + } + } + ep = flatmap_val(*ep); + break; + case matchPushM: + if (!is_map(*ep)) { + FAIL(); + } + n = *pc++; + if (is_flatmap(*ep)) { + if (flatmap_get_size(flatmap_val(*ep)) < n) { + FAIL(); + } + } else { + ASSERT(is_hashmap(*ep)); + if (hashmap_size(*ep) < n) { + FAIL(); + } + } + *sp++ = flatmap_val(*ep++); + break; + case matchKey: + t = (Eterm) *pc++; + tp = erts_maps_get(t, make_boxed(ep)); + if (!tp) { + FAIL(); + } + *sp++ = ep; + ep = tp; + break; case matchPop: ep = *(--sp); break; + case matchSwap: + tp = sp[-1]; + sp[-1] = sp[-2]; + sp[-2] = tp; + break; case matchBind: n = *pc++; variables[n].term = *ep++; break; case matchCmp: n = *pc++; - if (!eq_rel(variables[n].term, base, *ep, base)) + if (!EQ(variables[n].term, *ep)) FAIL(); ++ep; break; case matchEqBin: t = (Eterm) *pc++; - if (!eq_rel(t,NULL,*ep,base)) + if (!EQ(t,*ep)) FAIL(); ++ep; break; case matchEqFloat: - if (!is_float_rel(*ep,base)) + if (!is_float(*ep)) FAIL(); - if (memcmp(float_val_rel(*ep,base) + 1, pc, sizeof(double))) + if (memcmp(float_val(*ep) + 1, pc, sizeof(double))) FAIL(); pc += TermWords(2); ++ep; break; case matchEqRef: { Eterm* epc = (Eterm*)pc; - if (!is_ref_rel(*ep,base)) + if (!is_ref(*ep)) FAIL(); - if (!eq_rel(make_internal_ref_rel(epc, epc), epc, *ep, base)) { + if (!EQ(make_internal_ref(epc), *ep)) { FAIL(); } i = thing_arityval(*epc); @@ -1927,9 +2008,9 @@ restart: break; } case matchEqBig: - if (!is_big_rel(*ep,base)) + if (!is_big(*ep)) FAIL(); - tp = big_val_rel(*ep,base); + tp = big_val(*ep); { Eterm *epc = (Eterm *) pc; if (*tp != *epc) @@ -1987,6 +2068,39 @@ restart: } *esp++ = t; break; + case matchMkFlatMap: + n = *pc++; + ehp = HAllocX(build_proc, MAP_HEADER_FLATMAP_SZ + n, HEAP_XTRA); + t = *--esp; + { + flatmap_t *m = (flatmap_t *)ehp; + m->thing_word = MAP_HEADER_FLATMAP; + m->size = n; + m->keys = t; + } + t = make_flatmap(ehp); + ehp += MAP_HEADER_FLATMAP_SZ; + while (n--) { + *ehp++ = *--esp; + } + *esp++ = t; + break; + case matchMkHashMap: + n = *pc++; + esp -= 2*n; + ehp = HAllocX(build_proc, 2*n, HEAP_XTRA); + { + ErtsHeapFactory factory; + Uint ix; + for (ix = 0; ix < 2*n; ix++){ + ehp[ix] = esp[ix]; + } + erts_factory_proc_init(&factory, build_proc); + t = erts_hashmap_from_array(&factory, ehp, n, 0); + erts_factory_close(&factory); + } + *esp++ = t; + break; case matchCall0: bif = (Eterm (*)(Process*, ...)) *pc++; t = (*bif)(build_proc, bif_args); @@ -2038,62 +2152,36 @@ restart: esp -= 2; esp[-1] = t; break; - - #if HALFWORD_HEAP - case matchPushVGuard: - if (!base) goto case_matchPushV; - /* Build NULL-based copy on pseudo heap for easy disposal */ - n = *pc++; - ASSERT(is_value(variables[n].term)); - ASSERT(!variables[n].proc); - variables[n].term = copy_object_rel(psp, variables[n].term, base); - *esp++ = variables[n].term; - #ifdef DEBUG - variables[n].proc = psp; - variables[n].base = NULL; - #endif - break; - #endif case matchPushVResult: if (!(in_flags & ERTS_PAM_COPY_RESULT)) goto case_matchPushV; - - /* Build (NULL-based) copy on callers heap */ - #if HALFWORD_HEAP - if (!do_catch && !c_p_checkpoint.p) { - heap_checkpoint_init(c_p, &c_p_checkpoint); - } - #endif + /* Build copy on callers heap */ n = *pc++; ASSERT(is_value(variables[n].term)); ASSERT(!variables[n].proc); - variables[n].term = copy_object_rel(c_p, variables[n].term, base); + variables[n].term = copy_object_x(variables[n].term, c_p, HEAP_XTRA); *esp++ = variables[n].term; #ifdef DEBUG variables[n].proc = c_p; - variables[n].base = NULL; #endif break; case matchPushV: case_matchPushV: n = *pc++; ASSERT(is_value(variables[n].term)); - ASSERT(!variables[n].base); *esp++ = variables[n].term; break; case matchPushExpr: if (in_flags & ERTS_PAM_COPY_RESULT) { Uint sz; Eterm* top; - sz = size_object_rel(term, base); + sz = size_object(term); top = HAllocX(build_proc, sz, HEAP_XTRA); if (in_flags & ERTS_PAM_CONTIGUOUS_TUPLE) { - ASSERT(is_tuple_rel(term,base)); - *esp++ = copy_shallow_rel(tuple_val_rel(term,base), sz, - &top, &MSO(build_proc), base); + ASSERT(is_tuple(term)); + *esp++ = copy_shallow(tuple_val(term), sz, &top, &MSO(build_proc)); } else { - *esp++ = copy_struct_rel(term, sz, &top, &MSO(build_proc), - base, NULL); + *esp++ = copy_struct(term, sz, &top, &MSO(build_proc)); } } else { @@ -2101,7 +2189,6 @@ restart: } break; case matchPushArrayAsList: - ASSERT_HALFWORD(base == NULL); n = arity; /* Only happens when 'term' is an array */ tp = termp; ehp = HAllocX(build_proc, n*2, HEAP_XTRA); @@ -2117,7 +2204,6 @@ restart: break; case matchPushArrayAsListU: /* This instruction is NOT efficient. */ - ASSERT_HALFWORD(base == NULL); *esp++ = dpm_array_to_list(build_proc, termp, arity); break; case matchTrue: @@ -2222,11 +2308,7 @@ restart: *esp++ = am_true; break; case matchIsSeqTrace: - if (SEQ_TRACE_TOKEN(c_p) != NIL -#ifdef USE_VM_PROBES - && SEQ_TRACE_TOKEN(c_p) != am_have_dt_utag -#endif - ) + if (have_seqtrace(SEQ_TRACE_TOKEN(c_p))) *esp++ = am_true; else *esp++ = am_false; @@ -2250,11 +2332,7 @@ restart: --esp; break; case matchGetSeqToken: - if (SEQ_TRACE_TOKEN(c_p) == NIL -#ifdef USE_VM_PROBES - || SEQ_TRACE_TOKEN(c_p) == am_have_dt_utag -#endif - ) + if (have_no_seqtrace(SEQ_TRACE_TOKEN(c_p))) *esp++ = NIL; else { Eterm sender = SEQ_TRACE_TOKEN_SENDER(c_p); @@ -2417,17 +2495,10 @@ restart: case matchHalt: goto success; default: - erl_exit(1, "Internal error: unexpected opcode in match program."); + erts_exit(ERTS_ERROR_EXIT, "Internal error: unexpected opcode in match program."); } } fail: -#if HALFWORD_HEAP - if (c_p_checkpoint.p) { - /* Dispose garbage built by guards on caller heap */ - heap_checkpoint_revert(&c_p_checkpoint); - c_p_checkpoint.p = NULL; - } -#endif *return_flags = 0U; if (fail_label >= 0) { /* We failed during a "TryMeElse", lets restart, with the next match @@ -2441,11 +2512,11 @@ success: #ifdef DMC_DEBUG if (*heap_fence != FENCE_PATTERN) { - erl_exit(1, "Heap fence overwritten in db_prog_match after op " + erts_exit(ERTS_ERROR_EXIT, "Heap fence overwritten in db_prog_match after op " "0x%08x, overwritten with 0x%08x.", save_op, *heap_fence); } if (*stack_fence != FENCE_PATTERN) { - erl_exit(1, "Stack fence overwritten in db_prog_match after op " + erts_exit(ERTS_ERROR_EXIT, "Stack fence overwritten in db_prog_match after op " "0x%08x, overwritten with 0x%08x.", save_op, *stack_fence); } @@ -2590,21 +2661,14 @@ Wterm db_do_read_element(DbUpdateHandle* handle, Sint position) { Eterm elem = handle->dbterm->tpl[position]; if (!is_header(elem)) { -#if HALFWORD_HEAP - if (!is_immed(elem) - && !handle->tb->common.compress - && !(handle->abs_vec && handle->abs_vec[position])) { - return rterm2wterm(elem, handle->dbterm->tpl); - } -#endif return elem; } ASSERT(((DbTableCommon*)handle->tb)->compress); - ASSERT(!handle->mustResize); + ASSERT(!(handle->flags & DB_MUST_RESIZE)); handle->dbterm = db_alloc_tmp_uncompressed(&handle->tb->common, handle->dbterm); - handle->mustResize = 1; + handle->flags |= DB_MUST_RESIZE; return handle->dbterm->tpl[position]; } @@ -2624,9 +2688,6 @@ void db_do_update_element(DbUpdateHandle* handle, Eterm* oldp; Uint newval_sz; Uint oldval_sz; -#if HALFWORD_HEAP - Eterm* old_base; -#endif if (is_both_immed(newval,oldval)) { handle->dbterm->tpl[position] = newval; @@ -2637,21 +2698,14 @@ void db_do_update_element(DbUpdateHandle* handle, #endif return; } - if (!handle->mustResize) { + if (!(handle->flags & DB_MUST_RESIZE)) { if (handle->tb->common.compress) { handle->dbterm = db_alloc_tmp_uncompressed(&handle->tb->common, handle->dbterm); - handle->mustResize = 1; + handle->flags |= DB_MUST_RESIZE; oldval = handle->dbterm->tpl[position]; - #if HALFWORD_HEAP - old_base = NULL; - #endif } else { - #if HALFWORD_HEAP - ASSERT(!handle->abs_vec); - old_base = handle->dbterm->tpl; - #endif if (is_boxed(newval)) { newp = boxed_val(newval); switch (*newp & _TAG_HEADER_MASK) { @@ -2661,7 +2715,7 @@ void db_do_update_element(DbUpdateHandle* handle, case _TAG_HEADER_HEAP_BIN: newval_sz = header_arity(*newp) + 1; if (is_boxed(oldval)) { - oldp = boxed_val_rel(oldval,old_base); + oldp = boxed_val(oldval); switch (*oldp & _TAG_HEADER_MASK) { case _TAG_HEADER_POS_BIG: case _TAG_HEADER_NEG_BIG: @@ -2681,40 +2735,20 @@ void db_do_update_element(DbUpdateHandle* handle, } } } -#if HALFWORD_HEAP - else { - old_base = (handle->tb->common.compress - || (handle->abs_vec && handle->abs_vec[position])) ? - NULL : handle->dbterm->tpl; - } -#endif /* Not possible for simple memcpy or dbterm is already non-contiguous, */ /* need to realloc... */ newval_sz = is_immed(newval) ? 0 : size_object(newval); new_size_set: - oldval_sz = is_immed(oldval) ? 0 : size_object_rel(oldval,old_base); + oldval_sz = is_immed(oldval) ? 0 : size_object(oldval); both_size_set: handle->new_size = handle->new_size - oldval_sz + newval_sz; /* write new value in old dbterm, finalize will make a flat copy */ handle->dbterm->tpl[position] = newval; - handle->mustResize = 1; - -#if HALFWORD_HEAP - if (old_base && newval_sz > 0) { - ASSERT(!handle->tb->common.compress); - if (!handle->abs_vec) { - int i = header_arity(handle->dbterm->tpl[0]); - handle->abs_vec = erts_alloc(ERTS_ALC_T_TMP, (i+1)*sizeof(char)); - sys_memset(handle->abs_vec, 0, i+1); - /* abs_vec[0] not used */ - } - handle->abs_vec[position] = 1; - } -#endif + handle->flags |= DB_MUST_RESIZE; } static ERTS_INLINE byte* db_realloc_term(DbTableCommon* tb, void* old, @@ -2813,7 +2847,7 @@ static void* copy_to_comp(DbTableCommon* tb, Eterm obj, DbTerm* dest, tpl[arity + 1] = alloc_size; tmp_offheap.first = NULL; - tpl[tb->keypos] = copy_struct_rel(key, size_object(key), &top.ep, &tmp_offheap, NULL, tpl); + tpl[tb->keypos] = copy_struct(key, size_object(key), &top.ep, &tmp_offheap); dest->first_oh = tmp_offheap.first; for (i=1; i<=arity; i++) { if (i != tb->keypos) { @@ -2832,7 +2866,7 @@ static void* copy_to_comp(DbTableCommon* tb, Eterm obj, DbTerm* dest, Eterm* dbg_top = erts_alloc(ERTS_ALC_T_DB_TERM, dest->size * sizeof(Eterm)); dest->debug_clone = dbg_top; tmp_offheap.first = dest->first_oh; - copy_struct_rel(obj, dest->size, &dbg_top, &tmp_offheap, NULL, dbg_top); + copy_struct(obj, dest->size, &dbg_top, &tmp_offheap); dest->first_oh = tmp_offheap.first; ASSERT(dbg_top == dest->debug_clone + dest->size); } @@ -2879,7 +2913,7 @@ void* db_store_term(DbTableCommon *tb, DbTerm* old, Uint offset, Eterm obj) newp->size = size; top = newp->tpl; tmp_offheap.first = NULL; - copy_struct_rel(obj, size, &top, &tmp_offheap, NULL, top); + copy_struct(obj, size, &top, &tmp_offheap); newp->first_oh = tmp_offheap.first; #ifdef DEBUG_CLONE newp->debug_clone = NULL; @@ -2962,29 +2996,8 @@ void db_finalize_resize(DbUpdateHandle* handle, Uint offset) tmp_offheap.first = NULL; - #if HALFWORD_HEAP - if (handle->abs_vec) { - int i, arity = header_arity(handle->dbterm->tpl[0]); - - top[0] = tpl[0]; - top += arity + 1; - for (i=1; i<=arity; i++) { - Eterm* src_base = handle->abs_vec[i] ? NULL : tpl; - - newDbTerm->tpl[i] = copy_struct_rel(tpl[i], - size_object_rel(tpl[i],src_base), - &top, &tmp_offheap, src_base, - newDbTerm->tpl); - } - newDbTerm->first_oh = tmp_offheap.first; - ASSERT((byte*)top <= (newp + alloc_sz)); - erts_free(ERTS_ALC_T_TMP, handle->abs_vec); - } - else - #endif /* HALFWORD_HEAP */ { - copy_struct_rel(make_tuple_rel(tpl,tpl), handle->new_size, &top, - &tmp_offheap, tpl, top); + copy_struct(make_tuple(tpl), handle->new_size, &top, &tmp_offheap); newDbTerm->first_oh = tmp_offheap.first; ASSERT((byte*)top == (newp + alloc_sz)); } @@ -2996,27 +3009,34 @@ Eterm db_copy_from_comp(DbTableCommon* tb, DbTerm* bp, Eterm** hpp, { Eterm* hp = *hpp; int i, arity = arityval(bp->tpl[0]); + ErtsHeapFactory factory; hp[0] = bp->tpl[0]; *hpp += arity + 1; - hp[tb->keypos] = copy_struct_rel(bp->tpl[tb->keypos], - size_object_rel(bp->tpl[tb->keypos], bp->tpl), - hpp, off_heap, bp->tpl, NULL); + hp[tb->keypos] = copy_struct(bp->tpl[tb->keypos], + size_object(bp->tpl[tb->keypos]), + hpp, off_heap); + + erts_factory_static_init(&factory, *hpp, bp->size - (arity+1), off_heap); + for (i=arity; i>0; i--) { if (i != tb->keypos) { if (is_immed(bp->tpl[i])) { hp[i] = bp->tpl[i]; } else { - hp[i] = erts_decode_ext_ets(hpp, off_heap, + hp[i] = erts_decode_ext_ets(&factory, elem2ext(bp->tpl, i)); } } } + *hpp = factory.hp; + erts_factory_close(&factory); + ASSERT((*hpp - hp) <= bp->size); #ifdef DEBUG_CLONE - ASSERT(eq_rel(make_tuple(hp),NULL,make_tuple(bp->debug_clone),bp->debug_clone)); + ASSERT(EQ(make_tuple(hp),make_tuple(bp->debug_clone))); #endif return make_tuple(hp); } @@ -3032,21 +3052,22 @@ Eterm db_copy_element_from_ets(DbTableCommon* tb, Process* p, if (tb->compress && pos != tb->keypos) { byte* ext = elem2ext(obj->tpl, pos); Sint sz = erts_decode_ext_size_ets(ext, db_alloced_size_comp(obj)) + extra; - Eterm* hp = HAlloc(p, sz); - Eterm* endp = hp + sz; - Eterm copy = erts_decode_ext_ets(&hp, &MSO(p), ext); - *hpp = hp; - hp += extra; - HRelease(p, endp, hp); + Eterm copy; + ErtsHeapFactory factory; + + erts_factory_proc_prealloc_init(&factory, p, sz); + copy = erts_decode_ext_ets(&factory, ext); + *hpp = erts_produce_heap(&factory, extra, 0); + erts_factory_close(&factory); #ifdef DEBUG_CLONE - ASSERT(eq_rel(copy, NULL, obj->debug_clone[pos], obj->debug_clone)); + ASSERT(EQ(copy, obj->debug_clone[pos])); #endif return copy; } else { - Uint sz = size_object_rel(obj->tpl[pos], obj->tpl); + Uint sz = size_object(obj->tpl[pos]); *hpp = HAlloc(p, sz + extra); - return copy_struct_rel(obj->tpl[pos], sz, hpp, &MSO(p), obj->tpl, NULL); + return copy_struct(obj->tpl[pos], sz, hpp, &MSO(p)); } } @@ -3151,39 +3172,86 @@ int db_is_variable(Eterm obj) return N; } +/* check if node is (or contains) a map + * return 1 if node contains a map + * return 0 otherwise + */ + +int db_has_map(Eterm node) { + DECLARE_ESTACK(s); + + ESTACK_PUSH(s,node); + while (!ESTACK_ISEMPTY(s)) { + node = ESTACK_POP(s); + if (is_list(node)) { + while (is_list(node)) { + ESTACK_PUSH(s,CAR(list_val(node))); + node = CDR(list_val(node)); + } + ESTACK_PUSH(s,node); /* Non wellformed list or [] */ + } else if (is_tuple(node)) { + Eterm *tuple = tuple_val(node); + int arity = arityval(*tuple); + while(arity--) { + ESTACK_PUSH(s,*(++tuple)); + } + } else if is_map(node) { + DESTROY_ESTACK(s); + return 1; + } + } + DESTROY_ESTACK(s); + return 0; +} /* check if obj is (or contains) a variable */ /* return 1 if obj contains a variable or underscore */ /* return 0 if obj is fully ground */ -int db_has_variable(Eterm obj) -{ - switch(obj & _TAG_PRIMARY_MASK) { - case TAG_PRIMARY_LIST: { - while (is_list(obj)) { - if (db_has_variable(CAR(list_val(obj)))) +int db_has_variable(Eterm node) { + DECLARE_ESTACK(s); + + ESTACK_PUSH(s,node); + while (!ESTACK_ISEMPTY(s)) { + node = ESTACK_POP(s); + switch(node & _TAG_PRIMARY_MASK) { + case TAG_PRIMARY_LIST: + while (is_list(node)) { + ESTACK_PUSH(s,CAR(list_val(node))); + node = CDR(list_val(node)); + } + ESTACK_PUSH(s,node); /* Non wellformed list or [] */ + break; + case TAG_PRIMARY_BOXED: + if (is_tuple(node)) { + Eterm *tuple = tuple_val(node); + int arity = arityval(*tuple); + while(arity--) { + ESTACK_PUSH(s,*(++tuple)); + } + } else if (is_flatmap(node)) { + Eterm *values = flatmap_get_values(flatmap_val(node)); + Uint size = flatmap_get_size(flatmap_val(node)); + ESTACK_PUSH(s, ((flatmap_t *) flatmap_val(node))->keys); + while (size--) { + ESTACK_PUSH(s, *(values++)); + } + } else if (is_map(node)) { /* other map-nodes or map-heads */ + Eterm *ptr = hashmap_val(node); + int i = hashmap_bitcount(MAP_HEADER_VAL(*ptr)); + ptr += MAP_HEADER_ARITY(*ptr); + while(i--) { ESTACK_PUSH(s, *++ptr); } + } + break; + case TAG_PRIMARY_IMMED1: + if (node == am_Underscore || db_is_variable(node) >= 0) { + DESTROY_ESTACK(s); return 1; - obj = CDR(list_val(obj)); - } - return(db_has_variable(obj)); /* Non wellformed list or [] */ - } - case TAG_PRIMARY_BOXED: - if (!BOXED_IS_TUPLE(obj)) { - return 0; - } else { - Eterm *tuple = tuple_val(obj); - int arity = arityval(*tuple++); - while(arity--) { - if (db_has_variable(*tuple)) - return 1; - tuple++; } - return(0); + break; } - case TAG_PRIMARY_IMMED1: - if (obj == am_Underscore || db_is_variable(obj) >= 0) - return 1; } + DESTROY_ESTACK(s); return 0; } @@ -3243,11 +3311,9 @@ static DMCRet dmc_one_term(DMCContext *context, { Sint n; Eterm *hp; - ErlHeapFragment *tmp_mb; Uint sz, sz2, sz3; Uint i, j; - switch (c & _TAG_PRIMARY_MASK) { case TAG_PRIMARY_IMMED1: if ((n = db_is_variable(c)) >= 0) { /* variable */ @@ -3334,30 +3400,24 @@ static DMCRet dmc_one_term(DMCContext *context, DMC_PUSH(*text, n); DMC_PUSH(*stack, c); break; + case (_TAG_HEADER_MAP >> _TAG_PRIMARY_SIZE): + if (is_flatmap(c)) + n = flatmap_get_size(flatmap_val(c)); + else + n = hashmap_size(c); + DMC_PUSH(*text, matchPushM); + ++(context->stack_used); + DMC_PUSH(*text, n); + DMC_PUSH(*stack, c); + break; case (_TAG_HEADER_REF >> _TAG_PRIMARY_SIZE): { Eterm* ref_val = internal_ref_val(c); DMC_PUSH(*text, matchEqRef); -#if HALFWORD_HEAP - { - union { - UWord u; - Uint t[2]; - } fiddle; - ASSERT(thing_arityval(ref_val[0]) == 3); - fiddle.t[0] = ref_val[0]; - fiddle.t[1] = ref_val[1]; - DMC_PUSH(*text, fiddle.u); - fiddle.t[0] = ref_val[2]; - fiddle.t[1] = ref_val[3]; - DMC_PUSH(*text, fiddle.u); - } -#else n = thing_arityval(ref_val[0]); for (i = 0; i <= n; ++i) { DMC_PUSH(*text, ref_val[i]); } -#endif break; } case (_TAG_HEADER_POS_BIG >> _TAG_PRIMARY_SIZE): @@ -3366,77 +3426,51 @@ static DMCRet dmc_one_term(DMCContext *context, Eterm* bval = big_val(c); n = thing_arityval(bval[0]); DMC_PUSH(*text, matchEqBig); -#if HALFWORD_HEAP - { - union { - UWord u; - Uint t[2]; - } fiddle; - ASSERT(n >= 1); - fiddle.t[0] = bval[0]; - fiddle.t[1] = bval[1]; - DMC_PUSH(*text, fiddle.u); - for (i = 2; i <= n; ++i) { - fiddle.t[0] = bval[i]; - if (++i <= n) { - fiddle.t[1] = bval[i]; - } else { - fiddle.t[1] = (Uint) 0; - } - DMC_PUSH(*text, fiddle.u); - } - } -#else for (i = 0; i <= n; ++i) { DMC_PUSH(*text, (Uint) bval[i]); } -#endif break; } case (_TAG_HEADER_FLOAT >> _TAG_PRIMARY_SIZE): DMC_PUSH(*text,matchEqFloat); -#if HALFWORD_HEAP - { - union { - UWord u; - Uint t[2]; - } fiddle; - fiddle.t[0] = float_val(c)[1]; - fiddle.t[1] = float_val(c)[2]; - DMC_PUSH(*text, fiddle.u); - } -#else DMC_PUSH(*text, (Uint) float_val(c)[1]); #ifdef ARCH_64 DMC_PUSH(*text, (Uint) 0); #else DMC_PUSH(*text, (Uint) float_val(c)[2]); #endif -#endif break; default: /* BINARY, FUN, VECTOR, or EXTERNAL */ - /* - ** Make a private copy... - */ - n = size_object(c); - tmp_mb = new_message_buffer(n); - hp = tmp_mb->mem; DMC_PUSH(*text, matchEqBin); - DMC_PUSH(*text, copy_struct(c, n, &hp, &(tmp_mb->off_heap))); - tmp_mb->next = context->save; - context->save = tmp_mb; + DMC_PUSH(*text, dmc_private_copy(context, c)); break; } break; } default: - erl_exit(1, "db_match_compile: " + erts_exit(ERTS_ERROR_EXIT, "db_match_compile: " "Bad object on heap: 0x%bex\n", c); } return retOk; } /* +** Make a private copy of a term in a context. +*/ + +static Eterm +dmc_private_copy(DMCContext *context, Eterm c) +{ + Uint n = size_object(c); + ErlHeapFragment *tmp_mb = new_message_buffer(n); + Eterm *hp = tmp_mb->mem; + Eterm copy = copy_struct(c, n, &hp, &(tmp_mb->off_heap)); + tmp_mb->next = context->save; + context->save = tmp_mb; + return copy; +} + +/* ** Match guard compilation */ @@ -3527,57 +3561,78 @@ static DMCRet dmc_list(DMCContext *context, return retOk; } -static DMCRet dmc_tuple(DMCContext *context, - DMCHeap *heap, - DMC_STACK_TYPE(UWord) *text, - Eterm t, - int *constant) +static void +dmc_rearrange_constants(DMCContext *context, DMC_STACK_TYPE(UWord) *text, + int textpos, Eterm *p, Uint nelems) { DMC_STACK_TYPE(UWord) instr_save; + Uint i; + + DMC_INIT_STACK(instr_save); + while (DMC_STACK_NUM(*text) > textpos) { + DMC_PUSH(instr_save, DMC_POP(*text)); + } + for (i = nelems; i--;) { + do_emit_constant(context, text, p[i]); + } + while(!DMC_EMPTY(instr_save)) { + DMC_PUSH(*text, DMC_POP(instr_save)); + } + DMC_FREE(instr_save); +} + +static DMCRet +dmc_array(DMCContext *context, DMCHeap *heap, DMC_STACK_TYPE(UWord) *text, + Eterm *p, Uint nelems, int *constant) +{ int all_constant = 1; int textpos = DMC_STACK_NUM(*text); - Eterm *p = tuple_val(t); - Uint nelems = arityval(*p); Uint i; - int c; - DMCRet ret; /* - ** We remember where we started to layout code, + ** We remember where we started to layout code, ** assume all is constant and back up and restart if not so. - ** The tuple should be laid out with the last element first, - ** so we can memcpy the tuple to the eheap. + ** The array should be laid out with the last element first, + ** so we can memcpy it to the eheap. */ - for (i = nelems; i > 0; --i) { - if ((ret = dmc_expr(context, heap, text, p[i], &c)) != retOk) - return ret; - if (!c && all_constant) { - all_constant = 0; - if (i < nelems) { - Uint j; + for (i = nelems; i--;) { + DMCRet ret; + int c; - /* - * Oops, we need to relayout the constants. - * Save the already laid out instructions. - */ - DMC_INIT_STACK(instr_save); - while (DMC_STACK_NUM(*text) > textpos) - DMC_PUSH(instr_save, DMC_POP(*text)); - for (j = nelems; j > i; --j) - do_emit_constant(context, text, p[j]); - while(!DMC_EMPTY(instr_save)) - DMC_PUSH(*text, DMC_POP(instr_save)); - DMC_FREE(instr_save); - } - } else if (c && !all_constant) { - /* push a constant */ - do_emit_constant(context, text, p[i]); - } + ret = dmc_expr(context, heap, text, p[i], &c); + if (ret != retOk) { + return ret; + } + if (!c && all_constant) { + all_constant = 0; + if (i < nelems - 1) { + dmc_rearrange_constants(context, text, textpos, + p + i + 1, nelems - i - 1); + } + } else if (c && !all_constant) { + do_emit_constant(context, text, p[i]); + } + } + *constant = all_constant; + return retOk; +} + +static DMCRet +dmc_tuple(DMCContext *context, DMCHeap *heap, DMC_STACK_TYPE(UWord) *text, + Eterm t, int *constant) +{ + int all_constant; + Eterm *p = tuple_val(t); + Uint nelems = arityval(*p); + DMCRet ret; + + ret = dmc_array(context, heap, text, p + 1, nelems, &all_constant); + if (ret != retOk) { + return ret; } - if (all_constant) { - *constant = 1; - return retOk; + *constant = 1; + return retOk; } DMC_PUSH(*text, matchMkTuple); DMC_PUSH(*text, nelems); @@ -3586,6 +3641,93 @@ static DMCRet dmc_tuple(DMCContext *context, return retOk; } +static DMCRet +dmc_map(DMCContext *context, DMCHeap *heap, DMC_STACK_TYPE(UWord) *text, + Eterm t, int *constant) +{ + int nelems; + int constant_values; + DMCRet ret; + if (is_flatmap(t)) { + flatmap_t *m = (flatmap_t *)flatmap_val(t); + Eterm *values = flatmap_get_values(m); + + nelems = flatmap_get_size(m); + ret = dmc_array(context, heap, text, values, nelems, &constant_values); + + if (ret != retOk) { + return ret; + } + if (constant_values) { + *constant = 1; + return retOk; + } + DMC_PUSH(*text, matchPushC); + DMC_PUSH(*text, dmc_private_copy(context, m->keys)); + if (++context->stack_used > context->stack_need) { + context->stack_need = context->stack_used; + } + DMC_PUSH(*text, matchMkFlatMap); + DMC_PUSH(*text, nelems); + context->stack_used -= nelems; + *constant = 0; + return retOk; + } else { + DECLARE_WSTACK(wstack); + Eterm *kv; + int c; + + ASSERT(is_hashmap(t)); + + hashmap_iterator_init(&wstack, t, 1); + constant_values = 1; + nelems = hashmap_size(t); + + while ((kv=hashmap_iterator_prev(&wstack)) != NULL) { + if ((ret = dmc_expr(context, heap, text, CDR(kv), &c)) != retOk) { + DESTROY_WSTACK(wstack); + return ret; + } + if (!c) + constant_values = 0; + } + + if (constant_values) { + *constant = 1; + DESTROY_WSTACK(wstack); + return retOk; + } + + *constant = 0; + + hashmap_iterator_init(&wstack, t, 1); + + while ((kv=hashmap_iterator_prev(&wstack)) != NULL) { + /* push key */ + if ((ret = dmc_expr(context, heap, text, CAR(kv), &c)) != retOk) { + DESTROY_WSTACK(wstack); + return ret; + } + if (c) { + do_emit_constant(context, text, CAR(kv)); + } + /* push value */ + if ((ret = dmc_expr(context, heap, text, CDR(kv), &c)) != retOk) { + DESTROY_WSTACK(wstack); + return ret; + } + if (c) { + do_emit_constant(context, text, CDR(kv)); + } + } + DMC_PUSH(*text, matchMkHashMap); + DMC_PUSH(*text, nelems); + context->stack_used -= nelems; + DESTROY_WSTACK(wstack); + return retOk; + } +} + static DMCRet dmc_whole_expression(DMCContext *context, DMCHeap *heap, DMC_STACK_TYPE(UWord) *text, @@ -3620,24 +3762,8 @@ static void dmc_add_pushv_variant(DMCContext *context, DMCHeap *heap, MatchOps instr = matchPushV; ASSERT(n < heap->vars_used && v->is_bound); - if (context->is_guard) { - #if HALFWORD_HEAP - if (!v->first_guard_label) { - v->first_guard_label = DMC_STACK_NUM(*text); - ASSERT(v->first_guard_label); - instr = matchPushVGuard; /* may be changed to PushVResult below */ - } - #endif - } - else { /* body */ - #if HALFWORD_HEAP - if (v->first_guard_label) { - /* Avoid double-copy, copy to result heap at first encounter in guard */ - DMC_POKE(*text, v->first_guard_label, matchPushVResult); - v->is_in_body = 1; - } - #endif - if (!v->is_in_body) { + if (!context->is_guard) { + if(!v->is_in_body) { instr = matchPushVResult; v->is_in_body = 1; } @@ -4553,7 +4679,7 @@ static DMCRet dmc_fun(DMCContext *context, DMC_PUSH(*text, matchCall3); break; default: - erl_exit(1,"ets:match() internal error, " + erts_exit(ERTS_ERROR_EXIT,"ets:match() internal error, " "guard with more than 3 arguments."); } DMC_PUSH(*text, (UWord) b->biff); @@ -4580,7 +4706,10 @@ static DMCRet dmc_expr(DMCContext *context, return ret; break; case TAG_PRIMARY_BOXED: - if (!BOXED_IS_TUPLE(t)) { + if (is_map(t)) { + return dmc_map(context, heap, text, t, constant); + } + if (!is_tuple(t)) { goto simple_term; } p = tuple_val(t); @@ -4830,7 +4959,7 @@ static Uint my_size_object(Eterm t) tmp == am_const) { sum += size_object(tuple_val(t)[2]); } else { - erl_exit(1,"Internal error, sizing unrecognized object in " + erts_exit(ERTS_ERROR_EXIT,"Internal error, sizing unrecognized object in " "(d)ets:match compilation."); } break; @@ -4855,7 +4984,7 @@ static Eterm my_copy_struct(Eterm t, Eterm **hp, ErlOffHeap* off_heap) *hp += 2; break; case TAG_PRIMARY_BOXED: - if (BOXED_IS_TUPLE(t)) { + if (is_tuple(t)) { if (arityval(*tuple_val(t)) == 1 && is_tuple(a = tuple_val(t)[1])) { Uint i,n; @@ -4875,7 +5004,7 @@ static Eterm my_copy_struct(Eterm t, Eterm **hp, ErlOffHeap* off_heap) sz = size_object(b); ret = copy_struct(b,sz,hp,off_heap); } else { - erl_exit(1, "Trying to constant-copy non constant expression " + erts_exit(ERTS_ERROR_EXIT, "Trying to constant-copy non constant expression " "0x%bex in (d)ets:match compilation.", t); } } else { @@ -5063,16 +5192,13 @@ Eterm db_match_dbterm(DbTableCommon* tb, Process* c_p, Binary* bprog, int all, DbTerm* obj, Eterm** hpp, Uint extra) { Uint32 dummy; - Eterm* base; Eterm res; if (tb->compress) { obj = db_alloc_tmp_uncompressed(tb, obj); - base = NULL; } - else base = HALFWORD_HEAP ? obj->tpl : NULL; - res = db_prog_match(c_p, bprog, make_tuple_rel(obj->tpl,base), base, NULL, 0, + res = db_prog_match(c_p, bprog, make_tuple(obj->tpl), NULL, 0, ERTS_PAM_COPY_RESULT|ERTS_PAM_CONTIGUOUS_TUPLE, &dummy); if (is_value(res) && hpp!=NULL) { @@ -5126,6 +5252,18 @@ void db_match_dis(Binary *bp) ++t; erts_printf("Tuple\t%beu\n", n); break; + case matchMap: + ++t; + n = *t; + ++t; + erts_printf("Map\t%beu\n", n); + break; + case matchKey: + ++t; + p = (Eterm) *t; + ++t; + erts_printf("Key\t%p (%T)\n", t, p); + break; case matchPushT: ++t; n = *t; @@ -5136,10 +5274,20 @@ void db_match_dis(Binary *bp) ++t; erts_printf("PushL\n"); break; + case matchPushM: + ++t; + n = *t; + ++t; + erts_printf("PushM\t%beu\n", n); + break; case matchPop: ++t; erts_printf("Pop\n"); break; + case matchSwap: + ++t; + erts_printf("Swap\n"); + break; case matchBind: ++t; n = *t; @@ -5171,7 +5319,7 @@ void db_match_dis(Binary *bp) first = 0; else erts_printf(", "); -#if defined(ARCH_64) && !HALFWORD_HEAP +#if defined(ARCH_64) erts_printf("0x%016bex", rt->data.ui[ri]); #else erts_printf("0x%08bex", rt->data.ui[ri]); @@ -5195,7 +5343,7 @@ void db_match_dis(Binary *bp) first = 0; else erts_printf(", "); -#if defined(ARCH_64) && !HALFWORD_HEAP +#if defined(ARCH_64) erts_printf("0x%016bex", *et); #else erts_printf("0x%08bex", *et); @@ -5252,6 +5400,18 @@ void db_match_dis(Binary *bp) ++t; erts_printf("MkTuple\t%beu\n", n); break; + case matchMkFlatMap: + ++t; + n = *t; + ++t; + erts_printf("MkFlatMap\t%beu\n", n); + break; + case matchMkHashMap: + ++t; + n = *t; + ++t; + erts_printf("MkHashMap\t%beu\n", n); + break; case matchOr: ++t; n = *t; @@ -5306,13 +5466,6 @@ void db_match_dis(Binary *bp) ++t; erts_printf("PushV\t%beu\n", n); break; - #if HALFWORD_HEAP - case matchPushVGuard: - n = (Uint) *++t; - ++t; - erts_printf("PushVGuard\t%beu\n", n); - break; - #endif case matchPushVResult: n = (Uint) *++t; ++t; diff --git a/erts/emulator/beam/erl_db_util.h b/erts/emulator/beam/erl_db_util.h index 328b19dfc9..911ed37aef 100644 --- a/erts/emulator/beam/erl_db_util.h +++ b/erts/emulator/beam/erl_db_util.h @@ -3,16 +3,17 @@ * * Copyright Ericsson AB 1998-2013. All Rights Reserved. * - * The contents of this file are subject to the Erlang Public License, - * Version 1.1, (the "License"); you may not use this file except in - * compliance with the License. You should have received a copy of the - * Erlang Public License along with this software. If not, it can be - * retrieved online at http://www.erlang.org/. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Software distributed under the License is distributed on an "AS IS" - * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See - * the License for the specific language governing rights and limitations - * under the License. + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. * * %CopyrightEnd% */ @@ -76,6 +77,9 @@ typedef struct db_term { union db_table; typedef union db_table DbTable; +#define DB_MUST_RESIZE 1 +#define DB_NEW_OBJECT 2 + /* Info about a database entry while it's being updated * (by update_counter or update_element) */ @@ -84,11 +88,8 @@ typedef struct { DbTerm* dbterm; void** bp; /* {Hash|Tree}DbTerm** */ Uint new_size; - int mustResize; + int flags; void* lck; -#if HALFWORD_HEAP - unsigned char* abs_vec; /* [i] true if dbterm->tpl[i] is absolute Eterm */ -#endif } DbUpdateHandle; @@ -165,6 +166,7 @@ typedef struct db_table_method DbTable* tb, /* [in out] */ Eterm continuation, Eterm* ret); + int (*db_take)(Process *, DbTable *, Eterm, Eterm *); int (*db_delete_all_objects)(Process* p, DbTable* db /* [in out] */ ); @@ -182,15 +184,14 @@ typedef struct db_table_method void *arg); void (*db_check_table)(DbTable* tb); - /* Lookup a dbterm for updating. Return false if not found. - */ - int (*db_lookup_dbterm)(DbTable*, Eterm key, - DbUpdateHandle* handle); /* [out] */ + /* Lookup a dbterm for updating. Return false if not found. */ + int (*db_lookup_dbterm)(Process *, DbTable *, Eterm key, Eterm obj, + DbUpdateHandle* handle); - /* Must be called for each db_lookup_dbterm that returned true, - ** even if dbterm was not updated. - */ - void (*db_finalize_dbterm)(DbUpdateHandle* handle); + /* Must be called for each db_lookup_dbterm that returned true, even if + ** dbterm was not updated. If the handle was of a new object and cret is + ** not DB_ERROR_NONE, the object is removed from the table. */ + void (*db_finalize_dbterm)(int cret, DbUpdateHandle* handle); } DbTableMethod; @@ -225,7 +226,10 @@ typedef struct db_table_common { DbTableMethod* meth; /* table methods */ erts_smp_atomic_t nitems; /* Total number of items in table */ erts_smp_atomic_t memory_size;/* Total memory size. NOTE: in bytes! */ - Uint megasec,sec,microsec; /* Last fixation time */ + struct { /* Last fixation time */ + ErtsMonotonicTime monotonic; + ErtsMonotonicTime offset; + } time; DbFixation* fixations; /* List of processes who have done safe_fixtable, "local" fixations not included. */ /* All 32-bit fields */ @@ -283,10 +287,10 @@ ERTS_GLB_INLINE Eterm db_copy_key(Process* p, DbTable* tb, DbTerm* obj) Eterm key = GETKEY(tb, obj->tpl); if IS_CONST(key) return key; else { - Uint size = size_object_rel(key, obj->tpl); + Uint size = size_object(key); Eterm* hp = HAlloc(p, size); - Eterm res = copy_struct_rel(key, size, &hp, &MSO(p), obj->tpl, NULL); - ASSERT(eq_rel(res,NULL,key,obj->tpl)); + Eterm res = copy_struct(key, size, &hp, &MSO(p)); + ASSERT(EQ(res,key)); return res; } } @@ -298,14 +302,14 @@ ERTS_GLB_INLINE Eterm db_copy_object_from_ets(DbTableCommon* tb, DbTerm* bp, return db_copy_from_comp(tb, bp, hpp, off_heap); } else { - return copy_shallow_rel(bp->tpl, bp->size, hpp, off_heap, bp->tpl); + return copy_shallow(bp->tpl, bp->size, hpp, off_heap); } } ERTS_GLB_INLINE int db_eq(DbTableCommon* tb, Eterm a, DbTerm* b) { if (!tb->compress) { - return eq_rel(a, NULL, make_tuple_rel(b->tpl,b->tpl), b->tpl); + return EQ(a, make_tuple(b->tpl)); } else { return db_eq_comp(tb, a, b); @@ -339,6 +343,7 @@ void* db_store_term(DbTableCommon *tb, DbTerm* old, Uint offset, Eterm obj); void* db_store_term_comp(DbTableCommon *tb, DbTerm* old, Uint offset, Eterm obj); Eterm db_copy_element_from_ets(DbTableCommon* tb, Process* p, DbTerm* obj, Uint pos, Eterm** hpp, Uint extra); +int db_has_map(Eterm obj); int db_has_variable(Eterm obj); int db_is_variable(Eterm obj); void db_do_update_element(DbUpdateHandle* handle, @@ -430,7 +435,7 @@ Binary *db_match_compile(Eterm *matchexpr, Eterm *guards, Eterm db_match_dbterm(DbTableCommon* tb, Process* c_p, Binary* bprog, int all, DbTerm* obj, Eterm** hpp, Uint extra); -Eterm db_prog_match(Process *p, Binary *prog, Eterm term, Eterm* base, +Eterm db_prog_match(Process *p, Binary *prog, Eterm term, Eterm *termp, int arity, enum erts_pam_run_flags in_flags, Uint32 *return_flags /* Zeroed on enter */); diff --git a/erts/emulator/beam/erl_debug.c b/erts/emulator/beam/erl_debug.c index 50bdc79506..2902c98864 100644 --- a/erts/emulator/beam/erl_debug.c +++ b/erts/emulator/beam/erl_debug.c @@ -3,16 +3,17 @@ * * Copyright Ericsson AB 1998-2013. All Rights Reserved. * - * The contents of this file are subject to the Erlang Public License, - * Version 1.1, (the "License"); you may not use this file except in - * compliance with the License. You should have received a copy of the - * Erlang Public License along with this software. If not, it can be - * retrieved online at http://www.erlang.org/. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Software distributed under the License is distributed on an "AS IS" - * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See - * the License for the specific language governing rights and limitations - * under the License. + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. * * %CopyrightEnd% */ @@ -188,6 +189,9 @@ pdisplay1(int to, void *to_arg, Process* p, Eterm obj) case BINARY_DEF: erts_print(to, to_arg, "#Bin"); break; + case MATCHSTATE_DEF: + erts_print(to, to_arg, "#Matchstate"); + break; default: erts_print(to, to_arg, "unknown object %x", obj); } @@ -251,14 +255,14 @@ void erts_check_stack(Process *p) Eterm *stack_end = p->htop; if (p->stop > stack_start) - erl_exit(1, + erts_exit(ERTS_ERROR_EXIT, "<%lu.%lu.%lu>: Stack underflow\n", internal_pid_channel_no(p->common.id), internal_pid_number(p->common.id), internal_pid_serial(p->common.id)); if (p->stop < stack_end) - erl_exit(1, + erts_exit(ERTS_ERROR_EXIT, "<%lu.%lu.%lu>: Stack overflow\n", internal_pid_channel_no(p->common.id), internal_pid_number(p->common.id), @@ -283,7 +287,7 @@ void erts_check_stack(Process *p) if (in_mbuf) continue; - erl_exit(1, + erts_exit(ERTS_ERROR_EXIT, "<%lu.%lu.%lu>: Wild stack pointer\n", internal_pid_channel_no(p->common.id), internal_pid_number(p->common.id), @@ -308,6 +312,8 @@ void erts_check_for_holes(Process* p) p->last_htop = HEAP_TOP(p); for (hf = MBUF(p); hf != 0; hf = hf->next) { + if (hf == p->heap_hfrag) + continue; if (hf == p->last_mbuf) { break; } @@ -368,7 +374,7 @@ void erts_check_memory(Process *p, Eterm *start, Eterm *end) #ifdef DEBUG if (hval == DEBUG_BAD_WORD) { print_untagged_memory(start, end); - erl_exit(1, "Uninitialized HAlloc'ed memory found @ 0x%0*lx!\n", + erts_exit(ERTS_ERROR_EXIT, "Uninitialized HAlloc'ed memory found @ 0x%0*lx!\n", PTR_SIZE,(unsigned long)(pos - 1)); } #endif @@ -381,7 +387,7 @@ void erts_check_memory(Process *p, Eterm *start, Eterm *end) if (verify_eterm(p,hval)) continue; - erl_exit(1, "Wild pointer found @ 0x%0*lx!\n", + erts_exit(ERTS_ERROR_EXIT, "Wild pointer found @ 0x%0*lx!\n", PTR_SIZE,(unsigned long)(pos - 1)); } } @@ -391,14 +397,14 @@ void verify_process(Process *p) #define VERIFY_AREA(name,ptr,sz) { \ int n = (sz); \ while (n--) if(!verify_eterm(p,*(ptr+n))) \ - erl_exit(1,"Wild pointer found in " name " of %T!\n",p->common.id); } + erts_exit(ERTS_ERROR_EXIT,"Wild pointer found in " name " of %T!\n",p->common.id); } #define VERIFY_ETERM(name,eterm) { \ if(!verify_eterm(p,eterm)) \ - erl_exit(1,"Wild pointer found in " name " of %T!\n",p->common.id); } + erts_exit(ERTS_ERROR_EXIT,"Wild pointer found in " name " of %T!\n",p->common.id); } - ErlMessage* mp = p->msg.first; + ErtsMessage* mp = p->msg.first; VERBOSE(DEBUG_MEMORY,("Verify process: %T...\n",p->common.id)); @@ -412,7 +418,7 @@ void verify_process(Process *p) erts_check_heap(p); if (p->dictionary) - VERIFY_AREA("dictionary",p->dictionary->data, p->dictionary->used); + VERIFY_AREA("dictionary", ERTS_PD_START(p->dictionary), ERTS_PD_SIZE(p->dictionary)); VERIFY_ETERM("seq trace token",p->seq_trace_token); VERIFY_ETERM("group leader",p->group_leader); VERIFY_ETERM("fvalue",p->fvalue); @@ -527,7 +533,7 @@ static void print_process_memory(Process *p) PTR_SIZE, "PCB", dashes, dashes, dashes, dashes); if (p->msg.first != NULL) { - ErlMessage* mp; + ErtsMessage* mp; erts_printf(" Message Queue:\n"); mp = p->msg.first; while (mp != NULL) { @@ -538,8 +544,8 @@ static void print_process_memory(Process *p) } if (p->dictionary != NULL) { - int n = p->dictionary->used; - Eterm *ptr = p->dictionary->data; + int n = ERTS_PD_SIZE(p->dictionary); + Eterm *ptr = ERTS_PD_START(p->dictionary); erts_printf(" Dictionary: "); while (n--) erts_printf("0x%0*lx ",PTR_SIZE,(unsigned long)ptr++); erts_printf("\n"); @@ -627,29 +633,4 @@ void print_memory_info(Process *p) } erts_printf("+-----------------%s-%s-%s-%s-+\n",dashes,dashes,dashes,dashes); } -#if !HEAP_ON_C_STACK && defined(DEBUG) -Eterm *erts_debug_allocate_tmp_heap(int size, Process *p) -{ - ErtsSchedulerData *sd = ((p == NULL) ? erts_get_scheduler_data() : ERTS_PROC_GET_SCHDATA(p)); - int offset = sd->num_tmp_heap_used; - - ASSERT(offset+size <= TMP_HEAP_SIZE); - return (sd->tmp_heap)+offset; -} -void erts_debug_use_tmp_heap(int size, Process *p) -{ - ErtsSchedulerData *sd = ((p == NULL) ? erts_get_scheduler_data() : ERTS_PROC_GET_SCHDATA(p)); - - sd->num_tmp_heap_used += size; - ASSERT(sd->num_tmp_heap_used <= TMP_HEAP_SIZE); -} -void erts_debug_unuse_tmp_heap(int size, Process *p) -{ - ErtsSchedulerData *sd = ((p == NULL) ? erts_get_scheduler_data() : ERTS_PROC_GET_SCHDATA(p)); - - sd->num_tmp_heap_used -= size; - ASSERT(sd->num_tmp_heap_used >= 0); -} -#endif #endif - diff --git a/erts/emulator/beam/erl_debug.h b/erts/emulator/beam/erl_debug.h index af51212281..029320691d 100644 --- a/erts/emulator/beam/erl_debug.h +++ b/erts/emulator/beam/erl_debug.h @@ -3,16 +3,17 @@ * * Copyright Ericsson AB 2004-2012. All Rights Reserved. * - * The contents of this file are subject to the Erlang Public License, - * Version 1.1, (the "License"); you may not use this file except in - * compliance with the License. You should have received a copy of the - * Erlang Public License along with this software. If not, it can be - * retrieved online at http://www.erlang.org/. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Software distributed under the License is distributed on an "AS IS" - * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See - * the License for the specific language governing rights and limitations - * under the License. + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. * * %CopyrightEnd% */ @@ -47,6 +48,7 @@ #define DEBUG_THREADS 0x0010 /* Thread-related stuff */ #define DEBUG_PROCESSES 0x0020 /* Process creation and removal */ #define DEBUG_MEMORY 0x0040 /* Display results of memory checks */ +#define DEBUG_SHCOPY 0x0080 /* Sharing-preserving copying of terms */ extern Uint32 verbose; @@ -91,10 +93,4 @@ extern void print_tagged_memory(Eterm *start, Eterm *end); extern void print_untagged_memory(Eterm *start, Eterm *end); extern void print_memory(Process *p); extern void print_memory_info(Process *p); -#if defined(DEBUG) && !HEAP_ON_C_STACK -extern Eterm *erts_debug_allocate_tmp_heap(int, Process *); -extern void erts_debug_use_tmp_heap(int, Process *); -extern void erts_debug_unuse_tmp_heap(int, Process *); -#endif - #endif /* _ERL_DEBUG_H_ */ diff --git a/erts/emulator/beam/erl_driver.h b/erts/emulator/beam/erl_driver.h index 3ecb379326..0234b9b0e4 100644 --- a/erts/emulator/beam/erl_driver.h +++ b/erts/emulator/beam/erl_driver.h @@ -3,16 +3,17 @@ * * Copyright Ericsson AB 1999-2014. All Rights Reserved. * - * The contents of this file are subject to the Erlang Public License, - * Version 1.1, (the "License"); you may not use this file except in - * compliance with the License. You should have received a copy of the - * Erlang Public License along with this software. If not, it can be - * retrieved online at http://www.erlang.org/. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Software distributed under the License is distributed on an "AS IS" - * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See - * the License for the specific language governing rights and limitations - * under the License. + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. * * %CopyrightEnd% */ @@ -36,52 +37,6 @@ # endif #endif -#ifdef SIZEOF_CHAR -# define SIZEOF_CHAR_SAVED__ SIZEOF_CHAR -# undef SIZEOF_CHAR -#endif -#ifdef SIZEOF_SHORT -# define SIZEOF_SHORT_SAVED__ SIZEOF_SHORT -# undef SIZEOF_SHORT -#endif -#ifdef SIZEOF_INT -# define SIZEOF_INT_SAVED__ SIZEOF_INT -# undef SIZEOF_INT -#endif -#ifdef SIZEOF_LONG -# define SIZEOF_LONG_SAVED__ SIZEOF_LONG -# undef SIZEOF_LONG -#endif -#ifdef SIZEOF_LONG_LONG -# define SIZEOF_LONG_LONG_SAVED__ SIZEOF_LONG_LONG -# undef SIZEOF_LONG_LONG -#endif -#ifdef HALFWORD_HEAP_EMULATOR -# define HALFWORD_HEAP_EMULATOR_SAVED__ HALFWORD_HEAP_EMULATOR -# undef HALFWORD_HEAP_EMULATOR -#endif -#include "erl_int_sizes_config.h" -#if defined(SIZEOF_CHAR_SAVED__) && SIZEOF_CHAR_SAVED__ != SIZEOF_CHAR -# error SIZEOF_CHAR mismatch -#endif -#if defined(SIZEOF_SHORT_SAVED__) && SIZEOF_SHORT_SAVED__ != SIZEOF_SHORT -# error SIZEOF_SHORT mismatch -#endif -#if defined(SIZEOF_INT_SAVED__) && SIZEOF_INT_SAVED__ != SIZEOF_INT -# error SIZEOF_INT mismatch -#endif -#if defined(SIZEOF_LONG_SAVED__) && SIZEOF_LONG_SAVED__ != SIZEOF_LONG -# error SIZEOF_LONG mismatch -#endif -#if defined(SIZEOF_LONG_LONG_SAVED__) && SIZEOF_LONG_LONG_SAVED__ != SIZEOF_LONG_LONG -# error SIZEOF_LONG_LONG mismatch -#endif - -/* This is OK to override by the NIF/driver implementor */ -#if defined(HALFWORD_HEAP_EMULATOR_SAVED__) && !defined(HALFWORD_HEAP_EMULATOR) -#define HALFWORD_HEAP_EMULATOR HALFWORD_HEAP_EMULATOR_SAVED__ -#endif - #include "erl_drv_nif.h" #include <stdlib.h> @@ -133,7 +88,7 @@ typedef struct { #define ERL_DRV_EXTENDED_MARKER (0xfeeeeeed) #define ERL_DRV_EXTENDED_MAJOR_VERSION 3 -#define ERL_DRV_EXTENDED_MINOR_VERSION 0 +#define ERL_DRV_EXTENDED_MINOR_VERSION 3 /* * The emulator will refuse to load a driver with a major version @@ -171,34 +126,19 @@ typedef struct { #define ERL_DRV_FLAG_USE_PORT_LOCKING (1 << 0) #define ERL_DRV_FLAG_SOFT_BUSY (1 << 1) #define ERL_DRV_FLAG_NO_BUSY_MSGQ (1 << 2) +#define ERL_DRV_FLAG_USE_INIT_ACK (1 << 3) /* * Integer types */ -#if defined(__WIN32__) && (SIZEOF_VOID_P == 8) -typedef unsigned __int64 ErlDrvTermData; -typedef unsigned __int64 ErlDrvUInt; -typedef signed __int64 ErlDrvSInt; -#else -typedef unsigned long ErlDrvTermData; -typedef unsigned long ErlDrvUInt; -typedef signed long ErlDrvSInt; -#endif -#if defined(__WIN32__) -typedef unsigned __int64 ErlDrvUInt64; -typedef __int64 ErlDrvSInt64; -#elif SIZEOF_LONG == 8 -typedef unsigned long ErlDrvUInt64; -typedef long ErlDrvSInt64; -#elif SIZEOF_LONG_LONG == 8 -typedef unsigned long long ErlDrvUInt64; -typedef long long ErlDrvSInt64; -#else -#error No 64-bit integer type -#endif +typedef ErlNapiUInt64 ErlDrvUInt64; +typedef ErlNapiSInt64 ErlDrvSInt64; +typedef ErlNapiUInt ErlDrvUInt; +typedef ErlNapiSInt ErlDrvSInt; +typedef ErlNapiUInt ErlDrvTermData; -#if defined(__WIN32__) +#if defined(__WIN32__) || defined(_WIN32) typedef ErlDrvUInt ErlDrvSizeT; typedef ErlDrvSInt ErlDrvSSizeT; #else @@ -249,6 +189,17 @@ typedef struct { unsigned long microsecs; } ErlDrvNowData; +typedef ErlDrvSInt64 ErlDrvTime; + +#define ERL_DRV_TIME_ERROR ((ErlDrvSInt64) ERTS_NAPI_TIME_ERROR__) + +typedef enum { + ERL_DRV_SEC = ERTS_NAPI_SEC__, + ERL_DRV_MSEC = ERTS_NAPI_MSEC__, + ERL_DRV_USEC = ERTS_NAPI_USEC__, + ERL_DRV_NSEC = ERTS_NAPI_NSEC__ +} ErlDrvTimeUnit; + /* * Error codes that can be return from driver. */ @@ -361,6 +312,9 @@ typedef struct erl_drv_entry { /* Called on behalf of driver_select when it is safe to release 'event'. A typical unix driver would call close(event) */ + void (*emergency_close)(ErlDrvData drv_data); + /* called when the port is closed abruptly. + specifically when erl_crash_dump is called. */ /* When adding entries here, dont forget to pad in obsolete/driver.h */ } ErlDrvEntry; @@ -382,22 +336,23 @@ typedef struct erl_drv_entry { #ifdef STATIC_ERLANG_DRIVER # define ERLANG_DRIVER_NAME(NAME) NAME ## _driver_init +# define ERL_DRIVER_EXPORT #else # define ERLANG_DRIVER_NAME(NAME) driver_init +# if defined(__GNUC__) && __GNUC__ >= 4 +# define ERL_DRIVER_EXPORT __attribute__ ((visibility("default"))) +# elif defined (__SUNPRO_C) && (__SUNPRO_C >= 0x550) +# define ERL_DRIVER_EXPORT __global +# else +# define ERL_DRIVER_EXPORT +# endif #endif -/* For windows dynamic drivers */ #ifndef ERL_DRIVER_TYPES_ONLY -#if defined(__WIN32__) -# define DRIVER_INIT(DRIVER_NAME) \ - __declspec(dllexport) ErlDrvEntry* ERLANG_DRIVER_NAME(DRIVER_NAME)(void); \ - __declspec(dllexport) ErlDrvEntry* ERLANG_DRIVER_NAME(DRIVER_NAME)(void) -#else -# define DRIVER_INIT(DRIVER_NAME) \ - ErlDrvEntry* ERLANG_DRIVER_NAME(DRIVER_NAME)(void); \ - ErlDrvEntry* ERLANG_DRIVER_NAME(DRIVER_NAME)(void) -#endif +#define DRIVER_INIT(DRIVER_NAME) \ + ERL_DRIVER_EXPORT ErlDrvEntry* ERLANG_DRIVER_NAME(DRIVER_NAME)(void); \ + ERL_DRIVER_EXPORT ErlDrvEntry* ERLANG_DRIVER_NAME(DRIVER_NAME)(void) #define ERL_DRV_BUSY_MSGQ_DISABLED (~((ErlDrvSizeT) 0)) #define ERL_DRV_BUSY_MSGQ_READ_ONLY ((ErlDrvSizeT) 0) @@ -681,8 +636,16 @@ EXTERN long driver_async(ErlDrvPort ix, EXTERN int driver_lock_driver(ErlDrvPort ix); /* Get the current 'now' timestamp (analogue to erlang:now()) */ -EXTERN int driver_get_now(ErlDrvNowData *now); +EXTERN int driver_get_now(ErlDrvNowData *now) ERL_DRV_DEPRECATED_FUNC; +/* Erlang Monotonic Time */ +EXTERN ErlDrvTime erl_drv_monotonic_time(ErlDrvTimeUnit time_unit); +/* Time offset between Erlang Monotonic Time and Erlang System Time */ +EXTERN ErlDrvTime erl_drv_time_offset(ErlDrvTimeUnit time_unit); +/* Time unit conversion */ +EXTERN ErlDrvTime erl_drv_convert_time_unit(ErlDrvTime val, + ErlDrvTimeUnit from, + ErlDrvTimeUnit to); /* These were removed from the ANSI version, now they're back. */ @@ -692,18 +655,14 @@ EXTERN int driver_dl_close(void *); EXTERN char *driver_dl_error(void); /* environment */ -EXTERN int erl_drv_putenv(char *key, char *value); -EXTERN int erl_drv_getenv(char *key, char *value, size_t *value_size); - -#ifdef __OSE__ -typedef ErlDrvUInt ErlDrvOseEventId; -EXTERN union SIGNAL *erl_drv_ose_get_signal(ErlDrvEvent ev); -EXTERN ErlDrvEvent erl_drv_ose_event_alloc(SIGSELECT sig, ErlDrvOseEventId handle, - ErlDrvOseEventId (*resolve_signal)(union SIGNAL *sig), void *extra); -EXTERN void erl_drv_ose_event_free(ErlDrvEvent ev); -EXTERN void erl_drv_ose_event_fetch(ErlDrvEvent ev, SIGSELECT *sig, - ErlDrvOseEventId *handle, void **extra); -#endif +EXTERN int erl_drv_putenv(const char *key, char *value); +EXTERN int erl_drv_getenv(const char *key, char *value, size_t *value_size); + +/* spawn start init ack */ +EXTERN void erl_drv_init_ack(ErlDrvPort ix, ErlDrvData res); + +/* set the pid seen in port_info */ +EXTERN void erl_drv_set_os_pid(ErlDrvPort ix, ErlDrvSInt pid); #endif /* !ERL_DRIVER_TYPES_ONLY */ diff --git a/erts/emulator/beam/erl_drv_nif.h b/erts/emulator/beam/erl_drv_nif.h index 3f829ea7ea..f6b946ae82 100644 --- a/erts/emulator/beam/erl_drv_nif.h +++ b/erts/emulator/beam/erl_drv_nif.h @@ -3,16 +3,17 @@ * * Copyright Ericsson AB 2010. All Rights Reserved. * - * The contents of this file are subject to the Erlang Public License, - * Version 1.1, (the "License"); you may not use this file except in - * compliance with the License. You should have received a copy of the - * Erlang Public License along with this software. If not, it can be - * retrieved online at http://www.erlang.org/. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Software distributed under the License is distributed on an "AS IS" - * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See - * the License for the specific language governing rights and limitations - * under the License. + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. * * %CopyrightEnd% */ @@ -35,6 +36,7 @@ typedef struct { int scheduler_threads; int nif_major_version; int nif_minor_version; + int dirty_scheduler_support; } ErlDrvSysInfo; typedef struct { @@ -48,6 +50,90 @@ typedef enum { } ErlDrvDirtyJobFlags; #endif +#ifdef SIZEOF_CHAR +# define SIZEOF_CHAR_SAVED__ SIZEOF_CHAR +# undef SIZEOF_CHAR +#endif +#ifdef SIZEOF_SHORT +# define SIZEOF_SHORT_SAVED__ SIZEOF_SHORT +# undef SIZEOF_SHORT +#endif +#ifdef SIZEOF_INT +# define SIZEOF_INT_SAVED__ SIZEOF_INT +# undef SIZEOF_INT +#endif +#ifdef SIZEOF_LONG +# define SIZEOF_LONG_SAVED__ SIZEOF_LONG +# undef SIZEOF_LONG +#endif +#ifdef SIZEOF_LONG_LONG +# define SIZEOF_LONG_LONG_SAVED__ SIZEOF_LONG_LONG +# undef SIZEOF_LONG_LONG +#endif +#ifdef HALFWORD_HEAP_EMULATOR +# define HALFWORD_HEAP_EMULATOR_SAVED__ HALFWORD_HEAP_EMULATOR +# undef HALFWORD_HEAP_EMULATOR +#endif +#include "erl_int_sizes_config.h" +#if defined(SIZEOF_CHAR_SAVED__) && SIZEOF_CHAR_SAVED__ != SIZEOF_CHAR +# error SIZEOF_CHAR mismatch +#endif +#if defined(SIZEOF_SHORT_SAVED__) && SIZEOF_SHORT_SAVED__ != SIZEOF_SHORT +# error SIZEOF_SHORT mismatch +#endif +#if defined(SIZEOF_INT_SAVED__) && SIZEOF_INT_SAVED__ != SIZEOF_INT +# error SIZEOF_INT mismatch +#endif +#if defined(SIZEOF_LONG_SAVED__) && SIZEOF_LONG_SAVED__ != SIZEOF_LONG +# error SIZEOF_LONG mismatch +#endif +#if defined(SIZEOF_LONG_LONG_SAVED__) && SIZEOF_LONG_LONG_SAVED__ != SIZEOF_LONG_LONG +# error SIZEOF_LONG_LONG mismatch +#endif + +#if !defined(__GNUC__) && (defined(__WIN32__) || defined(_WIN32) || defined(_WIN32_)) +typedef unsigned __int64 ErlNapiUInt64; +typedef signed __int64 ErlNapiSInt64; +#define ERL_NAPI_SINT64_MAX__ 9223372036854775807i64 +#define ERL_NAPI_SINT64_MIN__ (-ERL_NAPI_SINT64_MAX__ - 1i64) +#elif SIZEOF_LONG == 8 +typedef unsigned long ErlNapiUInt64; +typedef signed long ErlNapiSInt64; +#define ERL_NAPI_SINT64_MAX__ 9223372036854775807L +#define ERL_NAPI_SINT64_MIN__ (-ERL_NAPI_SINT64_MAX__ - 1L) +#elif SIZEOF_LONG_LONG == 8 +typedef unsigned long long ErlNapiUInt64; +typedef signed long long ErlNapiSInt64; +#define ERL_NAPI_SINT64_MAX__ 9223372036854775807LL +#define ERL_NAPI_SINT64_MIN__ (-ERL_NAPI_SINT64_MAX__ - 1LL) +#else +# error No 64-bit integer type +#endif + +#if SIZEOF_VOID_P == 8 +typedef ErlNapiUInt64 ErlNapiUInt; +typedef ErlNapiSInt64 ErlNapiSInt; +#elif SIZEOF_VOID_P == 4 +# if SIZEOF_LONG == SIZEOF_VOID_P +typedef unsigned long ErlNapiUInt; +typedef signed long ErlNapiSInt; +# elif SIZEOF_INT == SIZEOF_VOID_P +typedef unsigned int ErlNapiUInt; +typedef signed int ErlNapiSInt; +# else +# error No 32-bit integer type +# endif +#else +# error Not support arch +#endif + +#define ERTS_NAPI_TIME_ERROR__ ERL_NAPI_SINT64_MIN__ + +#define ERTS_NAPI_SEC__ 0 +#define ERTS_NAPI_MSEC__ 1 +#define ERTS_NAPI_USEC__ 2 +#define ERTS_NAPI_NSEC__ 3 + #endif /* __ERL_DRV_NIF_H__ */ diff --git a/erts/emulator/beam/erl_drv_thread.c b/erts/emulator/beam/erl_drv_thread.c index 147249f751..184f8e8931 100644 --- a/erts/emulator/beam/erl_drv_thread.c +++ b/erts/emulator/beam/erl_drv_thread.c @@ -3,16 +3,17 @@ * * Copyright Ericsson AB 2007-2011. All Rights Reserved. * - * The contents of this file are subject to the Erlang Public License, - * Version 1.1, (the "License"); you may not use this file except in - * compliance with the License. You should have received a copy of the - * Erlang Public License along with this software. If not, it can be - * retrieved online at http://www.erlang.org/. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Software distributed under the License is distributed on an "AS IS" - * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See - * the License for the specific language governing rights and limitations - * under the License. + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. * * %CopyrightEnd% */ @@ -42,7 +43,7 @@ fatal_error(int err, char *func) else estr = "Unknown error"; } - erl_exit(ERTS_ABORT_EXIT, "Fatal error in %s: %s [%d]\n", func, estr, err); + erts_exit(ERTS_ABORT_EXIT, "Fatal error in %s: %s [%d]\n", func, estr, err); } #define ERL_DRV_TSD_KEYS_INC 10 @@ -601,17 +602,16 @@ erl_drv_thread_create(char *name, #ifdef USE_THREADS int res; struct ErlDrvTid_ *dtid; - ethr_thr_opts ethr_opts; + ethr_thr_opts ethr_opts = ETHR_THR_OPTS_DEFAULT_INITER; ethr_thr_opts *use_opts; - ethr_thr_opts def_ethr_opts = ETHR_THR_OPTS_DEFAULT_INITER; - if (!opts) + if (!opts && !name) use_opts = NULL; else { - sys_memcpy((void *) ðr_opts, - (void *) &def_ethr_opts, - sizeof(ethr_thr_opts)); - ethr_opts.suggested_stack_size = opts->suggested_stack_size; + if(opts) + ethr_opts.suggested_stack_size = opts->suggested_stack_size; + + ethr_opts.name = name; use_opts = ðr_opts; } diff --git a/erts/emulator/beam/erl_fun.c b/erts/emulator/beam/erl_fun.c index 88947b5536..ed555aa92f 100644 --- a/erts/emulator/beam/erl_fun.c +++ b/erts/emulator/beam/erl_fun.c @@ -3,16 +3,17 @@ * * Copyright Ericsson AB 2000-2010. All Rights Reserved. * - * The contents of this file are subject to the Erlang Public License, - * Version 1.1, (the "License"); you may not use this file except in - * compliance with the License. You should have received a copy of the - * Erlang Public License along with this software. If not, it can be - * retrieved online at http://www.erlang.org/. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Software distributed under the License is distributed on an "AS IS" - * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See - * the License for the specific language governing rights and limitations - * under the License. + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. * * %CopyrightEnd% */ @@ -65,6 +66,9 @@ erts_init_fun_table(void) f.cmp = (HCMP_FUN) fun_cmp; f.alloc = (HALLOC_FUN) fun_alloc; f.free = (HFREE_FUN) fun_free; + f.meta_alloc = (HMALLOC_FUN) erts_alloc; + f.meta_free = (HMFREE_FUN) erts_free; + f.meta_print = (HMPRINT_FUN) erts_print; hash_init(ERTS_ALC_T_FUN_TABLE, &erts_fun_table, "fun_table", 16, f); } @@ -184,7 +188,7 @@ erts_erase_fun_entry(ErlFunEntry* fe) #endif { if (fe->address != unloaded_fun) - erl_exit(1, + erts_exit(ERTS_ERROR_EXIT, "Internal error: " "Invalid reference count found on #Fun<%T.%d.%d>: " " About to erase fun still referred by code.\n", diff --git a/erts/emulator/beam/erl_fun.h b/erts/emulator/beam/erl_fun.h index b673ef6b3c..0024b1ff71 100644 --- a/erts/emulator/beam/erl_fun.h +++ b/erts/emulator/beam/erl_fun.h @@ -3,16 +3,17 @@ * * Copyright Ericsson AB 2000-2012. All Rights Reserved. * - * The contents of this file are subject to the Erlang Public License, - * Version 1.1, (the "License"); you may not use this file except in - * compliance with the License. You should have received a copy of the - * Erlang Public License along with this software. If not, it can be - * retrieved online at http://www.erlang.org/. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Software distributed under the License is distributed on an "AS IS" - * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See - * the License for the specific language governing rights and limitations - * under the License. + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. * * %CopyrightEnd% */ diff --git a/erts/emulator/beam/erl_gc.c b/erts/emulator/beam/erl_gc.c index aa15d2cc57..0d125c5598 100644 --- a/erts/emulator/beam/erl_gc.c +++ b/erts/emulator/beam/erl_gc.c @@ -3,16 +3,17 @@ * * Copyright Ericsson AB 2002-2014. All Rights Reserved. * - * The contents of this file are subject to the Erlang Public License, - * Version 1.1, (the "License"); you may not use this file except in - * compliance with the License. You should have received a copy of the - * Erlang Public License along with this software. If not, it can be - * retrieved online at http://www.erlang.org/. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Software distributed under the License is distributed on an "AS IS" - * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See - * the License for the specific language governing rights and limitations - * under the License. + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. * * %CopyrightEnd% */ @@ -20,6 +21,8 @@ # include "config.h" #endif +#define ERL_WANT_GC_INTERNALS__ + #include "sys.h" #include "erl_vm.h" #include "global.h" @@ -37,12 +40,24 @@ #include "hipe_mode_switch.h" #endif #include "dtrace-wrapper.h" +#include "erl_bif_unique.h" #define ERTS_INACT_WR_PB_LEAVE_MUCH_LIMIT 1 #define ERTS_INACT_WR_PB_LEAVE_MUCH_PERCENTAGE 20 #define ERTS_INACT_WR_PB_LEAVE_LIMIT 10 #define ERTS_INACT_WR_PB_LEAVE_PERCENTAGE 10 +#if defined(DEBUG) || 0 +#define ERTS_GC_DEBUG +#else +#undef ERTS_GC_DEBUG +#endif +#ifdef ERTS_GC_DEBUG +# define ERTS_GC_ASSERT ASSERT +#else +# define ERTS_GC_ASSERT(B) ((void) 1) +#endif + /* * Returns number of elements in an array. */ @@ -55,7 +70,7 @@ erts_fprintf(stderr, "stop=%p\n", (p)->stop); \ erts_fprintf(stderr, "htop=%p\n", (p)->htop); \ erts_fprintf(stderr, "heap=%p\n", (p)->heap); \ - erl_exit(ERTS_ABORT_EXIT, "%s, line %d: %T: Overrun stack and heap\n", \ + erts_exit(ERTS_ABORT_EXIT, "%s, line %d: %T: Overrun stack and heap\n", \ __FILE__,__LINE__,(P)->common.id); \ } @@ -63,10 +78,10 @@ #define ErtsGcQuickSanityCheck(P) \ do { \ ASSERT((P)->heap < (P)->hend); \ - ASSERT((P)->heap_sz == (P)->hend - (P)->heap); \ + ASSERT((p)->abandoned_heap || (P)->heap_sz == (P)->hend - (P)->heap); \ ASSERT((P)->heap <= (P)->htop && (P)->htop <= (P)->hend); \ ASSERT((P)->heap <= (P)->stop && (P)->stop <= (P)->hend); \ - ASSERT((P)->heap <= (P)->high_water && (P)->high_water <= (P)->hend);\ + ASSERT((p)->abandoned_heap || ((P)->heap <= (P)->high_water && (P)->high_water <= (P)->hend)); \ OverRunCheck((P)); \ } while (0) #else @@ -94,19 +109,33 @@ typedef struct { static Uint setup_rootset(Process*, Eterm*, int, Rootset*); static void cleanup_rootset(Rootset *rootset); -static Uint combined_message_size(Process* p); static void remove_message_buffers(Process* p); -static int major_collection(Process* p, int need, Eterm* objv, int nobj, Uint *recl); -static int minor_collection(Process* p, int need, Eterm* objv, int nobj, Uint *recl); -static void do_minor(Process *p, Uint new_sz, Eterm* objv, int nobj); -static Eterm* sweep_rootset(Rootset *rootset, Eterm* htop, char* src, Uint src_size); -static Eterm* sweep_one_area(Eterm* n_hp, Eterm* n_htop, char* src, Uint src_size); -static Eterm* sweep_one_heap(Eterm* heap_ptr, Eterm* heap_end, Eterm* htop, - char* src, Uint src_size); -static Eterm* collect_heap_frags(Process* p, Eterm* heap, - Eterm* htop, Eterm* objv, int nobj); -static Uint adjust_after_fullsweep(Process *p, Uint size_before, - int need, Eterm *objv, int nobj); +static Eterm *full_sweep_heaps(Process *p, + int hibernate, + Eterm *n_heap, Eterm* n_htop, + char *oh, Uint oh_size, + Eterm *objv, int nobj); +static int garbage_collect(Process* p, ErlHeapFragment *live_hf_end, + int need, Eterm* objv, int nobj); +static int major_collection(Process* p, ErlHeapFragment *live_hf_end, + int need, Eterm* objv, int nobj, Uint *recl); +static int minor_collection(Process* p, ErlHeapFragment *live_hf_end, + int need, Eterm* objv, int nobj, Uint *recl); +static void do_minor(Process *p, ErlHeapFragment *live_hf_end, + char *mature, Uint mature_size, + Uint new_sz, Eterm* objv, int nobj); +static Eterm *sweep_new_heap(Eterm *n_hp, Eterm *n_htop, + char* old_heap, Uint old_heap_size); +static Eterm *sweep_heaps(Eterm *n_hp, Eterm *n_htop, + char* old_heap, Uint old_heap_size); +static Eterm* sweep_literal_area(Eterm* n_hp, Eterm* n_htop, + char* old_heap, Uint old_heap_size, + char* src, Uint src_size); +static Eterm* sweep_literals_to_old_heap(Eterm* heap_ptr, Eterm* heap_end, Eterm* htop, + char* src, Uint src_size); +static Eterm* collect_live_heap_frags(Process* p, ErlHeapFragment *live_hf_end, + Eterm* heap, Eterm* htop, Eterm* objv, int nobj); +static int adjust_after_fullsweep(Process *p, int need, Eterm *objv, int nobj); static void shrink_new_heap(Process *p, Uint new_sz, Eterm *objv, int nobj); static void grow_new_heap(Process *p, Uint new_sz, Eterm* objv, int nobj); static void sweep_off_heap(Process *p, int fullsweep); @@ -116,16 +145,16 @@ static void offset_rootset(Process *p, Sint offs, char* area, Uint area_size, Eterm* objv, int nobj); static void offset_off_heap(Process* p, Sint offs, char* area, Uint area_size); static void offset_mqueue(Process *p, Sint offs, char* area, Uint area_size); +static void move_msgq_to_heap(Process *p); static void init_gc_info(ErtsGCInfo *gcip); #ifdef HARDDEBUG static void disallow_heap_frag_ref_in_heap(Process* p); static void disallow_heap_frag_ref_in_old_heap(Process* p); -static void disallow_heap_frag_ref(Process* p, Eterm* n_htop, Eterm* objv, int nobj); #endif -#if defined(ARCH_64) && !HALFWORD_HEAP +#if defined(ARCH_64) # define MAX_HEAP_SIZES 154 #else # define MAX_HEAP_SIZES 59 @@ -144,26 +173,24 @@ typedef struct { erts_smp_atomic32_t refc; } ErtsGCInfoReq; -#if !HALFWORD_HEAP -ERTS_SCHED_PREF_QUICK_ALLOC_IMPL(gcireq, - ErtsGCInfoReq, - 5, - ERTS_ALC_T_GC_INFO_REQ) -#else -static ERTS_INLINE ErtsGCInfoReq * -gcireq_alloc(void) +static ERTS_INLINE int +gc_cost(Uint gc_moved_live_words, Uint resize_moved_words) { - return erts_alloc(ERTS_ALC_T_GC_INFO_REQ, - sizeof(ErtsGCInfoReq)); -} + Sint reds; -static ERTS_INLINE void -gcireq_free(ErtsGCInfoReq *ptr) -{ - erts_free(ERTS_ALC_T_GC_INFO_REQ, ptr); + reds = gc_moved_live_words/10; + reds += resize_moved_words/100; + if (reds < 1) + return 1; + if (reds > INT_MAX) + return INT_MAX; + return (int) reds; } -#endif +ERTS_SCHED_PREF_QUICK_ALLOC_IMPL(gcireq, + ErtsGCInfoReq, + 5, + ERTS_ALC_T_GC_INFO_REQ) /* * Initialize GC global data. */ @@ -173,15 +200,15 @@ erts_init_gc(void) int i = 0, ix; Sint max_heap_size = 0; - ASSERT(offsetof(ProcBin,thing_word) == offsetof(struct erl_off_heap_header,thing_word)); - ASSERT(offsetof(ProcBin,thing_word) == offsetof(ErlFunThing,thing_word)); - ASSERT(offsetof(ProcBin,thing_word) == offsetof(ExternalThing,header)); - ASSERT(offsetof(ProcBin,size) == offsetof(struct erl_off_heap_header,size)); - ASSERT(offsetof(ProcBin,size) == offsetof(ErlSubBin,size)); - ASSERT(offsetof(ProcBin,size) == offsetof(ErlHeapBin,size)); - ASSERT(offsetof(ProcBin,next) == offsetof(struct erl_off_heap_header,next)); - ASSERT(offsetof(ProcBin,next) == offsetof(ErlFunThing,next)); - ASSERT(offsetof(ProcBin,next) == offsetof(ExternalThing,next)); + ERTS_CT_ASSERT(offsetof(ProcBin,thing_word) == offsetof(struct erl_off_heap_header,thing_word)); + ERTS_CT_ASSERT(offsetof(ProcBin,thing_word) == offsetof(ErlFunThing,thing_word)); + ERTS_CT_ASSERT(offsetof(ProcBin,thing_word) == offsetof(ExternalThing,header)); + ERTS_CT_ASSERT(offsetof(ProcBin,size) == offsetof(struct erl_off_heap_header,size)); + ERTS_CT_ASSERT(offsetof(ProcBin,size) == offsetof(ErlSubBin,size)); + ERTS_CT_ASSERT(offsetof(ProcBin,size) == offsetof(ErlHeapBin,size)); + ERTS_CT_ASSERT(offsetof(ProcBin,next) == offsetof(struct erl_off_heap_header,next)); + ERTS_CT_ASSERT(offsetof(ProcBin,next) == offsetof(ErlFunThing,next)); + ERTS_CT_ASSERT(offsetof(ProcBin,next) == offsetof(ExternalThing,next)); erts_test_long_gc_sleep = 0; @@ -205,7 +232,7 @@ erts_init_gc(void) } - /* for 32 bit we want max_heap_size to be MAX(32bit) / 4 [words] (and halfword) + /* for 32 bit we want max_heap_size to be MAX(32bit) / 4 [words] * for 64 bit we want max_heap_size to be MAX(52bit) / 8 [words] */ @@ -229,10 +256,7 @@ erts_init_gc(void) init_gc_info(&esdp->gc_info); } -#if !HALFWORD_HEAP init_gcireq_alloc(); -#endif - } /* @@ -265,7 +289,7 @@ erts_next_heap_size(Uint size, Uint offset) low = mid + 1; } } - erl_exit(1, "no next heap size found: %beu, offset %beu\n", size, offset); + erts_exit(ERTS_ERROR_EXIT, "no next heap size found: %beu, offset %beu\n", size, offset); } return 0; } @@ -348,10 +372,19 @@ erts_offset_off_heap(ErlOffHeap *ohp, Sint offs, Eterm* low, Eterm* high) #undef ptr_within Eterm -erts_gc_after_bif_call(Process* p, Eterm result, Eterm* regs, Uint arity) +erts_gc_after_bif_call_lhf(Process* p, ErlHeapFragment *live_hf_end, + Eterm result, Eterm* regs, Uint arity) { int cost; + if (p->flags & F_HIBERNATE_SCHED) { + /* + * We just hibernated. We do *not* want to mess + * up the hibernation by an ordinary GC... + */ + return result; + } + if (is_non_value(result)) { if (p->freason == TRAP) { #if HIPE @@ -359,21 +392,28 @@ erts_gc_after_bif_call(Process* p, Eterm result, Eterm* regs, Uint arity) regs = ERTS_PROC_GET_SCHDATA(p)->x_reg_array; } #endif - cost = erts_garbage_collect(p, 0, regs, p->arity); + cost = garbage_collect(p, live_hf_end, 0, regs, p->arity); } else { - cost = erts_garbage_collect(p, 0, regs, arity); + cost = garbage_collect(p, live_hf_end, 0, regs, arity); } } else { Eterm val[1]; val[0] = result; - cost = erts_garbage_collect(p, 0, val, 1); + cost = garbage_collect(p, live_hf_end, 0, val, 1); result = val[0]; } BUMP_REDS(p, cost); return result; } +Eterm +erts_gc_after_bif_call(Process* p, Eterm result, Eterm* regs, Uint arity) +{ + return erts_gc_after_bif_call_lhf(p, ERTS_INVALID_HFRAG_PTR, + result, regs, arity); +} + static ERTS_INLINE void reset_active_writer(Process *p) { struct erl_off_heap_header* ptr; @@ -387,6 +427,139 @@ static ERTS_INLINE void reset_active_writer(Process *p) } } +#define ERTS_DELAY_GC_EXTRA_FREE 40 +#define ERTS_ABANDON_HEAP_COST 10 + +static int +delay_garbage_collection(Process *p, ErlHeapFragment *live_hf_end, int need) +{ + ErlHeapFragment *hfrag; + Eterm *orig_heap, *orig_hend, *orig_htop, *orig_stop; + Eterm *stop, *hend; + Uint hsz, ssz; + int reds_left; + + ERTS_HOLE_CHECK(p); + + if ((p->flags & F_DISABLE_GC) + && p->live_hf_end == ERTS_INVALID_HFRAG_PTR) { + /* + * A BIF yielded with disabled GC. Remember + * heap fragments created by the BIF until we + * do next GC. + */ + p->live_hf_end = live_hf_end; + } + + if (need == 0) + return 1; + + /* + * Satisfy need in a heap fragment... + */ + ASSERT(need > 0); + + orig_heap = p->heap; + orig_hend = p->hend; + orig_htop = p->htop; + orig_stop = p->stop; + + ssz = orig_hend - orig_stop; + hsz = ssz + need + ERTS_DELAY_GC_EXTRA_FREE; + + hfrag = new_message_buffer(hsz); + hfrag->next = p->mbuf; + p->mbuf = hfrag; + p->mbuf_sz += hsz; + p->heap = p->htop = &hfrag->mem[0]; + p->hend = hend = &hfrag->mem[hsz]; + p->stop = stop = hend - ssz; + sys_memcpy((void *) stop, (void *) orig_stop, ssz * sizeof(Eterm)); + + if (p->abandoned_heap) { + /* Active heap already in a fragment; adjust it... */ + ErlHeapFragment *hfrag = ((ErlHeapFragment *) + (((char *) orig_heap) + - offsetof(ErlHeapFragment, mem))); + Uint unused = orig_hend - orig_htop; + ASSERT(hfrag->used_size == hfrag->alloc_size); + ASSERT(hfrag->used_size >= unused); + hfrag->used_size -= unused; + p->mbuf_sz -= unused; + } + else { + /* Do not leave a hole in the abandoned heap... */ + if (orig_htop < orig_hend) { + *orig_htop = make_pos_bignum_header(orig_hend-orig_htop-1); + if (orig_htop + 1 < orig_hend) { + orig_hend[-1] = (Uint) (orig_htop - orig_heap); + p->flags |= F_ABANDONED_HEAP_USE; + } + } + p->abandoned_heap = orig_heap; + } + +#ifdef CHECK_FOR_HOLES + p->last_htop = p->htop; + p->heap_hfrag = hfrag; +#endif + + /* Make sure that we do a proper GC as soon as possible... */ + p->flags |= F_FORCE_GC; + reds_left = ERTS_BIF_REDS_LEFT(p); + if (reds_left > ERTS_ABANDON_HEAP_COST) { + int vreds = reds_left - ERTS_ABANDON_HEAP_COST; + ERTS_VBUMP_REDS(p, vreds); + } + return ERTS_ABANDON_HEAP_COST; +} + +static ERTS_FORCE_INLINE Uint +young_gen_usage(Process *p) +{ + Uint hsz; + Eterm *aheap; + + hsz = p->mbuf_sz; + + if (p->flags & F_ON_HEAP_MSGQ) { + ErtsMessage *mp; + for (mp = p->msg.first; mp; mp = mp->next) + if (mp->data.attached) + hsz += erts_msg_attached_data_size(mp); + } + + aheap = p->abandoned_heap; + if (!aheap) + hsz += p->htop - p->heap; + else { + /* used in orig heap */ + if (p->flags & F_ABANDONED_HEAP_USE) + hsz += aheap[p->heap_sz-1]; + else + hsz += p->heap_sz; + /* Remove unused part in latest fragment */ + hsz -= p->hend - p->htop; + } + return hsz; +} + +#define ERTS_GET_ORIG_HEAP(Proc, Heap, HTop) \ + do { \ + Eterm *aheap__ = (Proc)->abandoned_heap; \ + if (!aheap__) { \ + (Heap) = (Proc)->heap; \ + (HTop) = (Proc)->htop; \ + } \ + else { \ + (Heap) = aheap__; \ + if ((Proc)->flags & F_ABANDONED_HEAP_USE) \ + (HTop) = aheap__ + aheap__[(Proc)->heap_sz-1]; \ + else \ + (HTop) = aheap__ + (Proc)->heap_sz; \ + } \ + } while (0) + /* * Garbage collect a process. * @@ -395,21 +568,28 @@ static ERTS_INLINE void reset_active_writer(Process *p) * objv: Array of terms to add to rootset; that is to preserve. * nobj: Number of objects in objv. */ -int -erts_garbage_collect(Process* p, int need, Eterm* objv, int nobj) +static int +garbage_collect(Process* p, ErlHeapFragment *live_hf_end, + int need, Eterm* objv, int nobj) { Uint reclaimed_now = 0; - int done = 0; - Uint ms1, s1, us1; + int reds; + ErtsMonotonicTime start_time = 0; /* Shut up faulty warning... */ ErtsSchedulerData *esdp; + ERTS_MSACC_PUSH_STATE_M(); #ifdef USE_VM_PROBES DTRACE_CHARBUF(pidbuf, DTRACE_TERM_BUF_SIZE); #endif - if (p->flags & F_DISABLE_GC) { - ASSERT(need == 0); - return 1; - } + if (p->flags & (F_DISABLE_GC|F_DELAY_GC)) + return delay_garbage_collection(p, live_hf_end, need); + + if (p->abandoned_heap) + live_hf_end = ERTS_INVALID_HFRAG_PTR; + else if (p->live_hf_end != ERTS_INVALID_HFRAG_PTR) + live_hf_end = p->live_hf_end; + + ERTS_MSACC_SET_STATE_CACHED_M(ERTS_MSACC_STATE_GC); esdp = erts_get_scheduler_data(); @@ -418,16 +598,13 @@ erts_garbage_collect(Process* p, int need, Eterm* objv, int nobj) } erts_smp_atomic32_read_bor_nob(&p->state, ERTS_PSFLG_GC); - if (erts_system_monitor_long_gc != 0) { - get_now(&ms1, &s1, &us1); - } + if (erts_system_monitor_long_gc != 0) + start_time = erts_get_monotonic_time(esdp); ERTS_CHK_OFFHEAP(p); ErtsGcQuickSanityCheck(p); - if (GEN_GCS(p) >= MAX_GEN_GCS(p)) { - FLAGS(p) |= F_NEED_FULLSWEEP; - } + #ifdef USE_VM_PROBES *pidbuf = '\0'; if (DTRACE_ENABLED(gc_major_start) @@ -440,17 +617,23 @@ erts_garbage_collect(Process* p, int need, Eterm* objv, int nobj) /* * Test which type of GC to do. */ - while (!done) { - if ((FLAGS(p) & F_NEED_FULLSWEEP) != 0) { - DTRACE2(gc_major_start, pidbuf, need); - done = major_collection(p, need, objv, nobj, &reclaimed_now); - DTRACE2(gc_major_end, pidbuf, reclaimed_now); - } else { - DTRACE2(gc_minor_start, pidbuf, need); - done = minor_collection(p, need, objv, nobj, &reclaimed_now); - DTRACE2(gc_minor_end, pidbuf, reclaimed_now); - } + + if (GEN_GCS(p) < MAX_GEN_GCS(p) && !(FLAGS(p) & F_NEED_FULLSWEEP)) { + DTRACE2(gc_minor_start, pidbuf, need); + reds = minor_collection(p, live_hf_end, need, objv, nobj, &reclaimed_now); + DTRACE2(gc_minor_end, pidbuf, reclaimed_now); + if (reds < 0) + goto do_major_collection; + } + else { + do_major_collection: + ERTS_MSACC_SET_STATE_CACHED_M_X(ERTS_MSACC_STATE_GC_FULL); + DTRACE2(gc_major_start, pidbuf, need); + reds = major_collection(p, live_hf_end, need, objv, nobj, &reclaimed_now); + DTRACE2(gc_major_end, pidbuf, reclaimed_now); + ERTS_MSACC_SET_STATE_CACHED_M_X(ERTS_MSACC_STATE_GC); } + reset_active_writer(p); /* @@ -468,16 +651,14 @@ erts_garbage_collect(Process* p, int need, Eterm* objv, int nobj) } if (erts_system_monitor_long_gc != 0) { - Uint ms2, s2, us2; - Sint t; + ErtsMonotonicTime end_time; + Uint gc_time; if (erts_test_long_gc_sleep) while (0 != erts_milli_sleep(erts_test_long_gc_sleep)); - get_now(&ms2, &s2, &us2); - t = ms2 - ms1; - t = t*1000000 + s2 - s1; - t = t*1000 + ((Sint) (us2 - us1))/1000; - if (t > 0 && (Uint)t > erts_system_monitor_long_gc) { - monitor_long_gc(p, t); + end_time = erts_get_monotonic_time(esdp); + gc_time = (Uint) ERTS_MONOTONIC_TO_MSEC(end_time - start_time); + if (gc_time && gc_time > erts_system_monitor_long_gc) { + monitor_long_gc(p, gc_time); } } if (erts_system_monitor_large_heap != 0) { @@ -491,6 +672,9 @@ erts_garbage_collect(Process* p, int need, Eterm* objv, int nobj) esdp->gc_info.reclaimed += reclaimed_now; FLAGS(p) &= ~F_FORCE_GC; + p->live_hf_end = ERTS_INVALID_HFRAG_PTR; + + ERTS_MSACC_POP_STATE_M(); #ifdef CHECK_FOR_HOLES /* @@ -512,15 +696,20 @@ erts_garbage_collect(Process* p, int need, Eterm* objv, int nobj) p->last_old_htop = p->old_htop; #endif - /* FIXME: This function should really return an Sint, i.e., a possibly - 64 bit wide signed integer, but that requires updating all the code - that calls it. For now, we just return INT_MAX if the result is too - large for an int. */ - { - Sint result = (HEAP_TOP(p) - HEAP_START(p)) / 10; - if (result >= INT_MAX) return INT_MAX; - else return (int) result; - } + return reds; +} + +int +erts_garbage_collect_nobump(Process* p, int need, Eterm* objv, int nobj) +{ + return garbage_collect(p, ERTS_INVALID_HFRAG_PTR, need, objv, nobj); +} + +void +erts_garbage_collect(Process* p, int need, Eterm* objv, int nobj) +{ + int reds = garbage_collect(p, ERTS_INVALID_HFRAG_PTR, need, objv, nobj); + BUMP_REDS(p, reds); } /* @@ -533,13 +722,11 @@ erts_garbage_collect_hibernate(Process* p) Uint heap_size; Eterm* heap; Eterm* htop; - Rootset rootset; - char* src; - Uint src_size; Uint actual_size; char* area; Uint area_size; Sint offs; + int reds; if (p->flags & F_DISABLE_GC) ERTS_INTERNAL_ERROR("GC disabled"); @@ -549,8 +736,6 @@ erts_garbage_collect_hibernate(Process* p) */ erts_smp_atomic32_read_bor_nob(&p->state, ERTS_PSFLG_GC); ErtsGcQuickSanityCheck(p); - ASSERT(p->mbuf_sz == 0); - ASSERT(p->mbuf == 0); ASSERT(p->stop == p->hend); /* Stack must be empty. */ /* @@ -558,46 +743,25 @@ erts_garbage_collect_hibernate(Process* p) */ - heap_size = p->heap_sz + (p->old_htop - p->old_heap); + heap_size = p->heap_sz + (p->old_htop - p->old_heap) + p->mbuf_sz; heap = (Eterm*) ERTS_HEAP_ALLOC(ERTS_ALC_T_TMP_HEAP, sizeof(Eterm)*heap_size); htop = heap; - (void) setup_rootset(p, p->arg_reg, p->arity, &rootset); -#if HIPE - hipe_empty_nstack(p); -#endif - - src = (char *) p->heap; - src_size = (char *) p->htop - src; - htop = sweep_rootset(&rootset, htop, src, src_size); - htop = sweep_one_area(heap, htop, src, src_size); - - if (p->old_heap) { - src = (char *) p->old_heap; - src_size = (char *) p->old_htop - src; - htop = sweep_rootset(&rootset, htop, src, src_size); - htop = sweep_one_area(heap, htop, src, src_size); - } - - cleanup_rootset(&rootset); + htop = full_sweep_heaps(p, + 1, + heap, + htop, + (char *) p->old_heap, + (char *) p->old_htop - (char *) p->old_heap, + p->arg_reg, + p->arity); - if (MSO(p).first) { - sweep_off_heap(p, 1); - } - - /* - * Update all pointers. - */ ERTS_HEAP_FREE(ERTS_ALC_T_HEAP, - (void*)HEAP_START(p), - HEAP_SIZE(p) * sizeof(Eterm)); - if (p->old_heap) { - ERTS_HEAP_FREE(ERTS_ALC_T_OLD_HEAP, - (void*)p->old_heap, - (p->old_hend - p->old_heap) * sizeof(Eterm)); - p->old_heap = p->old_htop = p->old_hend = 0; - } + (p->abandoned_heap + ? p->abandoned_heap + : p->heap), + p->heap_sz * sizeof(Eterm)); p->heap = heap; p->high_water = htop; @@ -612,6 +776,7 @@ erts_garbage_collect_hibernate(Process* p) } FLAGS(p) &= ~F_FORCE_GC; + p->live_hf_end = ERTS_INVALID_HFRAG_PTR; /* * Move the heap to its final destination. @@ -631,6 +796,8 @@ erts_garbage_collect_hibernate(Process* p) sys_memcpy((void *) heap, (void *) p->heap, actual_size*sizeof(Eterm)); ERTS_HEAP_FREE(ERTS_ALC_T_TMP_HEAP, p->heap, p->heap_sz*sizeof(Eterm)); + remove_message_buffers(p); + p->stop = p->hend = heap + heap_size; offs = heap - p->heap; @@ -659,15 +826,18 @@ erts_garbage_collect_hibernate(Process* p) ErtsGcQuickSanityCheck(p); erts_smp_atomic32_read_band_nob(&p->state, ~ERTS_PSFLG_GC); + + reds = gc_cost(actual_size, actual_size); + BUMP_REDS(p, reds); } void erts_garbage_collect_literals(Process* p, Eterm* literals, - Uint lit_size, + Uint byte_lit_size, struct erl_off_heap_header* oh) { - Uint byte_lit_size = sizeof(Eterm)*lit_size; + Uint lit_size = byte_lit_size / sizeof(Eterm); Uint old_heap_size; Eterm* temp_lit; Sint offs; @@ -677,7 +847,7 @@ erts_garbage_collect_literals(Process* p, Eterm* literals, Uint area_size; Eterm* old_htop; Uint n; - struct erl_off_heap_header** prev; + struct erl_off_heap_header** prev = NULL; if (p->flags & F_DISABLE_GC) return; @@ -743,7 +913,7 @@ erts_garbage_collect_literals(Process* p, Eterm* literals, if (IS_MOVED_BOXED(val)) { ASSERT(is_boxed(val)); *g_ptr++ = val; - } else if (in_area(ptr, area, area_size)) { + } else if (ErtsInArea(ptr, area, area_size)) { MOVE_BOXED(ptr,val,old_htop,g_ptr++); } else { g_ptr++; @@ -754,7 +924,7 @@ erts_garbage_collect_literals(Process* p, Eterm* literals, val = *ptr; if (IS_MOVED_CONS(val)) { /* Moved */ *g_ptr++ = ptr[1]; - } else if (in_area(ptr, area, area_size)) { + } else if (ErtsInArea(ptr, area, area_size)) { MOVE_CONS(ptr,val,old_htop,g_ptr++); } else { g_ptr++; @@ -774,8 +944,10 @@ erts_garbage_collect_literals(Process* p, Eterm* literals, * Now we'll have to go through all heaps updating all other references. */ - old_htop = sweep_one_heap(p->heap, p->htop, old_htop, area, area_size); - old_htop = sweep_one_area(p->old_heap, old_htop, area, area_size); + old_htop = sweep_literals_to_old_heap(p->heap, p->htop, old_htop, area, area_size); + old_htop = sweep_literal_area(p->old_heap, old_htop, + (char *) p->old_heap, sizeof(Eterm)*old_heap_size, + area, area_size); ASSERT(p->old_htop <= old_htop && old_htop <= p->old_hend); p->old_htop = old_htop; @@ -786,10 +958,10 @@ erts_garbage_collect_literals(Process* p, Eterm* literals, */ if (oh) { - prev = &MSO(p).first; - while (*prev) { - prev = &(*prev)->next; - } + prev = &MSO(p).first; + while (*prev) { + prev = &(*prev)->next; + } } /* @@ -818,6 +990,10 @@ erts_garbage_collect_literals(Process* p, Eterm* literals, oh = oh->next; } + if (prev) { + *prev = NULL; + } + /* * We no longer need this temporary area. */ @@ -830,15 +1006,18 @@ erts_garbage_collect_literals(Process* p, Eterm* literals, } static int -minor_collection(Process* p, int need, Eterm* objv, int nobj, Uint *recl) +minor_collection(Process* p, ErlHeapFragment *live_hf_end, + int need, Eterm* objv, int nobj, Uint *recl) { - Uint mature = HIGH_WATER(p) - HEAP_START(p); + Eterm *mature = p->abandoned_heap ? p->abandoned_heap : p->heap; + Uint mature_size = p->high_water - mature; + Uint size_before = young_gen_usage(p); /* * Allocate an old heap if we don't have one and if we'll need one. */ - if (OLD_HEAP(p) == NULL && mature != 0) { + if (OLD_HEAP(p) == NULL && mature_size != 0) { Eterm* n_old; /* Note: We choose a larger heap size than strictly needed, @@ -846,7 +1025,7 @@ minor_collection(Process* p, int need, Eterm* objv, int nobj, Uint *recl) * This improved Estone by more than 1200 estones on my computer * (Ultra Sparc 10). */ - Uint new_sz = erts_next_heap_size(HEAP_TOP(p) - HEAP_START(p), 1); + Uint new_sz = erts_next_heap_size(size_before, 1); /* Create new, empty old_heap */ n_old = (Eterm *) ERTS_HEAP_ALLOC(ERTS_ALC_T_OLD_HEAP, @@ -862,35 +1041,34 @@ minor_collection(Process* p, int need, Eterm* objv, int nobj, Uint *recl) */ if (OLD_HEAP(p) && - ((mature <= OLD_HEND(p) - OLD_HTOP(p)) && - ((BIN_VHEAP_MATURE(p) < ( BIN_OLD_VHEAP_SZ(p) - BIN_OLD_VHEAP(p)))) && - ((BIN_OLD_VHEAP_SZ(p) > BIN_OLD_VHEAP(p))) ) ) { - ErlMessage *msgp; - Uint size_after; - Uint need_after; - Uint stack_size = STACK_SZ_ON_HEAP(p); - Uint fragments = MBUF_SIZE(p) + combined_message_size(p); - Uint size_before = fragments + (HEAP_TOP(p) - HEAP_START(p)); - Uint new_sz = next_heap_size(p, HEAP_SIZE(p) + fragments, 0); - - do_minor(p, new_sz, objv, nobj); + ((mature_size <= OLD_HEND(p) - OLD_HTOP(p)) && + ((BIN_OLD_VHEAP_SZ(p) > BIN_OLD_VHEAP(p))) ) ) { + Eterm *prev_old_htop; + Uint stack_size, size_after, adjust_size, need_after, new_sz, new_mature; + + stack_size = p->hend - p->stop; + new_sz = stack_size + size_before; + new_sz = next_heap_size(p, new_sz, 0); + + prev_old_htop = p->old_htop; + do_minor(p, live_hf_end, (char *) mature, mature_size*sizeof(Eterm), + new_sz, objv, nobj); + + if (p->flags & F_ON_HEAP_MSGQ) + move_msgq_to_heap(p); + + new_mature = p->old_htop - prev_old_htop; + + size_after = new_mature; + size_after += HEAP_TOP(p) - HEAP_START(p) + p->mbuf_sz; + *recl += (size_before - size_after); - /* - * Copy newly received message onto the end of the new heap. - */ - ErtsGcQuickSanityCheck(p); - for (msgp = p->msg.first; msgp; msgp = msgp->next) { - if (msgp->data.attached) { - erts_move_msg_attached_data_to_heap(&p->htop, &p->off_heap, msgp); - ErtsGcQuickSanityCheck(p); - } - } ErtsGcQuickSanityCheck(p); GEN_GCS(p)++; - size_after = HEAP_TOP(p) - HEAP_START(p); - need_after = size_after + need + stack_size; - *recl += (size_before - size_after); + need_after = ((HEAP_TOP(p) - HEAP_START(p)) + + need + + stack_size); /* * Excessively large heaps should be shrunk, but @@ -901,6 +1079,8 @@ minor_collection(Process* p, int need, Eterm* objv, int nobj, Uint *recl) * the heap size is substantial, we don't want to shrink. */ + adjust_size = 0; + if ((HEAP_SIZE(p) > 3000) && (4 * need_after < HEAP_SIZE(p)) && ((HEAP_SIZE(p) > 8000) || (HEAP_SIZE(p) > (OLD_HEND(p) - OLD_HEAP(p))))) { @@ -922,26 +1102,33 @@ minor_collection(Process* p, int need, Eterm* objv, int nobj, Uint *recl) : next_heap_size(p, wanted, 0); if (wanted < HEAP_SIZE(p)) { shrink_new_heap(p, wanted, objv, nobj); + adjust_size = p->htop - p->heap; } - ASSERT(HEAP_SIZE(p) == next_heap_size(p, HEAP_SIZE(p), 0)); - return 1; /* We are done. */ + goto done; } if (HEAP_SIZE(p) >= need_after) { /* * The heap size turned out to be just right. We are done. */ - ASSERT(HEAP_SIZE(p) == next_heap_size(p, HEAP_SIZE(p), 0)); - return 1; + goto done; } + + grow_new_heap(p, next_heap_size(p, need_after, 0), objv, nobj); + adjust_size = p->htop - p->heap; + + done: + ASSERT(HEAP_SIZE(p) == next_heap_size(p, HEAP_SIZE(p), 0)); + ASSERT(MBUF(p) == NULL); + + return gc_cost(size_after, adjust_size); } /* - * Still not enough room after minor collection. Must force a major collection. + * Not enough room for a minor collection. Must force a major collection. */ - FLAGS(p) |= F_NEED_FULLSWEEP; - return 0; + return -1; } /* @@ -995,7 +1182,9 @@ static ERTS_INLINE void offset_nstack(Process* p, Sint offs, #endif /* HIPE */ static void -do_minor(Process *p, Uint new_sz, Eterm* objv, int nobj) +do_minor(Process *p, ErlHeapFragment *live_hf_end, + char *mature, Uint mature_size, + Uint new_sz, Eterm* objv, int nobj) { Rootset rootset; /* Rootset for GC (stack, dictionary, etc). */ Roots* roots; @@ -1004,17 +1193,24 @@ do_minor(Process *p, Uint new_sz, Eterm* objv, int nobj) Eterm* ptr; Eterm val; Eterm gval; - char* heap = (char *) HEAP_START(p); - Uint heap_size = (char *) HEAP_TOP(p) - heap; - Uint mature_size = (char *) HIGH_WATER(p) - heap; Eterm* old_htop = OLD_HTOP(p); Eterm* n_heap; + char* oh = (char *) OLD_HEAP(p); + Uint oh_size = (char *) OLD_HTOP(p) - oh; + + VERBOSE(DEBUG_SHCOPY, ("[pid=%T] MINOR GC: %p %p %p %p\n", p->common.id, + HEAP_START(p), HEAP_END(p), OLD_HEAP(p), OLD_HEND(p))); n_htop = n_heap = (Eterm*) ERTS_HEAP_ALLOC(ERTS_ALC_T_HEAP, sizeof(Eterm)*new_sz); - if (MBUF(p) != NULL) { - n_htop = collect_heap_frags(p, n_heap, n_htop, objv, nobj); + if (live_hf_end != ERTS_INVALID_HFRAG_PTR) { + /* + * Move heap frags that we know are completely live + * directly into the new young heap generation. + */ + n_htop = collect_live_heap_frags(p, live_hf_end, n_heap, n_htop, + objv, nobj); } n = setup_rootset(p, objv, nobj, &rootset); @@ -1037,9 +1233,9 @@ do_minor(Process *p, Uint new_sz, Eterm* objv, int nobj) if (IS_MOVED_BOXED(val)) { ASSERT(is_boxed(val)); *g_ptr++ = val; - } else if (in_area(ptr, heap, mature_size)) { + } else if (ErtsInArea(ptr, mature, mature_size)) { MOVE_BOXED(ptr,val,old_htop,g_ptr++); - } else if (in_area(ptr, heap, heap_size)) { + } else if (ErtsInYoungGen(gval, ptr, oh, oh_size)) { MOVE_BOXED(ptr,val,n_htop,g_ptr++); } else { g_ptr++; @@ -1052,9 +1248,9 @@ do_minor(Process *p, Uint new_sz, Eterm* objv, int nobj) val = *ptr; if (IS_MOVED_CONS(val)) { /* Moved */ *g_ptr++ = ptr[1]; - } else if (in_area(ptr, heap, mature_size)) { + } else if (ErtsInArea(ptr, mature, mature_size)) { MOVE_CONS(ptr,val,old_htop,g_ptr++); - } else if (in_area(ptr, heap, heap_size)) { + } else if (ErtsInYoungGen(gval, ptr, oh, oh_size)) { MOVE_CONS(ptr,val,n_htop,g_ptr++); } else { g_ptr++; @@ -1079,7 +1275,7 @@ do_minor(Process *p, Uint new_sz, Eterm* objv, int nobj) */ if (mature_size == 0) { - n_htop = sweep_one_area(n_heap, n_htop, heap, heap_size); + n_htop = sweep_new_heap(n_heap, n_htop, oh, oh_size); } else { Eterm* n_hp = n_heap; Eterm* ptr; @@ -1096,9 +1292,9 @@ do_minor(Process *p, Uint new_sz, Eterm* objv, int nobj) if (IS_MOVED_BOXED(val)) { ASSERT(is_boxed(val)); *n_hp++ = val; - } else if (in_area(ptr, heap, mature_size)) { + } else if (ErtsInArea(ptr, mature, mature_size)) { MOVE_BOXED(ptr,val,old_htop,n_hp++); - } else if (in_area(ptr, heap, heap_size)) { + } else if (ErtsInYoungGen(gval, ptr, oh, oh_size)) { MOVE_BOXED(ptr,val,n_htop,n_hp++); } else { n_hp++; @@ -1110,9 +1306,9 @@ do_minor(Process *p, Uint new_sz, Eterm* objv, int nobj) val = *ptr; if (IS_MOVED_CONS(val)) { *n_hp++ = ptr[1]; - } else if (in_area(ptr, heap, mature_size)) { + } else if (ErtsInArea(ptr, mature, mature_size)) { MOVE_CONS(ptr,val,old_htop,n_hp++); - } else if (in_area(ptr, heap, heap_size)) { + } else if (ErtsInYoungGen(gval, ptr, oh, oh_size)) { MOVE_CONS(ptr,val,n_htop,n_hp++); } else { n_hp++; @@ -1132,10 +1328,10 @@ do_minor(Process *p, Uint new_sz, Eterm* objv, int nobj) if (IS_MOVED_BOXED(val)) { *origptr = val; mb->base = binary_bytes(val); - } else if (in_area(ptr, heap, mature_size)) { + } else if (ErtsInArea(ptr, mature, mature_size)) { MOVE_BOXED(ptr,val,old_htop,origptr); mb->base = binary_bytes(mb->orig); - } else if (in_area(ptr, heap, heap_size)) { + } else if (ErtsInYoungGen(*origptr, ptr, oh, oh_size)) { MOVE_BOXED(ptr,val,n_htop,origptr); mb->base = binary_bytes(mb->orig); } @@ -1156,9 +1352,8 @@ do_minor(Process *p, Uint new_sz, Eterm* objv, int nobj) * may point to the old (soon to be deleted) new_heap. */ - if (OLD_HTOP(p) < old_htop) { - old_htop = sweep_one_area(OLD_HTOP(p), old_htop, heap, heap_size); - } + if (OLD_HTOP(p) < old_htop) + old_htop = sweep_new_heap(OLD_HTOP(p), old_htop, oh, oh_size); OLD_HTOP(p) = old_htop; HIGH_WATER(p) = n_htop; @@ -1190,8 +1385,12 @@ do_minor(Process *p, Uint new_sz, Eterm* objv, int nobj) #endif ERTS_HEAP_FREE(ERTS_ALC_T_HEAP, - (void*)HEAP_START(p), + (p->abandoned_heap + ? p->abandoned_heap + : HEAP_START(p)), HEAP_SIZE(p) * sizeof(Eterm)); + p->abandoned_heap = NULL; + p->flags &= ~F_ABANDONED_HEAP_USE; HEAP_START(p) = n_heap; HEAP_TOP(p) = n_htop; HEAP_SIZE(p) = new_sz; @@ -1208,38 +1407,30 @@ do_minor(Process *p, Uint new_sz, Eterm* objv, int nobj) */ static int -major_collection(Process* p, int need, Eterm* objv, int nobj, Uint *recl) +major_collection(Process* p, ErlHeapFragment *live_hf_end, + int need, Eterm* objv, int nobj, Uint *recl) { - Rootset rootset; - Roots* roots; - Uint size_before; + Uint size_before, size_after, stack_size; Eterm* n_heap; Eterm* n_htop; - char* src = (char *) HEAP_START(p); - Uint src_size = (char *) HEAP_TOP(p) - src; char* oh = (char *) OLD_HEAP(p); Uint oh_size = (char *) OLD_HTOP(p) - oh; - Uint n; - Uint new_sz; - Uint fragments = MBUF_SIZE(p) + combined_message_size(p); - ErlMessage *msgp; + Uint new_sz, stk_sz; + int adjusted; - size_before = fragments + (HEAP_TOP(p) - HEAP_START(p)); + VERBOSE(DEBUG_SHCOPY, ("[pid=%T] MAJOR GC: %p %p %p %p\n", p->common.id, + HEAP_START(p), HEAP_END(p), OLD_HEAP(p), OLD_HEND(p))); /* * Do a fullsweep GC. First figure out the size of the heap * to receive all live data. */ - new_sz = HEAP_SIZE(p) + fragments + (OLD_HTOP(p) - OLD_HEAP(p)); - /* - * We used to do - * - * new_sz += STACK_SZ_ON_HEAP(p); - * - * here for no obvious reason. (The stack size is already counted once - * in HEAP_SIZE(p).) - */ + size_before = young_gen_usage(p); + size_before += p->old_htop - p->old_heap; + stack_size = p->hend - p->stop; + + new_sz = stack_size + size_before; new_sz = next_heap_size(p, new_sz, 0); /* @@ -1253,13 +1444,76 @@ major_collection(Process* p, int need, Eterm* objv, int nobj, Uint *recl) n_htop = n_heap = (Eterm *) ERTS_HEAP_ALLOC(ERTS_ALC_T_HEAP, sizeof(Eterm)*new_sz); - /* - * Get rid of heap fragments. - */ + if (live_hf_end != ERTS_INVALID_HFRAG_PTR) { + /* + * Move heap frags that we know are completely live + * directly into the heap. + */ + n_htop = collect_live_heap_frags(p, live_hf_end, n_heap, n_htop, + objv, nobj); + } - if (MBUF(p) != NULL) { - n_htop = collect_heap_frags(p, n_heap, n_htop, objv, nobj); + n_htop = full_sweep_heaps(p, 0, n_heap, n_htop, oh, oh_size, objv, nobj); + + /* Move the stack to the end of the heap */ + stk_sz = HEAP_END(p) - p->stop; + sys_memcpy(n_heap + new_sz - stk_sz, p->stop, stk_sz * sizeof(Eterm)); + p->stop = n_heap + new_sz - stk_sz; + +#ifdef USE_VM_PROBES + if (HEAP_SIZE(p) != new_sz && DTRACE_ENABLED(process_heap_grow)) { + DTRACE_CHARBUF(pidbuf, DTRACE_TERM_BUF_SIZE); + + dtrace_proc_str(p, pidbuf); + DTRACE3(process_heap_grow, pidbuf, HEAP_SIZE(p), new_sz); } +#endif + + ERTS_HEAP_FREE(ERTS_ALC_T_HEAP, + (p->abandoned_heap + ? p->abandoned_heap + : HEAP_START(p)), + p->heap_sz * sizeof(Eterm)); + p->abandoned_heap = NULL; + p->flags &= ~F_ABANDONED_HEAP_USE; + HEAP_START(p) = n_heap; + HEAP_TOP(p) = n_htop; + HEAP_SIZE(p) = new_sz; + HEAP_END(p) = n_heap + new_sz; + GEN_GCS(p) = 0; + + HIGH_WATER(p) = HEAP_TOP(p); + + remove_message_buffers(p); + + if (p->flags & F_ON_HEAP_MSGQ) + move_msgq_to_heap(p); + + ErtsGcQuickSanityCheck(p); + + size_after = HEAP_TOP(p) - HEAP_START(p) + p->mbuf_sz; + *recl += size_before - size_after; + + adjusted = adjust_after_fullsweep(p, need, objv, nobj); + +#ifdef HARDDEBUG + disallow_heap_frag_ref_in_heap(p); +#endif + ErtsGcQuickSanityCheck(p); + + return gc_cost(size_after, adjusted ? size_after : 0); +} + +static Eterm * +full_sweep_heaps(Process *p, + int hibernate, + Eterm *n_heap, Eterm* n_htop, + char *oh, Uint oh_size, + Eterm *objv, int nobj) +{ + Rootset rootset; + Roots *roots; + Uint n; /* * Copy all top-level terms directly referenced by the rootset to @@ -1267,7 +1521,14 @@ major_collection(Process* p, int need, Eterm* objv, int nobj, Uint *recl) */ n = setup_rootset(p, objv, nobj, &rootset); - n_htop = fullsweep_nstack(p, n_htop); + +#ifdef HIPE + if (hibernate) + hipe_empty_nstack(p); + else + n_htop = fullsweep_nstack(p, n_htop); +#endif + roots = rootset.roots; while (n--) { Eterm* g_ptr = roots->v; @@ -1287,7 +1548,7 @@ major_collection(Process* p, int need, Eterm* objv, int nobj, Uint *recl) if (IS_MOVED_BOXED(val)) { ASSERT(is_boxed(val)); *g_ptr++ = val; - } else if (in_area(ptr, src, src_size) || in_area(ptr, oh, oh_size)) { + } else if (!erts_is_literal(gval, ptr)) { MOVE_BOXED(ptr,val,n_htop,g_ptr++); } else { g_ptr++; @@ -1300,7 +1561,7 @@ major_collection(Process* p, int need, Eterm* objv, int nobj, Uint *recl) val = *ptr; if (IS_MOVED_CONS(val)) { *g_ptr++ = ptr[1]; - } else if (in_area(ptr, src, src_size) || in_area(ptr, oh, oh_size)) { + } else if (!erts_is_literal(gval, ptr)) { MOVE_CONS(ptr,val,n_htop,g_ptr++); } else { g_ptr++; @@ -1325,74 +1586,7 @@ major_collection(Process* p, int need, Eterm* objv, int nobj, Uint *recl) * until all is copied. */ - if (oh_size == 0) { - n_htop = sweep_one_area(n_heap, n_htop, src, src_size); - } else { - Eterm* n_hp = n_heap; - - while (n_hp != n_htop) { - Eterm* ptr; - Eterm val; - Eterm gval = *n_hp; - - switch (primary_tag(gval)) { - case TAG_PRIMARY_BOXED: { - ptr = boxed_val(gval); - val = *ptr; - if (IS_MOVED_BOXED(val)) { - ASSERT(is_boxed(val)); - *n_hp++ = val; - } else if (in_area(ptr, src, src_size) || in_area(ptr, oh, oh_size)) { - MOVE_BOXED(ptr,val,n_htop,n_hp++); - } else { - n_hp++; - } - break; - } - case TAG_PRIMARY_LIST: { - ptr = list_val(gval); - val = *ptr; - if (IS_MOVED_CONS(val)) { - *n_hp++ = ptr[1]; - } else if (in_area(ptr, src, src_size) || in_area(ptr, oh, oh_size)) { - MOVE_CONS(ptr,val,n_htop,n_hp++); - } else { - n_hp++; - } - break; - } - case TAG_PRIMARY_HEADER: { - if (!header_is_thing(gval)) - n_hp++; - else { - if (header_is_bin_matchstate(gval)) { - ErlBinMatchState *ms = (ErlBinMatchState*) n_hp; - ErlBinMatchBuffer *mb = &(ms->mb); - Eterm* origptr; - origptr = &(mb->orig); - ptr = boxed_val(*origptr); - val = *ptr; - if (IS_MOVED_BOXED(val)) { - *origptr = val; - mb->base = binary_bytes(*origptr); - } else if (in_area(ptr, src, src_size) || - in_area(ptr, oh, oh_size)) { - MOVE_BOXED(ptr,val,n_htop,origptr); - mb->base = binary_bytes(*origptr); - ptr = boxed_val(*origptr); - val = *ptr; - } - } - n_hp += (thing_arityval(gval)+1); - } - break; - } - default: - n_hp++; - break; - } - } - } + n_htop = sweep_heaps(n_heap, n_htop, oh, oh_size); if (MSO(p).first) { sweep_off_heap(p, 1); @@ -1405,73 +1599,26 @@ major_collection(Process* p, int need, Eterm* objv, int nobj, Uint *recl) OLD_HEAP(p) = OLD_HTOP(p) = OLD_HEND(p) = NULL; } - /* Move the stack to the end of the heap */ - n = HEAP_END(p) - p->stop; - sys_memcpy(n_heap + new_sz - n, p->stop, n * sizeof(Eterm)); - p->stop = n_heap + new_sz - n; - -#ifdef USE_VM_PROBES - if (HEAP_SIZE(p) != new_sz && DTRACE_ENABLED(process_heap_grow)) { - DTRACE_CHARBUF(pidbuf, DTRACE_TERM_BUF_SIZE); - - dtrace_proc_str(p, pidbuf); - DTRACE3(process_heap_grow, pidbuf, HEAP_SIZE(p), new_sz); - } -#endif - - ERTS_HEAP_FREE(ERTS_ALC_T_HEAP, - (void *) HEAP_START(p), - (HEAP_END(p) - HEAP_START(p)) * sizeof(Eterm)); - HEAP_START(p) = n_heap; - HEAP_TOP(p) = n_htop; - HEAP_SIZE(p) = new_sz; - HEAP_END(p) = n_heap + new_sz; - GEN_GCS(p) = 0; - - HIGH_WATER(p) = HEAP_TOP(p); - - ErtsGcQuickSanityCheck(p); - - /* - * Copy newly received message onto the end of the new heap. - */ - for (msgp = p->msg.first; msgp; msgp = msgp->next) { - if (msgp->data.attached) { - erts_move_msg_attached_data_to_heap(&p->htop, &p->off_heap, msgp); - ErtsGcQuickSanityCheck(p); - } - } - - *recl += adjust_after_fullsweep(p, size_before, need, objv, nobj); - -#ifdef HARDDEBUG - disallow_heap_frag_ref_in_heap(p); -#endif - remove_message_buffers(p); - - ErtsGcQuickSanityCheck(p); - return 1; /* We are done. */ + return n_htop; } -static Uint -adjust_after_fullsweep(Process *p, Uint size_before, int need, Eterm *objv, int nobj) +static int +adjust_after_fullsweep(Process *p, int need, Eterm *objv, int nobj) { - Uint wanted, sz, size_after, need_after; + int adjusted = 0; + Uint wanted, sz, need_after; Uint stack_size = STACK_SZ_ON_HEAP(p); - Uint reclaimed_now; - - size_after = (HEAP_TOP(p) - HEAP_START(p)); - reclaimed_now = (size_before - size_after); /* * Resize the heap if needed. */ - need_after = size_after + need + stack_size; + need_after = (HEAP_TOP(p) - HEAP_START(p)) + need + stack_size; if (HEAP_SIZE(p) < need_after) { /* Too small - grow to match requested need */ sz = next_heap_size(p, need_after, 0); grow_new_heap(p, sz, objv, nobj); + adjusted = 1; } else if (3 * HEAP_SIZE(p) < 4 * need_after){ /* Need more than 75% of current, postpone to next GC.*/ FLAGS(p) |= F_HEAP_GROW; @@ -1488,28 +1635,10 @@ adjust_after_fullsweep(Process *p, Uint size_before, int need, Eterm *objv, int if (sz < HEAP_SIZE(p)) { shrink_new_heap(p, sz, objv, nobj); + adjusted = 1; } } - - return reclaimed_now; -} - -/* - * Return the size of all message buffers that are NOT linked in the - * mbuf list. - */ -static Uint -combined_message_size(Process* p) -{ - Uint sz = 0; - ErlMessage *msgp; - - for (msgp = p->msg.first; msgp; msgp = msgp->next) { - if (msgp->data.attached) { - sz += erts_msg_attached_data_size(msgp); - } - } - return sz; + return adjusted; } /* @@ -1522,6 +1651,10 @@ remove_message_buffers(Process* p) free_message_buffer(MBUF(p)); MBUF(p) = NULL; } + if (p->msg_frag) { + erts_cleanup_messages(p->msg_frag); + p->msg_frag = NULL; + } MBUF_SIZE(p) = 0; } #ifdef HARDDEBUG @@ -1533,64 +1666,6 @@ remove_message_buffers(Process* p) * For performance reasons, we use _unchecked_list_val(), _unchecked_boxed_val(), * and so on to avoid a function call. */ - -static void -disallow_heap_frag_ref(Process* p, Eterm* n_htop, Eterm* objv, int nobj) -{ - ErlHeapFragment* mbuf; - ErlHeapFragment* qb; - Eterm gval; - Eterm* ptr; - Eterm val; - - ASSERT(p->htop != NULL); - mbuf = MBUF(p); - - while (nobj--) { - gval = *objv; - - switch (primary_tag(gval)) { - - case TAG_PRIMARY_BOXED: { - ptr = _unchecked_boxed_val(gval); - val = *ptr; - if (IS_MOVED_BOXED(val)) { - ASSERT(is_boxed(val)); - objv++; - } else { - for (qb = mbuf; qb != NULL; qb = qb->next) { - if (in_area(ptr, qb->mem, qb->alloc_size*sizeof(Eterm))) { - abort(); - } - } - objv++; - } - break; - } - - case TAG_PRIMARY_LIST: { - ptr = _unchecked_list_val(gval); - val = *ptr; - if (IS_MOVED_CONS(val)) { - objv++; - } else { - for (qb = mbuf; qb != NULL; qb = qb->next) { - if (in_area(ptr, qb->mem, qb->alloc_size*sizeof(Eterm))) { - abort(); - } - } - objv++; - } - break; - } - - default: { - objv++; - break; - } - } - } -} static void disallow_heap_frag_ref_in_heap(Process* p) @@ -1618,9 +1693,9 @@ disallow_heap_frag_ref_in_heap(Process* p) switch (primary_tag(val)) { case TAG_PRIMARY_BOXED: ptr = _unchecked_boxed_val(val); - if (!in_area(ptr, heap, heap_size)) { + if (!ErtsInArea(ptr, heap, heap_size)) { for (qb = MBUF(p); qb != NULL; qb = qb->next) { - if (in_area(ptr, qb->mem, qb->alloc_size*sizeof(Eterm))) { + if (ErtsInArea(ptr, qb->mem, qb->alloc_size*sizeof(Eterm))) { abort(); } } @@ -1628,9 +1703,9 @@ disallow_heap_frag_ref_in_heap(Process* p) break; case TAG_PRIMARY_LIST: ptr = _unchecked_list_val(val); - if (!in_area(ptr, heap, heap_size)) { + if (!ErtsInArea(ptr, heap, heap_size)) { for (qb = MBUF(p); qb != NULL; qb = qb->next) { - if (in_area(ptr, qb->mem, qb->alloc_size*sizeof(Eterm))) { + if (ErtsInArea(ptr, qb->mem, qb->alloc_size*sizeof(Eterm))) { abort(); } } @@ -1672,26 +1747,26 @@ disallow_heap_frag_ref_in_old_heap(Process* p) val = *hp++; switch (primary_tag(val)) { case TAG_PRIMARY_BOXED: - ptr = (Eterm *) EXPAND_POINTER(val); - if (!in_area(ptr, old_heap, old_heap_size)) { - if (in_area(ptr, new_heap, new_heap_size)) { + ptr = (Eterm *) val; + if (!ErtsInArea(ptr, old_heap, old_heap_size)) { + if (ErtsInArea(ptr, new_heap, new_heap_size)) { abort(); } for (qb = MBUF(p); qb != NULL; qb = qb->next) { - if (in_area(ptr, qb->mem, qb->alloc_size*sizeof(Eterm))) { + if (ErtsInArea(ptr, qb->mem, qb->alloc_size*sizeof(Eterm))) { abort(); } } } break; case TAG_PRIMARY_LIST: - ptr = (Eterm *) EXPAND_POINTER(val); - if (!in_area(ptr, old_heap, old_heap_size)) { - if (in_area(ptr, new_heap, new_heap_size)) { + ptr = (Eterm *) val; + if (!ErtsInArea(ptr, old_heap, old_heap_size)) { + if (ErtsInArea(ptr, new_heap, new_heap_size)) { abort(); } for (qb = MBUF(p); qb != NULL; qb = qb->next) { - if (in_area(ptr, qb->mem, qb->alloc_size*sizeof(Eterm))) { + if (ErtsInArea(ptr, qb->mem, qb->alloc_size*sizeof(Eterm))) { abort(); } } @@ -1700,7 +1775,7 @@ disallow_heap_frag_ref_in_old_heap(Process* p) case TAG_PRIMARY_HEADER: if (header_is_thing(val)) { hp += _unchecked_thing_arityval(val); - if (!in_area(hp, old_heap, old_heap_size+1)) { + if (!ErtsInArea(hp, old_heap, old_heap_size+1)) { abort(); } } @@ -1710,66 +1785,30 @@ disallow_heap_frag_ref_in_old_heap(Process* p) } #endif -static Eterm* -sweep_rootset(Rootset* rootset, Eterm* htop, char* src, Uint src_size) +typedef enum { + ErtsSweepNewHeap, + ErtsSweepHeaps, + ErtsSweepLiteralArea +} ErtsSweepType; + +static ERTS_FORCE_INLINE Eterm * +sweep(Eterm *n_hp, Eterm *n_htop, + ErtsSweepType type, + char *oh, Uint ohsz, + char *src, Uint src_size) { - Roots* roots = rootset->roots; - Uint n = rootset->num_roots; Eterm* ptr; - Eterm gval; Eterm val; + Eterm gval; - while (n--) { - Eterm* g_ptr = roots->v; - Uint g_sz = roots->sz; - - roots++; - while (g_sz--) { - gval = *g_ptr; - - switch (primary_tag(gval)) { - case TAG_PRIMARY_BOXED: { - ptr = boxed_val(gval); - val = *ptr; - if (IS_MOVED_BOXED(val)) { - ASSERT(is_boxed(val)); - *g_ptr++ = val; - } else if (in_area(ptr, src, src_size)) { - MOVE_BOXED(ptr,val,htop,g_ptr++); - } else { - g_ptr++; - } - break; - } - case TAG_PRIMARY_LIST: { - ptr = list_val(gval); - val = *ptr; - if (IS_MOVED_CONS(val)) { - *g_ptr++ = ptr[1]; - } else if (in_area(ptr, src, src_size)) { - MOVE_CONS(ptr,val,htop,g_ptr++); - } else { - g_ptr++; - } - break; - } - - default: - g_ptr++; - break; - } - } - } - return htop; -} - +#undef ERTS_IS_IN_SWEEP_AREA -static Eterm* -sweep_one_area(Eterm* n_hp, Eterm* n_htop, char* src, Uint src_size) -{ - Eterm* ptr; - Eterm val; - Eterm gval; +#define ERTS_IS_IN_SWEEP_AREA(TPtr, Ptr) \ + (type == ErtsSweepHeaps \ + ? !erts_is_literal((TPtr), (Ptr)) \ + : (type == ErtsSweepNewHeap \ + ? ErtsInYoungGen((TPtr), (Ptr), oh, ohsz) \ + : ErtsInArea((Ptr), src, src_size))) while (n_hp != n_htop) { ASSERT(n_hp < n_htop); @@ -1781,7 +1820,7 @@ sweep_one_area(Eterm* n_hp, Eterm* n_htop, char* src, Uint src_size) if (IS_MOVED_BOXED(val)) { ASSERT(is_boxed(val)); *n_hp++ = val; - } else if (in_area(ptr, src, src_size)) { + } else if (ERTS_IS_IN_SWEEP_AREA(gval, ptr)) { MOVE_BOXED(ptr,val,n_htop,n_hp++); } else { n_hp++; @@ -1793,7 +1832,7 @@ sweep_one_area(Eterm* n_hp, Eterm* n_htop, char* src, Uint src_size) val = *ptr; if (IS_MOVED_CONS(val)) { *n_hp++ = ptr[1]; - } else if (in_area(ptr, src, src_size)) { + } else if (ERTS_IS_IN_SWEEP_AREA(gval, ptr)) { MOVE_CONS(ptr,val,n_htop,n_hp++); } else { n_hp++; @@ -1814,7 +1853,7 @@ sweep_one_area(Eterm* n_hp, Eterm* n_htop, char* src, Uint src_size) if (IS_MOVED_BOXED(val)) { *origptr = val; mb->base = binary_bytes(*origptr); - } else if (in_area(ptr, src, src_size)) { + } else if (ERTS_IS_IN_SWEEP_AREA(*origptr, ptr)) { MOVE_BOXED(ptr,val,n_htop,origptr); mb->base = binary_bytes(*origptr); } @@ -1829,10 +1868,41 @@ sweep_one_area(Eterm* n_hp, Eterm* n_htop, char* src, Uint src_size) } } return n_htop; +#undef ERTS_IS_IN_SWEEP_AREA +} + +static Eterm * +sweep_new_heap(Eterm *n_hp, Eterm *n_htop, char* old_heap, Uint old_heap_size) +{ + return sweep(n_hp, n_htop, + ErtsSweepNewHeap, + old_heap, old_heap_size, + NULL, 0); +} + +static Eterm * +sweep_heaps(Eterm *n_hp, Eterm *n_htop, char* old_heap, Uint old_heap_size) +{ + return sweep(n_hp, n_htop, + ErtsSweepHeaps, + old_heap, old_heap_size, + NULL, 0); +} + +static Eterm * +sweep_literal_area(Eterm *n_hp, Eterm *n_htop, + char* old_heap, Uint old_heap_size, + char* src, Uint src_size) +{ + return sweep(n_hp, n_htop, + ErtsSweepLiteralArea, + old_heap, old_heap_size, + src, src_size); } static Eterm* -sweep_one_heap(Eterm* heap_ptr, Eterm* heap_end, Eterm* htop, char* src, Uint src_size) +sweep_literals_to_old_heap(Eterm* heap_ptr, Eterm* heap_end, Eterm* htop, + char* src, Uint src_size) { while (heap_ptr < heap_end) { Eterm* ptr; @@ -1846,7 +1916,7 @@ sweep_one_heap(Eterm* heap_ptr, Eterm* heap_end, Eterm* htop, char* src, Uint sr if (IS_MOVED_BOXED(val)) { ASSERT(is_boxed(val)); *heap_ptr++ = val; - } else if (in_area(ptr, src, src_size)) { + } else if (ErtsInArea(ptr, src, src_size)) { MOVE_BOXED(ptr,val,htop,heap_ptr++); } else { heap_ptr++; @@ -1858,7 +1928,7 @@ sweep_one_heap(Eterm* heap_ptr, Eterm* heap_end, Eterm* htop, char* src, Uint sr val = *ptr; if (IS_MOVED_CONS(val)) { *heap_ptr++ = ptr[1]; - } else if (in_area(ptr, src, src_size)) { + } else if (ErtsInArea(ptr, src, src_size)) { MOVE_CONS(ptr,val,htop,heap_ptr++); } else { heap_ptr++; @@ -1869,6 +1939,21 @@ sweep_one_heap(Eterm* heap_ptr, Eterm* heap_end, Eterm* htop, char* src, Uint sr if (!header_is_thing(gval)) { heap_ptr++; } else { + if (header_is_bin_matchstate(gval)) { + ErlBinMatchState *ms = (ErlBinMatchState*) heap_ptr; + ErlBinMatchBuffer *mb = &(ms->mb); + Eterm* origptr; + origptr = &(mb->orig); + ptr = boxed_val(*origptr); + val = *ptr; + if (IS_MOVED_BOXED(val)) { + *origptr = val; + mb->base = binary_bytes(*origptr); + } else if (ErtsInArea(ptr, src, src_size)) { + MOVE_BOXED(ptr,val,htop,origptr); + mb->base = binary_bytes(*origptr); + } + } heap_ptr += (thing_arityval(gval)+1); } break; @@ -1915,32 +2000,22 @@ move_one_area(Eterm* n_htop, char* src, Uint src_size) */ static Eterm* -collect_heap_frags(Process* p, Eterm* n_hstart, Eterm* n_htop, - Eterm* objv, int nobj) +collect_live_heap_frags(Process* p, ErlHeapFragment *live_hf_end, + Eterm* n_hstart, Eterm* n_htop, + Eterm* objv, int nobj) { ErlHeapFragment* qb; char* frag_begin; Uint frag_size; /* - * We don't allow references to a heap fragments from the stack, heap, - * or process dictionary. - */ -#ifdef HARDDEBUG - disallow_heap_frag_ref(p, n_htop, p->stop, STACK_START(p) - p->stop); - if (p->dictionary != NULL) { - disallow_heap_frag_ref(p, n_htop, p->dictionary->data, p->dictionary->used); - } - disallow_heap_frag_ref_in_heap(p); -#endif - - /* * Move the heap fragments to the new heap. Note that no GC is done on * the heap fragments. Any garbage will thus be moved as well and survive * until next GC. */ qb = MBUF(p); - while (qb != NULL) { + while (qb != live_hf_end) { + ASSERT(!qb->off_heap.first); /* process fragments use the MSO(p) list */ frag_size = qb->used_size * sizeof(Eterm); if (frag_size != 0) { frag_begin = (char *) qb->mem; @@ -1951,12 +2026,177 @@ collect_heap_frags(Process* p, Eterm* n_hstart, Eterm* n_htop, return n_htop; } +static ERTS_INLINE void +copy_one_frag(Eterm** hpp, ErlOffHeap* off_heap, + ErlHeapFragment *bp, Eterm *refs, int nrefs) +{ + Uint sz; + int i; + Sint offs; + struct erl_off_heap_header* oh; + Eterm *fhp, *hp; + + OH_OVERHEAD(off_heap, bp->off_heap.overhead); + sz = bp->used_size; + + fhp = bp->mem; + hp = *hpp; + offs = hp - fhp; + + oh = NULL; + while (sz--) { + Uint cpy_sz; + Eterm val = *fhp++; + + switch (primary_tag(val)) { + case TAG_PRIMARY_IMMED1: + *hp++ = val; + break; + case TAG_PRIMARY_LIST: + case TAG_PRIMARY_BOXED: + *hp++ = offset_ptr(val, offs); + break; + case TAG_PRIMARY_HEADER: + *hp++ = val; + switch (val & _HEADER_SUBTAG_MASK) { + case ARITYVAL_SUBTAG: + break; + case REFC_BINARY_SUBTAG: + case FUN_SUBTAG: + case EXTERNAL_PID_SUBTAG: + case EXTERNAL_PORT_SUBTAG: + case EXTERNAL_REF_SUBTAG: + oh = (struct erl_off_heap_header*) (hp-1); + cpy_sz = thing_arityval(val); + goto cpy_words; + default: + cpy_sz = header_arity(val); + + cpy_words: + ASSERT(sz >= cpy_sz); + sz -= cpy_sz; + while (cpy_sz >= 8) { + cpy_sz -= 8; + *hp++ = *fhp++; + *hp++ = *fhp++; + *hp++ = *fhp++; + *hp++ = *fhp++; + *hp++ = *fhp++; + *hp++ = *fhp++; + *hp++ = *fhp++; + *hp++ = *fhp++; + } + switch (cpy_sz) { + case 7: *hp++ = *fhp++; + case 6: *hp++ = *fhp++; + case 5: *hp++ = *fhp++; + case 4: *hp++ = *fhp++; + case 3: *hp++ = *fhp++; + case 2: *hp++ = *fhp++; + case 1: *hp++ = *fhp++; + default: break; + } + if (oh) { + /* Add to offheap list */ + oh->next = off_heap->first; + off_heap->first = oh; + ASSERT(*hpp <= (Eterm*)oh); + ASSERT(hp > (Eterm*)oh); + oh = NULL; + } + break; + } + break; + } + } + + ASSERT(bp->used_size == hp - *hpp); + *hpp = hp; + + for (i = 0; i < nrefs; i++) { + if (is_not_immed(refs[i])) + refs[i] = offset_ptr(refs[i], offs); + } + bp->off_heap.first = NULL; +} + +static void +move_msgq_to_heap(Process *p) +{ + ErtsMessage **mpp = &p->msg.first; + + while (*mpp) { + ErtsMessage *mp = *mpp; + + if (mp->data.attached) { + ErlHeapFragment *bp; + ErtsHeapFactory factory; + + erts_factory_proc_prealloc_init(&factory, p, + erts_msg_attached_data_size(mp)); + + if (is_non_value(ERL_MESSAGE_TERM(mp))) { + if (mp->data.dist_ext) { + ASSERT(mp->data.dist_ext->heap_size >= 0); + if (is_not_nil(ERL_MESSAGE_TOKEN(mp))) { + bp = erts_dist_ext_trailer(mp->data.dist_ext); + ERL_MESSAGE_TOKEN(mp) = copy_struct(ERL_MESSAGE_TOKEN(mp), + bp->used_size, + &factory.hp, + factory.off_heap); + erts_cleanup_offheap(&bp->off_heap); + } + ERL_MESSAGE_TERM(mp) = erts_decode_dist_ext(&factory, + mp->data.dist_ext); + erts_free_dist_ext_copy(mp->data.dist_ext); + mp->data.dist_ext = NULL; + } + } + else { + + if (mp->data.attached == ERTS_MSG_COMBINED_HFRAG) + bp = &mp->hfrag; + else + bp = mp->data.heap_frag; + + if (bp->next) + erts_move_multi_frags(&factory.hp, factory.off_heap, bp, + mp->m, ERL_MESSAGE_REF_ARRAY_SZ, 0); + else + copy_one_frag(&factory.hp, factory.off_heap, bp, + mp->m, ERL_MESSAGE_REF_ARRAY_SZ); + + if (mp->data.attached != ERTS_MSG_COMBINED_HFRAG) { + mp->data.heap_frag = NULL; + free_message_buffer(bp); + } + else { + ErtsMessage *tmp = erts_alloc_message(0, NULL); + sys_memcpy((void *) tmp->m, (void *) mp->m, + sizeof(Eterm)*ERL_MESSAGE_REF_ARRAY_SZ); + tmp->next = mp->next; + if (p->msg.save == &mp->next) + p->msg.save = &tmp->next; + if (p->msg.last == &mp->next) + p->msg.last = &tmp->next; + *mpp = tmp; + mp->next = NULL; + erts_cleanup_messages(mp); + mp = tmp; + } + } + + erts_factory_close(&factory); + } + + mpp = &(*mpp)->next; + } +} + static Uint setup_rootset(Process *p, Eterm *objv, int nobj, Rootset *rootset) { - Uint avail; Roots* roots; - ErlMessage* mp; Uint n; n = 0; @@ -1968,8 +2208,8 @@ setup_rootset(Process *p, Eterm *objv, int nobj, Rootset *rootset) ++n; if (p->dictionary != NULL) { - roots[n].v = p->dictionary->data; - roots[n].sz = p->dictionary->used; + roots[n].v = ERTS_PD_START(p->dictionary); + roots[n].sz = ERTS_PD_SIZE(p->dictionary); ++n; } if (nobj > 0) { @@ -1979,7 +2219,7 @@ setup_rootset(Process *p, Eterm *objv, int nobj, Rootset *rootset) } ASSERT((is_nil(p->seq_trace_token) || - is_tuple(follow_moved(p->seq_trace_token)) || + is_tuple(follow_moved(p->seq_trace_token, (Eterm) 0)) || is_atom(p->seq_trace_token))); if (is_not_immed(p->seq_trace_token)) { roots[n].v = &p->seq_trace_token; @@ -1997,7 +2237,7 @@ setup_rootset(Process *p, Eterm *objv, int nobj, Rootset *rootset) is_internal_pid(ERTS_TRACER_PROC(p)) || is_internal_port(ERTS_TRACER_PROC(p))); - ASSERT(is_pid(follow_moved(p->group_leader))); + ASSERT(is_pid(follow_moved(p->group_leader, (Eterm) 0))); if (is_not_immed(p->group_leader)) { roots[n].v = &p->group_leader; roots[n].sz = 1; @@ -2018,33 +2258,63 @@ setup_rootset(Process *p, Eterm *objv, int nobj, Rootset *rootset) roots[n].sz = 1; n++; } + + /* + * If a NIF has saved arguments, they need to be added + */ + if (ERTS_PROC_GET_NIF_TRAP_EXPORT(p)) { + Eterm* argv; + int argc; + if (erts_setup_nif_gc(p, &argv, &argc)) { + roots[n].v = argv; + roots[n].sz = argc; + n++; + } + } + ASSERT(n <= rootset->size); - mp = p->msg.first; - avail = rootset->size - n; - while (mp != NULL) { - if (avail == 0) { - Uint new_size = 2*rootset->size; - if (roots == rootset->def) { - roots = erts_alloc(ERTS_ALC_T_ROOTSET, - new_size*sizeof(Roots)); - sys_memcpy(roots, rootset->def, sizeof(rootset->def)); - } else { - roots = erts_realloc(ERTS_ALC_T_ROOTSET, - (void *) roots, - new_size*sizeof(Roots)); - } + switch (p->flags & (F_OFF_HEAP_MSGQ|F_OFF_HEAP_MSGQ_CHNG)) { + case F_OFF_HEAP_MSGQ|F_OFF_HEAP_MSGQ_CHNG: + (void) erts_move_messages_off_heap(p); + case F_OFF_HEAP_MSGQ: + break; + case F_OFF_HEAP_MSGQ_CHNG: + case 0: { + /* + * We do not have off heap message queue enabled, i.e. we + * need to add message queue to rootset... + */ + ErtsMessage *mp; + + /* Ensure large enough rootset... */ + if (n + p->msg.len > rootset->size) { + Uint new_size = n + p->msg.len; + ERTS_GC_ASSERT(roots == rootset->def); + roots = erts_alloc(ERTS_ALC_T_ROOTSET, + new_size*sizeof(Roots)); + sys_memcpy(roots, rootset->def, n*sizeof(Roots)); rootset->size = new_size; - avail = new_size - n; } - if (mp->data.attached == NULL) { - roots[n].v = mp->m; - roots[n].sz = 2; - n++; - avail--; + + for (mp = p->msg.first; mp; mp = mp->next) { + + if (!mp->data.attached) { + /* + * Message may refer data on heap; + * add it to rootset... + */ + roots[n].v = mp->m; + roots[n].sz = ERL_MESSAGE_REF_ARRAY_SZ; + n++; + } } - mp = mp->next; + break; + } } + + ASSERT(rootset->size >= n); + rootset->roots = roots; rootset->num_roots = n; return n; @@ -2284,11 +2554,11 @@ sweep_off_heap(Process *p, int fullsweep) */ while (ptr) { if (IS_MOVED_BOXED(ptr->thing_word)) { - ASSERT(!in_area(ptr, oheap, oheap_sz)); + ASSERT(!ErtsInArea(ptr, oheap, oheap_sz)); *prev = ptr = (struct erl_off_heap_header*) boxed_val(ptr->thing_word); ASSERT(!IS_MOVED_BOXED(ptr->thing_word)); if (ptr->thing_word == HEADER_PROC_BIN) { - int to_new_heap = !in_area(ptr, oheap, oheap_sz); + int to_new_heap = !ErtsInArea(ptr, oheap, oheap_sz); ASSERT(to_new_heap == !seen_mature || (!to_new_heap && (seen_mature=1))); if (to_new_heap) { bin_vheap += ptr->size / sizeof(Eterm); @@ -2302,7 +2572,7 @@ sweep_off_heap(Process *p, int fullsweep) ptr = ptr->next; } } - else if (!in_area(ptr, oheap, oheap_sz)) { + else if (!ErtsInArea(ptr, oheap, oheap_sz)) { /* garbage */ switch (thing_subtag(ptr->thing_word)) { case REFC_BINARY_SUBTAG: @@ -2334,7 +2604,7 @@ sweep_off_heap(Process *p, int fullsweep) * generational collection - keep objects in list. */ while (ptr) { - ASSERT(in_area(ptr, oheap, oheap_sz)); + ASSERT(ErtsInArea(ptr, oheap, oheap_sz)); ASSERT(!IS_MOVED_BOXED(ptr->thing_word)); if (ptr->thing_word == HEADER_PROC_BIN) { BIN_OLD_VHEAP(p) += ptr->size / sizeof(Eterm); /* for binary gc (words)*/ @@ -2353,7 +2623,6 @@ sweep_off_heap(Process *p, int fullsweep) } BIN_VHEAP_SZ(p) = next_vheap_size(p, bin_vheap, BIN_VHEAP_SZ(p)); MSO(p).overhead = bin_vheap; - BIN_VHEAP_MATURE(p) = bin_vheap; /* * If we got any shrink candidates, check them out. @@ -2386,7 +2655,6 @@ sweep_off_heap(Process *p, int fullsweep) } pb->val = erts_bin_realloc(pb->val, new_size); - pb->val->orig_size = new_size; pb->bytes = (byte *) pb->val->orig_bytes; } } @@ -2425,7 +2693,7 @@ offset_heap(Eterm* hp, Uint sz, Sint offs, char* area, Uint area_size) switch (primary_tag(val)) { case TAG_PRIMARY_LIST: case TAG_PRIMARY_BOXED: - if (in_area(ptr_val(val), area, area_size)) { + if (ErtsInArea(ptr_val(val), area, area_size)) { *hp = offset_ptr(val, offs); } hp++; @@ -2447,7 +2715,7 @@ offset_heap(Eterm* hp, Uint sz, Sint offs, char* area, Uint area_size) { struct erl_off_heap_header* oh = (struct erl_off_heap_header*) hp; - if (in_area(oh->next, area, area_size)) { + if (ErtsInArea(oh->next, area, area_size)) { Eterm** uptr = (Eterm **) (void *) &oh->next; *uptr += offs; /* Patch the mso chain */ } @@ -2457,7 +2725,7 @@ offset_heap(Eterm* hp, Uint sz, Sint offs, char* area, Uint area_size) { ErlBinMatchState *ms = (ErlBinMatchState*) hp; ErlBinMatchBuffer *mb = &(ms->mb); - if (in_area(ptr_val(mb->orig), area, area_size)) { + if (ErtsInArea(ptr_val(mb->orig), area, area_size)) { mb->orig = offset_ptr(mb->orig, offs); mb->base = binary_bytes(mb->orig); } @@ -2487,7 +2755,7 @@ offset_heap_ptr(Eterm* hp, Uint sz, Sint offs, char* area, Uint area_size) switch (primary_tag(val)) { case TAG_PRIMARY_LIST: case TAG_PRIMARY_BOXED: - if (in_area(ptr_val(val), area, area_size)) { + if (ErtsInArea(ptr_val(val), area, area_size)) { *hp = offset_ptr(val, offs); } hp++; @@ -2502,7 +2770,7 @@ offset_heap_ptr(Eterm* hp, Uint sz, Sint offs, char* area, Uint area_size) static void offset_off_heap(Process* p, Sint offs, char* area, Uint area_size) { - if (MSO(p).first && in_area((Eterm *)MSO(p).first, area, area_size)) { + if (MSO(p).first && ErtsInArea((Eterm *)MSO(p).first, area, area_size)) { Eterm** uptr = (Eterm**) (void *) &MSO(p).first; *uptr += offs; } @@ -2514,35 +2782,39 @@ offset_off_heap(Process* p, Sint offs, char* area, Uint area_size) static void offset_mqueue(Process *p, Sint offs, char* area, Uint area_size) { - ErlMessage* mp = p->msg.first; - - while (mp != NULL) { - Eterm mesg = ERL_MESSAGE_TERM(mp); - if (is_value(mesg)) { - switch (primary_tag(mesg)) { - case TAG_PRIMARY_LIST: - case TAG_PRIMARY_BOXED: - if (in_area(ptr_val(mesg), area, area_size)) { - ERL_MESSAGE_TERM(mp) = offset_ptr(mesg, offs); + ErtsMessage* mp = p->msg.first; + + if ((p->flags & (F_OFF_HEAP_MSGQ|F_OFF_HEAP_MSGQ_CHNG)) != F_OFF_HEAP_MSGQ) { + + while (mp != NULL) { + Eterm mesg = ERL_MESSAGE_TERM(mp); + if (is_value(mesg)) { + switch (primary_tag(mesg)) { + case TAG_PRIMARY_LIST: + case TAG_PRIMARY_BOXED: + if (ErtsInArea(ptr_val(mesg), area, area_size)) { + ERL_MESSAGE_TERM(mp) = offset_ptr(mesg, offs); + } + break; } - break; } - } - mesg = ERL_MESSAGE_TOKEN(mp); - if (is_boxed(mesg) && in_area(ptr_val(mesg), area, area_size)) { - ERL_MESSAGE_TOKEN(mp) = offset_ptr(mesg, offs); - } + mesg = ERL_MESSAGE_TOKEN(mp); + if (is_boxed(mesg) && ErtsInArea(ptr_val(mesg), area, area_size)) { + ERL_MESSAGE_TOKEN(mp) = offset_ptr(mesg, offs); + } #ifdef USE_VM_PROBES - mesg = ERL_MESSAGE_DT_UTAG(mp); - if (is_boxed(mesg) && in_area(ptr_val(mesg), area, area_size)) { - ERL_MESSAGE_DT_UTAG(mp) = offset_ptr(mesg, offs); - } + mesg = ERL_MESSAGE_DT_UTAG(mp); + if (is_boxed(mesg) && ErtsInArea(ptr_val(mesg), area, area_size)) { + ERL_MESSAGE_DT_UTAG(mp) = offset_ptr(mesg, offs); + } #endif - ASSERT((is_nil(ERL_MESSAGE_TOKEN(mp)) || - is_tuple(ERL_MESSAGE_TOKEN(mp)) || - is_atom(ERL_MESSAGE_TOKEN(mp)))); - mp = mp->next; + ASSERT((is_nil(ERL_MESSAGE_TOKEN(mp)) || + is_tuple(ERL_MESSAGE_TOKEN(mp)) || + is_atom(ERL_MESSAGE_TOKEN(mp)))); + mp = mp->next; + } + } } @@ -2551,8 +2823,8 @@ offset_one_rootset(Process *p, Sint offs, char* area, Uint area_size, Eterm* objv, int nobj) { if (p->dictionary) { - offset_heap(p->dictionary->data, - p->dictionary->used, + offset_heap(ERTS_PD_START(p->dictionary), + ERTS_PD_SIZE(p->dictionary), offs, area, area_size); } @@ -2601,7 +2873,7 @@ reply_gc_info(void *vgcirp) Eterm **hpp; Uint sz, *szp; ErlOffHeap *ohp = NULL; - ErlHeapFragment *bp = NULL; + ErtsMessage *mp = NULL; ASSERT(esdp); @@ -2627,16 +2899,13 @@ reply_gc_info(void *vgcirp) if (hpp) break; - hp = erts_alloc_message_heap(sz, &bp, &ohp, rp, &rp_locks); + mp = erts_alloc_message_heap(rp, &rp_locks, sz, &hp, &ohp); + szp = NULL; hpp = &hp; } - erts_queue_message(rp, &rp_locks, bp, msg, NIL -#ifdef USE_VM_PROBES - , NIL -#endif - ); + erts_queue_message(rp, &rp_locks, mp, msg, NIL); if (gcirp->req_sched == esdp->no) rp_locks &= ~ERTS_PROC_LOCK_MAIN; @@ -2644,7 +2913,7 @@ reply_gc_info(void *vgcirp) if (rp_locks) erts_smp_proc_unlock(rp, rp_locks); - erts_smp_proc_dec_refc(rp); + erts_proc_dec_refc(rp); if (erts_smp_atomic32_dec_read_nob(&gcirp->refc) == 0) gcireq_free(vgcirp); @@ -2668,7 +2937,7 @@ erts_gc_info_request(Process *c_p) erts_smp_atomic32_init_nob(&gcirp->refc, (erts_aint32_t) erts_no_schedulers); - erts_smp_proc_add_refc(c_p, (Sint32) erts_no_schedulers); + erts_proc_add_refc(c_p, (Sint) erts_no_schedulers); #ifdef ERTS_SMP if (erts_no_schedulers > 1) @@ -2683,41 +2952,105 @@ erts_gc_info_request(Process *c_p) return ref; } +Eterm +erts_process_gc_info(Process *p, Uint *sizep, Eterm **hpp) +{ + ERTS_DECL_AM(bin_vheap_size); + ERTS_DECL_AM(bin_vheap_block_size); + ERTS_DECL_AM(bin_old_vheap_size); + ERTS_DECL_AM(bin_old_vheap_block_size); + Eterm tags[] = { + /* If you increase the number of elements here, make sure to update + any call sites as they may have stack allocations that depend + on the number of elements here. */ + am_old_heap_block_size, + am_heap_block_size, + am_mbuf_size, + am_recent_size, + am_stack_size, + am_old_heap_size, + am_heap_size, + AM_bin_vheap_size, + AM_bin_vheap_block_size, + AM_bin_old_vheap_size, + AM_bin_old_vheap_block_size + }; + UWord values[] = { + OLD_HEAP(p) ? OLD_HEND(p) - OLD_HEAP(p) : 0, + HEAP_SIZE(p), + MBUF_SIZE(p), + HIGH_WATER(p) - HEAP_START(p), + STACK_START(p) - p->stop, + OLD_HEAP(p) ? OLD_HTOP(p) - OLD_HEAP(p) : 0, + HEAP_TOP(p) - HEAP_START(p), + MSO(p).overhead, + BIN_VHEAP_SZ(p), + BIN_OLD_VHEAP(p), + BIN_OLD_VHEAP_SZ(p) + }; + + Eterm res = THE_NON_VALUE; + + ERTS_CT_ASSERT(sizeof(values)/sizeof(*values) == sizeof(tags)/sizeof(*tags)); + ERTS_CT_ASSERT(sizeof(values)/sizeof(*values) == ERTS_PROCESS_GC_INFO_MAX_TERMS); + + res = erts_bld_atom_uword_2tup_list(hpp, + sizep, + sizeof(values)/sizeof(*values), + tags, + values); + + return res; +} + #if defined(DEBUG) || defined(ERTS_OFFHEAP_DEBUG) static int within2(Eterm *ptr, Process *p, Eterm *real_htop) { - ErlHeapFragment* bp = MBUF(p); - ErlMessage* mp = p->msg.first; - Eterm *htop = real_htop ? real_htop : HEAP_TOP(p); + ErlHeapFragment* bp; + ErtsMessage* mp; + Eterm *htop, *heap; + + if (p->abandoned_heap) + ERTS_GET_ORIG_HEAP(p, heap, htop); + else { + heap = p->heap; + htop = real_htop ? real_htop : HEAP_TOP(p); + } if (OLD_HEAP(p) && (OLD_HEAP(p) <= ptr && ptr < OLD_HEND(p))) { return 1; } - if (HEAP_START(p) <= ptr && ptr < htop) { + if (heap <= ptr && ptr < htop) { return 1; } - while (bp != NULL) { - if (bp->mem <= ptr && ptr < bp->mem + bp->used_size) { - return 1; - } - bp = bp->next; - } + + mp = p->msg_frag; + bp = p->mbuf; + + if (bp) + goto search_heap_frags; + while (mp) { - if (mp->data.attached) { - ErlHeapFragment *hfp; - if (is_value(ERL_MESSAGE_TERM(mp))) - hfp = mp->data.heap_frag; - else if (is_not_nil(ERL_MESSAGE_TOKEN(mp))) - hfp = erts_dist_ext_trailer(mp->data.dist_ext); - else - hfp = NULL; - if (hfp && hfp->mem <= ptr && ptr < hfp->mem + hfp->used_size) + + if (mp->data.attached == ERTS_MSG_COMBINED_HFRAG) + bp = &mp->hfrag; + else + bp = mp->data.heap_frag; + + mp = mp->next; + + search_heap_frags: + + while (bp) { + if (bp->mem <= ptr && ptr < bp->mem + bp->used_size) { return 1; + } + bp = bp->next; } - mp = mp->next; } + return 0; } @@ -2734,16 +3067,16 @@ within(Eterm *ptr, Process *p) #define ERTS_CHK_OFFHEAP_ASSERT(EXP) \ do { \ if (!(EXP)) \ - erl_exit(ERTS_ABORT_EXIT, \ + erts_exit(ERTS_ABORT_EXIT, \ "%s:%d: Assertion failed: %s\n", \ __FILE__, __LINE__, #EXP); \ } while (0) + #ifdef ERTS_OFFHEAP_DEBUG_CHK_CIRCULAR_LIST # define ERTS_OFFHEAP_VISITED_BIT ((Eterm) 1 << 31) #endif - void erts_check_off_heap2(Process *p, Eterm *htop) { @@ -2772,7 +3105,7 @@ erts_check_off_heap2(Process *p, Eterm *htop) } ERTS_CHK_OFFHEAP_ASSERT(refc >= 1); #ifdef ERTS_OFFHEAP_DEBUG_CHK_CIRCULAR_LIST - ERTS_CHK_OFFHEAP_ASSERT(!(u.hdr->thing_word & ERTS_EXTERNAL_VISITED_BIT)); + ERTS_CHK_OFFHEAP_ASSERT(!(u.hdr->thing_word & ERTS_OFFHEAP_VISITED_BIT)); u.hdr->thing_word |= ERTS_OFFHEAP_VISITED_BIT; #endif if (old) { @@ -2785,7 +3118,7 @@ erts_check_off_heap2(Process *p, Eterm *htop) } } -#ifdef ERTS_OFFHEAP_DEBUG_CHK_CIRCULAR_EXTERNAL_LIST +#ifdef ERTS_OFFHEAP_DEBUG_CHK_CIRCULAR_LIST for (u.hdr = MSO(p).first; u.hdr; u.hdr = u.hdr->next) u.hdr->thing_word &= ~ERTS_OFFHEAP_VISITED_BIT; #endif diff --git a/erts/emulator/beam/erl_gc.h b/erts/emulator/beam/erl_gc.h index 5203dda263..2cedd9361f 100644 --- a/erts/emulator/beam/erl_gc.h +++ b/erts/emulator/beam/erl_gc.h @@ -3,16 +3,17 @@ * * Copyright Ericsson AB 2007-2011. All Rights Reserved. * - * The contents of this file are subject to the Erlang Public License, - * Version 1.1, (the "License"); you may not use this file except in - * compliance with the License. You should have received a copy of the - * Erlang Public License along with this software. If not, it can be - * retrieved online at http://www.erlang.org/. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Software distributed under the License is distributed on an "AS IS" - * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See - * the License for the specific language governing rights and limitations - * under the License. + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. * * %CopyrightEnd% */ @@ -20,10 +21,12 @@ #ifndef __ERL_GC_H__ #define __ERL_GC_H__ -#include "erl_map.h" +#if defined(ERL_WANT_GC_INTERNALS__) || defined(ERTS_DO_INCL_GLB_INLINE_FUNC_DEF) /* GC declarations shared by beam/erl_gc.c and hipe/hipe_gc.c */ +#include "erl_map.h" + #if defined(DEBUG) && !ERTS_GLB_INLINE_INCL_FUNC_DEF # define HARDDEBUG 1 #endif @@ -53,7 +56,10 @@ do { \ nelts = header_arity(HDR); \ switch ((HDR) & _HEADER_SUBTAG_MASK) { \ case SUB_BINARY_SUBTAG: nelts++; break; \ - case MAP_SUBTAG: nelts+=map_get_size(PTR) + 1; break; \ + case MAP_SUBTAG: \ + if (is_flatmap_header(HDR)) nelts+=flatmap_get_size(PTR) + 1; \ + else nelts += hashmap_bitcount(MAP_HEADER_VAL(HDR)); \ + break; \ case FUN_SUBTAG: nelts+=((ErlFunThing*)(PTR))->num_free+1; break; \ } \ gval = make_boxed(HTOP); \ @@ -61,22 +67,20 @@ do { \ *HTOP++ = HDR; \ *PTR++ = gval; \ while (nelts--) *HTOP++ = *PTR++; \ - \ } while(0) -#define in_area(ptr,start,nbytes) \ - ((UWord)((char*)(ptr) - (char*)(start)) < (nbytes)) - -extern Uint erts_test_long_gc_sleep; - #if defined(DEBUG) || defined(ERTS_OFFHEAP_DEBUG) int within(Eterm *ptr, Process *p); #endif -ERTS_GLB_INLINE Eterm follow_moved(Eterm term); +#define ErtsInYoungGen(TPtr, Ptr, OldHeap, OldHeapSz) \ + (!erts_is_literal((TPtr), (Ptr)) \ + & !ErtsInArea((Ptr), (OldHeap), (OldHeapSz))) + +ERTS_GLB_INLINE Eterm follow_moved(Eterm term, Eterm xptr_tag); #if ERTS_GLB_INLINE_INCL_FUNC_DEF -ERTS_GLB_INLINE Eterm follow_moved(Eterm term) +ERTS_GLB_INLINE Eterm follow_moved(Eterm term, Eterm xptr_tag) { Eterm* ptr; switch (primary_tag(term)) { @@ -84,17 +88,72 @@ ERTS_GLB_INLINE Eterm follow_moved(Eterm term) break; case TAG_PRIMARY_BOXED: ptr = boxed_val(term); - if (IS_MOVED_BOXED(*ptr)) term = *ptr; + if (IS_MOVED_BOXED(*ptr)) term = (*ptr) | xptr_tag; break; case TAG_PRIMARY_LIST: ptr = list_val(term); - if (IS_MOVED_CONS(ptr[0])) term = ptr[1]; + if (IS_MOVED_CONS(ptr[0])) term = (ptr[1]) | xptr_tag; break; default: ASSERT(!"strange tag in follow_moved"); } return term; } + #endif +#endif /* ERL_GC_C__ || HIPE_GC_C__ */ + +/* + * Global exported + */ + +#define ERTS_IS_GC_DESIRED_INTERNAL(Proc, HTop, STop) \ + ((((STop) - (HTop) < (Proc)->mbuf_sz)) \ + | ((Proc)->off_heap.overhead > (Proc)->bin_vheap_sz) \ + | !!((Proc)->flags & F_FORCE_GC)) + +#define ERTS_IS_GC_DESIRED(Proc) \ + ERTS_IS_GC_DESIRED_INTERNAL((Proc), (Proc)->htop, (Proc)->stop) + +#define ERTS_FORCE_GC_INTERNAL(Proc, FCalls) \ + do { \ + (Proc)->flags |= F_FORCE_GC; \ + ERTS_VBUMP_ALL_REDS_INTERNAL((Proc), (FCalls)); \ + } while (0) + +#define ERTS_FORCE_GC(Proc) \ + ERTS_FORCE_GC_INTERNAL((Proc), (Proc)->fcalls) + +extern Uint erts_test_long_gc_sleep; + +typedef struct { + Uint64 reclaimed; + Uint64 garbage_cols; +} ErtsGCInfo; + +#define ERTS_PROCESS_GC_INFO_MAX_TERMS (11) /* number of elements in process_gc_info*/ +#define ERTS_PROCESS_GC_INFO_MAX_SIZE \ + (ERTS_PROCESS_GC_INFO_MAX_TERMS * (2/*cons*/ + 3/*2-tuple*/ + BIG_UINT_HEAP_SIZE)) +Eterm erts_process_gc_info(struct process*, Uint *, Eterm **); + +void erts_gc_info(ErtsGCInfo *gcip); +void erts_init_gc(void); +int erts_garbage_collect_nobump(struct process*, int, Eterm*, int); +void erts_garbage_collect(struct process*, int, Eterm*, int); +void erts_garbage_collect_hibernate(struct process* p); +Eterm erts_gc_after_bif_call_lhf(struct process* p, ErlHeapFragment *live_hf_end, + Eterm result, Eterm* regs, Uint arity); +Eterm erts_gc_after_bif_call(struct process* p, Eterm result, Eterm* regs, Uint arity); +void erts_garbage_collect_literals(struct process* p, Eterm* literals, + Uint lit_size, + struct erl_off_heap_header* oh); +Uint erts_next_heap_size(Uint, Uint); +Eterm erts_heap_sizes(struct process* p); + +void erts_offset_off_heap(struct erl_off_heap*, Sint, Eterm*, Eterm*); +void erts_offset_heap_ptr(Eterm*, Uint, Sint, Eterm*, Eterm*); +void erts_offset_heap(Eterm*, Uint, Sint, Eterm*, Eterm*); +void erts_free_heap_frags(struct process* p); + #endif /* __ERL_GC_H__ */ diff --git a/erts/emulator/beam/erl_goodfit_alloc.c b/erts/emulator/beam/erl_goodfit_alloc.c index e9d8249ee1..9b4aad9d91 100644 --- a/erts/emulator/beam/erl_goodfit_alloc.c +++ b/erts/emulator/beam/erl_goodfit_alloc.c @@ -3,16 +3,17 @@ * * Copyright Ericsson AB 2003-2013. All Rights Reserved. * - * The contents of this file are subject to the Erlang Public License, - * Version 1.1, (the "License"); you may not use this file except in - * compliance with the License. You should have received a copy of the - * Erlang Public License along with this software. If not, it can be - * retrieved online at http://www.erlang.org/. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Software distributed under the License is distributed on an "AS IS" - * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See - * the License for the specific language governing rights and limitations - * under the License. + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. * * %CopyrightEnd% */ @@ -570,8 +571,8 @@ info_options(Allctr_t *allctr, if (hpp || szp) { if (!atoms_initialized) - erl_exit(1, "%s:%d: Internal error: Atoms not initialized", - __FILE__, __LINE__);; + erts_exit(ERTS_ERROR_EXIT, "%s:%d: Internal error: Atoms not initialized", + __FILE__, __LINE__); res = NIL; add_2tup(hpp, szp, &res, am.as, am.gf); diff --git a/erts/emulator/beam/erl_goodfit_alloc.h b/erts/emulator/beam/erl_goodfit_alloc.h index 385de0da23..ababdbd0a1 100644 --- a/erts/emulator/beam/erl_goodfit_alloc.h +++ b/erts/emulator/beam/erl_goodfit_alloc.h @@ -3,16 +3,17 @@ * * Copyright Ericsson AB 2003-2013. All Rights Reserved. * - * The contents of this file are subject to the Erlang Public License, - * Version 1.1, (the "License"); you may not use this file except in - * compliance with the License. You should have received a copy of the - * Erlang Public License along with this software. If not, it can be - * retrieved online at http://www.erlang.org/. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Software distributed under the License is distributed on an "AS IS" - * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See - * the License for the specific language governing rights and limitations - * under the License. + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. * * %CopyrightEnd% */ diff --git a/erts/emulator/beam/erl_hl_timer.c b/erts/emulator/beam/erl_hl_timer.c new file mode 100644 index 0000000000..491d2f8c84 --- /dev/null +++ b/erts/emulator/beam/erl_hl_timer.c @@ -0,0 +1,3173 @@ +/* + * %CopyrightBegin% + * + * Copyright Ericsson AB 2015. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * %CopyrightEnd% + */ + +/* + * Description: High level timers implementing BIF timers + * as well as process and port timers. + * + * Author: Rickard Green + */ + +#ifdef HAVE_CONFIG_H +# include "config.h" +#endif + +#include "sys.h" +#include "global.h" +#include "bif.h" +#include "erl_bif_unique.h" +#define ERTS_WANT_TIMER_WHEEL_API +#include "erl_time.h" +#include "erl_hl_timer.h" + +#define ERTS_TMR_CHECK_CANCEL_ON_CREATE 0 + +#if 0 +# define ERTS_HLT_HARD_DEBUG +#endif +#if 0 +# define ERTS_HLT_DEBUG +#endif + +#if defined(ERTS_HLT_HARD_DEBUG) || defined(DEBUG) +# if defined(ERTS_HLT_HARD_DEBUG) +# undef ERTS_RBT_HARD_DEBUG +# define ERTS_RBT_HARD_DEBUG 1 +# endif +# ifndef ERTS_HLT_DEBUG +# define ERTS_HLT_DEBUG 1 +# endif +#endif + +#undef ERTS_HLT_ASSERT +#if defined(ERTS_HLT_DEBUG) +# define ERTS_HLT_ASSERT(E) ERTS_ASSERT(E) +# undef ERTS_RBT_DEBUG +# define ERTS_RBT_DEBUG +#else +# define ERTS_HLT_ASSERT(E) ((void) 1) +#endif + +#if defined(ERTS_HLT_HARD_DEBUG) && defined(__GNUC__) +#warning "* * * * * * * * * * * * * * * * * *" +#warning "* ERTS_HLT_HARD_DEBUG IS ENABLED! *" +#warning "* * * * * * * * * * * * * * * * * *" +#endif + +#ifdef ERTS_HLT_HARD_DEBUG +# define ERTS_HLT_HDBG_CHK_SRV(SRV) hdbg_chk_srv((SRV)) +static void hdbg_chk_srv(ErtsHLTimerService *srv); +#else +# define ERTS_HLT_HDBG_CHK_SRV(SRV) ((void) 1) +#endif + +#if ERTS_REF_NUMBERS != 3 +#error "ERTS_REF_NUMBERS changed. Update me..." +#endif + +typedef enum { + ERTS_TMR_BIF, + ERTS_TMR_PROC, + ERTS_TMR_PORT, + ERTS_TMR_CALLBACK +} ErtsTmrType; + +#define ERTS_BIF_TIMER_SHORT_TIME 5000 + +#ifdef ERTS_SMP +# define ERTS_HLT_SMP_MEMBAR_LoadLoad_LoadStore \ + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore) +#else +# define ERTS_HLT_SMP_MEMBAR_LoadLoad_LoadStore +#endif + +/* Bit 0 to 9 contains scheduler id (see mask below) */ +#define ERTS_TMR_ROFLG_HLT (((Uint32) 1) << 10) +#define ERTS_TMR_ROFLG_BIF_TMR (((Uint32) 1) << 11) +#define ERTS_TMR_ROFLG_PRE_ALC (((Uint32) 1) << 12) +#define ERTS_TMR_ROFLG_REG_NAME (((Uint32) 1) << 13) +#define ERTS_TMR_ROFLG_PROC (((Uint32) 1) << 14) +#define ERTS_TMR_ROFLG_PORT (((Uint32) 1) << 15) +#define ERTS_TMR_ROFLG_CALLBACK (((Uint32) 1) << 16) +#ifdef ERTS_BTM_ACCESSOR_SUPPORT +#define ERTS_TMR_ROFLG_ABIF_TMR (((Uint32) 1) << 17) +#endif + +#define ERTS_TMR_ROFLG_SID_MASK \ + (ERTS_TMR_ROFLG_HLT - (Uint32) 1) + +#define ERTS_TMR_STATE_ACTIVE ((erts_aint32_t) 0) +#define ERTS_TMR_STATE_CANCELED ((erts_aint32_t) 1) +#define ERTS_TMR_STATE_TIMED_OUT ((erts_aint32_t) 2) + +typedef struct ErtsHLTimer_ ErtsHLTimer; + +#define ERTS_HLT_PFLG_RED (((UWord) 1) << 0) +#define ERTS_HLT_PFLG_SAME_TIME (((UWord) 1) << 1) + +#define ERTS_HLT_PFLGS_MASK \ + (ERTS_HLT_PFLG_RED|ERTS_HLT_PFLG_SAME_TIME) + +#define ERTS_HLT_PFIELD_NOT_IN_TABLE (~((UWord) 0)) + +typedef struct { + UWord parent; /* parent pointer and flags... */ + union { + struct { + ErtsHLTimer *right; + ErtsHLTimer *left; + } t; + struct { + ErtsHLTimer *prev; + ErtsHLTimer *next; + } l; + } u; + ErtsHLTimer *same_time; +} ErtsHLTimerTimeTree; + +typedef struct { + UWord parent; /* parent pointer and flags... */ + ErtsHLTimer *right; + ErtsHLTimer *left; +} ErtsHLTimerTree; + +typedef struct { + Uint32 roflgs; + erts_smp_atomic32_t refc; + union { + void *arg; + erts_atomic_t next; + } u; +} ErtsTmrHead; + +struct ErtsHLTimer_ { + ErtsTmrHead head; /* NEED to be first! */ + union { + ErtsThrPrgrLaterOp cleanup; + ErtsHLTimerTimeTree tree; + } time; + ErtsMonotonicTime timeout; + union { + Process *proc; + Port *port; + Eterm name; + void (*callback)(void *); + } receiver; + +#ifdef ERTS_HLT_HARD_DEBUG + int pending_timeout; +#endif + + erts_smp_atomic32_t state; + + /* BIF timer only fields follow... */ + struct { + Uint32 refn[ERTS_REF_NUMBERS]; + ErtsHLTimerTree proc_tree; + ErtsHLTimerTree tree; + Eterm message; + ErlHeapFragment *bp; + } btm; +#ifdef ERTS_BTM_ACCESSOR_SUPPORT + struct { + Eterm accessor; + ErtsHLTimerTree tree; + } abtm; +#endif +}; + +#define ERTS_HL_PTIMER_SIZE offsetof(ErtsHLTimer, btm) +#ifdef ERTS_BTM_ACCESSOR_SUPPORT +#define ERTS_BIF_TIMER_SIZE offsetof(ErtsHLTimer, abtm) +#define ERTS_ABIF_TIMER_SIZE sizeof(ErtsHLTimer) +#else +#define ERTS_BIF_TIMER_SIZE sizeof(ErtsHLTimer) +#endif + +typedef struct { + ErtsTmrHead head; /* NEED to be first! */ + union { + void *p; + void (*callback)(void *); + } u; + ErtsTWheelTimer tw_tmr; +} ErtsTWTimer; + +typedef union { + ErtsTmrHead head; + ErtsHLTimer hlt; + ErtsTWTimer twt; +} ErtsTimer; + +#ifdef SMALL_MEMORY +#define BIF_TIMER_PREALC_SZ 10 +#define PTIMER_PREALC_SZ 10 +#else +#define BIF_TIMER_PREALC_SZ 100 +#define PTIMER_PREALC_SZ 100 +#endif + +ERTS_SCHED_PREF_PALLOC_IMPL(bif_timer_pre, + ErtsHLTimer, + BIF_TIMER_PREALC_SZ) + +ERTS_SCHED_PREF_QUICK_ALLOC_IMPL(tw_timer, + ErtsTWTimer, + PTIMER_PREALC_SZ, + ERTS_ALC_T_LL_PTIMER) + +#ifdef ERTS_HLT_DEBUG +#define ERTS_TMR_TIMEOUT_YIELD_LIMIT 5 +#else +#define ERTS_TMR_TIMEOUT_YIELD_LIMIT 100 +#endif +#define ERTS_TMR_CANCELED_TIMER_LIMIT 100 +#define ERTS_TMR_CANCELED_TIMER_SMALL_LIMIT 5 + +#define ERTS_TMR_TIMEOUT_YIELD_STATE_T same_time_list_yield_state_t +#define ERTS_TMR_YIELDING_TIMEOUT_STATE_INITER {NULL, {0}} +typedef struct { + int dummy; +} ERTS_TMR_TIMEOUT_YIELD_STATE_T; + +typedef struct { + ErtsTmrHead marker; + erts_atomic_t last; +} ErtsHLTCncldTmrQTail; + +#ifdef ERTS_SMP + +typedef struct { + /* + * This structure needs to be cache line aligned for best + * performance. + */ + union { + /* + * Modified by threads returning canceled + * timers to this timer service. + */ + ErtsHLTCncldTmrQTail data; + char align__[ERTS_ALC_CACHE_LINE_ALIGN_SIZE( + sizeof(ErtsHLTCncldTmrQTail))]; + } tail; + /* + * Everything below this point is *only* accessed by the + * thread managing this timer service. + */ + struct { + ErtsTimer *first; + ErtsTimer *unref_end; + struct { + ErtsThrPrgrVal thr_progress; + int thr_progress_reached; + ErtsTimer *unref_end; + } next; + int used_marker; + } head; +} ErtsHLTCncldTmrQ; + +#endif /* ERTS_SMP */ + +typedef struct { + ErtsHLTimer *root; + ERTS_TMR_TIMEOUT_YIELD_STATE_T state; +} ErtsYieldingTimeoutState; + +struct ErtsHLTimerService_ { +#ifdef ERTS_SMP + ErtsHLTCncldTmrQ canceled_queue; +#endif + ErtsHLTimer *time_tree; + ErtsHLTimer *btm_tree; + ErtsHLTimer *next_timeout; + ErtsYieldingTimeoutState yield; + ErtsTWheelTimer service_timer; +}; + +static ERTS_INLINE int +refn_is_lt(Uint32 *x, Uint32 *y) +{ + /* !0 if x < y */ + if (x[2] < y[2]) + return 1; + if (x[2] != y[2]) + return 0; + if (x[1] < y[1]) + return 1; + if (x[1] != y[1]) + return 0; + return x[0] < y[0]; +} + +#define ERTS_RBT_PREFIX time +#define ERTS_RBT_T ErtsHLTimer +#define ERTS_RBT_KEY_T ErtsMonotonicTime +#define ERTS_RBT_FLAGS_T UWord +#define ERTS_RBT_INIT_EMPTY_TNODE(T) \ + do { \ + (T)->time.tree.parent = (UWord) NULL; \ + (T)->time.tree.u.t.right = NULL; \ + (T)->time.tree.u.t.left = NULL; \ + } while (0) +#define ERTS_RBT_IS_RED(T) \ + ((int) ((T)->time.tree.parent & ERTS_HLT_PFLG_RED)) +#define ERTS_RBT_SET_RED(T) \ + ((T)->time.tree.parent |= ERTS_HLT_PFLG_RED) +#define ERTS_RBT_IS_BLACK(T) \ + (!ERTS_RBT_IS_RED((T))) +#define ERTS_RBT_SET_BLACK(T) \ + ((T)->time.tree.parent &= ~ERTS_HLT_PFLG_RED) +#define ERTS_RBT_GET_FLAGS(T) \ + ((T)->time.tree.parent & ERTS_HLT_PFLGS_MASK) +#define ERTS_RBT_SET_FLAGS(T, F) \ + do { \ + ERTS_HLT_ASSERT((((UWord) (F)) & ~ERTS_HLT_PFLGS_MASK) == 0); \ + (T)->time.tree.parent &= ~ERTS_HLT_PFLGS_MASK; \ + (T)->time.tree.parent |= (F); \ + } while (0) +#define ERTS_RBT_GET_PARENT(T) \ + ((ErtsHLTimer *) ((T)->time.tree.parent & ~ERTS_HLT_PFLGS_MASK)) +#define ERTS_RBT_SET_PARENT(T, P) \ + do { \ + ERTS_HLT_ASSERT((((UWord) (P)) & ERTS_HLT_PFLGS_MASK) == 0); \ + (T)->time.tree.parent &= ERTS_HLT_PFLGS_MASK; \ + (T)->time.tree.parent |= (UWord) (P); \ + } while (0) +#define ERTS_RBT_GET_RIGHT(T) ((T)->time.tree.u.t.right) +#define ERTS_RBT_SET_RIGHT(T, R) ((T)->time.tree.u.t.right = (R)) +#define ERTS_RBT_GET_LEFT(T) ((T)->time.tree.u.t.left) +#define ERTS_RBT_SET_LEFT(T, L) ((T)->time.tree.u.t.left = (L)) +#define ERTS_RBT_GET_KEY(T) ((T)->timeout) +#define ERTS_RBT_IS_LT(KX, KY) ((KX) < (KY)) +#define ERTS_RBT_IS_EQ(KX, KY) ((KX) == (KY)) +#define ERTS_RBT_WANT_DELETE +#define ERTS_RBT_WANT_SMALLEST +#define ERTS_RBT_WANT_LOOKUP_INSERT +#define ERTS_RBT_WANT_REPLACE +#define ERTS_RBT_WANT_FOREACH +#ifdef ERTS_HLT_HARD_DEBUG +# define ERTS_RBT_WANT_LOOKUP +#endif +#define ERTS_RBT_UNDEF + +#include "erl_rbtree.h" + +/* Use circular list for timers at same time */ + +static ERTS_INLINE void +same_time_list_insert(ErtsHLTimer **root, ErtsHLTimer *tmr) +{ + ErtsHLTimer *first = *root; + if (!first) { + ERTS_HLT_ASSERT((((UWord) root) & ERTS_HLT_PFLG_SAME_TIME) == 0); + tmr->time.tree.parent = ((UWord) root) | ERTS_HLT_PFLG_SAME_TIME; + tmr->time.tree.u.l.next = tmr; + tmr->time.tree.u.l.prev = tmr; + *root = tmr; + } + else { + tmr->time.tree.parent = ERTS_HLT_PFLG_SAME_TIME; + tmr->time.tree.u.l.next = first; + tmr->time.tree.u.l.prev = first->time.tree.u.l.prev; + first->time.tree.u.l.prev = tmr; + tmr->time.tree.u.l.prev->time.tree.u.l.next = tmr; + } +} + +static ERTS_INLINE void +same_time_list_delete(ErtsHLTimer *tmr) +{ + ErtsHLTimer **root, *next; + + root = (ErtsHLTimer **) (tmr->time.tree.parent & ~ERTS_HLT_PFLG_SAME_TIME); + next = tmr->time.tree.u.l.next; + + ERTS_HLT_ASSERT((tmr->time.tree.parent + == (((UWord) root) | ERTS_HLT_PFLG_SAME_TIME)) + || (tmr->time.tree.parent + == ERTS_HLT_PFLG_SAME_TIME)); + + if (next == tmr) { + ERTS_HLT_ASSERT(root && *root == tmr); + ERTS_HLT_ASSERT(tmr->time.tree.u.l.prev == tmr); + *root = NULL; + } + else { + if (root) { + ERTS_HLT_ASSERT(*root == tmr); + *root = next; + next->time.tree.parent = ((UWord) root) | ERTS_HLT_PFLG_SAME_TIME; + } + tmr->time.tree.u.l.next->time.tree.u.l.prev = tmr->time.tree.u.l.prev; + tmr->time.tree.u.l.prev->time.tree.u.l.next = next; + } +} + +static ERTS_INLINE void +same_time_list_new_root(ErtsHLTimer **root) +{ + ErtsHLTimer *tmr = *root; + if (tmr) { + ERTS_HLT_ASSERT(root); + tmr->time.tree.parent = ((UWord) root) | ERTS_HLT_PFLG_SAME_TIME; + } +} + +static ERTS_INLINE int +same_time_list_foreach_destroy_yielding(ErtsHLTimer **root, + void (*op)(ErtsHLTimer *, void *), + void *arg, + ERTS_TMR_TIMEOUT_YIELD_STATE_T *ys, + Sint ylimit) +{ + Sint ycnt = ylimit; + ErtsHLTimer *end, *tmr = *root; + if (!tmr) + return 0; + + ERTS_HLT_ASSERT(tmr->time.tree.parent + == (((UWord) root) | ERTS_HLT_PFLG_SAME_TIME)); + + end = tmr->time.tree.u.l.prev; + end->time.tree.u.l.next = NULL; + + while (1) { + ErtsHLTimer *op_tmr = tmr; + + ERTS_HLT_ASSERT((tmr->time.tree.parent + == (((UWord) root) | ERTS_HLT_PFLG_SAME_TIME)) + || (tmr->time.tree.parent + == ERTS_HLT_PFLG_SAME_TIME)); + + tmr = tmr->time.tree.u.l.next; + (*op)(op_tmr, arg); + if (!tmr) { + *root = NULL; + return 0; + } + if (--ycnt <= 0) { + /* Make new circle of timers left to process... */ + *root = tmr; + end->time.tree.u.l.next = tmr; + tmr->time.tree.u.l.prev = end; + tmr->time.tree.parent = ((UWord) root) | ERTS_HLT_PFLG_SAME_TIME; + return 1; + } + } +} + +static ERTS_INLINE void +same_time_list_foreach(ErtsHLTimer *root, + void (*op)(ErtsHLTimer *, void *), + void *arg) +{ + if (root) { + ErtsHLTimer *tmr = root; + do { + (*op)(tmr, arg); + tmr = tmr->time.tree.u.l.next; + } while (root != tmr); + } +} + +#ifdef ERTS_HLT_HARD_DEBUG + +static ERTS_INLINE ErtsHLTimer * +same_time_list_lookup(ErtsHLTimer *root, ErtsHLTimer *x) +{ + if (root) { + ErtsHLTimer *tmr = root; + do { + if (tmr == x) + return tmr; + tmr = tmr->time.tree.u.l.next; + } while (root != tmr); + } + return NULL; +} + +#endif /* ERTS_HLT_HARD_DEBUG */ + +#define ERTS_RBT_PREFIX btm +#define ERTS_RBT_T ErtsHLTimer +#define ERTS_RBT_KEY_T Uint32 * +#define ERTS_RBT_FLAGS_T UWord +#define ERTS_RBT_INIT_EMPTY_TNODE(T) \ + do { \ + (T)->btm.tree.parent = (UWord) NULL; \ + (T)->btm.tree.right = NULL; \ + (T)->btm.tree.left = NULL; \ + } while (0) +#define ERTS_RBT_IS_RED(T) \ + ((int) ((T)->btm.tree.parent & ERTS_HLT_PFLG_RED)) +#define ERTS_RBT_SET_RED(T) \ + ((T)->btm.tree.parent |= ERTS_HLT_PFLG_RED) +#define ERTS_RBT_IS_BLACK(T) \ + (!ERTS_RBT_IS_RED((T))) +#define ERTS_RBT_SET_BLACK(T) \ + ((T)->btm.tree.parent &= ~ERTS_HLT_PFLG_RED) +#define ERTS_RBT_GET_FLAGS(T) \ + ((T)->btm.tree.parent & ERTS_HLT_PFLGS_MASK) +#define ERTS_RBT_SET_FLAGS(T, F) \ + do { \ + ERTS_HLT_ASSERT((((UWord) (F)) & ~ERTS_HLT_PFLGS_MASK) == 0); \ + (T)->btm.tree.parent &= ~ERTS_HLT_PFLGS_MASK; \ + (T)->btm.tree.parent |= (F); \ + } while (0) +#define ERTS_RBT_GET_PARENT(T) \ + ((ErtsHLTimer *) ((T)->btm.tree.parent & ~ERTS_HLT_PFLGS_MASK)) +#define ERTS_RBT_SET_PARENT(T, P) \ + do { \ + ERTS_HLT_ASSERT((((UWord) (P)) & ERTS_HLT_PFLGS_MASK) == 0); \ + (T)->btm.tree.parent &= ERTS_HLT_PFLGS_MASK; \ + (T)->btm.tree.parent |= (UWord) (P); \ + } while (0) +#define ERTS_RBT_GET_RIGHT(T) ((T)->btm.tree.right) +#define ERTS_RBT_SET_RIGHT(T, R) ((T)->btm.tree.right = (R)) +#define ERTS_RBT_GET_LEFT(T) ((T)->btm.tree.left) +#define ERTS_RBT_SET_LEFT(T, L) ((T)->btm.tree.left = (L)) +#define ERTS_RBT_GET_KEY(T) ((T)->btm.refn) +#define ERTS_RBT_IS_LT(KX, KY) refn_is_lt((KX), (KY)) +#define ERTS_RBT_IS_EQ(KX, KY) \ + (((KX)[0] == (KY)[0]) & ((KX)[1] == (KY)[1]) & ((KX)[2] == (KY)[2])) +#define ERTS_RBT_WANT_DELETE +#define ERTS_RBT_WANT_INSERT +#define ERTS_RBT_WANT_LOOKUP +#define ERTS_RBT_WANT_FOREACH +#define ERTS_RBT_UNDEF + +#include "erl_rbtree.h" + +#define ERTS_RBT_PREFIX proc_btm +#define ERTS_RBT_T ErtsHLTimer +#define ERTS_RBT_KEY_T Uint32 * +#define ERTS_RBT_FLAGS_T UWord +#define ERTS_RBT_INIT_EMPTY_TNODE(T) \ + do { \ + (T)->btm.proc_tree.parent = (UWord) NULL; \ + (T)->btm.proc_tree.right = NULL; \ + (T)->btm.proc_tree.left = NULL; \ + } while (0) +#define ERTS_RBT_IS_RED(T) \ + ((int) ((T)->btm.proc_tree.parent & ERTS_HLT_PFLG_RED)) +#define ERTS_RBT_SET_RED(T) \ + ((T)->btm.proc_tree.parent |= ERTS_HLT_PFLG_RED) +#define ERTS_RBT_IS_BLACK(T) \ + (!ERTS_RBT_IS_RED((T))) +#define ERTS_RBT_SET_BLACK(T) \ + ((T)->btm.proc_tree.parent &= ~ERTS_HLT_PFLG_RED) +#define ERTS_RBT_GET_FLAGS(T) \ + ((T)->btm.proc_tree.parent & ERTS_HLT_PFLGS_MASK) +#define ERTS_RBT_SET_FLAGS(T, F) \ + do { \ + ERTS_HLT_ASSERT((((UWord) (F)) & ~ERTS_HLT_PFLGS_MASK) == 0); \ + (T)->btm.proc_tree.parent &= ~ERTS_HLT_PFLGS_MASK; \ + (T)->btm.proc_tree.parent |= (F); \ + } while (0) +#define ERTS_RBT_GET_PARENT(T) \ + ((ErtsHLTimer *) ((T)->btm.proc_tree.parent & ~ERTS_HLT_PFLGS_MASK)) +#define ERTS_RBT_SET_PARENT(T, P) \ + do { \ + ERTS_HLT_ASSERT((((UWord) (P)) & ERTS_HLT_PFLGS_MASK) == 0); \ + (T)->btm.proc_tree.parent &= ERTS_HLT_PFLGS_MASK; \ + (T)->btm.proc_tree.parent |= (UWord) (P); \ + } while (0) +#define ERTS_RBT_GET_RIGHT(T) ((T)->btm.proc_tree.right) +#define ERTS_RBT_SET_RIGHT(T, R) ((T)->btm.proc_tree.right = (R)) +#define ERTS_RBT_GET_LEFT(T) ((T)->btm.proc_tree.left) +#define ERTS_RBT_SET_LEFT(T, L) ((T)->btm.proc_tree.left = (L)) +#define ERTS_RBT_GET_KEY(T) ((T)->btm.refn) +#define ERTS_RBT_IS_LT(KX, KY) refn_is_lt((KX), (KY)) +#define ERTS_RBT_IS_EQ(KX, KY) \ + (((KX)[0] == (KY)[0]) & ((KX)[1] == (KY)[1]) & ((KX)[2] == (KY)[2])) +#define ERTS_RBT_WANT_DELETE +#define ERTS_RBT_WANT_INSERT +#define ERTS_RBT_WANT_LOOKUP +#define ERTS_RBT_WANT_FOREACH_DESTROY_YIELDING +#define ERTS_RBT_UNDEF + +#include "erl_rbtree.h" + +#ifdef ERTS_BTM_ACCESSOR_SUPPORT + +#define ERTS_RBT_PREFIX abtm +#define ERTS_RBT_T ErtsHLTimer +#define ERTS_RBT_KEY_T Uint32 * +#define ERTS_RBT_FLAGS_T UWord +#define ERTS_RBT_INIT_EMPTY_TNODE(T) \ + do { \ + (T)->abtm.tree.parent = (UWord) NULL; \ + (T)->abtm.tree.right = NULL; \ + (T)->abtm.tree.left = NULL; \ + } while (0) +#define ERTS_RBT_IS_RED(T) \ + ((int) ((T)->abtm.tree.parent & ERTS_HLT_PFLG_RED)) +#define ERTS_RBT_SET_RED(T) \ + ((T)->abtm.tree.parent |= ERTS_HLT_PFLG_RED) +#define ERTS_RBT_IS_BLACK(T) \ + (!ERTS_RBT_IS_RED((T))) +#define ERTS_RBT_SET_BLACK(T) \ + ((T)->abtm.tree.parent &= ~ERTS_HLT_PFLG_RED) +#define ERTS_RBT_GET_FLAGS(T) \ + ((T)->abtm.tree.parent & ERTS_HLT_PFLGS_MASK) +#define ERTS_RBT_SET_FLAGS(T, F) \ + do { \ + ERTS_HLT_ASSERT((((UWord) (F)) & ~ERTS_HLT_PFLGS_MASK) == 0); \ + (T)->abtm.tree.parent &= ~ERTS_HLT_PFLGS_MASK; \ + (T)->abtm.tree.parent |= (F); \ + } while (0) +#define ERTS_RBT_GET_PARENT(T) \ + ((ErtsHLTimer *) ((T)->abtm.tree.parent & ~ERTS_HLT_PFLGS_MASK)) +#define ERTS_RBT_SET_PARENT(T, P) \ + do { \ + ERTS_HLT_ASSERT((((UWord) (P)) & ERTS_HLT_PFLGS_MASK) == 0); \ + (T)->abtm.tree.parent &= ERTS_HLT_PFLGS_MASK; \ + (T)->abtm.tree.parent |= (UWord) (P); \ + } while (0) +#define ERTS_RBT_GET_RIGHT(T) ((T)->abtm.tree.right) +#define ERTS_RBT_SET_RIGHT(T, R) ((T)->abtm.tree.right = (R)) +#define ERTS_RBT_GET_LEFT(T) ((T)->abtm.tree.left) +#define ERTS_RBT_SET_LEFT(T, L) ((T)->abtm.tree.left = (L)) +#define ERTS_RBT_GET_KEY(T) ((T)->btm.refn) +#define ERTS_RBT_IS_LT(KX, KY) refn_is_lt((KX), (KY)) +#define ERTS_RBT_IS_EQ(KX, KY) \ + (((KX)[0] == (KY)[0]) & ((KX)[1] == (KY)[1]) & ((KX)[2] == (KY)[2])) +#define ERTS_RBT_WANT_DELETE +#define ERTS_RBT_WANT_INSERT +#define ERTS_RBT_WANT_LOOKUP +#define ERTS_RBT_WANT_FOREACH_DESTROY_YIELDING +#define ERTS_RBT_UNDEF + +#include "erl_rbtree.h" + +#endif /* ERTS_BTM_ACCESSOR_SUPPORT */ + +#ifdef ERTS_SMP +static void init_canceled_queue(ErtsHLTCncldTmrQ *cq); +#endif + +void +erts_hl_timer_init(void) +{ + init_tw_timer_alloc(); + init_bif_timer_pre_alloc(); +} + +ErtsHLTimerService * +erts_create_timer_service(void) +{ + ErtsYieldingTimeoutState init_yield = ERTS_TMR_YIELDING_TIMEOUT_STATE_INITER; + ErtsHLTimerService *srv; + + srv = erts_alloc_permanent_cache_aligned(ERTS_ALC_T_TIMER_SERVICE, + sizeof(ErtsHLTimerService)); + srv->time_tree = NULL; + srv->btm_tree = NULL; + srv->next_timeout = NULL; + srv->yield = init_yield; + erts_twheel_init_timer(&srv->service_timer); + +#ifdef ERTS_SMP + init_canceled_queue(&srv->canceled_queue); +#endif + + return srv; +} + +size_t +erts_timer_type_size(ErtsAlcType_t type) +{ + switch (type) { + case ERTS_ALC_T_LL_PTIMER: return sizeof(ErtsTWTimer); + case ERTS_ALC_T_HL_PTIMER: return ERTS_HL_PTIMER_SIZE; + case ERTS_ALC_T_BIF_TIMER: return ERTS_BIF_TIMER_SIZE; +#ifdef ERTS_BTM_ACCESSOR_SUPPORT + case ERTS_ALC_T_ABIF_TIMER: return ERTS_ABIF_TIMER_SIZE; +#endif + default: ERTS_INTERNAL_ERROR("Unknown type"); + } + return 0; +} + +static ERTS_INLINE ErtsMonotonicTime +get_timeout_pos(ErtsMonotonicTime now, ErtsMonotonicTime msec) +{ + ErtsMonotonicTime timeout_pos; + if (msec <= 0) + return ERTS_MONOTONIC_TO_CLKTCKS(now); + timeout_pos = ERTS_MONOTONIC_TO_CLKTCKS(now-1); + timeout_pos += ERTS_MSEC_TO_CLKTCKS(msec) + 1; + return timeout_pos; +} + +static ERTS_INLINE Sint64 +get_time_left(ErtsSchedulerData *esdp, ErtsMonotonicTime timeout_pos) +{ + ErtsMonotonicTime now = erts_get_monotonic_time(esdp); + + now = ERTS_MONOTONIC_TO_CLKTCKS(now-1)+1; + if (timeout_pos <= now) + return (Sint64) 0; + return (Sint64) ERTS_CLKTCKS_TO_MSEC(timeout_pos - now); +} + +static ERTS_INLINE int +proc_timeout_common(Process *proc, void *tmr) +{ + if (tmr == (void *) erts_smp_atomic_cmpxchg_mb(&proc->common.timer, + ERTS_PTMR_TIMEDOUT, + (erts_aint_t) tmr)) { + erts_aint32_t state = erts_smp_atomic32_read_acqb(&proc->state); + if (!(state & (ERTS_PSFLG_ACTIVE|ERTS_PSFLG_EXITING))) + erts_schedule_process(proc, state, 0); + return 1; + } + return 0; +} + +static ERTS_INLINE int +port_timeout_common(Port *port, void *tmr) +{ + if (tmr == (void *) erts_smp_atomic_cmpxchg_mb(&port->common.timer, + ERTS_PTMR_TIMEDOUT, + (erts_aint_t) tmr)) { + erts_port_task_schedule(port->common.id, + &port->timeout_task, + ERTS_PORT_TASK_TIMEOUT); + return 1; + } + return 0; +} + +/* + * Basic timer wheel timer stuff + */ + +static void +scheduled_tw_timer_destroy(void *vtmr) +{ + tw_timer_free((ErtsTWTimer *) vtmr); +} + +static void +schedule_tw_timer_destroy(ErtsTWTimer *tmr) +{ + /* + * Reference to process/port can be + * dropped at once... + */ + if (tmr->head.roflgs & ERTS_TMR_ROFLG_PROC) + erts_proc_dec_refc((Process *) tmr->u.p); + else if (tmr->head.roflgs & ERTS_TMR_ROFLG_PORT) + erts_port_dec_refc((Port *) tmr->u.p); + + erts_schedule_thr_prgr_later_cleanup_op( + scheduled_tw_timer_destroy, + (void *) tmr, + &tmr->tw_tmr.u.cleanup, + sizeof(ErtsTWTimer)); +} + +static ERTS_INLINE void +tw_timer_dec_refc(ErtsTWTimer *tmr) +{ + if (erts_smp_atomic32_dec_read_relb(&tmr->head.refc) == 0) { + ERTS_HLT_SMP_MEMBAR_LoadLoad_LoadStore; + schedule_tw_timer_destroy(tmr); + } +} + +static void +tw_proc_timeout(void *vtwtp) +{ + ErtsTWTimer *twtp = (ErtsTWTimer *) vtwtp; + Process *proc = (Process *) twtp->u.p; + if (proc_timeout_common(proc, vtwtp)) + tw_timer_dec_refc(twtp); + tw_timer_dec_refc(twtp); +} + +static void +tw_port_timeout(void *vtwtp) +{ + ErtsTWTimer *twtp = (ErtsTWTimer *) vtwtp; + Port *port = (Port *) twtp->u.p; + if (port_timeout_common(port, vtwtp)) + tw_timer_dec_refc(twtp); + tw_timer_dec_refc(twtp); +} + +static void +tw_ptimer_cancel(void *vtwtp) +{ + tw_timer_dec_refc((ErtsTWTimer *) vtwtp); +} + +static void +cancel_tw_timer(ErtsSchedulerData *esdp, ErtsTWTimer *tmr) +{ + ERTS_HLT_ASSERT((tmr->head.roflgs & ERTS_TMR_ROFLG_SID_MASK) + == (Uint32) esdp->no); + erts_twheel_cancel_timer(esdp->timer_wheel, &tmr->tw_tmr); +} + +static void +tw_callback_timeout(void *vtwtp) +{ + ErtsTWTimer *twtp = (ErtsTWTimer *) vtwtp; + void (*callback)(void *) = twtp->u.callback; + void *arg = twtp->head.u.arg; + tw_timer_dec_refc(twtp); + (*callback)(arg); +} + +static ErtsTWTimer * +create_tw_timer(ErtsSchedulerData *esdp, + ErtsTmrType type, void *p, + void (*callback)(void *), void *arg, + ErtsMonotonicTime timeout_pos) +{ + ErtsTWTimer *tmr; + void (*timeout_func)(void *); + void (*cancel_func)(void *); + erts_aint32_t refc; + + tmr = tw_timer_alloc(); + erts_twheel_init_timer(&tmr->tw_tmr); + + tmr->head.roflgs = (Uint32) esdp->no; + ERTS_HLT_ASSERT((tmr->head.roflgs + & ~ERTS_TMR_ROFLG_SID_MASK) == 0); + + switch (type) { + + case ERTS_TMR_PROC: + tmr->u.p = p; + tmr->head.roflgs |= ERTS_TMR_ROFLG_PROC; + timeout_func = tw_proc_timeout; + cancel_func = tw_ptimer_cancel; + erts_proc_inc_refc((Process *) p); + refc = 2; + break; + + case ERTS_TMR_PORT: + tmr->u.p = p; + tmr->head.roflgs |= ERTS_TMR_ROFLG_PORT; + timeout_func = tw_port_timeout; + cancel_func = tw_ptimer_cancel; + erts_port_inc_refc((Port *) p); + refc = 2; + break; + + case ERTS_TMR_CALLBACK: + tmr->head.u.arg = arg; + tmr->u.callback = callback; + + tmr->head.roflgs |= ERTS_TMR_ROFLG_CALLBACK; + timeout_func = tw_callback_timeout; + cancel_func = NULL; + refc = 1; + break; + + default: + ERTS_INTERNAL_ERROR("Unsupported timer type"); + return NULL; + } + + erts_smp_atomic32_init_nob(&tmr->head.refc, refc); + + erts_twheel_set_timer(esdp->timer_wheel, + &tmr->tw_tmr, + timeout_func, + cancel_func, + tmr, + timeout_pos); + + return tmr; +} + +/* + * Basic high level timer stuff + */ + +static ERTS_INLINE void +hl_timer_destroy(ErtsHLTimer *tmr) +{ + Uint32 roflgs = tmr->head.roflgs; + if (!(roflgs & ERTS_TMR_ROFLG_BIF_TMR)) + erts_free(ERTS_ALC_T_HL_PTIMER, tmr); + else { + if (roflgs & ERTS_TMR_ROFLG_PRE_ALC) + bif_timer_pre_free(tmr); +#ifdef ERTS_BTM_ACCESSOR_SUPPORT + else if (roflgs & ERTS_TMR_ROFLG_ABIF_TMR) + erts_free(ERTS_ALC_T_ABIF_TIMER, tmr); +#endif + else + erts_free(ERTS_ALC_T_BIF_TIMER, tmr); + } +} + +static void +scheduled_hl_timer_destroy(void *vtmr) +{ + hl_timer_destroy((ErtsHLTimer *) vtmr); +} + +static void +schedule_hl_timer_destroy(ErtsHLTimer *tmr, Uint32 roflgs) +{ + UWord size; + + /* + * Reference to process/port can be dropped + * at once... + */ + + ERTS_HLT_ASSERT(erts_smp_atomic32_read_nob(&tmr->head.refc) == 0); + + if (roflgs & ERTS_TMR_ROFLG_REG_NAME) { + ERTS_HLT_ASSERT(is_atom(tmr->receiver.name)); + } + else if (roflgs & ERTS_TMR_ROFLG_PROC) { + ERTS_HLT_ASSERT(tmr->receiver.proc); + erts_proc_dec_refc(tmr->receiver.proc); + } + else if (roflgs & ERTS_TMR_ROFLG_PORT) { + ERTS_HLT_ASSERT(tmr->receiver.port); + erts_port_dec_refc(tmr->receiver.port); + } + + if (!(roflgs & ERTS_TMR_ROFLG_BIF_TMR)) + size = ERTS_HL_PTIMER_SIZE; + else { + /* + * Message buffer can be dropped at + * once... + */ + size = sizeof(ErtsHLTimer); + } + + erts_schedule_thr_prgr_later_cleanup_op( + scheduled_hl_timer_destroy, tmr, + &tmr->time.cleanup, size); +} + +static ERTS_INLINE void +hl_timer_pre_dec_refc(ErtsHLTimer *tmr) +{ +#ifdef ERTS_HLT_DEBUG + erts_aint_t refc; + refc = erts_smp_atomic32_dec_read_nob(&tmr->head.refc); + ERTS_HLT_ASSERT(refc > 0); +#else + erts_smp_atomic32_dec_nob(&tmr->head.refc); +#endif +} + +static ERTS_INLINE void +hl_timer_dec_refc(ErtsHLTimer *tmr, Uint32 roflgs) +{ + if (erts_smp_atomic32_dec_read_relb(&tmr->head.refc) == 0) { + ERTS_HLT_SMP_MEMBAR_LoadLoad_LoadStore; + schedule_hl_timer_destroy(tmr, roflgs); + } +} + +static void hlt_service_timeout(void *vesdp); +#ifdef ERTS_SMP +static void handle_canceled_queue(ErtsSchedulerData *esdp, + ErtsHLTCncldTmrQ *cq, + int use_limit, + int ops_limit, + int *need_thr_progress, + ErtsThrPrgrVal *thr_prgr_p, + int *need_more_work); +#endif + +static ERTS_INLINE void +check_canceled_queue(ErtsSchedulerData *esdp, ErtsHLTimerService *srv) +{ +#if defined(ERTS_SMP) && ERTS_TMR_CHECK_CANCEL_ON_CREATE + ErtsHLTCncldTmrQ *cq = &srv->canceled_queue; + if (cq->head.first != cq->head.unref_end) + handle_canceled_queue(esdp, cq, 1, + ERTS_TMR_CANCELED_TIMER_SMALL_LIMIT, + NULL, NULL, NULL); +#endif +} + +#ifdef ERTS_BTM_ACCESSOR_SUPPORT + +static void +hlt_delete_abtm(ErtsHLTimer *tmr) +{ + Process *proc; + + ERTS_HLT_ASSERT(tmr->head.roflgs & ERTS_TMR_ROFLG_ABIF_TMR); + + proc = erts_proc_lookup(tmr->abtm.accessor); + + if (proc) { + int deref = 0; + erts_smp_proc_lock(proc, ERTS_PROC_LOCK_BTM); + if (tmr->abtm.tree.parent != ERTS_HLT_PFIELD_NOT_IN_TABLE) { + abtm_rbt_delete(&proc->accessor_bif_timers, tmr); + deref = 1; + tmr->abtm.tree.parent = ERTS_HLT_PFIELD_NOT_IN_TABLE; + } + erts_smp_proc_unlock(proc, ERTS_PROC_LOCK_BTM); + if (deref) + hl_timer_pre_dec_refc(tmr); + } +} + +#endif + +static ErtsHLTimer * +create_hl_timer(ErtsSchedulerData *esdp, + ErtsMonotonicTime timeout_pos, + int short_time, ErtsTmrType type, + void *rcvrp, Eterm rcvr, Eterm acsr, + Eterm msg, Uint32 *refn, + void (*callback)(void *), void *arg) +{ + ErtsHLTimerService *srv = esdp->timer_service; + ErtsHLTimer *tmr, *st_tmr; + erts_aint32_t refc; + Uint32 roflgs; + + ERTS_HLT_HDBG_CHK_SRV(srv); + + check_canceled_queue(esdp, srv); + + ERTS_HLT_ASSERT((esdp->no & ~ERTS_TMR_ROFLG_SID_MASK) == 0); + + roflgs = ((Uint32) esdp->no) | ERTS_TMR_ROFLG_HLT; + + if (type != ERTS_TMR_BIF) { + + tmr = erts_alloc(ERTS_ALC_T_HL_PTIMER, + ERTS_HL_PTIMER_SIZE); + tmr->timeout = timeout_pos; + + switch (type) { + + case ERTS_TMR_PROC: + ERTS_HLT_ASSERT(is_internal_pid(rcvr)); + + erts_proc_inc_refc((Process *) rcvrp); + tmr->receiver.proc = (Process *) rcvrp; + roflgs |= ERTS_TMR_ROFLG_PROC; + refc = 2; + break; + + case ERTS_TMR_PORT: + ERTS_HLT_ASSERT(is_internal_port(rcvr)); + erts_port_inc_refc((Port *) rcvrp); + tmr->receiver.port = (Port *) rcvrp; + roflgs |= ERTS_TMR_ROFLG_PORT; + refc = 2; + break; + + case ERTS_TMR_CALLBACK: + roflgs |= ERTS_TMR_ROFLG_CALLBACK; + tmr->receiver.callback = callback; + tmr->head.u.arg = arg; + refc = 1; + break; + + default: + ERTS_INTERNAL_ERROR("Unsupported timer type"); + return NULL; + } + + } + else { /* ERTS_TMR_BIF */ + Uint hsz; +#ifdef ERTS_BTM_ACCESSOR_SUPPORT + int is_abif_tmr = is_value(acsr) && acsr != rcvr; +#endif + + if (short_time) { + tmr = bif_timer_pre_alloc(); + if (!tmr) + goto alloc_bif_timer; + roflgs |= ERTS_TMR_ROFLG_PRE_ALC; + } + else { + alloc_bif_timer: +#ifdef ERTS_BTM_ACCESSOR_SUPPORT + if (is_abif_tmr) + tmr = erts_alloc(ERTS_ALC_T_ABIF_TIMER, + ERTS_ABIF_TIMER_SIZE); + else +#endif + tmr = erts_alloc(ERTS_ALC_T_BIF_TIMER, + ERTS_BIF_TIMER_SIZE); + } + + tmr->timeout = timeout_pos; + + roflgs |= ERTS_TMR_ROFLG_BIF_TMR; + if (is_internal_pid(rcvr)) { + roflgs |= ERTS_TMR_ROFLG_PROC; + tmr->receiver.proc = (Process *) rcvrp; + refc = 2; + } + else { + ERTS_HLT_ASSERT(is_atom(rcvr)); + roflgs |= ERTS_TMR_ROFLG_REG_NAME; + tmr->receiver.name = rcvr; + refc = 1; + } + + hsz = is_immed(msg) ? ((Uint) 0) : size_object(msg); + if (!hsz) { + tmr->btm.message = msg; + tmr->btm.bp = NULL; + } + else { + ErlHeapFragment *bp = new_message_buffer(hsz); + Eterm *hp = bp->mem; + tmr->btm.message = copy_struct(msg, hsz, &hp, &bp->off_heap); + tmr->btm.bp = bp; + } + tmr->btm.refn[0] = refn[0]; + tmr->btm.refn[1] = refn[1]; + tmr->btm.refn[2] = refn[2]; + + tmr->btm.proc_tree.parent = ERTS_HLT_PFIELD_NOT_IN_TABLE; + +#ifdef ERTS_BTM_ACCESSOR_SUPPORT + if (is_abif_tmr) { + Process *aproc; + roflgs |= ERTS_TMR_ROFLG_ABIF_TMR; + tmr->abtm.accessor = acsr; + aproc = erts_proc_lookup(acsr); + if (!aproc) + tmr->abtm.tree.parent = ERTS_HLT_PFIELD_NOT_IN_TABLE; + else { + refc++; + erts_smp_proc_lock(aproc, ERTS_PROC_LOCK_BTM); + abtm_rbt_insert(&aproc->accessor_bif_timers, tmr); + erts_smp_proc_unlock(aproc, ERTS_PROC_LOCK_BTM); + } + } +#endif + + btm_rbt_insert(&srv->btm_tree, tmr); + } + + tmr->head.roflgs = roflgs; + erts_smp_atomic32_init_nob(&tmr->head.refc, refc); + erts_smp_atomic32_init_nob(&tmr->state, ERTS_TMR_STATE_ACTIVE); + + if (!srv->next_timeout + || tmr->timeout < srv->next_timeout->timeout) { + if (srv->next_timeout) + erts_twheel_cancel_timer(esdp->timer_wheel, + &srv->service_timer); + erts_twheel_set_timer(esdp->timer_wheel, + &srv->service_timer, + hlt_service_timeout, + NULL, + (void *) esdp, + tmr->timeout); + srv->next_timeout = tmr; + } + + st_tmr = time_rbt_lookup_insert(&srv->time_tree, tmr); + tmr->time.tree.same_time = st_tmr; + if (st_tmr) + same_time_list_insert(&st_tmr->time.tree.same_time, tmr); + +#ifdef ERTS_HLT_HARD_DEBUG + tmr->pending_timeout = 0; +#endif + + ERTS_HLT_HDBG_CHK_SRV(srv); + + return tmr; +} + +static ERTS_INLINE void +hlt_bif_timer_timeout(ErtsHLTimer *tmr, Uint32 roflgs) +{ + ErtsProcLocks proc_locks = ERTS_PROC_LOCKS_MSG_SEND; + Process *proc; + int queued_message = 0; + int dec_refc = 0; + Uint32 is_reg_name = (roflgs & ERTS_TMR_ROFLG_REG_NAME); + ERTS_HLT_ASSERT(roflgs & ERTS_TMR_ROFLG_BIF_TMR); + +#ifdef ERTS_BTM_ACCESSOR_SUPPORT + if (tmr->head.roflgs & ERTS_TMR_ROFLG_ABIF_TMR) + hlt_delete_abtm(tmr); +#endif + + if (is_reg_name) { + Eterm pid; + ERTS_HLT_ASSERT(is_atom(tmr->receiver.name)); + pid = erts_whereis_name_to_id(NULL, tmr->receiver.name); + proc = erts_proc_lookup(pid); + } + else { + ERTS_HLT_ASSERT(roflgs & ERTS_TMR_ROFLG_PROC); + ERTS_HLT_ASSERT(tmr->receiver.proc); + + proc = tmr->receiver.proc; + proc_locks |= ERTS_PROC_LOCK_BTM; + } + if (proc) { + erts_smp_proc_lock(proc, proc_locks); + /* + * If process is exiting, let it clean up + * the btm tree by itself (it may be in + * the middle of tree destruction). + */ + if (!ERTS_PROC_IS_EXITING(proc)) { + ErtsMessage *mp = erts_alloc_message(0, NULL); + mp->data.heap_frag = tmr->btm.bp; + erts_queue_message(proc, &proc_locks, mp, + tmr->btm.message, NIL); + erts_smp_proc_unlock(proc, ERTS_PROC_LOCKS_MSG_SEND); + queued_message = 1; + proc_locks &= ~ERTS_PROC_LOCKS_MSG_SEND; + tmr->btm.bp = NULL; + if (tmr->btm.proc_tree.parent != ERTS_HLT_PFIELD_NOT_IN_TABLE) { + proc_btm_rbt_delete(&proc->bif_timers, tmr); + tmr->btm.proc_tree.parent = ERTS_HLT_PFIELD_NOT_IN_TABLE; + dec_refc = 1; + } + } + if (proc_locks) + erts_smp_proc_unlock(proc, proc_locks); + if (dec_refc) + hl_timer_pre_dec_refc(tmr); + } + if (!queued_message && tmr->btm.bp) + free_message_buffer(tmr->btm.bp); +} + +static ERTS_INLINE void +hlt_proc_timeout(ErtsHLTimer *tmr) +{ + if (proc_timeout_common(tmr->receiver.proc, (void *) tmr)) + hl_timer_dec_refc(tmr, tmr->head.roflgs); +} + +static ERTS_INLINE void +hlt_port_timeout(ErtsHLTimer *tmr) +{ + if (port_timeout_common(tmr->receiver.port, (void *) tmr)) + hl_timer_dec_refc(tmr, tmr->head.roflgs); +} + +static void hlt_timeout(ErtsHLTimer *tmr, void *vsrv) +{ + ErtsHLTimerService *srv = (ErtsHLTimerService *) vsrv; + Uint32 roflgs; + erts_aint32_t state; + + ERTS_HLT_HDBG_CHK_SRV(srv); + + roflgs = tmr->head.roflgs; + ERTS_HLT_ASSERT(roflgs & ERTS_TMR_ROFLG_HLT); + + state = erts_smp_atomic32_cmpxchg_acqb(&tmr->state, + ERTS_TMR_STATE_TIMED_OUT, + ERTS_TMR_STATE_ACTIVE); + + ERTS_HLT_ASSERT(state == ERTS_TMR_STATE_CANCELED + || state == ERTS_TMR_STATE_ACTIVE); + + if (state == ERTS_TMR_STATE_ACTIVE) { + + if (roflgs & ERTS_TMR_ROFLG_BIF_TMR) + hlt_bif_timer_timeout(tmr, roflgs); + else if (roflgs & ERTS_TMR_ROFLG_PROC) + hlt_proc_timeout(tmr); + else if (roflgs & ERTS_TMR_ROFLG_PORT) + hlt_port_timeout(tmr); + else { + ERTS_HLT_ASSERT(roflgs & ERTS_TMR_ROFLG_CALLBACK); + (*tmr->receiver.callback)(tmr->head.u.arg); + } + + } + + tmr->time.tree.parent = ERTS_HLT_PFIELD_NOT_IN_TABLE; + if ((roflgs & ERTS_TMR_ROFLG_BIF_TMR) + && tmr->btm.tree.parent != ERTS_HLT_PFIELD_NOT_IN_TABLE) { + btm_rbt_delete(&srv->btm_tree, tmr); + tmr->btm.tree.parent = ERTS_HLT_PFIELD_NOT_IN_TABLE; + } + + ERTS_HLT_HDBG_CHK_SRV(srv); + + hl_timer_dec_refc(tmr, roflgs); +} + +#ifdef ERTS_HLT_HARD_DEBUG +static void +set_pending_timeout(ErtsHLTimer *tmr, void *unused) +{ + tmr->pending_timeout = -1; +} +#endif + +static void +hlt_service_timeout(void *vesdp) +{ + ErtsSchedulerData *esdp = (ErtsSchedulerData *) vesdp; + ErtsHLTimerService *srv = esdp->timer_service; + ErtsHLTimer *tmr = srv->next_timeout; + int yield; + + ERTS_HLT_HDBG_CHK_SRV(srv); + + ERTS_HLT_ASSERT(esdp == erts_get_scheduler_data()); + + ERTS_HLT_ASSERT(!srv->yield.root || srv->yield.root == tmr); + ERTS_HLT_ASSERT(tmr); + ERTS_HLT_ASSERT(tmr->timeout <= erts_get_monotonic_time(esdp)); + + if (!srv->yield.root) { + ERTS_HLT_ASSERT(tmr->time.tree.parent + != ERTS_HLT_PFIELD_NOT_IN_TABLE); + time_rbt_delete(&srv->time_tree, tmr); + tmr->time.tree.parent = ERTS_HLT_PFIELD_NOT_IN_TABLE; +#ifdef ERTS_HLT_HARD_DEBUG + tmr->pending_timeout = 1; + if (tmr->time.tree.same_time) + same_time_list_foreach(tmr->time.tree.same_time, set_pending_timeout, NULL); +#endif + } + + if (!tmr->time.tree.same_time && !srv->yield.root) + yield = 0; + else { + yield = same_time_list_foreach_destroy_yielding( + &tmr->time.tree.same_time, hlt_timeout, (void *) srv, + &srv->yield.state, ERTS_TMR_TIMEOUT_YIELD_LIMIT); + } + + if (yield) + srv->yield.root = tmr; + else { + srv->yield.root = NULL; + hlt_timeout(tmr, (void *) srv); + + tmr = time_rbt_smallest(srv->time_tree); + srv->next_timeout = tmr; + } + + ERTS_HLT_HDBG_CHK_SRV(srv); + + if (tmr) + erts_twheel_set_timer(esdp->timer_wheel, + &srv->service_timer, + hlt_service_timeout, + NULL, + vesdp, + tmr->timeout); +} + +static void +hlt_delete_timer(ErtsSchedulerData *esdp, ErtsHLTimer *tmr) +{ + ErtsHLTimerService *srv = esdp->timer_service; + + ERTS_HLT_HDBG_CHK_SRV(srv); + + if (tmr->head.roflgs & ERTS_TMR_ROFLG_BIF_TMR) { + + if (tmr->btm.tree.parent != ERTS_HLT_PFIELD_NOT_IN_TABLE) { + btm_rbt_delete(&srv->btm_tree, tmr); + tmr->btm.tree.parent = ERTS_HLT_PFIELD_NOT_IN_TABLE; + } + +#ifdef ERTS_BTM_ACCESSOR_SUPPORT + if (tmr->head.roflgs & ERTS_TMR_ROFLG_ABIF_TMR) + hlt_delete_abtm(tmr); +#endif + } + + if (tmr->time.tree.parent == ERTS_HLT_PFIELD_NOT_IN_TABLE) { + /* Already removed... */ + ERTS_HLT_HDBG_CHK_SRV(srv); + return; + } + + if (tmr->time.tree.parent & ERTS_HLT_PFLG_SAME_TIME) { + same_time_list_delete(tmr); + } + else if (tmr->time.tree.same_time) { + ErtsHLTimer *st_container; + + ERTS_HLT_ASSERT((tmr->time.tree.parent & ERTS_HLT_PFLG_SAME_TIME) == 0); + st_container = tmr->time.tree.same_time->time.tree.u.l.prev; + + ERTS_HLT_ASSERT(st_container); + ERTS_HLT_ASSERT(st_container->time.tree.parent + & ERTS_HLT_PFLG_SAME_TIME); + ERTS_HLT_ASSERT(tmr->timeout == st_container->timeout); + + same_time_list_delete(st_container); + st_container->time.tree.same_time = tmr->time.tree.same_time; + same_time_list_new_root(&st_container->time.tree.same_time); + + time_rbt_replace(&srv->time_tree, tmr, st_container); + ERTS_HLT_ASSERT((st_container->time.tree.parent + & ERTS_HLT_PFLG_SAME_TIME) == 0); + + if (srv->next_timeout == tmr) + srv->next_timeout = st_container; + } + else { + ERTS_HLT_ASSERT((tmr->time.tree.parent & ERTS_HLT_PFLG_SAME_TIME) == 0); + time_rbt_delete(&srv->time_tree, tmr); + if (tmr == srv->next_timeout) { + ErtsHLTimer *smlst; + erts_twheel_cancel_timer(esdp->timer_wheel, + &srv->service_timer); + smlst = time_rbt_smallest(srv->time_tree); + srv->next_timeout = smlst; + if (smlst) { + ERTS_HLT_ASSERT(smlst->timeout > tmr->timeout); + erts_twheel_set_timer(esdp->timer_wheel, + &srv->service_timer, + hlt_service_timeout, + NULL, + (void *) esdp, + smlst->timeout); + } + } + } + tmr->time.tree.parent = ERTS_HLT_PFIELD_NOT_IN_TABLE; + + hl_timer_dec_refc(tmr, tmr->head.roflgs); + + ERTS_HLT_HDBG_CHK_SRV(srv); +} + +/* + * Pass canceled timers back to originating scheduler + */ + +static ERTS_INLINE void +cleanup_sched_local_canceled_timer(ErtsSchedulerData *esdp, + ErtsTimer *tmr) +{ + Uint32 roflgs = tmr->head.roflgs; + ERTS_HLT_ASSERT(esdp == erts_get_scheduler_data()); + ERTS_HLT_ASSERT((tmr->head.roflgs & ERTS_TMR_ROFLG_SID_MASK) + == (Uint32) esdp->no); + if (roflgs & ERTS_TMR_ROFLG_HLT) { + hlt_delete_timer(esdp, &tmr->hlt); + hl_timer_dec_refc(&tmr->hlt, roflgs); + } + else { + cancel_tw_timer(esdp, &tmr->twt); + tw_timer_dec_refc(&tmr->twt); + } +} + +#ifdef ERTS_SMP + +static void +init_canceled_queue(ErtsHLTCncldTmrQ *cq) +{ + erts_atomic_init_nob(&cq->tail.data.marker.u.next, ERTS_AINT_NULL); + erts_atomic_init_nob(&cq->tail.data.last, + (erts_aint_t) &cq->tail.data.marker); + cq->head.first = (ErtsTimer *) &cq->tail.data.marker; + cq->head.unref_end = (ErtsTimer *) &cq->tail.data.marker; + cq->head.next.thr_progress = erts_thr_progress_current(); + cq->head.next.thr_progress_reached = 1; + cq->head.next.unref_end = (ErtsTimer *) &cq->tail.data.marker; + cq->head.used_marker = 1; +} + +static ERTS_INLINE int +cq_enqueue(ErtsHLTCncldTmrQ *cq, ErtsTimer *tmr, int cinit) +{ + erts_aint_t itmp; + ErtsTimer *enq, *this = tmr; + + erts_atomic_init_nob(&this->head.u.next, ERTS_AINT_NULL); + /* Enqueue at end of list... */ + + enq = (ErtsTimer *) erts_atomic_read_nob(&cq->tail.data.last); + itmp = erts_atomic_cmpxchg_relb(&enq->head.u.next, + (erts_aint_t) this, + ERTS_AINT_NULL); + if (itmp == ERTS_AINT_NULL) { + /* We are required to move last pointer */ +#ifdef DEBUG + ASSERT(ERTS_AINT_NULL == erts_atomic_read_nob(&this->head.u.next)); + ASSERT(((erts_aint_t) enq) + == erts_atomic_xchg_relb(&cq->tail.data.last, + (erts_aint_t) this)); +#else + erts_atomic_set_relb(&cq->tail.data.last, (erts_aint_t) this); +#endif + return 1; + } + else { + /* + * We *need* to insert element somewhere in between the + * last element we read earlier and the actual last element. + */ + int i = cinit; + + while (1) { + erts_aint_t itmp2; + erts_atomic_set_nob(&this->head.u.next, itmp); + itmp2 = erts_atomic_cmpxchg_relb(&enq->head.u.next, + (erts_aint_t) this, + itmp); + if (itmp == itmp2) + return 0; /* inserted this */ + if ((i & 1) == 0) + itmp = itmp2; + else { + enq = (ErtsTimer *) itmp2; + itmp = erts_atomic_read_acqb(&enq->head.u.next); + ASSERT(itmp != ERTS_AINT_NULL); + } + i++; + } + } +} + +static ERTS_INLINE erts_aint_t +check_insert_marker(ErtsHLTCncldTmrQ *cq, erts_aint_t ilast) +{ + if (!cq->head.used_marker + && cq->head.unref_end == (ErtsTimer *) ilast) { + erts_aint_t itmp; + ErtsTimer *last = (ErtsTimer *) ilast; + + erts_atomic_init_nob(&cq->tail.data.marker.u.next, ERTS_AINT_NULL); + itmp = erts_atomic_cmpxchg_relb(&last->head.u.next, + (erts_aint_t) &cq->tail.data.marker, + ERTS_AINT_NULL); + if (itmp == ERTS_AINT_NULL) { + ilast = (erts_aint_t) &cq->tail.data.marker; + cq->head.used_marker = !0; + erts_atomic_set_relb(&cq->tail.data.last, ilast); + } + } + return ilast; +} + +static ERTS_INLINE ErtsTimer * +cq_dequeue(ErtsHLTCncldTmrQ *cq) +{ + ErtsTimer *tmr; + + if (cq->head.first == cq->head.unref_end) + return NULL; + + tmr = cq->head.first; + if (tmr == (ErtsTimer *) &cq->tail.data.marker) { + ASSERT(cq->head.used_marker); + cq->head.used_marker = 0; + tmr = (ErtsTimer *) erts_atomic_read_nob(&tmr->head.u.next); + if (tmr == cq->head.unref_end) { + cq->head.first = tmr; + return NULL; + } + } + + cq->head.first = (ErtsTimer *) erts_atomic_read_nob(&tmr->head.u.next); + + ASSERT(cq->head.first); + + return tmr; +} + +static int +cq_check_incoming(ErtsSchedulerData *esdp, ErtsHLTCncldTmrQ *cq) +{ + erts_aint_t ilast = erts_atomic_read_nob(&cq->tail.data.last); + if (((ErtsTimer *) ilast) == (ErtsTimer *) &cq->tail.data.marker + && cq->head.first == (ErtsTimer *) &cq->tail.data.marker) { + /* Nothing more to do... */ + return 0; + } + + if (cq->head.next.thr_progress_reached + || erts_thr_progress_has_reached(cq->head.next.thr_progress)) { + cq->head.next.thr_progress_reached = 1; + /* Move unreferenced end pointer forward... */ + + ERTS_HLT_SMP_MEMBAR_LoadLoad_LoadStore; + + cq->head.unref_end = cq->head.next.unref_end; + + ilast = check_insert_marker(cq, ilast); + + if (cq->head.unref_end != (ErtsTimer *) ilast) { + cq->head.next.unref_end = (ErtsTimer *) ilast; + cq->head.next.thr_progress = erts_thr_progress_later(esdp); + cq->head.next.thr_progress_reached = 0; + } + } + return 1; +} + +static ERTS_INLINE void +store_earliest_thr_prgr(ErtsThrPrgrVal *prev_val, ErtsHLTCncldTmrQ *cq) +{ + if (!cq->head.next.thr_progress_reached + && (*prev_val == ERTS_THR_PRGR_INVALID + || erts_thr_progress_cmp(cq->head.next.thr_progress, + *prev_val) < 0)) { + *prev_val = cq->head.next.thr_progress; + } +} + +static void +handle_canceled_queue(ErtsSchedulerData *esdp, + ErtsHLTCncldTmrQ *cq, + int use_limit, + int ops_limit, + int *need_thr_progress, + ErtsThrPrgrVal *thr_prgr_p, + int *need_more_work) +{ + int need_thr_prgr = 0; + int need_mr_wrk = 0; + int have_checked_incoming = 0; + int ops = 0; + + ERTS_HLT_ASSERT(cq == &esdp->timer_service->canceled_queue); + + while (1) { + ErtsTimer *tmr = cq_dequeue(cq); + + if (tmr) + cleanup_sched_local_canceled_timer(esdp, tmr); + else { + if (have_checked_incoming) + break; + need_thr_prgr = cq_check_incoming(esdp, cq); + if (need_thr_progress) { + *need_thr_progress |= need_thr_prgr; + if (need_thr_prgr) + store_earliest_thr_prgr(thr_prgr_p, cq); + } + have_checked_incoming = 1; + continue; + } + + if (use_limit && ++ops >= ops_limit) { + if (cq->head.first != cq->head.unref_end) { + need_mr_wrk = 1; + if (need_more_work) + *need_more_work |= 1; + } + break; + } + } + + if (need_thr_progress && !(need_thr_prgr | need_mr_wrk)) { + need_thr_prgr = cq_check_incoming(esdp, cq); + *need_thr_progress |= need_thr_prgr; + if (need_thr_prgr) + store_earliest_thr_prgr(thr_prgr_p, cq); + } +} + +void +erts_handle_canceled_timers(void *vesdp, + int *need_thr_progress, + ErtsThrPrgrVal *thr_prgr_p, + int *need_more_work) +{ + ErtsSchedulerData *esdp = (ErtsSchedulerData *) vesdp; + ERTS_HLT_ASSERT(esdp == erts_get_scheduler_data()); + + handle_canceled_queue(esdp, &esdp->timer_service->canceled_queue, + 1, ERTS_TMR_CANCELED_TIMER_LIMIT, + need_thr_progress, thr_prgr_p, + need_more_work); +} + +#endif /* ERTS_SMP */ + +static void +queue_canceled_timer(ErtsSchedulerData *esdp, int rsched_id, ErtsTimer *tmr) +{ +#ifdef ERTS_SMP + ErtsHLTCncldTmrQ *cq; + cq = &ERTS_SCHEDULER_IX(rsched_id-1)->timer_service->canceled_queue; + if (cq_enqueue(cq, tmr, rsched_id - (int) esdp->no)) + erts_notify_canceled_timer(esdp, rsched_id); +#else + ERTS_INTERNAL_ERROR("Unexpected enqueue of canceled timer"); +#endif +} + +static void +continue_cancel_ptimer(ErtsSchedulerData *esdp, ErtsTimer *tmr) +{ +#ifdef ERTS_SMP + Uint32 sid = (tmr->head.roflgs & ERTS_TMR_ROFLG_SID_MASK); + + if (esdp->no != sid) + queue_canceled_timer(esdp, sid, tmr); + else +#endif + cleanup_sched_local_canceled_timer(esdp, tmr); +} + +/* + * BIF timer specific + */ + +Uint erts_bif_timer_memory_size(void) +{ + return (Uint) 0; +} + +static BIF_RETTYPE +setup_bif_timer(Process *c_p, ErtsMonotonicTime timeout_pos, + int short_time, Eterm rcvr, Eterm acsr, + Eterm msg, int wrap) +{ + BIF_RETTYPE ret; + Eterm ref, tmo_msg, *hp; + ErtsHLTimer *tmr; + ErtsSchedulerData *esdp; + DeclareTmpHeap(tmp_hp, 4, c_p); + + if (is_not_internal_pid(rcvr) && is_not_atom(rcvr)) + goto badarg; + + esdp = ERTS_PROC_GET_SCHDATA(c_p); + + hp = HAlloc(c_p, REF_THING_SIZE); + ref = erts_sched_make_ref_in_buffer(esdp, hp); + + ASSERT(erts_get_ref_numbers_thr_id( + internal_ref_numbers(ref)) == (Uint32) esdp->no); + + UseTmpHeap(4, c_p); + + tmo_msg = wrap ? TUPLE3(tmp_hp, am_timeout, ref, msg) : msg; + + tmr = create_hl_timer(esdp, timeout_pos, short_time, + ERTS_TMR_BIF, NULL, rcvr, acsr, tmo_msg, + internal_ref_numbers(ref), NULL, NULL); + + UnUseTmpHeap(4, c_p); + + if (is_internal_pid(rcvr)) { + Process *proc = erts_pid2proc_opt(c_p, ERTS_PROC_LOCK_MAIN, + rcvr, ERTS_PROC_LOCK_BTM, + ERTS_P2P_FLG_INC_REFC); + if (!proc) { + if (tmr->btm.bp) + free_message_buffer(tmr->btm.bp); + hlt_delete_timer(esdp, tmr); + hl_timer_destroy(tmr); + } + else { + proc_btm_rbt_insert(&proc->bif_timers, tmr); + erts_smp_proc_unlock(proc, ERTS_PROC_LOCK_BTM); + tmr->receiver.proc = proc; + } + } + + ERTS_BIF_PREP_RET(ret, ref); + return ret; + +badarg: + + ERTS_BIF_PREP_ERROR(ret, c_p, BADARG); + return ret; +} + +static int +cancel_bif_timer(ErtsHLTimer *tmr) +{ + erts_aint_t state; + Uint32 roflgs; + int res; + + state = erts_smp_atomic32_cmpxchg_acqb(&tmr->state, + ERTS_TMR_STATE_CANCELED, + ERTS_TMR_STATE_ACTIVE); + if (state != ERTS_TMR_STATE_ACTIVE) + return 0; + + if (tmr->btm.bp) + free_message_buffer(tmr->btm.bp); + + res = -1; + + roflgs = tmr->head.roflgs; + if (roflgs & ERTS_TMR_ROFLG_PROC) { + Process *proc = tmr->receiver.proc; + ERTS_HLT_ASSERT(!(tmr->head.roflgs & ERTS_TMR_ROFLG_REG_NAME)); + + erts_smp_proc_lock(proc, ERTS_PROC_LOCK_BTM); + /* + * If process is exiting, let it clean up + * the btm tree by itself (it may be in + * the middle of tree destruction). + */ + if (!ERTS_PROC_IS_EXITING(proc) + && tmr->btm.proc_tree.parent != ERTS_HLT_PFIELD_NOT_IN_TABLE) { + proc_btm_rbt_delete(&proc->bif_timers, tmr); + tmr->btm.proc_tree.parent = ERTS_HLT_PFIELD_NOT_IN_TABLE; + res = 1; + } + erts_smp_proc_unlock(proc, ERTS_PROC_LOCK_BTM); + } + + return res; +} + +static ERTS_INLINE Eterm +access_sched_local_btm(Process *c_p, Eterm pid, + Eterm tref, Uint32 *trefn, + Uint32 *rrefn, + int async, int cancel, + int return_res, + int info) +{ + ErtsSchedulerData *esdp; + ErtsHLTimerService *srv; + ErtsHLTimer *tmr; + Sint64 time_left; + Process *proc; + ErtsProcLocks proc_locks; + + time_left = -1; + + if (!c_p) + esdp = erts_get_scheduler_data(); + else { + esdp = ERTS_PROC_GET_SCHDATA(c_p); + ERTS_HLT_ASSERT(esdp == erts_get_scheduler_data()); + } + + ERTS_HLT_ASSERT(erts_get_ref_numbers_thr_id(trefn) + == (Uint32) esdp->no); + + srv = esdp->timer_service; + + tmr = btm_rbt_lookup(srv->btm_tree, trefn); + if (tmr) { + if (!cancel) { + erts_aint32_t state = erts_smp_atomic32_read_acqb(&tmr->state); + if (state == ERTS_TMR_STATE_ACTIVE) + time_left = get_time_left(esdp, tmr->timeout); + } + else { + int cncl_res = cancel_bif_timer(tmr); + if (cncl_res) { + + time_left = get_time_left(esdp, tmr->timeout); + + if (cncl_res > 0) + hl_timer_dec_refc(tmr, tmr->head.roflgs); + + hlt_delete_timer(esdp, tmr); + } + } + } + + if (!info) + return am_ok; + + if (return_res) { + ERTS_HLT_ASSERT(c_p); + if (time_left < 0) + return am_false; + else if (time_left <= (Sint64) MAX_SMALL) + return make_small((Sint) time_left); + else { + Uint hsz = ERTS_SINT64_HEAP_SIZE(time_left); + Eterm *hp = HAlloc(c_p, hsz); + return erts_sint64_to_big(time_left, &hp); + } + } + + if (c_p) { + proc = c_p; + proc_locks = ERTS_PROC_LOCK_MAIN; + } + else { + proc = erts_proc_lookup(pid); + proc_locks = 0; + } + + if (proc) { + Uint hsz; + ErtsMessage *mp; + Eterm *hp, msg, ref, result; + ErlOffHeap *ohp; + Uint32 *refn; +#ifdef ERTS_HLT_DEBUG + Eterm *hp_end; +#endif + + hsz = REF_THING_SIZE; + if (async) { + refn = trefn; /* timer ref */ + hsz += 4; /* 3-tuple */ + } + else { + refn = rrefn; /* request ref */ + hsz += 3; /* 2-tuple */ + } + + ERTS_HLT_ASSERT(refn); + + if (time_left > (Sint64) MAX_SMALL) + hsz += ERTS_SINT64_HEAP_SIZE(time_left); + + mp = erts_alloc_message_heap(proc, &proc_locks, + hsz, &hp, &ohp); + +#ifdef ERTS_HLT_DEBUG + hp_end = hp + hsz; +#endif + + if (time_left < 0) + result = am_false; + else if (time_left <= (Sint64) MAX_SMALL) + result = make_small((Sint) time_left); + else + result = erts_sint64_to_big(time_left, &hp); + + write_ref_thing(hp, + refn[0], + refn[1], + refn[2]); + ref = make_internal_ref(hp); + hp += REF_THING_SIZE; + + msg = (async + ? TUPLE3(hp, (cancel + ? am_cancel_timer + : am_read_timer), ref, result) + : TUPLE2(hp, ref, result)); + + ERTS_HLT_ASSERT(hp + (async ? 4 : 3) == hp_end); + + erts_queue_message(proc, &proc_locks, mp, msg, NIL); + + if (c_p) + proc_locks &= ~ERTS_PROC_LOCK_MAIN; + if (proc_locks) + erts_smp_proc_unlock(proc, proc_locks); + } + + return am_ok; +} + +#define ERTS_BTM_REQ_FLG_ASYNC (((Uint32) 1) << 0) +#define ERTS_BTM_REQ_FLG_CANCEL (((Uint32) 1) << 1) +#define ERTS_BTM_REQ_FLG_INFO (((Uint32) 1) << 2) + +typedef struct { + Eterm pid; + Uint32 trefn[ERTS_REF_NUMBERS]; + Uint32 rrefn[ERTS_REF_NUMBERS]; + Uint32 flags; +} ErtsBifTimerRequest; + +static void +bif_timer_access_request(void *vreq) +{ + ErtsBifTimerRequest *req = (ErtsBifTimerRequest *) vreq; + int async = (int) (req->flags & ERTS_BTM_REQ_FLG_ASYNC); + int cancel = (int) (req->flags & ERTS_BTM_REQ_FLG_CANCEL); + int info = (int) (req->flags & ERTS_BTM_REQ_FLG_INFO); + (void) access_sched_local_btm(NULL, req->pid, THE_NON_VALUE, + req->trefn, req->rrefn, async, + cancel, 0, info); + erts_free(ERTS_ALC_T_TIMER_REQUEST, vreq); +} + +static int +try_access_sched_remote_btm(ErtsSchedulerData *esdp, + Process *c_p, Uint32 sid, + Uint32 *trefn, + int async, int cancel, + int info, Eterm *resp) +{ + ErtsHLTimer *tmr; + Sint64 time_left; + + ERTS_HLT_ASSERT(c_p); + + /* + * Check if the timer is aimed at current + * process of if this process is an accessor + * of the timer... + */ + erts_smp_proc_lock(c_p, ERTS_PROC_LOCK_BTM); + tmr = proc_btm_rbt_lookup(c_p->bif_timers, trefn); +#ifdef ERTS_BTM_ACCESSOR_SUPPORT + if (!tmr) + tmr = abtm_rbt_lookup(c_p->accessor_bif_timers, trefn); +#endif + erts_smp_proc_unlock(c_p, ERTS_PROC_LOCK_BTM); + if (!tmr) + return 0; + + if (!cancel) { + erts_aint32_t state = erts_smp_atomic32_read_acqb(&tmr->state); + if (state == ERTS_TMR_STATE_ACTIVE) + time_left = get_time_left(esdp, tmr->timeout); + else + time_left = -1; + } + else { + int cncl_res = cancel_bif_timer(tmr); + if (!cncl_res) + time_left = -1; + else { + time_left = get_time_left(esdp, tmr->timeout); + if (cncl_res > 0) + queue_canceled_timer(esdp, sid, (ErtsTimer *) tmr); + } + } + + if (!info) { + *resp = am_ok; + return 1; + } + + if (!async) { + if (time_left < 0) + *resp = am_false; + else if (time_left <= (Sint64) MAX_SMALL) + *resp = make_small((Sint) time_left); + else { + Uint hsz = ERTS_SINT64_HEAP_SIZE(time_left); + Eterm *hp = HAlloc(c_p, hsz); + *resp = erts_sint64_to_big(time_left, &hp); + } + } + else { + ErtsMessage *mp; + Eterm tag, res, msg, tref; + Uint hsz; + Eterm *hp; + ErtsProcLocks proc_locks = ERTS_PROC_LOCK_MAIN; + ErlOffHeap *ohp; + + hsz = 4 + REF_THING_SIZE; + if (time_left > (Sint64) MAX_SMALL) + hsz += ERTS_SINT64_HEAP_SIZE(time_left); + + mp = erts_alloc_message_heap(c_p, &proc_locks, + hsz, &hp, &ohp); + if (cancel) + tag = am_cancel_timer; + else + tag = am_read_timer; + + write_ref_thing(hp, + trefn[0], + trefn[1], + trefn[2]); + tref = make_internal_ref(hp); + hp += REF_THING_SIZE; + + if (time_left < 0) + res = am_false; + else if (time_left <= (Sint64) MAX_SMALL) + res = make_small((Sint) time_left); + else + res = erts_sint64_to_big(time_left, &hp); + + msg = TUPLE3(hp, tag, tref, res); + + erts_queue_message(c_p, &proc_locks, mp, msg, NIL); + + proc_locks &= ~ERTS_PROC_LOCK_MAIN; + if (proc_locks) + erts_smp_proc_unlock(c_p, proc_locks); + + *resp = am_ok; + } + return 1; +} + +static BIF_RETTYPE +access_bif_timer(Process *c_p, Eterm tref, int cancel, int async, int info) +{ + BIF_RETTYPE ret; + ErtsSchedulerData *esdp; + Uint32 sid; + Uint32 *trefn; + Eterm res; + + if (is_not_internal_ref(tref)) { + if (is_not_ref(tref)) + goto badarg; + else + goto no_timer; + } + + esdp = ERTS_PROC_GET_SCHDATA(c_p); + + trefn = internal_ref_numbers(tref); + sid = erts_get_ref_numbers_thr_id(trefn); + if (sid < 1 || erts_no_schedulers < sid) + goto no_timer; + + if (sid == (Uint32) esdp->no) { + res = access_sched_local_btm(c_p, c_p->common.id, + tref, trefn, NULL, + async, cancel, !async, + info); + ERTS_BIF_PREP_RET(ret, res); + } + else if (try_access_sched_remote_btm(esdp, c_p, + sid, trefn, + async, cancel, + info, &res)) { + ERTS_BIF_PREP_RET(ret, res); + } + else { + /* + * Schedule access for execution on + * remote scheduler... + */ + ErtsBifTimerRequest *req = erts_alloc(ERTS_ALC_T_TIMER_REQUEST, + sizeof(ErtsBifTimerRequest)); + + req->flags = 0; + if (cancel) + req->flags |= ERTS_BTM_REQ_FLG_CANCEL; + if (async) + req->flags |= ERTS_BTM_REQ_FLG_ASYNC; + if (info) + req->flags |= ERTS_BTM_REQ_FLG_INFO; + + req->pid = c_p->common.id; + + req->trefn[0] = trefn[0]; + req->trefn[1] = trefn[1]; + req->trefn[2] = trefn[2]; + + if (async) + ERTS_BIF_PREP_RET(ret, am_ok); + else { + Eterm *hp, rref; + Uint32 *rrefn; + + hp = HAlloc(c_p, REF_THING_SIZE); + rref = erts_sched_make_ref_in_buffer(esdp, hp); + rrefn = internal_ref_numbers(rref); + + req->rrefn[0] = rrefn[0]; + req->rrefn[1] = rrefn[1]; + req->rrefn[2] = rrefn[2]; + + erts_smp_proc_lock(c_p, ERTS_PROC_LOCKS_MSG_RECEIVE); + + if (ERTS_PROC_PENDING_EXIT(c_p)) + ERTS_VBUMP_ALL_REDS(c_p); + else { + /* + * Caller needs to wait for a message containing + * the ref that we just created. No such message + * can exist in callers message queue at this time. + * We therefore move the save pointer of the + * callers message queue to the end of the queue. + * + * NOTE: It is of vital importance that the caller + * immediately do a receive unconditionaly + * waiting for the message with the reference; + * otherwise, next receive will *not* work + * as expected! + */ + ERTS_SMP_MSGQ_MV_INQ2PRIVQ(c_p); + c_p->msg.save = c_p->msg.last; + } + erts_smp_proc_unlock(c_p, ERTS_PROC_LOCKS_MSG_RECEIVE); + + ERTS_BIF_PREP_TRAP1(ret, erts_await_result, c_p, rref); + } + + erts_schedule_misc_aux_work(sid, + bif_timer_access_request, + (void *) req); + } + + return ret; + +badarg: + ERTS_BIF_PREP_ERROR(ret, c_p, BADARG); + return ret; + +no_timer: + ERTS_BIF_PREP_RET(ret, am_false); + return ret; + +} + +static ERTS_INLINE int +bool_arg(Eterm val, int *argp) +{ + switch (val) { + case am_true: *argp = 1; return 1; + case am_false: *argp = 0; return 1; + default: return 0; + } +} + +static ERTS_INLINE int +parse_bif_timer_options(Eterm option_list, int *async, int *info, + int *abs, Eterm *accessor) +{ + Eterm list = option_list; + + if (async) + *async = 0; + if (info) + *info = 1; + if (abs) + *abs = 0; + if (accessor) + *accessor = THE_NON_VALUE; + + while (is_list(list)) { + Eterm *consp, *tp, opt; + + consp = list_val(list); + opt = CAR(consp); + if (is_not_tuple(opt)) + return 0; + + tp = tuple_val(opt); + if (arityval(tp[0]) != 2) + return 0; + + switch (tp[1]) { + case am_async: + if (!async || !bool_arg(tp[2], async)) + return 0; + break; + case am_info: + if (!info || !bool_arg(tp[2], info)) + return 0; + break; + case am_abs: + if (!abs || !bool_arg(tp[2], abs)) + return 0; + break; +#ifdef ERTS_BTM_ACCESSOR_SUPPORT + case am_accessor: + if (!accessor || is_not_internal_pid(tp[2])) + return 0; + *accessor = tp[2]; + break; +#endif + default: + return 0; + } + + list = CDR(consp); + } + + if (is_not_nil(list)) + return 0; + return 1; +} + +static void +exit_cancel_bif_timer(ErtsHLTimer *tmr, void *vesdp) +{ + ErtsSchedulerData *esdp = (ErtsSchedulerData *) vesdp; + Uint32 sid, roflgs; + erts_aint_t state; + + state = erts_smp_atomic32_cmpxchg_acqb(&tmr->state, + ERTS_TMR_STATE_CANCELED, + ERTS_TMR_STATE_ACTIVE); + + roflgs = tmr->head.roflgs; + sid = roflgs & ERTS_TMR_ROFLG_SID_MASK; + + ERTS_HLT_ASSERT(sid == erts_get_ref_numbers_thr_id(tmr->btm.refn)); + ERTS_HLT_ASSERT(tmr->btm.proc_tree.parent + != ERTS_HLT_PFIELD_NOT_IN_TABLE); + + tmr->btm.proc_tree.parent = ERTS_HLT_PFIELD_NOT_IN_TABLE; + + if (sid == (Uint32) esdp->no) { + if (state == ERTS_TMR_STATE_ACTIVE) { + if (tmr->btm.bp) + free_message_buffer(tmr->btm.bp); + hlt_delete_timer(esdp, tmr); + } + hl_timer_dec_refc(tmr, roflgs); + } + else { + if (state == ERTS_TMR_STATE_ACTIVE) { + if (tmr->btm.bp) + free_message_buffer(tmr->btm.bp); + queue_canceled_timer(esdp, sid, (ErtsTimer *) tmr); + } + else + hl_timer_dec_refc(tmr, roflgs); + } +} + +#ifdef ERTS_HLT_DEBUG +# define ERTS_BTM_MAX_DESTROY_LIMIT 2 +#else +# define ERTS_BTM_MAX_DESTROY_LIMIT 50 +#endif + +typedef struct { + ErtsBifTimers *bif_timers; + union { + proc_btm_rbt_yield_state_t proc_btm_yield_state; +#ifdef ERTS_BTM_ACCESSOR_SUPPORT + abtm_rbt_yield_state_t abtm_yield_state; +#endif + } u; +} ErtsBifTimerYieldState; + +int erts_cancel_bif_timers(Process *p, ErtsBifTimers *btm, void **vyspp) +{ + ErtsSchedulerData *esdp = ERTS_PROC_GET_SCHDATA(p); + ErtsBifTimerYieldState ys = {btm, {ERTS_RBT_YIELD_STAT_INITER}}; + ErtsBifTimerYieldState *ysp; + int res; + + ysp = (ErtsBifTimerYieldState *) *vyspp; + if (!ysp) + ysp = &ys; + + res = proc_btm_rbt_foreach_destroy_yielding(&ysp->bif_timers, + exit_cancel_bif_timer, + (void *) esdp, + &ysp->u.proc_btm_yield_state, + ERTS_BTM_MAX_DESTROY_LIMIT); + + if (res == 0) { + if (ysp != &ys) + erts_free(ERTS_ALC_T_BTM_YIELD_STATE, ysp); + *vyspp = NULL; + } + else { + + if (ysp == &ys) { + ysp = erts_alloc(ERTS_ALC_T_BTM_YIELD_STATE, + sizeof(ErtsBifTimerYieldState)); + sys_memcpy((void *) ysp, (void *) &ys, + sizeof(ErtsBifTimerYieldState)); + } + + *vyspp = (void *) ysp; + } + + return res; +} + +#ifdef ERTS_BTM_ACCESSOR_SUPPORT + +static void +detach_bif_timer(ErtsHLTimer *tmr, void *vesdp) +{ + tmr->abtm.tree.parent = ERTS_HLT_PFIELD_NOT_IN_TABLE; + hl_timer_dec_refc(tmr, tmr->head.roflgs); +} + +int erts_detach_accessor_bif_timers(Process *p, ErtsBifTimers *btm, void **vyspp) +{ + ErtsSchedulerData *esdp = ERTS_PROC_GET_SCHDATA(p); + ErtsBifTimerYieldState ys = {btm, {ERTS_RBT_YIELD_STAT_INITER}}; + ErtsBifTimerYieldState *ysp; + int res; + + ysp = (ErtsBifTimerYieldState *) *vyspp; + if (!ysp) + ysp = &ys; + + res = abtm_rbt_foreach_destroy_yielding(&ysp->bif_timers, + detach_bif_timer, + (void *) esdp, + &ysp->u.abtm_yield_state, + ERTS_BTM_MAX_DESTROY_LIMIT); + + if (res == 0) { + if (ysp != &ys) + erts_free(ERTS_ALC_T_BTM_YIELD_STATE, ysp); + *vyspp = NULL; + } + else { + + if (ysp == &ys) { + ysp = erts_alloc(ERTS_ALC_T_BTM_YIELD_STATE, + sizeof(ErtsBifTimerYieldState)); + sys_memcpy((void *) ysp, (void *) &ys, + sizeof(ErtsBifTimerYieldState)); + } + + *vyspp = (void *) ysp; + } + + return res; +} + +#endif /* ERTS_BTM_ACCESSOR_SUPPORT */ + +static ERTS_INLINE int +parse_timeout_pos(ErtsSchedulerData *esdp, Eterm arg, + ErtsMonotonicTime *conv_arg, int abs, + ErtsMonotonicTime *tposp, int *stimep) +{ + ErtsMonotonicTime t; + + if (!term_to_Sint64(arg, &t)) { + ERTS_HLT_ASSERT(!is_small(arg)); + if (!is_big(arg)) + return -1; + + if (abs || !big_sign(arg)) + return 1; + + return -1; + } + + if (conv_arg) + *conv_arg = t; + + if (abs) { + t += -1*ERTS_MONOTONIC_OFFSET_MSEC; /* external to internal */ + if (t < ERTS_MONOTONIC_TO_MSEC(ERTS_MONOTONIC_BEGIN)) + return 1; + if (t > ERTS_MONOTONIC_TO_MSEC(ERTS_MONOTONIC_END)) + return 1; + *stimep = (t - ERTS_MONOTONIC_TO_MSEC(esdp->last_monotonic_time) + < ERTS_BIF_TIMER_SHORT_TIME); + *tposp = ERTS_MSEC_TO_CLKTCKS(t); + } + else { + ErtsMonotonicTime now, ticks; + + if (t < 0) + return -1; + + ticks = ERTS_MSEC_TO_CLKTCKS(t); + + if (ERTS_CLKTCK_RESOLUTION > 1000 && ticks < 0) + return 1; + + ERTS_HLT_ASSERT(ticks >= 0); + + now = erts_get_monotonic_time(esdp); + ticks += ERTS_MONOTONIC_TO_CLKTCKS(now-1); + ticks += 1; + + if (ticks < ERTS_MONOTONIC_TO_CLKTCKS(ERTS_MONOTONIC_BEGIN)) + return 1; + if (ticks > ERTS_MONOTONIC_TO_CLKTCKS(ERTS_MONOTONIC_END)) + return 1; + + *stimep = (t < ERTS_BIF_TIMER_SHORT_TIME); + *tposp = ticks; + } + + return 0; +} + +/* + * + * The BIF timer BIFs... + */ + +BIF_RETTYPE send_after_3(BIF_ALIST_3) +{ + ErtsMonotonicTime timeout_pos; + int short_time, tres; + + tres = parse_timeout_pos(ERTS_PROC_GET_SCHDATA(BIF_P), BIF_ARG_1, NULL, + 0, &timeout_pos, &short_time); + if (tres != 0) + BIF_ERROR(BIF_P, BADARG); + + return setup_bif_timer(BIF_P, timeout_pos, short_time, + BIF_ARG_2, BIF_ARG_2, BIF_ARG_3, 0); +} + +BIF_RETTYPE send_after_4(BIF_ALIST_4) +{ + ErtsMonotonicTime timeout_pos; + Eterm accessor; + int short_time, abs, tres; + + if (!parse_bif_timer_options(BIF_ARG_4, NULL, NULL, &abs, &accessor)) + BIF_ERROR(BIF_P, BADARG); + + tres = parse_timeout_pos(ERTS_PROC_GET_SCHDATA(BIF_P), BIF_ARG_1, NULL, + abs, &timeout_pos, &short_time); + if (tres != 0) + BIF_ERROR(BIF_P, BADARG); + + return setup_bif_timer(BIF_P, timeout_pos, short_time, + BIF_ARG_2, accessor, BIF_ARG_3, 0); +} + +BIF_RETTYPE start_timer_3(BIF_ALIST_3) +{ + ErtsMonotonicTime timeout_pos; + int short_time, tres; + + tres = parse_timeout_pos(ERTS_PROC_GET_SCHDATA(BIF_P), BIF_ARG_1, NULL, + 0, &timeout_pos, &short_time); + if (tres != 0) + BIF_ERROR(BIF_P, BADARG); + + return setup_bif_timer(BIF_P, timeout_pos, short_time, + BIF_ARG_2, BIF_ARG_2, BIF_ARG_3, !0); +} + +BIF_RETTYPE start_timer_4(BIF_ALIST_4) +{ + ErtsMonotonicTime timeout_pos; + Eterm accessor; + int short_time, abs, tres; + + if (!parse_bif_timer_options(BIF_ARG_4, NULL, NULL, &abs, &accessor)) + BIF_ERROR(BIF_P, BADARG); + + tres = parse_timeout_pos(ERTS_PROC_GET_SCHDATA(BIF_P), BIF_ARG_1, NULL, + abs, &timeout_pos, &short_time); + if (tres != 0) + BIF_ERROR(BIF_P, BADARG); + + return setup_bif_timer(BIF_P, timeout_pos, short_time, + BIF_ARG_2, accessor, BIF_ARG_3, !0); +} + +BIF_RETTYPE cancel_timer_1(BIF_ALIST_1) +{ + return access_bif_timer(BIF_P, BIF_ARG_1, 1, 0, 1); +} + +BIF_RETTYPE cancel_timer_2(BIF_ALIST_2) +{ + BIF_RETTYPE ret; + int async, info; + + if (parse_bif_timer_options(BIF_ARG_2, &async, &info, NULL, NULL)) + return access_bif_timer(BIF_P, BIF_ARG_1, 1, async, info); + + ERTS_BIF_PREP_ERROR(ret, BIF_P, BADARG); + return ret; +} + +BIF_RETTYPE read_timer_1(BIF_ALIST_1) +{ + return access_bif_timer(BIF_P, BIF_ARG_1, 0, 0, 1); +} + +BIF_RETTYPE read_timer_2(BIF_ALIST_2) +{ + BIF_RETTYPE ret; + int async; + + if (parse_bif_timer_options(BIF_ARG_2, &async, NULL, NULL, NULL)) + return access_bif_timer(BIF_P, BIF_ARG_1, 0, async, 1); + + ERTS_BIF_PREP_ERROR(ret, BIF_P, BADARG); + return ret; +} + +static void +start_callback_timer(ErtsSchedulerData *esdp, + int twt, + ErtsMonotonicTime timeout_pos, + void (*callback)(void *), + void *arg) + +{ + if (twt) + create_tw_timer(esdp, ERTS_TMR_CALLBACK, NULL, + callback, arg, timeout_pos); + else + create_hl_timer(esdp, timeout_pos, 0, + ERTS_TMR_CALLBACK, NULL, + NIL, THE_NON_VALUE, NIL, + NULL, callback, arg); +} + +typedef struct { + int twt; + ErtsMonotonicTime timeout_pos; + void (*callback)(void *); + void *arg; +} ErtsStartCallbackTimerRequest; + +static void +scheduled_start_callback_timer(void *vsctr) +{ + ErtsStartCallbackTimerRequest *sctr + = (ErtsStartCallbackTimerRequest *) vsctr; + + start_callback_timer(erts_get_scheduler_data(), + sctr->twt, + sctr->timeout_pos, + sctr->callback, + sctr->arg); + + erts_free(ERTS_ALC_T_TIMER_REQUEST, vsctr); +} + +void +erts_start_timer_callback(ErtsMonotonicTime tmo, + void (*callback)(void *), + void *arg) +{ + ErtsSchedulerData *esdp; + ErtsMonotonicTime timeout_pos; + int twt; + + esdp = erts_get_scheduler_data(); + timeout_pos = get_timeout_pos(erts_get_monotonic_time(esdp), + tmo); + twt = tmo < ERTS_TIMER_WHEEL_MSEC; + + if (esdp) + start_callback_timer(esdp, + twt, + timeout_pos, + callback, + arg); + else { + ErtsStartCallbackTimerRequest *sctr; + sctr = erts_alloc(ERTS_ALC_T_TIMER_REQUEST, + sizeof(ErtsStartCallbackTimerRequest)); + sctr->twt = twt; + sctr->timeout_pos = timeout_pos; + sctr->callback = callback; + sctr->arg = arg; + erts_schedule_misc_aux_work(1, + scheduled_start_callback_timer, + (void *) sctr); + } +} + +/* + * Process and Port timer functionality. + * + * NOTE! These are only allowed to be called by a + * scheduler thread that currently is + * executing the process or port. + */ + +static ERTS_INLINE void +set_proc_timer_common(Process *c_p, ErtsSchedulerData *esdp, Sint64 tmo, + ErtsMonotonicTime timeout_pos, int short_time) +{ + void *tmr; + check_canceled_queue(esdp, esdp->timer_service); + + if (tmo == 0) + c_p->flags |= F_TIMO; + else { + + c_p->flags |= F_INSLPQUEUE; + c_p->flags &= ~F_TIMO; + + if (tmo < ERTS_TIMER_WHEEL_MSEC) + tmr = (void *) create_tw_timer(esdp, ERTS_TMR_PROC, (void *) c_p, + NULL, NULL, timeout_pos); + else + tmr = (void *) create_hl_timer(esdp, timeout_pos, short_time, + ERTS_TMR_PROC, (void *) c_p, + c_p->common.id, THE_NON_VALUE, + NIL, NULL, NULL, NULL); + erts_smp_atomic_set_relb(&c_p->common.timer, (erts_aint_t) tmr); + } +} + +int +erts_set_proc_timer_term(Process *c_p, Eterm etmo) +{ + ErtsSchedulerData *esdp = ERTS_PROC_GET_SCHDATA(c_p); + ErtsMonotonicTime tmo, timeout_pos; + int short_time, tres; + + ERTS_HLT_ASSERT(erts_smp_atomic_read_nob(&c_p->common.timer) + == ERTS_PTMR_NONE); + + tres = parse_timeout_pos(esdp, etmo, &tmo, 0, + &timeout_pos, &short_time); + if (tres != 0) + return tres; + + if ((tmo >> 32) != 0) + return 1; + + set_proc_timer_common(c_p, esdp, tmo, timeout_pos, short_time); + return 0; +} + +void +erts_set_proc_timer_uword(Process *c_p, UWord tmo) +{ + ErtsSchedulerData *esdp = ERTS_PROC_GET_SCHDATA(c_p); + + ERTS_HLT_ASSERT(erts_smp_atomic_read_nob(&c_p->common.timer) + == ERTS_PTMR_NONE); + +#ifndef ARCH_32 + ERTS_HLT_ASSERT((tmo >> 32) == (UWord) 0); +#endif + + if (tmo == 0) + c_p->flags |= F_TIMO; + else { + ErtsMonotonicTime timeout_pos; + timeout_pos = get_timeout_pos(erts_get_monotonic_time(esdp), + (ErtsMonotonicTime) tmo); + set_proc_timer_common(c_p, esdp, (ErtsMonotonicTime) tmo, + timeout_pos, + tmo < ERTS_BIF_TIMER_SHORT_TIME); + } +} + +void +erts_cancel_proc_timer(Process *c_p) +{ + erts_aint_t tval; + tval = erts_smp_atomic_xchg_acqb(&c_p->common.timer, + ERTS_PTMR_NONE); + c_p->flags &= ~(F_INSLPQUEUE|F_TIMO); + if (tval == ERTS_PTMR_NONE) + return; + if (tval == ERTS_PTMR_TIMEDOUT) { + erts_smp_atomic_set_nob(&c_p->common.timer, ERTS_PTMR_NONE); + return; + } + continue_cancel_ptimer(ERTS_PROC_GET_SCHDATA(c_p), + (ErtsTimer *) tval); +} + +void +erts_set_port_timer(Port *c_prt, Sint64 tmo) +{ + void *tmr; + ErtsSchedulerData *esdp = erts_get_scheduler_data(); + ErtsMonotonicTime timeout_pos; + + if (erts_smp_atomic_read_nob(&c_prt->common.timer) != ERTS_PTMR_NONE) + erts_cancel_port_timer(c_prt); + + check_canceled_queue(esdp, esdp->timer_service); + + timeout_pos = get_timeout_pos(erts_get_monotonic_time(esdp), tmo); + + if (tmo < ERTS_TIMER_WHEEL_MSEC) + tmr = (void *) create_tw_timer(esdp, ERTS_TMR_PORT, (void *) c_prt, + NULL, NULL, timeout_pos); + else + tmr = (void *) create_hl_timer(esdp, timeout_pos, 0, ERTS_TMR_PORT, + (void *) c_prt, c_prt->common.id, + THE_NON_VALUE, NIL, NULL, NULL, NULL); + erts_smp_atomic_set_relb(&c_prt->common.timer, (erts_aint_t) tmr); +} + +void +erts_cancel_port_timer(Port *c_prt) +{ + erts_aint_t tval; + tval = erts_smp_atomic_xchg_acqb(&c_prt->common.timer, + ERTS_PTMR_NONE); + if (tval == ERTS_PTMR_NONE) + return; + if (tval == ERTS_PTMR_TIMEDOUT) { + while (!erts_port_task_is_scheduled(&c_prt->timeout_task)) + erts_thr_yield(); + erts_port_task_abort(&c_prt->timeout_task); + erts_smp_atomic_set_nob(&c_prt->common.timer, ERTS_PTMR_NONE); + return; + } + continue_cancel_ptimer(erts_get_scheduler_data(), + (ErtsTimer *) tval); +} + +Sint64 +erts_read_port_timer(Port *c_prt) +{ + ErtsTimer *tmr; + erts_aint_t itmr; + ErtsMonotonicTime timeout_pos; + + itmr = erts_smp_atomic_read_acqb(&c_prt->common.timer); + if (itmr == ERTS_PTMR_NONE) + return (Sint64) -1; + if (itmr == ERTS_PTMR_TIMEDOUT) + return (Sint64) 0; + tmr = (ErtsTimer *) itmr; + if (tmr->head.roflgs & ERTS_TMR_ROFLG_HLT) + timeout_pos = tmr->hlt.timeout; + else + timeout_pos = tmr->twt.tw_tmr.timeout_pos; + return get_time_left(NULL, timeout_pos); +} + +/* + * Debug stuff... + */ + +typedef struct { + int to; + void *to_arg; + ErtsMonotonicTime now; +} ErtsBTMPrint; + +static void +btm_print(ErtsHLTimer *tmr, void *vbtmp) +{ + ErtsBTMPrint *btmp = (ErtsBTMPrint *) vbtmp; + ErtsMonotonicTime left; + Eterm receiver; + + if (tmr->timeout <= btmp->now) + left = 0; + left = ERTS_CLKTCKS_TO_MSEC(tmr->timeout - btmp->now); + + receiver = ((tmr->head.roflgs & ERTS_TMR_ROFLG_REG_NAME) + ? tmr->receiver.name + : tmr->receiver.proc->common.id); + + erts_print(btmp->to, btmp->to_arg, + "=timer:%T\n" + "Message: %T\n" + "Time left: %b64d\n", + receiver, + tmr->btm.message, + (Sint64) left); +} + +void +erts_print_bif_timer_info(int to, void *to_arg) +{ + ErtsBTMPrint btmp; + int six; + + if (!ERTS_IS_CRASH_DUMPING) + ERTS_INTERNAL_ERROR("Not crash dumping"); + + btmp.to = to; + btmp.to_arg = to_arg; + btmp.now = erts_get_monotonic_time(NULL); + btmp.now = ERTS_MONOTONIC_TO_CLKTCKS(btmp.now); + + for (six = 0; six < erts_no_schedulers; six++) { + ErtsHLTimerService *srv = + erts_aligned_scheduler_data[six].esd.timer_service; + btm_rbt_foreach(srv->btm_tree, btm_print, (void *) &btmp); + } +} + +typedef struct { + void (*func)(Eterm, + Eterm, + ErlHeapFragment *, + void *); + void *arg; +} ErtsBTMForeachDebug; + +static void +debug_btm_foreach(ErtsHLTimer *tmr, void *vbtmfd) +{ + if (erts_smp_atomic32_read_nob(&tmr->state) == ERTS_TMR_STATE_ACTIVE) { + ErtsBTMForeachDebug *btmfd = (ErtsBTMForeachDebug *) vbtmfd; + (*btmfd->func)(((tmr->head.roflgs & ERTS_TMR_ROFLG_REG_NAME) + ? tmr->receiver.name + : tmr->receiver.proc->common.id), + tmr->btm.message, + tmr->btm.bp, + btmfd->arg); + } +} + +void +erts_debug_bif_timer_foreach(void (*func)(Eterm, + Eterm, + ErlHeapFragment *, + void *), + void *arg) +{ + ErtsBTMForeachDebug btmfd; + int six; + + btmfd.func = func; + btmfd.arg = arg; + + if (!erts_smp_thr_progress_is_blocking()) + ERTS_INTERNAL_ERROR("Not blocking thread progress"); + + for (six = 0; six < erts_no_schedulers; six++) { + ErtsHLTimerService *srv = + erts_aligned_scheduler_data[six].esd.timer_service; + btm_rbt_foreach(srv->btm_tree, + debug_btm_foreach, + (void *) &btmfd); + } +} + +typedef struct { + void (*tclbk)(void *); + void (*func)(void *, + ErtsMonotonicTime, + void *); + void *arg; +} ErtsDebugForeachCallbackTimer; + +static void +debug_callback_timer_foreach_list(ErtsHLTimer *tmr, void *vdfct) +{ + ErtsDebugForeachCallbackTimer *dfct + = (ErtsDebugForeachCallbackTimer *) vdfct; + + if ((tmr->head.roflgs & ERTS_TMR_ROFLG_CALLBACK) + && (tmr->receiver.callback == dfct->tclbk)) + (*dfct->func)(dfct->arg, + tmr->timeout, + tmr->head.u.arg); +} + +static void +debug_callback_timer_foreach(ErtsHLTimer *tmr, void *vdfct) +{ + ErtsDebugForeachCallbackTimer *dfct + = (ErtsDebugForeachCallbackTimer *) vdfct; + + if (tmr->time.tree.same_time) + same_time_list_foreach(tmr->time.tree.same_time, + debug_callback_timer_foreach_list, + vdfct); + + if ((tmr->head.roflgs & ERTS_TMR_ROFLG_CALLBACK) + && (tmr->receiver.callback == dfct->tclbk)) + (*dfct->func)(dfct->arg, + tmr->timeout, + tmr->head.u.arg); +} + +static void +debug_tw_callback_timer(void *vdfct, + ErtsMonotonicTime timeout_pos, + void *vtwtp) +{ + ErtsTWTimer *twtp = (ErtsTWTimer *) vtwtp; + ErtsDebugForeachCallbackTimer *dfct + = (ErtsDebugForeachCallbackTimer *) vdfct; + + if (twtp->u.callback == dfct->tclbk) + (*dfct->func)(dfct->arg, + timeout_pos, + twtp->head.u.arg); +} + +void +erts_debug_callback_timer_foreach(void (*tclbk)(void *), + void (*func)(void *, + ErtsMonotonicTime, + void *), + void *arg) +{ + int six; + ErtsDebugForeachCallbackTimer dfct; + + dfct.tclbk = tclbk; + dfct.func = func; + dfct.arg = arg; + + if (!erts_smp_thr_progress_is_blocking()) + ERTS_INTERNAL_ERROR("Not blocking thread progress"); + + for (six = 0; six < erts_no_schedulers; six++) { + ErtsHLTimerService *srv = + erts_aligned_scheduler_data[six].esd.timer_service; + ErtsTimerWheel *twheel = + erts_aligned_scheduler_data[six].esd.timer_wheel; + + erts_twheel_debug_foreach(twheel, + tw_callback_timeout, + debug_tw_callback_timer, + (void *) &dfct); + + if (srv->yield.root) + debug_callback_timer_foreach(srv->yield.root, + (void *) &dfct); + + time_rbt_foreach(srv->time_tree, + debug_callback_timer_foreach, + (void *) &dfct); + } +} + +#ifdef ERTS_HLT_HARD_DEBUG + +typedef struct { + ErtsHLTimerService *srv; + int found_root; + ErtsHLTimer **rootpp; +} ErtsHdbgHLT; + +static void +st_hdbg_func(ErtsHLTimer *tmr, void *vhdbg) +{ + ErtsHdbgHLT *hdbg = (ErtsHdbgHLT *) vhdbg; + ErtsHLTimer **rootpp; + ERTS_HLT_ASSERT(tmr->time.tree.parent & ERTS_HLT_PFLG_SAME_TIME); + if (tmr->time.tree.parent == ERTS_HLT_PFLG_SAME_TIME) { + ERTS_HLT_ASSERT(tmr != *hdbg->rootpp); + } + else { + rootpp = (ErtsHLTimer **) (tmr->time.tree.parent + & ~ERTS_HLT_PFLG_SAME_TIME); + ERTS_HLT_ASSERT(rootpp == hdbg->rootpp); + ERTS_HLT_ASSERT(tmr == *rootpp); + ERTS_HLT_ASSERT(!hdbg->found_root); + hdbg->found_root = 1; + } + ERTS_HLT_ASSERT(tmr->time.tree.u.l.next->time.tree.u.l.prev == tmr); + ERTS_HLT_ASSERT(tmr->time.tree.u.l.prev->time.tree.u.l.next == tmr); + ERTS_HLT_ASSERT(btm_rbt_lookup(hdbg->srv->btm_tree, tmr->btm.refn) == tmr); +} + +static void +tt_hdbg_func(ErtsHLTimer *tmr, void *vhdbg) +{ + ErtsHdbgHLT *hdbg = (ErtsHdbgHLT *) vhdbg; + ErtsHLTimer *prnt; + ERTS_HLT_ASSERT((tmr->time.tree.parent & ERTS_HLT_PFLG_SAME_TIME) == 0); + prnt = (ErtsHLTimer *) (tmr->time.tree.parent & ~ERTS_HLT_PFLGS_MASK); + if (prnt) { + ERTS_HLT_ASSERT(prnt->time.tree.u.t.left == tmr + || prnt->time.tree.u.t.right == tmr); + } + else { + ERTS_HLT_ASSERT(!hdbg->found_root); + hdbg->found_root = 1; + ERTS_HLT_ASSERT(tmr == *hdbg->rootpp); + } + if (tmr->time.tree.u.t.left) { + prnt = (ErtsHLTimer *) (tmr->time.tree.u.t.left->time.tree.parent + & ~ERTS_HLT_PFLGS_MASK); + ERTS_HLT_ASSERT(tmr == prnt); + } + if (tmr->time.tree.u.t.right) { + prnt = (ErtsHLTimer *) (tmr->time.tree.u.t.right->time.tree.parent + & ~ERTS_HLT_PFLGS_MASK); + ERTS_HLT_ASSERT(tmr == prnt); + } + if (tmr->head.roflgs & ERTS_TMR_ROFLG_BIF_TMR) + ERTS_HLT_ASSERT(btm_rbt_lookup(hdbg->srv->btm_tree, tmr->btm.refn) == tmr); + if (tmr->time.tree.same_time) { + ErtsHdbgHLT st_hdbg; + st_hdbg.srv = hdbg->srv; + st_hdbg.found_root = 0; + st_hdbg.rootpp = &tmr->time.tree.same_time; + same_time_list_foreach(tmr->time.tree.same_time, st_hdbg_func, (void *) &st_hdbg); + ERTS_HLT_ASSERT(st_hdbg.found_root); + } +} + +static void +bt_hdbg_func(ErtsHLTimer *tmr, void *vhdbg) +{ + ErtsHdbgHLT *hdbg = (ErtsHdbgHLT *) vhdbg; + ErtsHLTimer *prnt; + ERTS_HLT_ASSERT((tmr->btm.tree.parent & ERTS_HLT_PFLG_SAME_TIME) == 0); + prnt = (ErtsHLTimer *) (tmr->btm.tree.parent & ~ERTS_HLT_PFLGS_MASK); + if (prnt) { + ERTS_HLT_ASSERT(prnt->btm.tree.left == tmr + || prnt->btm.tree.right == tmr); + } + else { + ERTS_HLT_ASSERT(!hdbg->found_root); + hdbg->found_root = 1; + ERTS_HLT_ASSERT(tmr == *hdbg->rootpp); + } + if (tmr->btm.tree.left) { + prnt = (ErtsHLTimer *) (tmr->btm.tree.left->btm.tree.parent + & ~ERTS_HLT_PFLGS_MASK); + ERTS_HLT_ASSERT(tmr == prnt); + } + if (tmr->btm.tree.right) { + prnt = (ErtsHLTimer *) (tmr->btm.tree.right->btm.tree.parent + & ~ERTS_HLT_PFLGS_MASK); + ERTS_HLT_ASSERT(tmr == prnt); + } + if (tmr->pending_timeout) { + if (tmr->pending_timeout > 0) /* container > 0 */ + ERTS_HLT_ASSERT(tmr->time.tree.parent == ERTS_HLT_PFIELD_NOT_IN_TABLE); + else { + ERTS_HLT_ASSERT(tmr->time.tree.parent != ERTS_HLT_PFIELD_NOT_IN_TABLE); + ERTS_HLT_ASSERT(tmr->time.tree.parent & ERTS_HLT_PFLG_SAME_TIME); + } + } + else { + ErtsHLTimer *ttmr = time_rbt_lookup(hdbg->srv->time_tree, tmr->timeout); + ERTS_HLT_ASSERT(ttmr); + if (ttmr != tmr) { + ERTS_HLT_ASSERT(ttmr->time.tree.same_time); + ERTS_HLT_ASSERT(tmr == same_time_list_lookup(ttmr->time.tree.same_time, tmr)); + } + } +} + +static void +hdbg_chk_srv(ErtsHLTimerService *srv) +{ + if (srv->time_tree) { + ErtsHdbgHLT hdbg; + hdbg.srv = srv; + hdbg.found_root = 0; + hdbg.rootpp = &srv->time_tree; + time_rbt_foreach(srv->time_tree, tt_hdbg_func, (void *) &hdbg); + ERTS_HLT_ASSERT(hdbg.found_root); + } + if (srv->btm_tree) { + ErtsHdbgHLT hdbg; + hdbg.srv = srv; + hdbg.found_root = 0; + hdbg.rootpp = &srv->btm_tree; + btm_rbt_foreach(srv->btm_tree, bt_hdbg_func, (void *) &hdbg); + ERTS_HLT_ASSERT(hdbg.found_root); + } +} + +#endif /* ERTS_HLT_HARD_DEBUG */ diff --git a/erts/emulator/beam/erl_hl_timer.h b/erts/emulator/beam/erl_hl_timer.h new file mode 100644 index 0000000000..0931bb8965 --- /dev/null +++ b/erts/emulator/beam/erl_hl_timer.h @@ -0,0 +1,88 @@ +/* + * %CopyrightBegin% + * + * Copyright Ericsson AB 2015. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * %CopyrightEnd% + */ + +#ifndef ERL_HL_TIMER_H__ +#define ERL_HL_TIMER_H__ + +typedef struct ErtsHLTimer_ ErtsBifTimers; +typedef struct ErtsHLTimerService_ ErtsHLTimerService; + +#include "sys.h" +#include "erl_process.h" +#define ERL_PORT_GET_PORT_TYPE_ONLY__ +#include "erl_port.h" +#undef ERL_PORT_GET_PORT_TYPE_ONLY__ +#include "erl_message.h" +#include "erl_alloc_types.h" + +#define ERTS_PTMR_NONE ((erts_aint_t) NULL) +#define ERTS_PTMR_TIMEDOUT (ERTS_PTMR_NONE + ((erts_aint_t) 1)) + +#define ERTS_PTMR_INIT(P) \ + erts_smp_atomic_init_nob(&(P)->common.timer, ERTS_PTMR_NONE) +#define ERTS_PTMR_IS_SET(P) \ + (ERTS_PTMR_NONE != erts_smp_atomic_read_nob(&(P)->common.timer)) +#define ERTS_PTMR_IS_TIMED_OUT(P) \ + (ERTS_PTMR_TIMEDOUT == erts_smp_atomic_read_nob(&(P)->common.timer)) + +#define ERTS_PTMR_CLEAR(P) \ + do { \ + ASSERT(ERTS_PTMR_IS_TIMED_OUT((P))); \ + erts_smp_atomic_set_nob(&(P)->common.timer, \ + ERTS_PTMR_NONE); \ + } while (0) + +size_t erts_timer_type_size(ErtsAlcType_t type); +int erts_set_proc_timer_term(Process *, Eterm); +void erts_set_proc_timer_uword(Process *, UWord); +void erts_cancel_proc_timer(Process *); +void erts_set_port_timer(Port *, Sint64); +void erts_cancel_port_timer(Port *); +Sint64 erts_read_port_timer(Port *); +int erts_cancel_bif_timers(Process *, ErtsBifTimers *, void **); +int erts_detach_accessor_bif_timers(Process *, ErtsBifTimers *, void **); +ErtsHLTimerService *erts_create_timer_service(void); +void erts_hl_timer_init(void); +void erts_start_timer_callback(ErtsMonotonicTime, + void (*)(void *), + void *); +#ifdef ERTS_SMP +void +erts_handle_canceled_timers(void *vesdp, + int *need_thr_progress, + ErtsThrPrgrVal *thr_prgr_p, + int *need_more_work); +#endif + +Uint erts_bif_timer_memory_size(void); +void erts_print_bif_timer_info(int to, void *to_arg); + +void erts_debug_bif_timer_foreach(void (*func)(Eterm, + Eterm, + ErlHeapFragment *, + void *), + void *arg); +void +erts_debug_callback_timer_foreach(void (*tclbk)(void *), + void (*func)(void *, + ErtsMonotonicTime, + void *), + void *arg); +#endif /* ERL_HL_TIMER_H__ */ diff --git a/erts/emulator/beam/erl_init.c b/erts/emulator/beam/erl_init.c index d54658f1ea..0c81a95705 100644 --- a/erts/emulator/beam/erl_init.c +++ b/erts/emulator/beam/erl_init.c @@ -3,16 +3,17 @@ * * Copyright Ericsson AB 1997-2013. All Rights Reserved. * - * The contents of this file are subject to the Erlang Public License, - * Version 1.1, (the "License"); you may not use this file except in - * compliance with the License. You should have received a copy of the - * Erlang Public License along with this software. If not, it can be - * retrieved online at http://www.erlang.org/. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Software distributed under the License is distributed on an "AS IS" - * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See - * the License for the specific language governing rights and limitations - * under the License. + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. * * %CopyrightEnd% */ @@ -35,7 +36,7 @@ #include "dist.h" #include "erl_mseg.h" #include "erl_threads.h" -#include "erl_bif_timer.h" +#include "erl_hl_timer.h" #include "erl_instrument.h" #include "erl_printf_term.h" #include "erl_misc_utils.h" @@ -45,6 +46,9 @@ #include "erl_thr_queue.h" #include "erl_async.h" #include "erl_ptab.h" +#include "erl_bif_unique.h" +#define ERTS_WANT_TIMER_WHEEL_API +#include "erl_time.h" #ifdef HIPE #include "hipe_mode_switch.h" /* for hipe_mode_switch_init() */ @@ -87,11 +91,6 @@ const int etp_arch_bits = 32; #else # error "Not 64-bit, nor 32-bit arch" #endif -#if HALFWORD_HEAP -const int etp_halfword = 1; -#else -const int etp_halfword = 0; -#endif #ifdef HIPE const int etp_hipe = 1; #else @@ -117,6 +116,8 @@ const int etp_big_endian = 1; #else const int etp_big_endian = 0; #endif +const Eterm etp_the_non_value = THE_NON_VALUE; + /* * Note about VxWorks: All variables must be initialized by executable code, * not by an initializer. Otherwise a new instance of the emulator will @@ -134,7 +135,11 @@ static void erl_init(int ncpu, int legacy_proc_tab, int port_tab_sz, int port_tab_sz_ignore_files, - int legacy_port_tab); + int legacy_port_tab, + int time_correction, + ErtsTimeWarpMode time_warp_mode, + int node_tab_delete_delay, + ErtsDbSpinCount db_spin_count); static erts_atomic_t exiting; @@ -147,7 +152,7 @@ volatile int erts_writing_erl_crash_dump = 0; int erts_initialized = 0; #if defined(USE_THREADS) && !defined(ERTS_SMP) -static erts_tid_t main_thread; +erts_tid_t erts_main_thread; #endif int erts_use_sender_punish; @@ -161,9 +166,6 @@ int H_MIN_SIZE; /* The minimum heap grain */ int BIN_VH_MIN_SIZE; /* The minimum binary virtual*/ Uint32 erts_debug_flags; /* Debug flags. */ -#ifdef ERTS_OPCODE_COUNTER_SUPPORT -int count_instructions; -#endif int erts_backtrace_depth; /* How many functions to show in a backtrace * in error codes. */ @@ -188,12 +190,10 @@ static int no_dirty_io_schedulers; Uint32 verbose; /* See erl_debug.h for information about verbose */ #endif -int erts_disable_tolerant_timeofday; /* Time correction can be disabled it is - * not and/or it is too slow. - */ - int erts_atom_table_size = ATOM_LIMIT; /* Maximum number of atoms */ +int erts_pd_initial_size = 8; /* must be power of 2 */ + int erts_modified_timing_level; int erts_no_crash_dump = 0; /* Use -d to suppress crash dump. */ @@ -264,11 +264,24 @@ this_rel_num(void) i++; this_rel = atoi(&this_rel_str[i]); if (this_rel < 1) - erl_exit(-1, "Unexpected ERLANG_OTP_RELEASE format\n"); + erts_exit(1, "Unexpected ERLANG_OTP_RELEASE format\n"); } return this_rel; } +static ERTS_INLINE void +set_default_time_adj(int *time_correction_p, ErtsTimeWarpMode *time_warp_mode_p) +{ + *time_correction_p = 1; + *time_warp_mode_p = ERTS_NO_TIME_WARP_MODE; + if (!erts_check_time_adj_support(*time_correction_p, + *time_warp_mode_p)) { + *time_correction_p = 0; + ASSERT(erts_check_time_adj_support(*time_correction_p, + *time_warp_mode_p)); + } +} + /* * Common error printout function, all error messages * that don't go to the error logger go through here. @@ -284,13 +297,24 @@ static int early_init(int *argc, char **argv); void erts_short_init(void) { - int ncpu = early_init(NULL, NULL); + + int ncpu; + int time_correction; + ErtsTimeWarpMode time_warp_mode; + + set_default_time_adj(&time_correction, + &time_warp_mode); + ncpu = early_init(NULL, NULL); erl_init(ncpu, ERTS_DEFAULT_MAX_PROCESSES, 0, ERTS_DEFAULT_MAX_PORTS, 0, - 0); + 0, + time_correction, + time_warp_mode, + ERTS_NODE_TAB_DELAY_GC_DEFAULT, + ERTS_DB_SPNCNT_NORMAL); erts_initialized = 1; } @@ -300,12 +324,17 @@ erl_init(int ncpu, int legacy_proc_tab, int port_tab_sz, int port_tab_sz_ignore_files, - int legacy_port_tab) + int legacy_port_tab, + int time_correction, + ErtsTimeWarpMode time_warp_mode, + int node_tab_delete_delay, + ErtsDbSpinCount db_spin_count) { init_benchmarking(); + erts_bif_unique_init(); erts_init_monitors(); - erts_init_time(); + erts_init_time(time_correction, time_warp_mode); erts_init_sys_common_misc(); erts_init_process(ncpu, proc_tab_sz, legacy_proc_tab); erts_init_scheduling(no_schedulers, @@ -316,6 +345,7 @@ erl_init(int ncpu, no_dirty_io_schedulers #endif ); + erts_late_init_time_sup(); erts_init_cpu_topology(); /* Must be after init_scheduling */ erts_init_gc(); /* Must be after init_scheduling */ erts_alloc_late_init(); @@ -324,7 +354,6 @@ erl_init(int ncpu, BIN_VH_MIN_SIZE = erts_next_heap_size(BIN_VH_MIN_SIZE, 0); erts_init_trace(); - erts_init_binary(); erts_init_bits(); erts_code_ix_init(); erts_init_fun_table(); @@ -337,10 +366,10 @@ erl_init(int ncpu, erts_ddll_init(); init_emulator(); erts_ptab_init(); /* Must be after init_emulator() */ + erts_init_binary(); /* Must be after init_emulator() */ erts_bp_init(); - init_db(); /* Must be after init_emulator */ - erts_bif_timer_init(); - erts_init_node_tables(); + init_db(db_spin_count); /* Must be after init_emulator */ + erts_init_node_tables(node_tab_delete_delay); init_dist(); erl_drv_thr_init(); erts_init_async(); @@ -352,25 +381,29 @@ erl_init(int ncpu, erts_init_bif_re(); erts_init_unicode(); /* after RE to get access to PCRE unicode */ erts_init_external(); + erts_init_map(); erts_delay_trap = erts_export_put(am_erlang, am_delay_trap, 2); erts_late_init_process(); #if HAVE_ERTS_MSEG erts_mseg_late_init(); /* Must be after timer (erts_init_time()) and thread initializations */ #endif + erl_sys_late_init(); #ifdef HIPE hipe_mode_switch_init(); /* Must be after init_load/beam_catches/init */ #endif packet_parser_init(); erl_nif_init(); + erts_msacc_init(); } -static void +static Eterm erl_first_process_otp(char* modname, void* code, unsigned size, int argc, char** argv) { int i; Eterm start_mod; Eterm args; + Eterm res; Eterm* hp; Process parent; ErlSpawnOpts so; @@ -379,7 +412,7 @@ erl_first_process_otp(char* modname, void* code, unsigned size, int argc, char** start_mod = erts_atom_put((byte *) modname, sys_strlen(modname), ERTS_ATOM_ENC_LATIN1, 1); if (erts_find_function(start_mod, am_start, 2, erts_active_code_ix()) == NULL) { - erl_exit(5, "No function %s:start/2\n", modname); + erts_exit(ERTS_ERROR_EXIT, "No function %s:start/2\n", modname); } /* @@ -400,12 +433,36 @@ erl_first_process_otp(char* modname, void* code, unsigned size, int argc, char** hp += 2; args = CONS(hp, env, args); - so.flags = 0; - (void) erl_create_process(&parent, start_mod, am_start, args, &so); + so.flags = erts_default_spo_flags|SPO_SYSTEM_PROC; + res = erl_create_process(&parent, start_mod, am_start, args, &so); erts_smp_proc_unlock(&parent, ERTS_PROC_LOCK_MAIN); erts_cleanup_empty_process(&parent); + return res; +} + +static Eterm +erl_system_process_otp(Eterm parent_pid, char* modname) +{ + Eterm start_mod; + Process* parent; + ErlSpawnOpts so; + Eterm res; + + start_mod = erts_atom_put((byte *) modname, sys_strlen(modname), ERTS_ATOM_ENC_LATIN1, 1); + if (erts_find_function(start_mod, am_start, 0, + erts_active_code_ix()) == NULL) { + erts_exit(ERTS_ERROR_EXIT, "No function %s:start/0\n", modname); + } + + parent = erts_pid2proc(NULL, 0, parent_pid, ERTS_PROC_LOCK_MAIN); + + so.flags = erts_default_spo_flags|SPO_SYSTEM_PROC; + res = erl_create_process(parent, start_mod, am_start, NIL, &so); + erts_smp_proc_unlock(parent, ERTS_PROC_LOCK_MAIN); + return res; } + Eterm erts_preloaded(Process* p) { @@ -475,12 +532,12 @@ load_preloaded(void) length = preload_p[i].size; module_name = erts_atom_put((byte *) name, sys_strlen(name), ERTS_ATOM_ENC_LATIN1, 1); if ((code = sys_preload_begin(&preload_p[i])) == 0) - erl_exit(1, "Failed to find preloaded code for module %s\n", + erts_exit(ERTS_ERROR_EXIT, "Failed to find preloaded code for module %s\n", name); res = erts_preload_module(NULL, 0, NIL, &module_name, code, length); sys_preload_end(&preload_p[i]); if (res != NIL) - erl_exit(1,"Failed loading preloaded module %s (%T)\n", + erts_exit(ERTS_ERROR_EXIT,"Failed loading preloaded module %s (%T)\n", name, res); i++; } @@ -509,9 +566,9 @@ void erts_usage(void) /* erts_fprintf(stderr, "-b func set the boot function (default boot)\n"); */ - erts_fprintf(stderr, "-c disable continuous date/time correction with\n"); - erts_fprintf(stderr, " respect to uptime\n"); - + erts_fprintf(stderr, "-c bool enable or disable time correction\n"); + erts_fprintf(stderr, "-C mode set time warp mode; valid modes are:\n"); + erts_fprintf(stderr, " no_time_warp|single_time_warp|multi_time_warp\n"); erts_fprintf(stderr, "-d don't write a crash dump for internally detected errors\n"); erts_fprintf(stderr, " (halt(String) will still produce a crash dump)\n"); erts_fprintf(stderr, "-fn[u|a|l] Control how filenames are interpreted\n"); @@ -519,6 +576,8 @@ void erts_usage(void) H_DEFAULT_SIZE); erts_fprintf(stderr, "-hmbs size set minimum binary virtual heap size in words (default %d)\n", VH_DEFAULT_SIZE); + erts_fprintf(stderr, "-hpds size initial process dictionary size (default %d)\n", + erts_pd_initial_size); /* erts_fprintf(stderr, "-i module set the boot module (default init)\n"); */ @@ -548,6 +607,8 @@ void erts_usage(void) erts_fprintf(stderr, " see the erl(1) documentation for more info.\n"); erts_fprintf(stderr, "-sct cput set cpu topology,\n"); erts_fprintf(stderr, " see the erl(1) documentation for more info.\n"); + erts_fprintf(stderr, "-secio bool enable/disable eager check I/O scheduling,\n"); + erts_fprintf(stderr, " see the erl(1) documentation for more info.\n"); #if ERTS_HAVE_SCHED_UTIL_BALANCING_SUPPORT_OPT erts_fprintf(stderr, "-sub bool enable/disable scheduler utilization balancing,\n"); #else @@ -592,15 +653,24 @@ void erts_usage(void) erts_fprintf(stderr, "-v turn on chatty mode (GCs will be reported etc)\n"); - erts_fprintf(stderr, "-W<i|w> set error logger warnings mapping,\n"); + erts_fprintf(stderr, "-W<i|w|e> set error logger warnings mapping,\n"); erts_fprintf(stderr, " see error_logger documentation for details\n"); + erts_fprintf(stderr, "-xmqd val set default message queue data flag for processes,\n"); + erts_fprintf(stderr, " valid values are: off_heap | on_heap | mixed\n"); erts_fprintf(stderr, "-zdbbl size set the distribution buffer busy limit in kilobytes\n"); erts_fprintf(stderr, " valid range is [1-%d]\n", INT_MAX/1024); + erts_fprintf(stderr, "-zdntgc time set delayed node table gc in seconds\n"); + erts_fprintf(stderr, " valid values are infinity or intergers in the range [0-%d]\n", + ERTS_NODE_TAB_DELAY_GC_MAX); +#if 0 + erts_fprintf(stderr, "-zebwt val set ets busy wait threshold, valid values are:\n"); + erts_fprintf(stderr, " none|very_short|short|medium|long|very_long|extremely_long\n"); +#endif erts_fprintf(stderr, "\n"); erts_fprintf(stderr, "Note that if the emulator is started with erlexec (typically\n"); erts_fprintf(stderr, "from the erl script), these flags should be specified with +.\n"); erts_fprintf(stderr, "\n\n"); - erl_exit(-1, ""); + erts_exit(1, ""); } #ifdef USE_THREADS @@ -675,11 +745,14 @@ early_init(int *argc, char **argv) /* char envbuf[21]; /* enough for any 64-bit integer */ size_t envbufsz; +#if defined(USE_THREADS) && !defined(ERTS_SMP) + erts_main_thread = erts_thr_self(); +#endif + erts_save_emu_args(*argc, argv); erts_sched_compact_load = 1; erts_printf_eterm_func = erts_printf_term; - erts_disable_tolerant_timeofday = 0; display_items = 200; erts_backtrace_depth = DEFAULT_BACKTRACE_SIZE; erts_async_max_threads = ERTS_DEFAULT_NO_ASYNC_THREADS; @@ -729,9 +802,6 @@ early_init(int *argc, char **argv) /* (erts_aint32_t) ((Uint16) -1)); erts_pre_init_process(); -#if defined(USE_THREADS) && !defined(ERTS_SMP) - main_thread = erts_thr_self(); -#endif /* * We need to know the number of schedulers to use before we @@ -1125,6 +1195,7 @@ early_init(int *argc, char **argv) /* erts_thr_late_init(&elid); } #endif + erts_msacc_early_init(); #ifdef ERTS_ENABLE_LOCK_CHECK erts_lc_late_init(); @@ -1185,7 +1256,14 @@ erl_start(int argc, char **argv) int port_tab_sz_ignore_files = 0; int legacy_proc_tab = 0; int legacy_port_tab = 0; + int time_correction; + ErtsTimeWarpMode time_warp_mode; + int node_tab_delete_delay = ERTS_NODE_TAB_DELAY_GC_DEFAULT; + ErtsDbSpinCount db_spin_count = ERTS_DB_SPNCNT_NORMAL; + Eterm otp_ring0_pid; + set_default_time_adj(&time_correction, + &time_warp_mode); envbufsz = sizeof(envbuf); if (erts_sys_getenv_raw(ERL_MAX_ETS_TABLES_ENV, envbuf, &envbufsz) == 0) @@ -1217,7 +1295,7 @@ erl_start(int argc, char **argv) verbose = DEBUG_DEFAULT; #endif - erts_error_logger_warnings = am_error; + erts_error_logger_warnings = am_warning; while (i < argc) { if (argv[i][0] != '-') { @@ -1352,6 +1430,7 @@ erl_start(int argc, char **argv) case 't': verbose |= DEBUG_THREADS; break; case 'p': verbose |= DEBUG_PROCESSES; break; case 'm': verbose |= DEBUG_MESSAGES; break; + case 'c': verbose |= DEBUG_SHCOPY; break; default : erts_fprintf(stderr,"Unknown verbose option: %c\n",*ch); } } @@ -1364,6 +1443,7 @@ erl_start(int argc, char **argv) if (verbose & DEBUG_THREADS) erts_printf("THREADS "); if (verbose & DEBUG_PROCESSES) erts_printf("PROCESSES "); if (verbose & DEBUG_MESSAGES) erts_printf("MESSAGES "); + if (verbose & DEBUG_SHCOPY) erts_printf("SHCOPY "); erts_printf("\n"); #else erts_fprintf(stderr, "warning: -v (only in debug compiled code)\n"); @@ -1392,7 +1472,7 @@ erl_start(int argc, char **argv) } erts_fprintf(stderr, "(" EMULATOR ") emulator version " ERLANG_VERSION "\n"); - erl_exit(0, ""); + erts_exit(0, ""); } break; @@ -1406,6 +1486,7 @@ erl_start(int argc, char **argv) * * h|ms - min_heap_size * h|mbs - min_bin_vheap_size + * h|pds - erts_pd_initial_size * */ if (has_prefix("mbs", sub_param)) { @@ -1423,6 +1504,14 @@ erl_start(int argc, char **argv) erts_usage(); } VERBOSE(DEBUG_SYSTEM, ("using minimum heap size %d\n", H_MIN_SIZE)); + } else if (has_prefix("pds", sub_param)) { + arg = get_arg(sub_param+3, argv[i+1], &i); + if (!erts_pd_set_initial_size(atoi(arg))) { + erts_fprintf(stderr, "bad initial process dictionary size %s\n", arg); + erts_usage(); + } + VERBOSE(DEBUG_SYSTEM, ("using initial process dictionary size %d\n", + erts_pd_initial_size)); } else { /* backward compatibility */ arg = get_arg(argv[i]+2, argv[i+1], &i); @@ -1674,6 +1763,22 @@ erl_start(int argc, char **argv) erts_usage(); } } + else if (has_prefix("ecio", sub_param)) { + arg = get_arg(sub_param+4, argv[i+1], &i); +#ifndef __OSE__ + if (sys_strcmp("true", arg) == 0) + erts_eager_check_io = 1; + else +#endif + if (sys_strcmp("false", arg) == 0) + erts_eager_check_io = 0; + else { + erts_fprintf(stderr, + "bad schedule eager check I/O value '%s'\n", + arg); + erts_usage(); + } + } else if (has_prefix("pp", sub_param)) { arg = get_arg(sub_param+2, argv[i+1], &i); if (sys_strcmp(arg, "true") == 0) @@ -1878,15 +1983,51 @@ erl_start(int argc, char **argv) } break; } + case 'C': + arg = get_arg(argv[i]+2, argv[i+1], &i); + if (sys_strcmp(arg, "no_time_warp") == 0) + time_warp_mode = ERTS_NO_TIME_WARP_MODE; + else if (sys_strcmp(arg, "single_time_warp") == 0) + time_warp_mode = ERTS_SINGLE_TIME_WARP_MODE; + else if (sys_strcmp(arg, "multi_time_warp") == 0) + time_warp_mode = ERTS_MULTI_TIME_WARP_MODE; + else { + erts_fprintf(stderr, + "Invalid time warp mode: %s\n", arg); + erts_usage(); + } + break; case 'c': - if (argv[i][2] == 0) { /* -c: documented option */ - erts_disable_tolerant_timeofday = 1; + if (sys_strcmp(argv[i]+2, "false") == 0) + goto time_correction_false; + else if (sys_strcmp(argv[i]+2, "true") == 0) + goto time_correction_true; + else if (argv[i][2] == '\0') { + if (i + 1 >= argc) + goto time_correction_false; + else { + if (sys_strcmp(argv[i+1], "false") == 0) { + (void) get_arg(argv[i]+2, argv[i+1], &i); + goto time_correction_false; + } + else if (sys_strcmp(argv[i+1], "true") == 0) { + (void) get_arg(argv[i]+2, argv[i+1], &i); + time_correction_true: + time_correction = 1; + break; + } + else { + time_correction_false: + time_correction = 0; + break; + } + } } -#ifdef ERTS_OPCODE_COUNTER_SUPPORT - else if (argv[i][2] == 'i') { /* -ci: undcoumented option*/ - count_instructions = 1; + else { + arg = get_arg(argv[i]+2, argv[i+1], &i); + erts_fprintf(stderr, "Invalid time correnction value: %s\n", arg); + erts_usage(); } -#endif break; case 'W': arg = get_arg(argv[i]+2, argv[i+1], &i); @@ -1894,22 +2035,49 @@ erl_start(int argc, char **argv) case 'i': erts_error_logger_warnings = am_info; break; + case 'e': + erts_error_logger_warnings = am_error; + break; case 'w': erts_error_logger_warnings = am_warning; break; - case 'e': /* The default */ - erts_error_logger_warnings = am_error; default: erts_fprintf(stderr, "unrecognized warning_map option %s\n", arg); erts_usage(); } break; + case 'x': { + char *sub_param = argv[i]+2; + if (has_prefix("mqd", sub_param)) { + arg = get_arg(sub_param+3, argv[i+1], &i); + if (sys_strcmp(arg, "mixed") == 0) + erts_default_spo_flags &= ~(SPO_ON_HEAP_MSGQ|SPO_OFF_HEAP_MSGQ); + else if (sys_strcmp(arg, "on_heap") == 0) { + erts_default_spo_flags &= ~SPO_OFF_HEAP_MSGQ; + erts_default_spo_flags |= SPO_ON_HEAP_MSGQ; + } + else if (sys_strcmp(arg, "off_heap") == 0) { + erts_default_spo_flags &= ~SPO_ON_HEAP_MSGQ; + erts_default_spo_flags |= SPO_OFF_HEAP_MSGQ; + } + else { + erts_fprintf(stderr, + "Invalid message_queue_data flag: %s\n", arg); + erts_usage(); + } + } else { + erts_fprintf(stderr, "bad -x option %s\n", argv[i]); + erts_usage(); + } + break; + } + case 'z': { char *sub_param = argv[i]+2; - int new_limit; if (has_prefix("dbbl", sub_param)) { + int new_limit; arg = get_arg(sub_param+4, argv[i+1], &i); new_limit = atoi(arg); if (new_limit < 1 || INT_MAX/1024 < new_limit) { @@ -1918,7 +2086,48 @@ erl_start(int argc, char **argv) } else { erts_dist_buf_busy_limit = new_limit*1024; } - } else { + } + else if (has_prefix("dntgc", sub_param)) { + long secs; + + arg = get_arg(sub_param+5, argv[i+1], &i); + if (sys_strcmp(arg, "infinity") == 0) + secs = ERTS_NODE_TAB_DELAY_GC_INFINITY; + else { + char *endptr; + errno = 0; + secs = strtol(arg, &endptr, 10); + if (errno != 0 || *arg == '\0' || *endptr != '\0' + || secs < 0 || ERTS_NODE_TAB_DELAY_GC_MAX < secs) { + erts_fprintf(stderr, "Invalid delayed node table gc: %s\n", arg); + erts_usage(); + } + } + node_tab_delete_delay = (int) secs; + } + else if (has_prefix("ebwt", sub_param)) { + arg = get_arg(sub_param+4, argv[i+1], &i); + if (sys_strcmp(arg, "none") == 0) + db_spin_count = ERTS_DB_SPNCNT_NONE; + else if (sys_strcmp(arg, "very_short") == 0) + db_spin_count = ERTS_DB_SPNCNT_VERY_LOW; + else if (sys_strcmp(arg, "short") == 0) + db_spin_count = ERTS_DB_SPNCNT_LOW; + else if (sys_strcmp(arg, "medium") == 0) + db_spin_count = ERTS_DB_SPNCNT_NORMAL; + else if (sys_strcmp(arg, "long") == 0) + db_spin_count = ERTS_DB_SPNCNT_HIGH; + else if (sys_strcmp(arg, "very_long") == 0) + db_spin_count = ERTS_DB_SPNCNT_VERY_HIGH; + else if (sys_strcmp(arg, "extremely_long") == 0) + db_spin_count = ERTS_DB_SPNCNT_EXTREMELY_HIGH; + else { + erts_fprintf(stderr, + "Invalid ets busy wait threshold: %s\n", arg); + erts_usage(); + } + } + else { erts_fprintf(stderr, "bad -z option %s\n", argv[i]); erts_usage(); } @@ -1932,6 +2141,30 @@ erl_start(int argc, char **argv) i++; } + if (!erts_check_time_adj_support(time_correction, time_warp_mode)) { + char *time_correction_str = time_correction ? "Enabled" : "Disabled"; + char *time_warp_str = "undefined"; + switch (time_warp_mode) { + case ERTS_NO_TIME_WARP_MODE: + time_warp_str = "no"; + break; + case ERTS_SINGLE_TIME_WARP_MODE: + time_warp_str = "single"; + break; + case ERTS_MULTI_TIME_WARP_MODE: + time_warp_str = "multi"; + break; + default: + time_warp_str = "undefined"; + break; + } + erts_fprintf(stderr, "%s time correction with %s time warp mode " + "is not supported on this platform\n", + time_correction_str, + time_warp_str); + erts_usage(); + } + /* Output format on windows for sprintf defaults to three exponents. * We use two-exponent to mimic normal sprintf behaviour. */ @@ -1955,6 +2188,7 @@ erl_start(int argc, char **argv) init_break_handler(); if (replace_intr) erts_replace_intr(); + sys_init_suspend_handler(); #endif boot_argc = argc - i; /* Number of arguments to init */ @@ -1965,7 +2199,11 @@ erl_start(int argc, char **argv) legacy_proc_tab, port_tab_sz, port_tab_sz_ignore_files, - legacy_port_tab); + legacy_port_tab, + time_correction, + time_warp_mode, + node_tab_delete_delay, + db_spin_count); load_preloaded(); erts_end_staging_code_ix(); @@ -1973,7 +2211,10 @@ erl_start(int argc, char **argv) erts_initialized = 1; - erl_first_process_otp("otp_ring0", NULL, 0, boot_argc, boot_argv); + otp_ring0_pid = erl_first_process_otp("otp_ring0", NULL, 0, + boot_argc, boot_argv); + + (void) erl_system_process_otp(otp_ring0_pid, "erts_code_purger"); #ifdef ERTS_SMP erts_start_schedulers(); @@ -1981,13 +2222,18 @@ erl_start(int argc, char **argv) erts_sys_main_thread(); /* May or may not return! */ #else - erts_thr_set_main_status(1, 1); + { + ErtsSchedulerData *esdp = erts_get_scheduler_data(); + erts_msacc_init_thread("scheduler", 1, 1); + erts_thr_set_main_status(1, 1); #if ERTS_USE_ASYNC_READY_Q - erts_get_scheduler_data()->aux_work_data.async_ready.queue - = erts_get_async_ready_queue(1); + esdp->aux_work_data.async_ready.queue + = erts_get_async_ready_queue(1); #endif - set_main_stack_size(); - process_main(); + set_main_stack_size(); + erts_sched_init_time_sup(esdp); + process_main(); + } #endif } @@ -2044,7 +2290,7 @@ system_cleanup(int flush_async) if (!flush_async || !erts_initialized #if defined(USE_THREADS) && !defined(ERTS_SMP) - || !erts_equal_tids(main_thread, erts_thr_self()) + || !erts_equal_tids(erts_main_thread, erts_thr_self()) #endif ) return; @@ -2059,21 +2305,17 @@ system_cleanup(int flush_async) } static __decl_noreturn void __noreturn -erl_exit_vv(int n, int flush_async, char *fmt, va_list args1, va_list args2) +erts_exit_vv(int n, int flush_async, char *fmt, va_list args1, va_list args2) { - unsigned int an; - system_cleanup(flush_async); save_statistics(); - an = abs(n); - if (erts_mtrace_enabled) - erts_mtrace_exit((Uint32) an); + erts_mtrace_exit((Uint32) n); /* Produce an Erlang core dump if error */ - if (((n > 0 && erts_no_crash_dump == 0) || n == ERTS_DUMP_EXIT) + if (((n == ERTS_ERROR_EXIT && erts_no_crash_dump == 0) || n == ERTS_DUMP_EXIT) && erts_initialized) { erl_crash_dump_v((char*) NULL, 0, fmt, args1); } @@ -2086,29 +2328,29 @@ erl_exit_vv(int n, int flush_async, char *fmt, va_list args1, va_list args2) exit(0); else if (n == ERTS_DUMP_EXIT) ERTS_EXIT_AFTER_DUMP(1); - else if (n > 0 || n == ERTS_ABORT_EXIT) + else if (n == ERTS_ERROR_EXIT || n == ERTS_ABORT_EXIT) abort(); - exit(an); + exit(n); } /* Exit without flushing async threads */ -__decl_noreturn void __noreturn erl_exit(int n, char *fmt, ...) +__decl_noreturn void __noreturn erts_exit(int n, char *fmt, ...) { va_list args1, args2; va_start(args1, fmt); va_start(args2, fmt); - erl_exit_vv(n, 0, fmt, args1, args2); + erts_exit_vv(n, 0, fmt, args1, args2); va_end(args2); va_end(args1); } /* Exit after flushing async threads */ -__decl_noreturn void __noreturn erl_exit_flush_async(int n, char *fmt, ...) +__decl_noreturn void __noreturn erts_flush_async_exit(int n, char *fmt, ...) { va_list args1, args2; va_start(args1, fmt); va_start(args2, fmt); - erl_exit_vv(n, 1, fmt, args1, args2); + erts_exit_vv(n, 1, fmt, args1, args2); va_end(args2); va_end(args1); } diff --git a/erts/emulator/beam/erl_instrument.c b/erts/emulator/beam/erl_instrument.c index df7c443387..12a72ad839 100644 --- a/erts/emulator/beam/erl_instrument.c +++ b/erts/emulator/beam/erl_instrument.c @@ -3,16 +3,17 @@ * * Copyright Ericsson AB 2003-2013. All Rights Reserved. * - * The contents of this file are subject to the Erlang Public License, - * Version 1.1, (the "License"); you may not use this file except in - * compliance with the License. You should have received a copy of the - * Erlang Public License along with this software. If not, it can be - * retrieved online at http://www.erlang.org/. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Software distributed under the License is distributed on an "AS IS" - * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See - * the License for the specific language governing rights and limitations - * under the License. + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. * * %CopyrightEnd% */ @@ -1226,7 +1227,7 @@ erts_instr_init(int stat, int map_stat) mem_anchor = NULL; /* Install instrumentation functions */ - ASSERT(sizeof(erts_allctrs) == sizeof(real_allctrs)); + ERTS_CT_ASSERT(sizeof(erts_allctrs) == sizeof(real_allctrs)); sys_memcpy((void *)real_allctrs,(void *)erts_allctrs,sizeof(erts_allctrs)); diff --git a/erts/emulator/beam/erl_instrument.h b/erts/emulator/beam/erl_instrument.h index 37b9b67139..cb3b1920d3 100644 --- a/erts/emulator/beam/erl_instrument.h +++ b/erts/emulator/beam/erl_instrument.h @@ -3,16 +3,17 @@ * * Copyright Ericsson AB 2003-2009. All Rights Reserved. * - * The contents of this file are subject to the Erlang Public License, - * Version 1.1, (the "License"); you may not use this file except in - * compliance with the License. You should have received a copy of the - * Erlang Public License along with this software. If not, it can be - * retrieved online at http://www.erlang.org/. - * - * Software distributed under the License is distributed on an "AS IS" - * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See - * the License for the specific language governing rights and limitations - * under the License. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. * * %CopyrightEnd% */ diff --git a/erts/emulator/beam/erl_lock_check.c b/erts/emulator/beam/erl_lock_check.c index 7e3a90779d..598bf84c0b 100644 --- a/erts/emulator/beam/erl_lock_check.c +++ b/erts/emulator/beam/erl_lock_check.c @@ -3,16 +3,17 @@ * * Copyright Ericsson AB 2005-2013. All Rights Reserved. * - * The contents of this file are subject to the Erlang Public License, - * Version 1.1, (the "License"); you may not use this file except in - * compliance with the License. You should have received a copy of the - * Erlang Public License along with this software. If not, it can be - * retrieved online at http://www.erlang.org/. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Software distributed under the License is distributed on an "AS IS" - * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See - * the License for the specific language governing rights and limitations - * under the License. + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. * * %CopyrightEnd% */ @@ -91,6 +92,7 @@ static erts_lc_lock_order_t erts_lock_order[] = { { "driver_list", NULL }, { "proc_link", "pid" }, { "proc_msgq", "pid" }, + { "proc_btm", "pid" }, { "dist_entry", "address" }, { "dist_entry_links", "address" }, { "code_write_permission", NULL }, @@ -111,9 +113,6 @@ static erts_lc_lock_order_t erts_lock_order[] = { { "environ", NULL }, #endif { "efile_drv", "address" }, -#if defined(ENABLE_CHILD_WAITER_THREAD) || defined(ERTS_SMP) - { "child_status", NULL }, -#endif { "drv_ev_state_grow", NULL, }, { "drv_ev_state", "address" }, { "safe_hash", "address" }, @@ -138,16 +137,17 @@ static erts_lc_lock_order_t erts_lock_order[] = { { "mmap_init_atoms", NULL }, { "drv_tsd", NULL }, { "async_enq_mtx", NULL }, + { "msacc_list_mutex", NULL }, + { "msacc_unmanaged_mutex", NULL }, #ifdef ERTS_SMP - { "sys_msg_q", NULL }, { "atom_tab", NULL }, - { "make_ref", NULL }, { "misc_op_list_pre_alloc_lock", "address" }, { "message_pre_alloc_lock", "address" }, { "ptimer_pre_alloc_lock", "address", }, { "btm_pre_alloc_lock", NULL, }, { "dist_entry_out_queue", "address" }, { "port_sched_lock", "port_id" }, + { "sys_msg_q", NULL }, { "port_table", NULL }, #endif { "mtrace_op", NULL }, @@ -155,9 +155,6 @@ static erts_lc_lock_order_t erts_lock_order[] = { { "instr", NULL }, { "alcu_allocator", "index" }, { "mseg", NULL }, -#if HALFWORD_HEAP - { "pmmap", NULL }, -#endif #ifdef ERTS_SMP { "port_task_pre_alloc_lock", "address" }, { "proclist_pre_alloc_lock", "address" }, @@ -168,6 +165,8 @@ static erts_lc_lock_order_t erts_lock_order[] = { { "timer_wheel", NULL }, { "system_block", NULL }, { "timeofday", NULL }, + { "get_time", NULL }, + { "get_corrected_time", NULL }, { "breakpoints", NULL }, { "pollsets_lock", NULL }, { "pix_lock", "address" }, @@ -184,10 +183,8 @@ static erts_lc_lock_order_t erts_lock_order[] = { { "efile_drv dtrace mutex", NULL }, #endif { "mtrace_buf", NULL }, -#ifdef __WIN32__ #ifdef ERTS_SMP - { "sys_gethrtime", NULL }, -#endif + { "os_monotonic_time", NULL }, #endif { "erts_alloc_hard_debug", NULL }, { "hard_dbg_mseg", NULL }, @@ -227,8 +224,7 @@ rw_op_str(Uint16 flags) case ERTS_LC_FLG_LO_READ: return " (r)"; case ERTS_LC_FLG_LO_WRITE: - erts_fprintf(stderr, "\nInternal error\n"); - lc_abort(); + ERTS_INTERNAL_ERROR("Only write flag present"); default: break; } @@ -270,9 +266,9 @@ union erts_lc_free_block_t_ { static ethr_tsd_key locks_key; -static erts_lc_locked_locks_t *erts_locked_locks; +static erts_lc_locked_locks_t *erts_locked_locks = NULL; -static erts_lc_free_block_t *free_blocks; +static erts_lc_free_block_t *free_blocks = NULL; #ifdef ERTS_LC_STATIC_ALLOC #define ERTS_LC_FB_CHUNK_SIZE 10000 @@ -311,8 +307,7 @@ static ERTS_INLINE void lc_free(void *p) static void *lc_core_alloc(void) { lc_unlock(); - erts_fprintf(stderr, "Lock checker out of memory!\n"); - lc_abort(); + ERTS_INTERNAL_ERROR("Lock checker out of memory!\n"); } #else @@ -325,8 +320,7 @@ static void *lc_core_alloc(void) fbs = (erts_lc_free_block_t *) malloc(sizeof(erts_lc_free_block_t) * ERTS_LC_FB_CHUNK_SIZE); if (!fbs) { - erts_fprintf(stderr, "Lock checker failed to allocate memory!\n"); - lc_abort(); + ERTS_INTERNAL_ERROR("Lock checker failed to allocate memory!"); } for (i = 1; i < ERTS_LC_FB_CHUNK_SIZE - 1; i++) { #ifdef DEBUG @@ -366,11 +360,11 @@ create_locked_locks(char *thread_name) { erts_lc_locked_locks_t *l_lcks = malloc(sizeof(erts_lc_locked_locks_t)); if (!l_lcks) - lc_abort(); + ERTS_INTERNAL_ERROR("Lock checker failed to allocate memory!"); l_lcks->thread_name = strdup(thread_name ? thread_name : "unknown"); if (!l_lcks->thread_name) - lc_abort(); + ERTS_INTERNAL_ERROR("Lock checker failed to allocate memory!"); l_lcks->emu_thread = 0; l_lcks->tid = erts_thr_self(); @@ -691,7 +685,7 @@ erts_lc_set_thread_name(char *thread_name) free((void *) l_lcks->thread_name); l_lcks->thread_name = strdup(thread_name ? thread_name : "unknown"); if (!l_lcks->thread_name) - lc_abort(); + ERTS_INTERNAL_ERROR("strdup failed"); } l_lcks->emu_thread = 1; } @@ -1330,7 +1324,7 @@ erts_lc_init(void) #endif /* #ifdef ERTS_LC_STATIC_ALLOC */ if (ethr_spinlock_init(&free_blocks_lock) != 0) - lc_abort(); + ERTS_INTERNAL_ERROR("spinlock_init failed"); erts_tsd_key_create(&locks_key,"erts_lock_check_key"); } diff --git a/erts/emulator/beam/erl_lock_check.h b/erts/emulator/beam/erl_lock_check.h index 3f7f417e61..66251ef4e8 100644 --- a/erts/emulator/beam/erl_lock_check.h +++ b/erts/emulator/beam/erl_lock_check.h @@ -3,16 +3,17 @@ * * Copyright Ericsson AB 2005-2012. All Rights Reserved. * - * The contents of this file are subject to the Erlang Public License, - * Version 1.1, (the "License"); you may not use this file except in - * compliance with the License. You should have received a copy of the - * Erlang Public License along with this software. If not, it can be - * retrieved online at http://www.erlang.org/. - * - * Software distributed under the License is distributed on an "AS IS" - * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See - * the License for the specific language governing rights and limitations - * under the License. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. * * %CopyrightEnd% */ diff --git a/erts/emulator/beam/erl_lock_count.c b/erts/emulator/beam/erl_lock_count.c index 6f44bf097b..bd00480ba2 100644 --- a/erts/emulator/beam/erl_lock_count.c +++ b/erts/emulator/beam/erl_lock_count.c @@ -3,16 +3,17 @@ * * Copyright Ericsson AB 2008-2012. All Rights Reserved. * - * The contents of this file are subject to the Erlang Public License, - * Version 1.1, (the "License"); you may not use this file except in - * compliance with the License. You should have received a copy of the - * Erlang Public License along with this software. If not, it can be - * retrieved online at http://www.erlang.org/. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Software distributed under the License is distributed on an "AS IS" - * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See - * the License for the specific language governing rights and limitations - * under the License. + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. * * %CopyrightEnd% */ @@ -20,7 +21,7 @@ /* * Description: Statistics for locks. * - * Author: Bj�rn-Egil Dahlberg + * Author: Björn-Egil Dahlberg * Date: 2008-07-03 */ @@ -49,7 +50,7 @@ const char *str_undefined = "undefined"; static ethr_tsd_key lcnt_thr_data_key; static int lcnt_n_thr; -static erts_lcnt_thread_data_t *lcnt_thread_data[4096]; +static erts_lcnt_thread_data_t *lcnt_thread_data[2048]; /* local functions */ @@ -61,15 +62,34 @@ static ERTS_INLINE void lcnt_unlock(void) { ethr_mutex_unlock(&lcnt_data_lock); } +const int log2_tab64[64] = { + 63, 0, 58, 1, 59, 47, 53, 2, + 60, 39, 48, 27, 54, 33, 42, 3, + 61, 51, 37, 40, 49, 18, 28, 20, + 55, 30, 34, 11, 43, 14, 22, 4, + 62, 57, 46, 52, 38, 26, 32, 41, + 50, 36, 17, 19, 29, 10, 13, 21, + 56, 45, 25, 31, 35, 16, 9, 12, + 44, 24, 15, 8, 23, 7, 6, 5}; + +static ERTS_INLINE int lcnt_log2(Uint64 v) { + v |= v >> 1; + v |= v >> 2; + v |= v >> 4; + v |= v >> 8; + v |= v >> 16; + v |= v >> 32; + return log2_tab64[((Uint64)((v - (v >> 1))*0x07EDD5E59A4E28C2)) >> 58]; +} static char* lcnt_lock_type(Uint16 flag) { switch(flag & ERTS_LCNT_LT_ALL) { - case ERTS_LCNT_LT_SPINLOCK: return "spinlock"; - case ERTS_LCNT_LT_RWSPINLOCK: return "rw_spinlock"; - case ERTS_LCNT_LT_MUTEX: return "mutex"; - case ERTS_LCNT_LT_RWMUTEX: return "rw_mutex"; - case ERTS_LCNT_LT_PROCLOCK: return "proclock"; - default: return ""; + case ERTS_LCNT_LT_SPINLOCK: return "spinlock"; + case ERTS_LCNT_LT_RWSPINLOCK: return "rw_spinlock"; + case ERTS_LCNT_LT_MUTEX: return "mutex"; + case ERTS_LCNT_LT_RWMUTEX: return "rw_mutex"; + case ERTS_LCNT_LT_PROCLOCK: return "proclock"; + default: return ""; } } @@ -81,58 +101,56 @@ static void lcnt_clear_stats(erts_lcnt_lock_stats_t *stats) { stats->timer_n = 0; stats->file = (char *)str_undefined; stats->line = 0; + sys_memzero(stats->hist.ns, sizeof(stats->hist.ns)); } static void lcnt_time(erts_lcnt_time_t *time) { -#ifdef HAVE_GETHRTIME - SysHrTime hr_time; - hr_time = sys_gethrtime(); - time->s = (unsigned long)(hr_time / 1000000000LL); - time->ns = (unsigned long)(hr_time - 1000000000LL*time->s); -#else - SysTimeval tv; - sys_gettimeofday(&tv); - time->s = tv.tv_sec; - time->ns = tv.tv_usec*1000LL; -#endif + /* + * erts_sys_hrtime() is the highest resolution + * we could find, it may or may not be monotonic... + */ + ErtsMonotonicTime mtime = erts_sys_hrtime(); + time->s = (unsigned long) (mtime / 1000000000LL); + time->ns = (unsigned long) (mtime - 1000000000LL*time->s); } static void lcnt_time_diff(erts_lcnt_time_t *d, erts_lcnt_time_t *t1, erts_lcnt_time_t *t0) { long ds; long dns; - + ds = t1->s - t0->s; dns = t1->ns - t0->ns; - + /* the difference should not be able to get bigger than 1 sec in ns*/ - + if (dns < 0) { - ds -= 1; - dns += 1000000000LL; + ds -= 1; + dns += 1000000000LL; } + ASSERT(ds >= 0); + d->s = ds; d->ns = dns; } -/* difference d must be positive */ +/* difference d must be non-negative */ static void lcnt_time_add(erts_lcnt_time_t *t, erts_lcnt_time_t *d) { - unsigned long ngns = 0; - t->s += d->s; t->ns += d->ns; - ngns = t->ns / 1000000000LL; + t->s += t->ns / 1000000000LL; t->ns = t->ns % 1000000000LL; - - t->s += ngns; } static erts_lcnt_thread_data_t *lcnt_thread_data_alloc(void) { erts_lcnt_thread_data_t *eltd; - + eltd = (erts_lcnt_thread_data_t*)malloc(sizeof(erts_lcnt_thread_data_t)); + if (!eltd) { + ERTS_INTERNAL_ERROR("Lock counter failed to allocate memory!"); + } eltd->timer_set = 0; eltd->lock_in_conflict = 0; @@ -147,7 +165,6 @@ static erts_lcnt_thread_data_t *lcnt_get_thread_data(void) { return (erts_lcnt_thread_data_t *)ethr_tsd_get(lcnt_thr_data_key); } - /* debug */ #if 0 @@ -158,96 +175,100 @@ static char* lock_opt(Uint16 flag) { return "--"; } -static void print_lock_x(erts_lcnt_lock_t *lock, Uint16 flag, char *action, char *extra) { - erts_aint_t colls, tries, w_state, r_state; - erts_lcnt_lock_stats_t *stats = NULL; - +static void print_lock_x(erts_lcnt_lock_t *lock, Uint16 flag, char *action) { + erts_aint_t w_state, r_state; char *type; - int i; - + + if (strcmp(lock->name, "run_queue") != 0) return; type = lcnt_lock_type(lock->flag); r_state = ethr_atomic_read(&lock->r_state); w_state = ethr_atomic_read(&lock->w_state); - if (lock->flag & flag) { - erts_printf("%20s [%30s] [r/w state %4ld/%4ld] id %T %s\r\n", - action, - lock->name, - r_state, - w_state, - lock->id, - extra); - } -} - -static void print_lock(erts_lcnt_lock_t *lock, char *action) { - if (strcmp(lock->name, "proc_main") == 0) { - print_lock_x(lock, ERTS_LCNT_LT_ALL, action, ""); + erts_fprintf(stderr,"%10s [%24s] [r/w state %4ld/%4ld] %2s id %T\r\n", + action, + lock->name, + r_state, + w_state, + type, + lock->id); } } - #endif static erts_lcnt_lock_stats_t *lcnt_get_lock_stats(erts_lcnt_lock_t *lock, char *file, unsigned int line) { unsigned int i; erts_lcnt_lock_stats_t *stats = NULL; - - for (i = 0; i < lock->n_stats; i++) { - if ((lock->stats[i].file == file) && (lock->stats[i].line == line)) { - return &(lock->stats[i]); - } - } - if (lock->n_stats < ERTS_LCNT_MAX_LOCK_LOCATIONS) { - stats = &lock->stats[lock->n_stats]; - lock->n_stats++; - stats->file = file; - stats->line = line; - return stats; + if (erts_lcnt_rt_options & ERTS_LCNT_OPT_LOCATION) { + for (i = 0; i < lock->n_stats; i++) { + if ((lock->stats[i].file == file) && (lock->stats[i].line == line)) { + return &(lock->stats[i]); + } + } + if (lock->n_stats < ERTS_LCNT_MAX_LOCK_LOCATIONS) { + stats = &lock->stats[lock->n_stats]; + lock->n_stats++; + stats->file = file; + stats->line = line; + return stats; + } } return &lock->stats[0]; +} +static void lcnt_update_stats_hist(erts_lcnt_hist_t *hist, erts_lcnt_time_t *time_wait) { + int idx; + unsigned long r; + + if (time_wait->s > 0 || time_wait->ns > ERTS_LCNT_HISTOGRAM_MAX_NS) { + idx = ERTS_LCNT_HISTOGRAM_SLOT_SIZE - 1; + } else { + r = time_wait->ns >> ERTS_LCNT_HISTOGRAM_RSHIFT; + if (r) idx = lcnt_log2(r); + else idx = 0; + } + hist->ns[idx]++; } -static void lcnt_update_stats(erts_lcnt_lock_stats_t *stats, int lock_in_conflict, erts_lcnt_time_t *time_wait) { - +static void lcnt_update_stats(erts_lcnt_lock_stats_t *stats, int lock_in_conflict, + erts_lcnt_time_t *time_wait) { + ethr_atomic_inc(&stats->tries); if (lock_in_conflict) - ethr_atomic_inc(&stats->colls); + ethr_atomic_inc(&stats->colls); if (time_wait) { - lcnt_time_add(&(stats->timer), time_wait); - stats->timer_n++; + lcnt_time_add(&(stats->timer), time_wait); + stats->timer_n++; + lcnt_update_stats_hist(&stats->hist,time_wait); } } -/* - * interface - */ +/* interface */ void erts_lcnt_init() { erts_lcnt_thread_data_t *eltd = NULL; - + /* init lock */ if (ethr_mutex_init(&lcnt_data_lock) != 0) abort(); /* init tsd */ lcnt_n_thr = 0; - ethr_tsd_key_create(&lcnt_thr_data_key,"lcnt_data"); lcnt_lock(); - erts_lcnt_rt_options = ERTS_LCNT_OPT_PROCLOCK | ERTS_LCNT_OPT_LOCATION; - + erts_lcnt_rt_options = ERTS_LCNT_OPT_LOCATION | ERTS_LCNT_OPT_PROCLOCK; eltd = lcnt_thread_data_alloc(); - ethr_tsd_set(lcnt_thr_data_key, eltd); - + /* init lcnt structure */ erts_lcnt_data = (erts_lcnt_data_t*)malloc(sizeof(erts_lcnt_data_t)); + if (!erts_lcnt_data) { + ERTS_INTERNAL_ERROR("Lock counter failed to allocate memory!"); + } erts_lcnt_data->current_locks = erts_lcnt_list_init(); erts_lcnt_data->deleted_locks = erts_lcnt_list_init(); @@ -267,8 +288,11 @@ void erts_lcnt_late_init() { erts_lcnt_lock_list_t *erts_lcnt_list_init(void) { erts_lcnt_lock_list_t *list; - + list = (erts_lcnt_lock_list_t*)malloc(sizeof(erts_lcnt_lock_list_t)); + if (!list) { + ERTS_INTERNAL_ERROR("Lock counter failed to allocate memory!"); + } list->head = NULL; list->tail = NULL; list->n = 0; @@ -278,14 +302,14 @@ erts_lcnt_lock_list_t *erts_lcnt_list_init(void) { /* only do this on the list with the deleted locks! */ void erts_lcnt_list_clear(erts_lcnt_lock_list_t *list) { erts_lcnt_lock_t *lock = NULL, - *next = NULL; + *next = NULL; lock = list->head; - + while(lock != NULL) { - next = lock->next; - free(lock); - lock = next; + next = lock->next; + free(lock); + lock = next; } list->head = NULL; @@ -298,26 +322,25 @@ void erts_lcnt_list_insert(erts_lcnt_lock_list_t *list, erts_lcnt_lock_t *lock) tail = list->tail; if (tail) { - tail->next = lock; - lock->prev = tail; + tail->next = lock; + lock->prev = tail; } else { - list->head = lock; - lock->prev = NULL; - ASSERT(!lock->next); + list->head = lock; + lock->prev = NULL; + ASSERT(!lock->next); } lock->next = NULL; list->tail = lock; - + list->n++; } void erts_lcnt_list_delete(erts_lcnt_lock_list_t *list, erts_lcnt_lock_t *lock) { - if (lock->next) lock->next->prev = lock->prev; if (lock->prev) lock->prev->next = lock->next; if (list->head == lock) list->head = lock->next; if (list->tail == lock) list->tail = lock->prev; - + lock->prev = NULL; lock->next = NULL; list->n--; @@ -330,16 +353,14 @@ void erts_lcnt_list_delete(erts_lcnt_lock_list_t *list, erts_lcnt_lock_t *lock) /* interface to erl_threads.h */ /* only lock on init and destroy, all others should use atomics */ void erts_lcnt_init_lock(erts_lcnt_lock_t *lock, char *name, Uint16 flag ) { - erts_lcnt_init_lock_x(lock, name, flag, am_undefined); + erts_lcnt_init_lock_x(lock, name, flag, NIL); } + void erts_lcnt_init_lock_x(erts_lcnt_lock_t *lock, char *name, Uint16 flag, Eterm id) { int i; - if (!name) { - lock->flag = 0; - return; - } + if (name == NULL) { ERTS_LCNT_CLEAR_FLAG(lock); return; } lcnt_lock(); - + lock->next = NULL; lock->prev = NULL; lock->flag = flag; @@ -348,44 +369,55 @@ void erts_lcnt_init_lock_x(erts_lcnt_lock_t *lock, char *name, Uint16 flag, Eter ethr_atomic_init(&lock->r_state, 0); ethr_atomic_init(&lock->w_state, 0); - #ifdef DEBUG ethr_atomic_init(&lock->flowstate, 0); #endif - + lock->n_stats = 1; for (i = 0; i < ERTS_LCNT_MAX_LOCK_LOCATIONS; i++) { - lcnt_clear_stats(&lock->stats[i]); + lcnt_clear_stats(&lock->stats[i]); } erts_lcnt_list_insert(erts_lcnt_data->current_locks, lock); - lcnt_unlock(); } - +/* init empty, instead of zero struct */ +void erts_lcnt_init_lock_empty(erts_lcnt_lock_t *lock) { + lock->next = NULL; + lock->prev = NULL; + lock->flag = 0; + lock->name = NULL; + lock->id = NIL; + ethr_atomic_init(&lock->r_state, 0); + ethr_atomic_init(&lock->w_state, 0); +#ifdef DEBUG + ethr_atomic_init(&lock->flowstate, 0); +#endif + lock->n_stats = 0; + sys_memzero(lock->stats, sizeof(lock->stats)); +} +/* destroy lock */ void erts_lcnt_destroy_lock(erts_lcnt_lock_t *lock) { - erts_lcnt_lock_t *deleted_lock; - - if (!ERTS_LCNT_LOCK_TYPE(lock)) return; - + if (ERTS_LCNT_IS_LOCK_INVALID(lock)) return; lcnt_lock(); if (erts_lcnt_rt_options & ERTS_LCNT_OPT_COPYSAVE) { - /* copy structure and insert the copy */ - - deleted_lock = (erts_lcnt_lock_t*)malloc(sizeof(erts_lcnt_lock_t)); - memcpy(deleted_lock, lock, sizeof(erts_lcnt_lock_t)); - - deleted_lock->next = NULL; - deleted_lock->prev = NULL; - - erts_lcnt_list_insert(erts_lcnt_data->deleted_locks, deleted_lock); + erts_lcnt_lock_t *deleted_lock; + /* copy structure and insert the copy */ + deleted_lock = (erts_lcnt_lock_t*)malloc(sizeof(erts_lcnt_lock_t)); + if (!deleted_lock) { + ERTS_INTERNAL_ERROR("Lock counter failed to allocate memory!"); + } + memcpy(deleted_lock, lock, sizeof(erts_lcnt_lock_t)); + deleted_lock->next = NULL; + deleted_lock->prev = NULL; + erts_lcnt_list_insert(erts_lcnt_data->deleted_locks, deleted_lock); } /* delete original */ erts_lcnt_list_delete(erts_lcnt_data->current_locks, lock); - lock->flag = 0; - + ERTS_LCNT_CLEAR_FLAG(lock); + lcnt_unlock(); } @@ -394,16 +426,15 @@ void erts_lcnt_destroy_lock(erts_lcnt_lock_t *lock) { void erts_lcnt_lock_opt(erts_lcnt_lock_t *lock, Uint16 option) { erts_aint_t r_state = 0, w_state = 0; erts_lcnt_thread_data_t *eltd; - + if (erts_lcnt_rt_options & ERTS_LCNT_OPT_SUSPEND) return; - if (!ERTS_LCNT_LOCK_TYPE(lock)) return; + if (ERTS_LCNT_IS_LOCK_INVALID(lock)) return; eltd = lcnt_get_thread_data(); - ASSERT(eltd); - + w_state = ethr_atomic_read(&lock->w_state); - + if (option & ERTS_LCNT_LO_WRITE) { r_state = ethr_atomic_read(&lock->r_state); ethr_atomic_inc( &lock->w_state); @@ -411,47 +442,47 @@ void erts_lcnt_lock_opt(erts_lcnt_lock_t *lock, Uint16 option) { if (option & ERTS_LCNT_LO_READ) { ethr_atomic_inc( &lock->r_state); } - + /* we cannot acquire w_lock if either w or r are taken */ /* we cannot acquire r_lock if w_lock is taken */ - + if ((w_state > 0) || (r_state > 0)) { - eltd->lock_in_conflict = 1; - if (eltd->timer_set == 0) - lcnt_time(&eltd->timer); - eltd->timer_set++; + eltd->lock_in_conflict = 1; + if (eltd->timer_set == 0) { + lcnt_time(&eltd->timer); + } + eltd->timer_set++; } else { - eltd->lock_in_conflict = 0; + eltd->lock_in_conflict = 0; } } void erts_lcnt_lock(erts_lcnt_lock_t *lock) { erts_aint_t w_state; erts_lcnt_thread_data_t *eltd; - + if (erts_lcnt_rt_options & ERTS_LCNT_OPT_SUSPEND) return; - if (!ERTS_LCNT_LOCK_TYPE(lock)) return; + if (ERTS_LCNT_IS_LOCK_INVALID(lock)) return; w_state = ethr_atomic_read(&lock->w_state); - ethr_atomic_inc( &lock->w_state); - + ethr_atomic_inc(&lock->w_state); eltd = lcnt_get_thread_data(); ASSERT(eltd); if (w_state > 0) { - eltd->lock_in_conflict = 1; - /* only set the timer if nobody else has it - * This should only happen when proc_locks aquires several locks - * 'atomicly'. All other locks will block the thread if w_state > 0 - * i.e. locked. - */ - if (eltd->timer_set == 0) - lcnt_time(&eltd->timer); - eltd->timer_set++; - + eltd->lock_in_conflict = 1; + /* only set the timer if nobody else has it + * This should only happen when proc_locks aquires several locks + * 'atomicly'. All other locks will block the thread if w_state > 0 + * i.e. locked. + */ + if (eltd->timer_set == 0) { + lcnt_time(&eltd->timer); + } + eltd->timer_set++; } else { - eltd->lock_in_conflict = 0; + eltd->lock_in_conflict = 0; } } @@ -459,18 +490,20 @@ void erts_lcnt_lock(erts_lcnt_lock_t *lock) { void erts_lcnt_lock_unaquire(erts_lcnt_lock_t *lock) { /* should check if this thread was "waiting" */ - if (erts_lcnt_rt_options & ERTS_LCNT_OPT_SUSPEND) return; - if (!ERTS_LCNT_LOCK_TYPE(lock)) return; + if (ERTS_LCNT_IS_LOCK_INVALID(lock)) return; - ethr_atomic_dec( &lock->w_state); + ethr_atomic_dec(&lock->w_state); } -/* erts_lcnt_lock_post - * used when we get a lock (i.e. directly after a lock operation) +/* + * erts_lcnt_lock_post + * + * Used when we get a lock (i.e. directly after a lock operation) * if the timer was set then we had to wait for the lock * lock_post will calculate the wait time. */ + void erts_lcnt_lock_post(erts_lcnt_lock_t *lock) { erts_lcnt_lock_post_x(lock, (char*)str_undefined, 0); } @@ -485,38 +518,31 @@ void erts_lcnt_lock_post_x(erts_lcnt_lock_t *lock, char *file, unsigned int line #endif if (erts_lcnt_rt_options & ERTS_LCNT_OPT_SUSPEND) return; - if (!ERTS_LCNT_LOCK_TYPE(lock)) return; - + if (ERTS_LCNT_IS_LOCK_INVALID(lock)) return; + #ifdef DEBUG if (!(lock->flag & (ERTS_LCNT_LT_RWMUTEX | ERTS_LCNT_LT_RWSPINLOCK))) { - flowstate = ethr_atomic_read(&lock->flowstate); - ASSERT(flowstate == 0); - ethr_atomic_inc( &lock->flowstate); + flowstate = ethr_atomic_read(&lock->flowstate); + ASSERT(flowstate == 0); + ethr_atomic_inc(&lock->flowstate); } #endif - + eltd = lcnt_get_thread_data(); - + ASSERT(eltd); /* if lock was in conflict, time it */ - - if (erts_lcnt_rt_options & ERTS_LCNT_OPT_LOCATION) { - stats = lcnt_get_lock_stats(lock, file, line); - } else { - stats = &lock->stats[0]; - } - + stats = lcnt_get_lock_stats(lock, file, line); if (eltd->timer_set) { - lcnt_time(&timer); - - lcnt_time_diff(&time_wait, &timer, &(eltd->timer)); - lcnt_update_stats(stats, eltd->lock_in_conflict, &time_wait); - - eltd->timer_set--; - ASSERT(eltd->timer_set >= 0); + lcnt_time(&timer); + + lcnt_time_diff(&time_wait, &timer, &(eltd->timer)); + lcnt_update_stats(stats, eltd->lock_in_conflict, &time_wait); + eltd->timer_set--; + ASSERT(eltd->timer_set >= 0); } else { - lcnt_update_stats(stats, eltd->lock_in_conflict, NULL); + lcnt_update_stats(stats, eltd->lock_in_conflict, NULL); } } @@ -525,27 +551,28 @@ void erts_lcnt_lock_post_x(erts_lcnt_lock_t *lock, char *file, unsigned int line void erts_lcnt_unlock_opt(erts_lcnt_lock_t *lock, Uint16 option) { if (erts_lcnt_rt_options & ERTS_LCNT_OPT_SUSPEND) return; - if (!ERTS_LCNT_LOCK_TYPE(lock)) return; + if (ERTS_LCNT_IS_LOCK_INVALID(lock)) return; if (option & ERTS_LCNT_LO_WRITE) ethr_atomic_dec(&lock->w_state); if (option & ERTS_LCNT_LO_READ ) ethr_atomic_dec(&lock->r_state); } void erts_lcnt_unlock(erts_lcnt_lock_t *lock) { -#ifdef DEBUG - erts_aint_t w_state; - erts_aint_t flowstate; -#endif if (erts_lcnt_rt_options & ERTS_LCNT_OPT_SUSPEND) return; - if (!ERTS_LCNT_LOCK_TYPE(lock)) return; + if (ERTS_LCNT_IS_LOCK_INVALID(lock)) return; #ifdef DEBUG - /* flowstate */ - flowstate = ethr_atomic_read(&lock->flowstate); - ASSERT(flowstate == 1); - ethr_atomic_dec( &lock->flowstate); - - /* write state */ - w_state = ethr_atomic_read(&lock->w_state); - ASSERT(w_state > 0) + { + erts_aint_t w_state; + erts_aint_t flowstate; + + /* flowstate */ + flowstate = ethr_atomic_read(&lock->flowstate); + ASSERT(flowstate == 1); + ethr_atomic_dec(&lock->flowstate); + + /* write state */ + w_state = ethr_atomic_read(&lock->w_state); + ASSERT(w_state > 0); + } #endif ethr_atomic_dec(&lock->w_state); } @@ -554,37 +581,34 @@ void erts_lcnt_unlock(erts_lcnt_lock_t *lock) { void erts_lcnt_trylock_opt(erts_lcnt_lock_t *lock, int res, Uint16 option) { if (erts_lcnt_rt_options & ERTS_LCNT_OPT_SUSPEND) return; - if (!ERTS_LCNT_LOCK_TYPE(lock)) return; + if (ERTS_LCNT_IS_LOCK_INVALID(lock)) return; /* Determine lock_state via res instead of state */ if (res != EBUSY) { - if (option & ERTS_LCNT_LO_WRITE) ethr_atomic_inc(&lock->w_state); - if (option & ERTS_LCNT_LO_READ ) ethr_atomic_inc(&lock->r_state); - lcnt_update_stats(&(lock->stats[0]), 0, NULL); + if (option & ERTS_LCNT_LO_WRITE) ethr_atomic_inc(&lock->w_state); + if (option & ERTS_LCNT_LO_READ ) ethr_atomic_inc(&lock->r_state); + lcnt_update_stats(&(lock->stats[0]), 0, NULL); } else { ethr_atomic_inc(&lock->stats[0].tries); ethr_atomic_inc(&lock->stats[0].colls); } } - + void erts_lcnt_trylock(erts_lcnt_lock_t *lock, int res) { /* Determine lock_state via res instead of state */ -#ifdef DEBUG - erts_aint_t flowstate; -#endif if (erts_lcnt_rt_options & ERTS_LCNT_OPT_SUSPEND) return; - if (!ERTS_LCNT_LOCK_TYPE(lock)) return; + if (ERTS_LCNT_IS_LOCK_INVALID(lock)) return; if (res != EBUSY) { - #ifdef DEBUG - flowstate = ethr_atomic_read(&lock->flowstate); - ASSERT(flowstate == 0); - ethr_atomic_inc( &lock->flowstate); + { + erts_aint_t flowstate; + flowstate = ethr_atomic_read(&lock->flowstate); + ASSERT(flowstate == 0); + ethr_atomic_inc( &lock->flowstate); + } #endif - ethr_atomic_inc(&lock->w_state); - - lcnt_update_stats(&(lock->stats[0]), 0, NULL); - + ethr_atomic_inc(&lock->w_state); + lcnt_update_stats(&(lock->stats[0]), 0, NULL); } else { ethr_atomic_inc(&lock->stats[0].tries); ethr_atomic_inc(&lock->stats[0].colls); @@ -639,13 +663,13 @@ void erts_lcnt_clear_counters(void) { lcnt_lock(); list = erts_lcnt_data->current_locks; - + for (lock = list->head; lock != NULL; lock = lock->next) { - for( i = 0; i < ERTS_LCNT_MAX_LOCK_LOCATIONS; i++) { - stats = &lock->stats[i]; - lcnt_clear_stats(stats); - } - lock->n_stats = 1; + for( i = 0; i < ERTS_LCNT_MAX_LOCK_LOCATIONS; i++) { + stats = &lock->stats[i]; + lcnt_clear_stats(stats); + } + lock->n_stats = 1; } /* empty deleted locks in lock list */ @@ -658,14 +682,14 @@ void erts_lcnt_clear_counters(void) { erts_lcnt_data_t *erts_lcnt_get_data(void) { erts_lcnt_time_t timer_stop; - + lcnt_lock(); - + lcnt_time(&timer_stop); lcnt_time_diff(&(erts_lcnt_data->duration), &timer_stop, &timer_start); - + lcnt_unlock(); - + return erts_lcnt_data; } diff --git a/erts/emulator/beam/erl_lock_count.h b/erts/emulator/beam/erl_lock_count.h index 75f7cd028b..4cc6a5c695 100644 --- a/erts/emulator/beam/erl_lock_count.h +++ b/erts/emulator/beam/erl_lock_count.h @@ -3,16 +3,17 @@ * * Copyright Ericsson AB 2008-2012. All Rights Reserved. * - * The contents of this file are subject to the Erlang Public License, - * Version 1.1, (the "License"); you may not use this file except in - * compliance with the License. You should have received a copy of the - * Erlang Public License along with this software. If not, it can be - * retrieved online at http://www.erlang.org/. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Software distributed under the License is distributed on an "AS IS" - * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See - * the License for the specific language governing rights and limitations - * under the License. + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. * * %CopyrightEnd% */ @@ -20,7 +21,7 @@ /* * Description: Statistics for locks. * - * Author: Bj�rn-Egil Dahlberg + * Author: Björn-Egil Dahlberg * Date: 2008-07-03 * Abstract: * Locks statistics internal representation. @@ -35,6 +36,10 @@ * | | | - collisions (including trylock busy) * | | | - timer (time spent in waiting for lock) * | | | - n_timer (collisions excluding trylock busy) + * | | | - histogram + * | | | | - # 0 = log2(lock wait_time ns) + * | | | | - ... + * | | | | - # n = log2(lock wait_time ns) * * Each instance of a lock is the unique lock, i.e. set and id in that set. * For each lock there is a set of statistics with where and what impact @@ -68,8 +73,17 @@ #include "ethread.h" +#define ERTS_LCNT_MAX_LOCK_LOCATIONS (10) -#define ERTS_LCNT_MAX_LOCK_LOCATIONS (10) +/* histogram */ +#define ERTS_LCNT_HISTOGRAM_MAX_NS (((unsigned long)1LL << 28) - 1) +#if 0 || defined(ERTS_HAVE_OS_MONOTONIC_TIME_SUPPORT) +#define ERTS_LCNT_HISTOGRAM_SLOT_SIZE (30) +#define ERTS_LCNT_HISTOGRAM_RSHIFT (0) +#else +#define ERTS_LCNT_HISTOGRAM_SLOT_SIZE (20) +#define ERTS_LCNT_HISTOGRAM_RSHIFT (10) +#endif #define ERTS_LCNT_LT_SPINLOCK (((Uint16) 1) << 0) #define ERTS_LCNT_LT_RWSPINLOCK (((Uint16) 1) << 1) @@ -82,28 +96,37 @@ #define ERTS_LCNT_LO_WRITE (((Uint16) 1) << 7) #define ERTS_LCNT_LO_READ_WRITE ( ERTS_LCNT_LO_READ \ - | ERTS_LCNT_LO_WRITE ) + | ERTS_LCNT_LO_WRITE ) #define ERTS_LCNT_LT_ALL ( ERTS_LCNT_LT_SPINLOCK \ - | ERTS_LCNT_LT_RWSPINLOCK \ - | ERTS_LCNT_LT_MUTEX \ - | ERTS_LCNT_LT_RWMUTEX \ - | ERTS_LCNT_LT_PROCLOCK ) + | ERTS_LCNT_LT_RWSPINLOCK \ + | ERTS_LCNT_LT_MUTEX \ + | ERTS_LCNT_LT_RWMUTEX \ + | ERTS_LCNT_LT_PROCLOCK ) + +#define ERTS_LCNT_LOCK_TYPE(lock) ((lock)->flag & ERTS_LCNT_LT_ALL) +#define ERTS_LCNT_IS_LOCK_INVALID(lock) (!((lock)->flag & ERTS_LCNT_LT_ALL)) +#define ERTS_LCNT_CLEAR_FLAG(lock) ((lock)->flag = 0) + /* runtime options */ -#define ERTS_LCNT_OPT_SUSPEND (((Uint16) 1) << 0) -#define ERTS_LCNT_OPT_LOCATION (((Uint16) 1) << 1) -#define ERTS_LCNT_OPT_PROCLOCK (((Uint16) 1) << 2) -#define ERTS_LCNT_OPT_COPYSAVE (((Uint16) 1) << 3) -#define ERTS_LCNT_OPT_PORTLOCK (((Uint16) 1) << 4) +#define ERTS_LCNT_OPT_SUSPEND (((Uint16) 1) << 0) +#define ERTS_LCNT_OPT_LOCATION (((Uint16) 1) << 1) +#define ERTS_LCNT_OPT_PROCLOCK (((Uint16) 1) << 2) +#define ERTS_LCNT_OPT_PORTLOCK (((Uint16) 1) << 3) +#define ERTS_LCNT_OPT_COPYSAVE (((Uint16) 1) << 4) typedef struct { unsigned long s; unsigned long ns; } erts_lcnt_time_t; - + extern erts_lcnt_time_t timer_start; +typedef struct { + Uint32 ns[ERTS_LCNT_HISTOGRAM_SLOT_SIZE]; /* log2 array of nano seconds occurences */ +} erts_lcnt_hist_t; + typedef struct erts_lcnt_lock_stats_s { /* "tries" and "colls" needs to be atomic since * trylock busy does not aquire a lock and there @@ -112,12 +135,13 @@ typedef struct erts_lcnt_lock_stats_s { char *file; /* which file the lock was taken */ unsigned int line; /* line number in file */ - + ethr_atomic_t tries; /* n tries to get lock */ ethr_atomic_t colls; /* n collisions of tries to get lock */ - + unsigned long timer_n; /* #times waited for lock */ erts_lcnt_time_t timer; /* total wait time for lock */ + erts_lcnt_hist_t hist; } erts_lcnt_lock_stats_t; /* rw locks uses both states, other locks only uses w_state */ @@ -137,7 +161,7 @@ typedef struct erts_lcnt_lock_s { /* statistics */ unsigned int n_stats; erts_lcnt_lock_stats_t stats[ERTS_LCNT_MAX_LOCK_LOCATIONS]; /* first entry is "undefined"*/ - + /* chains for list handling */ /* data is hold by lcnt_lock */ struct erts_lcnt_lock_s *prev; @@ -149,7 +173,7 @@ typedef struct { erts_lcnt_lock_t *tail; unsigned long n; } erts_lcnt_lock_list_t; - + typedef struct { erts_lcnt_time_t duration; /* time since last clear */ erts_lcnt_lock_list_t *current_locks; @@ -187,6 +211,7 @@ void erts_lcnt_list_delete(erts_lcnt_lock_list_t *list, erts_lcnt_lock_t *lock); /* lock operations (global) */ void erts_lcnt_init_lock(erts_lcnt_lock_t *lock, char *name, Uint16 flag); void erts_lcnt_init_lock_x(erts_lcnt_lock_t *lock, char *name, Uint16 flag, Eterm id); +void erts_lcnt_init_lock_empty(erts_lcnt_lock_t *lock); void erts_lcnt_destroy_lock(erts_lcnt_lock_t *lock); void erts_lcnt_lock(erts_lcnt_lock_t *lock); @@ -208,7 +233,5 @@ void erts_lcnt_clear_counters(void); char *erts_lcnt_lock_type(Uint16 type); erts_lcnt_data_t *erts_lcnt_get_data(void); -#define ERTS_LCNT_LOCK_TYPE(lockp) ((lockp)->flag & ERTS_LCNT_LT_ALL) - #endif /* ifdef ERTS_ENABLE_LOCK_COUNT */ #endif /* ifndef ERTS_LOCK_COUNT_H__ */ diff --git a/erts/emulator/beam/erl_map.c b/erts/emulator/beam/erl_map.c index fdd2d0c0f6..4b6931f848 100644 --- a/erts/emulator/beam/erl_map.c +++ b/erts/emulator/beam/erl_map.c @@ -3,19 +3,23 @@ * * Copyright Ericsson AB 2014. All Rights Reserved. * - * The contents of this file are subject to the Erlang Public License, - * Version 1.1, (the "License"); you may not use this file except in - * compliance with the License. You should have received a copy of the - * Erlang Public License along with this software. If not, it can be - * retrieved online at http://www.erlang.org/. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Software distributed under the License is distributed on an "AS IS" - * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See - * the License for the specific language governing rights and limitations - * under the License. + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. * * %CopyrightEnd% * + * hashmaps are an adaption of Rich Hickeys Persistent HashMaps + * which were an adaption of Phil Bagwells - Hash Array Mapped Tries + * * Author: Björn-Egil Dahlberg */ @@ -28,7 +32,10 @@ #include "global.h" #include "erl_process.h" #include "error.h" +#define ERL_WANT_HIPE_BIF_WRAPPER__ #include "bif.h" +#undef ERL_WANT_HIPE_BIF_WRAPPER__ +#include "erl_binary.h" #include "erl_map.h" @@ -58,41 +65,91 @@ * - maps:size/1 * - maps:without/2 * + * DEBUG: for sharing calculation + * - erts_internal:map_to_tuple_keys/1 */ +#ifndef DECL_AM +#define DECL_AM(S) Eterm AM_ ## S = am_atom_put(#S, sizeof(#S) - 1) +#endif + +/* for hashmap_from_list/1 */ +typedef struct { + Uint32 hx; + Uint32 skip; + Uint i; + Eterm val; +} hxnode_t; + + +static Eterm flatmap_merge(Process *p, Eterm nodeA, Eterm nodeB); +static BIF_RETTYPE map_merge_mixed(Process *p, Eterm flat, Eterm tree, int swap_args); +struct HashmapMergeContext_; +static BIF_RETTYPE hashmap_merge(Process *p, Eterm nodeA, Eterm nodeB, int swap_args, + struct HashmapMergeContext_*); +static Export hashmap_merge_trap_export; +static BIF_RETTYPE maps_merge_trap_1(BIF_ALIST_1); +static Uint hashmap_subtree_size(Eterm node); +static Eterm hashmap_to_list(Process *p, Eterm map); +static Eterm hashmap_keys(Process *p, Eterm map); +static Eterm hashmap_values(Process *p, Eterm map); +static Eterm hashmap_delete(Process *p, Uint32 hx, Eterm key, Eterm node); +static Eterm flatmap_from_validated_list(Process *p, Eterm list, Uint size); +static Eterm hashmap_from_validated_list(Process *p, Eterm list, Uint size); +static Eterm hashmap_from_unsorted_array(ErtsHeapFactory*, hxnode_t *hxns, Uint n, int reject_dupkeys); +static Eterm hashmap_from_sorted_unique_array(ErtsHeapFactory*, hxnode_t *hxns, Uint n, int is_root); +static Eterm hashmap_from_chunked_array(ErtsHeapFactory*, hxnode_t *hxns, Uint n, Uint size, int is_root); +static Eterm hashmap_info(Process *p, Eterm node); +static Eterm hashmap_bld_tuple_uint(Uint **hpp, Uint *szp, Uint n, Uint nums[]); +static int hxnodecmp(hxnode_t* a, hxnode_t* b); +static int hxnodecmpkey(hxnode_t* a, hxnode_t* b); + + +void erts_init_map(void) { + erts_init_trap_export(&hashmap_merge_trap_export, + am_maps, am_merge_trap, 1, + &maps_merge_trap_1); + return; +} + + /* erlang:map_size/1 * the corresponding instruction is implemented in: * beam/erl_bif_guard.c */ BIF_RETTYPE map_size_1(BIF_ALIST_1) { - if (is_map(BIF_ARG_1)) { - Eterm *hp; - Uint hsz = 0; - map_t *mp = (map_t*)map_val(BIF_ARG_1); - Uint n = map_get_size(mp); - - erts_bld_uint(NULL, &hsz, n); + if (is_flatmap(BIF_ARG_1)) { + flatmap_t *mp = (flatmap_t*)flatmap_val(BIF_ARG_1); + BIF_RET(make_small(flatmap_get_size(mp))); + } else if (is_hashmap(BIF_ARG_1)) { + Eterm *head, *hp, res; + Uint size, hsz=0; + + head = hashmap_val(BIF_ARG_1); + size = head[1]; + (void) erts_bld_uint(NULL, &hsz, size); hp = HAlloc(BIF_P, hsz); - BIF_RET(erts_bld_uint(&hp, NULL, n)); + res = erts_bld_uint(&hp, NULL, size); + BIF_RET(res); } - BIF_ERROR(BIF_P, BADARG); + BIF_P->fvalue = BIF_ARG_1; + BIF_ERROR(BIF_P, BADMAP); } -/* maps:to_list/1 - */ +/* maps:to_list/1 */ BIF_RETTYPE maps_to_list_1(BIF_ALIST_1) { - if (is_map(BIF_ARG_1)) { + if (is_flatmap(BIF_ARG_1)) { Uint n; Eterm* hp; Eterm *ks,*vs, res, tup; - map_t *mp = (map_t*)map_val(BIF_ARG_1); + flatmap_t *mp = (flatmap_t*)flatmap_val(BIF_ARG_1); - ks = map_get_keys(mp); - vs = map_get_values(mp); - n = map_get_size(mp); + ks = flatmap_get_keys(mp); + vs = flatmap_get_values(mp); + n = flatmap_get_size(mp); hp = HAlloc(BIF_P, (2 + 3) * n); res = NIL; @@ -102,108 +159,97 @@ BIF_RETTYPE maps_to_list_1(BIF_ALIST_1) { } BIF_RET(res); + } else if (is_hashmap(BIF_ARG_1)) { + return hashmap_to_list(BIF_P, BIF_ARG_1); } - BIF_ERROR(BIF_P, BADARG); + BIF_P->fvalue = BIF_ARG_1; + BIF_ERROR(BIF_P, BADMAP); } /* maps:find/2 * return value if key *matches* a key in the map */ -int erts_maps_find(Eterm key, Eterm map, Eterm *value) { +const Eterm * +erts_maps_get(Eterm key, Eterm map) +{ + Uint32 hx; + if (is_flatmap(map)) { + Eterm *ks, *vs; + flatmap_t *mp; + Uint n, i; - Eterm *ks,*vs; - map_t *mp; - Uint n,i; + mp = (flatmap_t *)flatmap_val(map); + n = flatmap_get_size(mp); - mp = (map_t*)map_val(map); - n = map_get_size(mp); - ks = map_get_keys(mp); - vs = map_get_values(mp); + if (n == 0) { + return NULL; + } - for( i = 0; i < n; i++) { - if (EQ(ks[i], key)) { - *value = vs[i]; - return 1; + ks = (Eterm *)tuple_val(mp->keys) + 1; + vs = flatmap_get_values(mp); + + if (is_immed(key)) { + for (i = 0; i < n; i++) { + if (ks[i] == key) { + return &vs[i]; + } + } + } + + for (i = 0; i < n; i++) { + if (EQ(ks[i], key)) { + return &vs[i]; + } } + return NULL; } - return 0; + ASSERT(is_hashmap(map)); + hx = hashmap_make_hash(key); + + return erts_hashmap_get(hx, key, map); } BIF_RETTYPE maps_find_2(BIF_ALIST_2) { if (is_map(BIF_ARG_2)) { - Eterm *hp, value,res; + Eterm *hp, res; + const Eterm *value; - if (erts_maps_find(BIF_ARG_1, BIF_ARG_2, &value)) { + value = erts_maps_get(BIF_ARG_1, BIF_ARG_2); + if (value) { hp = HAlloc(BIF_P, 3); res = make_tuple(hp); *hp++ = make_arityval(2); *hp++ = am_ok; - *hp++ = value; + *hp++ = *value; BIF_RET(res); } - BIF_RET(am_error); } - BIF_ERROR(BIF_P, BADARG); + BIF_P->fvalue = BIF_ARG_2; + BIF_ERROR(BIF_P, BADMAP); } + /* maps:get/2 * return value if key *matches* a key in the map - * exception bad_key if none matches + * exception badkey if none matches */ - -int erts_maps_get(Eterm key, Eterm map, Eterm *value) { - Eterm *ks,*vs; - map_t *mp; - Uint n,i; - - mp = (map_t*)map_val(map); - n = map_get_size(mp); - - if (n == 0) - return 0; - - ks = map_get_keys(mp); - vs = map_get_values(mp); - - if (is_immed(key)) { - for( i = 0; i < n; i++) { - if (ks[i] == key) { - *value = vs[i]; - return 1; - } - } - } - - for( i = 0; i < n; i++) { - if (EQ(ks[i], key)) { - *value = vs[i]; - return 1; - } - } - return 0; -} - BIF_RETTYPE maps_get_2(BIF_ALIST_2) { if (is_map(BIF_ARG_2)) { - Eterm *hp; - Eterm value, error; - char *s_error; + const Eterm *value; - if (erts_maps_get(BIF_ARG_1, BIF_ARG_2, &value)) { - BIF_RET(value); + value = erts_maps_get(BIF_ARG_1, BIF_ARG_2); + if (value) { + BIF_RET(*value); } - s_error = "bad_key"; - error = am_atom_put(s_error, sys_strlen(s_error)); - - hp = HAlloc(BIF_P, 3); - BIF_P->fvalue = TUPLE2(hp, error, BIF_ARG_1); - BIF_ERROR(BIF_P, EXC_ERROR_2); + BIF_P->fvalue = BIF_ARG_1; + BIF_ERROR(BIF_P, BADKEY); } - BIF_ERROR(BIF_P, BADARG); + BIF_P->fvalue = BIF_ARG_2; + BIF_ERROR(BIF_P, BADMAP); } /* maps:from_list/1 @@ -211,13 +257,8 @@ BIF_RETTYPE maps_get_2(BIF_ALIST_2) { */ BIF_RETTYPE maps_from_list_1(BIF_ALIST_1) { - Eterm *kv, item = BIF_ARG_1; - Eterm *hp, *thp,*vs, *ks, keys, res; - map_t *mp; - Uint size = 0, unused_size = 0; - Sint c = 0; - Sint idx = 0; - + Eterm item = BIF_ARG_1, res, *kv; + Uint size = 0; if (is_list(item) || is_nil(item)) { /* Calculate size and check validity */ @@ -238,450 +279,1329 @@ BIF_RETTYPE maps_from_list_1(BIF_ALIST_1) { if (is_not_nil(item)) goto error; - hp = HAlloc(BIF_P, 3 + 1 + (2 * size)); - thp = hp; + if (size > MAP_SMALL_MAP_LIMIT) { + BIF_RET(hashmap_from_validated_list(BIF_P, BIF_ARG_1, size)); + } else { + BIF_RET(flatmap_from_validated_list(BIF_P, BIF_ARG_1, size)); + } + } + +error: + + BIF_ERROR(BIF_P, BADARG); +} + +static Eterm flatmap_from_validated_list(Process *p, Eterm list, Uint size) { + Eterm *kv, item = list; + Eterm *hp, *thp,*vs, *ks, keys, res; + flatmap_t *mp; + Uint unused_size = 0; + Sint c = 0; + Sint idx = 0; + + + hp = HAlloc(p, 3 + 1 + (2 * size)); + thp = hp; + keys = make_tuple(hp); + *hp++ = make_arityval(size); + ks = hp; + hp += size; + mp = (flatmap_t*)hp; + res = make_flatmap(mp); + hp += MAP_HEADER_FLATMAP_SZ; + vs = hp; + + mp->thing_word = MAP_HEADER_FLATMAP; + mp->size = size; /* set later, might shrink*/ + mp->keys = keys; + + if (size == 0) + return res; + + /* first entry */ + kv = tuple_val(CAR(list_val(item))); + ks[0] = kv[1]; + vs[0] = kv[2]; + size = 1; + item = CDR(list_val(item)); + + /* insert sort key/value pairs */ + while(is_list(item)) { + + kv = tuple_val(CAR(list_val(item))); + + /* compare ks backwards + * idx represent word index to be written (hole position). + * We cannot copy the elements when searching since we might + * have an equal key. So we search for just the index first =( + * + * It is perhaps faster to move the values in the first pass. + * Check for uniqueness during insert phase and then have a + * second phace compacting the map if duplicates are found + * during insert. .. or do someother sort .. shell-sort perhaps. + */ + + idx = size; + + while(idx > 0 && (c = CMP_TERM(kv[1],ks[idx-1])) < 0) { idx--; } + + if (c == 0) { + /* last compare was equal, + * i.e. we have to release memory + * and overwrite that key/value + */ + ks[idx-1] = kv[1]; + vs[idx-1] = kv[2]; + unused_size++; + } else { + Uint i = size; + while(i > idx) { + ks[i] = ks[i-1]; + vs[i] = vs[i-1]; + i--; + } + ks[idx] = kv[1]; + vs[idx] = kv[2]; + size++; + } + item = CDR(list_val(item)); + } + + if (unused_size) { + /* the key tuple is embedded in the heap + * write a bignum to clear it. + */ + /* release values as normal since they are on the top of the heap */ + + ks[size] = make_pos_bignum_header(unused_size - 1); + HRelease(p, vs + size + unused_size, vs + size); + } + + *thp = make_arityval(size); + mp->size = size; + return res; +} + +#define swizzle32(D,S) \ + do { \ + (D) = ((S) & 0x0000000f) << 28 | ((S) & 0x000000f0) << 20 \ + | ((S) & 0x00000f00) << 12 | ((S) & 0x0000f000) << 4 \ + | ((S) & 0x000f0000) >> 4 | ((S) & 0x00f00000) >> 12 \ + | ((S) & 0x0f000000) >> 20 | ((S) & 0xf0000000) >> 28; \ + } while(0) + +#define maskval(V,L) (((V) >> ((7 - (L))*4)) & 0xf) +#define cdepth(V1,V2) (hashmap_clz((V1) ^ (V2)) >> 2) + +static Eterm hashmap_from_validated_list(Process *p, Eterm list, Uint size) { + Eterm item = list; + Eterm *hp; + Eterm *kv, res; + Uint32 sw, hx; + Uint ix = 0; + hxnode_t *hxns; + ErtsHeapFactory factory; + DeclareTmpHeap(tmp,2,p); + ASSERT(size > 0); + + hp = HAlloc(p, (2 * size)); + + /* create tmp hx values and leaf ptrs */ + hxns = (hxnode_t *)erts_alloc(ERTS_ALC_T_TMP, size * sizeof(hxnode_t)); + + UseTmpHeap(2,p); + while(is_list(item)) { + res = CAR(list_val(item)); + kv = tuple_val(res); + hx = hashmap_restore_hash(tmp,0,kv[1]); + swizzle32(sw,hx); + hxns[ix].hx = sw; + hxns[ix].val = CONS(hp, kv[1], kv[2]); hp += 2; + hxns[ix].skip = 1; /* will be reassigned in from_array */ + hxns[ix].i = ix; + ix++; + item = CDR(list_val(item)); + } + UnUseTmpHeap(2,p); + + erts_factory_proc_init(&factory, p); + res = hashmap_from_unsorted_array(&factory, hxns, size, 0); + erts_factory_close(&factory); + + erts_free(ERTS_ALC_T_TMP, (void *) hxns); + ERTS_VERIFY_UNUSED_TEMP_ALLOC(p); + + if (hashmap_size(res) <= MAP_SMALL_MAP_LIMIT) { + DECLARE_WSTACK(wstack); + Eterm *kv, *ks, *vs; + flatmap_t *mp; + Eterm keys; + Uint n = hashmap_size(res); + + /* build flat structure */ + hp = HAlloc(p, 3 + 1 + (2 * n)); keys = make_tuple(hp); - *hp++ = make_arityval(size); + *hp++ = make_arityval(n); ks = hp; - hp += size; - mp = (map_t*)hp; - res = make_map(mp); - hp += MAP_HEADER_SIZE; + hp += n; + mp = (flatmap_t*)hp; + hp += MAP_HEADER_FLATMAP_SZ; vs = hp; - mp->thing_word = MAP_HEADER; - mp->size = size; /* set later, might shrink*/ + mp->thing_word = MAP_HEADER_FLATMAP; + mp->size = n; mp->keys = keys; - if (size == 0) - BIF_RET(res); + hashmap_iterator_init(&wstack, res, 0); - item = BIF_ARG_1; + while ((kv=hashmap_iterator_next(&wstack)) != NULL) { + *ks++ = CAR(kv); + *vs++ = CDR(kv); + } - /* first entry */ - kv = tuple_val(CAR(list_val(item))); - ks[0] = kv[1]; - vs[0] = kv[2]; - size = 1; - item = CDR(list_val(item)); + /* it cannot have multiple keys */ + erts_validate_and_sort_flatmap(mp); - /* insert sort key/value pairs */ - while(is_list(item)) { + DESTROY_WSTACK(wstack); + return make_flatmap(mp); + } - kv = tuple_val(CAR(list_val(item))); - - /* compare ks backwards - * idx represent word index to be written (hole position). - * We cannot copy the elements when searching since we might - * have an equal key. So we search for just the index first =( - * - * It is perhaps faster to move the values in the first pass. - * Check for uniqueness during insert phase and then have a - * second phace compacting the map if duplicates are found - * during insert. .. or do someother sort .. shell-sort perhaps. - */ + return res; +} - idx = size; +Eterm erts_hashmap_from_array(ErtsHeapFactory* factory, Eterm *leafs, Uint n, + int reject_dupkeys) { + Uint32 sw, hx; + Uint ix; + hxnode_t *hxns; + Eterm res; + + /* create tmp hx values and leaf ptrs */ + hxns = (hxnode_t *)erts_alloc(ERTS_ALC_T_TMP, n * sizeof(hxnode_t)); + + for (ix = 0; ix < n; ix++) { + hx = hashmap_make_hash(*leafs); + swizzle32(sw,hx); + hxns[ix].hx = sw; + hxns[ix].val = make_list(leafs); + hxns[ix].skip = 1; + hxns[ix].i = ix; + leafs += 2; + } - while(idx > 0 && (c = CMP_TERM(kv[1],ks[idx-1])) < 0) { idx--; } + res = hashmap_from_unsorted_array(factory, hxns, n, reject_dupkeys); - if (c == 0) { - /* last compare was equal, - * i.e. we have to release memory - * and overwrite that key/value - */ - ks[idx-1] = kv[1]; - vs[idx-1] = kv[2]; - unused_size++; - } else { - Uint i = size; - while(i > idx) { - ks[i] = ks[i-1]; - vs[i] = vs[i-1]; - i--; + erts_free(ERTS_ALC_T_TMP, (void *) hxns); + + return res; +} + + +Eterm erts_hashmap_from_ks_and_vs_extra(Process *p, Eterm *ks, Eterm *vs, Uint n, + Eterm key, Eterm value) { + Uint32 sw, hx; + Uint i,sz; + hxnode_t *hxns; + ErtsHeapFactory factory; + Eterm *hp, res; + + sz = (key == THE_NON_VALUE) ? n : (n + 1); + ASSERT(sz > MAP_SMALL_MAP_LIMIT); + hp = HAlloc(p, 2 * sz); + + /* create tmp hx values and leaf ptrs */ + hxns = (hxnode_t *)erts_alloc(ERTS_ALC_T_TMP, sz * sizeof(hxnode_t)); + + for(i = 0; i < n; i++) { + hx = hashmap_make_hash(ks[i]); + swizzle32(sw,hx); + hxns[i].hx = sw; + hxns[i].val = CONS(hp, ks[i], vs[i]); hp += 2; + hxns[i].skip = 1; /* will be reassigned in from_array */ + hxns[i].i = i; + } + + if (key != THE_NON_VALUE) { + hx = hashmap_make_hash(key); + swizzle32(sw,hx); + hxns[i].hx = sw; + hxns[i].val = CONS(hp, key, value); hp += 2; + hxns[i].skip = 1; + hxns[i].i = i; + } + + erts_factory_proc_init(&factory, p); + res = hashmap_from_unsorted_array(&factory, hxns, sz, 0); + erts_factory_close(&factory); + + erts_free(ERTS_ALC_T_TMP, (void *) hxns); + ERTS_VERIFY_UNUSED_TEMP_ALLOC(p); + + return res; +} + +static Eterm hashmap_from_unsorted_array(ErtsHeapFactory* factory, + hxnode_t *hxns, Uint n, + int reject_dupkeys) { + Uint jx = 0, ix = 0, lx, cx; + Eterm res; + + if (n == 0) { + Eterm *hp; + hp = erts_produce_heap(factory, 2, 0); + hp[0] = MAP_HEADER_HAMT_HEAD_BITMAP(0); + hp[1] = 0; + + return make_hashmap(hp); + } + + /* sort and compact array (remove non-unique entries) */ + qsort(hxns, n, sizeof(hxnode_t), (int (*)(const void *, const void *)) hxnodecmp); + + ix = 0, cx = 0; + while(ix < n - 1) { + if (hxns[ix].hx == hxns[ix+1].hx) { + + /* find region of equal hash values */ + jx = ix + 1; + while(jx < n && hxns[ix].hx == hxns[jx].hx) { jx++; } + /* find all correct keys from region + * (last in list but now hash sorted so we check highest id instead) */ + + /* resort with keys instead of hash value within region */ + + qsort(&hxns[ix], jx - ix, sizeof(hxnode_t), + (int (*)(const void *, const void *)) hxnodecmpkey); + + while(ix < jx) { + lx = ix; + while(++ix < jx && EQ(CAR(list_val(hxns[ix].val)), + CAR(list_val(hxns[lx].val)))) { + if (reject_dupkeys) + return THE_NON_VALUE; + + if (hxns[ix].i > hxns[lx].i) { + lx = ix; + } } - ks[idx] = kv[1]; - vs[idx] = kv[2]; - size++; + hxns[cx].hx = hxns[lx].hx; + hxns[cx].val = hxns[lx].val; + cx++; } - item = CDR(list_val(item)); + ix = jx; + continue; + } + if (ix > cx) { + hxns[cx].hx = hxns[ix].hx; + hxns[cx].val = hxns[ix].val; } + cx++; + ix++; + } - if (unused_size) { - /* the key tuple is embedded in the heap - * write a bignum to clear it. - */ - /* release values as normal since they are on the top of the heap */ + if (ix < n) { + hxns[cx].hx = hxns[ix].hx; + hxns[cx].val = hxns[ix].val; + cx++; + } - ks[size] = make_pos_bignum_header(unused_size - 1); - HRelease(BIF_P, vs + size + unused_size, vs + size); - } + if (cx > 1) { + /* recursive decompose array */ + res = hashmap_from_sorted_unique_array(factory, hxns, cx, 0); + } else { + Eterm *hp; - *thp = make_arityval(size); - mp->size = size; - BIF_RET(res); + /* we only have one item, either because n was 1 or + * because we hade multiples of the same key. + * + * hash value has been swizzled, need to drag it down to get the + * correct slot. */ + + hp = erts_produce_heap(factory, HAMT_HEAD_BITMAP_SZ(1), 0); + hp[0] = MAP_HEADER_HAMT_HEAD_BITMAP(1 << ((hxns[0].hx >> 0x1c) & 0xf)); + hp[1] = 1; + hp[2] = hxns[0].val; + res = make_hashmap(hp); } -error: + return res; +} - BIF_ERROR(BIF_P, BADARG); +static Eterm hashmap_from_sorted_unique_array(ErtsHeapFactory* factory, + hxnode_t *hxns, Uint n, int lvl) { + Eterm res = NIL; + Uint i,ix,jx,elems; + Uint32 sw, hx; + Eterm val; + hxnode_t *tmp; + DeclareTmpHeapNoproc(th,2); + UseTmpHeapNoproc(2); + ASSERT(lvl < 32); + ix = 0; + elems = 1; + while (ix < n - 1) { + if (hxns[ix].hx == hxns[ix+1].hx) { + jx = ix + 1; + while (jx < n && hxns[ix].hx == hxns[jx].hx) { jx++; } + tmp = (hxnode_t *)erts_alloc(ERTS_ALC_T_TMP, ((jx - ix)) * sizeof(hxnode_t)); + + for(i = 0; i < jx - ix; i++) { + val = hxns[i + ix].val; + hx = hashmap_restore_hash(th, lvl + 8, CAR(list_val(val))); + swizzle32(sw,hx); + tmp[i].hx = sw; + tmp[i].val = val; + tmp[i].i = i; + tmp[i].skip = 1; + } + + qsort(tmp, jx - ix, sizeof(hxnode_t), (int (*)(const void *, const void *)) hxnodecmp); + + hxns[ix].skip = jx - ix; + hxns[ix].val = hashmap_from_sorted_unique_array(factory, tmp, jx - ix, lvl + 8); + erts_free(ERTS_ALC_T_TMP, (void *) tmp); + ix = jx; + if (ix < n) { elems++; } + continue; + } + hxns[ix].skip = 1; + elems++; + ix++; + } + + res = hashmap_from_chunked_array(factory, hxns, elems, n, !lvl); + + ERTS_FACTORY_HOLE_CHECK(factory); + + UnUseTmpHeapNoproc(2); + return res; } -/* maps:is_key/2 - */ +#define HALLOC_EXTRA 200 +static Eterm hashmap_from_chunked_array(ErtsHeapFactory *factory, hxnode_t *hxns, Uint n, + Uint size, int is_root) { + Uint ix, d, dn, dc, slot, elems; + Uint32 v, vp, vn, hdr; + Uint bp, sz; + DECLARE_ESTACK(stack); + Eterm res = NIL, *hp = NULL, *nhp; -BIF_RETTYPE maps_is_key_2(BIF_ALIST_2) { - if (is_map(BIF_ARG_2)) { - Eterm *ks, key; - map_t *mp; - Uint n,i; - mp = (map_t*)map_val(BIF_ARG_2); - key = BIF_ARG_1; - n = map_get_size(mp); - ks = map_get_keys(mp); + /* if we get here with only one element then + * we have eight levels of collisions + */ - if (n == 0) - BIF_RET(am_false); + if (n == 1) { + res = hxns[0].val; + v = hxns[0].hx; + for (d = 7; d > 0; d--) { + slot = maskval(v,d); + hp = erts_produce_heap(factory, HAMT_NODE_BITMAP_SZ(1), HALLOC_EXTRA); + hp[0] = MAP_HEADER_HAMT_NODE_BITMAP(1 << slot); + hp[1] = res; + res = make_hashmap(hp); + } - if (is_immed(key)) { - for( i = 0; i < n; i++) { - if (ks[i] == key) { - BIF_RET(am_true); - } + slot = maskval(v,0); + hp = erts_produce_heap(factory, (is_root ? 3 : 2), 0); + + if (is_root) { + hp[0] = MAP_HEADER_HAMT_HEAD_BITMAP(1 << slot); + hp[1] = size; + hp[2] = res; + } else { + hp[0] = MAP_HEADER_HAMT_NODE_BITMAP(1 << slot); + hp[1] = res; + } + return make_hashmap(hp); + } + + /* push initial nodes on the stack, + * this is the starting depth */ + + ix = 0; + d = 0; + vp = hxns[ix].hx; + v = hxns[ix + hxns[ix].skip].hx; + + ASSERT(vp > v); + slot = maskval(vp,d); + + while(slot == maskval(v,d)) { + ESTACK_PUSH(stack, 1 << slot); + d++; + slot = maskval(vp,d); + } + + res = hxns[ix].val; + + if (hxns[ix].skip > 1) { + dc = 7; + /* build collision nodes */ + while (dc > d) { + hp = erts_produce_heap(factory, HAMT_NODE_BITMAP_SZ(1), HALLOC_EXTRA); + hp[0] = MAP_HEADER_HAMT_NODE_BITMAP(1 << maskval(vp,dc)); + hp[1] = res; + res = make_hashmap(hp); + dc--; + } + } + + ESTACK_PUSH2(stack,res,1 << slot); + + /* all of the other nodes .. */ + elems = n - 2; /* remove first and last elements */ + while(elems--) { + hdr = ESTACK_POP(stack); + ix = ix + hxns[ix].skip; + + /* determine if node or subtree should be built by looking + * at the next value. */ + + vn = hxns[ix + hxns[ix].skip].hx; + dn = cdepth(v,vn); + ASSERT(v > vn); + + res = hxns[ix].val; + + if (hxns[ix].skip > 1) { + int wat = (d > dn) ? d : dn; + dc = 7; + /* build collision nodes */ + while (dc > wat) { + hp = erts_produce_heap(factory, HAMT_NODE_BITMAP_SZ(1), HALLOC_EXTRA); + hp[0] = MAP_HEADER_HAMT_NODE_BITMAP(1 << maskval(v,dc)); + hp[1] = res; + res = make_hashmap(hp); + dc--; } } - for( i = 0; i < n; i++) { - if (EQ(ks[i], key)) { - BIF_RET(am_true); + /* next depth is higher (implies collision) */ + if (d < dn) { + /* hdr is the popped one initially */ + while(d < dn) { + slot = maskval(v, d); + bp = 1 << slot; + ESTACK_PUSH(stack, hdr | bp); + d++; + hdr = 0; /* clear hdr for all other collisions */ + } + + slot = maskval(v, d); + bp = 1 << slot; + /* no more collisions */ + ESTACK_PUSH2(stack,res,bp); + } else if (d == dn) { + /* no collisions at all */ + slot = maskval(v, d); + bp = 1 << slot; + ESTACK_PUSH2(stack,res,hdr | bp); + } else { + /* dn < n, we have a drop and we are done + * build nodes and subtree */ + while (dn != d) { + slot = maskval(v, d); + bp = 1 << slot; + /* OR bitposition before sz calculation to handle + * redundant collisions */ + hdr |= bp; + sz = hashmap_bitcount(hdr); + hp = erts_produce_heap(factory, HAMT_NODE_BITMAP_SZ(sz), HALLOC_EXTRA); + nhp = hp; + *hp++ = MAP_HEADER_HAMT_NODE_BITMAP(hdr); + *hp++ = res; sz--; + while (sz--) { *hp++ = ESTACK_POP(stack); } + ASSERT((hp - nhp) < 18); + res = make_hashmap(nhp); + + /* we need to pop the next hdr and push if we don't need it */ + + hdr = ESTACK_POP(stack); + d--; } + ESTACK_PUSH2(stack,res,hdr); } - BIF_RET(am_false); + + vp = v; + v = vn; + d = dn; + ERTS_FACTORY_HOLE_CHECK(factory); } - BIF_ERROR(BIF_P, BADARG); + + /* v and vp are reused from above */ + dn = cdepth(vp,v); + ix = ix + hxns[ix].skip; + res = hxns[ix].val; + + if (hxns[ix].skip > 1) { + dc = 7; + /* build collision nodes */ + while (dc > dn) { + hp = erts_produce_heap(factory, HAMT_NODE_BITMAP_SZ(1), HALLOC_EXTRA); + hp[0] = MAP_HEADER_HAMT_NODE_BITMAP(1 << maskval(v,dc)); + hp[1] = res; + res = make_hashmap(hp); + dc--; + } + } + + hdr = ESTACK_POP(stack); + /* pop remaining subtree if any */ + while (dn) { + slot = maskval(v, dn); + bp = 1 << slot; + /* OR bitposition before sz calculation to handle + * redundant collisions */ + hdr |= bp; + sz = hashmap_bitcount(hdr); + hp = erts_produce_heap(factory, HAMT_NODE_BITMAP_SZ(sz), HALLOC_EXTRA); + nhp = hp; + *hp++ = MAP_HEADER_HAMT_NODE_BITMAP(hdr); + *hp++ = res; sz--; + + while (sz--) { *hp++ = ESTACK_POP(stack); } + res = make_hashmap(nhp); + hdr = ESTACK_POP(stack); + dn--; + } + + /* and finally the root .. */ + + slot = maskval(v, dn); + bp = 1 << slot; + hdr |= bp; + sz = hashmap_bitcount(hdr); + hp = erts_produce_heap(factory, sz + /* hdr + item */ (is_root ? 2 : 1), 0); + nhp = hp; + + if (is_root) { + *hp++ = (hdr == 0xffff) ? MAP_HEADER_HAMT_HEAD_ARRAY : MAP_HEADER_HAMT_HEAD_BITMAP(hdr); + *hp++ = size; + } else { + *hp++ = MAP_HEADER_HAMT_NODE_BITMAP(hdr); + } + + *hp++ = res; sz--; + while (sz--) { *hp++ = ESTACK_POP(stack); } + + res = make_hashmap(nhp); + + ASSERT(ESTACK_COUNT(stack) == 0); + DESTROY_ESTACK(stack); + ERTS_FACTORY_HOLE_CHECK(factory); + return res; +} +#undef HALLOC_EXTRA + +static int hxnodecmpkey(hxnode_t *a, hxnode_t *b) { + Sint c = CMP_TERM(CAR(list_val(a->val)), CAR(list_val(b->val))); +#if ERTS_SIZEOF_ETERM <= SIZEOF_INT + return c; +#else + return c > 0 ? 1 : (c < 0 ? -1 : 0); +#endif } -/* maps:keys/1 - */ +static int hxnodecmp(hxnode_t *a, hxnode_t *b) { + if (a->hx < b->hx) + return 1; + else if (a->hx == b->hx) + return 0; + else + return -1; +} + +/* maps:is_key/2 */ + +BIF_RETTYPE maps_is_key_2(BIF_ALIST_2) { + if (is_map(BIF_ARG_2)) { + BIF_RET(erts_maps_get(BIF_ARG_1, BIF_ARG_2) ? am_true : am_false); + } + BIF_P->fvalue = BIF_ARG_2; + BIF_ERROR(BIF_P, BADMAP); +} + +/* maps:keys/1 */ BIF_RETTYPE maps_keys_1(BIF_ALIST_1) { - if (is_map(BIF_ARG_1)) { + if (is_flatmap(BIF_ARG_1)) { Eterm *hp, *ks, res = NIL; - map_t *mp; + flatmap_t *mp; Uint n; - mp = (map_t*)map_val(BIF_ARG_1); - n = map_get_size(mp); + mp = (flatmap_t*)flatmap_val(BIF_ARG_1); + n = flatmap_get_size(mp); if (n == 0) BIF_RET(res); hp = HAlloc(BIF_P, (2 * n)); - ks = map_get_keys(mp); + ks = flatmap_get_keys(mp); while(n--) { res = CONS(hp, ks[n], res); hp += 2; } BIF_RET(res); + } else if (is_hashmap(BIF_ARG_1)) { + BIF_RET(hashmap_keys(BIF_P, BIF_ARG_1)); } - BIF_ERROR(BIF_P, BADARG); + BIF_P->fvalue = BIF_ARG_1; + BIF_ERROR(BIF_P, BADMAP); } -/* maps:merge/2 - */ + +/* maps:merge/2 */ + +HIPE_WRAPPER_BIF_DISABLE_GC(maps_merge, 2) BIF_RETTYPE maps_merge_2(BIF_ALIST_2) { - if (is_map(BIF_ARG_1) && is_map(BIF_ARG_2)) { - Eterm *hp,*thp; - Eterm tup; - Eterm *ks,*vs,*ks1,*vs1,*ks2,*vs2; - map_t *mp1,*mp2,*mp_new; - Uint n1,n2,i1,i2,need,unused_size=0; - int c = 0; - - mp1 = (map_t*)map_val(BIF_ARG_1); - mp2 = (map_t*)map_val(BIF_ARG_2); - n1 = map_get_size(mp1); - n2 = map_get_size(mp2); - - need = MAP_HEADER_SIZE + 1 + 2*(n1 + n2); - - hp = HAlloc(BIF_P, need); - thp = hp; - tup = make_tuple(thp); - ks = hp + 1; hp += 1 + n1 + n2; - mp_new = (map_t*)hp; hp += MAP_HEADER_SIZE; - vs = hp; hp += n1 + n2; - - mp_new->thing_word = MAP_HEADER; - mp_new->size = 0; - mp_new->keys = tup; - - i1 = 0; i2 = 0; - ks1 = map_get_keys(mp1); - vs1 = map_get_values(mp1); - ks2 = map_get_keys(mp2); - vs2 = map_get_values(mp2); - - while(i1 < n1 && i2 < n2) { - c = CMP_TERM(ks1[i1],ks2[i2]); - if ( c == 0) { - /* use righthand side arguments map value, - * but advance both maps */ - *ks++ = ks2[i2]; - *vs++ = vs2[i2]; - i1++, i2++, unused_size++; - } else if ( c < 0) { - *ks++ = ks1[i1]; - *vs++ = vs1[i1]; - i1++; - } else { - *ks++ = ks2[i2]; - *vs++ = vs2[i2]; - i2++; - } + if (is_flatmap(BIF_ARG_1)) { + if (is_flatmap(BIF_ARG_2)) { + BIF_RET(flatmap_merge(BIF_P, BIF_ARG_1, BIF_ARG_2)); + } else if (is_hashmap(BIF_ARG_2)) { + /* Will always become a tree */ + return map_merge_mixed(BIF_P, BIF_ARG_1, BIF_ARG_2, 0); } + BIF_P->fvalue = BIF_ARG_2; + } else if (is_hashmap(BIF_ARG_1)) { + if (is_hashmap(BIF_ARG_2)) { + return hashmap_merge(BIF_P, BIF_ARG_1, BIF_ARG_2, 0, NULL); + } else if (is_flatmap(BIF_ARG_2)) { + /* Will always become a tree */ + return map_merge_mixed(BIF_P, BIF_ARG_2, BIF_ARG_1, 1); + } + BIF_P->fvalue = BIF_ARG_2; + } else { + BIF_P->fvalue = BIF_ARG_1; + } + BIF_ERROR(BIF_P, BADMAP); +} + +static Eterm flatmap_merge(Process *p, Eterm nodeA, Eterm nodeB) { + Eterm *hp,*thp; + Eterm tup; + Eterm *ks,*vs,*ks1,*vs1,*ks2,*vs2; + flatmap_t *mp1,*mp2,*mp_new; + Uint n,n1,n2,i1,i2,need,unused_size=0; + Sint c = 0; - /* copy remaining */ - while (i1 < n1) { + mp1 = (flatmap_t*)flatmap_val(nodeA); + mp2 = (flatmap_t*)flatmap_val(nodeB); + n1 = flatmap_get_size(mp1); + n2 = flatmap_get_size(mp2); + + need = MAP_HEADER_FLATMAP_SZ + 1 + 2 * (n1 + n2); + + hp = HAlloc(p, need); + thp = hp; + tup = make_tuple(thp); + ks = hp + 1; hp += 1 + n1 + n2; + mp_new = (flatmap_t*)hp; hp += MAP_HEADER_FLATMAP_SZ; + vs = hp; hp += n1 + n2; + + mp_new->thing_word = MAP_HEADER_FLATMAP; + mp_new->size = 0; + mp_new->keys = tup; + + i1 = 0; i2 = 0; + ks1 = flatmap_get_keys(mp1); + vs1 = flatmap_get_values(mp1); + ks2 = flatmap_get_keys(mp2); + vs2 = flatmap_get_values(mp2); + + while(i1 < n1 && i2 < n2) { + c = CMP_TERM(ks1[i1],ks2[i2]); + if (c == 0) { + /* use righthand side arguments map value, + * but advance both maps */ + *ks++ = ks2[i2]; + *vs++ = vs2[i2]; + i1++, i2++, unused_size++; + } else if (c < 0) { *ks++ = ks1[i1]; *vs++ = vs1[i1]; i1++; - } - - while (i2 < n2) { + } else { *ks++ = ks2[i2]; *vs++ = vs2[i2]; i2++; } + } - if (unused_size) { - /* the key tuple is embedded in the heap, write a bignum to clear it. - * - * release values as normal since they are on the top of the heap - * size = n1 + n1 - unused_size - */ + /* copy remaining */ + while (i1 < n1) { + *ks++ = ks1[i1]; + *vs++ = vs1[i1]; + i1++; + } + + while (i2 < n2) { + *ks++ = ks2[i2]; + *vs++ = vs2[i2]; + i2++; + } + + if (unused_size) { + /* the key tuple is embedded in the heap, write a bignum to clear it. + * + * release values as normal since they are on the top of the heap + * size = n1 + n1 - unused_size + */ + + *ks = make_pos_bignum_header(unused_size - 1); + HRelease(p, vs + unused_size, vs); + } + + n = n1 + n2 - unused_size; + *thp = make_arityval(n); + mp_new->size = n; + + /* Reshape map to a hashmap if the map exceeds the limit */ - *ks = make_pos_bignum_header(unused_size - 1); - HRelease(BIF_P, vs + unused_size, vs); + if (n > MAP_SMALL_MAP_LIMIT) { + Uint32 hx,sw; + Uint i; + Eterm res; + hxnode_t *hxns; + ErtsHeapFactory factory; + + ks = flatmap_get_keys(mp_new); + vs = flatmap_get_values(mp_new); + + hp = HAlloc(p, 2 * n); + + hxns = (hxnode_t *)erts_alloc(ERTS_ALC_T_TMP,n * sizeof(hxnode_t)); + + for (i = 0; i < n; i++) { + hx = hashmap_make_hash(ks[i]); + swizzle32(sw,hx); + hxns[i].hx = sw; + hxns[i].val = CONS(hp, ks[i], vs[i]); hp += 2; + hxns[i].skip = 1; + hxns[i].i = i; } - mp_new->size = n1 + n2 - unused_size; - *thp = make_arityval(n1 + n2 - unused_size); + erts_factory_proc_init(&factory, p); + res = hashmap_from_unsorted_array(&factory, hxns, n, 0); + erts_factory_close(&factory); - BIF_RET(make_map(mp_new)); + erts_free(ERTS_ALC_T_TMP, (void *) hxns); + ERTS_VERIFY_UNUSED_TEMP_ALLOC(p); + + return res; } - BIF_ERROR(BIF_P, BADARG); + + return make_flatmap(mp_new); } -/* maps:new/2 - */ -BIF_RETTYPE maps_new_0(BIF_ALIST_0) { - Eterm* hp; - Eterm tup; - map_t *mp; +static Eterm map_merge_mixed(Process *p, Eterm flat, Eterm tree, int swap_args) { + Eterm *ks, *vs, *hp, res; + flatmap_t *mp; + Uint n, i; + hxnode_t *hxns; + Uint32 sw, hx; + ErtsHeapFactory factory; - hp = HAlloc(BIF_P, (MAP_HEADER_SIZE + 1)); - tup = make_tuple(hp); - *hp++ = make_arityval(0); + /* convert flat to tree */ - mp = (map_t*)hp; - mp->thing_word = MAP_HEADER; - mp->size = 0; - mp->keys = tup; + ASSERT(is_flatmap(flat)); + ASSERT(is_hashmap(tree)); + + mp = (flatmap_t*)flatmap_val(flat); + n = flatmap_get_size(mp); + + ks = flatmap_get_keys(mp); + vs = flatmap_get_values(mp); + + hp = HAlloc(p, 2 * n); - BIF_RET(make_map(mp)); + hxns = (hxnode_t *)erts_alloc(ERTS_ALC_T_TMP, n * sizeof(hxnode_t)); + + for (i = 0; i < n; i++) { + hx = hashmap_make_hash(ks[i]); + swizzle32(sw,hx); + hxns[i].hx = sw; + hxns[i].val = CONS(hp, ks[i], vs[i]); hp += 2; + hxns[i].skip = 1; + hxns[i].i = i; + } + + erts_factory_proc_init(&factory, p); + res = hashmap_from_unsorted_array(&factory, hxns, n, 0); + erts_factory_close(&factory); + + erts_free(ERTS_ALC_T_TMP, (void *) hxns); + ERTS_VERIFY_UNUSED_TEMP_ALLOC(p); + + return hashmap_merge(p, res, tree, swap_args, NULL); } -/* maps:put/3 - */ +#define PSTACK_TYPE struct HashmapMergePStackType +struct HashmapMergePStackType { + Eterm nodeA, nodeB; + Eterm *srcA, *srcB; + Uint32 abm, bbm, rbm; /* node bitmaps */ + int mix; /* &1: there are unique A stuff in node + * &2: there are unique B stuff in node */ + int ix; + Eterm array[16]; /* temp node construction area */ +}; + +typedef struct HashmapMergeContext_ { + Uint size; /* total key-value counter */ + unsigned int lvl; + Eterm trap_bin; + ErtsPStack pstack; +#ifdef DEBUG + Eterm dbg_map_A, dbg_map_B; +#endif +} HashmapMergeContext; -Eterm erts_maps_put(Process *p, Eterm key, Eterm value, Eterm map) { - Sint n,i; - Sint c = 0; - Eterm* hp, *shp; - Eterm *ks,*vs, res, tup; - map_t *mp = (map_t*)map_val(map); +static void hashmap_merge_ctx_destructor(Binary* ctx_bin) +{ + HashmapMergeContext* ctx = (HashmapMergeContext*) ERTS_MAGIC_BIN_DATA(ctx_bin); + ASSERT(ERTS_MAGIC_BIN_DESTRUCTOR(ctx_bin) == hashmap_merge_ctx_destructor); - n = map_get_size(mp); + PSTACK_DESTROY_SAVED(&ctx->pstack); +} - if (n == 0) { - hp = HAlloc(p, MAP_HEADER_SIZE + 1 + 2); - tup = make_tuple(hp); - *hp++ = make_arityval(1); - *hp++ = key; - res = make_map(hp); - *hp++ = MAP_HEADER; - *hp++ = 1; - *hp++ = tup; - *hp++ = value; +BIF_RETTYPE maps_merge_trap_1(BIF_ALIST_1) { + Binary* ctx_bin = ((ProcBin *) binary_val(BIF_ARG_1))->val; - return res; - } + ASSERT(ERTS_MAGIC_BIN_DESTRUCTOR(ctx_bin) == hashmap_merge_ctx_destructor); - ks = map_get_keys(mp); - vs = map_get_values(mp); + return hashmap_merge(BIF_P, NIL, NIL, 0, + (HashmapMergeContext*) ERTS_MAGIC_BIN_DATA(ctx_bin)); +} - /* only allocate for values, - * assume key-tuple will be intact +#define HALLOC_EXTRA 200 +#define MAP_MERGE_LOOP_FACTOR 8 + +static BIF_RETTYPE hashmap_merge(Process *p, Eterm map_A, Eterm map_B, + int swap_args, HashmapMergeContext* ctx) { +#define PSTACK_TYPE struct HashmapMergePStackType + PSTACK_DECLARE(s, 4); + HashmapMergeContext local_ctx; + struct HashmapMergePStackType* sp; + Uint32 hx; + Eterm res = THE_NON_VALUE; + Eterm hdrA, hdrB; + Eterm *hp, *nhp; + Eterm trap_ret; + Sint initial_reds = (Sint) (ERTS_BIF_REDS_LEFT(p) * MAP_MERGE_LOOP_FACTOR); + Sint reds = initial_reds; + DeclareTmpHeap(th,2,p); + UseTmpHeap(2,p); + + /* + * Strategy: Do depth-first traversal of both trees (at the same time) + * and merge each pair of nodes. */ - hp = HAlloc(p, MAP_HEADER_SIZE + n); - shp = hp; /* save hp, used if optimistic update fails */ - res = make_map(hp); - *hp++ = MAP_HEADER; - *hp++ = n; - *hp++ = mp->keys; - - if (is_immed(key)) { - for( i = 0; i < n; i ++) { - if (ks[i] == key) { - *hp++ = value; - vs++; - c = 1; - } else { - *hp++ = *vs++; + PSTACK_CHANGE_ALLOCATOR(s, ERTS_ALC_T_SAVED_ESTACK); + + if (ctx == NULL) { /* first call */ + hashmap_head_t* a = (hashmap_head_t*) hashmap_val(map_A); + hashmap_head_t* b = (hashmap_head_t*) hashmap_val(map_B); + + sp = PSTACK_PUSH(s); + sp->srcA = swap_args ? &map_B : &map_A; + sp->srcB = swap_args ? &map_A : &map_B; + sp->mix = 0; + local_ctx.size = a->size + b->size; + local_ctx.lvl = 0; + #ifdef DEBUG + local_ctx.dbg_map_A = map_A; + local_ctx.dbg_map_B = map_B; + local_ctx.trap_bin = THE_NON_VALUE; + #endif + ctx = &local_ctx; + } + else { + PSTACK_RESTORE(s, &ctx->pstack); + sp = PSTACK_TOP(s); + goto resume_from_trap; + } + +recurse: + + sp->nodeA = *sp->srcA; + sp->nodeB = *sp->srcB; + + if (sp->nodeA == sp->nodeB) { + res = sp->nodeA; + ctx->size -= is_list(sp->nodeB) ? 1 : hashmap_subtree_size(sp->nodeB); + } + else { + if (is_list(sp->nodeA)) { /* A is LEAF */ + Eterm keyA = CAR(list_val(sp->nodeA)); + + if (is_list(sp->nodeB)) { /* LEAF + LEAF */ + Eterm keyB = CAR(list_val(sp->nodeB)); + + if (EQ(keyA, keyB)) { + --ctx->size; + res = sp->nodeB; + sp->mix = 2; /* We assume values differ. + + Don't spend time comparing big values. + - Might waste some heap space for internal + nodes that could otherwise be reused. */ + goto merge_nodes; + } + } + hx = hashmap_restore_hash(th, ctx->lvl, keyA); + sp->abm = 1 << hashmap_index(hx); + /* keep srcA pointing at the leaf */ + } + else { /* A is NODE */ + sp->srcA = boxed_val(sp->nodeA); + hdrA = *sp->srcA++; + ASSERT(is_header(hdrA)); + switch (hdrA & _HEADER_MAP_SUBTAG_MASK) { + case HAMT_SUBTAG_HEAD_ARRAY: { + sp->srcA++; + sp->abm = 0xffff; + break; + } + case HAMT_SUBTAG_HEAD_BITMAP: sp->srcA++; + case HAMT_SUBTAG_NODE_BITMAP: { + sp->abm = MAP_HEADER_VAL(hdrA); + break; + } + default: + erts_exit(ERTS_ABORT_EXIT, "bad header %ld\r\n", hdrA); + } + } + + if (is_list(sp->nodeB)) { /* B is LEAF */ + Eterm keyB = CAR(list_val(sp->nodeB)); + + hx = hashmap_restore_hash(th, ctx->lvl, keyB); + sp->bbm = 1 << hashmap_index(hx); + /* keep srcB pointing at the leaf */ + } + else { /* B is NODE */ + sp->srcB = boxed_val(sp->nodeB); + hdrB = *sp->srcB++; + ASSERT(is_header(hdrB)); + switch (hdrB & _HEADER_MAP_SUBTAG_MASK) { + case HAMT_SUBTAG_HEAD_ARRAY: { + sp->srcB++; + sp->bbm = 0xffff; + break; + } + case HAMT_SUBTAG_HEAD_BITMAP: sp->srcB++; + case HAMT_SUBTAG_NODE_BITMAP: { + sp->bbm = MAP_HEADER_VAL(hdrB); + break; + } + default: + erts_exit(ERTS_ABORT_EXIT, "bad header %ld\r\n", hdrB); + } + } + } + +merge_nodes: + + for (;;) { + if (is_value(res)) { /* We have a complete (sub-)tree or leaf */ + int child_mix; + if (ctx->lvl == 0) + break; + + /* Pop from stack and continue build parent node */ + ctx->lvl--; + child_mix = sp->mix; + sp = PSTACK_POP(s); + sp->array[sp->ix++] = res; + sp->mix |= child_mix; + res = THE_NON_VALUE; + if (sp->rbm) { + sp->srcA++; + sp->srcB++; } + } else { /* Start build a node */ + sp->ix = 0; + sp->rbm = sp->abm | sp->bbm; + ASSERT(!(sp->rbm == 0 && ctx->lvl > 0)); } - } else { - for( i = 0; i < n; i ++) { - if (EQ(ks[i], key)) { - *hp++ = value; - vs++; - c = 1; + + if (--reds <= 0) { + goto trap; + } +resume_from_trap: + + while (sp->rbm) { + Uint32 next = sp->rbm & (sp->rbm-1); + Uint32 bit = sp->rbm ^ next; + sp->rbm = next; + if (sp->abm & bit) { + if (sp->bbm & bit) { + /* Bit clash. Push and resolve by recursive merge */ + Eterm* srcA = sp->srcA; + Eterm* srcB = sp->srcB; + ctx->lvl++; + sp = PSTACK_PUSH(s); + sp->srcA = srcA; + sp->srcB = srcB; + sp->mix = 0; + goto recurse; + } else { + sp->array[sp->ix++] = *sp->srcA++; + sp->mix |= 1; + } } else { - *hp++ = *vs++; + ASSERT(sp->bbm & bit); + sp->array[sp->ix++] = *sp->srcB++; + sp->mix |= 2; } } + + switch (sp->mix) { + case 0: /* Nodes A and B contain the *EXACT* same sub-trees + => fall through and reuse nodeA */ + + case 1: /* Only unique A stuff => reuse nodeA */ + res = sp->nodeA; + break; + + case 2: /* Only unique B stuff => reuse nodeB */ + res = sp->nodeB; + break; + + case 3: /* We have a mix => must build new node */ + ASSERT(sp->ix == hashmap_bitcount(sp->abm | sp->bbm)); + if (ctx->lvl == 0) { + nhp = HAllocX(p, HAMT_HEAD_BITMAP_SZ(sp->ix), HALLOC_EXTRA); + hp = nhp; + *hp++ = (sp->ix == 16 ? MAP_HEADER_HAMT_HEAD_ARRAY + : MAP_HEADER_HAMT_HEAD_BITMAP(sp->abm | sp->bbm)); + *hp++ = ctx->size; + } else { + nhp = HAllocX(p, HAMT_NODE_BITMAP_SZ(sp->ix), HALLOC_EXTRA); + hp = nhp; + *hp++ = MAP_HEADER_HAMT_NODE_BITMAP(sp->abm | sp->bbm); + } + sys_memcpy(hp, sp->array, sp->ix * sizeof(Eterm)); + res = make_boxed(nhp); + break; + default: + erts_exit(ERTS_ABORT_EXIT, "strange mix %d\r\n", sp->mix); + } } - if (c) - return res; + /* Done */ - /* need to make a new tuple, - * use old hp since it needs to be recreated anyway. - */ - tup = make_tuple(shp); - *shp++ = make_arityval(n+1); +#ifdef DEBUG + { + Eterm *head = hashmap_val(res); + Uint size = head[1]; + Uint real_size = hashmap_subtree_size(res); + ASSERT(size == real_size); + } +#endif - hp = HAlloc(p, 3 + n + 1); - res = make_map(hp); - *hp++ = MAP_HEADER; - *hp++ = n + 1; - *hp++ = tup; + if (ctx != &local_ctx) { + ASSERT(ctx->trap_bin != THE_NON_VALUE); + ASSERT(p->flags & F_DISABLE_GC); + erts_set_gc_state(p, 1); + } + else { + ASSERT(ctx->trap_bin == THE_NON_VALUE); + ASSERT(!(p->flags & F_DISABLE_GC)); + } + PSTACK_DESTROY(s); + UnUseTmpHeap(2,p); + BUMP_REDS(p, (initial_reds - reds) / MAP_MERGE_LOOP_FACTOR); + return res; - ks = map_get_keys(mp); - vs = map_get_values(mp); +trap: /* Yield */ - ASSERT(n >= 0); + if (ctx == &local_ctx) { + Binary* ctx_b = erts_create_magic_binary(sizeof(HashmapMergeContext), + hashmap_merge_ctx_destructor); + ctx = ERTS_MAGIC_BIN_DATA(ctx_b); + sys_memcpy(ctx, &local_ctx, sizeof(HashmapMergeContext)); + hp = HAlloc(p, PROC_BIN_SIZE); + ASSERT(ctx->trap_bin == THE_NON_VALUE); + ctx->trap_bin = erts_mk_magic_binary_term(&hp, &MSO(p), ctx_b); - /* copy map in order */ - while (n && ((c = CMP_TERM(*ks, key)) < 0)) { - *shp++ = *ks++; - *hp++ = *vs++; - n--; + erts_set_gc_state(p, 0); + } + else { + ASSERT(ctx->trap_bin != THE_NON_VALUE); + ASSERT(p->flags & F_DISABLE_GC); } - *shp++ = key; - *hp++ = value; + PSTACK_SAVE(s, &ctx->pstack); - ASSERT(n >= 0); + BUMP_ALL_REDS(p); + ERTS_BIF_PREP_TRAP1(trap_ret, &hashmap_merge_trap_export, + p, ctx->trap_bin); + UnUseTmpHeap(2,p); + return trap_ret; +} - while(n--) { - *shp++ = *ks++; - *hp++ = *vs++; +static Uint hashmap_subtree_size(Eterm node) { + DECLARE_WSTACK(stack); + Uint size = 0; + + hashmap_iterator_init(&stack, node, 0); + while (hashmap_iterator_next(&stack)) { + size++; } - /* we have one word remaining - * this will work out fine once we get the size word - * in the header. - */ - *shp = make_pos_bignum_header(0); - return res; + DESTROY_WSTACK(stack); + return size; +} + + +static int hash_cmp(Uint32 ha, Uint32 hb) +{ + int i; + for (i=0; i<8; i++) { + int cmp = (int)(ha & 0xF) - (int)(hb & 0xF); + if (cmp) + return cmp; + ha >>= 4; + hb >>= 4; + } + return 0; +} + +int hashmap_key_hash_cmp(Eterm* ap, Eterm* bp) +{ + unsigned int lvl = 0; + DeclareTmpHeapNoproc(th,2); + UseTmpHeapNoproc(2); + + if (ap && bp) { + ASSERT(CMP_TERM(CAR(ap), CAR(bp)) != 0); + for (;;) { + Uint32 ha = hashmap_restore_hash(th, lvl, CAR(ap)); + Uint32 hb = hashmap_restore_hash(th, lvl, CAR(bp)); + int cmp = hash_cmp(ha, hb); + if (cmp) { + UnUseTmpHeapNoproc(2); + return cmp; + } + lvl += 8; + } + } + UnUseTmpHeapNoproc(2); + return ap ? -1 : 1; } +/* maps:new/0 */ + +BIF_RETTYPE maps_new_0(BIF_ALIST_0) { + Eterm* hp; + Eterm tup; + flatmap_t *mp; + + hp = HAlloc(BIF_P, (MAP_HEADER_FLATMAP_SZ + 1)); + tup = make_tuple(hp); + *hp++ = make_arityval(0); + + mp = (flatmap_t*)hp; + mp->thing_word = MAP_HEADER_FLATMAP; + mp->size = 0; + mp->keys = tup; + + BIF_RET(make_flatmap(mp)); +} + +/* maps:put/3 */ + BIF_RETTYPE maps_put_3(BIF_ALIST_3) { if (is_map(BIF_ARG_3)) { BIF_RET(erts_maps_put(BIF_P, BIF_ARG_1, BIF_ARG_2, BIF_ARG_3)); } - BIF_ERROR(BIF_P, BADARG); + BIF_P->fvalue = BIF_ARG_3; + BIF_ERROR(BIF_P, BADMAP); } -/* maps:remove/3 - */ +/* maps:remove/3 */ int erts_maps_remove(Process *p, Eterm key, Eterm map, Eterm *res) { - Sint n; - Uint need; - Eterm *hp_start; - Eterm *thp, *mhp; - Eterm *ks, *vs, tup; - map_t *mp = (map_t*)map_val(map); + Uint32 hx; + if (is_flatmap(map)) { + Sint n; + Uint need; + Eterm *hp_start; + Eterm *thp, *mhp; + Eterm *ks, *vs, tup; + flatmap_t *mp = (flatmap_t*)flatmap_val(map); + + n = flatmap_get_size(mp); + + if (n == 0) { + *res = map; + return 1; + } - n = map_get_size(mp); + ks = flatmap_get_keys(mp); + vs = flatmap_get_values(mp); - if (n == 0) { - *res = map; - return 1; - } + /* Assume key exists. + * Release allocated if it didn't. + * Allocate key tuple first. + */ - ks = map_get_keys(mp); - vs = map_get_values(mp); + need = n + 1 - 1 + 3 + n - 1; /* tuple - 1 + map - 1 */ + hp_start = HAlloc(p, need); + thp = hp_start; + mhp = thp + n; /* offset with tuple heap size */ - /* Assume key exists. - * Release allocated if it didn't. - * Allocate key tuple first. - */ + tup = make_tuple(thp); + *thp++ = make_arityval(n - 1); - need = n + 1 - 1 + 3 + n - 1; /* tuple - 1 + map - 1 */ - hp_start = HAlloc(p, need); - thp = hp_start; - mhp = thp + n; /* offset with tuple heap size */ + *res = make_flatmap(mhp); + *mhp++ = MAP_HEADER_FLATMAP; + *mhp++ = n - 1; + *mhp++ = tup; - tup = make_tuple(thp); - *thp++ = make_arityval(n - 1); - - *res = make_map(mhp); - *mhp++ = MAP_HEADER; - *mhp++ = n - 1; - *mhp++ = tup; - - if (is_immed(key)) { - while (1) { - if (*ks == key) { - goto found_key; - } else if (--n) { - *mhp++ = *vs++; - *thp++ = *ks++; - } else - break; - } - } else { - while(1) { - if (EQ(*ks, key)) { - goto found_key; - } else if (--n) { - *mhp++ = *vs++; - *thp++ = *ks++; - } else - break; + if (is_immed(key)) { + while (1) { + if (*ks == key) { + goto found_key; + } else if (--n) { + *mhp++ = *vs++; + *thp++ = *ks++; + } else + break; + } + } else { + while(1) { + if (EQ(*ks, key)) { + goto found_key; + } else if (--n) { + *mhp++ = *vs++; + *thp++ = *ks++; + } else + break; + } } - } - /* Not found, remove allocated memory - * and return previous map. - */ - HRelease(p, hp_start + need, hp_start); + /* Not found, remove allocated memory + * and return previous map. + */ + HRelease(p, hp_start + need, hp_start); - *res = map; - return 1; + *res = map; + return 1; found_key: - /* Copy rest of keys and values */ - if (--n) { - sys_memcpy(mhp, vs+1, n*sizeof(Eterm)); - sys_memcpy(thp, ks+1, n*sizeof(Eterm)); + /* Copy rest of keys and values */ + if (--n) { + sys_memcpy(mhp, vs+1, n*sizeof(Eterm)); + sys_memcpy(thp, ks+1, n*sizeof(Eterm)); + } + return 1; } + ASSERT(is_hashmap(map)); + hx = hashmap_make_hash(key); + *res = hashmap_delete(p, hx, key, map); return 1; } @@ -692,32 +1612,32 @@ BIF_RETTYPE maps_remove_2(BIF_ALIST_2) { BIF_RET(res); } } - BIF_ERROR(BIF_P, BADARG); + BIF_P->fvalue = BIF_ARG_2; + BIF_ERROR(BIF_P, BADMAP); } -/* maps:update/3 - */ - int erts_maps_update(Process *p, Eterm key, Eterm value, Eterm map, Eterm *res) { + Uint32 hx; + if (is_flatmap(map)) { Sint n,i; Eterm* hp,*shp; Eterm *ks,*vs; - map_t *mp = (map_t*)map_val(map); + flatmap_t *mp = (flatmap_t*)flatmap_val(map); - if ((n = map_get_size(mp)) == 0) { + if ((n = flatmap_get_size(mp)) == 0) { return 0; } - ks = map_get_keys(mp); - vs = map_get_values(mp); + ks = flatmap_get_keys(mp); + vs = flatmap_get_values(mp); /* only allocate for values, * assume key-tuple will be intact */ - hp = HAlloc(p, MAP_HEADER_SIZE + n); + hp = HAlloc(p, MAP_HEADER_FLATMAP_SZ + n); shp = hp; - *hp++ = MAP_HEADER; + *hp++ = MAP_HEADER_FLATMAP; *hp++ = n; *hp++ = mp->keys; @@ -739,7 +1659,7 @@ int erts_maps_update(Process *p, Eterm key, Eterm value, Eterm map, Eterm *res) } } - HRelease(p, shp + MAP_HEADER_SIZE + n, shp); + HRelease(p, shp + MAP_HEADER_FLATMAP_SZ + n, shp); return 0; found_key: @@ -747,56 +1667,939 @@ found_key: vs++; if (++i < n) sys_memcpy(hp, vs, (n - i)*sizeof(Eterm)); - *res = make_map(shp); + *res = make_flatmap(shp); return 1; + } + + ASSERT(is_hashmap(map)); + hx = hashmap_make_hash(key); + *res = erts_hashmap_insert(p, hx, key, value, map, 1); + if (is_value(*res)) + return 1; + + return 0; +} + +Eterm erts_maps_put(Process *p, Eterm key, Eterm value, Eterm map) { + Uint32 hx; + Eterm res; + if (is_flatmap(map)) { + Sint n,i; + Sint c = 0; + Eterm* hp, *shp; + Eterm *ks, *vs, tup; + flatmap_t *mp = (flatmap_t*)flatmap_val(map); + + n = flatmap_get_size(mp); + + if (n == 0) { + hp = HAlloc(p, MAP_HEADER_FLATMAP_SZ + 1 + 2); + tup = make_tuple(hp); + *hp++ = make_arityval(1); + *hp++ = key; + res = make_flatmap(hp); + *hp++ = MAP_HEADER_FLATMAP; + *hp++ = 1; + *hp++ = tup; + *hp++ = value; + + return res; + } + + ks = flatmap_get_keys(mp); + vs = flatmap_get_values(mp); + + /* only allocate for values, + * assume key-tuple will be intact + */ + + hp = HAlloc(p, MAP_HEADER_FLATMAP_SZ + n); + shp = hp; /* save hp, used if optimistic update fails */ + res = make_flatmap(hp); + *hp++ = MAP_HEADER_FLATMAP; + *hp++ = n; + *hp++ = mp->keys; + + if (is_immed(key)) { + for( i = 0; i < n; i ++) { + if (ks[i] == key) { + *hp++ = value; + vs++; + c = 1; + } else { + *hp++ = *vs++; + } + } + } else { + for( i = 0; i < n; i ++) { + if (EQ(ks[i], key)) { + *hp++ = value; + vs++; + c = 1; + } else { + *hp++ = *vs++; + } + } + } + + if (c) + return res; + + /* the map will grow */ + + if (n >= MAP_SMALL_MAP_LIMIT) { + HRelease(p, shp + MAP_HEADER_FLATMAP_SZ + n, shp); + ks = flatmap_get_keys(mp); + vs = flatmap_get_values(mp); + + res = erts_hashmap_from_ks_and_vs_extra(p,ks,vs,n,key,value); + + return res; + } + + /* still a small map. need to make a new tuple, + * use old hp since it needs to be recreated anyway. */ + + tup = make_tuple(shp); + *shp++ = make_arityval(n+1); + + hp = HAlloc(p, 3 + n + 1); + res = make_flatmap(hp); + *hp++ = MAP_HEADER_FLATMAP; + *hp++ = n + 1; + *hp++ = tup; + + ks = flatmap_get_keys(mp); + vs = flatmap_get_values(mp); + + ASSERT(n >= 0); + + /* copy map in order */ + while (n && ((c = CMP_TERM(*ks, key)) < 0)) { + *shp++ = *ks++; + *hp++ = *vs++; + n--; + } + + *shp++ = key; + *hp++ = value; + + ASSERT(n >= 0); + + while(n--) { + *shp++ = *ks++; + *hp++ = *vs++; + } + /* we have one word remaining + * this will work out fine once we get the size word + * in the header. + */ + *shp = make_pos_bignum_header(0); + return res; + } + ASSERT(is_hashmap(map)); + + hx = hashmap_make_hash(key); + res = erts_hashmap_insert(p, hx, key, value, map, 0); + ASSERT(is_hashmap(res)); + + return res; } +/* maps:update/3 */ + BIF_RETTYPE maps_update_3(BIF_ALIST_3) { - if (is_map(BIF_ARG_3)) { + if (is_not_map(BIF_ARG_3)) { + BIF_P->fvalue = BIF_ARG_3; + BIF_ERROR(BIF_P, BADMAP); + } else { Eterm res; if (erts_maps_update(BIF_P, BIF_ARG_1, BIF_ARG_2, BIF_ARG_3, &res)) { BIF_RET(res); } + BIF_P->fvalue = BIF_ARG_1; + BIF_ERROR(BIF_P, BADKEY); } - BIF_ERROR(BIF_P, BADARG); } -/* maps:values/1 - */ +/* maps:values/1 */ BIF_RETTYPE maps_values_1(BIF_ALIST_1) { - if (is_map(BIF_ARG_1)) { + if (is_flatmap(BIF_ARG_1)) { Eterm *hp, *vs, res = NIL; - map_t *mp; + flatmap_t *mp; Uint n; - mp = (map_t*)map_val(BIF_ARG_1); - n = map_get_size(mp); + mp = (flatmap_t*)flatmap_val(BIF_ARG_1); + n = flatmap_get_size(mp); if (n == 0) BIF_RET(res); hp = HAlloc(BIF_P, (2 * n)); - vs = map_get_values(mp); + vs = flatmap_get_values(mp); while(n--) { res = CONS(hp, vs[n], res); hp += 2; } BIF_RET(res); + } else if (is_hashmap(BIF_ARG_1)) { + BIF_RET(hashmap_values(BIF_P, BIF_ARG_1)); } - BIF_ERROR(BIF_P, BADARG); + BIF_P->fvalue = BIF_ARG_1; + BIF_ERROR(BIF_P, BADMAP); +} + +static Eterm hashmap_to_list(Process *p, Eterm node) { + DECLARE_WSTACK(stack); + Eterm *hp, *kv; + Eterm res = NIL; + + hp = HAlloc(p, hashmap_size(node) * (2 + 3)); + hashmap_iterator_init(&stack, node, 0); + while ((kv=hashmap_iterator_next(&stack)) != NULL) { + Eterm tup = TUPLE2(hp, CAR(kv), CDR(kv)); + hp += 3; + res = CONS(hp, tup, res); + hp += 2; + } + DESTROY_WSTACK(stack); + return res; +} + +void hashmap_iterator_init(ErtsWStack* s, Eterm node, int reverse) { + Eterm hdr = *hashmap_val(node); + Uint sz; + + switch(hdr & _HEADER_MAP_SUBTAG_MASK) { + case HAMT_SUBTAG_HEAD_ARRAY: + sz = 16; + break; + case HAMT_SUBTAG_HEAD_BITMAP: + case HAMT_SUBTAG_NODE_BITMAP: + sz = hashmap_bitcount(MAP_HEADER_VAL(hdr)); + break; + default: + erts_exit(ERTS_ABORT_EXIT, "bad header"); + } + + WSTACK_PUSH3((*s), (UWord)THE_NON_VALUE, /* end marker */ + (UWord)(!reverse ? 0 : sz+1), + (UWord)node); +} + +Eterm* hashmap_iterator_next(ErtsWStack* s) { + Eterm node, *ptr, hdr; + Uint32 sz; + Uint idx; + + for (;;) { + ASSERT(!WSTACK_ISEMPTY((*s))); + node = (Eterm) WSTACK_POP((*s)); + if (is_non_value(node)) { + return NULL; + } + idx = (Uint) WSTACK_POP((*s)); + for (;;) { + ASSERT(is_boxed(node)); + ptr = boxed_val(node); + hdr = *ptr; + ASSERT(is_header(hdr)); + switch(hdr & _HEADER_MAP_SUBTAG_MASK) { + case HAMT_SUBTAG_HEAD_ARRAY: + ptr++; + sz = 16; + break; + case HAMT_SUBTAG_HEAD_BITMAP: + ptr++; + case HAMT_SUBTAG_NODE_BITMAP: + sz = hashmap_bitcount(MAP_HEADER_VAL(hdr)); + ASSERT(sz < 17); + break; + default: + erts_exit(ERTS_ABORT_EXIT, "bad header"); + } + + idx++; + + if (idx <= sz) { + WSTACK_PUSH2((*s), (UWord)idx, (UWord)node); + + if (is_list(ptr[idx])) { + return list_val(ptr[idx]); + } + ASSERT(is_boxed(ptr[idx])); + node = ptr[idx]; + idx = 0; + } + else + break; /* and pop parent node */ + } + } +} + +Eterm* hashmap_iterator_prev(ErtsWStack* s) { + Eterm node, *ptr, hdr; + Uint32 sz; + Uint idx; + + for (;;) { + ASSERT(!WSTACK_ISEMPTY((*s))); + node = (Eterm) WSTACK_POP((*s)); + if (is_non_value(node)) { + return NULL; + } + idx = (Uint) WSTACK_POP((*s)); + for (;;) { + ASSERT(is_boxed(node)); + ptr = boxed_val(node); + hdr = *ptr; + ASSERT(is_header(hdr)); + switch(hdr & _HEADER_MAP_SUBTAG_MASK) { + case HAMT_SUBTAG_HEAD_ARRAY: + ptr++; + sz = 16; + break; + case HAMT_SUBTAG_HEAD_BITMAP: + ptr++; + case HAMT_SUBTAG_NODE_BITMAP: + sz = hashmap_bitcount(MAP_HEADER_VAL(hdr)); + ASSERT(sz < 17); + break; + default: + erts_exit(ERTS_ERROR_EXIT, "bad header"); + } + + if (idx > sz) + idx = sz; + else + idx--; + + if (idx >= 1) { + WSTACK_PUSH2((*s), (UWord)idx, (UWord)node); + + if (is_list(ptr[idx])) { + return list_val(ptr[idx]); + } + ASSERT(is_boxed(ptr[idx])); + node = ptr[idx]; + idx = 17; + } + else + break; /* and pop parent node */ + } + } +} + +const Eterm * +erts_hashmap_get(Uint32 hx, Eterm key, Eterm node) +{ + Eterm *ptr, hdr, *res; + Uint ix, lvl = 0; + Uint32 hval,bp; + DeclareTmpHeapNoproc(th,2); + UseTmpHeapNoproc(2); + + ASSERT(is_boxed(node)); + ptr = boxed_val(node); + hdr = *ptr; + ASSERT(is_header(hdr)); + ASSERT(is_hashmap_header_head(hdr)); + ptr++; + + for (;;) { + hval = MAP_HEADER_VAL(hdr); + ix = hashmap_index(hx); + if (hval != 0xffff) { + bp = 1 << ix; + if (!(bp & hval)) { + /* not occupied */ + res = NULL; + break; + } + ix = hashmap_bitcount(hval & (bp - 1)); + } + node = ptr[ix+1]; + + if (is_list(node)) { /* LEAF NODE [K|V] */ + ptr = list_val(node); + res = EQ(CAR(ptr), key) ? &(CDR(ptr)) : NULL; + break; + } + + hx = hashmap_shift_hash(th,hx,lvl,key); + + ASSERT(is_boxed(node)); + ptr = boxed_val(node); + hdr = *ptr; + ASSERT(is_header(hdr)); + ASSERT(!is_hashmap_header_head(hdr)); + } + + UnUseTmpHeapNoproc(2); + return res; +} + +Eterm erts_hashmap_insert(Process *p, Uint32 hx, Eterm key, Eterm value, + Eterm map, int is_update) { + Uint size, upsz; + Eterm *hp, res = THE_NON_VALUE; + DECLARE_ESTACK(stack); + if (erts_hashmap_insert_down(hx, key, map, &size, &upsz, &stack, is_update)) { + hp = HAlloc(p, size); + res = erts_hashmap_insert_up(hp, key, value, &upsz, &stack); + } + + DESTROY_ESTACK(stack); + ERTS_VERIFY_UNUSED_TEMP_ALLOC(p); + ERTS_HOLE_CHECK(p); + + return res; +} + + +int erts_hashmap_insert_down(Uint32 hx, Eterm key, Eterm node, Uint *sz, + Uint *update_size, ErtsEStack *sp, int is_update) { + Eterm *ptr; + Eterm hdr, ckey; + Uint32 ix, cix, bp, hval, chx; + Uint slot, lvl = 0, clvl; + Uint size = 0, n = 0; + DeclareTmpHeapNoproc(th,2); + + *update_size = 1; + + UseTmpHeapNoproc(2); + for (;;) { + switch(primary_tag(node)) { + case TAG_PRIMARY_LIST: /* LEAF NODE [K|V] */ + ptr = list_val(node); + ckey = CAR(ptr); + if (EQ(ckey, key)) { + *update_size = 0; + goto unroll; + } + if (is_update) { + UnUseTmpHeapNoproc(2); + return 0; + } + goto insert_subnodes; + case TAG_PRIMARY_BOXED: + ptr = boxed_val(node); + hdr = *ptr; + ASSERT(is_header(hdr)); + + switch(hdr & _HEADER_MAP_SUBTAG_MASK) { + case HAMT_SUBTAG_HEAD_ARRAY: + ix = hashmap_index(hx); + hx = hashmap_shift_hash(th,hx,lvl,key); + size += HAMT_HEAD_ARRAY_SZ; + ESTACK_PUSH2(*sp, ix, node); + node = ptr[ix+2]; + break; + case HAMT_SUBTAG_NODE_BITMAP: + hval = MAP_HEADER_VAL(hdr); + ix = hashmap_index(hx); + bp = 1 << ix; + if (hval == 0xffff) { + slot = ix; + n = 16; + } else { + slot = hashmap_bitcount(hval & (bp - 1)); + n = hashmap_bitcount(hval); + } + + ESTACK_PUSH4(*sp, n, bp, slot, node); + + if (!(bp & hval)) { /* not occupied */ + if (is_update) { + UnUseTmpHeapNoproc(2); + return 0; + } + size += HAMT_NODE_BITMAP_SZ(n+1); + goto unroll; + } + + hx = hashmap_shift_hash(th,hx,lvl,key); + node = ptr[slot+1]; + ASSERT(HAMT_NODE_BITMAP_SZ(n) <= 17); + size += HAMT_NODE_BITMAP_SZ(n); + break; + + case HAMT_SUBTAG_HEAD_BITMAP: + hval = MAP_HEADER_VAL(hdr); + ix = hashmap_index(hx); + bp = 1 << ix; + slot = hashmap_bitcount(hval & (bp - 1)); + n = hashmap_bitcount(hval); + + ESTACK_PUSH4(*sp, n, bp, slot, node); + + /* occupied */ + if (bp & hval) { + hx = hashmap_shift_hash(th,hx,lvl,key); + node = ptr[slot+2]; + ASSERT(HAMT_HEAD_BITMAP_SZ(n) <= 18); + size += HAMT_HEAD_BITMAP_SZ(n); + break; + } + /* not occupied */ + if (is_update) { + UnUseTmpHeapNoproc(2); + return 0; + } + size += HAMT_HEAD_BITMAP_SZ(n+1); + goto unroll; + default: + erts_exit(ERTS_ERROR_EXIT, "bad header tag %ld\r\n", hdr & _HEADER_MAP_SUBTAG_MASK); + break; + } + break; + default: + erts_exit(ERTS_ERROR_EXIT, "bad primary tag %p\r\n", node); + break; + } + } +insert_subnodes: + clvl = lvl; + chx = hashmap_restore_hash(th,clvl,ckey); + size += HAMT_NODE_BITMAP_SZ(2); + ix = hashmap_index(hx); + cix = hashmap_index(chx); + + while (cix == ix) { + ESTACK_PUSH4(*sp, 0, 1 << ix, 0, MAP_HEADER_HAMT_NODE_BITMAP(0)); + size += HAMT_NODE_BITMAP_SZ(1); + hx = hashmap_shift_hash(th,hx,lvl,key); + chx = hashmap_shift_hash(th,chx,clvl,ckey); + ix = hashmap_index(hx); + cix = hashmap_index(chx); + } + ESTACK_PUSH3(*sp, cix, ix, node); + +unroll: + *sz = size + /* res cons */ 2; + UnUseTmpHeapNoproc(2); + return 1; +} + +Eterm erts_hashmap_insert_up(Eterm *hp, Eterm key, Eterm value, + Uint *update_size, ErtsEStack *sp) { + Eterm node, *ptr, hdr; + Eterm res; + Eterm *nhp = NULL; + Uint32 ix, cix, bp, hval; + Uint slot, n; + /* Needed for halfword */ + DeclareTmpHeapNoproc(fake,1); + UseTmpHeapNoproc(1); + + res = CONS(hp, key, value); hp += 2; + + do { + node = ESTACK_POP(*sp); + switch(primary_tag(node)) { + case TAG_PRIMARY_LIST: + ix = (Uint32) ESTACK_POP(*sp); + cix = (Uint32) ESTACK_POP(*sp); + + nhp = hp; + *hp++ = MAP_HEADER_HAMT_NODE_BITMAP((1 << ix) | (1 << cix)); + if (ix < cix) { + *hp++ = res; + *hp++ = node; + } else { + *hp++ = node; + *hp++ = res; + } + res = make_hashmap(nhp); + break; + case TAG_PRIMARY_HEADER: + /* subnodes, fake it */ + *fake = node; + node = make_boxed(fake); + case TAG_PRIMARY_BOXED: + ptr = boxed_val(node); + hdr = *ptr; + ASSERT(is_header(hdr)); + + switch(hdr & _HEADER_MAP_SUBTAG_MASK) { + case HAMT_SUBTAG_HEAD_ARRAY: + slot = (Uint) ESTACK_POP(*sp); + nhp = hp; + n = HAMT_HEAD_ARRAY_SZ - 2; + *hp++ = MAP_HEADER_HAMT_HEAD_ARRAY; ptr++; + *hp++ = (*ptr++) + *update_size; + while(n--) { *hp++ = *ptr++; } + nhp[slot+2] = res; + res = make_hashmap(nhp); + break; + case HAMT_SUBTAG_NODE_BITMAP: + slot = (Uint) ESTACK_POP(*sp); + bp = (Uint32) ESTACK_POP(*sp); + n = (Uint32) ESTACK_POP(*sp); + hval = MAP_HEADER_VAL(hdr); + nhp = hp; + *hp++ = MAP_HEADER_HAMT_NODE_BITMAP(hval | bp); ptr++; + + n -= slot; + while(slot--) { *hp++ = *ptr++; } + *hp++ = res; + if (hval & bp) { ptr++; n--; } + while(n--) { *hp++ = *ptr++; } + + res = make_hashmap(nhp); + break; + case HAMT_SUBTAG_HEAD_BITMAP: + slot = (Uint) ESTACK_POP(*sp); + bp = (Uint32) ESTACK_POP(*sp); + n = (Uint32) ESTACK_POP(*sp); + hval = MAP_HEADER_VAL(hdr); + nhp = hp; + *hp++ = MAP_HEADER_HAMT_HEAD_BITMAP(hval | bp); ptr++; + *hp++ = (*ptr++) + *update_size; + + n -= slot; + while(slot--) { *hp++ = *ptr++; } + *hp++ = res; + if (hval & bp) { ptr++; n--; } + while(n--) { *hp++ = *ptr++; } + + if ((hval | bp) == 0xffff) { + *nhp = MAP_HEADER_HAMT_HEAD_ARRAY; + } + res = make_hashmap(nhp); + break; + default: + erts_exit(ERTS_ERROR_EXIT, "bad header tag %x\r\n", hdr & _HEADER_MAP_SUBTAG_MASK); + break; + } + break; + default: + erts_exit(ERTS_ERROR_EXIT, "bad primary tag %x\r\n", primary_tag(node)); + break; + } + + } while(!ESTACK_ISEMPTY(*sp)); + + UnUseTmpHeapNoproc(1); + return res; +} + +static Eterm hashmap_keys(Process* p, Eterm node) { + DECLARE_WSTACK(stack); + hashmap_head_t* root; + Eterm *hp, *kv; + Eterm res = NIL; + + root = (hashmap_head_t*) boxed_val(node); + hp = HAlloc(p, root->size * 2); + hashmap_iterator_init(&stack, node, 0); + while ((kv=hashmap_iterator_next(&stack)) != NULL) { + res = CONS(hp, CAR(kv), res); + hp += 2; + } + DESTROY_WSTACK(stack); + return res; +} + +static Eterm hashmap_values(Process* p, Eterm node) { + DECLARE_WSTACK(stack); + hashmap_head_t* root; + Eterm *hp, *kv; + Eterm res = NIL; + + root = (hashmap_head_t*) boxed_val(node); + hp = HAlloc(p, root->size * 2); + hashmap_iterator_init(&stack, node, 0); + while ((kv=hashmap_iterator_next(&stack)) != NULL) { + res = CONS(hp, CDR(kv), res); + hp += 2; + } + DESTROY_WSTACK(stack); + return res; +} + +static Eterm hashmap_delete(Process *p, Uint32 hx, Eterm key, Eterm map) { + Eterm *hp = NULL, *nhp = NULL, *hp_end = NULL; + Eterm *ptr; + Eterm hdr, res = map, node = map; + Uint32 ix, bp, hval; + Uint slot, lvl = 0; + Uint size = 0, n = 0; + DECLARE_ESTACK(stack); + DeclareTmpHeapNoproc(th,2); + UseTmpHeapNoproc(2); + + for (;;) { + switch(primary_tag(node)) { + case TAG_PRIMARY_LIST: + if (EQ(CAR(list_val(node)), key)) { + goto unroll; + } + goto not_found; + case TAG_PRIMARY_BOXED: + ptr = boxed_val(node); + hdr = *ptr; + ASSERT(is_header(hdr)); + + switch(hdr & _HEADER_MAP_SUBTAG_MASK) { + case HAMT_SUBTAG_HEAD_ARRAY: + ix = hashmap_index(hx); + hx = hashmap_shift_hash(th,hx,lvl,key); + size += HAMT_HEAD_ARRAY_SZ; + ESTACK_PUSH2(stack, ix, node); + node = ptr[ix+2]; + break; + case HAMT_SUBTAG_NODE_BITMAP: + hval = MAP_HEADER_VAL(hdr); + ix = hashmap_index(hx); + bp = 1 << ix; + if (hval == 0xffff) { + slot = ix; + n = 16; + } else if (bp & hval) { + slot = hashmap_bitcount(hval & (bp - 1)); + n = hashmap_bitcount(hval); + } else { + /* not occupied */ + goto not_found; + } + + ESTACK_PUSH4(stack, n, bp, slot, node); + + hx = hashmap_shift_hash(th,hx,lvl,key); + node = ptr[slot+1]; + ASSERT(HAMT_NODE_BITMAP_SZ(n) <= 17); + size += HAMT_NODE_BITMAP_SZ(n); + break; + + case HAMT_SUBTAG_HEAD_BITMAP: + hval = MAP_HEADER_VAL(hdr); + ix = hashmap_index(hx); + bp = 1 << ix; + slot = hashmap_bitcount(hval & (bp - 1)); + n = hashmap_bitcount(hval); + + ESTACK_PUSH4(stack, n, bp, slot, node); + + /* occupied */ + if (bp & hval) { + hx = hashmap_shift_hash(th,hx,lvl,key); + node = ptr[slot+2]; + ASSERT(HAMT_HEAD_BITMAP_SZ(n) <= 18); + size += HAMT_HEAD_BITMAP_SZ(n); + break; + } + /* not occupied */ + goto not_found; + default: + erts_exit(ERTS_ERROR_EXIT, "bad header tag %ld\r\n", hdr & _HEADER_MAP_SUBTAG_MASK); + break; + } + break; + default: + erts_exit(ERTS_ERROR_EXIT, "bad primary tag %p\r\n", node); + break; + } + } + +unroll: + /* the size is bounded and atleast one less than the previous size */ + size -= 1; + n = hashmap_size(map) - 1; + + if (n <= MAP_SMALL_MAP_LIMIT) { + DECLARE_WSTACK(wstack); + Eterm *kv, *ks, *vs; + flatmap_t *mp; + Eterm keys; + + DESTROY_ESTACK(stack); + + /* build flat structure */ + hp = HAlloc(p, 3 + 1 + (2 * n)); + keys = make_tuple(hp); + *hp++ = make_arityval(n); + ks = hp; + hp += n; + mp = (flatmap_t*)hp; + hp += MAP_HEADER_FLATMAP_SZ; + vs = hp; + + mp->thing_word = MAP_HEADER_FLATMAP; + mp->size = n; + mp->keys = keys; + + hashmap_iterator_init(&wstack, map, 0); + + while ((kv=hashmap_iterator_next(&wstack)) != NULL) { + if (EQ(CAR(kv),key)) + continue; + *ks++ = CAR(kv); + *vs++ = CDR(kv); + } + + /* it cannot have multiple keys */ + erts_validate_and_sort_flatmap(mp); + + DESTROY_WSTACK(wstack); + UnUseTmpHeapNoproc(2); + return make_flatmap(mp); + } + + hp = HAlloc(p, size); + hp_end = hp + size; + res = THE_NON_VALUE; + + do { + node = ESTACK_POP(stack); + + /* all nodes are things */ + ptr = boxed_val(node); + hdr = *ptr; + ASSERT(is_header(hdr)); + + switch(hdr & _HEADER_MAP_SUBTAG_MASK) { + case HAMT_SUBTAG_HEAD_ARRAY: + ix = (Uint) ESTACK_POP(stack); + nhp = hp; + if (res == THE_NON_VALUE) { + n = 16; + n -= ix; + *hp++ = MAP_HEADER_HAMT_HEAD_BITMAP(0xffff ^ (1 << ix)); ptr++; + *hp++ = (*ptr++) - 1; + while(ix--) { *hp++ = *ptr++; } + ptr++; n--; + while(n--) { *hp++ = *ptr++; } + res = make_hashmap(nhp); + } else { + n = 16; + *hp++ = MAP_HEADER_HAMT_HEAD_ARRAY; ptr++; + *hp++ = (*ptr++) - 1; + while(n--) { *hp++ = *ptr++; } + nhp[ix+2] = res; + res = make_hashmap(nhp); + } + break; + case HAMT_SUBTAG_NODE_BITMAP: + slot = (Uint) ESTACK_POP(stack); + bp = (Uint32) ESTACK_POP(stack); + n = (Uint32) ESTACK_POP(stack); + nhp = hp; + + /* bitmap change matrix + * res | none leaf bitmap + * ---------------------------- + * n=1 | remove remove keep + * n=2 | other keep keep + * n>2 | shrink keep keep + * + * other: (remember, n is 2) + * shrink if the other bitmap value is a bitmap node + * remove if the other bitmap value is a leaf + * + * remove: + * this bitmap node is removed, res is moved up in tree (could be none) + * this is a special case of shrink + * + * keep: + * the current path index is still used down in the tree, need to keep it + * copy as usual with the updated res + * + * shrink: + * the current path index is no longer used down in the tree, remove it (shrink) + */ + if (res == THE_NON_VALUE) { + if (n == 1) { + break; + } else if (n == 2) { + if (slot == 0) { + ix = 2; /* off by one 'cause hdr */ + } else { + ix = 1; /* off by one 'cause hdr */ + } + if (primary_tag(ptr[ix]) == TAG_PRIMARY_LIST) { + res = ptr[ix]; + } else { + hval = MAP_HEADER_VAL(hdr); + *hp++ = MAP_HEADER_HAMT_NODE_BITMAP(hval ^ bp); + *hp++ = ptr[ix]; + res = make_hashmap(nhp); + } + } else { + /* n > 2 */ + hval = MAP_HEADER_VAL(hdr); + *hp++ = MAP_HEADER_HAMT_NODE_BITMAP(hval ^ bp); ptr++; + n -= slot; + while(slot--) { *hp++ = *ptr++; } + ptr++; n--; + while(n--) { *hp++ = *ptr++; } + res = make_hashmap(nhp); + } + } else if (primary_tag(res) == TAG_PRIMARY_LIST && n == 1) { + break; + } else { + /* res is bitmap or leaf && n > 1, keep */ + n -= slot; + *hp++ = *ptr++; + while(slot--) { *hp++ = *ptr++; } + *hp++ = res; + ptr++; n--; + while(n--) { *hp++ = *ptr++; } + res = make_hashmap(nhp); + } + break; + case HAMT_SUBTAG_HEAD_BITMAP: + slot = (Uint) ESTACK_POP(stack); + bp = (Uint32) ESTACK_POP(stack); + n = (Uint32) ESTACK_POP(stack); + nhp = hp; + + if (res != THE_NON_VALUE) { + *hp++ = *ptr++; + *hp++ = (*ptr++) - 1; + n -= slot; + while(slot--) { *hp++ = *ptr++; } + *hp++ = res; + ptr++; n--; + while(n--) { *hp++ = *ptr++; } + } else { + hval = MAP_HEADER_VAL(hdr); + *hp++ = MAP_HEADER_HAMT_HEAD_BITMAP(hval ^ bp); ptr++; + *hp++ = (*ptr++) - 1; + n -= slot; + while(slot--) { *hp++ = *ptr++; } + ptr++; n--; + while(n--) { *hp++ = *ptr++; } + } + res = make_hashmap(nhp); + break; + default: + erts_exit(ERTS_ERROR_EXIT, "bad header tag %x\r\n", hdr & _HEADER_MAP_SUBTAG_MASK); + break; + } + } while(!ESTACK_ISEMPTY(stack)); + HRelease(p, hp_end, hp); +not_found: + DESTROY_ESTACK(stack); + ERTS_VERIFY_UNUSED_TEMP_ALLOC(p); + ERTS_HOLE_CHECK(p); + UnUseTmpHeapNoproc(2); + return res; } -int erts_validate_and_sort_map(map_t* mp) + +int erts_validate_and_sort_flatmap(flatmap_t* mp) { - Eterm *ks = map_get_keys(mp); - Eterm *vs = map_get_values(mp); - Uint sz = map_get_size(mp); + Eterm *ks = flatmap_get_keys(mp); + Eterm *vs = flatmap_get_values(mp); + Uint sz = flatmap_get_size(mp); Uint ix,jx; Eterm tmp; - int c; + Sint c; /* sort */ @@ -819,3 +2622,345 @@ int erts_validate_and_sort_map(map_t* mp) } return 1; } + +#if 0 /* Can't get myself to remove this beautiful piece of code + for probabilistic overestimation of nr of nodes in a hashmap */ + +/* Really rough estimate of sqrt(x) + * Guaranteed not to be less than sqrt(x) + */ +static int int_sqrt_ceiling(Uint x) +{ + int n; + + if (x <= 2) + return x; + + n = erts_fit_in_bits_uint(x-1); + if (n & 1) { + /* Calc: sqrt(2^n) = 2^(n/2) * sqrt(2) ~= 2^(n/2) * 3 / 2 */ + return (1 << (n/2 - 1)) * 3; + } + else { + /* Calc: sqrt(2^n) = 2^(n/2) */ + return 1 << (n / 2); + } +} + +/* May not be enough if hashing is broken (not uniform) + * or if hell freezes over. + */ +Uint hashmap_overestimated_node_count(Uint k) +{ + /* k is nr of key-value pairs. + N(k) is expected nr of nodes in hamt. + + Observation: + For uniformly distributed hash values, average of N varies between + 0.3*k and 0.4*k (with a beautiful sine curve) + and standard deviation of N is about sqrt(k)/3. + + Assuming normal probability distribution, we overestimate nr of nodes + by 15 std.devs above the average, which gives a probability for overrun + less than 1.0e-49 (same magnitude as a git SHA1 collision). + */ + return 2*k/5 + 1 + (15/3)*int_sqrt_ceiling(k); +} +#endif + +BIF_RETTYPE erts_debug_map_info_1(BIF_ALIST_1) { + if (is_hashmap(BIF_ARG_1)) { + BIF_RET(hashmap_info(BIF_P,BIF_ARG_1)); + } else if (is_flatmap(BIF_ARG_1)) { + BIF_ERROR(BIF_P, BADARG); + } else { + BIF_P->fvalue = BIF_ARG_1; + BIF_ERROR(BIF_P, BADMAP); + } +} + +/* + * erts_internal:map_to_tuple_keys/1 + * + * Used in erts_debug:size/1 + */ + +BIF_RETTYPE erts_internal_map_to_tuple_keys_1(BIF_ALIST_1) { + if (is_flatmap(BIF_ARG_1)) { + flatmap_t *mp = (flatmap_t*)flatmap_val(BIF_ARG_1); + BIF_RET(mp->keys); + } else if (is_hashmap(BIF_ARG_1)) { + BIF_ERROR(BIF_P, BADARG); + } else { + BIF_P->fvalue = BIF_ARG_1; + BIF_ERROR(BIF_P, BADMAP); + } +} + +/* + * erts_internal:term_type/1 + * + * Used in erts_debug:size/1 + */ + +BIF_RETTYPE erts_internal_term_type_1(BIF_ALIST_1) { + Eterm obj = BIF_ARG_1; + switch (primary_tag(obj)) { + case TAG_PRIMARY_LIST: + BIF_RET(ERTS_MAKE_AM("list")); + case TAG_PRIMARY_BOXED: { + Eterm hdr = *boxed_val(obj); + ASSERT(is_header(hdr)); + switch (hdr & _TAG_HEADER_MASK) { + case ARITYVAL_SUBTAG: + BIF_RET(ERTS_MAKE_AM("tuple")); + case EXPORT_SUBTAG: + BIF_RET(ERTS_MAKE_AM("export")); + case FUN_SUBTAG: + BIF_RET(ERTS_MAKE_AM("fun")); + case MAP_SUBTAG: + switch (MAP_HEADER_TYPE(hdr)) { + case MAP_HEADER_TAG_FLATMAP_HEAD : + BIF_RET(ERTS_MAKE_AM("flatmap")); + case MAP_HEADER_TAG_HAMT_HEAD_BITMAP : + case MAP_HEADER_TAG_HAMT_HEAD_ARRAY : + BIF_RET(ERTS_MAKE_AM("hashmap")); + case MAP_HEADER_TAG_HAMT_NODE_BITMAP : + BIF_RET(ERTS_MAKE_AM("hashmap_node")); + default: + erts_exit(ERTS_ABORT_EXIT, "term_type: bad map header type %d\n", MAP_HEADER_TYPE(hdr)); + } + case REFC_BINARY_SUBTAG: + BIF_RET(ERTS_MAKE_AM("refc_binary")); + case HEAP_BINARY_SUBTAG: + BIF_RET(ERTS_MAKE_AM("heap_binary")); + case SUB_BINARY_SUBTAG: + BIF_RET(ERTS_MAKE_AM("sub_binary")); + case BIN_MATCHSTATE_SUBTAG: + BIF_RET(ERTS_MAKE_AM("matchstate")); + case POS_BIG_SUBTAG: + case NEG_BIG_SUBTAG: + BIF_RET(ERTS_MAKE_AM("bignum")); + case REF_SUBTAG: + BIF_RET(ERTS_MAKE_AM("reference")); + case EXTERNAL_REF_SUBTAG: + BIF_RET(ERTS_MAKE_AM("external_reference")); + case EXTERNAL_PID_SUBTAG: + BIF_RET(ERTS_MAKE_AM("external_pid")); + case EXTERNAL_PORT_SUBTAG: + BIF_RET(ERTS_MAKE_AM("external_port")); + case FLOAT_SUBTAG: + BIF_RET(ERTS_MAKE_AM("hfloat")); + default: + erts_exit(ERTS_ABORT_EXIT, "term_type: Invalid tag (0x%X)\n", hdr); + } + } + case TAG_PRIMARY_IMMED1: + switch (obj & _TAG_IMMED1_MASK) { + case _TAG_IMMED1_SMALL: + BIF_RET(ERTS_MAKE_AM("fixnum")); + case _TAG_IMMED1_PID: + BIF_RET(ERTS_MAKE_AM("pid")); + case _TAG_IMMED1_PORT: + BIF_RET(ERTS_MAKE_AM("port")); + case _TAG_IMMED1_IMMED2: + switch (obj & _TAG_IMMED2_MASK) { + case _TAG_IMMED2_ATOM: + BIF_RET(ERTS_MAKE_AM("atom")); + case _TAG_IMMED2_CATCH: + BIF_RET(ERTS_MAKE_AM("catch")); + case _TAG_IMMED2_NIL: + BIF_RET(ERTS_MAKE_AM("nil")); + default: + erts_exit(ERTS_ABORT_EXIT, "term_type: Invalid tag (0x%X)\n", obj); + } + default: + erts_exit(ERTS_ABORT_EXIT, "term_type: Invalid tag (0x%X)\n", obj); + } + default: + erts_exit(ERTS_ABORT_EXIT, "term_type: Invalid tag (0x%X)\n", obj); + } +} + +/* + * erts_internal:map_hashmap_children/1 + * + * Used in erts_debug:size/1 + */ + +BIF_RETTYPE erts_internal_map_hashmap_children_1(BIF_ALIST_1) { + if (is_map(BIF_ARG_1)) { + Eterm node = BIF_ARG_1; + Eterm *ptr, hdr, *hp, res = NIL; + Uint sz = 0; + ptr = boxed_val(node); + hdr = *ptr; + ASSERT(is_header(hdr)); + + switch(hdr & _HEADER_MAP_SUBTAG_MASK) { + case HAMT_SUBTAG_HEAD_FLATMAP: + BIF_ERROR(BIF_P, BADARG); + case HAMT_SUBTAG_HEAD_BITMAP: + ptr++; + case HAMT_SUBTAG_NODE_BITMAP: + ptr++; + sz = hashmap_bitcount(MAP_HEADER_VAL(hdr)); + break; + case HAMT_SUBTAG_HEAD_ARRAY: + sz = 16; + ptr += 2; + break; + default: + erts_exit(ERTS_ERROR_EXIT, "bad header\r\n"); + break; + } + ASSERT(sz < 17); + hp = HAlloc(BIF_P, 2*sz); + while(sz--) { res = CONS(hp, *ptr++, res); hp += 2; } + BIF_RET(res); + } + BIF_P->fvalue = BIF_ARG_1; + BIF_ERROR(BIF_P, BADMAP); +} + + +static Eterm hashmap_info(Process *p, Eterm node) { + Eterm *hp; + Eterm res = NIL, info = NIL; + Eterm *ptr, tup, hdr; + Uint sz; + DECL_AM(depth); + DECL_AM(leafs); + DECL_AM(bitmaps); + DECL_AM(arrays); + Uint nleaf=0, nbitmap=0, narray=0; + Uint bitmap_usage[16], leaf_usage[16]; + Uint lvl = 0, clvl; + DECLARE_ESTACK(stack); + + for (sz = 0; sz < 16; sz++) { + bitmap_usage[sz] = 0; + leaf_usage[sz] = 0; + } + + ptr = boxed_val(node); + ESTACK_PUSH(stack, 0); + ESTACK_PUSH(stack, node); + do { + node = ESTACK_POP(stack); + clvl = ESTACK_POP(stack); + if (lvl < clvl) + lvl = clvl; + switch(primary_tag(node)) { + case TAG_PRIMARY_LIST: + nleaf++; + leaf_usage[clvl] += 1; + break; + case TAG_PRIMARY_BOXED: + ptr = boxed_val(node); + hdr = *ptr; + ASSERT(is_header(hdr)); + switch(hdr & _HEADER_MAP_SUBTAG_MASK) { + case HAMT_SUBTAG_NODE_BITMAP: + nbitmap++; + sz = hashmap_bitcount(MAP_HEADER_VAL(hdr)); + ASSERT(sz < 17); + bitmap_usage[sz-1] += 1; + while(sz--) { + ESTACK_PUSH(stack, clvl + 1); + ESTACK_PUSH(stack, ptr[sz+1]); + } + break; + case HAMT_SUBTAG_HEAD_BITMAP: + nbitmap++; + sz = hashmap_bitcount(MAP_HEADER_VAL(hdr)); + bitmap_usage[sz-1] += 1; + while(sz--) { + ESTACK_PUSH(stack, clvl + 1); + ESTACK_PUSH(stack, ptr[sz+2]); + } + break; + case HAMT_SUBTAG_HEAD_ARRAY: + narray++; + sz = 16; + while(sz--) { + ESTACK_PUSH(stack, clvl + 1); + ESTACK_PUSH(stack, ptr[sz+2]); + } + break; + default: + erts_exit(ERTS_ERROR_EXIT, "bad header\r\n"); + break; + } + } + } while(!ESTACK_ISEMPTY(stack)); + + + /* size */ + sz = 0; + hashmap_bld_tuple_uint(NULL,&sz,16,leaf_usage); + hashmap_bld_tuple_uint(NULL,&sz,16,bitmap_usage); + + /* alloc */ + hp = HAlloc(p, 2+3 + 3*(2+4) + sz); + + info = hashmap_bld_tuple_uint(&hp,NULL,16,leaf_usage); + tup = TUPLE3(hp, AM_leafs, make_small(nleaf),info); hp += 4; + res = CONS(hp, tup, res); hp += 2; + + info = hashmap_bld_tuple_uint(&hp,NULL,16,bitmap_usage); + tup = TUPLE3(hp, AM_bitmaps, make_small(nbitmap), info); hp += 4; + res = CONS(hp, tup, res); hp += 2; + + tup = TUPLE3(hp, AM_arrays, make_small(narray),NIL); hp += 4; + res = CONS(hp, tup, res); hp += 2; + + tup = TUPLE2(hp, AM_depth, make_small(lvl)); hp += 3; + res = CONS(hp, tup, res); hp += 2; + + DESTROY_ESTACK(stack); + ERTS_HOLE_CHECK(p); + return res; +} + +static Eterm hashmap_bld_tuple_uint(Uint **hpp, Uint *szp, Uint n, Uint nums[]) { + Eterm res = THE_NON_VALUE; + Eterm *ts = (Eterm *)erts_alloc(ERTS_ALC_T_TMP, n * sizeof(Eterm)); + Uint i; + + for (i = 0; i < n; i++) { + ts[i] = erts_bld_uint(hpp, szp, nums[i]); + } + res = erts_bld_tuplev(hpp, szp, n, ts); + erts_free(ERTS_ALC_T_TMP, (void *) ts); + return res; +} + + +/* implementation of builtin emulations */ + +#if !ERTS_AT_LEAST_GCC_VSN__(3, 4, 0) +/* Count leading zeros emulation */ +Uint32 hashmap_clz(Uint32 x) { + Uint32 y; + int n = 32; + y = x >>16; if (y != 0) {n = n -16; x = y;} + y = x >> 8; if (y != 0) {n = n - 8; x = y;} + y = x >> 4; if (y != 0) {n = n - 4; x = y;} + y = x >> 2; if (y != 0) {n = n - 2; x = y;} + y = x >> 1; if (y != 0) return n - 2; + return n - x; +} + +const Uint32 SK5 = 0x55555555, SK3 = 0x33333333; +const Uint32 SKF0 = 0xF0F0F0F, SKFF = 0xFF00FF; + +/* CTPOP emulation */ +Uint32 hashmap_bitcount(Uint32 x) { + x -= ((x >> 1 ) & SK5); + x = (x & SK3 ) + ((x >> 2 ) & SK3 ); + x = (x & SKF0) + ((x >> 4 ) & SKF0); + x += x >> 8; + return (x + (x >> 16)) & 0x3F; +} +#endif diff --git a/erts/emulator/beam/erl_map.h b/erts/emulator/beam/erl_map.h index cfacb2ec28..052fa99f03 100644 --- a/erts/emulator/beam/erl_map.h +++ b/erts/emulator/beam/erl_map.h @@ -3,16 +3,17 @@ * * Copyright Ericsson AB 2014. All Rights Reserved. * - * The contents of this file are subject to the Erlang Public License, - * Version 1.1, (the "License"); you may not use this file except in - * compliance with the License. You should have received a copy of the - * Erlang Public License along with this software. If not, it can be - * retrieved online at http://www.erlang.org/. - * - * Software distributed under the License is distributed on an "AS IS" - * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See - * the License for the specific language governing rights and limitations - * under the License. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. * * %CopyrightEnd% */ @@ -22,13 +23,23 @@ #define __ERL_MAP_H__ #include "sys.h" + +/* instrinsic wrappers */ +#if ERTS_AT_LEAST_GCC_VSN__(3, 4, 0) +#define hashmap_clz(x) ((Uint32) __builtin_clz((unsigned int)(x))) +#define hashmap_bitcount(x) ((Uint32) __builtin_popcount((unsigned int) (x))) +#else +Uint32 hashmap_clz(Uint32 x); +Uint32 hashmap_bitcount(Uint32 x); +#endif + /* MAP */ -typedef struct map_s { +typedef struct flatmap_s { Eterm thing_word; Uint size; Eterm keys; /* tuple */ -} map_t; +} flatmap_t; /* map node * * ----------- @@ -42,31 +53,144 @@ typedef struct map_s { * ----------- */ +/* the head-node is a bitmap or array with an untagged size */ + + +#define hashmap_size(x) (((hashmap_head_t*) hashmap_val(x))->size) +#define hashmap_make_hash(Key) make_internal_hash(Key) + +#define hashmap_restore_hash(Heap,Lvl,Key) \ + (((Lvl) < 8) ? hashmap_make_hash(Key) >> (4*(Lvl)) : hashmap_make_hash(CONS(Heap, make_small((Lvl)>>3), (Key))) >> (4*((Lvl) & 7))) +#define hashmap_shift_hash(Heap,Hx,Lvl,Key) \ + (((++(Lvl)) & 7) ? (Hx) >> 4 : hashmap_make_hash(CONS(Heap, make_small((Lvl)>>3), Key))) /* erl_term.h stuff */ -#define make_map(x) make_boxed((Eterm*)(x)) -#define make_map_rel(x, BASE) make_boxed_rel((Eterm*)(x),(BASE)) -#define is_map(x) (is_boxed((x)) && is_map_header(*boxed_val((x)))) -#define is_map_rel(RTERM,BASE) is_map(rterm2wterm(RTERM,BASE)) -#define is_not_map(x) (!is_map((x))) -#define is_map_header(x) (((x) & (_TAG_HEADER_MASK)) == _TAG_HEADER_MAP) -#define header_is_map(x) ((((x) & (_HEADER_SUBTAG_MASK)) == MAP_SUBTAG)) -#define map_val(x) (_unchecked_boxed_val((x))) -#define map_val_rel(RTERM, BASE) map_val(rterm2wterm(RTERM, BASE)) - -#define map_get_values(x) (((Eterm *)(x)) + 3) -#define map_get_keys(x) (((Eterm *)tuple_val(((map_t *)(x))->keys)) + 1) -#define map_get_size(x) (((map_t*)(x))->size) - -#define MAP_HEADER _make_header(1,_TAG_HEADER_MAP) -#define MAP_HEADER_SIZE (sizeof(map_t) / sizeof(Eterm)) - -Eterm erts_maps_put(Process *p, Eterm key, Eterm value, Eterm map); -int erts_maps_update(Process *p, Eterm key, Eterm value, Eterm map, Eterm *res); -int erts_maps_find(Eterm key, Eterm map, Eterm *value); -int erts_maps_get(Eterm key, Eterm map, Eterm *value); -int erts_maps_remove(Process *p, Eterm key, Eterm map, Eterm *res); -int erts_validate_and_sort_map(map_t* map); +#define flatmap_get_values(x) (((Eterm *)(x)) + 3) +#define flatmap_get_keys(x) (((Eterm *)tuple_val(((flatmap_t *)(x))->keys)) + 1) +#define flatmap_get_size(x) (((flatmap_t*)(x))->size) + +#ifdef DEBUG +#define MAP_SMALL_MAP_LIMIT (3) +#else +#define MAP_SMALL_MAP_LIMIT (32) #endif +struct ErtsWStack_; +struct ErtsEStack_; + +Eterm erts_maps_put(Process *p, Eterm key, Eterm value, Eterm map); +int erts_maps_update(Process *p, Eterm key, Eterm value, Eterm map, Eterm *res); +int erts_maps_remove(Process *p, Eterm key, Eterm map, Eterm *res); + +Eterm erts_hashmap_insert(Process *p, Uint32 hx, Eterm key, Eterm value, + Eterm node, int is_update); +int erts_hashmap_insert_down(Uint32 hx, Eterm key, Eterm node, Uint *sz, + Uint *upsz, struct ErtsEStack_ *sp, int is_update); +Eterm erts_hashmap_insert_up(Eterm *hp, Eterm key, Eterm value, + Uint *upsz, struct ErtsEStack_ *sp); + +int erts_validate_and_sort_flatmap(flatmap_t* map); +void hashmap_iterator_init(struct ErtsWStack_* s, Eterm node, int reverse); +Eterm* hashmap_iterator_next(struct ErtsWStack_* s); +Eterm* hashmap_iterator_prev(struct ErtsWStack_* s); +int hashmap_key_hash_cmp(Eterm* ap, Eterm* bp); +Eterm erts_hashmap_from_array(ErtsHeapFactory*, Eterm *leafs, Uint n, int reject_dupkeys); + +#define erts_hashmap_from_ks_and_vs(P, KS, VS, N) \ + erts_hashmap_from_ks_and_vs_extra((P), (KS), (VS), (N), THE_NON_VALUE, THE_NON_VALUE); + +Eterm erts_hashmap_from_ks_and_vs_extra(Process *p, Eterm *ks, Eterm *vs, Uint n, + Eterm k, Eterm v); + +const Eterm *erts_maps_get(Eterm key, Eterm map); + +const Eterm *erts_hashmap_get(Uint32 hx, Eterm key, Eterm map); + +/* hamt nodes v2.0 + * + * node :: leaf | array | bitmap + * head + */ +typedef struct hashmap_head_s { + Eterm thing_word; + Uint size; + Eterm items[1]; +} hashmap_head_t; + +/* thing_word tagscheme + * Need two bits for map subtags + * + * Original HEADER representation: + * + * aaaaaaaaaaaaaaaa aaaaaaaaaatttt00 arity:26, tag:4 + * + * For maps we have: + * + * vvvvvvvvvvvvvvvv aaaaaaaamm111100 val:16, arity:8, mtype:2 + * + * unsure about trailing zeros + * + * map-tag: + * 00 - flat map tag (non-hamt) -> val:16 = #items + * 01 - map-node bitmap tag -> val:16 = bitmap + * 10 - map-head (array-node) -> val:16 = 0xffff + * 11 - map-head (bitmap-node) -> val:16 = bitmap + */ + +/* erl_map.h stuff */ + +#define is_hashmap_header_head(x) ((MAP_HEADER_TYPE(x) & (0x2))) + +#define MAKE_MAP_HEADER(Type,Arity,Val) \ + (_make_header(((((Uint16)(Val)) << MAP_HEADER_ARITY_SZ) | (Arity)) << MAP_HEADER_TAG_SZ | (Type) , _TAG_HEADER_MAP)) + +#define MAP_HEADER_FLATMAP \ + MAKE_MAP_HEADER(MAP_HEADER_TAG_FLATMAP_HEAD,0x1,0x0) + +#define MAP_HEADER_HAMT_HEAD_ARRAY \ + MAKE_MAP_HEADER(MAP_HEADER_TAG_HAMT_HEAD_ARRAY,0x1,0xffff) + +#define MAP_HEADER_HAMT_HEAD_BITMAP(Bmp) \ + MAKE_MAP_HEADER(MAP_HEADER_TAG_HAMT_HEAD_BITMAP,0x1,Bmp) + +#define MAP_HEADER_HAMT_NODE_BITMAP(Bmp) \ + MAKE_MAP_HEADER(MAP_HEADER_TAG_HAMT_NODE_BITMAP,0x0,Bmp) + +#define MAP_HEADER_FLATMAP_SZ (sizeof(flatmap_t) / sizeof(Eterm)) + +#define HAMT_NODE_ARRAY_SZ (17) +#define HAMT_HEAD_ARRAY_SZ (18) +#define HAMT_NODE_BITMAP_SZ(n) (1 + n) +#define HAMT_HEAD_BITMAP_SZ(n) (2 + n) + +/* 2 bits maps tag + 4 bits subtag + 2 ignore bits */ +#define _HEADER_MAP_SUBTAG_MASK (0xfc) +/* 1 bit map tag + 1 ignore bit + 4 bits subtag + 2 ignore bits */ +#define _HEADER_MAP_HASHMAP_HEAD_MASK (0xbc) + +#define HAMT_SUBTAG_NODE_BITMAP ((MAP_HEADER_TAG_HAMT_NODE_BITMAP << _HEADER_ARITY_OFFS) | MAP_SUBTAG) +#define HAMT_SUBTAG_HEAD_ARRAY ((MAP_HEADER_TAG_HAMT_HEAD_ARRAY << _HEADER_ARITY_OFFS) | MAP_SUBTAG) +#define HAMT_SUBTAG_HEAD_BITMAP ((MAP_HEADER_TAG_HAMT_HEAD_BITMAP << _HEADER_ARITY_OFFS) | MAP_SUBTAG) +#define HAMT_SUBTAG_HEAD_FLATMAP ((MAP_HEADER_TAG_FLATMAP_HEAD << _HEADER_ARITY_OFFS) | MAP_SUBTAG) + +#define hashmap_index(hash) (((Uint32)hash) & 0xf) + +/* hashmap heap size: + [one cons cell + one list term in parent node] per key + [one header + one boxed term in parent node] per inner node + [one header + one size word] for root node + Observed average number of nodes per key is about 0.35. +*/ +#define HASHMAP_WORDS_PER_KEY 3 +#define HASHMAP_WORDS_PER_NODE 2 +#ifdef DEBUG +# define HASHMAP_ESTIMATED_TOT_NODE_SIZE(KEYS) \ + (HASHMAP_WORDS_PER_NODE * (KEYS) * 3/10) /* slightly under estimated */ +#else +# define HASHMAP_ESTIMATED_TOT_NODE_SIZE(KEYS) \ + (HASHMAP_WORDS_PER_NODE * (KEYS) * 4/10) /* slightly over estimated */ +#endif +#define HASHMAP_ESTIMATED_HEAP_SIZE(KEYS) \ + ((KEYS)*HASHMAP_WORDS_PER_KEY + HASHMAP_ESTIMATED_TOT_NODE_SIZE(KEYS)) +#endif diff --git a/erts/emulator/beam/erl_math.c b/erts/emulator/beam/erl_math.c index 16d4fdc09c..b46cc37495 100644 --- a/erts/emulator/beam/erl_math.c +++ b/erts/emulator/beam/erl_math.c @@ -3,16 +3,17 @@ * * Copyright Ericsson AB 1997-2009. All Rights Reserved. * - * The contents of this file are subject to the Erlang Public License, - * Version 1.1, (the "License"); you may not use this file except in - * compliance with the License. You should have received a copy of the - * Erlang Public License along with this software. If not, it can be - * retrieved online at http://www.erlang.org/. - * - * Software distributed under the License is distributed on an "AS IS" - * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See - * the License for the specific language governing rights and limitations - * under the License. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. * * %CopyrightEnd% */ @@ -207,6 +208,24 @@ BIF_RETTYPE math_log_1(BIF_ALIST_1) return math_call_1(BIF_P, log, BIF_ARG_1); } +#ifdef HAVE_LOG2 +static double +log2_wrapper(double x) +{ + return log2(x); +} +#else +static double +log2_wrapper(double x) +{ + return log(x) / 0.6931471805599453; /* log(2.0); */ +} +#endif + +BIF_RETTYPE math_log2_1(BIF_ALIST_1) +{ + return math_call_1(BIF_P, log2_wrapper, BIF_ARG_1); +} BIF_RETTYPE math_log10_1(BIF_ALIST_1) { diff --git a/erts/emulator/beam/erl_message.c b/erts/emulator/beam/erl_message.c index 0eb8117980..88efb2c59f 100644 --- a/erts/emulator/beam/erl_message.c +++ b/erts/emulator/beam/erl_message.c @@ -3,16 +3,17 @@ * * Copyright Ericsson AB 1997-2012. All Rights Reserved. * - * The contents of this file are subject to the Erlang Public License, - * Version 1.1, (the "License"); you may not use this file except in - * compliance with the License. You should have received a copy of the - * Erlang Public License along with this software. If not, it can be - * retrieved online at http://www.erlang.org/. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Software distributed under the License is distributed on an "AS IS" - * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See - * the License for the specific language governing rights and limitations - * under the License. + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. * * %CopyrightEnd% */ @@ -32,8 +33,8 @@ #include "erl_binary.h" #include "dtrace-wrapper.h" -ERTS_SCHED_PREF_QUICK_ALLOC_IMPL(message, - ErlMessage, +ERTS_SCHED_PREF_QUICK_ALLOC_IMPL(message_ref, + ErtsMessageRef, ERL_MESSAGE_BUF_SZ, ERTS_ALC_T_MSG_REF) @@ -43,27 +44,20 @@ ERTS_SCHED_PREF_QUICK_ALLOC_IMPL(message, #undef HARD_DEBUG #endif - - - -#ifdef DEBUG -static ERTS_INLINE int in_heapfrag(const Eterm* ptr, const ErlHeapFragment *bp) +void +init_message(void) { - return ((unsigned)(ptr - bp->mem) < bp->used_size); + init_message_ref_alloc(); } -#endif - -void -init_message(void) +void *erts_alloc_message_ref(void) { - init_message_alloc(); + return (void *) message_ref_alloc(); } -void -free_message(ErlMessage* mp) +void erts_free_message_ref(void *mp) { - message_free(mp); + message_ref_free((ErtsMessageRef *) mp); } /* Allocate message buffer (size in words) */ @@ -73,7 +67,8 @@ new_message_buffer(Uint size) ErlHeapFragment* bp; bp = (ErlHeapFragment*) ERTS_HEAP_ALLOC(ERTS_ALC_T_HEAP_FRAG, ERTS_HEAP_FRAG_SIZE(size)); - ERTS_INIT_HEAP_FRAG(bp, size); + ERTS_INIT_HEAP_FRAG(bp, size, size); + VERBOSE(DEBUG_SHCOPY, ("[pid=%T] new message buffer %p\n", erts_get_current_pid(), bp->mem)); return bp; } @@ -93,9 +88,6 @@ erts_resize_message_buffer(ErlHeapFragment *bp, Uint size, #endif ErlHeapFragment* nbp; - /* ToDo: Make use of 'used_size' to avoid realloc - when shrinking just a few words */ - #ifdef DEBUG { Uint off_sz = size < bp->used_size ? size : bp->used_size; @@ -110,8 +102,10 @@ erts_resize_message_buffer(ErlHeapFragment *bp, Uint size, } #endif - if (size == bp->used_size) + if (size >= (bp->used_size - bp->used_size / 16)) { + bp->used_size = size; return bp; + } #ifdef HARD_DEBUG dbg_brefs = erts_alloc(ERTS_ALC_T_UNDEF, sizeof(Eterm *)*brefs_size); @@ -203,100 +197,57 @@ free_message_buffer(ErlHeapFragment* bp) }while (bp != NULL); } -static ERTS_INLINE void -link_mbuf_to_proc(Process *proc, ErlHeapFragment *bp) +void +erts_cleanup_messages(ErtsMessage *msgp) { - if (bp) { - /* Link the message buffer */ - bp->next = MBUF(proc); - MBUF(proc) = bp; - MBUF_SIZE(proc) += bp->used_size; - FLAGS(proc) |= F_FORCE_GC; - - /* Move any off_heap's into the process */ - if (bp->off_heap.first != NULL) { - struct erl_off_heap_header** next_p = &bp->off_heap.first; - while (*next_p != NULL) { - next_p = &((*next_p)->next); + ErtsMessage *mp = msgp; + while (mp) { + ErtsMessage *fmp; + ErlHeapFragment *bp; + if (is_non_value(ERL_MESSAGE_TERM(mp))) { + if (is_not_immed(ERL_MESSAGE_TOKEN(mp))) { + bp = (ErlHeapFragment *) mp->data.dist_ext->ext_endp; + erts_cleanup_offheap(&bp->off_heap); + } + if (mp->data.dist_ext) + erts_free_dist_ext_copy(mp->data.dist_ext); + } + else { + if (mp->data.attached != ERTS_MSG_COMBINED_HFRAG) + bp = mp->data.heap_frag; + else { + bp = mp->hfrag.next; + erts_cleanup_offheap(&mp->hfrag.off_heap); } - *next_p = MSO(proc).first; - MSO(proc).first = bp->off_heap.first; - bp->off_heap.first = NULL; - OH_OVERHEAD(&(MSO(proc)), bp->off_heap.overhead); + if (bp) + free_message_buffer(bp); } + fmp = mp; + mp = mp->next; + erts_free_message(fmp); } } -Eterm -erts_msg_distext2heap(Process *pp, - ErtsProcLocks *plcksp, - ErlHeapFragment **bpp, - Eterm *tokenp, - ErtsDistExternal *dist_extp) +ErtsMessage * +erts_realloc_shrink_message(ErtsMessage *mp, Uint sz, Eterm *brefs, Uint brefs_size) { - Eterm msg; - Uint tok_sz = 0; - Eterm *hp = NULL; - Eterm *hp_end = NULL; - ErlOffHeap *ohp; - Sint sz; - - *bpp = NULL; - sz = erts_decode_dist_ext_size(dist_extp); - if (sz < 0) - goto decode_error; - if (is_not_nil(*tokenp)) { - ErlHeapFragment *heap_frag = erts_dist_ext_trailer(dist_extp); - tok_sz = heap_frag->used_size; - sz += tok_sz; - } - if (pp) - hp = erts_alloc_message_heap(sz, bpp, &ohp, pp, plcksp); - else { - *bpp = new_message_buffer(sz); - hp = (*bpp)->mem; - ohp = &(*bpp)->off_heap; - } - hp_end = hp + sz; - msg = erts_decode_dist_ext(&hp, ohp, dist_extp); - if (is_non_value(msg)) - goto decode_error; - if (is_not_nil(*tokenp)) { - ErlHeapFragment *heap_frag = erts_dist_ext_trailer(dist_extp); - *tokenp = copy_struct(*tokenp, tok_sz, &hp, ohp); - erts_cleanup_offheap(&heap_frag->off_heap); - } - erts_free_dist_ext_copy(dist_extp); - if (hp_end != hp) { - if (!(*bpp)) { - HRelease(pp, hp_end, hp); - } - else { - Uint final_size = hp - &(*bpp)->mem[0]; - Eterm brefs[2] = {msg, *tokenp}; - ASSERT(sz - (hp_end - hp) == final_size); - *bpp = erts_resize_message_buffer(*bpp, final_size, &brefs[0], 2); - msg = brefs[0]; - *tokenp = brefs[1]; - } + ErtsMessage *nmp = erts_realloc(ERTS_ALC_T_MSG, mp, + sizeof(ErtsMessage) + (sz - 1)*sizeof(Eterm)); + if (nmp != mp) { + Eterm *sp = &mp->hfrag.mem[0]; + Eterm *ep = sp + sz; + Sint offs = &nmp->hfrag.mem[0] - sp; + erts_offset_off_heap(&nmp->hfrag.off_heap, offs, sp, ep); + erts_offset_heap(&nmp->hfrag.mem[0], sz, offs, sp, ep); + if (brefs && brefs_size) + erts_offset_heap_ptr(brefs, brefs_size, offs, sp, ep); } - return msg; - decode_error: - if (is_not_nil(*tokenp)) { - ErlHeapFragment *heap_frag = erts_dist_ext_trailer(dist_extp); - erts_cleanup_offheap(&heap_frag->off_heap); - } - erts_free_dist_ext_copy(dist_extp); - if (*bpp) { - free_message_buffer(*bpp); - *bpp = NULL; - } - else if (hp) { - HRelease(pp, hp_end, hp); - } - return THE_NON_VALUE; - } + nmp->hfrag.used_size = sz; + nmp->hfrag.alloc_size = sz; + + return nmp; +} void erts_queue_dist_message(Process *rcvr, @@ -304,7 +255,7 @@ erts_queue_dist_message(Process *rcvr, ErtsDistExternal *dist_ext, Eterm token) { - ErlMessage* mp; + ErtsMessage* mp; #ifdef USE_VM_PROBES Sint tok_label = 0; Sint tok_lastcnt = 0; @@ -316,7 +267,17 @@ erts_queue_dist_message(Process *rcvr, ERTS_SMP_LC_ASSERT(*rcvr_locks == erts_proc_lc_my_proc_locks(rcvr)); - mp = message_alloc(); + mp = erts_alloc_message(0, NULL); + mp->data.dist_ext = dist_ext; + + ERL_MESSAGE_TERM(mp) = THE_NON_VALUE; +#ifdef USE_VM_PROBES + ERL_MESSAGE_DT_UTAG(mp) = NIL; + if (token == am_have_dt_utag) + ERL_MESSAGE_TOKEN(mp) = NIL; + else +#endif + ERL_MESSAGE_TOKEN(mp) = token; #ifdef ERTS_SMP if (!(*rcvr_locks & ERTS_PROC_LOCK_MSGQ)) { @@ -335,68 +296,46 @@ erts_queue_dist_message(Process *rcvr, if (!(*rcvr_locks & ERTS_PROC_LOCK_MSGQ)) erts_smp_proc_unlock(rcvr, ERTS_PROC_LOCK_MSGQ); /* Drop message if receiver is exiting or has a pending exit ... */ - if (is_not_nil(token)) { - ErlHeapFragment *heap_frag; - heap_frag = erts_dist_ext_trailer(mp->data.dist_ext); - erts_cleanup_offheap(&heap_frag->off_heap); - } - erts_free_dist_ext_copy(dist_ext); - message_free(mp); + erts_cleanup_messages(mp); } else #endif if (IS_TRACED_FL(rcvr, F_TRACE_RECEIVE)) { /* Ahh... need to decode it in order to trace it... */ - ErlHeapFragment *mbuf; - Eterm msg; if (!(*rcvr_locks & ERTS_PROC_LOCK_MSGQ)) erts_smp_proc_unlock(rcvr, ERTS_PROC_LOCK_MSGQ); - message_free(mp); - msg = erts_msg_distext2heap(rcvr, rcvr_locks, &mbuf, &token, dist_ext); - if (is_value(msg)) -#ifdef USE_VM_PROBES - if (DTRACE_ENABLED(message_queued)) { - DTRACE_CHARBUF(receiver_name, DTRACE_TERM_BUF_SIZE); - - dtrace_proc_str(rcvr, receiver_name); - if (token != NIL && token != am_have_dt_utag) { - tok_label = signed_val(SEQ_TRACE_T_LABEL(token)); - tok_lastcnt = signed_val(SEQ_TRACE_T_LASTCNT(token)); - tok_serial = signed_val(SEQ_TRACE_T_SERIAL(token)); - } - DTRACE6(message_queued, - receiver_name, size_object(msg), rcvr->msg.len, - tok_label, tok_lastcnt, tok_serial); - } -#endif - erts_queue_message(rcvr, rcvr_locks, mbuf, msg, token + if (!erts_decode_dist_message(rcvr, *rcvr_locks, mp, 0)) + erts_free_message(mp); + else { + Eterm msg = ERL_MESSAGE_TERM(mp); + token = ERL_MESSAGE_TOKEN(mp); #ifdef USE_VM_PROBES - , NIL + if (DTRACE_ENABLED(message_queued)) { + DTRACE_CHARBUF(receiver_name, DTRACE_TERM_BUF_SIZE); + + dtrace_proc_str(rcvr, receiver_name); + if (have_seqtrace(token)) { + tok_label = signed_val(SEQ_TRACE_T_LABEL(token)); + tok_lastcnt = signed_val(SEQ_TRACE_T_LASTCNT(token)); + tok_serial = signed_val(SEQ_TRACE_T_SERIAL(token)); + } + DTRACE6(message_queued, + receiver_name, size_object(msg), rcvr->msg.len, + tok_label, tok_lastcnt, tok_serial); + } #endif - ); + erts_queue_message(rcvr, rcvr_locks, mp, msg, token); + } } else { /* Enqueue message on external format */ - ERL_MESSAGE_TERM(mp) = THE_NON_VALUE; -#ifdef USE_VM_PROBES - ERL_MESSAGE_DT_UTAG(mp) = NIL; - if (token == am_have_dt_utag) { - ERL_MESSAGE_TOKEN(mp) = NIL; - } else { -#endif - ERL_MESSAGE_TOKEN(mp) = token; -#ifdef USE_VM_PROBES - } -#endif - mp->next = NULL; - #ifdef USE_VM_PROBES if (DTRACE_ENABLED(message_queued)) { DTRACE_CHARBUF(receiver_name, DTRACE_TERM_BUF_SIZE); dtrace_proc_str(rcvr, receiver_name); - if (token != NIL && token != am_have_dt_utag) { + if (have_seqtrace(token)) { tok_label = signed_val(SEQ_TRACE_T_LABEL(token)); tok_lastcnt = signed_val(SEQ_TRACE_T_LASTCNT(token)); tok_serial = signed_val(SEQ_TRACE_T_SERIAL(token)); @@ -409,13 +348,19 @@ erts_queue_dist_message(Process *rcvr, tok_label, tok_lastcnt, tok_serial); } #endif - mp->data.dist_ext = dist_ext; + LINK_MESSAGE(rcvr, mp); if (!(*rcvr_locks & ERTS_PROC_LOCK_MSGQ)) erts_smp_proc_unlock(rcvr, ERTS_PROC_LOCK_MSGQ); - erts_proc_notify_new_message(rcvr); + erts_proc_notify_new_message(rcvr, +#ifdef ERTS_SMP + *rcvr_locks +#else + 0 +#endif + ); } } @@ -423,9 +368,9 @@ erts_queue_dist_message(Process *rcvr, static Sint queue_message(Process *c_p, Process* receiver, - ErtsProcLocks *receiver_locks, erts_aint32_t *receiver_state, - ErlHeapFragment* bp, + ErtsProcLocks *receiver_locks, + ErtsMessage* mp, Eterm message, Eterm seq_trace_token #ifdef USE_VM_PROBES @@ -434,31 +379,24 @@ queue_message(Process *c_p, ) { Sint res; - ErlMessage* mp; int locked_msgq = 0; - erts_aint_t state; - -#ifndef ERTS_SMP - ASSERT(bp != NULL || receiver->mbuf == NULL); -#endif + erts_aint32_t state; ERTS_SMP_LC_ASSERT(*receiver_locks == erts_proc_lc_my_proc_locks(receiver)); - mp = message_alloc(); - - if (receiver_state) - state = *receiver_state; - else - state = erts_smp_atomic32_read_acqb(&receiver->state); - #ifdef ERTS_SMP - if (state & (ERTS_PSFLG_EXITING|ERTS_PSFLG_PENDING_EXIT)) - goto exiting; - if (!(*receiver_locks & ERTS_PROC_LOCK_MSGQ)) { if (erts_smp_proc_trylock(receiver, ERTS_PROC_LOCK_MSGQ) == EBUSY) { ErtsProcLocks need_locks = ERTS_PROC_LOCK_MSGQ; + + if (receiver_state) + state = *receiver_state; + else + state = erts_smp_atomic32_read_nob(&receiver->state); + if (state & (ERTS_PSFLG_EXITING|ERTS_PSFLG_PENDING_EXIT)) + goto exiting; + if (*receiver_locks & ERTS_PROC_LOCK_STATUS) { erts_smp_proc_unlock(receiver, ERTS_PROC_LOCK_STATUS); need_locks |= ERTS_PROC_LOCK_STATUS; @@ -466,13 +404,12 @@ queue_message(Process *c_p, erts_smp_proc_lock(receiver, need_locks); } locked_msgq = 1; - state = erts_smp_atomic32_read_nob(&receiver->state); - if (receiver_state) - *receiver_state = state; } #endif + state = erts_smp_atomic32_read_nob(&receiver->state); + if (state & (ERTS_PSFLG_PENDING_EXIT|ERTS_PSFLG_EXITING)) { #ifdef ERTS_SMP exiting: @@ -480,9 +417,7 @@ queue_message(Process *c_p, /* Drop message if receiver is exiting or has a pending exit... */ if (locked_msgq) erts_smp_proc_unlock(receiver, ERTS_PROC_LOCK_MSGQ); - if (bp) - free_message_buffer(bp); - message_free(mp); + erts_cleanup_messages(mp); return 0; } @@ -491,13 +426,9 @@ queue_message(Process *c_p, #ifdef USE_VM_PROBES ERL_MESSAGE_DT_UTAG(mp) = dt_utag; #endif - mp->next = NULL; - mp->data.heap_frag = bp; -#ifndef ERTS_SMP res = receiver->msg.len; -#else - res = receiver->msg_inq.len; +#ifdef ERTS_SMP if (*receiver_locks & ERTS_PROC_LOCK_MAIN) { /* * We move 'in queue' to 'private queue' and place @@ -507,7 +438,7 @@ queue_message(Process *c_p, * we don't need to include the 'in queue' in * the root set when garbage collecting. */ - res += receiver->msg.len; + res += receiver->msg_inq.len; ERTS_SMP_MSGQ_MV_INQ2PRIVQ(receiver); LINK_MESSAGE_PRIVQ(receiver, mp); } @@ -542,7 +473,13 @@ queue_message(Process *c_p, if (locked_msgq) erts_smp_proc_unlock(receiver, ERTS_PROC_LOCK_MSGQ); - erts_proc_notify_new_message(receiver); + erts_proc_notify_new_message(receiver, +#ifdef ERTS_SMP + *receiver_locks +#else + 0 +#endif + ); #ifndef ERTS_SMP ERTS_HOLE_CHECK(receiver); @@ -551,21 +488,21 @@ queue_message(Process *c_p, } void -erts_queue_message(Process* receiver, - ErtsProcLocks *receiver_locks, - ErlHeapFragment* bp, - Eterm message, - Eterm seq_trace_token #ifdef USE_VM_PROBES - , Eterm dt_utag +erts_queue_message_probe(Process* receiver, ErtsProcLocks *receiver_locks, + ErtsMessage* mp, + Eterm message, Eterm seq_trace_token, Eterm dt_utag) +#else +erts_queue_message(Process* receiver, ErtsProcLocks *receiver_locks, + ErtsMessage* mp, + Eterm message, Eterm seq_trace_token) #endif - ) { queue_message(NULL, receiver, - receiver_locks, NULL, - bp, + receiver_locks, + mp, message, seq_trace_token #ifdef USE_VM_PROBES @@ -575,245 +512,37 @@ erts_queue_message(Process* receiver, } void -erts_link_mbuf_to_proc(struct process *proc, ErlHeapFragment *bp) -{ - Eterm* htop = HEAP_TOP(proc); - - link_mbuf_to_proc(proc, bp); - if (htop < HEAP_LIMIT(proc)) { - *htop = make_pos_bignum_header(HEAP_LIMIT(proc)-htop-1); - HEAP_TOP(proc) = HEAP_LIMIT(proc); - } -} - -/* - * Moves content of message buffer attached to a message into a heap. - * The message buffer is deallocated. - */ -void -erts_move_msg_mbuf_to_heap(Eterm** hpp, ErlOffHeap* off_heap, ErlMessage *msg) +erts_link_mbuf_to_proc(Process *proc, ErlHeapFragment *first_bp) { - struct erl_off_heap_header* oh; - Eterm term, token, *fhp, *hp; - Sint offs; - Uint sz; - ErlHeapFragment *bp; -#ifdef USE_VM_PROBES - Eterm utag; -#endif - -#ifdef HARD_DEBUG - struct erl_off_heap_header* dbg_oh_start = off_heap->first; - Eterm dbg_term, dbg_token; - ErlHeapFragment *dbg_bp; - Uint *dbg_hp, *dbg_thp_start; - Uint dbg_term_sz, dbg_token_sz; -#ifdef USE_VM_PROBES - Eterm dbg_utag; - Uint dbg_utag_sz; -#endif -#endif - - bp = msg->data.heap_frag; - term = ERL_MESSAGE_TERM(msg); - token = ERL_MESSAGE_TOKEN(msg); -#ifdef USE_VM_PROBES - utag = ERL_MESSAGE_DT_UTAG(msg); -#endif - if (!bp) { -#ifdef USE_VM_PROBES - ASSERT(is_immed(term) && is_immed(token) && is_immed(utag)); -#else - ASSERT(is_immed(term) && is_immed(token)); -#endif - return; - } - -#ifdef HARD_DEBUG - dbg_term_sz = size_object(term); - dbg_token_sz = size_object(token); - dbg_bp = new_message_buffer(dbg_term_sz + dbg_token_sz); -#ifdef USE_VM_PROBES - dbg_utag_sz = size_object(utag); - dbg_bp = new_message_buffer(dbg_term_sz + dbg_token_sz + dbg_utag_sz ); -#endif - /*ASSERT(dbg_term_sz + dbg_token_sz == erts_msg_used_frag_sz(msg)); - Copied size may be smaller due to removed SubBins's or garbage. - Copied size may be larger due to duplicated shared terms. - */ - dbg_hp = dbg_bp->mem; - dbg_term = copy_struct(term, dbg_term_sz, &dbg_hp, &dbg_bp->off_heap); - dbg_token = copy_struct(token, dbg_token_sz, &dbg_hp, &dbg_bp->off_heap); -#ifdef USE_VM_PROBES - dbg_utag = copy_struct(utag, dbg_utag_sz, &dbg_hp, &dbg_bp->off_heap); -#endif - dbg_thp_start = *hpp; -#endif - - if (bp->next != NULL) { - move_multi_frags(hpp, off_heap, bp, msg->m, -#ifdef USE_VM_PROBES - 3 -#else - 2 -#endif - ); - goto copy_done; - } - - OH_OVERHEAD(off_heap, bp->off_heap.overhead); - sz = bp->used_size; - - ASSERT(is_immed(term) || in_heapfrag(ptr_val(term),bp)); - ASSERT(is_immed(token) || in_heapfrag(ptr_val(token),bp)); - - fhp = bp->mem; - hp = *hpp; - offs = hp - fhp; - - oh = NULL; - while (sz--) { - Uint cpy_sz; - Eterm val = *fhp++; - - switch (primary_tag(val)) { - case TAG_PRIMARY_IMMED1: - *hp++ = val; - break; - case TAG_PRIMARY_LIST: - case TAG_PRIMARY_BOXED: - ASSERT(in_heapfrag(ptr_val(val), bp)); - *hp++ = offset_ptr(val, offs); - break; - case TAG_PRIMARY_HEADER: - *hp++ = val; - switch (val & _HEADER_SUBTAG_MASK) { - case ARITYVAL_SUBTAG: - break; - case REFC_BINARY_SUBTAG: - case FUN_SUBTAG: - case EXTERNAL_PID_SUBTAG: - case EXTERNAL_PORT_SUBTAG: - case EXTERNAL_REF_SUBTAG: - oh = (struct erl_off_heap_header*) (hp-1); - cpy_sz = thing_arityval(val); - goto cpy_words; - default: - cpy_sz = header_arity(val); - - cpy_words: - ASSERT(sz >= cpy_sz); - sz -= cpy_sz; - while (cpy_sz >= 8) { - cpy_sz -= 8; - *hp++ = *fhp++; - *hp++ = *fhp++; - *hp++ = *fhp++; - *hp++ = *fhp++; - *hp++ = *fhp++; - *hp++ = *fhp++; - *hp++ = *fhp++; - *hp++ = *fhp++; + if (first_bp) { + ErlHeapFragment *bp = first_bp; + + while (1) { + /* Move any off_heap's into the process */ + if (bp->off_heap.first != NULL) { + struct erl_off_heap_header** next_p = &bp->off_heap.first; + while (*next_p != NULL) { + next_p = &((*next_p)->next); } - switch (cpy_sz) { - case 7: *hp++ = *fhp++; - case 6: *hp++ = *fhp++; - case 5: *hp++ = *fhp++; - case 4: *hp++ = *fhp++; - case 3: *hp++ = *fhp++; - case 2: *hp++ = *fhp++; - case 1: *hp++ = *fhp++; - default: break; - } - if (oh) { - /* Add to offheap list */ - oh->next = off_heap->first; - off_heap->first = oh; - ASSERT(*hpp <= (Eterm*)oh); - ASSERT(hp > (Eterm*)oh); - oh = NULL; - } - break; + *next_p = MSO(proc).first; + MSO(proc).first = bp->off_heap.first; + bp->off_heap.first = NULL; + OH_OVERHEAD(&(MSO(proc)), bp->off_heap.overhead); } - break; + MBUF_SIZE(proc) += bp->used_size; + if (!bp->next) + break; + bp = bp->next; } - } - - ASSERT(bp->used_size == hp - *hpp); - *hpp = hp; - - if (is_not_immed(token)) { - ASSERT(in_heapfrag(ptr_val(token), bp)); - ERL_MESSAGE_TOKEN(msg) = offset_ptr(token, offs); -#ifdef HARD_DEBUG - ASSERT(dbg_thp_start <= ptr_val(ERL_MESSAGE_TOKEN(msg))); - ASSERT(hp > ptr_val(ERL_MESSAGE_TOKEN(msg))); -#endif - } - - if (is_not_immed(term)) { - ASSERT(in_heapfrag(ptr_val(term),bp)); - ERL_MESSAGE_TERM(msg) = offset_ptr(term, offs); -#ifdef HARD_DEBUG - ASSERT(dbg_thp_start <= ptr_val(ERL_MESSAGE_TERM(msg))); - ASSERT(hp > ptr_val(ERL_MESSAGE_TERM(msg))); -#endif - } -#ifdef USE_VM_PROBES - if (is_not_immed(utag)) { - ASSERT(in_heapfrag(ptr_val(utag), bp)); - ERL_MESSAGE_DT_UTAG(msg) = offset_ptr(utag, offs); -#ifdef HARD_DEBUG - ASSERT(dbg_thp_start <= ptr_val(ERL_MESSAGE_DT_UTAG(msg))); - ASSERT(hp > ptr_val(ERL_MESSAGE_DT_UTAG(msg))); -#endif - } -#endif -copy_done: - -#ifdef HARD_DEBUG - { - int i, j; - ErlHeapFragment* frag; - { - struct erl_off_heap_header* dbg_oh = off_heap->first; - i = j = 0; - while (dbg_oh != dbg_oh_start) { - dbg_oh = dbg_oh->next; - i++; - } - for (frag=bp; frag; frag=frag->next) { - dbg_oh = frag->off_heap.first; - while (dbg_oh) { - dbg_oh = dbg_oh->next; - j++; - } - } - ASSERT(i == j); - } + /* Link the message buffer */ + bp->next = MBUF(proc); + MBUF(proc) = first_bp; } -#endif - - - bp->off_heap.first = NULL; - free_message_buffer(bp); - msg->data.heap_frag = NULL; - -#ifdef HARD_DEBUG - ASSERT(eq(ERL_MESSAGE_TERM(msg), dbg_term)); - ASSERT(eq(ERL_MESSAGE_TOKEN(msg), dbg_token)); -#ifdef USE_VM_PROBES - ASSERT(eq(ERL_MESSAGE_DT_UTAG(msg), dbg_utag)); -#endif - free_message_buffer(dbg_bp); -#endif - } - Uint -erts_msg_attached_data_size_aux(ErlMessage *msg) +erts_msg_attached_data_size_aux(ErtsMessage *msg) { Sint sz; ASSERT(is_non_value(ERL_MESSAGE_TERM(msg))); @@ -842,29 +571,87 @@ erts_msg_attached_data_size_aux(ErlMessage *msg) return sz; } -void -erts_move_msg_attached_data_to_heap(Eterm **hpp, ErlOffHeap *ohp, ErlMessage *msg) +ErtsMessage * +erts_try_alloc_message_on_heap(Process *pp, + erts_aint32_t *psp, + ErtsProcLocks *plp, + Uint sz, + Eterm **hpp, + ErlOffHeap **ohpp, + int *on_heap_p) { - if (is_value(ERL_MESSAGE_TERM(msg))) - erts_move_msg_mbuf_to_heap(hpp, ohp, msg); - else if (msg->data.dist_ext) { - ASSERT(msg->data.dist_ext->heap_size >= 0); - if (is_not_nil(ERL_MESSAGE_TOKEN(msg))) { - ErlHeapFragment *heap_frag; - heap_frag = erts_dist_ext_trailer(msg->data.dist_ext); - ERL_MESSAGE_TOKEN(msg) = copy_struct(ERL_MESSAGE_TOKEN(msg), - heap_frag->used_size, - hpp, - ohp); - erts_cleanup_offheap(&heap_frag->off_heap); +#ifdef ERTS_SMP + int locked_main = 0; +#endif + ErtsMessage *mp; + + ASSERT(!(*psp & ERTS_PSFLG_OFF_HEAP_MSGQ)); + + if ( +#if defined(ERTS_SMP) + *plp & ERTS_PROC_LOCK_MAIN +#else + 1 +#endif + ) { +#ifdef ERTS_SMP + try_on_heap: +#endif + if ((*psp & (ERTS_PSFLG_EXITING|ERTS_PSFLG_PENDING_EXIT)) + || (pp->flags & F_DISABLE_GC) + || HEAP_LIMIT(pp) - HEAP_TOP(pp) <= sz) { + /* + * The heap is either potentially in an inconsistent + * state, or not large enough. + */ +#ifdef ERTS_SMP + if (locked_main) { + *plp &= ~ERTS_PROC_LOCK_MAIN; + erts_smp_proc_unlock(pp, ERTS_PROC_LOCK_MAIN); + } +#endif + goto in_message_fragment; } - ERL_MESSAGE_TERM(msg) = erts_decode_dist_ext(hpp, - ohp, - msg->data.dist_ext); - erts_free_dist_ext_copy(msg->data.dist_ext); - msg->data.dist_ext = NULL; + + *hpp = HEAP_TOP(pp); + HEAP_TOP(pp) = *hpp + sz; + *ohpp = &MSO(pp); + mp = erts_alloc_message(0, NULL); + mp->data.attached = NULL; + *on_heap_p = !0; } - /* else: bad external detected when calculating size */ +#ifdef ERTS_SMP + else if (erts_smp_proc_trylock(pp, ERTS_PROC_LOCK_MAIN) == 0) { + locked_main = 1; + *psp = erts_smp_atomic32_read_nob(&pp->state); + *plp |= ERTS_PROC_LOCK_MAIN; + goto try_on_heap; + } +#endif + else { + in_message_fragment: + if (!((*psp) & ERTS_PSFLG_ON_HEAP_MSGQ)) { + mp = erts_alloc_message(sz, hpp); + *ohpp = sz == 0 ? NULL : &mp->hfrag.off_heap; + } + else { + mp = erts_alloc_message(0, NULL); + if (!sz) { + *hpp = NULL; + *ohpp = NULL; + } + else { + ErlHeapFragment *bp; + bp = new_message_buffer(sz); + *hpp = &bp->mem[0]; + mp->data.heap_frag = bp; + *ohpp = &bp->off_heap; + } + } + *on_heap_p = 0; + } + + return mp; } /* @@ -879,7 +666,8 @@ erts_send_message(Process* sender, unsigned flags) { Uint msize; - ErlHeapFragment* bp = NULL; + ErtsMessage* mp; + ErlOffHeap *ohp; Eterm token = NIL; Sint res = 0; #ifdef USE_VM_PROBES @@ -888,67 +676,96 @@ erts_send_message(Process* sender, Sint tok_label = 0; Sint tok_lastcnt = 0; Sint tok_serial = 0; + Eterm utag = NIL; +#endif + erts_aint32_t receiver_state; +#ifdef SHCOPY_SEND + erts_shcopy_t info; #endif BM_STOP_TIMER(system); BM_MESSAGE(message,sender,receiver); BM_START_TIMER(send); - #ifdef USE_VM_PROBES +#ifdef USE_VM_PROBES *sender_name = *receiver_name = '\0'; - if (DTRACE_ENABLED(message_send)) { + if (DTRACE_ENABLED(message_send)) { erts_snprintf(sender_name, sizeof(DTRACE_CHARBUF_NAME(sender_name)), "%T", sender->common.id); erts_snprintf(receiver_name, sizeof(DTRACE_CHARBUF_NAME(receiver_name)), "%T", receiver->common.id); } #endif + + receiver_state = erts_smp_atomic32_read_nob(&receiver->state); + if (SEQ_TRACE_TOKEN(sender) != NIL && !(flags & ERTS_SND_FLG_NO_SEQ_TRACE)) { Eterm* hp; Eterm stoken = SEQ_TRACE_TOKEN(sender); Uint seq_trace_size = 0; #ifdef USE_VM_PROBES Uint dt_utag_size = 0; - Eterm utag = NIL; -#endif - - BM_SWAP_TIMER(send,size); - msize = size_object(message); - BM_SWAP_TIMER(size,send); - -#ifdef USE_VM_PROBES - if (stoken != am_have_dt_utag) { #endif + BM_SWAP_TIMER(send,size); + /* SHCOPY corrupts the heap between + * copy_shared_calculate, and + * copy_shared_perform. (it inserts move_markers like the gc). + * Make sure we don't use the heap between those instances. + */ + if (have_seqtrace(stoken)) { seq_trace_update_send(sender); seq_trace_output(stoken, message, SEQ_TRACE_SEND, receiver->common.id, sender); seq_trace_size = 6; /* TUPLE5 */ -#ifdef USE_VM_PROBES - } - if (DT_UTAG_FLAGS(sender) & DT_UTAG_SPREADING) { - dt_utag_size = size_object(DT_UTAG(sender)); - } else if (stoken == am_have_dt_utag ) { - stoken = NIL; } +#ifdef USE_VM_PROBES + if (DT_UTAG_FLAGS(sender) & DT_UTAG_SPREADING) { + dt_utag_size = size_object(DT_UTAG(sender)); + } else if (stoken == am_have_dt_utag ) { + stoken = NIL; + } #endif - bp = new_message_buffer(msize + seq_trace_size +#ifdef SHCOPY_SEND + INITIALIZE_SHCOPY(info); + msize = copy_shared_calculate(message, &info); +#else + msize = size_object(message); +#endif + BM_SWAP_TIMER(size,send); + + mp = erts_alloc_message_heap_state(receiver, + &receiver_state, + receiver_locks, + (msize #ifdef USE_VM_PROBES - + dt_utag_size + + dt_utag_size #endif - ); - hp = bp->mem; + + seq_trace_size), + &hp, + &ohp); BM_SWAP_TIMER(send,copy); - token = copy_struct(stoken, - seq_trace_size, - &hp, - &bp->off_heap); - message = copy_struct(message, msize, &hp, &bp->off_heap); +#ifdef SHCOPY_SEND + if (is_not_immed(message)) + message = copy_shared_perform(message, msize, &info, &hp, ohp); + DESTROY_SHCOPY(info); +#else + if (is_not_immed(message)) + message = copy_struct(message, msize, &hp, ohp); +#endif + if (is_immed(stoken)) + token = stoken; + else + token = copy_struct(stoken, seq_trace_size, &hp, ohp); + #ifdef USE_VM_PROBES if (DT_UTAG_FLAGS(sender) & DT_UTAG_SPREADING) { - utag = copy_struct(DT_UTAG(sender), dt_utag_size, &hp, &bp->off_heap); + if (is_immed(DT_UTAG(sender))) + utag = DT_UTAG(sender); + else + utag = copy_struct(DT_UTAG(sender), dt_utag_size, &hp, ohp); #ifdef DTRACE_TAG_HARDDEBUG erts_fprintf(stderr, "Dtrace -> (%T) Spreading tag (%T) with " @@ -961,7 +778,7 @@ erts_send_message(Process* sender, #ifdef USE_VM_PROBES if (DTRACE_ENABLED(message_send)) { - if (stoken != NIL && stoken != am_have_dt_utag) { + if (have_seqtrace(stoken)) { tok_label = signed_val(SEQ_TRACE_T_LABEL(stoken)); tok_lastcnt = signed_val(SEQ_TRACE_T_LASTCNT(stoken)); tok_serial = signed_val(SEQ_TRACE_T_SERIAL(stoken)); @@ -970,140 +787,63 @@ erts_send_message(Process* sender, msize, tok_label, tok_lastcnt, tok_serial); } #endif - res = queue_message(NULL, - receiver, - receiver_locks, - NULL, - bp, - message, - token -#ifdef USE_VM_PROBES - , utag -#endif - ); - BM_SWAP_TIMER(send,system); - } else if (sender == receiver) { - /* Drop message if receiver has a pending exit ... */ -#ifdef ERTS_SMP - ErtsProcLocks need_locks = (~(*receiver_locks) - & (ERTS_PROC_LOCK_MSGQ - | ERTS_PROC_LOCK_STATUS)); - if (need_locks) { - *receiver_locks |= need_locks; - if (erts_smp_proc_trylock(receiver, need_locks) == EBUSY) { - if (need_locks == ERTS_PROC_LOCK_MSGQ) { - erts_smp_proc_unlock(receiver, ERTS_PROC_LOCK_STATUS); - need_locks = ERTS_PROC_LOCK_MSGQ|ERTS_PROC_LOCK_STATUS; - } - erts_smp_proc_lock(receiver, need_locks); - } - } - if (!ERTS_PROC_PENDING_EXIT(receiver)) -#endif - { - ErlMessage* mp = message_alloc(); - - DTRACE6(message_send, sender_name, receiver_name, - size_object(message), tok_label, tok_lastcnt, tok_serial); - mp->data.attached = NULL; - ERL_MESSAGE_TERM(mp) = message; - ERL_MESSAGE_TOKEN(mp) = NIL; -#ifdef USE_VM_PROBES - ERL_MESSAGE_DT_UTAG(mp) = NIL; -#endif - mp->next = NULL; - /* - * We move 'in queue' to 'private queue' and place - * message at the end of 'private queue' in order - * to ensure that the 'in queue' doesn't contain - * references into the heap. By ensuring this, - * we don't need to include the 'in queue' in - * the root set when garbage collecting. - */ - - ERTS_SMP_MSGQ_MV_INQ2PRIVQ(receiver); - LINK_MESSAGE_PRIVQ(receiver, mp); - - res = receiver->msg.len; - - if (IS_TRACED_FL(receiver, F_TRACE_RECEIVE)) { - trace_receive(receiver, message); - } - } - BM_SWAP_TIMER(send,system); } else { -#ifdef ERTS_SMP - ErlOffHeap *ohp; Eterm *hp; - erts_aint32_t state; - - BM_SWAP_TIMER(send,size); - msize = size_object(message); - BM_SWAP_TIMER(size,send); - hp = erts_alloc_message_heap_state(msize, - &bp, - &ohp, - receiver, - receiver_locks, - &state); - BM_SWAP_TIMER(send,copy); - message = copy_struct(message, msize, &hp, ohp); - BM_MESSAGE_COPIED(msz); - BM_SWAP_TIMER(copy,send); - DTRACE6(message_send, sender_name, receiver_name, - msize, tok_label, tok_lastcnt, tok_serial); - res = queue_message(sender, - receiver, - receiver_locks, - &state, - bp, - message, - token -#ifdef USE_VM_PROBES - , NIL + + if (receiver == sender && !(receiver_state & ERTS_PSFLG_OFF_HEAP_MSGQ)) { + mp = erts_alloc_message(0, NULL); + msize = 0; + } + else { + BM_SWAP_TIMER(send,size); +#ifdef SHCOPY_SEND + INITIALIZE_SHCOPY(info); + msize = copy_shared_calculate(message, &info); +#else + msize = size_object(message); #endif - ); - BM_SWAP_TIMER(send,system); + BM_SWAP_TIMER(size,send); + + mp = erts_alloc_message_heap_state(receiver, + &receiver_state, + receiver_locks, + msize, + &hp, + &ohp); + BM_SWAP_TIMER(send,copy); +#ifdef SHCOPY_SEND + if (is_not_immed(message)) + message = copy_shared_perform(message, msize, &info, &hp, ohp); + DESTROY_SHCOPY(info); #else - ErlMessage* mp = message_alloc(); - Eterm *hp; - BM_SWAP_TIMER(send,size); - msize = size_object(message); - BM_SWAP_TIMER(size,send); - - if (receiver->stop - receiver->htop <= msize) { - BM_SWAP_TIMER(send,system); - erts_garbage_collect(receiver, msize, receiver->arg_reg, receiver->arity); - BM_SWAP_TIMER(system,send); + if (is_not_immed(message)) + message = copy_struct(message, msize, &hp, ohp); +#endif + BM_MESSAGE_COPIED(msz); + BM_SWAP_TIMER(copy,send); } - hp = receiver->htop; - receiver->htop = hp + msize; - BM_SWAP_TIMER(send,copy); - message = copy_struct(message, msize, &hp, &receiver->off_heap); - BM_MESSAGE_COPIED(msize); - BM_SWAP_TIMER(copy,send); DTRACE6(message_send, sender_name, receiver_name, - (uint32_t)msize, tok_label, tok_lastcnt, tok_serial); - ERL_MESSAGE_TERM(mp) = message; - ERL_MESSAGE_TOKEN(mp) = NIL; + msize, tok_label, tok_lastcnt, tok_serial); + } + + res = queue_message(sender, + receiver, + &receiver_state, + receiver_locks, + mp, + message, + token #ifdef USE_VM_PROBES - ERL_MESSAGE_DT_UTAG(mp) = NIL; + , utag #endif - mp->next = NULL; - mp->data.attached = NULL; - LINK_MESSAGE(receiver, mp); - res = receiver->msg.len; - erts_proc_notify_new_message(receiver); + ); - if (IS_TRACED_FL(receiver, F_TRACE_RECEIVE)) { - trace_receive(receiver, message); - } - BM_SWAP_TIMER(send,system); -#endif /* #ifndef ERTS_SMP */ - } - return res; + BM_SWAP_TIMER(send,system); + + return res; } + /* * This function delivers an EXIT message to a process * which is trapping EXITs. @@ -1121,53 +861,1025 @@ erts_deliver_exit_message(Eterm from, Process *to, ErtsProcLocks *to_locksp, Uint sz_from; Eterm* hp; Eterm temptoken; - ErlHeapFragment* bp = NULL; - - if (token != NIL -#ifdef USE_VM_PROBES - && token != am_have_dt_utag + ErtsMessage* mp; + ErlOffHeap *ohp; +#ifdef SHCOPY_SEND + erts_shcopy_t info; #endif - ) { + if (have_seqtrace(token)) { ASSERT(is_tuple(token)); - sz_reason = size_object(reason); sz_token = size_object(token); sz_from = size_object(from); - bp = new_message_buffer(sz_reason + sz_from + sz_token + 4); - hp = bp->mem; - mess = copy_struct(reason, sz_reason, &hp, &bp->off_heap); - from_copy = copy_struct(from, sz_from, &hp, &bp->off_heap); +#ifdef SHCOPY_SEND + INITIALIZE_SHCOPY(info); + sz_reason = copy_shared_calculate(reason, &info); +#else + sz_reason = size_object(reason); +#endif + mp = erts_alloc_message_heap(to, to_locksp, + sz_reason + sz_from + sz_token + 4, + &hp, &ohp); +#ifdef SHCOPY_SEND + mess = copy_shared_perform(reason, sz_reason, &info, &hp, ohp); + DESTROY_SHCOPY(info); +#else + mess = copy_struct(reason, sz_reason, &hp, ohp); +#endif + from_copy = copy_struct(from, sz_from, &hp, ohp); save = TUPLE3(hp, am_EXIT, from_copy, mess); hp += 4; /* the trace token must in this case be updated by the caller */ seq_trace_output(token, save, SEQ_TRACE_SEND, to->common.id, NULL); - temptoken = copy_struct(token, sz_token, &hp, &bp->off_heap); - erts_queue_message(to, to_locksp, bp, save, temptoken -#ifdef USE_VM_PROBES - , NIL -#endif - ); + temptoken = copy_struct(token, sz_token, &hp, ohp); + erts_queue_message(to, to_locksp, mp, save, temptoken); } else { - ErlOffHeap *ohp; - sz_reason = size_object(reason); sz_from = IS_CONST(from) ? 0 : size_object(from); +#ifdef SHCOPY_SEND + INITIALIZE_SHCOPY(info); + sz_reason = copy_shared_calculate(reason, &info); +#else + sz_reason = size_object(reason); +#endif + mp = erts_alloc_message_heap(to, to_locksp, + sz_reason+sz_from+4, &hp, &ohp); - hp = erts_alloc_message_heap(sz_reason+sz_from+4, - &bp, - &ohp, - to, - to_locksp); - +#ifdef SHCOPY_SEND + mess = copy_shared_perform(reason, sz_reason, &info, &hp, ohp); + DESTROY_SHCOPY(info); +#else mess = copy_struct(reason, sz_reason, &hp, ohp); +#endif from_copy = (IS_CONST(from) ? from : copy_struct(from, sz_from, &hp, ohp)); save = TUPLE3(hp, am_EXIT, from_copy, mess); - erts_queue_message(to, to_locksp, bp, save, NIL + erts_queue_message(to, to_locksp, mp, save, NIL); + } +} + +void erts_save_message_in_proc(Process *p, ErtsMessage *msgp) +{ + ErlHeapFragment *hfp; + + if (msgp->data.attached == ERTS_MSG_COMBINED_HFRAG) + hfp = &msgp->hfrag; + else if (msgp->data.attached) { + hfp = msgp->data.heap_frag; + } + else { + erts_free_message(msgp); + return; /* Nothing to save */ + } + + while (1) { + struct erl_off_heap_header *ohhp = hfp->off_heap.first; + if (ohhp) { + for ( ; ohhp->next; ohhp = ohhp->next) + ; + ohhp->next = p->off_heap.first; + p->off_heap.first = hfp->off_heap.first; + hfp->off_heap.first = NULL; + } + p->off_heap.overhead += hfp->off_heap.overhead; + hfp->off_heap.overhead = 0; + p->mbuf_sz += hfp->used_size; + + if (!hfp->next) + break; + hfp = hfp->next; + } + + msgp->next = p->msg_frag; + p->msg_frag = msgp; +} + +Sint +erts_move_messages_off_heap(Process *c_p) +{ + int reds = 1; + /* + * Move all messages off heap. This *only* occurs when the + * process had off heap message disabled and just enabled + * it... + */ + ErtsMessage *mp; + + reds += c_p->msg.len / 10; + + ASSERT(erts_smp_atomic32_read_nob(&c_p->state) + & ERTS_PSFLG_OFF_HEAP_MSGQ); + ASSERT(c_p->flags & F_OFF_HEAP_MSGQ_CHNG); + + for (mp = c_p->msg.first; mp; mp = mp->next) { + Uint msg_sz, token_sz; #ifdef USE_VM_PROBES - , NIL + Uint utag_sz; #endif - ); + Eterm *hp; + ErlHeapFragment *hfrag; + + if (mp->data.attached) + continue; + + if (is_immed(ERL_MESSAGE_TERM(mp)) +#ifdef USE_VM_PROBES + && is_immed(ERL_MESSAGE_DT_UTAG(mp)) +#endif + && is_not_immed(ERL_MESSAGE_TOKEN(mp))) + continue; + + /* + * The message refers into the heap. Copy the message + * from the heap into a heap fragment and attach + * it to the message... + */ + msg_sz = size_object(ERL_MESSAGE_TERM(mp)); +#ifdef USE_VM_PROBES + utag_sz = size_object(ERL_MESSAGE_DT_UTAG(mp)); +#endif + token_sz = size_object(ERL_MESSAGE_TOKEN(mp)); + + hfrag = new_message_buffer(msg_sz +#ifdef USE_VM_PROBES + + utag_sz +#endif + + token_sz); + hp = hfrag->mem; + if (is_not_immed(ERL_MESSAGE_TERM(mp))) + ERL_MESSAGE_TERM(mp) = copy_struct(ERL_MESSAGE_TERM(mp), + msg_sz, &hp, + &hfrag->off_heap); + if (is_not_immed(ERL_MESSAGE_TOKEN(mp))) + ERL_MESSAGE_TOKEN(mp) = copy_struct(ERL_MESSAGE_TOKEN(mp), + token_sz, &hp, + &hfrag->off_heap); +#ifdef USE_VM_PROBES + if (is_not_immed(ERL_MESSAGE_DT_UTAG(mp))) + ERL_MESSAGE_DT_UTAG(mp) = copy_struct(ERL_MESSAGE_DT_UTAG(mp), + utag_sz, &hp, + &hfrag->off_heap); +#endif + mp->data.heap_frag = hfrag; + reds += 1; + } + + return reds; +} + +Sint +erts_complete_off_heap_message_queue_change(Process *c_p) +{ + int reds = 1; + + ERTS_SMP_LC_ASSERT(ERTS_PROC_LOCK_MAIN == erts_proc_lc_my_proc_locks(c_p)); + ASSERT(c_p->flags & F_OFF_HEAP_MSGQ_CHNG); + ASSERT(erts_smp_atomic32_read_nob(&c_p->state) & ERTS_PSFLG_OFF_HEAP_MSGQ); + + /* + * This job was first initiated when the process changed to off heap + * message queue management. Since then ERTS_PSFLG_OFF_HEAP_MSGQ + * has been set. However, the management state might have been changed + * again (multiple times) since then. Check users last requested state + * (the flags F_OFF_HEAP_MSGQ, and F_ON_HEAP_MSGQ), and make the state + * consistent with that. + */ + + if (!(c_p->flags & F_OFF_HEAP_MSGQ)) + erts_smp_atomic32_read_band_nob(&c_p->state, + ~ERTS_PSFLG_OFF_HEAP_MSGQ); + else { + reds += 2; + erts_smp_proc_lock(c_p, ERTS_PROC_LOCK_MSGQ); + ERTS_SMP_MSGQ_MV_INQ2PRIVQ(c_p); + erts_smp_proc_unlock(c_p, ERTS_PROC_LOCK_MSGQ); + reds += erts_move_messages_off_heap(c_p); + } + c_p->flags &= ~F_OFF_HEAP_MSGQ_CHNG; + return reds; +} + +typedef struct { + Eterm pid; + ErtsThrPrgrLaterOp lop; +} ErtsChangeOffHeapMessageQueue; + +static void +change_off_heap_msgq(void *vcohmq) +{ + ErtsChangeOffHeapMessageQueue *cohmq; + /* + * Now we've waited thread progress which ensures that all + * messages to the process are enqueued off heap. Schedule + * completion of this change as a system task on the process + * itself. This in order to avoid lock contention on its + * main lock. We will be called in + * erts_complete_off_heap_message_queue_change() (above) when + * the system task has been selected for execution. + */ + cohmq = (ErtsChangeOffHeapMessageQueue *) vcohmq; + erts_schedule_complete_off_heap_message_queue_change(cohmq->pid); + erts_free(ERTS_ALC_T_MSGQ_CHNG, vcohmq); +} + +Eterm +erts_change_message_queue_management(Process *c_p, Eterm new_state) +{ + Eterm res; + +#ifdef DEBUG + if (c_p->flags & F_OFF_HEAP_MSGQ) { + ASSERT(erts_smp_atomic32_read_nob(&c_p->state) + & ERTS_PSFLG_OFF_HEAP_MSGQ); + } + else { + if (c_p->flags & F_OFF_HEAP_MSGQ_CHNG) { + ASSERT(erts_smp_atomic32_read_nob(&c_p->state) + & ERTS_PSFLG_OFF_HEAP_MSGQ); + } + else { + ASSERT(!(erts_smp_atomic32_read_nob(&c_p->state) + & ERTS_PSFLG_OFF_HEAP_MSGQ)); + } + } +#endif + + switch (c_p->flags & (F_OFF_HEAP_MSGQ|F_ON_HEAP_MSGQ)) { + + case F_OFF_HEAP_MSGQ: + res = am_off_heap; + + switch (new_state) { + case am_off_heap: + break; + case am_on_heap: + c_p->flags |= F_ON_HEAP_MSGQ; + erts_smp_atomic32_read_bor_nob(&c_p->state, + ERTS_PSFLG_ON_HEAP_MSGQ); + /* fall through */ + case am_mixed: + c_p->flags &= ~F_OFF_HEAP_MSGQ; + /* + * We are not allowed to clear ERTS_PSFLG_OFF_HEAP_MSGQ + * if a off heap change is ongoing. It will be adjusted + * when the change completes... + */ + if (!(c_p->flags & F_OFF_HEAP_MSGQ_CHNG)) { + /* Safe to clear ERTS_PSFLG_OFF_HEAP_MSGQ... */ + erts_smp_atomic32_read_band_nob(&c_p->state, + ~ERTS_PSFLG_OFF_HEAP_MSGQ); + } + break; + default: + res = THE_NON_VALUE; /* badarg */ + break; + } + break; + + case F_ON_HEAP_MSGQ: + res = am_on_heap; + + switch (new_state) { + case am_on_heap: + break; + case am_mixed: + c_p->flags &= ~F_ON_HEAP_MSGQ; + erts_smp_atomic32_read_band_nob(&c_p->state, + ~ERTS_PSFLG_ON_HEAP_MSGQ); + break; + case am_off_heap: + c_p->flags &= ~F_ON_HEAP_MSGQ; + erts_smp_atomic32_read_band_nob(&c_p->state, + ~ERTS_PSFLG_ON_HEAP_MSGQ); + goto change_to_off_heap; + default: + res = THE_NON_VALUE; /* badarg */ + break; + } + break; + + case 0: + res = am_mixed; + + switch (new_state) { + case am_mixed: + break; + case am_on_heap: + c_p->flags |= F_ON_HEAP_MSGQ; + erts_smp_atomic32_read_bor_nob(&c_p->state, + ERTS_PSFLG_ON_HEAP_MSGQ); + break; + case am_off_heap: + goto change_to_off_heap; + default: + res = THE_NON_VALUE; /* badarg */ + break; + } + break; + + default: + res = am_error; + ERTS_INTERNAL_ERROR("Inconsistent message queue management state"); + break; + } + + return res; + +change_to_off_heap: + + c_p->flags |= F_OFF_HEAP_MSGQ; + + /* + * We do not have to schedule a change if + * we have an ongoing off heap change... + */ + if (!(c_p->flags & F_OFF_HEAP_MSGQ_CHNG)) { + ErtsChangeOffHeapMessageQueue *cohmq; + /* + * Need to set ERTS_PSFLG_OFF_HEAP_MSGQ and wait + * thread progress before completing the change in + * order to ensure that all senders observe that + * messages should be passed off heap. When the + * change has completed, GC does not need to inspect + * the message queue at all. + */ + erts_smp_atomic32_read_bor_nob(&c_p->state, + ERTS_PSFLG_OFF_HEAP_MSGQ); + c_p->flags |= F_OFF_HEAP_MSGQ_CHNG; + cohmq = erts_alloc(ERTS_ALC_T_MSGQ_CHNG, + sizeof(ErtsChangeOffHeapMessageQueue)); + cohmq->pid = c_p->common.id; + erts_schedule_thr_prgr_later_op(change_off_heap_msgq, + (void *) cohmq, + &cohmq->lop); } + + return res; } +int +erts_decode_dist_message(Process *proc, ErtsProcLocks proc_locks, + ErtsMessage *msgp, int force_off_heap) +{ + ErtsHeapFactory factory; + Eterm msg; + ErlHeapFragment *bp; + Sint need; + int decode_in_heap_frag; + + decode_in_heap_frag = (force_off_heap + || !(proc_locks & ERTS_PROC_LOCK_MAIN) + || (proc->flags & F_OFF_HEAP_MSGQ)); + + if (msgp->data.dist_ext->heap_size >= 0) + need = msgp->data.dist_ext->heap_size; + else { + need = erts_decode_dist_ext_size(msgp->data.dist_ext); + if (need < 0) { + /* bad msg; remove it... */ + if (is_not_immed(ERL_MESSAGE_TOKEN(msgp))) { + bp = erts_dist_ext_trailer(msgp->data.dist_ext); + erts_cleanup_offheap(&bp->off_heap); + } + erts_free_dist_ext_copy(msgp->data.dist_ext); + msgp->data.dist_ext = NULL; + return 0; + } + + msgp->data.dist_ext->heap_size = need; + } + + if (is_not_immed(ERL_MESSAGE_TOKEN(msgp))) { + bp = erts_dist_ext_trailer(msgp->data.dist_ext); + need += bp->used_size; + } + + if (decode_in_heap_frag) + erts_factory_heap_frag_init(&factory, new_message_buffer(need)); + else + erts_factory_proc_prealloc_init(&factory, proc, need); + + ASSERT(msgp->data.dist_ext->heap_size >= 0); + if (is_not_immed(ERL_MESSAGE_TOKEN(msgp))) { + ErlHeapFragment *heap_frag; + heap_frag = erts_dist_ext_trailer(msgp->data.dist_ext); + ERL_MESSAGE_TOKEN(msgp) = copy_struct(ERL_MESSAGE_TOKEN(msgp), + heap_frag->used_size, + &factory.hp, + factory.off_heap); + erts_cleanup_offheap(&heap_frag->off_heap); + } + + msg = erts_decode_dist_ext(&factory, msgp->data.dist_ext); + ERL_MESSAGE_TERM(msgp) = msg; + erts_free_dist_ext_copy(msgp->data.dist_ext); + msgp->data.attached = NULL; + + if (is_non_value(msg)) { + erts_factory_undo(&factory); + return 0; + } + + erts_factory_trim_and_close(&factory, msgp->m, + ERL_MESSAGE_REF_ARRAY_SZ); + + ASSERT(!msgp->data.heap_frag); + + if (decode_in_heap_frag) + msgp->data.heap_frag = factory.heap_frags; + + return 1; +} + +/* + * ERTS_INSPECT_MSGQ_KEEP_OH_MSGS == 0 will move off heap messages + * into the heap of the inspected process if off_heap_message_queue + * is false when process_info(_, messages) is called. That is, the + * following GC will have more data in the rootset compared to the + * scenario when process_info(_, messages) had not been called. + * + * ERTS_INSPECT_MSGQ_KEEP_OH_MSGS != 0 will keep off heap messages + * off heap when process_info(_, messages) is called regardless of + * the off_heap_message_queue setting of the process. That is, it + * will change the following execution of the process as little as + * possible. + */ +#define ERTS_INSPECT_MSGQ_KEEP_OH_MSGS 1 + +Uint +erts_prep_msgq_for_inspection(Process *c_p, Process *rp, + ErtsProcLocks rp_locks, ErtsMessageInfo *mip) +{ + Uint tot_heap_size; + ErtsMessage* mp; + Sint i; + int self_on_heap; + + /* + * Prepare the message queue for inspection + * by process_info(). + * + * + * - Decode all messages on external format + * - Remove all corrupt dist messages from queue + * - Save pointer to, and heap size need of each + * message in the mip array. + * - Return total heap size need for all messages + * that needs to be copied. + * + * If ERTS_INSPECT_MSGQ_KEEP_OH_MSGS == 0: + * - In case off heap messages is disabled and + * we are inspecting our own queue, move all + * off heap data into the heap. + */ + + self_on_heap = c_p == rp && !(c_p->flags & F_OFF_HEAP_MSGQ); + + tot_heap_size = 0; + i = 0; + mp = rp->msg.first; + while (mp) { + Eterm msg = ERL_MESSAGE_TERM(mp); + + mip[i].size = 0; + + if (is_non_value(msg)) { + /* Dist message on external format; decode it... */ + if (mp->data.attached) + erts_decode_dist_message(rp, rp_locks, mp, + ERTS_INSPECT_MSGQ_KEEP_OH_MSGS); + + msg = ERL_MESSAGE_TERM(mp); + + if (is_non_value(msg)) { + ErtsMessage **mpp; + ErtsMessage *bad_mp = mp; + /* + * Bad distribution message; remove + * it from the queue... + */ + ASSERT(!mp->data.attached); + + mpp = i == 0 ? &rp->msg.first : &mip[i-1].msgp->next; + + if (rp->msg.save == &bad_mp->next) + rp->msg.save = mpp; + if (rp->msg.last == &bad_mp->next) + rp->msg.last = mpp; + mp = mp->next; + *mpp = mp; + rp->msg.len--; + bad_mp->next = NULL; + erts_cleanup_messages(bad_mp); + continue; + } + } + + ASSERT(is_value(msg)); + +#if ERTS_INSPECT_MSGQ_KEEP_OH_MSGS + if (is_not_immed(msg) && (!self_on_heap || mp->data.attached)) { + Uint sz = size_object(msg); + mip[i].size = sz; + tot_heap_size += sz; + } +#else + if (self_on_heap) { + if (mp->data.attached) { + ErtsMessage *tmp = NULL; + if (mp->data.attached != ERTS_MSG_COMBINED_HFRAG) { + erts_link_mbuf_to_proc(rp, mp->data.heap_frag); + mp->data.attached = NULL; + } + else { + /* + * Need to replace the message reference since + * we will get references to the message data + * from the heap... + */ + ErtsMessage **mpp; + tmp = erts_alloc_message(0, NULL); + sys_memcpy((void *) tmp->m, (void *) mp->m, + sizeof(Eterm)*ERL_MESSAGE_REF_ARRAY_SZ); + mpp = i == 0 ? &rp->msg.first : &mip[i-1].msgp->next; + tmp->next = mp->next; + if (rp->msg.save == &mp->next) + rp->msg.save = &tmp->next; + if (rp->msg.last == &mp->next) + rp->msg.last = &tmp->next; + *mpp = tmp; + erts_save_message_in_proc(rp, mp); + mp = tmp; + } + } + } + else if (is_not_immed(msg)) { + Uint sz = size_object(msg); + mip[i].size = sz; + tot_heap_size += sz; + } + +#endif + + mip[i].msgp = mp; + i++; + mp = mp->next; + } + + return tot_heap_size; +} + +void erts_factory_proc_init(ErtsHeapFactory* factory, + Process* p) +{ + erts_factory_proc_prealloc_init(factory, p, HEAP_LIMIT(p) - HEAP_TOP(p)); +} + +void erts_factory_proc_prealloc_init(ErtsHeapFactory* factory, + Process* p, + Sint size) +{ + ErlHeapFragment *bp = p->mbuf; + factory->mode = FACTORY_HALLOC; + factory->p = p; + factory->hp_start = HAlloc(p, size); + factory->hp = factory->hp_start; + factory->hp_end = factory->hp_start + size; + factory->off_heap = &p->off_heap; + factory->message = NULL; + factory->off_heap_saved.first = p->off_heap.first; + factory->off_heap_saved.overhead = p->off_heap.overhead; + factory->heap_frags_saved = bp; + factory->heap_frags_saved_used = bp ? bp->used_size : 0; + factory->heap_frags = NULL; /* not used */ + factory->alloc_type = 0; /* not used */ +} + +void erts_factory_heap_frag_init(ErtsHeapFactory* factory, + ErlHeapFragment* bp) +{ + factory->mode = FACTORY_HEAP_FRAGS; + factory->p = NULL; + factory->hp_start = bp->mem; + factory->hp = bp->mem; + factory->hp_end = bp->mem + bp->alloc_size; + factory->off_heap = &bp->off_heap; + factory->message = NULL; + factory->heap_frags = bp; + factory->heap_frags_saved = NULL; + factory->heap_frags_saved_used = 0; + factory->alloc_type = ERTS_ALC_T_HEAP_FRAG; + ASSERT(!bp->next); + factory->off_heap_saved.first = factory->off_heap->first; + factory->off_heap_saved.overhead = factory->off_heap->overhead; + + ASSERT(factory->hp >= factory->hp_start && factory->hp <= factory->hp_end); +} + + +ErtsMessage * +erts_factory_message_create(ErtsHeapFactory* factory, + Process *proc, + ErtsProcLocks *proc_locksp, + Uint sz) +{ + Eterm *hp; + ErlOffHeap *ohp; + ErtsMessage *msgp; + int on_heap; + erts_aint32_t state; + + state = erts_smp_atomic32_read_nob(&proc->state); + + if (state & ERTS_PSFLG_OFF_HEAP_MSGQ) { + msgp = erts_alloc_message(sz, &hp); + ohp = sz == 0 ? NULL : &msgp->hfrag.off_heap; + on_heap = 0; + } + else { + msgp = erts_try_alloc_message_on_heap(proc, &state, + proc_locksp, + sz, &hp, &ohp, + &on_heap); + } + + if (on_heap) { + ERTS_SMP_ASSERT(*proc_locksp & ERTS_PROC_LOCK_MAIN); + ASSERT(ohp == &proc->off_heap); + factory->mode = FACTORY_HALLOC; + factory->p = proc; + factory->heap_frags_saved = proc->mbuf; + factory->heap_frags_saved_used = proc->mbuf ? proc->mbuf->used_size : 0; + } + else { + factory->mode = FACTORY_MESSAGE; + factory->p = NULL; + factory->heap_frags_saved = NULL; + factory->heap_frags_saved_used = 0; + + if (msgp->data.attached == ERTS_MSG_COMBINED_HFRAG) { + ASSERT(!msgp->hfrag.next); + factory->heap_frags = NULL; + } + else { + ASSERT(!msgp->data.heap_frag + || !msgp->data.heap_frag->next); + factory->heap_frags = msgp->data.heap_frag; + } + } + factory->hp_start = hp; + factory->hp = hp; + factory->hp_end = hp + sz; + factory->message = msgp; + factory->off_heap = ohp; + factory->alloc_type = ERTS_ALC_T_HEAP_FRAG; + if (ohp) { + factory->off_heap_saved.first = ohp->first; + factory->off_heap_saved.overhead = ohp->overhead; + } + else { + factory->off_heap_saved.first = NULL; + factory->off_heap_saved.overhead = 0; + } + + ASSERT(factory->hp >= factory->hp_start && factory->hp <= factory->hp_end); + + return msgp; +} + +void erts_factory_selfcontained_message_init(ErtsHeapFactory* factory, + ErtsMessage *msgp, + Eterm *hp) +{ + ErlHeapFragment* bp; + if (msgp->data.attached == ERTS_MSG_COMBINED_HFRAG) { + bp = &msgp->hfrag; + factory->heap_frags = NULL; + } + else { + bp = msgp->data.heap_frag; + factory->heap_frags = bp; + } + factory->mode = FACTORY_MESSAGE; + factory->p = NULL; + factory->hp_start = bp->mem; + factory->hp = hp; + factory->hp_end = bp->mem + bp->alloc_size; + factory->message = msgp; + factory->off_heap = &bp->off_heap; + factory->heap_frags_saved = NULL; + factory->heap_frags_saved_used = 0; + factory->alloc_type = ERTS_ALC_T_HEAP_FRAG; + ASSERT(!bp->next); + factory->off_heap_saved.first = factory->off_heap->first; + factory->off_heap_saved.overhead = factory->off_heap->overhead; + + ASSERT(factory->hp >= factory->hp_start && factory->hp <= factory->hp_end); +} + +/* One static sized heap that must suffice. + No extra heap fragments will be allocated. +*/ +void erts_factory_static_init(ErtsHeapFactory* factory, + Eterm* hp, + Uint size, + ErlOffHeap* off_heap) +{ + factory->mode = FACTORY_STATIC; + factory->hp_start = hp; + factory->hp = hp; + factory->hp_end = hp + size; + factory->off_heap = off_heap; + factory->off_heap_saved.first = factory->off_heap->first; + factory->off_heap_saved.overhead = factory->off_heap->overhead; +} + +/* A temporary heap with default buffer allocated/freed by client. + * factory_close is same as factory_undo + */ +void erts_factory_tmp_init(ErtsHeapFactory* factory, Eterm* hp, Uint size, + Uint32 atype) +{ + factory->mode = FACTORY_TMP; + factory->hp_start = hp; + factory->hp = hp; + factory->hp_end = hp + size; + factory->heap_frags = NULL; + factory->off_heap_saved.first = NULL; + factory->off_heap_saved.overhead = 0; + factory->off_heap = &factory->off_heap_saved; + factory->alloc_type = atype; +} + +/* When we know the term is an immediate and need no heap. +*/ +void erts_factory_dummy_init(ErtsHeapFactory* factory) +{ + factory->mode = FACTORY_CLOSED; +} + +static void reserve_heap(ErtsHeapFactory*, Uint need, Uint xtra); + +Eterm* erts_produce_heap(ErtsHeapFactory* factory, Uint need, Uint xtra) +{ + Eterm* res; + + ASSERT((unsigned int)factory->mode > (unsigned int)FACTORY_CLOSED); + if (factory->hp + need > factory->hp_end) { + reserve_heap(factory, need, xtra); + } + res = factory->hp; + factory->hp += need; + return res; +} + +Eterm* erts_reserve_heap(ErtsHeapFactory* factory, Uint need) +{ + ASSERT((unsigned int)factory->mode > (unsigned int)FACTORY_CLOSED); + if (factory->hp + need > factory->hp_end) { + reserve_heap(factory, need, 200); + } + return factory->hp; +} + +static void reserve_heap(ErtsHeapFactory* factory, Uint need, Uint xtra) +{ + ErlHeapFragment* bp; + + switch (factory->mode) { + case FACTORY_HALLOC: + HRelease(factory->p, factory->hp_end, factory->hp); + factory->hp = HAllocX(factory->p, need, xtra); + factory->hp_end = factory->hp + need; + return; + + case FACTORY_MESSAGE: + if (!factory->heap_frags) { + ASSERT(factory->message->data.attached == ERTS_MSG_COMBINED_HFRAG); + bp = &factory->message->hfrag; + } + else { + /* Fall through */ + case FACTORY_HEAP_FRAGS: + case FACTORY_TMP: + bp = factory->heap_frags; + } + + if (bp) { + ASSERT(factory->hp > bp->mem); + ASSERT(factory->hp <= factory->hp_end); + ASSERT(factory->hp_end == bp->mem + bp->alloc_size); + + bp->used_size = factory->hp - bp->mem; + } + bp = (ErlHeapFragment*) ERTS_HEAP_ALLOC(factory->alloc_type, + ERTS_HEAP_FRAG_SIZE(need+xtra)); + bp->next = factory->heap_frags; + factory->heap_frags = bp; + bp->alloc_size = need + xtra; + bp->used_size = need; + bp->off_heap.first = NULL; + bp->off_heap.overhead = 0; + + factory->hp = bp->mem; + factory->hp_end = bp->mem + bp->alloc_size; + return; + + case FACTORY_STATIC: + case FACTORY_CLOSED: + default: + ASSERT(!"Invalid factory mode"); + } +} + +void erts_factory_close(ErtsHeapFactory* factory) +{ + ErlHeapFragment* bp; + + switch (factory->mode) { + case FACTORY_HALLOC: + HRelease(factory->p, factory->hp_end, factory->hp); + break; + + case FACTORY_MESSAGE: + if (!factory->heap_frags) { + if (factory->message->data.attached == ERTS_MSG_COMBINED_HFRAG) + bp = &factory->message->hfrag; + else + bp = NULL; + } + else { + if (factory->message->data.attached == ERTS_MSG_COMBINED_HFRAG) + factory->message->hfrag.next = factory->heap_frags; + else + factory->message->data.heap_frag = factory->heap_frags; + + /* Fall through */ + case FACTORY_HEAP_FRAGS: + bp = factory->heap_frags; + } + + if (bp) { + ASSERT(factory->hp >= bp->mem); + ASSERT(factory->hp <= factory->hp_end); + ASSERT(factory->hp_end == bp->mem + bp->alloc_size); + + bp->used_size = factory->hp - bp->mem; + } + break; + case FACTORY_TMP: + erts_factory_undo(factory); + break; + case FACTORY_STATIC: break; + case FACTORY_CLOSED: break; + default: + ASSERT(!"Invalid factory mode"); + } + factory->mode = FACTORY_CLOSED; +} + +void erts_factory_trim_and_close(ErtsHeapFactory* factory, + Eterm *brefs, Uint brefs_size) +{ + ErlHeapFragment *bp; + + switch (factory->mode) { + case FACTORY_MESSAGE: { + ErtsMessage *mp = factory->message; + if (mp->data.attached == ERTS_MSG_COMBINED_HFRAG) { + if (!mp->hfrag.next) { + Uint sz = factory->hp - factory->hp_start; + mp = erts_shrink_message(mp, sz, brefs, brefs_size); + factory->message = mp; + factory->mode = FACTORY_CLOSED; + return; + } + /*else we don't trim multi fragmented messages for now (off_heap...) */ + break; + } + /* Fall through... */ + } + case FACTORY_HEAP_FRAGS: + bp = factory->heap_frags; + if (!bp) + break; + if (bp->next == NULL) { + Uint used_sz = factory->hp - bp->mem; + ASSERT(used_sz <= bp->alloc_size); + if (used_sz > 0) + bp = erts_resize_message_buffer(bp, used_sz, + brefs, brefs_size); + else { + free_message_buffer(bp); + bp = NULL; + } + factory->heap_frags = bp; + if (factory->mode == FACTORY_MESSAGE) + factory->message->data.heap_frag = bp; + factory->mode = FACTORY_CLOSED; + return; + } + /*else we don't trim multi fragmented messages for now (off_heap...) */ + default: + break; + } + erts_factory_close(factory); +} + +void erts_factory_undo(ErtsHeapFactory* factory) +{ + ErlHeapFragment* bp; + struct erl_off_heap_header *hdr, **hdr_nextp; + + switch (factory->mode) { + case FACTORY_HALLOC: + case FACTORY_STATIC: + /* Cleanup off-heap + */ + hdr_nextp = NULL; + for (hdr = factory->off_heap->first; + hdr != factory->off_heap_saved.first; + hdr = hdr->next) { + + hdr_nextp = &hdr->next; + } + + if (hdr_nextp != NULL) { + *hdr_nextp = NULL; + erts_cleanup_offheap(factory->off_heap); + factory->off_heap->first = factory->off_heap_saved.first; + factory->off_heap->overhead = factory->off_heap_saved.overhead; + } + + if (factory->mode == FACTORY_HALLOC) { + /* Free heap frags + */ + bp = factory->p->mbuf; + if (bp != factory->heap_frags_saved) { + do { + ErlHeapFragment *next_bp = bp->next; + ASSERT(bp->off_heap.first == NULL); + ERTS_HEAP_FREE(ERTS_ALC_T_HEAP_FRAG, (void *) bp, + ERTS_HEAP_FRAG_SIZE(bp->alloc_size)); + bp = next_bp; + } while (bp != factory->heap_frags_saved); + + factory->p->mbuf = bp; + } + + /* Rollback heap top + */ + + if (HEAP_START(factory->p) <= factory->hp_start + && factory->hp_start <= HEAP_LIMIT(factory->p)) { + HEAP_TOP(factory->p) = factory->hp_start; + } + + /* Fix last heap frag */ + if (factory->heap_frags_saved) { + ASSERT(factory->heap_frags_saved == factory->p->mbuf); + if (factory->hp_start != factory->heap_frags_saved->mem) + factory->heap_frags_saved->used_size = factory->heap_frags_saved_used; + else { + factory->p->mbuf = factory->p->mbuf->next; + ERTS_HEAP_FREE(ERTS_ALC_T_HEAP_FRAG, factory->heap_frags_saved, + ERTS_HEAP_FRAG_SIZE(factory->heap_frags_saved->alloc_size)); + } + } + } + break; + + case FACTORY_MESSAGE: + if (factory->message->data.attached == ERTS_MSG_COMBINED_HFRAG) + factory->message->hfrag.next = factory->heap_frags; + else + factory->message->data.heap_frag = factory->heap_frags; + erts_cleanup_messages(factory->message); + break; + case FACTORY_TMP: + case FACTORY_HEAP_FRAGS: + erts_cleanup_offheap(factory->off_heap); + factory->off_heap->first = NULL; + + bp = factory->heap_frags; + while (bp != NULL) { + ErlHeapFragment* next_bp = bp->next; + + ASSERT(bp->off_heap.first == NULL); + ERTS_HEAP_FREE(factory->alloc_type, (void *) bp, + ERTS_HEAP_FRAG_SIZE(bp->alloc_size)); + bp = next_bp; + } + break; + + case FACTORY_CLOSED: break; + default: + ASSERT(!"Invalid factory mode"); + } + factory->mode = FACTORY_CLOSED; +#ifdef DEBUG + factory->p = NULL; + factory->hp = NULL; + factory->heap_frags = NULL; +#endif +} diff --git a/erts/emulator/beam/erl_message.h b/erts/emulator/beam/erl_message.h index 0f3bb8d281..60035d15ae 100644 --- a/erts/emulator/beam/erl_message.h +++ b/erts/emulator/beam/erl_message.h @@ -3,16 +3,17 @@ * * Copyright Ericsson AB 1997-2012. All Rights Reserved. * - * The contents of this file are subject to the Erlang Public License, - * Version 1.1, (the "License"); you may not use this file except in - * compliance with the License. You should have received a copy of the - * Erlang Public License along with this software. If not, it can be - * retrieved online at http://www.erlang.org/. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Software distributed under the License is distributed on an "AS IS" - * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See - * the License for the specific language governing rights and limitations - * under the License. + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. * * %CopyrightEnd% */ @@ -23,6 +24,8 @@ struct proc_bin; struct external_thing_; +typedef struct erl_mesg ErtsMessage; + /* * This struct represents data that must be updated by structure copy, * but is stored outside of any heap. @@ -31,9 +34,6 @@ struct external_thing_; struct erl_off_heap_header { Eterm thing_word; Uint size; -#if HALFWORD_HEAP - void* dummy_ptr_padding__; -#endif struct erl_off_heap_header* next; }; @@ -51,9 +51,58 @@ typedef struct erl_off_heap { (OHP)->first = NULL; \ (OHP)->overhead = 0; \ } while (0) + +typedef struct { + enum { + FACTORY_CLOSED = 0, + FACTORY_HALLOC, + FACTORY_MESSAGE, + FACTORY_HEAP_FRAGS, + FACTORY_STATIC, + FACTORY_TMP + } mode; + Process* p; + Eterm* hp_start; + Eterm* hp; + Eterm* hp_end; + ErtsMessage *message; + struct erl_heap_fragment* heap_frags; + struct erl_heap_fragment* heap_frags_saved; + Uint heap_frags_saved_used; + ErlOffHeap* off_heap; + ErlOffHeap off_heap_saved; + Uint32 alloc_type; +} ErtsHeapFactory; + +void erts_factory_proc_init(ErtsHeapFactory*, Process*); +void erts_factory_proc_prealloc_init(ErtsHeapFactory*, Process*, Sint size); +void erts_factory_heap_frag_init(ErtsHeapFactory*, struct erl_heap_fragment*); +ErtsMessage *erts_factory_message_create(ErtsHeapFactory *, Process *, + ErtsProcLocks *, Uint sz); +void erts_factory_selfcontained_message_init(ErtsHeapFactory*, ErtsMessage *, Eterm *); +void erts_factory_static_init(ErtsHeapFactory*, Eterm* hp, Uint size, ErlOffHeap*); +void erts_factory_tmp_init(ErtsHeapFactory*, Eterm* hp, Uint size, Uint32 atype); +void erts_factory_dummy_init(ErtsHeapFactory*); + +Eterm* erts_produce_heap(ErtsHeapFactory*, Uint need, Uint xtra); +Eterm* erts_reserve_heap(ErtsHeapFactory*, Uint need); +void erts_factory_close(ErtsHeapFactory*); +void erts_factory_trim_and_close(ErtsHeapFactory*,Eterm *brefs, Uint brefs_size); +void erts_factory_undo(ErtsHeapFactory*); + +#ifdef CHECK_FOR_HOLES +# define ERTS_FACTORY_HOLE_CHECK(f) do { \ + /*if ((f)->p) erts_check_for_holes((f)->p);*/ \ + } while (0) +#else +# define ERTS_FACTORY_HOLE_CHECK(p) +#endif + #include "external.h" #include "erl_process.h" +#define ERTS_INVALID_HFRAG_PTR ((ErlHeapFragment *) ~((UWord) 7)) + /* * This struct represents a heap fragment, which is used when there * isn't sufficient room in the process heap and we can't do a GC. @@ -68,33 +117,53 @@ struct erl_heap_fragment { Eterm mem[1]; /* Data */ }; -typedef struct erl_mesg { - struct erl_mesg* next; /* Next message */ - union { - ErtsDistExternal *dist_ext; - ErlHeapFragment *heap_frag; - void *attached; - } data; +/* m[0] = message, m[1] = seq trace token */ +#define ERL_MESSAGE_REF_ARRAY_SZ 2 +#define ERL_MESSAGE_TERM(mp) ((mp)->m[0]) +#define ERL_MESSAGE_TOKEN(mp) ((mp)->m[1]) + #ifdef USE_VM_PROBES - Eterm m[3]; /* m[0] = message, m[1] = seq trace token, m[3] = dynamic trace user tag */ +/* m[2] = dynamic trace user tag */ +#undef ERL_MESSAGE_REF_ARRAY_SZ +#define ERL_MESSAGE_REF_ARRAY_SZ 3 +#define ERL_MESSAGE_DT_UTAG(mp) ((mp)->m[2]) #else - Eterm m[2]; /* m[0] = message, m[1] = seq trace token */ #endif -} ErlMessage; -#define ERL_MESSAGE_TERM(mp) ((mp)->m[0]) -#define ERL_MESSAGE_TOKEN(mp) ((mp)->m[1]) #ifdef USE_VM_PROBES -#define ERL_MESSAGE_DT_UTAG(mp) ((mp)->m[2]) +#define have_no_seqtrace(T) ((T) == NIL || (T) == am_have_dt_utag) +#else +#define have_no_seqtrace(T) ((T) == NIL) #endif +#define have_seqtrace(T) (!have_no_seqtrace(T)) + +#define ERL_MESSAGE_REF_FIELDS__ \ + ErtsMessage *next; /* Next message */ \ + union { \ + ErtsDistExternal *dist_ext; \ + ErlHeapFragment *heap_frag; \ + void *attached; \ + } data; \ + Eterm m[ERL_MESSAGE_REF_ARRAY_SZ] + + +typedef struct erl_msg_ref__ { + ERL_MESSAGE_REF_FIELDS__; +} ErtsMessageRef; + +struct erl_mesg { + ERL_MESSAGE_REF_FIELDS__; + + ErlHeapFragment hfrag; +}; /* Size of default message buffer (erl_message.c) */ #define ERL_MESSAGE_BUF_SZ 500 typedef struct { - ErlMessage* first; - ErlMessage** last; /* point to the last next pointer */ - ErlMessage** save; + ErtsMessage* first; + ErtsMessage** last; /* point to the last next pointer */ + ErtsMessage** save; Sint len; /* queue length */ /* @@ -102,14 +171,14 @@ typedef struct { * recv_set/1 instructions. */ BeamInstr* mark; /* address to rec_loop/2 instruction */ - ErlMessage** saved_last; /* saved last pointer */ + ErtsMessage** saved_last; /* saved last pointer */ } ErlMessageQueue; #ifdef ERTS_SMP typedef struct { - ErlMessage* first; - ErlMessage** last; /* point to the last next pointer */ + ErtsMessage* first; + ErtsMessage** last; /* point to the last next pointer */ Sint len; /* queue length */ } ErlMessageInQueue; @@ -124,7 +193,7 @@ typedef struct { *(p)->msg.last = (mp); \ (p)->msg.last = &(mp)->next; \ (p)->msg.len++; \ -} while(0) +} while (0) #ifdef ERTS_SMP @@ -160,7 +229,7 @@ do { \ /* Unlink current message */ #define UNLINK_MESSAGE(p,msgp) do { \ - ErlMessage* __mp = (msgp)->next; \ + ErtsMessage* __mp = (msgp)->next; \ *(p)->msg.save = __mp; \ (p)->msg.len--; \ if (__mp == NULL) \ @@ -176,102 +245,196 @@ do { \ #define SAVE_MESSAGE(p) \ (p)->msg.save = &(*(p)->msg.save)->next -/* - * ErtsMoveMsgAttachmentIntoProc() moves data attached to a message - * onto the heap of a process. The attached data is the content of - * the the message either on the internal format or on the external - * format, and also possibly a seq trace token on the internal format. - * If the message content is on the external format, the decode might - * fail. If the decoding fails, ERL_MESSAGE_TERM(M) will contain - * THE_NON_VALUE. That is, ERL_MESSAGE_TERM(M) *has* to be checked - * afterwards and taken care of appropriately. - * - * ErtsMoveMsgAttachmentIntoProc() will shallow copy to heap if - * possible; otherwise, move to heap via garbage collection. - * - * ErtsMoveMsgAttachmentIntoProc() is used when receiveing messages - * in process_main() and in hipe_check_get_msg(). - */ - -#define ErtsMoveMsgAttachmentIntoProc(M, P, ST, HT, FC, SWPO, SWPI) \ -do { \ - if ((M)->data.attached) { \ - Uint need__ = erts_msg_attached_data_size((M)); \ - if ((ST) - (HT) >= need__) { \ - Uint *htop__ = (HT); \ - erts_move_msg_attached_data_to_heap(&htop__, &MSO((P)), (M));\ - ASSERT(htop__ - (HT) <= need__); \ - (HT) = htop__; \ - } \ - else { \ - { SWPO ; } \ - (FC) -= erts_garbage_collect((P), 0, NULL, 0); \ - { SWPI ; } \ - } \ - ASSERT(!(M)->data.attached); \ - } \ -} while (0) - #define ERTS_SND_FLG_NO_SEQ_TRACE (((unsigned) 1) << 0) #define ERTS_HEAP_FRAG_SIZE(DATA_WORDS) \ (sizeof(ErlHeapFragment) - sizeof(Eterm) + (DATA_WORDS)*sizeof(Eterm)) -#define ERTS_INIT_HEAP_FRAG(HEAP_FRAG_P, DATA_WORDS) \ -do { \ - (HEAP_FRAG_P)->next = NULL; \ - (HEAP_FRAG_P)->alloc_size = (DATA_WORDS); \ - (HEAP_FRAG_P)->used_size = (DATA_WORDS); \ - (HEAP_FRAG_P)->off_heap.first = NULL; \ - (HEAP_FRAG_P)->off_heap.overhead = 0; \ -} while (0) +#define ERTS_INIT_HEAP_FRAG(HEAP_FRAG_P, USED_WORDS, DATA_WORDS) \ + do { \ + (HEAP_FRAG_P)->next = NULL; \ + (HEAP_FRAG_P)->alloc_size = (DATA_WORDS); \ + (HEAP_FRAG_P)->used_size = (USED_WORDS); \ + (HEAP_FRAG_P)->off_heap.first = NULL; \ + (HEAP_FRAG_P)->off_heap.overhead = 0; \ + } while (0) void init_message(void); -void free_message(ErlMessage *); ErlHeapFragment* new_message_buffer(Uint); ErlHeapFragment* erts_resize_message_buffer(ErlHeapFragment *, Uint, Eterm *, Uint); void free_message_buffer(ErlHeapFragment *); void erts_queue_dist_message(Process*, ErtsProcLocks*, ErtsDistExternal *, Eterm); -void erts_queue_message(Process*, ErtsProcLocks*, ErlHeapFragment*, Eterm, Eterm #ifdef USE_VM_PROBES - , Eterm dt_utag +void erts_queue_message_probe(Process*, ErtsProcLocks*, ErtsMessage*, + Eterm message, Eterm seq_trace_token, Eterm dt_utag); +#define erts_queue_message(RP,RL,BP,Msg,SEQ) \ + erts_queue_message_probe((RP),(RL),(BP),(Msg),(SEQ),NIL) +#else +void erts_queue_message(Process*, ErtsProcLocks*, ErtsMessage*, + Eterm message, Eterm seq_trace_token); +#define erts_queue_message_probe(RP,RL,BP,Msg,SEQ,TAG) \ + erts_queue_message((RP),(RL),(BP),(Msg),(SEQ)) #endif -); void erts_deliver_exit_message(Eterm, Process*, ErtsProcLocks *, Eterm, Eterm); Sint erts_send_message(Process*, Process*, ErtsProcLocks*, Eterm, unsigned); void erts_link_mbuf_to_proc(Process *proc, ErlHeapFragment *bp); -void erts_move_msg_mbuf_to_heap(Eterm**, ErlOffHeap*, ErlMessage *); +Uint erts_msg_attached_data_size_aux(ErtsMessage *msg); -Uint erts_msg_attached_data_size_aux(ErlMessage *msg); -void erts_move_msg_attached_data_to_heap(Eterm **, ErlOffHeap *, ErlMessage *); +void erts_cleanup_offheap(ErlOffHeap *offheap); +void erts_save_message_in_proc(Process *p, ErtsMessage *msg); +Sint erts_move_messages_off_heap(Process *c_p); +Sint erts_complete_off_heap_message_queue_change(Process *c_p); +Eterm erts_change_message_queue_management(Process *c_p, Eterm new_state); -Eterm erts_msg_distext2heap(Process *, ErtsProcLocks *, ErlHeapFragment **, - Eterm *, ErtsDistExternal *); +int erts_decode_dist_message(Process *, ErtsProcLocks, ErtsMessage *, int); -void erts_cleanup_offheap(ErlOffHeap *offheap); +void erts_cleanup_messages(ErtsMessage *mp); + +typedef struct { + Uint size; + ErtsMessage *msgp; +} ErtsMessageInfo; + +Uint erts_prep_msgq_for_inspection(Process *c_p, + Process *rp, + ErtsProcLocks rp_locks, + ErtsMessageInfo *mip); + +void *erts_alloc_message_ref(void); +void erts_free_message_ref(void *); +#define ERTS_SMALL_FIX_MSG_SZ 10 +#define ERTS_MEDIUM_FIX_MSG_SZ 20 +#define ERTS_LARGE_FIX_MSG_SZ 30 -ERTS_GLB_INLINE Uint erts_msg_used_frag_sz(const ErlMessage *msg); -ERTS_GLB_INLINE Uint erts_msg_attached_data_size(ErlMessage *msg); +void *erts_alloc_small_message(void); +void erts_free_small_message(void *mp); + +typedef struct { + ErtsMessage m; + Eterm data[ERTS_SMALL_FIX_MSG_SZ-1]; +} ErtsSmallFixSzMessage; + +typedef struct { + ErtsMessage m; + Eterm data[ERTS_MEDIUM_FIX_MSG_SZ-1]; +} ErtsMediumFixSzMessage; + +typedef struct { + ErtsMessage m; + Eterm data[ERTS_LARGE_FIX_MSG_SZ-1]; +} ErtsLargeFixSzMessage; + +ErtsMessage *erts_try_alloc_message_on_heap(Process *pp, + erts_aint32_t *psp, + ErtsProcLocks *plp, + Uint sz, + Eterm **hpp, + ErlOffHeap **ohpp, + int *on_heap_p); +ErtsMessage *erts_realloc_shrink_message(ErtsMessage *mp, Uint sz, + Eterm *brefs, Uint brefs_size); + +ERTS_GLB_FORCE_INLINE ErtsMessage *erts_alloc_message(Uint sz, Eterm **hpp); +ERTS_GLB_FORCE_INLINE ErtsMessage *erts_shrink_message(ErtsMessage *mp, Uint sz, + Eterm *brefs, Uint brefs_size); +ERTS_GLB_FORCE_INLINE void erts_free_message(ErtsMessage *mp); +ERTS_GLB_INLINE Uint erts_used_frag_sz(const ErlHeapFragment*); +ERTS_GLB_INLINE Uint erts_msg_attached_data_size(ErtsMessage *msg); + +#define ERTS_MSG_COMBINED_HFRAG ((void *) 0x1) #if ERTS_GLB_INLINE_INCL_FUNC_DEF -ERTS_GLB_INLINE Uint erts_msg_used_frag_sz(const ErlMessage *msg) + +ERTS_GLB_FORCE_INLINE ErtsMessage *erts_alloc_message(Uint sz, Eterm **hpp) +{ + ErtsMessage *mp; + + if (sz == 0) { + mp = erts_alloc_message_ref(); + mp->next = NULL; + ERL_MESSAGE_TERM(mp) = NIL; + mp->data.attached = NULL; + if (hpp) + *hpp = NULL; + return mp; + } + + mp = erts_alloc(ERTS_ALC_T_MSG, + sizeof(ErtsMessage) + (sz - 1)*sizeof(Eterm)); + + mp->next = NULL; + ERL_MESSAGE_TERM(mp) = NIL; + mp->data.attached = ERTS_MSG_COMBINED_HFRAG; + ERTS_INIT_HEAP_FRAG(&mp->hfrag, sz, sz); + + if (hpp) + *hpp = &mp->hfrag.mem[0]; + + return mp; +} + +ERTS_GLB_FORCE_INLINE ErtsMessage * +erts_shrink_message(ErtsMessage *mp, Uint sz, Eterm *brefs, Uint brefs_size) +{ + if (sz == 0) { + ErtsMessage *nmp; + if (!mp->data.attached) + return mp; + ASSERT(mp->data.attached == ERTS_MSG_COMBINED_HFRAG); + nmp = erts_alloc_message_ref(); +#ifdef DEBUG + if (brefs && brefs_size) { + int i; + for (i = 0; i < brefs_size; i++) + ASSERT(is_non_value(brefs[i]) || is_immed(brefs[i])); + } +#endif + erts_free(ERTS_ALC_T_MSG, mp); + return nmp; + } + + ASSERT(mp->data.attached == ERTS_MSG_COMBINED_HFRAG); + ASSERT(mp->hfrag.used_size >= sz); + + if (sz >= (mp->hfrag.alloc_size - mp->hfrag.alloc_size / 16)) { + mp->hfrag.used_size = sz; + return mp; + } + + return erts_realloc_shrink_message(mp, sz, brefs, brefs_size); +} + +ERTS_GLB_FORCE_INLINE void erts_free_message(ErtsMessage *mp) +{ + if (mp->data.attached != ERTS_MSG_COMBINED_HFRAG) + erts_free_message_ref(mp); + else + erts_free(ERTS_ALC_T_MSG, mp); +} + +ERTS_GLB_INLINE Uint erts_used_frag_sz(const ErlHeapFragment* bp) { - const ErlHeapFragment *bp; Uint sz = 0; - for (bp = msg->data.heap_frag; bp!=NULL; bp=bp->next) { + for ( ; bp!=NULL; bp=bp->next) { sz += bp->used_size; } return sz; } -ERTS_GLB_INLINE Uint erts_msg_attached_data_size(ErlMessage *msg) +ERTS_GLB_INLINE Uint erts_msg_attached_data_size(ErtsMessage *msg) { ASSERT(msg->data.attached); - if (is_value(ERL_MESSAGE_TERM(msg))) - return erts_msg_used_frag_sz(msg); + if (is_value(ERL_MESSAGE_TERM(msg))) { + ErlHeapFragment *bp; + if (msg->data.attached == ERTS_MSG_COMBINED_HFRAG) + bp = &msg->hfrag; + else + bp = msg->data.heap_frag; + return erts_used_frag_sz(bp); + } else if (msg->data.dist_ext->heap_size < 0) return erts_msg_attached_data_size_aux(msg); else { diff --git a/erts/emulator/beam/erl_monitors.c b/erts/emulator/beam/erl_monitors.c index 244a2b26db..bd899fa2f9 100644 --- a/erts/emulator/beam/erl_monitors.c +++ b/erts/emulator/beam/erl_monitors.c @@ -3,16 +3,17 @@ * * Copyright Ericsson AB 2004-2013. All Rights Reserved. * - * The contents of this file are subject to the Erlang Public License, - * Version 1.1, (the "License"); you may not use this file except in - * compliance with the License. You should have received a copy of the - * Erlang Public License along with this software. If not, it can be - * retrieved online at http://www.erlang.org/. - * - * Software distributed under the License is distributed on an "AS IS" - * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See - * the License for the specific language governing rights and limitations - * under the License. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. * * %CopyrightEnd% */ @@ -355,7 +356,7 @@ void erts_add_monitor(ErtsMonitor **root, Uint type, Eterm ref, Eterm pid, tstack[tpos++] = this; this = &((*this)->right); } else { /* Equal key is an error for monitors */ - erl_exit(1,"Insertion of already present monitor!"); + erts_exit(ERTS_ERROR_EXIT,"Insertion of already present monitor!"); break; } } diff --git a/erts/emulator/beam/erl_monitors.h b/erts/emulator/beam/erl_monitors.h index fb11dbbd22..901ed8940b 100644 --- a/erts/emulator/beam/erl_monitors.h +++ b/erts/emulator/beam/erl_monitors.h @@ -3,16 +3,17 @@ * * Copyright Ericsson AB 2004-2013. All Rights Reserved. * - * The contents of this file are subject to the Erlang Public License, - * Version 1.1, (the "License"); you may not use this file except in - * compliance with the License. You should have received a copy of the - * Erlang Public License along with this software. If not, it can be - * retrieved online at http://www.erlang.org/. - * - * Software distributed under the License is distributed on an "AS IS" - * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See - * the License for the specific language governing rights and limitations - * under the License. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. * * %CopyrightEnd% */ @@ -82,6 +83,7 @@ /* Type tags for monitors */ #define MON_ORIGIN 1 #define MON_TARGET 3 +#define MON_TIME_OFFSET 7 /* Type tags for links */ #define LINK_PID 1 /* ...Or port */ @@ -103,7 +105,7 @@ typedef struct erts_monitor_or_link { typedef struct erts_monitor { struct erts_monitor *left, *right; Sint16 balance; - Uint16 type; /* MON_ORIGIN | MON_TARGET */ + Uint16 type; /* MON_ORIGIN | MON_TARGET | MON_TIME_OFFSET */ Eterm ref; Eterm pid; /* In case of distributed named monitor, this is the nodename atom in MON_ORIGIN process, otherwise a pid or diff --git a/erts/emulator/beam/erl_msacc.c b/erts/emulator/beam/erl_msacc.c new file mode 100644 index 0000000000..71e3fd8b6e --- /dev/null +++ b/erts/emulator/beam/erl_msacc.c @@ -0,0 +1,486 @@ +/* + * %CopyrightBegin% + * + * Copyright Ericsson AB 2014-2015. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * %CopyrightEnd% + */ + +/* + * Description: Microstate accounting. + * + * We keep track of the different states that the + * Erlang VM threads are in, in order to provide + * performance/debugging statistics. There is a + * small overhead in enabling this, but in the big + * scheme of things it should be negligible. + */ + +#ifdef HAVE_CONFIG_H +# include "config.h" +#endif + +#define ERTS_MSACC_STATE_STRINGS 1 + +#include "sys.h" +#include "global.h" +#include "erl_threads.h" +#include "erl_bif_unique.h" +#include "erl_map.h" +#include "erl_msacc.h" + +#if ERTS_ENABLE_MSACC + +static Eterm erts_msacc_gather_stats(ErtsMsAcc *msacc, Eterm **hpp, Uint *szp); +static void erts_msacc_reset(ErtsMsAcc *msacc); +static ErtsMsAcc* get_msacc(void); + +#ifdef USE_THREADS +erts_tsd_key_t ERTS_WRITE_UNLIKELY(erts_msacc_key); +#else +ErtsMsAcc *ERTS_WRITE_UNLIKELY(erts_msacc) = NULL; +#endif +int ERTS_WRITE_UNLIKELY(erts_msacc_enabled); + +static Eterm *erts_msacc_state_atoms = NULL; +static erts_rwmtx_t msacc_mutex; +static ErtsMsAcc *msacc_managed = NULL; +#ifdef USE_THREADS +static ErtsMsAcc *msacc_unmanaged = NULL; +static Uint msacc_unmanaged_count = 0; +#endif + +/* we have to split initiation as atoms are not inited in early init */ +void erts_msacc_early_init(void) { +#ifndef ERTS_MSACC_ALWAYS_ON + erts_msacc_enabled = 0; +#endif + erts_rwmtx_init(&msacc_mutex,"msacc_list_mutex"); +#ifdef USE_THREADS + erts_tsd_key_create(&erts_msacc_key,"erts_msacc_key"); +#else + erts_msacc = NULL; +#endif +} + +void erts_msacc_init(void) { + int i; + erts_msacc_state_atoms = erts_alloc(ERTS_ALC_T_MSACC, + sizeof(Eterm)*ERTS_MSACC_STATE_COUNT); + for (i = 0; i < ERTS_MSACC_STATE_COUNT; i++) { + erts_msacc_state_atoms[i] = am_atom_put(erts_msacc_states[i], + strlen(erts_msacc_states[i])); + } +} + +void erts_msacc_init_thread(char *type, int id, int managed) { + ErtsMsAcc *msacc; + + msacc = erts_alloc(ERTS_ALC_T_MSACC, sizeof(ErtsMsAcc)); + + msacc->type = strdup(type); + msacc->id = make_small(id); + msacc->unmanaged = !managed; + msacc->tid = erts_thr_self(); + msacc->perf_counter = 0; + +#ifdef USE_THREADS + erts_rwmtx_rwlock(&msacc_mutex); + if (!managed) { + erts_mtx_init(&msacc->mtx,"msacc_unmanaged_mutex"); + msacc->next = msacc_unmanaged; + msacc_unmanaged = msacc; + msacc_unmanaged_count++; + ERTS_MSACC_TSD_SET(msacc); + } else { + msacc->next = msacc_managed; + msacc_managed = msacc; + } + erts_rwmtx_rwunlock(&msacc_mutex); +#else + msacc_managed = msacc; +#endif + + erts_msacc_reset(msacc); + +#ifdef ERTS_MSACC_ALWAYS_ON + ERTS_MSACC_TSD_SET(msacc); + msacc->perf_counter = erts_sys_perf_counter(); + msacc->state = ERTS_MSACC_STATE_OTHER; +#endif +} + +/* + * Creates a structure looking like this + * #{ type => scheduler, id => 1, counters => #{ State1 => Counter1 ... StateN => CounterN}} + */ +static +Eterm erts_msacc_gather_stats(ErtsMsAcc *msacc, Eterm **hpp, Uint *szp) { + int i; + Eterm *hp; + Eterm key, state_key, state_map; + Eterm res = THE_NON_VALUE; + flatmap_t *map; + + if (szp) { + *szp += MAP_HEADER_FLATMAP_SZ + 1 + 2*(3); + *szp += MAP_HEADER_FLATMAP_SZ + 1 + 2*(ERTS_MSACC_STATE_COUNT); + for (i = 0; i < ERTS_MSACC_STATE_COUNT; i++) { + (void)erts_bld_sint64(NULL,szp,(Sint64)msacc->perf_counters[i]); +#ifdef ERTS_MSACC_STATE_COUNTERS + (void)erts_bld_uint64(NULL,szp,msacc->state_counters[i]); + *szp += 3; /* tuple to put state+perf counter in */ +#endif + } + } + + if (hpp) { + Eterm counters[ERTS_MSACC_STATE_COUNT]; + hp = *hpp; + for (i = 0; i < ERTS_MSACC_STATE_COUNT; i++) { + Eterm counter = erts_bld_sint64(&hp,NULL,(Sint64)msacc->perf_counters[i]); +#ifdef ERTS_MSACC_STATE_COUNTERS + Eterm counter__ = erts_bld_uint64(&hp,NULL,msacc->state_counters[i]); + counters[i] = TUPLE2(hp,counter,counter__); + hp += 3; +#else + counters[i] = counter; +#endif + } + + key = TUPLE3(hp,am_counters,am_id,am_type); + hp += 4; + + state_key = make_tuple(hp); + hp[0] = make_arityval(ERTS_MSACC_STATE_COUNT); + + for (i = 0; i < ERTS_MSACC_STATE_COUNT; i++) + hp[1+i] = erts_msacc_state_atoms[i]; + hp += 1 + ERTS_MSACC_STATE_COUNT; + + map = (flatmap_t*)hp; + hp += MAP_HEADER_FLATMAP_SZ; + map->thing_word = MAP_HEADER_FLATMAP; + map->size = ERTS_MSACC_STATE_COUNT; + map->keys = state_key; + for (i = 0; i < ERTS_MSACC_STATE_COUNT; i++) + hp[i] = counters[i]; + hp += ERTS_MSACC_STATE_COUNT; + state_map = make_flatmap(map); + + map = (flatmap_t*)hp; + hp += MAP_HEADER_FLATMAP_SZ; + map->thing_word = MAP_HEADER_FLATMAP; + map->size = 3; + map->keys = key; + hp[0] = state_map; + hp[1] = msacc->id; + hp[2] = am_atom_put(msacc->type,strlen(msacc->type)); + hp += 3; + + *hpp = hp; + res = make_flatmap(map); + } + + return res; +} + +typedef struct { + int action; + Process *proc; + Eterm ref; + Eterm ref_heap[REF_THING_SIZE]; + Uint req_sched; + erts_smp_atomic32_t refc; +} ErtsMSAccReq; + +static ErtsMsAcc* get_msacc(void) { + ErtsMsAcc *msacc; + erts_rwmtx_rlock(&msacc_mutex); + msacc = msacc_managed; + while (!erts_equal_tids(msacc->tid,erts_thr_self())) { + msacc = msacc->next; + ASSERT(msacc != NULL); + } + erts_rwmtx_runlock(&msacc_mutex); + return msacc; +} + +static void send_reply(ErtsMsAcc *msacc, ErtsMSAccReq *msaccrp) { + ErtsSchedulerData *esdp = erts_get_scheduler_data(); + Process *rp = msaccrp->proc; + ErtsMessage *msgp = NULL; + Eterm **hpp, *hp; + Eterm ref_copy = NIL, msg; + Uint sz, *szp; + ErlOffHeap *ohp = NULL; + ErtsProcLocks rp_locks = (esdp && msaccrp->req_sched == esdp->no + ? ERTS_PROC_LOCK_MAIN : 0); + + sz = 0; + hpp = NULL; + szp = &sz; + + if (msacc->unmanaged) erts_mtx_lock(&msacc->mtx); + + while (1) { + if (hpp) + ref_copy = STORE_NC(hpp, ohp, msaccrp->ref); + else + *szp += REF_THING_SIZE; + + if (msaccrp->action != ERTS_MSACC_GATHER) + msg = ref_copy; + else { + msg = erts_msacc_gather_stats(msacc, hpp, szp); + msg = erts_bld_tuple(hpp, szp, 2, ref_copy, msg); + } + if (hpp) + break; + + msgp = erts_alloc_message_heap(rp, &rp_locks, sz, &hp, &ohp); + hpp = &hp; + szp = NULL; + } + + if (msacc->unmanaged) erts_mtx_unlock(&msacc->mtx); + + erts_queue_message(rp, &rp_locks, msgp, msg, NIL); + + if (esdp && msaccrp->req_sched == esdp->no) + rp_locks &= ~ERTS_PROC_LOCK_MAIN; + + if (rp_locks) + erts_smp_proc_unlock(rp, rp_locks); + +} + +static void +reply_msacc(void *vmsaccrp) +{ + ErtsMsAcc *msacc = ERTS_MSACC_TSD_GET(); + ErtsMSAccReq *msaccrp = (ErtsMSAccReq *) vmsaccrp; + + ASSERT(!msacc || !msacc->unmanaged); + + if (msaccrp->action == ERTS_MSACC_ENABLE && !msacc) { + msacc = get_msacc(); + + msacc->perf_counter = erts_sys_perf_counter(); + + msacc->state = ERTS_MSACC_STATE_OTHER; + + ERTS_MSACC_TSD_SET(msacc); + + } else if (msaccrp->action == ERTS_MSACC_DISABLE && msacc) { + ERTS_MSACC_TSD_SET(NULL); + } else if (msaccrp->action == ERTS_MSACC_RESET) { + msacc = msacc ? msacc : get_msacc(); + erts_msacc_reset(msacc); + } else if (msaccrp->action == ERTS_MSACC_GATHER && !msacc) { + msacc = get_msacc(); + } + + ASSERT(!msacc || !msacc->unmanaged); + + send_reply(msacc, msaccrp); + + erts_proc_dec_refc(msaccrp->proc); + + if (erts_smp_atomic32_dec_read_nob(&msaccrp->refc) == 0) + erts_free(ERTS_ALC_T_MSACC, vmsaccrp); +} + +static void erts_msacc_reset(ErtsMsAcc *msacc) { + int i; + if (msacc->unmanaged) erts_mtx_lock(&msacc->mtx); + + for (i = 0; i < ERTS_MSACC_STATE_COUNT; i++) { + msacc->perf_counters[i] = 0; +#ifdef ERTS_MSACC_STATE_COUNTERS + msacc->state_counters[i] = 0; +#endif + } + + if (msacc->unmanaged) erts_mtx_unlock(&msacc->mtx); +} + +#endif /* ERTS_ENABLE_MSACC */ + + +/* + * This function is responsible for enabling, disabling, resetting and + * gathering data related to microstate accounting. + * + * Managed threads and unmanaged threads are handled differently. + * - managed threads get a misc_aux job telling them to switch on msacc + * - unmanaged have some fields protected by a mutex that has to be taken + * before any values can be updated + * + * For performance reasons there is also a global value erts_msacc_enabled + * that controls the state of all threads. Statistics gathering is only on + * if erts_msacc_enabled && msacc is true. + */ +Eterm +erts_msacc_request(Process *c_p, int action, Eterm *threads) +{ +#ifdef ERTS_ENABLE_MSACC + ErtsMsAcc *msacc = ERTS_MSACC_TSD_GET(); + ErtsSchedulerData *esdp = ERTS_PROC_GET_SCHDATA(c_p); + Eterm ref; + ErtsMSAccReq *msaccrp; + Eterm *hp; + + +#ifdef ERTS_MSACC_ALWAYS_ON + if (action == ERTS_MSACC_ENABLE || action == ERTS_MSACC_DISABLE) + return THE_NON_VALUE; +#else + /* take care of double enable, and double disable here */ + if (msacc && action == ERTS_MSACC_ENABLE) { + return THE_NON_VALUE; + } else if (!msacc && action == ERTS_MSACC_DISABLE) { + return THE_NON_VALUE; + } +#endif + + ref = erts_make_ref(c_p); + + msaccrp = erts_alloc(ERTS_ALC_T_MSACC, sizeof(ErtsMSAccReq)); + hp = &msaccrp->ref_heap[0]; + + msaccrp->action = action; + msaccrp->proc = c_p; + msaccrp->ref = STORE_NC(&hp, NULL, ref); + msaccrp->req_sched = esdp->no; + +#ifdef ERTS_SMP + *threads = erts_no_schedulers; + *threads += 1; /* aux thread */ +#else + *threads = 1; +#endif + + erts_smp_atomic32_init_nob(&msaccrp->refc,(erts_aint32_t)*threads); + + erts_proc_add_refc(c_p, *threads); + + if (erts_no_schedulers > 1) + erts_schedule_multi_misc_aux_work(1, + erts_no_schedulers, + reply_msacc, + (void *) msaccrp); +#ifdef ERTS_SMP + /* aux thread */ + erts_schedule_misc_aux_work(0, reply_msacc, (void *) msaccrp); +#endif + +#ifdef USE_THREADS + /* Manage unmanaged threads */ + switch (action) { + case ERTS_MSACC_GATHER: { + Uint unmanaged_count; + ErtsMsAcc *msacc, **unmanaged; + int i = 0; + + /* we copy a list of pointers here so that we do not have to have + the msacc_mutex when sending messages */ + erts_rwmtx_rlock(&msacc_mutex); + unmanaged_count = msacc_unmanaged_count; + unmanaged = erts_alloc(ERTS_ALC_T_MSACC, + sizeof(ErtsMsAcc*)*unmanaged_count); + + for (i = 0, msacc = msacc_unmanaged; + i < unmanaged_count; + i++, msacc = msacc->next) { + unmanaged[i] = msacc; + } + erts_rwmtx_runlock(&msacc_mutex); + + for (i = 0; i < unmanaged_count; i++) { + erts_mtx_lock(&unmanaged[i]->mtx); + if (unmanaged[i]->perf_counter) { + ErtsSysPerfCounter perf_counter; + /* if enabled update stats */ + perf_counter = erts_sys_perf_counter(); + unmanaged[i]->perf_counters[unmanaged[i]->state] += + perf_counter - unmanaged[i]->perf_counter; + unmanaged[i]->perf_counter = perf_counter; + } + erts_mtx_unlock(&unmanaged[i]->mtx); + send_reply(unmanaged[i],msaccrp); + } + erts_free(ERTS_ALC_T_MSACC,unmanaged); + /* We have just sent unmanaged_count messages, so bump no of threads */ + *threads += unmanaged_count; + break; + } + case ERTS_MSACC_RESET: { + ErtsMsAcc *msacc; + erts_rwmtx_rlock(&msacc_mutex); + for (msacc = msacc_unmanaged; msacc != NULL; msacc = msacc->next) + erts_msacc_reset(msacc); + erts_rwmtx_runlock(&msacc_mutex); + break; + } + case ERTS_MSACC_ENABLE: { + erts_rwmtx_rlock(&msacc_mutex); + for (msacc = msacc_unmanaged; msacc != NULL; msacc = msacc->next) { + erts_mtx_lock(&msacc->mtx); + msacc->perf_counter = erts_sys_perf_counter(); + /* we assume the unmanaged thread is sleeping */ + msacc->state = ERTS_MSACC_STATE_SLEEP; + erts_mtx_unlock(&msacc->mtx); + } + erts_rwmtx_runlock(&msacc_mutex); + break; + } + case ERTS_MSACC_DISABLE: { + ErtsSysPerfCounter perf_counter; + erts_rwmtx_rlock(&msacc_mutex); + /* make sure to update stats with latest results */ + for (msacc = msacc_unmanaged; msacc != NULL; msacc = msacc->next) { + erts_mtx_lock(&msacc->mtx); + perf_counter = erts_sys_perf_counter(); + msacc->perf_counters[msacc->state] += perf_counter - msacc->perf_counter; + msacc->perf_counter = 0; + erts_mtx_unlock(&msacc->mtx); + } + erts_rwmtx_runlock(&msacc_mutex); + break; + } + default: { ASSERT(0); } + } + +#endif + + *threads = make_small(*threads); + + reply_msacc((void *) msaccrp); + +#ifndef ERTS_MSACC_ALWAYS_ON + /* enable/disable the global value */ + if (action == ERTS_MSACC_ENABLE) { + erts_msacc_enabled = 1; + } else if (action == ERTS_MSACC_DISABLE) { + erts_msacc_enabled = 0; + } +#endif + + return ref; +#else + return THE_NON_VALUE; +#endif +} diff --git a/erts/emulator/beam/erl_msacc.h b/erts/emulator/beam/erl_msacc.h new file mode 100644 index 0000000000..284388f7aa --- /dev/null +++ b/erts/emulator/beam/erl_msacc.h @@ -0,0 +1,409 @@ +/* + * %CopyrightBegin% + * + * Copyright Ericsson AB 2014-2015. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * %CopyrightEnd% + */ + +#ifndef ERL_MSACC_H__ +#define ERL_MSACC_H__ + +/* Can be enabled/disabled via configure */ +#if ERTS_ENABLE_MSACC == 2 +#define ERTS_MSACC_EXTENDED_STATES 1 +#endif + +/* Uncomment this to also count the number of + transitions to a state. This will add a count + to the counter map. */ +/* #define ERTS_MSACC_STATE_COUNTERS 1 */ + +/* Uncomment this to make msacc to always be on, + this reduces overhead a little bit when profiling */ +/* #define ERTS_MSACC_ALWAYS_ON 1 */ + +#define ERTS_MSACC_DISABLE 0 +#define ERTS_MSACC_ENABLE 1 +#define ERTS_MSACC_RESET 2 +#define ERTS_MSACC_GATHER 3 + +/* + * When adding a new state, you have to: + * * Add it here + * * Increment ERTS_MSACC_STATE_COUNT + * * Add string value to erts_msacc_states + * * Have to be in alphabetical order! + * * Only add states to the non-extended section after + * careful benchmarking to make sure the overhead + * when disabled is minimal. + */ + +#ifndef ERTS_MSACC_EXTENDED_STATES +#define ERTS_MSACC_STATE_AUX 0 +#define ERTS_MSACC_STATE_CHECK_IO 1 +#define ERTS_MSACC_STATE_EMULATOR 2 +#define ERTS_MSACC_STATE_GC 3 +#define ERTS_MSACC_STATE_OTHER 4 +#define ERTS_MSACC_STATE_PORT 5 +#define ERTS_MSACC_STATE_SLEEP 6 + +#define ERTS_MSACC_STATE_COUNT 7 + +#if ERTS_MSACC_STATE_STRINGS && ERTS_ENABLE_MSACC +static char *erts_msacc_states[] = { + "aux", + "check_io", + "emulator", + "gc", + "other", + "port", + "sleep" +}; +#endif + +#else + +#define ERTS_MSACC_STATE_ALLOC 0 +#define ERTS_MSACC_STATE_AUX 1 +#define ERTS_MSACC_STATE_BIF 2 +#define ERTS_MSACC_STATE_BUSY_WAIT 3 +#define ERTS_MSACC_STATE_CHECK_IO 4 +#define ERTS_MSACC_STATE_EMULATOR 5 +#define ERTS_MSACC_STATE_ETS 6 +#define ERTS_MSACC_STATE_GC 7 +#define ERTS_MSACC_STATE_GC_FULL 8 +#define ERTS_MSACC_STATE_NIF 9 +#define ERTS_MSACC_STATE_OTHER 10 +#define ERTS_MSACC_STATE_PORT 11 +#define ERTS_MSACC_STATE_SEND 12 +#define ERTS_MSACC_STATE_SLEEP 13 +#define ERTS_MSACC_STATE_TIMERS 14 + +#define ERTS_MSACC_STATE_COUNT 15 + +#if ERTS_MSACC_STATE_STRINGS +static char *erts_msacc_states[] = { + "alloc", + "aux", + "bif", + "busy_wait", + "check_io", + "emulator", + "ets", + "gc", + "gc_full", + "nif", + "other", + "port", + "send", + "sleep", + "timers" +}; +#endif + +#endif + +typedef struct erl_msacc_t_ ErtsMsAcc; + +struct erl_msacc_t_ { + + /* the the values below are protected by mtx iff unmanaged = 1 */ + ErtsSysPerfCounter perf_counter; + ErtsSysPerfCounter perf_counters[ERTS_MSACC_STATE_COUNT]; +#ifdef ERTS_MSACC_STATE_COUNTERS + Uint64 state_counters[ERTS_MSACC_STATE_COUNT]; +#endif + Uint state; + + /* protected by msacc_mutex in erl_msacc.c, and should be constant */ + int unmanaged; + erts_mtx_t mtx; + ErtsMsAcc *next; + erts_tid_t tid; + Eterm id; + char *type; +}; + +#if ERTS_ENABLE_MSACC + +#define ERTS_MSACC_INLINE ERTS_GLB_INLINE + +#ifdef USE_THREADS +extern erts_tsd_key_t erts_msacc_key; +#else +extern ErtsMsAcc *erts_msacc; +#endif + +#ifdef ERTS_MSACC_ALWAYS_ON +#define erts_msacc_enabled 1 +#else +extern int erts_msacc_enabled; +#endif + +#ifdef USE_THREADS +#define ERTS_MSACC_TSD_GET() erts_tsd_get(erts_msacc_key) +#define ERTS_MSACC_TSD_SET(tsd) erts_tsd_set(erts_msacc_key,tsd) +#else +#define ERTS_MSACC_TSD_GET() erts_msacc +#define ERTS_MSACC_TSD_SET(tsd) erts_msacc = tsd +#endif + +void erts_msacc_early_init(void); +void erts_msacc_init(void); +void erts_msacc_init_thread(char *type, int id, int liberty); + +/* The defines below are used to instrument the vm code + * with different state changes. There are two variants + * of each define. One that has a cached ErtsMsAcc * + * that it can use, and one that does not. + * The cached values are necessary to have in order to get + * low enough overhead when running without msacc enabled. + * + * The two most common patterns to use the function with are: + * + * ERTS_MSACC_PUSH_AND_SET_STATE(ERTS_MSACC_STATE_NEWSTATE); + * ... call other function in new state ... + * ERTS_MSACC_POP_STATE(); + * + * Note that the erts_msacc_push* function declare new variables, so + * to conform with C89 we have to call it in the beginning of a function. + * We might not want to change state it the beginning though, so we use this: + * + * ERTS_MSACC_PUSH_STATE(); + * ... some other code ... + * ERTS_MSACC_SET_STATE_CACHED(ERTS_MSACC_STATE_NEWSTATE); + * ... call other function in new state ... + * ERTS_MSACC_POP_STATE(); + * + * Notice that we used the cached version of set_state as push_state already + * read the erts_msacc_enabled to the cache. + * + * Most macros also have other variants with the suffix _m which means that + * they are known to only be called in managed threads, or with the _x suffix + * which means that it should only be used in an emulator compiled with + * extended states. + * + * Here is a listing of the entire api: + * + * void ERTS_MSACC_DECLARE_CACHE() + * void ERTS_MSACC_UPDATE_CACHE() + * void ERTS_MSACC_IS_ENABLED() + * void ERTS_MSACC_IS_ENABLED_CACHED() + * + * void ERTS_MSACC_PUSH_STATE() + * void ERTS_MSACC_SET_STATE(int state) + * void ERTS_MSACC_PUSH_AND_SET_STATE(int state) + * + * void ERTS_MSACC_PUSH_STATE_CACHED() + * void ERTS_MSACC_SET_STATE_CACHED(int state) + * void ERTS_MSACC_PUSH_AND_SET_STATE_CACHED(int state) + * void ERTS_MSACC_POP_STATE() + * + * void ERTS_MSACC_PUSH_STATE_M() + * void ERTS_MSACC_PUSH_STATE_CACHED_M() + * void ERTS_MSACC_SET_STATE_CACHED_M(int state) + * void ERTS_MSACC_SET_STATE_M(int state) + * void ERTS_MSACC_POP_STATE_M() + * void ERTS_MSACC_PUSH_AND_SET_STATE_M(int state) + * + * Most functions are also available with an _x suffix that are only enabled + * when using the extra states. If they are not, just add them to the end + * of this file. + */ + +/* cache handling functions */ +#define ERTS_MSACC_IS_ENABLED() ERTS_UNLIKELY(erts_msacc_enabled) +#define ERTS_MSACC_DECLARE_CACHE() \ + ErtsMsAcc *ERTS_MSACC_UPDATE_CACHE(); \ + ERTS_DECLARE_DUMMY(Uint __erts_msacc_state) = ERTS_MSACC_STATE_OTHER; +#define ERTS_MSACC_IS_ENABLED_CACHED() ERTS_UNLIKELY(__erts_msacc_cache != NULL) +#define ERTS_MSACC_UPDATE_CACHE() \ + __erts_msacc_cache = erts_msacc_enabled ? ERTS_MSACC_TSD_GET() : NULL + + +/* The defines below implicitly declare and load a new cache */ +#define ERTS_MSACC_PUSH_STATE() \ + ERTS_MSACC_DECLARE_CACHE(); \ + ERTS_MSACC_PUSH_STATE_CACHED() +#define ERTS_MSACC_SET_STATE(state) \ + ERTS_MSACC_DECLARE_CACHE(); \ + ERTS_MSACC_SET_STATE_CACHED(state) +#define ERTS_MSACC_PUSH_AND_SET_STATE(state) \ + ERTS_MSACC_PUSH_STATE(); ERTS_MSACC_SET_STATE_CACHED(state) + +/* The defines below need an already declared cache to work */ +#define ERTS_MSACC_PUSH_STATE_CACHED() \ + __erts_msacc_state = ERTS_MSACC_IS_ENABLED_CACHED() ? \ + erts_msacc_get_state_um__(__erts_msacc_cache) : ERTS_MSACC_STATE_OTHER +#define ERTS_MSACC_SET_STATE_CACHED(state) \ + if (ERTS_MSACC_IS_ENABLED_CACHED()) \ + erts_msacc_set_state_um__(__erts_msacc_cache, state, 1) +#define ERTS_MSACC_PUSH_AND_SET_STATE_CACHED(state) \ + ERTS_MSACC_PUSH_STATE_CACHED(); ERTS_MSACC_SET_STATE_CACHED(state) +#define ERTS_MSACC_POP_STATE() \ + if (ERTS_MSACC_IS_ENABLED_CACHED()) \ + erts_msacc_set_state_um__(__erts_msacc_cache, __erts_msacc_state, 0) + +/* Only use these defines when we know that we have in a managed thread */ +#define ERTS_MSACC_PUSH_STATE_M() \ + ERTS_MSACC_DECLARE_CACHE(); \ + ERTS_MSACC_PUSH_STATE_CACHED_M() +#define ERTS_MSACC_PUSH_STATE_CACHED_M() \ + __erts_msacc_state = ERTS_MSACC_IS_ENABLED_CACHED() ? \ + erts_msacc_get_state_m__(__erts_msacc_cache) : ERTS_MSACC_STATE_OTHER +#define ERTS_MSACC_SET_STATE_M(state) \ + ERTS_MSACC_DECLARE_CACHE(); \ + ERTS_MSACC_SET_STATE_CACHED_M(state) +#define ERTS_MSACC_SET_STATE_CACHED_M(state) \ + if (ERTS_MSACC_IS_ENABLED_CACHED()) \ + erts_msacc_set_state_m__(__erts_msacc_cache, state, 1) +#define ERTS_MSACC_POP_STATE_M() \ + if (ERTS_MSACC_IS_ENABLED_CACHED()) \ + erts_msacc_set_state_m__(__erts_msacc_cache, __erts_msacc_state, 0) +#define ERTS_MSACC_PUSH_AND_SET_STATE_M(state) \ + ERTS_MSACC_PUSH_STATE_M(); ERTS_MSACC_SET_STATE_CACHED_M(state) + +ERTS_MSACC_INLINE +void erts_msacc_set_state_um__(ErtsMsAcc *msacc,Uint state,int increment); +ERTS_MSACC_INLINE +void erts_msacc_set_state_m__(ErtsMsAcc *msacc,Uint state,int increment); + +ERTS_MSACC_INLINE +Uint erts_msacc_get_state_um__(ErtsMsAcc *msacc); +ERTS_MSACC_INLINE +Uint erts_msacc_get_state_m__(ErtsMsAcc *msacc); + + +#if ERTS_GLB_INLINE_INCL_FUNC_DEF + +ERTS_MSACC_INLINE +Uint erts_msacc_get_state_um__(ErtsMsAcc *msacc) { + Uint state; + if (msacc->unmanaged) + erts_mtx_lock(&msacc->mtx); + state = msacc->state; + if (msacc->unmanaged) + erts_mtx_unlock(&msacc->mtx); + return state; +} + +ERTS_MSACC_INLINE +Uint erts_msacc_get_state_m__(ErtsMsAcc *msacc) { + return msacc->state; +} + +ERTS_MSACC_INLINE +void erts_msacc_set_state_um__(ErtsMsAcc *msacc, Uint new_state, int increment) { + if (ERTS_UNLIKELY(msacc->unmanaged)) { + erts_mtx_lock(&msacc->mtx); + msacc->state = new_state; + if (ERTS_LIKELY(!msacc->perf_counter)) { + erts_mtx_unlock(&msacc->mtx); + return; + } + } + + erts_msacc_set_state_m__(msacc,new_state,increment); + + if (ERTS_UNLIKELY(msacc->unmanaged)) + erts_mtx_unlock(&msacc->mtx); +} + +ERTS_MSACC_INLINE +void erts_msacc_set_state_m__(ErtsMsAcc *msacc, Uint new_state, int increment) { + ErtsSysPerfCounter prev_perf_counter; + Sint64 diff; + + if (new_state == msacc->state) + return; + + prev_perf_counter = msacc->perf_counter; + msacc->perf_counter = erts_sys_perf_counter(); + diff = msacc->perf_counter - prev_perf_counter; + ASSERT(diff >= 0); + msacc->perf_counters[msacc->state] += diff; +#ifdef ERTS_MSACC_STATE_COUNTERS + msacc->state_counters[new_state] += increment; +#endif + msacc->state = new_state; +} + +#endif /* ERTS_GLB_INLINE_INCL_FUNC_DEF */ + +#else + +#define ERTS_MSACC_IS_ENABLED() 0 +#define erts_msacc_early_init() +#define erts_msacc_init() +#define erts_msacc_init_thread(type, id, liberty) +#define ERTS_MSACC_PUSH_STATE() +#define ERTS_MSACC_PUSH_STATE_CACHED() +#define ERTS_MSACC_POP_STATE() +#define ERTS_MSACC_SET_STATE(state) +#define ERTS_MSACC_SET_STATE_CACHED(state) +#define ERTS_MSACC_PUSH_AND_SET_STATE(state) +#define ERTS_MSACC_PUSH_AND_SET_STATE_CACHED(state) +#define ERTS_MSACC_UPDATE_CACHE() +#define ERTS_MSACC_IS_ENABLED_CACHED() +#define ERTS_MSACC_DECLARE_CACHE() +#define ERTS_MSACC_PUSH_STATE_M() +#define ERTS_MSACC_PUSH_STATE_CACHED_M() +#define ERTS_MSACC_SET_STATE_CACHED_M(state) +#define ERTS_MSACC_POP_STATE_M() +#define ERTS_MSACC_PUSH_AND_SET_STATE_M(state) + + +#endif /* ERTS_ENABLE_MSACC */ + +#ifndef ERTS_MSACC_EXTENDED_STATES + +#define ERTS_MSACC_PUSH_STATE_X() +#define ERTS_MSACC_POP_STATE_X() +#define ERTS_MSACC_SET_STATE_X(state) +#define ERTS_MSACC_SET_STATE_M_X(state) +#define ERTS_MSACC_SET_STATE_CACHED_X(state) +#define ERTS_MSACC_PUSH_AND_SET_STATE_X(state) +#define ERTS_MSACC_PUSH_AND_SET_STATE_CACHED_X(state) +#define ERTS_MSACC_UPDATE_CACHE_X() +#define ERTS_MSACC_IS_ENABLED_CACHED_X() 0 +#define ERTS_MSACC_DECLARE_CACHE_X() +#define ERTS_MSACC_PUSH_STATE_M_X() +#define ERTS_MSACC_PUSH_STATE_CACHED_M_X() +#define ERTS_MSACC_SET_STATE_CACHED_M_X(state) +#define ERTS_MSACC_POP_STATE_M_X() +#define ERTS_MSACC_PUSH_AND_SET_STATE_M_X(state) + +#else + +#define ERTS_MSACC_PUSH_STATE_X() ERTS_MSACC_PUSH_STATE() +#define ERTS_MSACC_POP_STATE_X() ERTS_MSACC_POP_STATE() +#define ERTS_MSACC_SET_STATE_X(state) ERTS_MSACC_SET_STATE(state) +#define ERTS_MSACC_SET_STATE_M_X(state) ERTS_MSACC_SET_STATE_M(state) +#define ERTS_MSACC_SET_STATE_CACHED_X(state) ERTS_MSACC_SET_STATE_CACHED(state) +#define ERTS_MSACC_PUSH_AND_SET_STATE_X(state) ERTS_MSACC_PUSH_AND_SET_STATE(state) +#define ERTS_MSACC_PUSH_AND_SET_STATE_CACHED_X(state) ERTS_MSACC_PUSH_AND_SET_STATE_CACHED(state) +#define ERTS_MSACC_UPDATE_CACHE_X() ERTS_MSACC_UPDATE_CACHE() +#define ERTS_MSACC_IS_ENABLED_CACHED_X() ERTS_MSACC_IS_ENABLED_CACHED() +#define ERTS_MSACC_DECLARE_CACHE_X() ERTS_MSACC_DECLARE_CACHE() +#define ERTS_MSACC_PUSH_STATE_M_X() ERTS_MSACC_PUSH_STATE_M() +#define ERTS_MSACC_PUSH_STATE_CACHED_M_X() ERTS_MSACC_PUSH_STATE_CACHED_M() +#define ERTS_MSACC_SET_STATE_CACHED_M_X(state) ERTS_MSACC_SET_STATE_CACHED_M(state) +#define ERTS_MSACC_POP_STATE_M_X() ERTS_MSACC_POP_STATE_M() +#define ERTS_MSACC_PUSH_AND_SET_STATE_M_X(state) ERTS_MSACC_PUSH_AND_SET_STATE_M(state) + +#endif /* !ERTS_MSACC_EXTENDED_STATES */ + +#endif /* ERL_MSACC_H__ */ diff --git a/erts/emulator/beam/erl_mtrace.c b/erts/emulator/beam/erl_mtrace.c index c8bb126687..fdaf02f37a 100644 --- a/erts/emulator/beam/erl_mtrace.c +++ b/erts/emulator/beam/erl_mtrace.c @@ -3,16 +3,17 @@ * * Copyright Ericsson AB 2003-2013. All Rights Reserved. * - * The contents of this file are subject to the Erlang Public License, - * Version 1.1, (the "License"); you may not use this file except in - * compliance with the License. You should have received a copy of the - * Erlang Public License along with this software. If not, it can be - * retrieved online at http://www.erlang.org/. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Software distributed under the License is distributed on an "AS IS" - * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See - * the License for the specific language governing rights and limitations - * under the License. + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. * * %CopyrightEnd% */ @@ -627,7 +628,7 @@ erts_mtrace_install_wrapper_functions(void) if (erts_mtrace_enabled) { int i; /* Install trace functions */ - ASSERT(sizeof(erts_allctrs) == sizeof(real_allctrs)); + ERTS_CT_ASSERT(sizeof(erts_allctrs) == sizeof(real_allctrs)); sys_memcpy((void *) real_allctrs, (void *) erts_allctrs, diff --git a/erts/emulator/beam/erl_mtrace.h b/erts/emulator/beam/erl_mtrace.h index 204543ddb0..b34eab3c4a 100644 --- a/erts/emulator/beam/erl_mtrace.h +++ b/erts/emulator/beam/erl_mtrace.h @@ -3,16 +3,17 @@ * * Copyright Ericsson AB 2003-2009. All Rights Reserved. * - * The contents of this file are subject to the Erlang Public License, - * Version 1.1, (the "License"); you may not use this file except in - * compliance with the License. You should have received a copy of the - * Erlang Public License along with this software. If not, it can be - * retrieved online at http://www.erlang.org/. - * - * Software distributed under the License is distributed on an "AS IS" - * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See - * the License for the specific language governing rights and limitations - * under the License. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. * * %CopyrightEnd% */ diff --git a/erts/emulator/beam/erl_nif.c b/erts/emulator/beam/erl_nif.c index ff551ea3af..d3030070b7 100644 --- a/erts/emulator/beam/erl_nif.c +++ b/erts/emulator/beam/erl_nif.c @@ -3,16 +3,17 @@ * * Copyright Ericsson AB 2009-2014. All Rights Reserved. * - * The contents of this file are subject to the Erlang Public License, - * Version 1.1, (the "License"); you may not use this file except in - * compliance with the License. You should have received a copy of the - * Erlang Public License along with this software. If not, it can be - * retrieved online at http://www.erlang.org/. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Software distributed under the License is distributed on an "AS IS" - * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See - * the License for the specific language governing rights and limitations - * under the License. + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. * * %CopyrightEnd% */ @@ -36,6 +37,7 @@ #include "erl_thr_progress.h" #include "dtrace-wrapper.h" #include "erl_process.h" +#include "erl_bif_unique.h" #if defined(USE_DYNAMIC_TRACE) && (defined(USE_DTRACE) || defined(USE_SYSTEMTAP)) #define HAVE_USE_DTRACE 1 #endif @@ -127,6 +129,7 @@ void erts_pre_nif(ErlNifEnv* env, Process* p, struct erl_module_nif* mod_nif) env->heap_frag = NULL; env->fpe_was_unmasked = erts_block_fpe(); env->tmp_obj_list = NULL; + env->exception_thrown = 0; } static void pre_nif_noproc(ErlNifEnv* env, struct erl_module_nif* mod_nif) @@ -206,6 +209,7 @@ static void flush_env(ErlNifEnv* env) */ static void cache_env(ErlNifEnv* env) { + env->heap_frag = MBUF(env->proc); if (env->heap_frag == NULL) { ASSERT(env->hp_end == HEAP_LIMIT(env->proc)); ASSERT(env->hp <= HEAP_TOP(env->proc)); @@ -213,10 +217,6 @@ static void cache_env(ErlNifEnv* env) env->hp = HEAP_TOP(env->proc); } else { - ASSERT(env->hp_end != HEAP_LIMIT(env->proc)); - ASSERT(env->hp_end - env->hp <= env->heap_frag->alloc_size); - env->heap_frag = MBUF(env->proc); - ASSERT(env->heap_frag != NULL); env->hp = env->heap_frag->mem + env->heap_frag->used_size; env->hp_end = env->heap_frag->mem + env->heap_frag->alloc_size; } @@ -311,6 +311,7 @@ int enif_send(ErlNifEnv* env, const ErlNifPid* to_pid, ErtsProcLocks rp_locks = 0; Process* rp; Process* c_p; + ErtsMessage *mp; ErlHeapFragment* frags; Eterm receiver = to_pid->pid; int flush_me = 0; @@ -327,14 +328,14 @@ int enif_send(ErlNifEnv* env, const ErlNifPid* to_pid, #ifdef ERTS_SMP c_p = NULL; #else - erl_exit(ERTS_ABORT_EXIT,"enif_send: env==NULL on non-SMP VM"); + erts_exit(ERTS_ABORT_EXIT,"enif_send: env==NULL on non-SMP VM"); #endif } rp = (scheduler ? erts_proc_lookup(receiver) : erts_pid2proc_opt(c_p, ERTS_PROC_LOCK_MAIN, - receiver, rp_locks, ERTS_P2P_FLG_SMP_INC_REFC)); + receiver, rp_locks, ERTS_P2P_FLG_INC_REFC)); if (rp == NULL) { ASSERT(env == NULL || receiver != c_p->common.id); return 0; @@ -344,7 +345,7 @@ int enif_send(ErlNifEnv* env, const ErlNifPid* to_pid, ASSERT(frags == MBUF(&menv->phony_proc)); if (frags != NULL) { /* Move all offheap's from phony proc to the first fragment. - Quick and dirty, but erts_move_msg_mbuf_to_heap doesn't care. */ + Quick and dirty... */ ASSERT(!is_offheap(&frags->off_heap)); frags->off_heap = MSO(&menv->phony_proc); clear_offheap(&MSO(&menv->phony_proc)); @@ -356,30 +357,61 @@ int enif_send(ErlNifEnv* env, const ErlNifPid* to_pid, if (flush_me) { flush_env(env); /* Needed for ERTS_HOLE_CHECK */ } - erts_queue_message(rp, &rp_locks, frags, msg, am_undefined -#ifdef USE_VM_PROBES - , NIL -#endif - ); + mp = erts_alloc_message(0, NULL); + mp->data.heap_frag = frags; + erts_queue_message(rp, &rp_locks, mp, msg, am_undefined); if (c_p == rp) rp_locks &= ~ERTS_PROC_LOCK_MAIN; if (rp_locks) erts_smp_proc_unlock(rp, rp_locks); if (!scheduler) - erts_smp_proc_dec_refc(rp); + erts_proc_dec_refc(rp); if (flush_me) { cache_env(env); } return 1; } +int +enif_port_command(ErlNifEnv *env, const ErlNifPort* to_port, + ErlNifEnv *msg_env, ERL_NIF_TERM msg) +{ + + ErtsSchedulerData *esdp = erts_get_scheduler_data(); + int scheduler = esdp ? esdp->no : 0; + Port *prt; + + if (scheduler == 0 || !env) + return 0; + + prt = erts_port_lookup(to_port->port_id, + (erts_port_synchronous_ops + ? ERTS_PORT_SFLGS_INVALID_DRIVER_LOOKUP + : ERTS_PORT_SFLGS_INVALID_LOOKUP)); + + if (!prt) + return 0; + + return erts_port_output_async(prt, env->proc->common.id, msg); +} + ERL_NIF_TERM enif_make_copy(ErlNifEnv* dst_env, ERL_NIF_TERM src_term) { Uint sz; Eterm* hp; +#ifdef SHCOPY + erts_shcopy_t info; + INITIALIZE_SHCOPY(info); + sz = copy_shared_calculate(src_term, &info); + hp = alloc_heap(dst_env, sz); + src_term = copy_shared_perform(src_term, sz, &info, &hp, &MSO(dst_env->proc)); + DESTROY_SHCOPY(info); + return src_term; +#else sz = size_object(src_term); hp = alloc_heap(dst_env, sz); return copy_struct(src_term, sz, &hp, &MSO(dst_env->proc)); +#endif } @@ -392,12 +424,28 @@ static int is_offheap(const ErlOffHeap* oh) ErlNifPid* enif_self(ErlNifEnv* caller_env, ErlNifPid* pid) { + if (caller_env->proc->common.id == ERTS_INVALID_PID) + return NULL; pid->pid = caller_env->proc->common.id; return pid; } + int enif_get_local_pid(ErlNifEnv* env, ERL_NIF_TERM term, ErlNifPid* pid) { - return is_internal_pid(term) ? (pid->pid=term, 1) : 0; + if (is_internal_pid(term)) { + pid->pid=term; + return 1; + } + return 0; +} + +int enif_get_local_port(ErlNifEnv* env, ERL_NIF_TERM term, ErlNifPort* port) +{ + if (is_internal_port(term)) { + port->port_id=term; + return 1; + } + return 0; } int enif_is_atom(ErlNifEnv* env, ERL_NIF_TERM term) @@ -447,7 +495,7 @@ int enif_is_list(ErlNifEnv* env, ERL_NIF_TERM term) int enif_is_exception(ErlNifEnv* env, ERL_NIF_TERM term) { - return term == THE_NON_VALUE; + return env->exception_thrown && term == THE_NON_VALUE; } int enif_is_number(ErlNifEnv* env, ERL_NIF_TERM term) @@ -472,6 +520,18 @@ int enif_inspect_binary(ErlNifEnv* env, Eterm bin_term, ErlNifBinary* bin) struct enif_tmp_obj_t* tmp; byte* raw_ptr; }u; + + if (is_boxed(bin_term) && *binary_val(bin_term) == HEADER_SUB_BIN) { + ErlSubBin* sb = (ErlSubBin*) binary_val(bin_term); + if (sb->is_writable) { + ProcBin* pb = (ProcBin*) binary_val(sb->orig); + ASSERT(pb->thing_word == HEADER_PROC_BIN); + if (pb->flags) { + erts_emasculate_writable_binary(pb); + sb->is_writable = 0; + } + } + } u.tmp = NULL; bin->data = erts_get_aligned_binary_bytes_extra(bin_term, &u.raw_ptr, allocator, sizeof(struct enif_tmp_obj_t)); @@ -539,9 +599,7 @@ int enif_alloc_binary(size_t size, ErlNifBinary* bin) if (refbin == NULL) { return 0; /* The NIF must take action */ } - refbin->flags = BIN_FLAG_DRV; /* BUGBUG: Flag? */ erts_refc_init(&refbin->refc, 1); - refbin->orig_size = (SWord) size; bin->size = size; bin->data = (unsigned char*) refbin->orig_bytes; @@ -561,7 +619,6 @@ int enif_realloc_binary(ErlNifBinary* bin, size_t size) if (!newbin) { return 0; } - newbin->orig_size = size; bin->ref_bin = newbin; bin->data = (unsigned char*) newbin->orig_bytes; bin->size = size; @@ -601,6 +658,68 @@ unsigned char* enif_make_new_binary(ErlNifEnv* env, size_t size, return binary_bytes(*termp); } +int enif_term_to_binary(ErlNifEnv *dst_env, ERL_NIF_TERM term, + ErlNifBinary *bin) +{ + Sint size; + byte *bp; + Binary* refbin; + + size = erts_encode_ext_size(term); + if (!enif_alloc_binary(size, bin)) + return 0; + + refbin = bin->ref_bin; + + bp = bin->data; + + erts_encode_ext(term, &bp); + + bin->size = bp - bin->data; + refbin->orig_size = bin->size; + + ASSERT(bin->data + bin->size == bp); + + return 1; +} + +size_t enif_binary_to_term(ErlNifEnv *dst_env, + const unsigned char* data, + size_t data_sz, + ERL_NIF_TERM *term, + ErlNifBinaryToTerm opts) +{ + Sint size; + ErtsHeapFactory factory; + byte *bp = (byte*) data; + + ERTS_CT_ASSERT(ERL_NIF_BIN2TERM_SAFE == ERTS_DIST_EXT_BTT_SAFE); + + if (opts & ~ERL_NIF_BIN2TERM_SAFE) { + return 0; + } + if ((size = erts_decode_ext_size(bp, data_sz)) < 0) + return 0; + + if (size > 0) { + flush_env(dst_env); + erts_factory_proc_prealloc_init(&factory, dst_env->proc, size); + } else { + erts_factory_dummy_init(&factory); + } + + *term = erts_decode_ext(&factory, &bp, (Uint32)opts); + + if (is_non_value(*term)) { + return 0; + } + erts_factory_close(&factory); + cache_env(dst_env); + + ASSERT(bp > data); + return bp - data; +} + int enif_is_identical(Eterm lhs, Eterm rhs) { return EQ(lhs,rhs); @@ -730,7 +849,22 @@ Eterm enif_make_sub_binary(ErlNifEnv* env, ERL_NIF_TERM bin_term, Eterm enif_make_badarg(ErlNifEnv* env) { - BIF_ERROR(env->proc, BADARG); + return enif_raise_exception(env, am_badarg); +} + +Eterm enif_raise_exception(ErlNifEnv* env, ERL_NIF_TERM reason) +{ + env->exception_thrown = 1; + env->proc->fvalue = reason; + BIF_ERROR(env->proc, EXC_ERROR); +} + +int enif_has_pending_exception(ErlNifEnv* env, ERL_NIF_TERM* reason) +{ + if (env->exception_thrown && reason != NULL) { + *reason = env->proc->fvalue; + } + return env->exception_thrown; } int enif_get_atom(ErlNifEnv* env, Eterm atom, char* buf, unsigned len, @@ -875,8 +1009,13 @@ int enif_get_list_cell(ErlNifEnv* env, Eterm term, Eterm* head, Eterm* tail) int enif_get_list_length(ErlNifEnv* env, Eterm term, unsigned* len) { - if (is_not_list(term) && is_not_nil(term)) return 0; - *len = erts_list_length(term); + Sint i; + Uint u; + + if ((i = erts_list_length(term)) < 0) return 0; + u = (Uint)i; + if ((unsigned)u != u) return 0; + *len = u; return 1; } @@ -952,8 +1091,12 @@ ERL_NIF_TERM enif_make_uint64(ErlNifEnv* env, ErlNifUInt64 i) ERL_NIF_TERM enif_make_double(ErlNifEnv* env, double d) { - Eterm* hp = alloc_heap(env,FLOAT_SIZE_OBJECT); + Eterm* hp; FloatDef f; + + if (!erts_isfinite(d)) + return enif_make_badarg(env); + hp = alloc_heap(env,FLOAT_SIZE_OBJECT); f.fd = d; PUT_DOUBLE(f, hp); return make_float(hp); @@ -966,6 +1109,8 @@ ERL_NIF_TERM enif_make_atom(ErlNifEnv* env, const char* name) ERL_NIF_TERM enif_make_atom_len(ErlNifEnv* env, const char* name, size_t len) { + if (len > MAX_ATOM_CHARACTERS) + return enif_make_badarg(env); return erts_atom_put((byte*)name, len, ERTS_ATOM_ENC_LATIN1, 1); } @@ -979,6 +1124,8 @@ int enif_make_existing_atom_len(ErlNifEnv* env, const char* name, size_t len, ERL_NIF_TERM* atom, ErlNifCharEncoding encoding) { ASSERT(encoding == ERL_NIF_LATIN1); + if (len > MAX_ATOM_CHARACTERS) + return 0; return erts_atom_get(name, len, atom, ERTS_ATOM_ENC_LATIN1); } @@ -1109,6 +1256,103 @@ int enif_make_reverse_list(ErlNifEnv* env, ERL_NIF_TERM term, ERL_NIF_TERM *list return 1; } +int enif_is_process_alive(ErlNifEnv* env, ErlNifPid *proc) +{ + ErtsProcLocks rp_locks = 0; /* We don't need any locks, + just to check if it is alive */ + Eterm target = proc->pid; + Process* rp; + Process* c_p; + int scheduler = erts_get_scheduler_id() != 0; + + if (env != NULL) { + c_p = env->proc; + if (target == c_p->common.id) { + /* We are alive! */ + return 1; + } + } + else { +#ifdef ERTS_SMP + c_p = NULL; +#else + erts_exit(ERTS_ABORT_EXIT,"enif_is_process_alive: " + "env==NULL on non-SMP VM"); +#endif + } + + rp = (scheduler + ? erts_proc_lookup(target) + : erts_pid2proc_opt(c_p, ERTS_PROC_LOCK_MAIN, + target, rp_locks, ERTS_P2P_FLG_INC_REFC)); + if (rp == NULL) { + ASSERT(env == NULL || target != c_p->common.id); + return 0; + } else { + if (!scheduler) + erts_proc_dec_refc(rp); + return 1; + } +} + +int enif_is_port_alive(ErlNifEnv *env, ErlNifPort *port) +{ + /* only allowed if called from scheduler */ + if (erts_get_scheduler_id() == 0) + erts_exit(ERTS_ABORT_EXIT,"enif_is_port_alive: called from non-scheduler"); + + return erts_port_lookup( + port->port_id, + (erts_port_synchronous_ops + ? ERTS_PORT_SFLGS_INVALID_DRIVER_LOOKUP + : ERTS_PORT_SFLGS_INVALID_LOOKUP)) != NULL; +} + +ERL_NIF_TERM +enif_now_time(ErlNifEnv *env) +{ + Uint mega, sec, micro; + Eterm *hp; + get_now(&mega, &sec, µ); + hp = alloc_heap(env, 4); + return TUPLE3(hp, make_small(mega), make_small(sec), make_small(micro)); +} + +ERL_NIF_TERM +enif_cpu_time(ErlNifEnv *env) +{ +#ifdef HAVE_ERTS_NOW_CPU + Uint mega, sec, micro; + Eterm *hp; + erts_get_now_cpu(&mega, &sec, µ); + hp = alloc_heap(env, 4); + return TUPLE3(hp, make_small(mega), make_small(sec), make_small(micro)); +#else + return enif_make_badarg(env); +#endif +} + +ERL_NIF_TERM +enif_make_unique_integer(ErlNifEnv *env, ErlNifUniqueInteger properties) +{ + int monotonic = properties & ERL_NIF_UNIQUE_MONOTONIC; + int positive = properties & ERL_NIF_UNIQUE_POSITIVE; + Eterm *hp; + Uint hsz; + + if (monotonic) { + Sint64 raw_unique = erts_raw_get_unique_monotonic_integer(); + hsz = erts_raw_unique_monotonic_integer_heap_size(raw_unique, positive); + hp = alloc_heap(env, hsz); + return erts_raw_make_unique_monotonic_integer_value(&hp, raw_unique, positive); + } else { + Uint64 raw_unique[ERTS_UNIQUE_INT_RAW_VALUES]; + erts_raw_get_unique_integer(raw_unique); + hsz = erts_raw_unique_integer_heap_size(raw_unique, positive); + hp = alloc_heap(env, hsz); + return erts_raw_make_unique_integer(&hp, raw_unique, positive); + } +} ErlNifMutex* enif_mutex_create(char *name) { return erl_drv_mutex_create(name); } void enif_mutex_destroy(ErlNifMutex *mtx) { erl_drv_mutex_destroy(mtx); } @@ -1142,6 +1386,27 @@ ErlNifTid enif_thread_self(void) { return erl_drv_thread_self(); } int enif_equal_tids(ErlNifTid tid1, ErlNifTid tid2) { return erl_drv_equal_tids(tid1,tid2); } void enif_thread_exit(void *resp) { erl_drv_thread_exit(resp); } int enif_thread_join(ErlNifTid tid, void **respp) { return erl_drv_thread_join(tid,respp); } +int enif_getenv(const char *key, char *value, size_t *value_size) { return erl_drv_getenv(key, value, value_size); } + +ErlNifTime enif_monotonic_time(ErlNifTimeUnit time_unit) +{ + return (ErlNifTime) erts_napi_monotonic_time((int) time_unit); +} + +ErlNifTime enif_time_offset(ErlNifTimeUnit time_unit) +{ + return (ErlNifTime) erts_napi_time_offset((int) time_unit); +} + +ErlNifTime +enif_convert_time_unit(ErlNifTime val, + ErlNifTimeUnit from, + ErlNifTimeUnit to) +{ + return (ErlNifTime) erts_napi_convert_time_unit((ErtsMonotonicTime) val, + (int) from, + (int) to); +} int enif_fprintf(void* filep, const char* format, ...) { @@ -1178,7 +1443,11 @@ typedef struct enif_resource_t struct enif_resource_type_t* type; #ifdef DEBUG erts_refc_t nif_refc; +# ifdef ARCH_32 + byte align__[4]; +# endif #endif + char data[1]; }ErlNifResource; @@ -1354,7 +1623,7 @@ static void rollback_opened_resource_types(void) static void nif_resource_dtor(Binary* bin) { - ErlNifResource* resource = (ErlNifResource*) ERTS_MAGIC_BIN_DATA(bin); + ErlNifResource* resource = (ErlNifResource*) ERTS_MAGIC_BIN_UNALIGNED_DATA(bin); ErlNifResourceType* type = resource->type; ASSERT(ERTS_MAGIC_BIN_DESTRUCTOR(bin) == &nif_resource_dtor); @@ -1375,8 +1644,10 @@ static void nif_resource_dtor(Binary* bin) void* enif_alloc_resource(ErlNifResourceType* type, size_t size) { - Binary* bin = erts_create_magic_binary(SIZEOF_ErlNifResource(size), &nif_resource_dtor); - ErlNifResource* resource = ERTS_MAGIC_BIN_DATA(bin); + Binary* bin = erts_create_magic_binary_x(SIZEOF_ErlNifResource(size), + &nif_resource_dtor, + 1); /* unaligned */ + ErlNifResource* resource = ERTS_MAGIC_BIN_UNALIGNED_DATA(bin); ASSERT(type->owner && type->next && type->prev); /* not allowed in load/upgrade */ resource->type = type; @@ -1391,7 +1662,7 @@ void* enif_alloc_resource(ErlNifResourceType* type, size_t size) void enif_release_resource(void* obj) { ErlNifResource* resource = DATA_TO_RESOURCE(obj); - ErtsBinary* bin = ERTS_MAGIC_BIN_FROM_DATA(resource); + ErtsBinary* bin = ERTS_MAGIC_BIN_FROM_UNALIGNED_DATA(resource); ASSERT(ERTS_MAGIC_BIN_DESTRUCTOR(bin) == &nif_resource_dtor); #ifdef DEBUG @@ -1405,7 +1676,7 @@ void enif_release_resource(void* obj) void enif_keep_resource(void* obj) { ErlNifResource* resource = DATA_TO_RESOURCE(obj); - ErtsBinary* bin = ERTS_MAGIC_BIN_FROM_DATA(resource); + ErtsBinary* bin = ERTS_MAGIC_BIN_FROM_UNALIGNED_DATA(resource); ASSERT(ERTS_MAGIC_BIN_DESTRUCTOR(bin) == &nif_resource_dtor); #ifdef DEBUG @@ -1417,7 +1688,7 @@ void enif_keep_resource(void* obj) ERL_NIF_TERM enif_make_resource(ErlNifEnv* env, void* obj) { ErlNifResource* resource = DATA_TO_RESOURCE(obj); - ErtsBinary* bin = ERTS_MAGIC_BIN_FROM_DATA(resource); + ErtsBinary* bin = ERTS_MAGIC_BIN_FROM_UNALIGNED_DATA(resource); Eterm* hp = alloc_heap(env,PROC_BIN_SIZE); return erts_mk_magic_binary_term(&hp, &MSO(env->proc), &bin->binary); } @@ -1446,7 +1717,7 @@ int enif_get_resource(ErlNifEnv* env, ERL_NIF_TERM term, ErlNifResourceType* typ return 0; / * Or should we allow "resource binaries" as handles? * / }*/ mbin = pb->val; - resource = (ErlNifResource*) ERTS_MAGIC_BIN_DATA(mbin); + resource = (ErlNifResource*) ERTS_MAGIC_BIN_UNALIGNED_DATA(mbin); if (ERTS_MAGIC_BIN_DESTRUCTOR(mbin) != &nif_resource_dtor || resource->type != type) { return 0; @@ -1458,8 +1729,8 @@ int enif_get_resource(ErlNifEnv* env, ERL_NIF_TERM term, ErlNifResourceType* typ size_t enif_sizeof_resource(void* obj) { ErlNifResource* resource = DATA_TO_RESOURCE(obj); - Binary* bin = &ERTS_MAGIC_BIN_FROM_DATA(resource)->binary; - return ERTS_MAGIC_BIN_DATA_SIZE(bin) - offsetof(ErlNifResource,data); + Binary* bin = &ERTS_MAGIC_BIN_FROM_UNALIGNED_DATA(resource)->binary; + return ERTS_MAGIC_BIN_UNALIGNED_DATA_SIZE(bin) - offsetof(ErlNifResource,data); } @@ -1513,84 +1784,279 @@ int enif_consume_timeslice(ErlNifEnv* env, int percent) return ERTS_BIF_REDS_LEFT(env->proc) == 0; } -#ifdef ERTS_DIRTY_SCHEDULERS - -/* NIFs exports need one more item than the Export struct provides, the - * erl_module_nif*, so the DirtyNifExport below adds that. The Export - * member must be first in the struct. +/* + * NIF exports need a few more items than the Export struct provides, + * including the erl_module_nif* and a NIF function pointer, so the + * NifExport below adds those. The Export member must be first in the + * struct. The saved_mfa, exception_thrown, saved_argc, rootset_extra, and + * rootset members are used to track the MFA, any pending exception, and + * arguments of the top NIF in case a chain of one or more + * enif_schedule_nif() calls results in an exception, since in that case + * the original MFA and registers have to be restored before returning to + * Erlang to ensure stacktrace information associated with the exception is + * correct. */ +typedef ERL_NIF_TERM (*NativeFunPtr)(ErlNifEnv*, int, const ERL_NIF_TERM[]); + typedef struct { Export exp; struct erl_module_nif* m; -} DirtyNifExport; + NativeFunPtr fp; + Eterm saved_mfa[3]; + int exception_thrown; + int saved_argc; + int rootset_extra; + Eterm rootset[1]; +} NifExport; -static void -alloc_proc_psd(Process* proc, DirtyNifExport **ep) +/* + * If a process has saved arguments, they need to be part of the GC + * rootset. The function below is called from setup_rootset() in + * erl_gc.c. This function is declared in erl_process.h. Any exception term + * saved in the NifExport is also made part of the GC rootset here; it + * always resides in rootset[0]. + */ +int +erts_setup_nif_gc(Process* proc, Eterm** objv, int* nobj) { + NifExport* ep = (NifExport*) ERTS_PROC_GET_NIF_TRAP_EXPORT(proc); + int gc = ep && (ep->saved_argc > 0 || ep->rootset[0] != NIL); + + if (gc) { + *objv = ep->rootset; + *nobj = 1 + ep->saved_argc; + } + return gc; +} + +/* + * Allocate a NifExport and set it in proc specific data + */ +static NifExport* +allocate_nif_sched_data(Process* proc, int argc) +{ + NifExport* ep; + size_t total; int i; - if (!*ep) { - *ep = erts_alloc(ERTS_ALC_T_PSD, sizeof(DirtyNifExport)); - sys_memset((void*) *ep, 0, sizeof(DirtyNifExport)); - for (i=0; i<ERTS_NUM_CODE_IX; i++) { - (*ep)->exp.addressv[i] = &(*ep)->exp.code[3]; - } - (*ep)->exp.code[3] = (BeamInstr) em_call_nif; + + total = sizeof(NifExport) + argc*sizeof(Eterm); + ep = erts_alloc(ERTS_ALC_T_NIF_TRAP_EXPORT, total); + sys_memset((void*) ep, 0, total); + ep->rootset_extra = argc; + ep->rootset[0] = NIL; + for (i=0; i<ERTS_NUM_CODE_IX; i++) { + ep->exp.addressv[i] = &ep->exp.code[3]; } - (void) ERTS_PROC_SET_DIRTY_SCHED_TRAP_EXPORT(proc, ERTS_PROC_LOCK_MAIN, &(*ep)->exp); + ep->exp.code[3] = (BeamInstr) em_call_nif; + (void) ERTS_PROC_SET_NIF_TRAP_EXPORT(proc, ep); + return ep; } +static ERTS_INLINE void +destroy_nif_export(NifExport *nif_export) +{ + erts_free(ERTS_ALC_T_NIF_TRAP_EXPORT, (void *) nif_export); +} + +void +erts_destroy_nif_export(void *nif_export) +{ + destroy_nif_export((NifExport *) nif_export); +} + +/* + * Initialize a NifExport struct. Create it if needed and store it in the + * proc. The direct_fp function is what will be invoked by op_call_nif, and + * the indirect_fp function, if not NULL, is what the direct_fp function + * will call. If the allocated NifExport isn't enough to hold all of argv, + * allocate a larger one. Save MFA and registers only if the need_save + * parameter is true. + */ static ERL_NIF_TERM -execute_dirty_nif_finalizer(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) +init_nif_sched_data(ErlNifEnv* env, NativeFunPtr direct_fp, NativeFunPtr indirect_fp, + int need_save, int argc, const ERL_NIF_TERM argv[]) { - Eterm* reg = ERTS_PROC_GET_SCHDATA(env->proc)->x_reg_array; - ERL_NIF_TERM result, dirty_result = (ERL_NIF_TERM) reg[0]; - typedef ERL_NIF_TERM (*FinalizerFP)(ErlNifEnv*, ERL_NIF_TERM); - FinalizerFP fp; -#if HAVE_INT64 && SIZEOF_LONG != 8 - ASSERT(sizeof(fp) <= sizeof(ErlNifUInt64)); - enif_get_uint64(env, reg[1], (ErlNifUInt64 *) &fp); -#else - ASSERT(sizeof(fp) <= sizeof(unsigned long)); - enif_get_ulong(env, reg[1], (unsigned long *) &fp); -#endif - result = (*fp)(env, dirty_result); - if (erts_refc_dectest(&env->mod_nif->rt_dtor_cnt, 0) == 0 - && env->mod_nif->mod == NULL) - close_lib(env->mod_nif); - return result; + Process* proc = env->proc; + Eterm* reg = ERTS_PROC_GET_SCHDATA(proc)->x_reg_array; + NifExport* ep; + int i; + + ep = (NifExport*) ERTS_PROC_GET_NIF_TRAP_EXPORT(proc); + if (!ep) + ep = allocate_nif_sched_data(proc, argc); + else if (need_save && ep->rootset_extra < argc) { + NifExport* new_ep = allocate_nif_sched_data(proc, argc); + destroy_nif_export(ep); + ep = new_ep; + } + if (env->exception_thrown) { + ep->exception_thrown = 1; + ep->rootset[0] = env->proc->fvalue; + } else { + ep->exception_thrown = 0; + ep->rootset[0] = NIL; + } + ERTS_VBUMP_ALL_REDS(proc); + for (i = 0; i < argc; i++) { + if (need_save) + ep->rootset[i+1] = reg[i]; + reg[i] = (Eterm) argv[i]; + } + if (need_save) { + ep->saved_mfa[0] = proc->current[0]; + ep->saved_mfa[1] = proc->current[1]; + ep->saved_mfa[2] = proc->current[2]; + ep->saved_argc = argc; + } + proc->i = (BeamInstr*) ep->exp.addressv[0]; + ep->exp.code[0] = (BeamInstr) proc->current[0]; + ep->exp.code[1] = (BeamInstr) proc->current[1]; + ep->exp.code[2] = argc; + ep->exp.code[4] = (BeamInstr) direct_fp; + ep->m = env->mod_nif; + ep->fp = indirect_fp; + proc->freason = TRAP; + proc->arity = argc; + return THE_NON_VALUE; } -#endif /* ERTS_DIRTY_SCHEDULERS */ +/* + * Restore saved MFA and registers. Registers are restored only when the + * exception flag is true. + */ +static void +restore_nif_mfa(Process* proc, NifExport* ep, int exception) +{ + int i; + Eterm* reg = ERTS_PROC_GET_SCHDATA(proc)->x_reg_array; -#ifdef ERL_NIF_DIRTY_SCHEDULER_SUPPORT + proc->current[0] = ep->saved_mfa[0]; + proc->current[1] = ep->saved_mfa[1]; + proc->current[2] = ep->saved_mfa[2]; + if (exception) + for (i = 0; i < ep->saved_argc; i++) + reg[i] = ep->rootset[i+1]; + ep->saved_argc = 0; + ep->saved_mfa[0] = THE_NON_VALUE; +} -ERL_NIF_TERM -enif_schedule_dirty_nif(ErlNifEnv* env, int flags, - ERL_NIF_TERM (*fp)(ErlNifEnv*, int, const ERL_NIF_TERM[]), - int argc, const ERL_NIF_TERM argv[]) +#ifdef ERTS_DIRTY_SCHEDULERS + +/* + * Finalize a dirty NIF call. This function is scheduled to cause the VM to + * switch the process off a dirty scheduler thread and back onto a regular + * scheduler thread, and then return the result from the dirty NIF. It also + * restores the original NIF MFA when necessary based on the value of + * ep->fp set by execute_dirty_nif via init_nif_sched_data -- non-NULL + * means restore, NULL means do not restore. + */ +static ERL_NIF_TERM +dirty_nif_finalizer(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) +{ + Process* proc = env->proc; + NifExport* ep; + + ASSERT(argc == 1); + ASSERT(!ERTS_SCHEDULER_IS_DIRTY(env->proc->scheduler_data)); + ep = (NifExport*) ERTS_PROC_GET_NIF_TRAP_EXPORT(proc); + ASSERT(ep); + ASSERT(!ep->exception_thrown); + if (ep->fp) + restore_nif_mfa(proc, ep, 0); + return argv[0]; +} + +/* Finalize a dirty NIF call that raised an exception. Otherwise same as + * the dirty_nif_finalizer() function. + */ +static ERL_NIF_TERM +dirty_nif_exception(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) +{ + Process* proc = env->proc; + NifExport* ep; + + ASSERT(!ERTS_SCHEDULER_IS_DIRTY(env->proc->scheduler_data)); + ep = (NifExport*) ERTS_PROC_GET_NIF_TRAP_EXPORT(proc); + ASSERT(ep); + ASSERT(ep->exception_thrown); + if (ep->fp) + restore_nif_mfa(proc, ep, 1); + return enif_raise_exception(env, ep->rootset[0]); +} + +/* + * Dirty NIF execution wrapper function. Invoke an application's dirty NIF, + * then check the result and schedule the appropriate finalizer function + * where needed. Also restore the original NIF MFA when appropriate. + */ +static ERL_NIF_TERM +execute_dirty_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) +{ + Process* proc = env->proc; + NativeFunPtr fp = (NativeFunPtr) proc->current[6]; + NifExport* ep; + ERL_NIF_TERM result; + + ASSERT(ERTS_SCHEDULER_IS_DIRTY(env->proc->scheduler_data)); + + /* + * Set ep->fp to NULL before the native call so we know later whether it scheduled another NIF for execution + */ + ep = (NifExport*) ERTS_PROC_GET_NIF_TRAP_EXPORT(proc); + ASSERT(ep); + ep->fp = NULL; + erts_smp_atomic32_read_band_mb(&proc->state, ~(ERTS_PSFLG_DIRTY_CPU_PROC + | ERTS_PSFLG_DIRTY_IO_PROC)); + result = (*fp)(env, argc, argv); + + if (erts_refc_dectest(&env->mod_nif->rt_dtor_cnt, 0) == 0 && env->mod_nif->mod == NULL) + close_lib(env->mod_nif); + /* + * If no more NIFs were scheduled by the native call via + * enif_schedule_nif(), then ep->fp will still be NULL as set above, in + * which case we need to restore the original NIF calling + * context. Reuse fp essentially as a boolean for this, passing it to + * init_nif_sched_data below. Both dirty_nif_exception and + * dirty_nif_finalizer then check ep->fp to decide whether or not to + * restore the original calling context. + */ + ep = (NifExport*) ERTS_PROC_GET_NIF_TRAP_EXPORT(proc); + ASSERT(ep); + if (ep->fp) + fp = NULL; + if (is_non_value(result) || env->exception_thrown) { + if (proc->freason != TRAP) { + return init_nif_sched_data(env, dirty_nif_exception, fp, 0, argc, argv); + } else { + if (ep->fp == NULL) + restore_nif_mfa(proc, ep, 1); + return THE_NON_VALUE; + } + } + else + return init_nif_sched_data(env, dirty_nif_finalizer, fp, 0, 1, &result); +} + +/* + * Dirty NIF scheduling wrapper function. Schedule a dirty NIF to execute + * via the execute_dirty_nif() wrapper function. The dirty scheduler thread + * type (CPU or I/O) is indicated in flags parameter. + */ +static ERTS_INLINE ERL_NIF_TERM +schedule_dirty_nif(ErlNifEnv* env, int flags, int argc, const ERL_NIF_TERM argv[]) { -#ifdef USE_THREADS erts_aint32_t state, n, a; Process* proc = env->proc; - Eterm* reg = ERTS_PROC_GET_SCHDATA(proc)->x_reg_array; - DirtyNifExport* ep = NULL; - int i; + NativeFunPtr fp = (NativeFunPtr) proc->current[6]; + NifExport* ep; + int need_save; - int chkflgs = (flags & (ERL_NIF_DIRTY_JOB_IO_BOUND|ERL_NIF_DIRTY_JOB_CPU_BOUND)); - if (chkflgs != ERL_NIF_DIRTY_JOB_IO_BOUND && chkflgs != ERL_NIF_DIRTY_JOB_CPU_BOUND) - return enif_make_badarg(env); + ASSERT(flags==ERL_NIF_DIRTY_JOB_IO_BOUND || flags==ERL_NIF_DIRTY_JOB_CPU_BOUND); a = erts_smp_atomic32_read_acqb(&proc->state); while (1) { n = state = a; - /* - * clear any current dirty flags and dirty queue indicators, - * in case the application is shifting a job from one type - * of dirty scheduler to the other - */ - n &= ~(ERTS_PSFLG_DIRTY_CPU_PROC|ERTS_PSFLG_DIRTY_IO_PROC - |ERTS_PSFLG_DIRTY_CPU_PROC_IN_Q|ERTS_PSFLG_DIRTY_IO_PROC_IN_Q); - if (chkflgs == ERL_NIF_DIRTY_JOB_CPU_BOUND) + if (flags == ERL_NIF_DIRTY_JOB_CPU_BOUND) n |= ERTS_PSFLG_DIRTY_CPU_PROC; else n |= ERTS_PSFLG_DIRTY_IO_PROC; @@ -1598,85 +2064,107 @@ enif_schedule_dirty_nif(ErlNifEnv* env, int flags, if (a == state) break; } - if (!(ep = (DirtyNifExport*) ERTS_PROC_GET_DIRTY_SCHED_TRAP_EXPORT(proc))) - alloc_proc_psd(proc, &ep); - ERTS_VBUMP_ALL_REDS(proc); - ep->exp.code[2] = argc; - for (i = 0; i < argc; i++) { - reg[i] = (Eterm) argv[i]; - } - proc->i = (BeamInstr*) ep->exp.addressv[0]; - ep->exp.code[4] = (BeamInstr) fp; - ep->m = env->mod_nif; - proc->freason = TRAP; - erts_refc_inc(&env->mod_nif->rt_dtor_cnt, 1); + ep = (NifExport*) ERTS_PROC_GET_NIF_TRAP_EXPORT(proc); + need_save = (ep == NULL || is_non_value(ep->saved_mfa[0])); + return init_nif_sched_data(env, execute_dirty_nif, fp, need_save, argc, argv); +} - return THE_NON_VALUE; -#else - return (*fp)(env, argc, argv); -#endif +static ERL_NIF_TERM +schedule_dirty_io_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) +{ + return schedule_dirty_nif(env, ERL_NIF_DIRTY_JOB_IO_BOUND, argc, argv); +} + +static ERL_NIF_TERM +schedule_dirty_cpu_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) +{ + return schedule_dirty_nif(env, ERL_NIF_DIRTY_JOB_CPU_BOUND, argc, argv); +} + +#endif /* ERTS_DIRTY_SCHEDULERS */ + +/* + * NIF execution wrapper used by enif_schedule_nif() for regular NIFs. It + * calls the actual NIF, restores original NIF MFA if necessary, and + * then returns the NIF result. + */ +static ERL_NIF_TERM +execute_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) +{ + Process* proc = env->proc; + NativeFunPtr fp = (NativeFunPtr) proc->current[6]; + NifExport* ep; + ERL_NIF_TERM result; + + ASSERT(!env->exception_thrown); + ep = (NifExport*) ERTS_PROC_GET_NIF_TRAP_EXPORT(proc); + ASSERT(ep); + ep->fp = NULL; + result = (*fp)(env, argc, argv); + ep = (NifExport*) ERTS_PROC_GET_NIF_TRAP_EXPORT(proc); + ASSERT(ep); + /* + * If no NIFs were scheduled by the native call via + * enif_schedule_nif(), then ep->fp will still be NULL as set above, in + * which case we need to restore the original NIF MFA. + */ + if (ep->fp == NULL) + restore_nif_mfa(proc, ep, env->exception_thrown); + return result; } ERL_NIF_TERM -enif_schedule_dirty_nif_finalizer(ErlNifEnv* env, ERL_NIF_TERM result, - ERL_NIF_TERM (*fp)(ErlNifEnv*, ERL_NIF_TERM)) +enif_schedule_nif(ErlNifEnv* env, const char* fun_name, int flags, + ERL_NIF_TERM (*fp)(ErlNifEnv*, int, const ERL_NIF_TERM[]), + int argc, const ERL_NIF_TERM argv[]) { -#ifdef USE_THREADS Process* proc = env->proc; - Eterm* reg = ERTS_PROC_GET_SCHDATA(proc)->x_reg_array; - DirtyNifExport* ep; - - erts_smp_atomic32_read_band_mb(&proc->state, - ~(ERTS_PSFLG_DIRTY_CPU_PROC - |ERTS_PSFLG_DIRTY_IO_PROC - |ERTS_PSFLG_DIRTY_CPU_PROC_IN_Q - |ERTS_PSFLG_DIRTY_IO_PROC_IN_Q)); - if (!(ep = (DirtyNifExport*) ERTS_PROC_GET_DIRTY_SCHED_TRAP_EXPORT(proc))) - alloc_proc_psd(proc, &ep); - ERTS_VBUMP_ALL_REDS(proc); - ep->exp.code[2] = 2; - reg[0] = (Eterm) result; -#if HAVE_INT64 && SIZEOF_LONG != 8 - ASSERT(sizeof(fp) <= sizeof(ErlNifUInt64)); - reg[1] = (Eterm) enif_make_uint64(env, (ErlNifUInt64) fp); -#else - ASSERT(sizeof(fp) <= sizeof(unsigned long)); - reg[1] = (Eterm) enif_make_ulong(env, (unsigned long) fp); -#endif - proc->i = (BeamInstr*) ep->exp.addressv[0]; - ep->exp.code[4] = (BeamInstr) execute_dirty_nif_finalizer; - proc->freason = TRAP; + NifExport* ep; + ERL_NIF_TERM fun_name_atom, result; + int need_save; - return THE_NON_VALUE; + if (argc > MAX_ARG) + return enif_make_badarg(env); + fun_name_atom = enif_make_atom(env, fun_name); + if (enif_is_exception(env, fun_name_atom)) + return fun_name_atom; + + ep = (NifExport*) ERTS_PROC_GET_NIF_TRAP_EXPORT(proc); + need_save = (ep == NULL || is_non_value(ep->saved_mfa[0])); + + if (flags) { +#ifdef ERTS_DIRTY_SCHEDULERS + NativeFunPtr sched_fun; + int chkflgs = (flags & (ERL_NIF_DIRTY_JOB_IO_BOUND|ERL_NIF_DIRTY_JOB_CPU_BOUND)); + if (chkflgs == ERL_NIF_DIRTY_JOB_IO_BOUND) + sched_fun = schedule_dirty_io_nif; + else if (chkflgs == ERL_NIF_DIRTY_JOB_CPU_BOUND) + sched_fun = schedule_dirty_cpu_nif; + else + return enif_make_badarg(env); + result = init_nif_sched_data(env, sched_fun, fp, need_save, argc, argv); #else - return (*fp)(env, result); + return enif_make_badarg(env); #endif -} + } + else + result = init_nif_sched_data(env, execute_nif, fp, need_save, argc, argv); -/* A simple finalizer that just returns its result argument */ -ERL_NIF_TERM -enif_dirty_nif_finalizer(ErlNifEnv* env, ERL_NIF_TERM result) -{ + ep = (NifExport*) ERTS_PROC_GET_NIF_TRAP_EXPORT(proc); + ASSERT(ep); + ep->exp.code[1] = (BeamInstr) fun_name_atom; return result; } +#ifdef ERL_NIF_DIRTY_SCHEDULER_SUPPORT + int enif_is_on_dirty_scheduler(ErlNifEnv* env) { return ERTS_SCHEDULER_IS_DIRTY(env->proc->scheduler_data); } -int -enif_have_dirty_schedulers() -{ -#ifdef USE_THREADS - return 1; -#else - return 0; -#endif -} - #endif /* ERL_NIF_DIRTY_SCHEDULER_SUPPORT */ /* Maps */ @@ -1688,29 +2176,33 @@ int enif_is_map(ErlNifEnv* env, ERL_NIF_TERM term) int enif_get_map_size(ErlNifEnv* env, ERL_NIF_TERM term, size_t *size) { - if (is_map(term)) { - map_t *mp; - mp = (map_t*)map_val(term); - *size = map_get_size(mp); + if (is_flatmap(term)) { + flatmap_t *mp; + mp = (flatmap_t*)flatmap_val(term); + *size = flatmap_get_size(mp); return 1; } + else if (is_hashmap(term)) { + *size = hashmap_size(term); + return 1; + } return 0; } ERL_NIF_TERM enif_make_new_map(ErlNifEnv* env) { - Eterm* hp = alloc_heap(env,MAP_HEADER_SIZE+1); + Eterm* hp = alloc_heap(env,MAP_HEADER_FLATMAP_SZ+1); Eterm tup; - map_t *mp; + flatmap_t *mp; tup = make_tuple(hp); *hp++ = make_arityval(0); - mp = (map_t*)hp; - mp->thing_word = MAP_HEADER; + mp = (flatmap_t*)hp; + mp->thing_word = MAP_HEADER_FLATMAP; mp->size = 0; mp->keys = tup; - return make_map(mp); + return make_flatmap(mp); } int enif_make_map_put(ErlNifEnv* env, @@ -1719,7 +2211,7 @@ int enif_make_map_put(ErlNifEnv* env, Eterm value, Eterm *map_out) { - if (is_not_map(map_in)) { + if (!is_map(map_in)) { return 0; } flush_env(env); @@ -1733,10 +2225,16 @@ int enif_get_map_value(ErlNifEnv* env, Eterm key, Eterm *value) { - if (is_not_map(map)) { + const Eterm *ret; + if (!is_map(map)) { return 0; } - return erts_maps_get(key, map, value); + ret = erts_maps_get(key, map); + if (ret) { + *value = *ret; + return 1; + } + return 0; } int enif_make_map_update(ErlNifEnv* env, @@ -1746,7 +2244,7 @@ int enif_make_map_update(ErlNifEnv* env, Eterm *map_out) { int res; - if (is_not_map(map_in)) { + if (!is_map(map_in)) { return 0; } @@ -1762,7 +2260,7 @@ int enif_make_map_remove(ErlNifEnv* env, Eterm *map_out) { int res; - if (is_not_map(map_in)) { + if (!is_map(map_in)) { return 0; } flush_env(env); @@ -1776,13 +2274,13 @@ int enif_map_iterator_create(ErlNifEnv *env, ErlNifMapIterator *iter, ErlNifMapIteratorEntry entry) { - if (is_map(map)) { - map_t *mp = (map_t*)map_val(map); + if (is_flatmap(map)) { + flatmap_t *mp = (flatmap_t*)flatmap_val(map); size_t offset; switch (entry) { - case ERL_NIF_MAP_ITERATOR_HEAD: offset = 0; break; - case ERL_NIF_MAP_ITERATOR_TAIL: offset = map_get_size(mp) - 1; break; + case ERL_NIF_MAP_ITERATOR_FIRST: offset = 0; break; + case ERL_NIF_MAP_ITERATOR_LAST: offset = flatmap_get_size(mp) - 1; break; default: goto error; } @@ -1791,14 +2289,37 @@ int enif_map_iterator_create(ErlNifEnv *env, */ iter->map = map; - iter->ks = ((Eterm *)map_get_keys(mp)) + offset; - iter->vs = ((Eterm *)map_get_values(mp)) + offset; - iter->t_limit = map_get_size(mp) + 1; + iter->u.flat.ks = ((Eterm *)flatmap_get_keys(mp)) + offset; + iter->u.flat.vs = ((Eterm *)flatmap_get_values(mp)) + offset; + iter->size = flatmap_get_size(mp); iter->idx = offset + 1; return 1; } - + else if (is_hashmap(map)) { + iter->map = map; + iter->size = hashmap_size(map); + iter->u.hash.wstack = erts_alloc(ERTS_ALC_T_NIF, sizeof(ErtsDynamicWStack)); + WSTACK_INIT(iter->u.hash.wstack, ERTS_ALC_T_NIF); + + switch (entry) { + case ERL_NIF_MAP_ITERATOR_FIRST: + iter->idx = 1; + hashmap_iterator_init(&iter->u.hash.wstack->ws, map, 0); + iter->u.hash.kv = hashmap_iterator_next(&iter->u.hash.wstack->ws); + break; + case ERL_NIF_MAP_ITERATOR_LAST: + iter->idx = hashmap_size(map); + hashmap_iterator_init(&iter->u.hash.wstack->ws, map, 1); + iter->u.hash.kv = hashmap_iterator_prev(&iter->u.hash.wstack->ws); + break; + default: + goto error; + } + ASSERT(!!iter->u.hash.kv == (iter->idx >= 1 && + iter->idx <= iter->size)); + return 1; + } error: #ifdef DEBUG iter->map = THE_NON_VALUE; @@ -1808,48 +2329,97 @@ error: void enif_map_iterator_destroy(ErlNifEnv *env, ErlNifMapIterator *iter) { - /* not used */ + if (is_hashmap(iter->map)) { + WSTACK_DESTROY(iter->u.hash.wstack->ws); + erts_free(ERTS_ALC_T_NIF, iter->u.hash.wstack); + } + else + ASSERT(is_flatmap(iter->map)); + #ifdef DEBUG iter->map = THE_NON_VALUE; #endif - } int enif_map_iterator_is_tail(ErlNifEnv *env, ErlNifMapIterator *iter) { - ASSERT(iter && is_map(iter->map)); - ASSERT(iter->idx >= 0 && (iter->idx <= map_get_size(map_val(iter->map)) + 1)); - return (iter->t_limit == 1 || iter->idx == iter->t_limit); + ASSERT(iter); + if (is_flatmap(iter->map)) { + ASSERT(iter->idx >= 0); + ASSERT(iter->idx <= flatmap_get_size(flatmap_val(iter->map)) + 1); + return (iter->size == 0 || iter->idx > iter->size); + } + else { + ASSERT(is_hashmap(iter->map)); + return iter->idx > iter->size; + } } int enif_map_iterator_is_head(ErlNifEnv *env, ErlNifMapIterator *iter) { - ASSERT(iter && is_map(iter->map)); - ASSERT(iter->idx >= 0 && (iter->idx <= map_get_size(map_val(iter->map)) + 1)); - return (iter->t_limit == 1 || iter->idx == 0); + ASSERT(iter); + if (is_flatmap(iter->map)) { + ASSERT(iter->idx >= 0); + ASSERT(iter->idx <= flatmap_get_size(flatmap_val(iter->map)) + 1); + return (iter->size == 0 || iter->idx == 0); + } + else { + ASSERT(is_hashmap(iter->map)); + return iter->idx == 0; + } } int enif_map_iterator_next(ErlNifEnv *env, ErlNifMapIterator *iter) { - ASSERT(iter && is_map(iter->map)); - if (iter->idx < iter->t_limit) { - iter->idx++; - iter->ks++; - iter->vs++; + ASSERT(iter); + if (is_flatmap(iter->map)) { + if (iter->idx <= iter->size) { + iter->idx++; + iter->u.flat.ks++; + iter->u.flat.vs++; + } + return (iter->idx <= iter->size); + } + else { + ASSERT(is_hashmap(iter->map)); + + if (iter->idx <= hashmap_size(iter->map)) { + if (iter->idx < 1) { + hashmap_iterator_init(&iter->u.hash.wstack->ws, iter->map, 0); + } + iter->u.hash.kv = hashmap_iterator_next(&iter->u.hash.wstack->ws); + iter->idx++; + ASSERT(!!iter->u.hash.kv == (iter->idx <= iter->size)); + } + return iter->idx <= iter->size; } - return (iter->idx != iter->t_limit); } int enif_map_iterator_prev(ErlNifEnv *env, ErlNifMapIterator *iter) { - ASSERT(iter && is_map(iter->map)); - if (iter->idx > 0) { - iter->idx--; - iter->ks--; - iter->vs--; + ASSERT(iter); + if (is_flatmap(iter->map)) { + if (iter->idx > 0) { + iter->idx--; + iter->u.flat.ks--; + iter->u.flat.vs--; + } + return iter->idx > 0; + } + else { + ASSERT(is_hashmap(iter->map)); + + if (iter->idx > 0) { + if (iter->idx > iter->size) { + hashmap_iterator_init(&iter->u.hash.wstack->ws, iter->map, 1); + } + iter->u.hash.kv = hashmap_iterator_prev(&iter->u.hash.wstack->ws); + iter->idx--; + ASSERT(!!iter->u.hash.kv == (iter->idx > 0)); + } + return iter->idx > 0; } - return (iter->idx > 0); } int enif_map_iterator_get_pair(ErlNifEnv *env, @@ -1857,15 +2427,25 @@ int enif_map_iterator_get_pair(ErlNifEnv *env, Eterm *key, Eterm *value) { - ASSERT(iter && is_map(iter->map)); - if (iter->idx > 0 && iter->idx < iter->t_limit) { - ASSERT(iter->ks >= map_get_keys(map_val(iter->map)) && - iter->ks < (map_get_keys(map_val(iter->map)) + map_get_size(map_val(iter->map)))); - ASSERT(iter->vs >= map_get_values(map_val(iter->map)) && - iter->vs < (map_get_values(map_val(iter->map)) + map_get_size(map_val(iter->map)))); - *key = *(iter->ks); - *value = *(iter->vs); - return 1; + ASSERT(iter); + if (is_flatmap(iter->map)) { + if (iter->idx > 0 && iter->idx <= iter->size) { + ASSERT(iter->u.flat.ks >= flatmap_get_keys(flatmap_val(iter->map)) && + iter->u.flat.ks < (flatmap_get_keys(flatmap_val(iter->map)) + flatmap_get_size(flatmap_val(iter->map)))); + ASSERT(iter->u.flat.vs >= flatmap_get_values(flatmap_val(iter->map)) && + iter->u.flat.vs < (flatmap_get_values(flatmap_val(iter->map)) + flatmap_get_size(flatmap_val(iter->map)))); + *key = *(iter->u.flat.ks); + *value = *(iter->u.flat.vs); + return 1; + } + } + else { + ASSERT(is_hashmap(iter->map)); + if (iter->idx > 0 && iter->idx <= iter->size) { + *key = CAR(iter->u.hash.kv); + *value = CDR(iter->u.hash.kv); + return 1; + } } return 0; } @@ -1875,17 +2455,17 @@ int enif_map_iterator_get_pair(ErlNifEnv *env, ***************************************************************************/ -static BeamInstr** get_func_pp(BeamInstr* mod_code, Eterm f_atom, unsigned arity) +static BeamInstr** get_func_pp(BeamCodeHeader* mod_code, Eterm f_atom, unsigned arity) { - int n = (int) mod_code[MI_NUM_FUNCTIONS]; + int n = (int) mod_code->num_functions; int j; for (j = 0; j < n; ++j) { - BeamInstr* code_ptr = (BeamInstr*) mod_code[MI_FUNCTIONS+j]; + BeamInstr* code_ptr = (BeamInstr*) mod_code->functions[j]; ASSERT(code_ptr[0] == (BeamInstr) BeamOp(op_i_func_info_IaaI)); if (f_atom == ((Eterm) code_ptr[3]) && arity == ((unsigned) code_ptr[4])) { - return (BeamInstr**) &mod_code[MI_FUNCTIONS+j]; + return (BeamInstr**) &mod_code->functions[j]; } } return NULL; @@ -1977,6 +2557,35 @@ static Eterm load_nif_error(Process* p, const char* atom, const char* format, .. return ret; } +/* + * The function below is for looping through ErlNifFunc arrays, helping + * provide backwards compatibility across the version 2.7 change that added + * the "flags" field to ErlNifFunc. + */ +static ErlNifFunc* next_func(ErlNifEntry* entry, int* incrp, ErlNifFunc* func) +{ + ASSERT(incrp); + if (!*incrp) { + if (entry->major > 2 || (entry->major == 2 && entry->minor >= 7)) + *incrp = sizeof(ErlNifFunc); + else { + /* + * ErlNifFuncV1 below is what ErlNifFunc was before the + * addition of the flags field for 2.7, and is needed to handle + * backward compatibility. + */ + typedef struct { + const char* name; + unsigned arity; + ERL_NIF_TERM (*fptr)(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]); + }ErlNifFuncV1; + *incrp = sizeof(ErlNifFuncV1); + } + } + return (ErlNifFunc*) ((char*)func + *incrp); +} + + BIF_RETTYPE load_nif_2(BIF_ALIST_2) { static const char bad_lib[] = "bad_lib"; @@ -2039,8 +2648,8 @@ BIF_RETTYPE load_nif_2(BIF_ALIST_2) if (init_func != NULL) handle = init_func; - if (!in_area(caller, mod->curr.code, mod->curr.code_length)) { - ASSERT(in_area(caller, mod->old.code, mod->old.code_length)); + if (!in_area(caller, mod->curr.code_hdr, mod->curr.code_length)) { + ASSERT(in_area(caller, mod->old.code_hdr, mod->old.code_length)); ret = load_nif_error(BIF_P, "old_code", "Calling load_nif from old " "module '%T' not allowed", mod_atom); @@ -2086,22 +2695,48 @@ BIF_RETTYPE load_nif_2(BIF_ALIST_2) } else { /*erts_fprintf(stderr, "Found module %T\r\n", mod_atom);*/ - + + int maybe_dirty_nifs = ((entry->major > 2 || (entry->major == 2 && entry->minor >= 7)) + && (entry->options & ERL_NIF_DIRTY_NIF_OPTION)); + int incr = 0; + ErlNifFunc* f = entry->funcs; for (i=0; i < entry->num_of_funcs && ret==am_ok; i++) { BeamInstr** code_pp; - ErlNifFunc* f = &entry->funcs[i]; if (!erts_atom_get(f->name, sys_strlen(f->name), &f_atom, ERTS_ATOM_ENC_LATIN1) - || (code_pp = get_func_pp(mod->curr.code, f_atom, f->arity))==NULL) { + || (code_pp = get_func_pp(mod->curr.code_hdr, f_atom, f->arity))==NULL) { ret = load_nif_error(BIF_P,bad_lib,"Function not found %T:%s/%u", mod_atom, f->name, f->arity); - } - else if (code_pp[1] - code_pp[0] < (5+3)) { + } + else if (maybe_dirty_nifs && f->flags) { + /* + * If the flags field is non-zero and this emulator was + * built with dirty scheduler support, check that the flags + * value is legal. But if this emulator was built without + * dirty scheduler support, treat a non-zero flags field as + * a load error. + */ +#ifdef ERTS_DIRTY_SCHEDULERS + if (f->flags != ERL_NIF_DIRTY_JOB_IO_BOUND && f->flags != ERL_NIF_DIRTY_JOB_CPU_BOUND) + ret = load_nif_error(BIF_P, bad_lib, "Illegal flags field value %d for NIF %T:%s/%u", + f->flags, mod_atom, f->name, f->arity); +#else + ret = load_nif_error(BIF_P, bad_lib, "NIF %T:%s/%u requires a runtime with dirty scheduler support.", + mod_atom, f->name, f->arity); +#endif + } +#ifdef ERTS_DIRTY_SCHEDULERS + else if (code_pp[1] - code_pp[0] < (5+4)) +#else + else if (code_pp[1] - code_pp[0] < (5+3)) +#endif + { ret = load_nif_error(BIF_P,bad_lib,"No explicit call to load_nif" - " in module (%T:%s/%u to small)", - mod_atom, entry->funcs[i].name, entry->funcs[i].arity); + " in module (%T:%s/%u too small)", + mod_atom, f->name, f->arity); } /*erts_fprintf(stderr, "Found NIF %T:%s/%u\r\n", - mod_atom, entry->funcs[i].name, entry->funcs[i].arity);*/ + mod_atom, f->name, f->arity);*/ + f = next_func(entry, &incr, f); } } @@ -2127,7 +2762,8 @@ BIF_RETTYPE load_nif_2(BIF_ALIST_2) * is deprecated and was only ment as a development feature not to * be used in production systems. (See warning below) */ - int k; + int k, old_incr = 0; + ErlNifFunc* old_func; lib->priv_data = mod->curr.nif->priv_data; ASSERT(mod->curr.nif->entry != NULL); @@ -2136,13 +2772,16 @@ BIF_RETTYPE load_nif_2(BIF_ALIST_2) goto error; } /* Check that no NIF is removed */ + old_func = mod->curr.nif->entry->funcs; for (k=0; k < mod->curr.nif->entry->num_of_funcs; k++) { - ErlNifFunc* old_func = &mod->curr.nif->entry->funcs[k]; + int incr = 0; + ErlNifFunc* f = entry->funcs; for (i=0; i < entry->num_of_funcs; i++) { - if (old_func->arity == entry->funcs[i].arity - && sys_strcmp(old_func->name, entry->funcs[i].name) == 0) { + if (old_func->arity == f->arity + && sys_strcmp(old_func->name, f->name) == 0) { break; } + f = next_func(entry, &incr, f); } if (i == entry->num_of_funcs) { ret = load_nif_error(BIF_P,reload,"Reloaded library missing " @@ -2150,7 +2789,8 @@ BIF_RETTYPE load_nif_2(BIF_ALIST_2) old_func->name, old_func->arity); goto error; } - } + old_func = next_func(mod->curr.nif->entry, &old_incr, old_func); + } erts_pre_nif(&env, BIF_P, lib); veto = entry->reload(&env, &lib->priv_data, BIF_ARG_2); erts_post_nif(&env); @@ -2197,13 +2837,17 @@ BIF_RETTYPE load_nif_2(BIF_ALIST_2) /* ** Everything ok, patch the beam code with op_call_nif */ - mod->curr.nif = lib; + + int incr = 0; + ErlNifFunc* f = entry->funcs; + + mod->curr.nif = lib; for (i=0; i < entry->num_of_funcs; i++) { BeamInstr* code_ptr; - erts_atom_get(entry->funcs[i].name, sys_strlen(entry->funcs[i].name), &f_atom, ERTS_ATOM_ENC_LATIN1); - code_ptr = *get_func_pp(mod->curr.code, f_atom, entry->funcs[i].arity); - + erts_atom_get(f->name, sys_strlen(f->name), &f_atom, ERTS_ATOM_ENC_LATIN1); + code_ptr = *get_func_pp(mod->curr.code_hdr, f_atom, f->arity); + if (code_ptr[1] == 0) { code_ptr[5+0] = (BeamInstr) BeamOp(op_call_nif); } @@ -2211,10 +2855,21 @@ BIF_RETTYPE load_nif_2(BIF_ALIST_2) GenericBp* g = (GenericBp *) code_ptr[1]; ASSERT(code_ptr[5+0] == (BeamInstr) BeamOp(op_i_generic_breakpoint)); - g->orig_instr = (BeamInstr) BeamOp(op_call_nif); - } - code_ptr[5+1] = (BeamInstr) entry->funcs[i].fptr; + g->orig_instr = (BeamInstr) BeamOp(op_call_nif); + } + if ((entry->major > 2 || (entry->major == 2 && entry->minor >= 7)) + && (entry->options & ERL_NIF_DIRTY_NIF_OPTION) && f->flags) { +#ifdef ERL_NIF_DIRTY_SCHEDULER_SUPPORT + code_ptr[5+3] = (BeamInstr) f->fptr; + code_ptr[5+1] = (f->flags == ERL_NIF_DIRTY_JOB_IO_BOUND) ? + (BeamInstr) schedule_dirty_io_nif : + (BeamInstr) schedule_dirty_cpu_nif; +#endif + } + else + code_ptr[5+1] = (BeamInstr) f->fptr; code_ptr[5+2] = (BeamInstr) lib; + f = next_func(entry, &incr, f); } } else { @@ -2289,6 +2944,8 @@ erts_unload_nif(struct erl_module_nif* lib) void erl_nif_init() { + ERTS_CT_ASSERT((offsetof(ErlNifResource,data) % 8) == ERTS_MAGIC_BIN_BYTES_TO_ALIGN); + resource_type_list.next = &resource_type_list; resource_type_list.prev = &resource_type_list; resource_type_list.dtor = NULL; diff --git a/erts/emulator/beam/erl_nif.h b/erts/emulator/beam/erl_nif.h index 5b93c2398e..bc1f59bc90 100644 --- a/erts/emulator/beam/erl_nif.h +++ b/erts/emulator/beam/erl_nif.h @@ -3,16 +3,17 @@ * * Copyright Ericsson AB 2009-2014. All Rights Reserved. * - * The contents of this file are subject to the Erlang Public License, - * Version 1.1, (the "License"); you may not use this file except in - * compliance with the License. You should have received a copy of the - * Erlang Public License along with this software. If not, it can be - * retrieved online at http://www.erlang.org/. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Software distributed under the License is distributed on an "AS IS" - * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See - * the License for the specific language governing rights and limitations - * under the License. + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. * * %CopyrightEnd% */ @@ -42,9 +43,16 @@ ** 2.5: R17 Maps API additions ** 2.6: R17 with maps ** R17 dirty schedulers +** 2.7: 17.3 add enif_schedule_nif +** remove enif_schedule_dirty_nif, enif_schedule_dirty_nif_finalizer, enif_dirty_nif_finalizer +** add ErlNifEntry options +** add ErlNifFunc flags +** 2.8: 18.0 add enif_has_pending_exception +** 2.9: 18.2 enif_getenv +** 2.10: Time API */ #define ERL_NIF_MAJOR_VERSION 2 -#define ERL_NIF_MINOR_VERSION 6 +#define ERL_NIF_MINOR_VERSION 10 /* * The emulator will refuse to load a nif-lib with a major version @@ -60,63 +68,31 @@ #include <stdlib.h> -#ifdef SIZEOF_CHAR -# define SIZEOF_CHAR_SAVED__ SIZEOF_CHAR -# undef SIZEOF_CHAR -#endif -#ifdef SIZEOF_SHORT -# define SIZEOF_SHORT_SAVED__ SIZEOF_SHORT -# undef SIZEOF_SHORT -#endif -#ifdef SIZEOF_INT -# define SIZEOF_INT_SAVED__ SIZEOF_INT -# undef SIZEOF_INT -#endif -#ifdef SIZEOF_LONG -# define SIZEOF_LONG_SAVED__ SIZEOF_LONG -# undef SIZEOF_LONG -#endif -#ifdef SIZEOF_LONG_LONG -# define SIZEOF_LONG_LONG_SAVED__ SIZEOF_LONG_LONG -# undef SIZEOF_LONG_LONG -#endif -#ifdef HALFWORD_HEAP_EMULATOR -# define HALFWORD_HEAP_EMULATOR_SAVED__ HALFWORD_HEAP_EMULATOR -# undef HALFWORD_HEAP_EMULATOR -#endif -#include "erl_int_sizes_config.h" - #ifdef __cplusplus extern "C" { #endif -#if (defined(__WIN32__) || defined(_WIN32) || defined(_WIN32_)) -typedef unsigned __int64 ErlNifUInt64; -typedef __int64 ErlNifSInt64; -#elif SIZEOF_LONG == 8 -typedef unsigned long ErlNifUInt64; -typedef long ErlNifSInt64; -#elif SIZEOF_LONG_LONG == 8 -typedef unsigned long long ErlNifUInt64; -typedef long long ErlNifSInt64; -#else -#error No 64-bit integer type -#endif +typedef ErlNapiUInt64 ErlNifUInt64; +typedef ErlNapiSInt64 ErlNifSInt64; +typedef ErlNapiUInt ErlNifUInt; +typedef ErlNapiSInt ErlNifSInt; -#ifdef HALFWORD_HEAP_EMULATOR -# define ERL_NIF_VM_VARIANT "beam.halfword" -typedef unsigned int ERL_NIF_TERM; -#else # define ERL_NIF_VM_VARIANT "beam.vanilla" -# if SIZEOF_LONG == SIZEOF_VOID_P -typedef unsigned long ERL_NIF_TERM; -# elif SIZEOF_LONG_LONG == SIZEOF_VOID_P -typedef unsigned long long ERL_NIF_TERM; -# endif -#endif +typedef ErlNifUInt ERL_NIF_TERM; typedef ERL_NIF_TERM ERL_NIF_UINT; +typedef ErlNifSInt64 ErlNifTime; + +#define ERL_NIF_TIME_ERROR ((ErlNifSInt64) ERTS_NAPI_TIME_ERROR__) + +typedef enum { + ERL_NIF_SEC = ERTS_NAPI_SEC__, + ERL_NIF_MSEC = ERTS_NAPI_MSEC__, + ERL_NIF_USEC = ERTS_NAPI_USEC__, + ERL_NIF_NSEC = ERTS_NAPI_NSEC__ +} ErlNifTimeUnit; + struct enif_environment_t; typedef struct enif_environment_t ErlNifEnv; @@ -125,8 +101,10 @@ typedef struct const char* name; unsigned arity; ERL_NIF_TERM (*fptr)(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]); + unsigned flags; }ErlNifFunc; + typedef struct enif_entry_t { int major; @@ -139,8 +117,11 @@ typedef struct enif_entry_t int (*upgrade)(ErlNifEnv*, void** priv_data, void** old_priv_data, ERL_NIF_TERM load_info); void (*unload) (ErlNifEnv*, void* priv_data); const char* vm_variant; + unsigned options; }ErlNifEntry; +/* Field bits for ErlNifEntry options */ +#define ERL_NIF_DIRTY_NIF_OPTION 1 typedef struct @@ -169,7 +150,12 @@ typedef enum typedef struct { ERL_NIF_TERM pid; /* internal, may change */ -}ErlNifPid; +} ErlNifPid; + +typedef struct +{ + ERL_NIF_TERM port_id; /* internal, may change */ +}ErlNifPort; typedef ErlDrvSysInfo ErlNifSysInfo; @@ -192,22 +178,44 @@ typedef enum typedef struct /* All fields all internal and may change */ { ERL_NIF_TERM map; - ERL_NIF_UINT t_limit; + ERL_NIF_UINT size; ERL_NIF_UINT idx; - ERL_NIF_TERM *ks; - ERL_NIF_TERM *vs; + union { + struct { + ERL_NIF_TERM *ks; + ERL_NIF_TERM *vs; + }flat; + struct { + struct ErtsDynamicWStack_* wstack; + ERL_NIF_TERM* kv; + }hash; + }u; void* __spare__[2]; /* for future additions to be ABI compatible (same struct size) */ } ErlNifMapIterator; typedef enum { - ERL_NIF_MAP_ITERATOR_HEAD = 1, - ERL_NIF_MAP_ITERATOR_TAIL = 2 + ERL_NIF_MAP_ITERATOR_FIRST = 1, + ERL_NIF_MAP_ITERATOR_LAST = 2, + + /* deprecated synonyms (undocumented in 17 and 18-rc) */ + ERL_NIF_MAP_ITERATOR_HEAD = ERL_NIF_MAP_ITERATOR_FIRST, + ERL_NIF_MAP_ITERATOR_TAIL = ERL_NIF_MAP_ITERATOR_LAST } ErlNifMapIteratorEntry; +typedef enum { + ERL_NIF_UNIQUE_POSITIVE = (1 << 0), + ERL_NIF_UNIQUE_MONOTONIC = (1 << 1) +} ErlNifUniqueInteger; + +typedef enum { + ERL_NIF_BIN2TERM_SAFE = 0x20000000 +} ErlNifBinaryToTerm; + #if (defined(__WIN32__) || defined(_WIN32) || defined(_WIN32_)) # define ERL_NIF_API_FUNC_DECL(RET_TYPE, NAME, ARGS) RET_TYPE (*NAME) ARGS typedef struct { # include "erl_nif_api_funcs.h" + void* erts_alc_test; } TWinDynNifCallbacks; extern TWinDynNifCallbacks WinDynNifCallbacks; # undef ERL_NIF_API_FUNC_DECL @@ -227,22 +235,33 @@ extern TWinDynNifCallbacks WinDynNifCallbacks; #if (defined(__WIN32__) || defined(_WIN32) || defined(_WIN32_)) # define ERL_NIF_INIT_GLOB TWinDynNifCallbacks WinDynNifCallbacks; -# ifdef STATIC_ERLANG_NIF -# define ERL_NIF_INIT_DECL(MODNAME) __declspec(dllexport) ErlNifEntry* MODNAME ## _nif_init(TWinDynNifCallbacks* callbacks) -# else -# define ERL_NIF_INIT_DECL(MODNAME) __declspec(dllexport) ErlNifEntry* nif_init(TWinDynNifCallbacks* callbacks) -# endif +# define ERL_NIF_INIT_ARGS TWinDynNifCallbacks* callbacks # define ERL_NIF_INIT_BODY memcpy(&WinDynNifCallbacks,callbacks,sizeof(TWinDynNifCallbacks)) +# define ERL_NIF_INIT_EXPORT __declspec(dllexport) #else # define ERL_NIF_INIT_GLOB +# define ERL_NIF_INIT_ARGS void # define ERL_NIF_INIT_BODY -# ifdef STATIC_ERLANG_NIF -# define ERL_NIF_INIT_DECL(MODNAME) ErlNifEntry* MODNAME ## _nif_init(void) +# if defined(__GNUC__) && __GNUC__ >= 4 +# define ERL_NIF_INIT_EXPORT __attribute__ ((visibility("default"))) +# elif defined (__SUNPRO_C) && (__SUNPRO_C >= 0x550) +# define ERL_NIF_INIT_EXPORT __global # else -# define ERL_NIF_INIT_DECL(MODNAME) ErlNifEntry* nif_init(void) +# define ERL_NIF_INIT_EXPORT # endif #endif +#ifdef STATIC_ERLANG_NIF +# define ERL_NIF_INIT_DECL(MODNAME) ErlNifEntry* MODNAME ## _nif_init(ERL_NIF_INIT_ARGS) +#else +# define ERL_NIF_INIT_DECL(MODNAME) ERL_NIF_INIT_EXPORT ErlNifEntry* nif_init(ERL_NIF_INIT_ARGS) +#endif + +#ifdef ERL_NIF_DIRTY_SCHEDULER_SUPPORT +# define ERL_NIF_ENTRY_OPTIONS ERL_NIF_DIRTY_NIF_OPTION +#else +# define ERL_NIF_ENTRY_OPTIONS 0 +#endif #ifdef __cplusplus } @@ -268,7 +287,8 @@ ERL_NIF_INIT_DECL(NAME) \ sizeof(FUNCS) / sizeof(*FUNCS), \ FUNCS, \ LOAD, RELOAD, UPGRADE, UNLOAD, \ - ERL_NIF_VM_VARIANT \ + ERL_NIF_VM_VARIANT, \ + ERL_NIF_ENTRY_OPTIONS \ }; \ ERL_NIF_INIT_BODY; \ return &entry; \ diff --git a/erts/emulator/beam/erl_nif_api_funcs.h b/erts/emulator/beam/erl_nif_api_funcs.h index d7c554e60b..35058afe7c 100644 --- a/erts/emulator/beam/erl_nif_api_funcs.h +++ b/erts/emulator/beam/erl_nif_api_funcs.h @@ -3,16 +3,17 @@ * * Copyright Ericsson AB 2009-2014. All Rights Reserved. * - * The contents of this file are subject to the Erlang Public License, - * Version 1.1, (the "License"); you may not use this file except in - * compliance with the License. You should have received a copy of the - * Erlang Public License along with this software. If not, it can be - * retrieved online at http://www.erlang.org/. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Software distributed under the License is distributed on an "AS IS" - * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See - * the License for the specific language governing rights and limitations - * under the License. + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. * * %CopyrightEnd% */ @@ -22,7 +23,7 @@ #endif /* -** WARNING: add new ERL_NIF_API_FUNC_DECL entries at the bottom of the list +** WARNING: Add new ERL_NIF_API_FUNC_DECL entries at the bottom of the list ** to keep compatibility on Windows!!! ** ** And don't forget to increase ERL_NIF_MINOR_VERSION in erl_nif.h @@ -141,14 +142,6 @@ ERL_NIF_API_FUNC_DECL(int,enif_is_number,(ErlNifEnv*, ERL_NIF_TERM term)); ERL_NIF_API_FUNC_DECL(void*,enif_dlopen,(const char* lib, void (*err_handler)(void*,const char*), void* err_arg)); ERL_NIF_API_FUNC_DECL(void*,enif_dlsym,(void* handle, const char* symbol, void (*err_handler)(void*,const char*), void* err_arg)); ERL_NIF_API_FUNC_DECL(int,enif_consume_timeslice,(ErlNifEnv*, int percent)); -#ifdef ERL_NIF_DIRTY_SCHEDULER_SUPPORT -ERL_NIF_API_FUNC_DECL(ERL_NIF_TERM,enif_schedule_dirty_nif,(ErlNifEnv*,int,ERL_NIF_TERM (*)(ErlNifEnv*,int,const ERL_NIF_TERM[]),int,const ERL_NIF_TERM[])); -ERL_NIF_API_FUNC_DECL(ERL_NIF_TERM,enif_schedule_dirty_nif_finalizer,(ErlNifEnv*,ERL_NIF_TERM,ERL_NIF_TERM (*)(ErlNifEnv*,ERL_NIF_TERM))); -ERL_NIF_API_FUNC_DECL(ERL_NIF_TERM,enif_dirty_nif_finalizer,(ErlNifEnv*,ERL_NIF_TERM)); -ERL_NIF_API_FUNC_DECL(int,enif_is_on_dirty_scheduler,(ErlNifEnv*)); -ERL_NIF_API_FUNC_DECL(int,enif_have_dirty_schedulers,(void)); -#endif - ERL_NIF_API_FUNC_DECL(int, enif_is_map, (ErlNifEnv* env, ERL_NIF_TERM term)); ERL_NIF_API_FUNC_DECL(int, enif_get_map_size, (ErlNifEnv* env, ERL_NIF_TERM term, size_t *size)); ERL_NIF_API_FUNC_DECL(ERL_NIF_TERM, enif_make_new_map, (ErlNifEnv* env)); @@ -163,12 +156,37 @@ ERL_NIF_API_FUNC_DECL(int, enif_map_iterator_is_tail, (ErlNifEnv *env, ErlNifMap ERL_NIF_API_FUNC_DECL(int, enif_map_iterator_next, (ErlNifEnv *env, ErlNifMapIterator *iter)); ERL_NIF_API_FUNC_DECL(int, enif_map_iterator_prev, (ErlNifEnv *env, ErlNifMapIterator *iter)); ERL_NIF_API_FUNC_DECL(int, enif_map_iterator_get_pair, (ErlNifEnv *env, ErlNifMapIterator *iter, ERL_NIF_TERM *key, ERL_NIF_TERM *value)); - +ERL_NIF_API_FUNC_DECL(ERL_NIF_TERM,enif_schedule_nif,(ErlNifEnv*,const char*,int,ERL_NIF_TERM (*)(ErlNifEnv*,int,const ERL_NIF_TERM[]),int,const ERL_NIF_TERM[])); +ERL_NIF_API_FUNC_DECL(int, enif_has_pending_exception, (ErlNifEnv *env, ERL_NIF_TERM* reason)); +ERL_NIF_API_FUNC_DECL(ERL_NIF_TERM, enif_raise_exception, (ErlNifEnv *env, ERL_NIF_TERM reason)); +ERL_NIF_API_FUNC_DECL(int,enif_getenv,(const char* key, char* value, size_t* value_size)); +ERL_NIF_API_FUNC_DECL(ErlNifTime, enif_monotonic_time, (ErlNifTimeUnit)); +ERL_NIF_API_FUNC_DECL(ErlNifTime, enif_time_offset, (ErlNifTimeUnit)); +ERL_NIF_API_FUNC_DECL(ErlNifTime, enif_convert_time_unit, (ErlNifTime, ErlNifTimeUnit, ErlNifTimeUnit)); +ERL_NIF_API_FUNC_DECL(ERL_NIF_TERM, enif_now_time, (ErlNifEnv *env)); +ERL_NIF_API_FUNC_DECL(ERL_NIF_TERM, enif_cpu_time, (ErlNifEnv *env)); +ERL_NIF_API_FUNC_DECL(ERL_NIF_TERM, enif_make_unique_integer, (ErlNifEnv *env, ErlNifUniqueInteger properties)); +ERL_NIF_API_FUNC_DECL(int, enif_is_process_alive, (ErlNifEnv *env, ErlNifPid *pid)); +ERL_NIF_API_FUNC_DECL(int, enif_is_port_alive, (ErlNifEnv *env, ErlNifPort *port_id)); +ERL_NIF_API_FUNC_DECL(int, enif_get_local_port, (ErlNifEnv* env, ERL_NIF_TERM, ErlNifPort* port_id)); +ERL_NIF_API_FUNC_DECL(int, enif_term_to_binary, (ErlNifEnv *env, ERL_NIF_TERM term, ErlNifBinary *bin)); +ERL_NIF_API_FUNC_DECL(size_t, enif_binary_to_term, (ErlNifEnv *env, const unsigned char* data, size_t sz, ERL_NIF_TERM *term, unsigned int opts)); +ERL_NIF_API_FUNC_DECL(int, enif_port_command, (ErlNifEnv *env, const ErlNifPort* to_port, ErlNifEnv *msg_env, ERL_NIF_TERM msg)); /* -** Add new entries here to keep compatibility on Windows!!! +** ADD NEW ENTRIES HERE (before this comment) !!! */ + + +/* + * Conditional EXPERIMENTAL stuff always last. + * Must be moved up and made unconditional to support binary backward + * compatibility on Windows. + */ +#ifdef ERL_NIF_DIRTY_SCHEDULER_SUPPORT +ERL_NIF_API_FUNC_DECL(int,enif_is_on_dirty_scheduler,(ErlNifEnv*)); #endif +#endif /* ERL_NIF_API_FUNC_DECL */ /* ** Please keep the ERL_NIF_API_FUNC_MACRO list below in the same order @@ -282,21 +300,12 @@ ERL_NIF_API_FUNC_DECL(int, enif_map_iterator_get_pair, (ErlNifEnv *env, ErlNifMa # define enif_make_int64 ERL_NIF_API_FUNC_MACRO(enif_make_int64) # define enif_make_uint64 ERL_NIF_API_FUNC_MACRO(enif_make_uint64) #endif - # define enif_is_exception ERL_NIF_API_FUNC_MACRO(enif_is_exception) # define enif_make_reverse_list ERL_NIF_API_FUNC_MACRO(enif_make_reverse_list) # define enif_is_number ERL_NIF_API_FUNC_MACRO(enif_is_number) # define enif_dlopen ERL_NIF_API_FUNC_MACRO(enif_dlopen) # define enif_dlsym ERL_NIF_API_FUNC_MACRO(enif_dlsym) # define enif_consume_timeslice ERL_NIF_API_FUNC_MACRO(enif_consume_timeslice) -#ifdef ERL_NIF_DIRTY_SCHEDULER_SUPPORT -# define enif_schedule_dirty_nif ERL_NIF_API_FUNC_MACRO(enif_schedule_dirty_nif) -# define enif_schedule_dirty_nif_finalizer ERL_NIF_API_FUNC_MACRO(enif_schedule_dirty_nif_finalizer) -# define enif_dirty_nif_finalizer ERL_NIF_API_FUNC_MACRO(enif_dirty_nif_finalizer) -# define enif_is_on_dirty_scheduler ERL_NIF_API_FUNC_MACRO(enif_is_on_dirty_scheduler) -# define enif_have_dirty_schedulers ERL_NIF_API_FUNC_MACRO(enif_have_dirty_schedulers) -#endif - # define enif_is_map ERL_NIF_API_FUNC_MACRO(enif_is_map) # define enif_get_map_size ERL_NIF_API_FUNC_MACRO(enif_get_map_size) # define enif_make_new_map ERL_NIF_API_FUNC_MACRO(enif_make_new_map) @@ -311,11 +320,36 @@ ERL_NIF_API_FUNC_DECL(int, enif_map_iterator_get_pair, (ErlNifEnv *env, ErlNifMa # define enif_map_iterator_next ERL_NIF_API_FUNC_MACRO(enif_map_iterator_next) # define enif_map_iterator_prev ERL_NIF_API_FUNC_MACRO(enif_map_iterator_prev) # define enif_map_iterator_get_pair ERL_NIF_API_FUNC_MACRO(enif_map_iterator_get_pair) +# define enif_schedule_nif ERL_NIF_API_FUNC_MACRO(enif_schedule_nif) +# define enif_has_pending_exception ERL_NIF_API_FUNC_MACRO(enif_has_pending_exception) +# define enif_raise_exception ERL_NIF_API_FUNC_MACRO(enif_raise_exception) +# define enif_getenv ERL_NIF_API_FUNC_MACRO(enif_getenv) +# define enif_monotonic_time ERL_NIF_API_FUNC_MACRO(enif_monotonic_time) +# define enif_time_offset ERL_NIF_API_FUNC_MACRO(enif_time_offset) +# define enif_convert_time_unit ERL_NIF_API_FUNC_MACRO(enif_convert_time_unit) +# define enif_now_time ERL_NIF_API_FUNC_MACRO(enif_now_time) +# define enif_cpu_time ERL_NIF_API_FUNC_MACRO(enif_cpu_time) +# define enif_make_unique_integer ERL_NIF_API_FUNC_MACRO(enif_make_unique_integer) +# define enif_is_process_alive ERL_NIF_API_FUNC_MACRO(enif_is_process_alive) +# define enif_is_port_alive ERL_NIF_API_FUNC_MACRO(enif_is_port_alive) +# define enif_get_local_port ERL_NIF_API_FUNC_MACRO(enif_get_local_port) +# define enif_term_to_binary ERL_NIF_API_FUNC_MACRO(enif_term_to_binary) +# define enif_binary_to_term ERL_NIF_API_FUNC_MACRO(enif_binary_to_term) +# define enif_port_command ERL_NIF_API_FUNC_MACRO(enif_port_command) /* -** Add new entries here +** ADD NEW ENTRIES HERE (before this comment) */ + +/* + * Conditional EXPERIMENTAL stuff always last + * Must be moved up and made unconditional to support binary backward + * compatibility on Windows. + */ +#ifdef ERL_NIF_DIRTY_SCHEDULER_SUPPORT +# define enif_is_on_dirty_scheduler ERL_NIF_API_FUNC_MACRO(enif_is_on_dirty_scheduler) #endif +#endif /* ERL_NIF_API_FUNC_MACRO */ #if defined(__GNUC__) && !(defined(__WIN32__) || defined(_WIN32) || defined(_WIN32_)) @@ -535,7 +569,7 @@ static ERL_NIF_INLINE ERL_NIF_TERM enif_make_list9(ErlNifEnv* env, #ifndef enif_make_pid -# define enif_make_pid(ENV, PID) ((const ERL_NIF_TERM)((PID)->pid)) +# define enif_make_pid(ENV, PID) ((void)(ENV),(const ERL_NIF_TERM)((PID)->pid)) #if SIZEOF_LONG == 8 # define enif_get_int64 enif_get_long diff --git a/erts/emulator/beam/erl_node_container_utils.h b/erts/emulator/beam/erl_node_container_utils.h index 17f6b32bb1..b6a3531e28 100644 --- a/erts/emulator/beam/erl_node_container_utils.h +++ b/erts/emulator/beam/erl_node_container_utils.h @@ -3,16 +3,17 @@ * * Copyright Ericsson AB 2001-2013. All Rights Reserved. * - * The contents of this file are subject to the Erlang Public License, - * Version 1.1, (the "License"); you may not use this file except in - * compliance with the License. You should have received a copy of the - * Erlang Public License along with this software. If not, it can be - * retrieved online at http://www.erlang.org/. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Software distributed under the License is distributed on an "AS IS" - * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See - * the License for the specific language governing rights and limitations - * under the License. + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. * * %CopyrightEnd% */ @@ -254,7 +255,7 @@ extern ErtsPTab erts_port; * Refs * \* */ -#if defined(ARCH_64) && !HALFWORD_HEAP +#if defined(ARCH_64) #define internal_ref_no_of_numbers(x) \ (internal_ref_data((x))[0]) @@ -327,8 +328,6 @@ extern ErtsPTab erts_port; : external_ref_channel_no((x))) #define is_ref(x) (is_internal_ref((x)) \ || is_external_ref((x))) -#define is_ref_rel(x,Base) (is_internal_ref_rel((x),Base) \ - || is_external_ref_rel((x),Base)) #define is_not_ref(x) (!is_ref(x)) #endif diff --git a/erts/emulator/beam/erl_node_tables.c b/erts/emulator/beam/erl_node_tables.c index c6d136f951..8617f42d7b 100644 --- a/erts/emulator/beam/erl_node_tables.c +++ b/erts/emulator/beam/erl_node_tables.c @@ -3,16 +3,17 @@ * * Copyright Ericsson AB 2001-2013. All Rights Reserved. * - * The contents of this file are subject to the Erlang Public License, - * Version 1.1, (the "License"); you may not use this file except in - * compliance with the License. You should have received a copy of the - * Erlang Public License along with this software. If not, it can be - * retrieved online at http://www.erlang.org/. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Software distributed under the License is distributed on an "AS IS" - * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See - * the License for the specific language governing rights and limitations - * under the License. + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. * * %CopyrightEnd% */ @@ -36,21 +37,24 @@ erts_smp_rwmtx_t erts_node_table_rwmtx; DistEntry *erts_hidden_dist_entries; DistEntry *erts_visible_dist_entries; -DistEntry *erts_not_connected_dist_entries; +DistEntry *erts_not_connected_dist_entries; /* including erts_this_dist_entry */ Sint erts_no_of_hidden_dist_entries; Sint erts_no_of_visible_dist_entries; -Sint erts_no_of_not_connected_dist_entries; +Sint erts_no_of_not_connected_dist_entries; /* including erts_this_dist_entry */ DistEntry *erts_this_dist_entry; ErlNode *erts_this_node; char erts_this_node_sysname_BUFFER[256], *erts_this_node_sysname = "uninitialized yet"; -static Uint node_entries; -static Uint dist_entries; +static Uint node_entries = 0; +static Uint dist_entries = 0; static int references_atoms_need_init = 1; +static ErtsMonotonicTime orig_node_tab_delete_delay; +static ErtsMonotonicTime node_tab_delete_delay; + /* -- The distribution table ---------------------------------------------- */ #ifdef DEBUG @@ -87,9 +91,6 @@ dist_table_alloc(void *dep_tmpl) erts_smp_rwmtx_opt_t rwmtx_opt = ERTS_SMP_RWMTX_OPT_DEFAULT_INITER; rwmtx_opt.type = ERTS_SMP_RWMTX_TYPE_FREQUENT_READ; - if(((DistEntry *) dep_tmpl) == erts_this_dist_entry) - return dep_tmpl; - sysname = ((DistEntry *) dep_tmpl)->sysname; chnl_nr = make_small((Uint) atom_val(sysname)); dep = (DistEntry *) erts_alloc(ERTS_ALC_T_DIST_ENTRY, sizeof(DistEntry)); @@ -128,7 +129,9 @@ dist_table_alloc(void *dep_tmpl) /* Link in */ - /* All new dist entries are "not connected" */ + /* All new dist entries are "not connected". + * erts_this_dist_entry is also always included among "not connected" + */ dep->next = erts_not_connected_dist_entries; if(erts_not_connected_dist_entries) { ASSERT(erts_not_connected_dist_entries->prev == NULL); @@ -145,9 +148,6 @@ dist_table_free(void *vdep) { DistEntry *dep = (DistEntry *) vdep; - if(dep == erts_this_dist_entry) - return; - ASSERT(is_nil(dep->cid)); ASSERT(dep->nlinks == NULL); ASSERT(dep->node_links == NULL); @@ -182,7 +182,7 @@ dist_table_free(void *vdep) #endif erts_free(ERTS_ALC_T_DIST_ENTRY, (void *) dep); - ASSERT(dist_entries > 1); + ASSERT(dist_entries > 0); dist_entries--; } @@ -290,21 +290,46 @@ DistEntry *erts_find_dist_entry(Eterm sysname) return res; } -void erts_delete_dist_entry(DistEntry *dep) +static void try_delete_dist_entry(void *vdep) +{ + DistEntry *dep = (DistEntry *) vdep; + erts_aint_t refc; + + erts_smp_rwmtx_rwlock(&erts_dist_table_rwmtx); + /* + * Another thread might have looked up this dist entry after + * we decided to delete it (refc became zero). If so, the other + * thread incremented refc twice. Once for the new reference + * and once for this thread. + * + * If refc reach -1, no one has used the entry since we + * set up the timer. Delete the entry. + * + * If refc reach 0, the entry is currently not in use + * but has been used since we set up the timer. Set up a + * new timer. + * + * If refc > 0, the entry is in use. Keep the entry. + */ + refc = erts_refc_dectest(&dep->refc, -1); + if (refc == -1) + (void) hash_erase(&erts_dist_table, (void *) dep); + erts_smp_rwmtx_rwunlock(&erts_dist_table_rwmtx); + + if (refc == 0) + erts_schedule_delete_dist_entry(dep); +} + +void erts_schedule_delete_dist_entry(DistEntry *dep) { ASSERT(dep != erts_this_dist_entry); - if(dep != erts_this_dist_entry) { - erts_smp_rwmtx_rwlock(&erts_dist_table_rwmtx); - /* - * Another thread might have looked up this dist entry after - * we decided to delete it (refc became zero). If so, the other - * thread incremented refc twice. Once for the new reference - * and once for this thread. Therefore, delete dist entry if - * refc is 0 or -1 after a decrement. - */ - if (erts_refc_dectest(&dep->refc, -1) <= 0) - (void) hash_erase(&erts_dist_table, (void *) dep); - erts_smp_rwmtx_rwunlock(&erts_dist_table_rwmtx); + if (dep != erts_this_dist_entry) { + if (node_tab_delete_delay == 0) + try_delete_dist_entry((void *) dep); + else if (node_tab_delete_delay > 0) + erts_start_timer_callback(node_tab_delete_delay, + try_delete_dist_entry, + (void *) dep); } } @@ -340,8 +365,7 @@ erts_dist_table_size(void) ASSERT(dist_entries == (erts_no_of_visible_dist_entries + erts_no_of_hidden_dist_entries - + erts_no_of_not_connected_dist_entries - + 1 /* erts_this_dist_entry */)); + + erts_no_of_not_connected_dist_entries)); #endif res = (hash_table_sz(&erts_dist_table) @@ -514,9 +538,6 @@ node_table_alloc(void *venp_tmpl) { ErlNode *enp; - if(((ErlNode *) venp_tmpl) == erts_this_node) - return venp_tmpl; - enp = (ErlNode *) erts_alloc(ERTS_ALC_T_NODE_ENTRY, sizeof(ErlNode)); node_entries++; @@ -534,8 +555,7 @@ node_table_free(void *venp) { ErlNode *enp = (ErlNode *) venp; - if(enp == erts_this_node) - return; + ERTS_SMP_LC_ASSERT(enp != erts_this_node || erts_thr_progress_is_blocking()); erts_deref_dist_entry(enp->dist_entry); #ifdef DEBUG @@ -543,7 +563,7 @@ node_table_free(void *venp) #endif erts_free(ERTS_ALC_T_NODE_ENTRY, venp); - ASSERT(node_entries > 1); + ASSERT(node_entries > 0); node_entries--; } @@ -556,14 +576,14 @@ erts_node_table_size(void) #endif int lock = !ERTS_IS_CRASH_DUMPING; if (lock) - erts_smp_rwmtx_rwlock(&erts_node_table_rwmtx); + erts_smp_rwmtx_rlock(&erts_node_table_rwmtx); #ifdef DEBUG hash_get_info(&hi, &erts_node_table); ASSERT(node_entries == hi.objs); #endif res = hash_table_sz(&erts_node_table) + node_entries*sizeof(ErlNode); if (lock) - erts_smp_rwmtx_rwunlock(&erts_node_table_rwmtx); + erts_smp_rwmtx_runlock(&erts_node_table_rwmtx); return res; } @@ -572,10 +592,10 @@ erts_node_table_info(int to, void *to_arg) { int lock = !ERTS_IS_CRASH_DUMPING; if (lock) - erts_smp_rwmtx_rwlock(&erts_node_table_rwmtx); + erts_smp_rwmtx_rlock(&erts_node_table_rwmtx); hash_info(to, to_arg, &erts_node_table); if (lock) - erts_smp_rwmtx_rwunlock(&erts_node_table_rwmtx); + erts_smp_rwmtx_runlock(&erts_node_table_rwmtx); } @@ -609,21 +629,46 @@ ErlNode *erts_find_or_insert_node(Eterm sysname, Uint creation) return res; } -void erts_delete_node(ErlNode *enp) +static void try_delete_node(void *venp) +{ + ErlNode *enp = (ErlNode *) venp; + erts_aint_t refc; + + erts_smp_rwmtx_rwlock(&erts_node_table_rwmtx); + /* + * Another thread might have looked up this node after we + * decided to delete it (refc became zero). If so, the other + * thread incremented refc twice. Once for the new reference + * and once for this thread. + * + * If refc reach -1, no one has used the entry since we + * set up the timer. Delete the entry. + * + * If refc reach 0, the entry is currently not in use + * but has been used since we set up the timer. Set up a + * new timer. + * + * If refc > 0, the entry is in use. Keep the entry. + */ + refc = erts_refc_dectest(&enp->refc, -1); + if (refc == -1) + (void) hash_erase(&erts_node_table, (void *) enp); + erts_smp_rwmtx_rwunlock(&erts_node_table_rwmtx); + + if (refc == 0) + erts_schedule_delete_node(enp); +} + +void erts_schedule_delete_node(ErlNode *enp) { ASSERT(enp != erts_this_node); - if(enp != erts_this_node) { - erts_smp_rwmtx_rwlock(&erts_node_table_rwmtx); - /* - * Another thread might have looked up this node after we - * decided to delete it (refc became zero). If so, the other - * thread incremented refc twice. Once for the new reference - * and once for this thread. Therefore, delete node if refc - * is 0 or -1 after a decrement. - */ - if (erts_refc_dectest(&enp->refc, -1) <= 0) - (void) hash_erase(&erts_node_table, (void *) enp); - erts_smp_rwmtx_rwunlock(&erts_node_table_rwmtx); + if (enp != erts_this_node) { + if (node_tab_delete_delay == 0) + try_delete_node((void *) enp); + else if (node_tab_delete_delay > 0) + erts_start_timer_callback(node_tab_delete_delay, + try_delete_node, + (void *) enp); } } @@ -651,7 +696,7 @@ static void print_node(void *venp, void *vpndp) erts_print(pndp->to, pndp->to_arg, " %d", enp->creation); #ifdef DEBUG erts_print(pndp->to, pndp->to_arg, " (refc=%ld)", - erts_refc_read(&enp->refc, 1)); + erts_refc_read(&enp->refc, 0)); #endif pndp->no_sysname++; } @@ -674,13 +719,13 @@ void erts_print_node_info(int to, pnd.no_total = 0; if (lock) - erts_smp_rwmtx_rwlock(&erts_node_table_rwmtx); + erts_smp_rwmtx_rlock(&erts_node_table_rwmtx); hash_foreach(&erts_node_table, print_node, (void *) &pnd); if (pnd.no_sysname != 0) { erts_print(to, to_arg, "\n"); } if (lock) - erts_smp_rwmtx_rwunlock(&erts_node_table_rwmtx); + erts_smp_rwmtx_runlock(&erts_node_table_rwmtx); if(no_sysname) *no_sysname = pnd.no_sysname; @@ -693,45 +738,70 @@ void erts_print_node_info(int to, void erts_set_this_node(Eterm sysname, Uint creation) { - erts_smp_rwmtx_rwlock(&erts_node_table_rwmtx); - erts_smp_rwmtx_rwlock(&erts_dist_table_rwmtx); + ERTS_SMP_LC_ASSERT(erts_thr_progress_is_blocking()); + ASSERT(erts_refc_read(&erts_this_dist_entry->refc, 2)); - (void) hash_erase(&erts_dist_table, (void *) erts_this_dist_entry); - erts_this_dist_entry->sysname = sysname; - erts_this_dist_entry->creation = creation; - (void) hash_put(&erts_dist_table, (void *) erts_this_dist_entry); + if (erts_refc_dectest(&erts_this_node->refc, 0) == 0) + try_delete_node(erts_this_node); - (void) hash_erase(&erts_node_table, (void *) erts_this_node); - erts_this_node->sysname = sysname; - erts_this_node->creation = creation; - erts_this_node_sysname = erts_this_node_sysname_BUFFER; - erts_snprintf(erts_this_node_sysname, sizeof(erts_this_node_sysname), - "%T", sysname); - (void) hash_put(&erts_node_table, (void *) erts_this_node); + if (erts_refc_dectest(&erts_this_dist_entry->refc, 0) == 0) + try_delete_dist_entry(erts_this_dist_entry); - erts_smp_rwmtx_rwunlock(&erts_dist_table_rwmtx); - erts_smp_rwmtx_rwunlock(&erts_node_table_rwmtx); + erts_this_node = NULL; /* to make sure refc is bumped for this node */ + erts_this_node = erts_find_or_insert_node(sysname, creation); + erts_this_dist_entry = erts_this_node->dist_entry; + erts_refc_inc(&erts_this_dist_entry->refc, 2); + + erts_this_node_sysname = erts_this_node_sysname_BUFFER; + erts_snprintf(erts_this_node_sysname, sizeof(erts_this_node_sysname_BUFFER), + "%T", sysname); } -void erts_init_node_tables(void) +Uint +erts_delayed_node_table_gc(void) +{ + if (node_tab_delete_delay < 0) + return (Uint) ERTS_NODE_TAB_DELAY_GC_INFINITY; + if (node_tab_delete_delay == 0) + return (Uint) 0; + return (Uint) ((node_tab_delete_delay-1)/1000 + 1); +} + +void erts_init_node_tables(int dd_sec) { erts_smp_rwmtx_opt_t rwmtx_opt = ERTS_SMP_RWMTX_OPT_DEFAULT_INITER; HashFunctions f; + ErlNode node_tmpl; + + if (dd_sec == ERTS_NODE_TAB_DELAY_GC_INFINITY) + node_tab_delete_delay = (ErtsMonotonicTime) -1; + else + node_tab_delete_delay = ((ErtsMonotonicTime) dd_sec)*1000; + + orig_node_tab_delete_delay = node_tab_delete_delay; rwmtx_opt.type = ERTS_SMP_RWMTX_TYPE_FREQUENT_READ; rwmtx_opt.lived = ERTS_SMP_RWMTX_LONG_LIVED; - f.hash = (H_FUN) dist_table_hash; - f.cmp = (HCMP_FUN) dist_table_cmp; - f.alloc = (HALLOC_FUN) dist_table_alloc; - f.free = (HFREE_FUN) dist_table_free; - - erts_this_dist_entry = erts_alloc(ERTS_ALC_T_DIST_ENTRY, sizeof(DistEntry)); - dist_entries = 1; + erts_smp_rwmtx_init_opt(&erts_node_table_rwmtx, &rwmtx_opt, "node_table"); + erts_smp_rwmtx_init_opt(&erts_dist_table_rwmtx, &rwmtx_opt, "dist_table"); + f.hash = (H_FUN) dist_table_hash; + f.cmp = (HCMP_FUN) dist_table_cmp; + f.alloc = (HALLOC_FUN) dist_table_alloc; + f.free = (HFREE_FUN) dist_table_free; + f.meta_alloc = (HMALLOC_FUN) erts_alloc; + f.meta_free = (HMFREE_FUN) erts_free; + f.meta_print = (HMPRINT_FUN) erts_print; hash_init(ERTS_ALC_T_DIST_TABLE, &erts_dist_table, "dist_table", 11, f); + f.hash = (H_FUN) node_table_hash; + f.cmp = (HCMP_FUN) node_table_cmp; + f.alloc = (HALLOC_FUN) node_table_alloc; + f.free = (HFREE_FUN) node_table_free; + hash_init(ERTS_ALC_T_NODE_TABLE, &erts_node_table, "node_table", 11, f); + erts_hidden_dist_entries = NULL; erts_visible_dist_entries = NULL; erts_not_connected_dist_entries = NULL; @@ -739,69 +809,23 @@ void erts_init_node_tables(void) erts_no_of_visible_dist_entries = 0; erts_no_of_not_connected_dist_entries = 0; - erts_this_dist_entry->next = NULL; - erts_this_dist_entry->prev = NULL; - erts_refc_init(&erts_this_dist_entry->refc, 1); /* erts_this_node */ - - erts_smp_rwmtx_init_opt_x(&erts_this_dist_entry->rwmtx, - &rwmtx_opt, - "dist_entry", - make_small(ERST_INTERNAL_CHANNEL_NO)); - erts_this_dist_entry->sysname = am_Noname; - erts_this_dist_entry->cid = NIL; - erts_this_dist_entry->connection_id = 0; - erts_this_dist_entry->status = 0; - erts_this_dist_entry->flags = 0; - erts_this_dist_entry->version = 0; - - erts_smp_mtx_init_x(&erts_this_dist_entry->lnk_mtx, - "dist_entry_links", - make_small(ERST_INTERNAL_CHANNEL_NO)); - erts_this_dist_entry->node_links = NULL; - erts_this_dist_entry->nlinks = NULL; - erts_this_dist_entry->monitors = NULL; - - erts_smp_mtx_init_x(&erts_this_dist_entry->qlock, - "dist_entry_out_queue", - make_small(ERST_INTERNAL_CHANNEL_NO)); - erts_this_dist_entry->qflgs = 0; - erts_this_dist_entry->qsize = 0; - erts_this_dist_entry->out_queue.first = NULL; - erts_this_dist_entry->out_queue.last = NULL; - erts_this_dist_entry->suspended = NULL; - - erts_this_dist_entry->finalized_out_queue.first = NULL; - erts_this_dist_entry->finalized_out_queue.last = NULL; - erts_smp_atomic_init_nob(&erts_this_dist_entry->dist_cmd_scheduled, 0); - erts_port_task_handle_init(&erts_this_dist_entry->dist_cmd); - erts_this_dist_entry->send = NULL; - erts_this_dist_entry->cache = NULL; - - (void) hash_put(&erts_dist_table, (void *) erts_this_dist_entry); + node_tmpl.sysname = am_Noname; + node_tmpl.creation = 0; + erts_this_node = hash_put(&erts_node_table, &node_tmpl); + /* +1 for erts_this_node */ + erts_refc_init(&erts_this_node->refc, 1); - f.hash = (H_FUN) node_table_hash; - f.cmp = (HCMP_FUN) node_table_cmp; - f.alloc = (HALLOC_FUN) node_table_alloc; - f.free = (HFREE_FUN) node_table_free; + ASSERT(erts_this_node->dist_entry != NULL); + erts_this_dist_entry = erts_this_node->dist_entry; + /* +1 for erts_this_dist_entry */ + /* +1 for erts_this_node->dist_entry */ + erts_refc_init(&erts_this_dist_entry->refc, 2); - hash_init(ERTS_ALC_T_NODE_TABLE, &erts_node_table, "node_table", 11, f); - - erts_this_node = erts_alloc(ERTS_ALC_T_NODE_ENTRY, sizeof(ErlNode)); - node_entries = 1; - erts_refc_init(&erts_this_node->refc, 1); /* The system itself */ - erts_this_node->sysname = am_Noname; - erts_this_node->creation = 0; - erts_this_node->dist_entry = erts_this_dist_entry; erts_this_node_sysname = erts_this_node_sysname_BUFFER; - erts_snprintf(erts_this_node_sysname, sizeof(erts_this_node_sysname), + erts_snprintf(erts_this_node_sysname, sizeof(erts_this_node_sysname_BUFFER), "%T", erts_this_node->sysname); - (void) hash_put(&erts_node_table, (void *) erts_this_node); - - erts_smp_rwmtx_init_opt(&erts_node_table_rwmtx, &rwmtx_opt, "node_table"); - erts_smp_rwmtx_init_opt(&erts_dist_table_rwmtx, &rwmtx_opt, "dist_table"); - references_atoms_need_init = 1; } @@ -847,6 +871,7 @@ static Eterm AM_dist_references; static Eterm AM_node_references; static Eterm AM_system; static Eterm AM_timer; +static Eterm AM_delayed_delete_timer; static void setup_reference_table(void); static Eterm reference_table_term(Uint **hpp, Uint *szp); @@ -881,8 +906,10 @@ typedef struct dist_referrer_ { int heap_ref; int node_ref; int ctrl_ref; + int system_ref; Eterm id; Uint creation; + Uint id_heap[ID_HEAP_SIZE]; } DistReferrer; typedef struct { @@ -931,6 +958,7 @@ erts_get_node_and_dist_references(struct process *proc) INIT_AM(node_references); INIT_AM(timer); INIT_AM(system); + INIT_AM(delayed_delete_timer); references_atoms_need_init = 0; } @@ -988,17 +1016,25 @@ insert_dist_referrer(ReferredDist *referred_dist, sizeof(DistReferrer)); drp->next = referred_dist->referrers; referred_dist->referrers = drp; - drp->id = id; + if(IS_CONST(id)) + drp->id = id; + else { + Uint *hp = &drp->id_heap[0]; + ASSERT(is_tuple(id)); + drp->id = copy_struct(id, size_object(id), &hp, NULL); + } drp->creation = creation; drp->heap_ref = 0; drp->node_ref = 0; drp->ctrl_ref = 0; + drp->system_ref = 0; } switch (type) { case NODE_REF: drp->node_ref++; break; case CTRL_REF: drp->ctrl_ref++; break; case HEAP_REF: drp->heap_ref++; break; + case SYSTEM_REF: drp->system_ref++; break; default: ASSERT(0); } } @@ -1017,7 +1053,7 @@ insert_dist_entry(DistEntry *dist, int type, Eterm id, Uint creation) } if(!rdp) - erl_exit(1, + erts_exit(ERTS_ERROR_EXIT, "Reference to non-existing distribution table entry found!\n"); insert_dist_referrer(rdp, type, id, creation); @@ -1078,7 +1114,7 @@ insert_node(ErlNode *node, int type, Eterm id) } if (!rnp) - erl_exit(1, "Reference to non-existing node table entry found!\n"); + erts_exit(ERTS_ERROR_EXIT, "Reference to non-existing node table entry found!\n"); insert_node_referrer(rnp, type, id); } @@ -1124,23 +1160,11 @@ insert_offheap(ErlOffHeap *oh, int type, Eterm id) break; } if (insert_bin) { -#if HALFWORD_HEAP - UWord val = (UWord) u.pb->val; - DeclareTmpHeapNoproc(id_heap,BIG_UINT_HEAP_SIZE*2); /* extra place allocated */ -#else DeclareTmpHeapNoproc(id_heap,BIG_UINT_HEAP_SIZE); -#endif Uint *hp = &id_heap[0]; InsertedBin *nib; -#if HALFWORD_HEAP - int actual_need = BIG_UWORD_HEAP_SIZE(val); - ASSERT(actual_need <= (BIG_UINT_HEAP_SIZE*2)); - UseTmpHeapNoproc(actual_need); - a.id = erts_bld_uword(&hp, NULL, (UWord) val); -#else UseTmpHeapNoproc(BIG_UINT_HEAP_SIZE); a.id = erts_bld_uint(&hp, NULL, (Uint) u.pb->val); -#endif erts_match_prog_foreach_offheap(u.pb->val, insert_offheap2, (void *) &a); @@ -1148,11 +1172,7 @@ insert_offheap(ErlOffHeap *oh, int type, Eterm id) nib->bin_val = u.pb->val; nib->next = inserted_bins; inserted_bins = nib; -#if HALFWORD_HEAP - UnUseTmpHeapNoproc(actual_need); -#else UnUseTmpHeapNoproc(BIG_UINT_HEAP_SIZE); -#endif } } break; @@ -1261,6 +1281,33 @@ insert_sys_msg(Eterm from, Eterm to, Eterm msg, ErlHeapFragment *bp) #endif static void +insert_delayed_delete_node(void *state, + ErtsMonotonicTime timeout_pos, + void *vnp) +{ + DeclareTmpHeapNoproc(heap,3); + UseTmpHeapNoproc(3); + insert_node((ErlNode *) vnp, + SYSTEM_REF, + TUPLE2(&heap[0], AM_system, AM_delayed_delete_timer)); + UnUseTmpHeapNoproc(3); +} + +static void +insert_delayed_delete_dist_entry(void *state, + ErtsMonotonicTime timeout_pos, + void *vdep) +{ + DeclareTmpHeapNoproc(heap,3); + UseTmpHeapNoproc(3); + insert_dist_entry((DistEntry *) vdep, + SYSTEM_REF, + TUPLE2(&heap[0], AM_system, AM_delayed_delete_timer), + 0); + UnUseTmpHeapNoproc(3); +} + +static void setup_reference_table(void) { ErlHeapFragment *hfp; @@ -1288,11 +1335,22 @@ setup_reference_table(void) /* Go through the hole system, and build a table of all references to ErlNode and DistEntry structures */ + erts_debug_callback_timer_foreach(try_delete_node, + insert_delayed_delete_node, + NULL); + erts_debug_callback_timer_foreach(try_delete_dist_entry, + insert_delayed_delete_dist_entry, + NULL); + UseTmpHeapNoproc(3); insert_node(erts_this_node, SYSTEM_REF, TUPLE2(&heap[0], AM_system, am_undefined)); + insert_dist_entry(erts_this_dist_entry, + SYSTEM_REF, + TUPLE2(&heap[0], AM_system, am_undefined), + erts_this_node->creation); UnUseTmpHeapNoproc(3); max = erts_ptab_max(&erts_proc); @@ -1300,67 +1358,55 @@ setup_reference_table(void) for (i = 0; i < max; i++) { Process *proc = erts_pix2proc(i); if (proc) { - ErlMessage *msg; + int mli; + ErtsMessage *msg_list[] = { + proc->msg.first, +#ifdef ERTS_SMP + proc->msg_inq.first, +#endif + proc->msg_frag}; /* Insert Heap */ insert_offheap(&(proc->off_heap), HEAP_REF, proc->common.id); - /* Insert message buffers */ + /* Insert heap fragments buffers */ for(hfp = proc->mbuf; hfp; hfp = hfp->next) insert_offheap(&(hfp->off_heap), HEAP_REF, proc->common.id); - /* Insert msg msg buffers */ - for (msg = proc->msg.first; msg; msg = msg->next) { - ErlHeapFragment *heap_frag = NULL; - if (msg->data.attached) { - if (is_value(ERL_MESSAGE_TERM(msg))) - heap_frag = msg->data.heap_frag; - else { - if (msg->data.dist_ext->dep) - insert_dist_entry(msg->data.dist_ext->dep, - HEAP_REF, proc->common.id, 0); - if (is_not_nil(ERL_MESSAGE_TOKEN(msg))) - heap_frag = erts_dist_ext_trailer(msg->data.dist_ext); + + /* Insert msg buffers */ + for (mli = 0; mli < sizeof(msg_list)/sizeof(msg_list[0]); mli++) { + ErtsMessage *msg; + for (msg = msg_list[mli]; msg; msg = msg->next) { + ErlHeapFragment *heap_frag = NULL; + if (msg->data.attached) { + if (msg->data.attached == ERTS_MSG_COMBINED_HFRAG) + heap_frag = &msg->hfrag; + else if (is_value(ERL_MESSAGE_TERM(msg))) + heap_frag = msg->data.heap_frag; + else { + if (msg->data.dist_ext->dep) + insert_dist_entry(msg->data.dist_ext->dep, + HEAP_REF, proc->common.id, 0); + if (is_not_nil(ERL_MESSAGE_TOKEN(msg))) + heap_frag = erts_dist_ext_trailer(msg->data.dist_ext); + } } - } - if (heap_frag) - insert_offheap(&(heap_frag->off_heap), - HEAP_REF, - proc->common.id); - } -#ifdef ERTS_SMP - for (msg = proc->msg_inq.first; msg; msg = msg->next) { - ErlHeapFragment *heap_frag = NULL; - if (msg->data.attached) { - if (is_value(ERL_MESSAGE_TERM(msg))) - heap_frag = msg->data.heap_frag; - else { - if (msg->data.dist_ext->dep) - insert_dist_entry(msg->data.dist_ext->dep, - HEAP_REF, proc->common.id, 0); - if (is_not_nil(ERL_MESSAGE_TOKEN(msg))) - heap_frag = erts_dist_ext_trailer(msg->data.dist_ext); + while (heap_frag) { + insert_offheap(&(heap_frag->off_heap), + HEAP_REF, + proc->common.id); + heap_frag = heap_frag->next; } } - if (heap_frag) - insert_offheap(&(heap_frag->off_heap), - HEAP_REF, - proc->common.id); } -#endif /* Insert links */ if (ERTS_P_LINKS(proc)) insert_links(ERTS_P_LINKS(proc), proc->common.id); if (ERTS_P_MONITORS(proc)) insert_monitors(ERTS_P_MONITORS(proc), proc->common.id); - /* Insert controller */ - { - DistEntry *dep = ERTS_PROC_GET_DIST_ENTRY(proc); - if (dep) - insert_dist_entry(dep, CTRL_REF, proc->common.id, 0); - } } } @@ -1469,7 +1515,7 @@ setup_reference_table(void) erts_db_foreach_table(insert_ets_table, NULL); /* Insert all bif timers */ - erts_bif_timer_foreach(insert_bif_timer, NULL); + erts_debug_bif_timer_foreach(insert_bif_timer, NULL); /* Insert node table (references to dist) */ hash_foreach(&erts_node_table, insert_erl_node, NULL); @@ -1601,7 +1647,7 @@ reference_table_term(Uint **hpp, Uint *szp) tup = MK_2TUP(referred_nodes[i].node->sysname, MK_UINT(referred_nodes[i].node->creation)); - tup = MK_3TUP(tup, MK_UINT(erts_refc_read(&referred_nodes[i].node->refc, 1)), nril); + tup = MK_3TUP(tup, MK_UINT(erts_refc_read(&referred_nodes[i].node->refc, 0)), nril); nl = MK_CONS(tup, nl); } @@ -1624,6 +1670,10 @@ reference_table_term(Uint **hpp, Uint *szp) tup = MK_2TUP(AM_heap, MK_UINT(drp->heap_ref)); drl = MK_CONS(tup, drl); } + if(drp->system_ref) { + tup = MK_2TUP(AM_system, MK_UINT(drp->system_ref)); + drl = MK_CONS(tup, drl); + } if (is_internal_pid(drp->id)) { ASSERT(!drp->node_ref); @@ -1633,6 +1683,14 @@ reference_table_term(Uint **hpp, Uint *szp) ASSERT(drp->ctrl_ref && !drp->node_ref); tup = MK_2TUP(AM_port, drp->id); } + else if (is_tuple(drp->id)) { + Eterm *t; + ASSERT(drp->system_ref && !drp->node_ref + && !drp->ctrl_ref && !drp->heap_ref); + t = tuple_val(drp->id); + ASSERT(2 == arityval(t[0])); + tup = MK_2TUP(t[1], t[2]); + } else { ASSERT(!drp->ctrl_ref && drp->node_ref); ASSERT(is_atom(drp->id)); @@ -1650,7 +1708,7 @@ reference_table_term(Uint **hpp, Uint *szp) /* DistList = [{Dist, Refc, ReferenceIdList}] */ tup = MK_3TUP(referred_dists[i].dist->sysname, - MK_UINT(erts_refc_read(&referred_dists[i].dist->refc, 1)), + MK_UINT(erts_refc_read(&referred_dists[i].dist->refc, 0)), dril); dl = MK_CONS(tup, dl); } @@ -1705,3 +1763,15 @@ delete_reference_table(void) } } +void +erts_debug_test_node_tab_delayed_delete(Sint64 millisecs) +{ + erts_smp_thr_progress_block(); + + if (millisecs < 0) + node_tab_delete_delay = orig_node_tab_delete_delay; + else + node_tab_delete_delay = millisecs; + + erts_smp_thr_progress_unblock(); +} diff --git a/erts/emulator/beam/erl_node_tables.h b/erts/emulator/beam/erl_node_tables.h index af60071ea5..fb2f2a5407 100644 --- a/erts/emulator/beam/erl_node_tables.h +++ b/erts/emulator/beam/erl_node_tables.h @@ -3,16 +3,17 @@ * * Copyright Ericsson AB 2001-2012. All Rights Reserved. * - * The contents of this file are subject to the Erlang Public License, - * Version 1.1, (the "License"); you may not use this file except in - * compliance with the License. You should have received a copy of the - * Erlang Public License along with this software. If not, it can be - * retrieved online at http://www.erlang.org/. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Software distributed under the License is distributed on an "AS IS" - * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See - * the License for the specific language governing rights and limitations - * under the License. + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. * * %CopyrightEnd% */ @@ -40,12 +41,17 @@ #include "sys.h" #include "hash.h" +#include "erl_alloc.h" #include "erl_process.h" #include "erl_monitors.h" #include "erl_smp.h" #define ERTS_PORT_TASK_ONLY_BASIC_TYPES__ #include "erl_port_task.h" #undef ERTS_PORT_TASK_ONLY_BASIC_TYPES__ + +#define ERTS_NODE_TAB_DELAY_GC_DEFAULT (60) +#define ERTS_NODE_TAB_DELAY_GC_MAX (100*1000*1000) +#define ERTS_NODE_TAB_DELAY_GC_INFINITY (ERTS_NODE_TAB_DELAY_GC_MAX+1) #define ERST_INTERNAL_CHANNEL_NO 0 @@ -61,7 +67,7 @@ #define ERTS_DE_QFLGS_ALL (ERTS_DE_QFLG_BUSY \ | ERTS_DE_QFLG_EXIT) -#if defined(ARCH_64) && !HALFWORD_HEAP +#if defined(ARCH_64) #define ERTS_DIST_OUTPUT_BUF_DBG_PATTERN ((Uint) 0xf713f713f713f713UL) #else #define ERTS_DIST_OUTPUT_BUF_DBG_PATTERN ((Uint) 0xf713f713) @@ -166,20 +172,21 @@ extern DistEntry *erts_this_dist_entry; extern ErlNode *erts_this_node; extern char *erts_this_node_sysname; /* must match erl_node_tables.c */ +Uint erts_delayed_node_table_gc(void); DistEntry *erts_channel_no_to_dist_entry(Uint); DistEntry *erts_sysname_to_connected_dist_entry(Eterm); DistEntry *erts_find_or_insert_dist_entry(Eterm); DistEntry *erts_find_dist_entry(Eterm); -void erts_delete_dist_entry(DistEntry *); +void erts_schedule_delete_dist_entry(DistEntry *); Uint erts_dist_table_size(void); void erts_dist_table_info(int, void *); void erts_set_dist_entry_not_connected(DistEntry *); void erts_set_dist_entry_connected(DistEntry *, Eterm, Uint); ErlNode *erts_find_or_insert_node(Eterm, Uint); -void erts_delete_node(ErlNode *); +void erts_schedule_delete_node(ErlNode *); void erts_set_this_node(Eterm, Uint); Uint erts_node_table_size(void); -void erts_init_node_tables(void); +void erts_init_node_tables(int); void erts_node_table_info(int, void *); void erts_print_node_info(int, void *, Eterm, int*, int*); Eterm erts_get_node_and_dist_references(struct process *); @@ -204,7 +211,7 @@ erts_deref_dist_entry(DistEntry *dep) { ASSERT(dep); if (erts_refc_dectest(&dep->refc, 0) == 0) - erts_delete_dist_entry(dep); + erts_schedule_delete_dist_entry(dep); } ERTS_GLB_INLINE void @@ -212,7 +219,7 @@ erts_deref_node_entry(ErlNode *np) { ASSERT(np); if (erts_refc_dectest(&np->refc, 0) == 0) - erts_delete_node(np); + erts_schedule_delete_node(np); } ERTS_GLB_INLINE void @@ -253,5 +260,6 @@ erts_smp_de_links_unlock(DistEntry *dep) #endif /* #if ERTS_GLB_INLINE_INCL_FUNC_DEF */ +void erts_debug_test_node_tab_delayed_delete(Sint64 millisecs); #endif diff --git a/erts/emulator/beam/erl_port.h b/erts/emulator/beam/erl_port.h index ad3f104a68..5b06bc7fd1 100644 --- a/erts/emulator/beam/erl_port.h +++ b/erts/emulator/beam/erl_port.h @@ -3,16 +3,17 @@ * * Copyright Ericsson AB 2012-2013. All Rights Reserved. * - * The contents of this file are subject to the Erlang Public License, - * Version 1.1, (the "License"); you may not use this file except in - * compliance with the License. You should have received a copy of the - * Erlang Public License along with this software. If not, it can be - * retrieved online at http://www.erlang.org/. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Software distributed under the License is distributed on an "AS IS" - * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See - * the License for the specific language governing rights and limitations - * under the License. + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. * * %CopyrightEnd% */ @@ -184,8 +185,13 @@ struct _erl_drv_port { int control_flags; /* Flags for port_control() */ ErlDrvPDL port_data_lock; - ErtsPrtSD *psd; /* Port specific data */ + erts_smp_atomic_t psd; /* Port specific data */ int reds; /* Only used while executing driver callbacks */ + + struct { + Eterm to; + Uint32 ref[ERTS_MAX_REF_NUMBERS]; + } *async_open_port; /* Reference used with async open port */ }; @@ -246,28 +252,56 @@ ERTS_GLB_INLINE void *erts_prtsd_set(Port *p, int ix, void *new); ERTS_GLB_INLINE void * erts_prtsd_get(Port *prt, int ix) { - return prt->psd ? prt->psd->data[ix] : NULL; + ErtsPrtSD *psd = (ErtsPrtSD *) erts_smp_atomic_read_nob(&prt->psd); + if (!psd) + return NULL; + ERTS_SMP_DATA_DEPENDENCY_READ_MEMORY_BARRIER; + return psd->data[ix]; } ERTS_GLB_INLINE void * erts_prtsd_set(Port *prt, int ix, void *data) { - if (prt->psd) { - void *old = prt->psd->data[ix]; - prt->psd->data[ix] = data; + ErtsPrtSD *psd, *new_psd; + void *old; + int i; + + psd = (ErtsPrtSD *) erts_smp_atomic_read_nob(&prt->psd); + + if (psd) { +#ifdef ERTS_SMP +#ifdef ETHR_ORDERED_READ_DEPEND + ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreStore); +#else + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreStore); +#endif +#endif + old = psd->data[ix]; + psd->data[ix] = data; return old; } - else { - prt->psd = erts_alloc(ERTS_ALC_T_PRTSD, sizeof(ErtsPrtSD)); - prt->psd->data[ix] = data; + + if (!data) return NULL; - } + + new_psd = erts_alloc(ERTS_ALC_T_PRTSD, sizeof(ErtsPrtSD)); + for (i = 0; i < ERTS_PRTSD_SIZE; i++) + new_psd->data[i] = NULL; + psd = (ErtsPrtSD *) erts_smp_atomic_cmpxchg_mb(&prt->psd, + (erts_aint_t) new_psd, + (erts_aint_t) NULL); + if (psd) + erts_free(ERTS_ALC_T_PRTSD, new_psd); + else + psd = new_psd; + old = psd->data[ix]; + psd->data[ix] = data; + return old; } #endif -extern erts_smp_atomic_t erts_bytes_out; /* no bytes written out */ -extern erts_smp_atomic_t erts_bytes_in; /* no bytes sent into the system */ +Eterm erts_request_io_bytes(Process *c_p); /* port status flags */ @@ -350,6 +384,7 @@ int erts_lc_is_port_locked(Port *); ERTS_GLB_INLINE void erts_port_inc_refc(Port *prt); ERTS_GLB_INLINE void erts_port_dec_refc(Port *prt); ERTS_GLB_INLINE void erts_port_add_refc(Port *prt, Sint32 add_refc); +ERTS_GLB_INLINE Sint erts_port_read_refc(Port *prt); ERTS_GLB_INLINE int erts_smp_port_trylock(Port *prt); ERTS_GLB_INLINE void erts_smp_port_lock(Port *prt); @@ -359,37 +394,26 @@ ERTS_GLB_INLINE void erts_smp_port_unlock(Port *prt); ERTS_GLB_INLINE void erts_port_inc_refc(Port *prt) { -#ifdef ERTS_SMP - erts_ptab_inc_refc(&prt->common); -#else - erts_atomic32_inc_nob(&prt->refc); -#endif + erts_ptab_atmc_inc_refc(&prt->common); } ERTS_GLB_INLINE void erts_port_dec_refc(Port *prt) { -#ifdef ERTS_SMP - int referred = erts_ptab_dec_test_refc(&prt->common); + int referred = erts_ptab_atmc_dec_test_refc(&prt->common); if (!referred) erts_port_free(prt); -#else - int refc = erts_atomic32_dec_read_nob(&prt->refc); - if (refc == 0) - erts_port_free(prt); -#endif } ERTS_GLB_INLINE void erts_port_add_refc(Port *prt, Sint32 add_refc) { -#ifdef ERTS_SMP - int referred = erts_ptab_add_test_refc(&prt->common, add_refc); + int referred = erts_ptab_atmc_add_test_refc(&prt->common, add_refc); if (!referred) erts_port_free(prt); -#else - int refc = erts_atomic32_add_read_nob(&prt->refc, add_refc); - if (refc == 0) - erts_port_free(prt); -#endif +} + +ERTS_GLB_INLINE Sint erts_port_read_refc(Port *prt) +{ + return erts_ptab_atmc_read_refc(&prt->common); } ERTS_GLB_INLINE int @@ -697,7 +721,7 @@ erts_drvport2port_state(ErlDrvPort drvport, erts_aint32_t *statep) Port *prt = ERTS_ErlDrvPort2Port(drvport); erts_aint32_t state; ASSERT(prt); - ERTS_LC_ASSERT(erts_lc_is_emu_thr()); +// ERTS_LC_ASSERT(erts_lc_is_emu_thr()); if (prt == ERTS_INVALID_ERL_DRV_PORT) return ERTS_INVALID_ERL_DRV_PORT; ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(prt) @@ -954,4 +978,11 @@ ErtsPortOpResult erts_port_control(Process *, Port *, unsigned int, Eterm, Eterm ErtsPortOpResult erts_port_call(Process *, Port *, unsigned int, Eterm, Eterm *); ErtsPortOpResult erts_port_info(Process *, Port *, Eterm, Eterm *); +int erts_port_output_async(Port *, Eterm, Eterm); + +/* + * Signals from ports to ports. Used by sys drivers. + */ +int erl_drv_port_control(Eterm, char, char*, ErlDrvSizeT); + #endif diff --git a/erts/emulator/beam/erl_port_task.c b/erts/emulator/beam/erl_port_task.c index 31d9a1e26e..b200344af5 100644 --- a/erts/emulator/beam/erl_port_task.c +++ b/erts/emulator/beam/erl_port_task.c @@ -3,16 +3,17 @@ * * Copyright Ericsson AB 2006-2013. All Rights Reserved. * - * The contents of this file are subject to the Erlang Public License, - * Version 1.1, (the "License"); you may not use this file except in - * compliance with the License. You should have received a copy of the - * Erlang Public License along with this software. If not, it can be - * retrieved online at http://www.erlang.org/. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Software distributed under the License is distributed on an "AS IS" - * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See - * the License for the specific language governing rights and limitations - * under the License. + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. * * %CopyrightEnd% */ @@ -32,6 +33,7 @@ #include "global.h" #include "erl_port_task.h" #include "dist.h" +#include "erl_check_io.h" #include "dtrace-wrapper.h" #include <stdarg.h> @@ -68,6 +70,13 @@ static void chk_task_queues(Port *pp, ErtsPortTask *execq, int processing_busy_q #define DTRACE_DRIVER(PROBE_NAME, PP) do {} while(0) #endif +#define ERTS_SMP_LC_VERIFY_RQ(RQ, PP) \ + do { \ + ERTS_SMP_LC_ASSERT(erts_smp_lc_runq_is_locked(runq)); \ + ERTS_SMP_LC_ASSERT((RQ) == ((ErtsRunQueue *) \ + erts_smp_atomic_read_nob(&(PP)->run_queue))); \ + } while (0) + erts_smp_atomic_t erts_port_task_outstanding_io_tasks; #define ERTS_PT_STATE_SCHEDULED 0 @@ -171,10 +180,9 @@ p2p_sig_data_to_task(ErtsProc2PortSigData *sigdp) return ptp; } -ErtsProc2PortSigData * -erts_port_task_alloc_p2p_sig_data(void) +static ERTS_INLINE ErtsProc2PortSigData * +p2p_sig_data_init(ErtsPortTask *ptp) { - ErtsPortTask *ptp = port_task_alloc(); ptp->type = ERTS_PORT_TASK_PROC_SIG; ptp->u.alive.flags = ERTS_PT_FLG_SIG_DEP; @@ -185,6 +193,31 @@ erts_port_task_alloc_p2p_sig_data(void) return &ptp->u.alive.td.psig.data; } +ErtsProc2PortSigData * +erts_port_task_alloc_p2p_sig_data(void) +{ + ErtsPortTask *ptp = port_task_alloc(); + + return p2p_sig_data_init(ptp); +} + +ErtsProc2PortSigData * +erts_port_task_alloc_p2p_sig_data_extra(size_t extra, void **extra_ptr) +{ + ErtsPortTask *ptp = erts_alloc(ERTS_ALC_T_PORT_TASK, + sizeof(ErtsPortTask) + extra); + + *extra_ptr = ptp+1; + + return p2p_sig_data_init(ptp); +} + +void +erts_port_task_free_p2p_sig_data(ErtsProc2PortSigData *sigdp) +{ + schedule_port_task_free(p2p_sig_data_to_task(sigdp)); +} + static ERTS_INLINE Eterm task_caller(ErtsPortTask *ptp) { @@ -543,6 +576,16 @@ reset_handle(ErtsPortTask *ptp) } static ERTS_INLINE void +reset_executed_io_task_handle(ErtsPortTask *ptp) +{ + if (ptp->u.alive.handle) { + ASSERT(ptp == handle2task(ptp->u.alive.handle)); + erts_io_notify_port_task_executed(ptp->u.alive.handle); + reset_port_task_handle(ptp->u.alive.handle); + } +} + +static ERTS_INLINE void set_handle(ErtsPortTask *ptp, ErtsPortTaskHandle *pthp) { ptp->u.alive.handle = pthp; @@ -798,12 +841,13 @@ schedule_port_task_handle_list_free(ErtsPortTaskHandleList *pthlp) static ERTS_INLINE void abort_nosuspend_task(Port *pp, ErtsPortTaskType type, - ErtsPortTaskTypeData *tdp) + ErtsPortTaskTypeData *tdp, + int bpq_data) { ASSERT(type == ERTS_PORT_TASK_PROC_SIG); - if (!pp->sched.taskq.bpq) + if (!bpq_data) tdp->psig.callback(NULL, ERTS_PORT_SFLG_INVALID, ERTS_PROC2PORT_SIG_ABORT_NOSUSPEND, @@ -991,6 +1035,7 @@ static ERTS_INLINE int finalize_exec(Port *pp, ErtsPortTask **execq, int processing_busy_q) { erts_aint32_t act; + unsigned int prof_runnable_ports; if (!processing_busy_q) pp->sched.taskq.local.first = *execq; @@ -1007,6 +1052,10 @@ finalize_exec(Port *pp, ErtsPortTask **execq, int processing_busy_q) if (act & ERTS_PTS_FLG_CHK_UNSET_BUSY_PORT_Q) act = check_unset_busy_port_q(pp, act, pp->sched.taskq.bpq); + prof_runnable_ports = erts_system_profile_flags.runnable_ports; + if (prof_runnable_ports) + erts_port_task_sched_lock(&pp->sched); + while (1) { erts_aint32_t new, exp; @@ -1018,12 +1067,24 @@ finalize_exec(Port *pp, ErtsPortTask **execq, int processing_busy_q) act = erts_smp_atomic32_cmpxchg_relb(&pp->sched.flags, new, exp); - ASSERT(!(act & ERTS_PTS_FLG_IN_RUNQ)); + ERTS_LC_ASSERT(!(act & ERTS_PTS_FLG_IN_RUNQ)); + ERTS_LC_ASSERT(!(act & ERTS_PTS_FLG_EXEC_IMM)); if (exp == act) break; } + if (prof_runnable_ports | IS_TRACED_FL(pp, F_TRACE_SCHED_PORTS)) { + /* trace port scheduling, out */ + if (IS_TRACED_FL(pp, F_TRACE_SCHED_PORTS)) + trace_sched_ports(pp, am_out); + if (prof_runnable_ports) { + if (!(act & (ERTS_PTS_FLG_EXEC_IMM|ERTS_PTS_FLG_HAVE_TASKS))) + profile_runnable_port(pp, am_inactive); + erts_port_task_sched_unlock(&pp->sched); + } + } + return (act & ERTS_PTS_FLG_HAVE_TASKS) != 0; } @@ -1345,7 +1406,7 @@ erts_port_task_abort_nosuspend_tasks(Port *pp) #endif schedule_port_task_handle_list_free(pthlp); - abort_nosuspend_task(pp, type, &td); + abort_nosuspend_task(pp, type, &td, pp->sched.taskq.bpq != NULL); } } @@ -1369,11 +1430,9 @@ erts_port_task_schedule(Eterm id, Port *pp; ErtsPortTask *ptp = NULL; erts_aint32_t act, add_flags; + unsigned int prof_runnable_ports; - if (pthp && erts_port_task_is_scheduled(pthp)) { - ASSERT(0); - erts_port_task_abort(pthp); - } + ERTS_LC_ASSERT(!pthp || !erts_port_task_is_scheduled(pthp)); ASSERT(is_internal_port(id)); @@ -1457,6 +1516,10 @@ erts_port_task_schedule(Eterm id, if (ns_pthlp) add_flags |= ERTS_PTS_FLG_HAVE_NS_TASKS; + prof_runnable_ports = erts_system_profile_flags.runnable_ports; + if (prof_runnable_ports) + erts_port_task_sched_lock(&pp->sched); + while (1) { erts_aint32_t new, exp; @@ -1481,6 +1544,13 @@ erts_port_task_schedule(Eterm id, goto done; /* Died after our task insert... */ } + if (prof_runnable_ports) { + if (!(act & ERTS_PTS_FLG_EXEC_IMM)) + profile_runnable_port(pp, am_active); + erts_port_task_sched_unlock(&pp->sched); + prof_runnable_ports = 0; + } + /* Enqueue port on run-queue */ runq = erts_port_runq(pp); @@ -1489,8 +1559,10 @@ erts_port_task_schedule(Eterm id, #ifdef ERTS_SMP xrunq = erts_check_emigration_need(runq, ERTS_PORT_PRIO_LEVEL); + ERTS_SMP_LC_ASSERT(runq != xrunq); + ERTS_SMP_LC_VERIFY_RQ(runq, pp); if (xrunq) { - /* Port emigrated ... */ + /* Emigrate port ... */ erts_smp_atomic_set_nob(&pp->run_queue, (erts_aint_t) xrunq); erts_smp_runq_unlock(runq); runq = erts_port_runq(pp); @@ -1500,10 +1572,6 @@ erts_port_task_schedule(Eterm id, #endif enqueue_port(runq, pp); - - if (erts_system_profile_flags.runnable_ports) { - profile_runnable_port(pp, am_active); - } erts_smp_runq_unlock(runq); @@ -1511,6 +1579,9 @@ erts_port_task_schedule(Eterm id, done: + if (prof_runnable_ports) + erts_port_task_sched_unlock(&pp->sched); + #ifdef ERTS_SMP if (dhndl != ERTS_THR_PRGR_DHANDLE_MANAGED) erts_port_dec_refc(pp); @@ -1525,7 +1596,7 @@ abort_nosuspend: erts_port_dec_refc(pp); #endif - abort_nosuspend_task(pp, ptp->type, &ptp->u.alive.td); + abort_nosuspend_task(pp, ptp->type, &ptp->u.alive.td, 0); ASSERT(ns_pthlp); erts_free(ERTS_ALC_T_PT_HNDL_LIST, ns_pthlp); @@ -1600,6 +1671,8 @@ erts_port_task_execute(ErtsRunQueue *runq, Port **curr_port_pp) erts_aint32_t state; int active; Uint64 start_time = 0; + ErtsSchedulerData *esdp = runq->scheduler; + ERTS_MSACC_PUSH_STATE_M(); ERTS_SMP_LC_ASSERT(erts_smp_lc_runq_is_locked(runq)); @@ -1609,12 +1682,13 @@ erts_port_task_execute(ErtsRunQueue *runq, Port **curr_port_pp) goto done; } + ERTS_SMP_LC_VERIFY_RQ(runq, pp); + erts_smp_runq_unlock(runq); *curr_port_pp = pp; if (erts_sched_stat.enabled) { - ErtsSchedulerData *esdp = erts_get_scheduler_data(); Uint old = ERTS_PORT_SCHED_ID(pp, esdp->no); int migrated = old && old != esdp->no; @@ -1641,6 +1715,7 @@ erts_port_task_execute(ErtsRunQueue *runq, Port **curr_port_pp) state = erts_atomic32_read_nob(&pp->state); pp->reds = ERTS_PORT_REDS_EXECUTE; + ERTS_MSACC_SET_STATE_CACHED_M(ERTS_MSACC_STATE_PORT); goto begin_handle_tasks; while (1) { @@ -1659,8 +1734,6 @@ erts_port_task_execute(ErtsRunQueue *runq, Port **curr_port_pp) goto aborted_port_task; } - reset_handle(ptp); - if (erts_system_monitor_long_schedule != 0) { start_time = erts_timestamp_millis(); } @@ -1671,20 +1744,27 @@ erts_port_task_execute(ErtsRunQueue *runq, Port **curr_port_pp) switch (ptp->type) { case ERTS_PORT_TASK_TIMEOUT: - reds = ERTS_PORT_REDS_TIMEOUT; - if (!(state & ERTS_PORT_SFLGS_DEAD)) { - DTRACE_DRIVER(driver_timeout, pp); - (*pp->drv_ptr->timeout)((ErlDrvData) pp->drv_data); - } + reset_handle(ptp); + if (!ERTS_PTMR_IS_TIMED_OUT(pp)) + reds = 0; + else { + ERTS_PTMR_CLEAR(pp); + reds = ERTS_PORT_REDS_TIMEOUT; + if (!(state & ERTS_PORT_SFLGS_DEAD)) { + DTRACE_DRIVER(driver_timeout, pp); + (*pp->drv_ptr->timeout)((ErlDrvData) pp->drv_data); + } + } break; case ERTS_PORT_TASK_INPUT: reds = ERTS_PORT_REDS_INPUT; ASSERT((state & ERTS_PORT_SFLGS_DEAD) == 0); DTRACE_DRIVER(driver_ready_input, pp); - /* NOTE some windows/ose drivers use ->ready_input + /* NOTE some windows drivers use ->ready_input for input and output */ (*pp->drv_ptr->ready_input)((ErlDrvData) pp->drv_data, ptp->u.alive.td.io.event); + reset_executed_io_task_handle(ptp); io_tasks_executed++; break; case ERTS_PORT_TASK_OUTPUT: @@ -1693,6 +1773,7 @@ erts_port_task_execute(ErtsRunQueue *runq, Port **curr_port_pp) DTRACE_DRIVER(driver_ready_output, pp); (*pp->drv_ptr->ready_output)((ErlDrvData) pp->drv_data, ptp->u.alive.td.io.event); + reset_executed_io_task_handle(ptp); io_tasks_executed++; break; case ERTS_PORT_TASK_EVENT: @@ -1702,10 +1783,12 @@ erts_port_task_execute(ErtsRunQueue *runq, Port **curr_port_pp) (*pp->drv_ptr->event)((ErlDrvData) pp->drv_data, ptp->u.alive.td.io.event, ptp->u.alive.td.io.event_data); + reset_executed_io_task_handle(ptp); io_tasks_executed++; break; case ERTS_PORT_TASK_PROC_SIG: { ErtsProc2PortSigData *sigdp = &ptp->u.alive.td.psig.data; + reset_handle(ptp); ASSERT((state & ERTS_PORT_SFLGS_DEAD) == 0); if (!pp->sched.taskq.bpq) reds = ptp->u.alive.td.psig.callback(pp, @@ -1723,10 +1806,11 @@ erts_port_task_execute(ErtsRunQueue *runq, Port **curr_port_pp) break; } case ERTS_PORT_TASK_DIST_CMD: + reset_handle(ptp); reds = erts_dist_command(pp, CONTEXT_REDS - pp->reds); break; default: - erl_exit(ERTS_ABORT_EXIT, + erts_exit(ERTS_ABORT_EXIT, "Invalid port task type: %d\n", (int) ptp->type); break; @@ -1764,11 +1848,8 @@ erts_port_task_execute(ErtsRunQueue *runq, Port **curr_port_pp) } erts_unblock_fpe(fpe_was_unmasked); + ERTS_MSACC_POP_STATE_M(); - /* trace port scheduling, out */ - if (IS_TRACED_FL(pp, F_TRACE_SCHED_PORTS)) { - trace_sched_ports(pp, am_out); - } if (io_tasks_executed) { ASSERT(erts_smp_atomic_read_nob(&erts_port_task_outstanding_io_tasks) @@ -1791,11 +1872,7 @@ erts_port_task_execute(ErtsRunQueue *runq, Port **curr_port_pp) erts_smp_runq_lock(runq); - if (!active) { - if (erts_system_profile_flags.runnable_ports) - profile_runnable_port(pp, am_inactive); - } - else { + if (active) { #ifdef ERTS_SMP ErtsRunQueue *xrunq; #endif @@ -1804,6 +1881,8 @@ erts_port_task_execute(ErtsRunQueue *runq, Port **curr_port_pp) #ifdef ERTS_SMP xrunq = erts_check_emigration_need(runq, ERTS_PORT_PRIO_LEVEL); + ERTS_SMP_LC_ASSERT(runq != xrunq); + ERTS_SMP_LC_VERIFY_RQ(runq, pp); if (!xrunq) { #endif enqueue_port(runq, pp); @@ -1811,7 +1890,7 @@ erts_port_task_execute(ErtsRunQueue *runq, Port **curr_port_pp) #ifdef ERTS_SMP } else { - /* Port emigrated ... */ + /* Emigrate port... */ erts_smp_atomic_set_nob(&pp->run_queue, (erts_aint_t) xrunq); erts_smp_runq_unlock(runq); @@ -1833,7 +1912,7 @@ erts_port_task_execute(ErtsRunQueue *runq, Port **curr_port_pp) runq->scheduler->reductions += reds; ERTS_SMP_LC_ASSERT(erts_smp_lc_runq_is_locked(runq)); - ERTS_PORT_REDUCTIONS_EXECUTED(runq, reds); + ERTS_PORT_REDUCTIONS_EXECUTED(esdp, runq, reds); return res; } @@ -1996,7 +2075,7 @@ begin_port_cleanup(Port *pp, ErtsPortTask **execqp, int *processing_busy_q_p) break; } default: - erl_exit(ERTS_ABORT_EXIT, + erts_exit(ERTS_ABORT_EXIT, "Invalid port task type: %d\n", (int) ptp->type); } diff --git a/erts/emulator/beam/erl_port_task.h b/erts/emulator/beam/erl_port_task.h index 1d30465ec9..56cef4e352 100644 --- a/erts/emulator/beam/erl_port_task.h +++ b/erts/emulator/beam/erl_port_task.h @@ -3,16 +3,17 @@ * * Copyright Ericsson AB 2006-2013. All Rights Reserved. * - * The contents of this file are subject to the Erlang Public License, - * Version 1.1, (the "License"); you may not use this file except in - * compliance with the License. You should have received a copy of the - * Erlang Public License along with this software. If not, it can be - * retrieved online at http://www.erlang.org/. - * - * Software distributed under the License is distributed on an "AS IS" - * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See - * the License for the specific language governing rights and limitations - * under the License. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. * * %CopyrightEnd% */ @@ -78,6 +79,7 @@ extern erts_smp_atomic_t erts_port_task_outstanding_io_tasks; #define ERTS_PTS_FLG_PARALLELISM (((erts_aint32_t) 1) << 9) #define ERTS_PTS_FLG_FORCE_SCHED (((erts_aint32_t) 1) << 10) #define ERTS_PTS_FLG_EXITING (((erts_aint32_t) 1) << 11) +#define ERTS_PTS_FLG_EXEC_IMM (((erts_aint32_t) 1) << 12) #define ERTS_PTS_FLGS_BUSY \ (ERTS_PTS_FLG_BUSY_PORT | ERTS_PTS_FLG_BUSY_PORT_Q) @@ -87,6 +89,7 @@ extern erts_smp_atomic_t erts_port_task_outstanding_io_tasks; | ERTS_PTS_FLG_HAVE_BUSY_TASKS \ | ERTS_PTS_FLG_HAVE_TASKS \ | ERTS_PTS_FLG_EXEC \ + | ERTS_PTS_FLG_EXEC_IMM \ | ERTS_PTS_FLG_FORCE_SCHED \ | ERTS_PTS_FLG_EXITING) @@ -154,7 +157,7 @@ erts_port_task_handle_init(ErtsPortTaskHandle *pthp) ERTS_GLB_INLINE int erts_port_task_is_scheduled(ErtsPortTaskHandle *pthp) { - return ((void *) erts_smp_atomic_read_nob(pthp)) != NULL; + return ((void *) erts_smp_atomic_read_acqb(pthp)) != NULL; } ERTS_GLB_INLINE void erts_port_task_pre_init_sched(ErtsPortTaskSched *ptsp, @@ -266,6 +269,8 @@ int erts_port_task_schedule(Eterm, void erts_port_task_free_port(Port *); int erts_port_is_scheduled(Port *); ErtsProc2PortSigData *erts_port_task_alloc_p2p_sig_data(void); +ErtsProc2PortSigData *erts_port_task_alloc_p2p_sig_data_extra(size_t extra, void **extra_ptr); +void erts_port_task_free_p2p_sig_data(ErtsProc2PortSigData *sigdp); #ifdef ERTS_SMP void erts_enqueue_port(ErtsRunQueue *rq, Port *pp); diff --git a/erts/emulator/beam/erl_printf_term.c b/erts/emulator/beam/erl_printf_term.c index d18760dc43..45ba4371dc 100644 --- a/erts/emulator/beam/erl_printf_term.c +++ b/erts/emulator/beam/erl_printf_term.c @@ -1,18 +1,19 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2005-2013. All Rights Reserved. + * Copyright Ericsson AB 2005-2014. All Rights Reserved. * - * The contents of this file are subject to the Erlang Public License, - * Version 1.1, (the "License"); you may not use this file except in - * compliance with the License. You should have received a copy of the - * Erlang Public License along with this software. If not, it can be - * retrieved online at http://www.erlang.org/. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Software distributed under the License is distributed on an "AS IS" - * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See - * the License for the specific language governing rights and limitations - * under the License. + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. * * %CopyrightEnd% */ @@ -25,6 +26,7 @@ #include "sys.h" #include "big.h" #include "erl_map.h" +#include "erl_binary.h" #define PRINT_CHAR(CNT, FN, ARG, C) \ do { \ @@ -115,13 +117,12 @@ do { \ /* return 0 if list is not a non-empty flat list of printable characters */ static int -is_printable_string(Eterm list, Eterm* base) -{ +is_printable_string(Eterm list) { int len = 0; int c; while(is_list(list)) { - Eterm* consp = list_val_rel(list, base); + Eterm* consp = list_val(list); Eterm hd = CAR(consp); if (!is_byte(hd)) @@ -138,6 +139,25 @@ is_printable_string(Eterm list, Eterm* base) return 0; } +static int is_printable_ascii(byte* bytep, Uint bytesize, Uint bitoffs) +{ + if (!bitoffs) { + while (bytesize--) { + if (*bytep < ' ' || *bytep >= 127) + return 0; + bytep++; + } + } else { + while (bytesize--) { + byte octet = (bytep[0] << bitoffs) | (bytep[1] >> (8-bitoffs)); + if (octet < ' ' || octet >= 127) + return 0; + bytep++; + } + } + return 1; +} + /* print a atom doing what quoting is necessary */ static int print_atom_name(fmtfn_t fn, void* arg, Eterm atom, long *dcount) { @@ -227,10 +247,19 @@ static int print_atom_name(fmtfn_t fn, void* arg, Eterm atom, long *dcount) #define PRT_PATCH_FUN_SIZE ((Eterm) 7) #define PRT_LAST_ARRAY_ELEMENT ((Eterm) 8) /* Note! Must be last... */ +#if 0 +static char *format_binary(Uint16 x, char *b) { + int z; + b[16] = '\0'; + for (z = 0; z < 16; z++) { + b[15-z] = ((x>>z) & 0x1) ? '1' : '0'; + } + return b; +} +#endif + static int -print_term(fmtfn_t fn, void* arg, Eterm obj, long *dcount, - Eterm* obj_base) /* ignored if !HALFWORD_HEAP */ -{ +print_term(fmtfn_t fn, void* arg, Eterm obj, long *dcount) { DECLARE_WSTACK(s); int res; int i; @@ -276,20 +305,16 @@ print_term(fmtfn_t fn, void* arg, Eterm obj, long *dcount, obj = (Eterm) popped.word; L_print_one_cons: { - Eterm* cons = list_val_rel(obj, obj_base); + Eterm* cons = list_val(obj); Eterm tl; obj = CAR(cons); tl = CDR(cons); if (is_not_nil(tl)) { if (is_list(tl)) { - WSTACK_PUSH(s, tl); - WSTACK_PUSH(s, PRT_ONE_CONS); - WSTACK_PUSH(s, PRT_COMMA); + WSTACK_PUSH3(s, tl, PRT_ONE_CONS, PRT_COMMA); } else { - WSTACK_PUSH(s, tl); - WSTACK_PUSH(s, PRT_TERM); - WSTACK_PUSH(s, PRT_BAR); + WSTACK_PUSH3(s, tl, PRT_TERM, PRT_BAR); } } } @@ -299,9 +324,7 @@ print_term(fmtfn_t fn, void* arg, Eterm obj, long *dcount, break; default: /* PRT_LAST_ARRAY_ELEMENT+1 and upwards */ obj = *popped.ptr; - WSTACK_PUSH(s, (UWord) (popped.ptr + 1)); - WSTACK_PUSH(s, val-1); - WSTACK_PUSH(s, PRT_COMMA); + WSTACK_PUSH3(s, (UWord) (popped.ptr + 1), val-1, PRT_COMMA); break; } break; @@ -318,11 +341,7 @@ print_term(fmtfn_t fn, void* arg, Eterm obj, long *dcount, PRINT_CHAR(res, fn, arg, '>'); goto L_done; } -#if HALFWORD_HEAP - wobj = is_immed(obj) ? (Wterm)obj : rterm2wterm(obj, obj_base); -#else wobj = (Wterm)obj; -#endif switch (tag_val_def(wobj)) { case NIL_DEF: PRINT_STRING(res, fn, arg, "[]"); @@ -398,10 +417,10 @@ print_term(fmtfn_t fn, void* arg, Eterm obj, long *dcount, PRINT_CHAR(res, fn, arg, '>'); break; case LIST_DEF: - if (is_printable_string(obj, obj_base)) { + if (is_printable_string(obj)) { int c; PRINT_CHAR(res, fn, arg, '"'); - nobj = list_val_rel(obj, obj_base); + nobj = list_val(obj); while (1) { if ((*dcount)-- <= 0) goto L_done; @@ -415,7 +434,7 @@ print_term(fmtfn_t fn, void* arg, Eterm obj, long *dcount, } if (is_not_list(*nobj)) break; - nobj = list_val_rel(*nobj, obj_base); + nobj = list_val(*nobj); } PRINT_CHAR(res, fn, arg, '"'); } else { @@ -431,8 +450,7 @@ print_term(fmtfn_t fn, void* arg, Eterm obj, long *dcount, WSTACK_PUSH(s,PRT_CLOSE_TUPLE); ++nobj; if (i > 0) { - WSTACK_PUSH(s, (UWord) nobj); - WSTACK_PUSH(s, PRT_LAST_ARRAY_ELEMENT+i-1); + WSTACK_PUSH2(s, (UWord) nobj, PRT_LAST_ARRAY_ELEMENT+i-1); } break; case FLOAT_DEF: { @@ -442,17 +460,66 @@ print_term(fmtfn_t fn, void* arg, Eterm obj, long *dcount, } break; case BINARY_DEF: - if (header_is_bin_matchstate(*boxed_val(wobj))) { - PRINT_STRING(res, fn, arg, "#MatchState"); - } - else { - ProcBin* pb = (ProcBin *) binary_val(wobj); - if (pb->size == 1) - PRINT_STRING(res, fn, arg, "<<1 byte>>"); - else { + { + byte* bytep; + Uint bytesize = binary_size(obj); + Uint bitoffs; + Uint bitsize; + byte octet; + ERTS_GET_BINARY_BYTES(obj, bytep, bitoffs, bitsize); + + if (bitsize || !bytesize + || !is_printable_ascii(bytep, bytesize, bitoffs)) { + int is_first = 1; PRINT_STRING(res, fn, arg, "<<"); - PRINT_UWORD(res, fn, arg, 'u', 0, 1, (ErlPfUWord) pb->size); - PRINT_STRING(res, fn, arg, " bytes>>"); + while (bytesize) { + if (is_first) + is_first = 0; + else + PRINT_CHAR(res, fn, arg, ','); + if (bitoffs) + octet = (bytep[0] << bitoffs) | (bytep[1] >> (8-bitoffs)); + else + octet = bytep[0]; + PRINT_UWORD(res, fn, arg, 'u', 0, 1, octet); + ++bytep; + --bytesize; + } + if (bitsize) { + Uint bits = bitoffs + bitsize; + octet = bytep[0]; + if (bits < 8) + octet >>= 8 - bits; + else if (bits > 8) { + bits -= 8; /* bits in last byte */ + octet <<= bits; + octet |= bytep[1] >> (8 - bits); + } + octet &= (1 << bitsize) - 1; + if (is_first) + is_first = 0; + else + PRINT_CHAR(res, fn, arg, ','); + PRINT_UWORD(res, fn, arg, 'u', 0, 1, octet); + PRINT_CHAR(res, fn, arg, ':'); + PRINT_UWORD(res, fn, arg, 'u', 0, 1, bitsize); + } + PRINT_STRING(res, fn, arg, ">>"); + } + else { + PRINT_STRING(res, fn, arg, "<<\""); + while (bytesize) { + if (bitoffs) + octet = (bytep[0] << bitoffs) | (bytep[1] >> (8-bitoffs)); + else + octet = bytep[0]; + if (octet == '"') + PRINT_CHAR(res, fn, arg, '\\'); + PRINT_CHAR(res, fn, arg, octet); + ++bytep; + --bytesize; + } + PRINT_STRING(res, fn, arg, "\">>"); } } break; @@ -489,37 +556,77 @@ print_term(fmtfn_t fn, void* arg, Eterm obj, long *dcount, } break; case MAP_DEF: - { - Uint n; - Eterm *ks, *vs; - map_t *mp = (map_t *)map_val(wobj); - n = map_get_size(mp); - ks = map_get_keys(mp); - vs = map_get_values(mp); - - PRINT_CHAR(res, fn, arg, '#'); - PRINT_CHAR(res, fn, arg, '{'); - WSTACK_PUSH(s, PRT_CLOSE_TUPLE); - if (n > 0) { - n--; - WSTACK_PUSH(s, vs[n]); - WSTACK_PUSH(s, PRT_TERM); - WSTACK_PUSH(s, PRT_ASSOC); - WSTACK_PUSH(s, ks[n]); - WSTACK_PUSH(s, PRT_TERM); - - while (n--) { - WSTACK_PUSH(s, PRT_COMMA); - WSTACK_PUSH(s, vs[n]); - WSTACK_PUSH(s, PRT_TERM); - WSTACK_PUSH(s, PRT_ASSOC); - WSTACK_PUSH(s, ks[n]); - WSTACK_PUSH(s, PRT_TERM); - } - } - } + if (is_flatmap(wobj)) { + Uint n; + Eterm *ks, *vs; + flatmap_t *mp = (flatmap_t *)flatmap_val(wobj); + n = flatmap_get_size(mp); + ks = flatmap_get_keys(mp); + vs = flatmap_get_values(mp); + + PRINT_CHAR(res, fn, arg, '#'); + PRINT_CHAR(res, fn, arg, '{'); + WSTACK_PUSH(s, PRT_CLOSE_TUPLE); + if (n > 0) { + n--; + WSTACK_PUSH5(s, vs[n], PRT_TERM, PRT_ASSOC, ks[n], PRT_TERM); + while (n--) { + WSTACK_PUSH6(s, PRT_COMMA, vs[n], PRT_TERM, PRT_ASSOC, + ks[n], PRT_TERM); + } + } + } else { + Uint n, mapval; + Eterm *head; + head = hashmap_val(wobj); + mapval = MAP_HEADER_VAL(*head); + switch (MAP_HEADER_TYPE(*head)) { + case MAP_HEADER_TAG_HAMT_HEAD_ARRAY: + case MAP_HEADER_TAG_HAMT_HEAD_BITMAP: + PRINT_STRING(res, fn, arg, "#<"); + PRINT_UWORD(res, fn, arg, 'x', 0, 1, mapval); + PRINT_STRING(res, fn, arg, ">{"); + WSTACK_PUSH(s,PRT_CLOSE_TUPLE); + n = hashmap_bitcount(mapval); + ASSERT(n < 17); + head += 2; + if (n > 0) { + n--; + WSTACK_PUSH(s, head[n]); + WSTACK_PUSH(s, PRT_TERM); + while (n--) { + WSTACK_PUSH(s, PRT_COMMA); + WSTACK_PUSH(s, head[n]); + WSTACK_PUSH(s, PRT_TERM); + } + } + break; + case MAP_HEADER_TAG_HAMT_NODE_BITMAP: + n = hashmap_bitcount(mapval); + head++; + PRINT_CHAR(res, fn, arg, '<'); + PRINT_UWORD(res, fn, arg, 'x', 0, 1, mapval); + PRINT_STRING(res, fn, arg, ">{"); + WSTACK_PUSH(s,PRT_CLOSE_TUPLE); + ASSERT(n < 17); + if (n > 0) { + n--; + WSTACK_PUSH(s, head[n]); + WSTACK_PUSH(s, PRT_TERM); + while (n--) { + WSTACK_PUSH(s, PRT_COMMA); + WSTACK_PUSH(s, head[n]); + WSTACK_PUSH(s, PRT_TERM); + } + } + break; + } + } + break; + case MATCHSTATE_DEF: + PRINT_STRING(res, fn, arg, "#MatchState"); break; - default: + default: PRINT_STRING(res, fn, arg, "<unknown:"); PRINT_POINTER(res, fn, arg, wobj); PRINT_CHAR(res, fn, arg, '>'); @@ -528,19 +635,17 @@ print_term(fmtfn_t fn, void* arg, Eterm obj, long *dcount, } L_done: - DESTROY_WSTACK(s); return res; } + int -erts_printf_term(fmtfn_t fn, void* arg, ErlPfEterm term, long precision, - ErlPfEterm* term_base) -{ +erts_printf_term(fmtfn_t fn, void* arg, ErlPfEterm term, long precision) { int res; - ASSERT(sizeof(ErlPfEterm) == sizeof(Eterm)); + ERTS_CT_ASSERT(sizeof(ErlPfEterm) == sizeof(Eterm)); - res = print_term(fn, arg, (Eterm)term, &precision, (Eterm*)term_base); + res = print_term(fn, arg, (Eterm)term, &precision); if (res < 0) return res; if (precision <= 0) diff --git a/erts/emulator/beam/erl_printf_term.h b/erts/emulator/beam/erl_printf_term.h index f92c99d713..4618457f27 100644 --- a/erts/emulator/beam/erl_printf_term.h +++ b/erts/emulator/beam/erl_printf_term.h @@ -3,16 +3,17 @@ * * Copyright Ericsson AB 2005-2013. All Rights Reserved. * - * The contents of this file are subject to the Erlang Public License, - * Version 1.1, (the "License"); you may not use this file except in - * compliance with the License. You should have received a copy of the - * Erlang Public License along with this software. If not, it can be - * retrieved online at http://www.erlang.org/. - * - * Software distributed under the License is distributed on an "AS IS" - * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See - * the License for the specific language governing rights and limitations - * under the License. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. * * %CopyrightEnd% */ @@ -21,6 +22,5 @@ #define ERL_PRINTF_TERM_H__ #include "erl_printf_format.h" -int erts_printf_term(fmtfn_t fn, void* arg, ErlPfEterm term, long precision, - ErlPfEterm* term_base); +int erts_printf_term(fmtfn_t fn, void* arg, ErlPfEterm term, long precision); #endif diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c index 2b468a9ad9..794af60b2f 100644 --- a/erts/emulator/beam/erl_process.c +++ b/erts/emulator/beam/erl_process.c @@ -3,16 +3,17 @@ * * Copyright Ericsson AB 1996-2014. All Rights Reserved. * - * The contents of this file are subject to the Erlang Public License, - * Version 1.1, (the "License"); you may not use this file except in - * compliance with the License. You should have received a copy of the - * Erlang Public License along with this software. If not, it can be - * retrieved online at http://www.erlang.org/. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Software distributed under the License is distributed on an "AS IS" - * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See - * the License for the specific language governing rights and limitations - * under the License. + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. * * %CopyrightEnd% */ @@ -43,8 +44,11 @@ #include "erl_async.h" #include "dtrace-wrapper.h" #include "erl_ptab.h" +#include "erl_bif_unique.h" +#define ERTS_WANT_TIMER_WHEEL_API +#include "erl_time.h" - +#define ERTS_CHECK_TIME_REDS CONTEXT_REDS #define ERTS_DELAYED_WAKEUP_INFINITY (~(Uint64) 0) #define ERTS_DELAYED_WAKEUP_REDUCTIONS ((Uint64) CONTEXT_REDS/2) @@ -54,11 +58,7 @@ #define ERTS_PROC_MIN_CONTEXT_SWITCH_REDS_COST (CONTEXT_REDS/10) -#ifndef ERTS_SCHED_MIN_SPIN #define ERTS_SCHED_SPIN_UNTIL_YIELD 100 -#else -#define ERTS_SCHED_SPIN_UNTIL_YIELD 1 -#endif #define ERTS_SCHED_SYS_SLEEP_SPINCOUNT_VERY_LONG 40 #define ERTS_SCHED_AUX_WORK_SLEEP_SPINCOUNT_FACT_VERY_LONG 1000 @@ -115,7 +115,7 @@ #define RUNQ_SET_RQ(X, RQ) erts_smp_atomic_set_nob((X), (erts_aint_t) (RQ)) #ifdef DEBUG -# if defined(ARCH_64) && !HALFWORD_HEAP +# if defined(ARCH_64) # define ERTS_DBG_SET_INVALID_RUNQP(RQP, N) \ (RUNQ_SET_RQ((RQP), (0xdeadbeefdead0003LL | ((N) << 4))) # define ERTS_DBG_VERIFY_VALID_RUNQP(RQP) \ @@ -148,13 +148,16 @@ extern BeamInstr beam_apply[]; extern BeamInstr beam_exit[]; extern BeamInstr beam_continue_exit[]; -int erts_sched_compact_load; -int erts_sched_balance_util = 0; -Uint erts_no_schedulers; -#ifdef ERTS_DIRTY_SCHEDULERS -Uint erts_no_dirty_cpu_schedulers; -Uint erts_no_dirty_io_schedulers; -#endif +int ERTS_WRITE_UNLIKELY(erts_default_spo_flags) = 0; +int ERTS_WRITE_UNLIKELY(erts_eager_check_io) = 1; +int ERTS_WRITE_UNLIKELY(erts_sched_compact_load); +int ERTS_WRITE_UNLIKELY(erts_sched_balance_util) = 0; +Uint ERTS_WRITE_UNLIKELY(erts_no_schedulers); +Uint ERTS_WRITE_UNLIKELY(erts_no_dirty_cpu_schedulers) = 0; +Uint ERTS_WRITE_UNLIKELY(erts_no_dirty_io_schedulers) = 0; + +static char *erts_aux_work_flag_descr[ERTS_SSI_AUX_WORK_NO_FLAGS] = {0}; +int erts_aux_work_no_flags = ERTS_SSI_AUX_WORK_NO_FLAGS; #define ERTS_THR_PRGR_LATER_CLEANUP_OP_THRESHOLD_VERY_LAZY (4*1024*1024) #define ERTS_THR_PRGR_LATER_CLEANUP_OP_THRESHOLD_LAZY (512*1024) @@ -183,84 +186,176 @@ int erts_disable_proc_not_running_opt; static ErtsAuxWorkData *aux_thread_aux_work_data; -#define ERTS_SCHDLR_SSPND_CHNG_WAITER (((erts_aint32_t) 1) << 0) +#define ERTS_SCHDLR_SSPND_CHNG_NMSB (((erts_aint32_t) 1) << 0) #define ERTS_SCHDLR_SSPND_CHNG_MSB (((erts_aint32_t) 1) << 1) #define ERTS_SCHDLR_SSPND_CHNG_ONLN (((erts_aint32_t) 1) << 2) +#define ERTS_SCHDLR_SSPND_CHNG_DCPU_ONLN (((erts_aint32_t) 1) << 3) -#ifndef DEBUG +typedef enum { + ERTS_SCHED_NORMAL, + ERTS_SCHED_DIRTY_CPU, + ERTS_SCHED_DIRTY_IO +} ErtsSchedType; -#define ERTS_SCHDLR_SSPND_CHNG_SET(VAL, OLD_VAL) \ - erts_smp_atomic32_set_nob(&schdlr_sspnd.changing, (VAL)) +typedef struct { + int ongoing; + ErtsProcList *blckrs; + ErtsProcList *chngq; +} ErtsMultiSchedulingBlock; -#ifdef ERTS_DIRTY_SCHEDULERS -#define ERTS_SCHDLR_SSPND_DIRTY_CPU_CHNG_SET(VAL, OLD_VAL) \ - erts_smp_atomic32_set_nob(&schdlr_sspnd.dirty_cpu_changing, (VAL)) -#define ERTS_SCHDLR_SSPND_DIRTY_IO_CHNG_SET(VAL, OLD_VAL) \ - erts_smp_atomic32_set_nob(&schdlr_sspnd.dirty_io_changing, (VAL)) -#endif +static struct { + erts_smp_mtx_t mtx; + Uint32 online; + Uint32 curr_online; + Uint32 active; + erts_smp_atomic32_t changing; + ErtsProcList *chngq; + Eterm changer; + ErtsMultiSchedulingBlock nmsb; /* Normal multi Scheduling Block */ + ErtsMultiSchedulingBlock msb; /* Multi Scheduling Block */ +} schdlr_sspnd; -#else +#define ERTS_SCHDLR_SSPND_S_BITS 10 +#define ERTS_SCHDLR_SSPND_DCS_BITS 11 +#define ERTS_SCHDLR_SSPND_DIS_BITS 11 -#define ERTS_SCHDLR_SSPND_CHNG_SET(VAL, OLD_VAL) \ -do { \ - erts_aint32_t old_val__; \ - old_val__ = erts_smp_atomic32_xchg_nob(&schdlr_sspnd.changing, \ - (VAL)); \ - ASSERT(old_val__ == (OLD_VAL)); \ -} while (0) +#define ERTS_SCHDLR_SSPND_S_MASK ((1 << ERTS_SCHDLR_SSPND_S_BITS)-1) +#define ERTS_SCHDLR_SSPND_DCS_MASK ((1 << ERTS_SCHDLR_SSPND_DCS_BITS)-1) +#define ERTS_SCHDLR_SSPND_DIS_MASK ((1 << ERTS_SCHDLR_SSPND_DIS_BITS)-1) -#ifdef ERTS_DIRTY_SCHEDULERS -#define ERTS_SCHDLR_SSPND_DIRTY_CPU_CHNG_SET(VAL, OLD_VAL) \ -do { \ - erts_aint32_t old_val__; \ - old_val__ = erts_smp_atomic32_xchg_nob(&schdlr_sspnd.dirty_cpu_changing, \ - (VAL)); \ - ASSERT(old_val__ == (OLD_VAL)); \ -} while (0) -#define ERTS_SCHDLR_SSPND_DIRTY_IO_CHNG_SET(VAL, OLD_VAL) \ -do { \ - erts_aint32_t old_val__; \ - old_val__ = erts_smp_atomic32_xchg_nob(&schdlr_sspnd.dirty_io_changing, \ - (VAL)); \ - ASSERT(old_val__ == (OLD_VAL)); \ -} while (0) -#endif +#define ERTS_SCHDLR_SSPND_S_SHIFT 0 +#define ERTS_SCHDLR_SSPND_DCS_SHIFT (ERTS_SCHDLR_SSPND_S_SHIFT \ + + ERTS_SCHDLR_SSPND_S_BITS) +#define ERTS_SCHDLR_SSPND_DIS_SHIFT (ERTS_SCHDLR_SSPND_DCS_SHIFT \ + + ERTS_SCHDLR_SSPND_DCS_BITS) +#if (ERTS_SCHDLR_SSPND_S_BITS \ + + ERTS_SCHDLR_SSPND_DCS_BITS \ + + ERTS_SCHDLR_SSPND_DIS_BITS) > 32 +# error Wont fit in Uint32 #endif - -static struct { - erts_smp_mtx_t mtx; - erts_smp_cnd_t cnd; - int online; - int curr_online; - int wait_curr_online; -#ifdef ERTS_DIRTY_SCHEDULERS - int dirty_cpu_online; - int dirty_cpu_curr_online; - int dirty_cpu_wait_curr_online; - int dirty_io_online; - int dirty_io_curr_online; - int dirty_io_wait_curr_online; +#if (ERTS_MAX_NO_OF_SCHEDULERS-1) > ERTS_SCHDLR_SSPND_S_MASK +# error Max no schedulers wont fit in its bit-field #endif - erts_smp_atomic32_t changing; - erts_smp_atomic32_t active; -#ifdef ERTS_DIRTY_SCHEDULERS - erts_smp_atomic32_t dirty_cpu_changing; - erts_smp_atomic32_t dirty_cpu_active; - erts_smp_atomic32_t dirty_io_changing; - erts_smp_atomic32_t dirty_io_active; +#if ERTS_MAX_NO_OF_DIRTY_CPU_SCHEDULERS > ERTS_SCHDLR_SSPND_DCS_MASK +# error Max no dirty cpu schedulers wont fit in its bit-field #endif - struct { - int ongoing; - long wait_active; -#ifdef ERTS_DIRTY_SCHEDULERS - long dirty_cpu_wait_active; - long dirty_io_wait_active; +#if ERTS_MAX_NO_OF_DIRTY_IO_SCHEDULERS > ERTS_SCHDLR_SSPND_DIS_MASK +# error Max no dirty io schedulers wont fit in its bit-field #endif - ErtsProcList *procs; - } msb; /* Multi Scheduling Block */ -} schdlr_sspnd; + +#define ERTS_SCHDLR_SSPND_MAKE_NSCHEDS_VAL(S, DCS, DIS) \ + ((((Uint32) (((S) & ERTS_SCHDLR_SSPND_S_MASK))-1) \ + << ERTS_SCHDLR_SSPND_S_SHIFT) \ + | ((((Uint32) ((DCS) & ERTS_SCHDLR_SSPND_DCS_MASK)) \ + << ERTS_SCHDLR_SSPND_DCS_SHIFT)) \ + | ((((Uint32) ((DIS) & ERTS_SCHDLR_SSPND_DIS_MASK)) \ + << ERTS_SCHDLR_SSPND_DIS_SHIFT))) + +static void init_scheduler_suspend(void); + +static ERTS_INLINE Uint32 +schdlr_sspnd_get_nscheds(Uint32 *valp, ErtsSchedType type) +{ + Uint32 res = (Uint32) (*valp); + switch (type) { + case ERTS_SCHED_NORMAL: + res >>= ERTS_SCHDLR_SSPND_S_SHIFT; + res &= (Uint32) ERTS_SCHDLR_SSPND_S_MASK; + res++; + break; + case ERTS_SCHED_DIRTY_CPU: + res >>= ERTS_SCHDLR_SSPND_DCS_SHIFT; + res &= (Uint32) ERTS_SCHDLR_SSPND_DCS_MASK; + break; + case ERTS_SCHED_DIRTY_IO: + res >>= ERTS_SCHDLR_SSPND_DIS_SHIFT; + res &= (Uint32) ERTS_SCHDLR_SSPND_DIS_MASK; + break; + default: + ERTS_INTERNAL_ERROR("Invalid scheduler type"); + return 0; + } + + return res; +} + +static ERTS_INLINE void +schdlr_sspnd_dec_nscheds(Uint32 *valp, ErtsSchedType type) +{ + ASSERT(schdlr_sspnd_get_nscheds(valp, type) > 0); + + switch (type) { + case ERTS_SCHED_NORMAL: + *valp -= ((Uint32) 1) << ERTS_SCHDLR_SSPND_S_SHIFT; + break; + case ERTS_SCHED_DIRTY_CPU: + *valp -= ((Uint32) 1) << ERTS_SCHDLR_SSPND_DCS_SHIFT; + break; + case ERTS_SCHED_DIRTY_IO: + *valp -= ((Uint32) 1) << ERTS_SCHDLR_SSPND_DIS_SHIFT; + break; + default: + ERTS_INTERNAL_ERROR("Invalid scheduler type"); + } +} + +static ERTS_INLINE void +schdlr_sspnd_inc_nscheds(Uint32 *valp, ErtsSchedType type) +{ + switch (type) { + case ERTS_SCHED_NORMAL: + ASSERT(schdlr_sspnd_get_nscheds(valp, type) + < ERTS_MAX_NO_OF_SCHEDULERS-1); + *valp += ((Uint32) 1) << ERTS_SCHDLR_SSPND_S_SHIFT; + break; + case ERTS_SCHED_DIRTY_CPU: + ASSERT(schdlr_sspnd_get_nscheds(valp, type) + < ERTS_MAX_NO_OF_DIRTY_CPU_SCHEDULERS); + *valp += ((Uint32) 1) << ERTS_SCHDLR_SSPND_DCS_SHIFT; + break; + case ERTS_SCHED_DIRTY_IO: + ASSERT(schdlr_sspnd_get_nscheds(valp, type) + < ERTS_MAX_NO_OF_DIRTY_IO_SCHEDULERS); + *valp += ((Uint32) 1) << ERTS_SCHDLR_SSPND_DIS_SHIFT; + break; + default: + ERTS_INTERNAL_ERROR("Invalid scheduler type"); + } +} + +static ERTS_INLINE void +schdlr_sspnd_set_nscheds(Uint32 *valp, ErtsSchedType type, Uint32 no) +{ + Uint32 val = *valp; + + switch (type) { + case ERTS_SCHED_NORMAL: + ASSERT(no > 0); + val &= ~(((Uint32) ERTS_SCHDLR_SSPND_S_MASK) + << ERTS_SCHDLR_SSPND_S_SHIFT); + val |= (((no-1) & ((Uint32) ERTS_SCHDLR_SSPND_S_MASK)) + << ERTS_SCHDLR_SSPND_S_SHIFT); + break; + case ERTS_SCHED_DIRTY_CPU: + val &= ~(((Uint32) ERTS_SCHDLR_SSPND_DCS_MASK) + << ERTS_SCHDLR_SSPND_DCS_SHIFT); + val |= ((no & ((Uint32) ERTS_SCHDLR_SSPND_DCS_MASK)) + << ERTS_SCHDLR_SSPND_DCS_SHIFT); + break; + case ERTS_SCHED_DIRTY_IO: + val &= ~(((Uint32) ERTS_SCHDLR_SSPND_DIS_MASK) + << ERTS_SCHDLR_SSPND_DIS_SHIFT); + val |= ((no & ((Uint32) ERTS_SCHDLR_SSPND_DIS_MASK)) + << ERTS_SCHDLR_SSPND_DIS_SHIFT); + break; + default: + ERTS_INTERNAL_ERROR("Invalid scheduler type"); + } + + *valp = val; +} static struct { erts_smp_mtx_t update_mtx; @@ -290,7 +385,7 @@ do { \ erts_sched_stat_t erts_sched_stat; #ifdef USE_THREADS -static erts_tsd_key_t sched_data_key; +static erts_tsd_key_t ERTS_WRITE_UNLIKELY(sched_data_key); #endif static erts_smp_atomic32_t function_calls; @@ -330,16 +425,16 @@ static ErtsAlignedSchedulerSleepInfo *aligned_dirty_io_sched_sleep_info; static Uint last_reductions; static Uint last_exact_reductions; -Uint erts_default_process_flags; -Eterm erts_system_monitor; -Eterm erts_system_monitor_long_gc; -Uint erts_system_monitor_long_schedule; -Eterm erts_system_monitor_large_heap; +Eterm ERTS_WRITE_UNLIKELY(erts_system_monitor); +Eterm ERTS_WRITE_UNLIKELY(erts_system_monitor_long_gc); +Uint ERTS_WRITE_UNLIKELY(erts_system_monitor_long_schedule); +Eterm ERTS_WRITE_UNLIKELY(erts_system_monitor_large_heap); struct erts_system_monitor_flags_t erts_system_monitor_flags; /* system performance monitor */ Eterm erts_system_profile; struct erts_system_profile_flags_t erts_system_profile_flags; +int erts_system_profile_ts_type = ERTS_TRACE_FLG_NOW_TIMESTAMP; #if ERTS_MAX_PROCESSES > 0x7fffffff #error "Need to store process_count in another type" @@ -347,7 +442,8 @@ struct erts_system_profile_flags_t erts_system_profile_flags; typedef enum { ERTS_PSTT_GC, /* Garbage Collect */ - ERTS_PSTT_CPC /* Check Process Code */ + ERTS_PSTT_CPC, /* Check Process Code */ + ERTS_PSTT_COHMQ /* Change off heap message queue */ } ErtsProcSysTaskType; #define ERTS_MAX_PROC_SYS_TASK_ARGS 2 @@ -420,8 +516,10 @@ do { \ do { \ ErtsRunQueue *RQVAR; \ int ix__; \ + int online__ = (int) schdlr_sspnd_get_nscheds(&schdlr_sspnd.online, \ + ERTS_SCHED_NORMAL); \ ERTS_SMP_LC_ASSERT(erts_smp_lc_mtx_is_locked(&schdlr_sspnd.mtx)); \ - for (ix__ = 0; ix__ < schdlr_sspnd.online; ix__++) { \ + for (ix__ = 0; ix__ < online__; ix__++) { \ RQVAR = ERTS_RUNQ_IX(ix__); \ erts_smp_runq_lock(RQVAR); \ { DO; } \ @@ -451,13 +549,12 @@ do { \ static void exec_misc_ops(ErtsRunQueue *); static void print_function_from_pc(int to, void *to_arg, BeamInstr* x); -static int stack_element_dump(int to, void *to_arg, Process* p, Eterm* sp, - int yreg); +static int stack_element_dump(int to, void *to_arg, Eterm* sp, int yreg); static void aux_work_timeout(void *unused); static void aux_work_timeout_early_init(int no_schedulers); static void aux_work_timeout_late_init(void); -static void setup_aux_work_timer(void); +static void setup_aux_work_timer(ErtsSchedulerData *esdp); static int execute_sys_tasks(Process *c_p, erts_aint32_t *statep, @@ -487,20 +584,20 @@ dbg_chk_aux_work_val(erts_aint32_t value) valid |= ERTS_SSI_AUX_WORK_MISC_THR_PRGR; valid |= ERTS_SSI_AUX_WORK_DD; valid |= ERTS_SSI_AUX_WORK_DD_THR_PRGR; + valid |= ERTS_SSI_AUX_WORK_CNCLD_TMRS; + valid |= ERTS_SSI_AUX_WORK_CNCLD_TMRS_THR_PRGR; valid |= ERTS_SSI_AUX_WORK_THR_PRGR_LATER_OP; #endif #if HAVE_ERTS_MSEG valid |= ERTS_SSI_AUX_WORK_MSEG_CACHE_CHECK; #endif -#ifdef ERTS_SMP_SCHEDULERS_NEED_TO_CHECK_CHILDREN - valid |= ERTS_SSI_AUX_WORK_CHECK_CHILDREN; -#endif #ifdef ERTS_SSI_AUX_WORK_REAP_PORTS valid |= ERTS_SSI_AUX_WORK_REAP_PORTS; #endif + valid |= ERTS_SSI_AUX_WORK_DEBUG_WAIT_COMPLETED; if (~valid & value) - erl_exit(ERTS_ABORT_EXIT, + erts_exit(ERTS_ABORT_EXIT, "Invalid aux_work value found: 0x%x\n", ~valid & value); } @@ -556,63 +653,78 @@ erts_pre_init_process(void) erts_tsd_key_create(&sched_data_key, "erts_sched_data_key"); #endif + erts_aux_work_flag_descr[ERTS_SSI_AUX_WORK_DELAYED_AW_WAKEUP_IX] + = "DELAYED_AW_WAKEUP"; + erts_aux_work_flag_descr[ERTS_SSI_AUX_WORK_DD_IX] + = "DD"; + erts_aux_work_flag_descr[ERTS_SSI_AUX_WORK_DD_THR_PRGR_IX] + = "DD_THR_PRGR"; + erts_aux_work_flag_descr[ERTS_SSI_AUX_WORK_FIX_ALLOC_DEALLOC_IX] + = "FIX_ALLOC_DEALLOC"; + erts_aux_work_flag_descr[ERTS_SSI_AUX_WORK_FIX_ALLOC_LOWER_LIM_IX] + = "FIX_ALLOC_LOWER_LIM"; + erts_aux_work_flag_descr[ERTS_SSI_AUX_WORK_THR_PRGR_LATER_OP_IX] + = "THR_PRGR_LATER_OP"; + erts_aux_work_flag_descr[ERTS_SSI_AUX_WORK_CNCLD_TMRS_IX] + = "CNCLD_TMRS"; + erts_aux_work_flag_descr[ERTS_SSI_AUX_WORK_CNCLD_TMRS_THR_PRGR_IX] + = "CNCLD_TMRS_THR_PRGR"; + erts_aux_work_flag_descr[ERTS_SSI_AUX_WORK_ASYNC_READY_IX] + = "ASYNC_READY"; + erts_aux_work_flag_descr[ERTS_SSI_AUX_WORK_ASYNC_READY_CLEAN_IX] + = "ASYNC_READY_CLEAN"; + erts_aux_work_flag_descr[ERTS_SSI_AUX_WORK_MISC_THR_PRGR_IX] + = "MISC_THR_PRGR"; + erts_aux_work_flag_descr[ERTS_SSI_AUX_WORK_MISC_IX] + = "MISC"; + erts_aux_work_flag_descr[ERTS_SSI_AUX_WORK_SET_TMO_IX] + = "SET_TMO"; + erts_aux_work_flag_descr[ERTS_SSI_AUX_WORK_MSEG_CACHE_CHECK_IX] + = "MSEG_CACHE_CHECK"; + erts_aux_work_flag_descr[ERTS_SSI_AUX_WORK_REAP_PORTS_IX] + = "REAP_PORTS"; + erts_aux_work_flag_descr[ERTS_SSI_AUX_WORK_DEBUG_WAIT_COMPLETED_IX] + = "DEBUG_WAIT_COMPLETED"; + #ifdef ERTS_ENABLE_LOCK_CHECK - { - int ix; - - erts_psd_required_locks[ERTS_PSD_ERROR_HANDLER].get_locks - = ERTS_PSD_ERROR_HANDLER_BUF_GET_LOCKS; - erts_psd_required_locks[ERTS_PSD_ERROR_HANDLER].set_locks - = ERTS_PSD_ERROR_HANDLER_BUF_SET_LOCKS; - - erts_psd_required_locks[ERTS_PSD_SAVED_CALLS_BUF].get_locks - = ERTS_PSD_SAVED_CALLS_BUF_GET_LOCKS; - erts_psd_required_locks[ERTS_PSD_SAVED_CALLS_BUF].set_locks - = ERTS_PSD_SAVED_CALLS_BUF_SET_LOCKS; - - erts_psd_required_locks[ERTS_PSD_SCHED_ID].get_locks - = ERTS_PSD_SCHED_ID_GET_LOCKS; - erts_psd_required_locks[ERTS_PSD_SCHED_ID].set_locks - = ERTS_PSD_SCHED_ID_SET_LOCKS; - - erts_psd_required_locks[ERTS_PSD_DIST_ENTRY].get_locks - = ERTS_PSD_DIST_ENTRY_GET_LOCKS; - erts_psd_required_locks[ERTS_PSD_DIST_ENTRY].set_locks - = ERTS_PSD_DIST_ENTRY_SET_LOCKS; - - erts_psd_required_locks[ERTS_PSD_CALL_TIME_BP].get_locks - = ERTS_PSD_CALL_TIME_BP_GET_LOCKS; - erts_psd_required_locks[ERTS_PSD_CALL_TIME_BP].set_locks - = ERTS_PSD_CALL_TIME_BP_SET_LOCKS; - - erts_psd_required_locks[ERTS_PSD_DELAYED_GC_TASK_QS].get_locks - = ERTS_PSD_DELAYED_GC_TASK_QS_GET_LOCKS; - erts_psd_required_locks[ERTS_PSD_DELAYED_GC_TASK_QS].set_locks - = ERTS_PSD_DELAYED_GC_TASK_QS_SET_LOCKS; -#ifdef ERTS_DIRTY_SCHEDULERS - erts_psd_required_locks[ERTS_PSD_DIRTY_SCHED_TRAP_EXPORT].get_locks - = ERTS_PSD_DIRTY_SCHED_TRAP_EXPORT_GET_LOCKS; - erts_psd_required_locks[ERTS_PSD_DIRTY_SCHED_TRAP_EXPORT].set_locks - = ERTS_PSD_DIRTY_SCHED_TRAP_EXPORT_SET_LOCKS; -#endif + erts_psd_required_locks[ERTS_PSD_ERROR_HANDLER].get_locks + = ERTS_PSD_ERROR_HANDLER_BUF_GET_LOCKS; + erts_psd_required_locks[ERTS_PSD_ERROR_HANDLER].set_locks + = ERTS_PSD_ERROR_HANDLER_BUF_SET_LOCKS; + + erts_psd_required_locks[ERTS_PSD_SAVED_CALLS_BUF].get_locks + = ERTS_PSD_SAVED_CALLS_BUF_GET_LOCKS; + erts_psd_required_locks[ERTS_PSD_SAVED_CALLS_BUF].set_locks + = ERTS_PSD_SAVED_CALLS_BUF_SET_LOCKS; + + erts_psd_required_locks[ERTS_PSD_SCHED_ID].get_locks + = ERTS_PSD_SCHED_ID_GET_LOCKS; + erts_psd_required_locks[ERTS_PSD_SCHED_ID].set_locks + = ERTS_PSD_SCHED_ID_SET_LOCKS; + + erts_psd_required_locks[ERTS_PSD_CALL_TIME_BP].get_locks + = ERTS_PSD_CALL_TIME_BP_GET_LOCKS; + erts_psd_required_locks[ERTS_PSD_CALL_TIME_BP].set_locks + = ERTS_PSD_CALL_TIME_BP_SET_LOCKS; - /* Check that we have locks for all entries */ - for (ix = 0; ix < ERTS_PSD_SIZE; ix++) { - ERTS_SMP_LC_ASSERT(erts_psd_required_locks[ix].get_locks); - ERTS_SMP_LC_ASSERT(erts_psd_required_locks[ix].set_locks); - } - } + erts_psd_required_locks[ERTS_PSD_DELAYED_GC_TASK_QS].get_locks + = ERTS_PSD_DELAYED_GC_TASK_QS_GET_LOCKS; + erts_psd_required_locks[ERTS_PSD_DELAYED_GC_TASK_QS].set_locks + = ERTS_PSD_DELAYED_GC_TASK_QS_SET_LOCKS; + + erts_psd_required_locks[ERTS_PSD_NIF_TRAP_EXPORT].get_locks + = ERTS_PSD_NIF_TRAP_EXPORT_GET_LOCKS; + erts_psd_required_locks[ERTS_PSD_NIF_TRAP_EXPORT].set_locks + = ERTS_PSD_NIF_TRAP_EXPORT_SET_LOCKS; #endif } -#ifdef ERTS_SMP static void release_process(void *vproc) { - erts_smp_proc_dec_refc((Process *) vproc); + erts_proc_dec_refc((Process *) vproc); } -#endif /* initialize the scheduler */ void @@ -628,20 +740,21 @@ erts_init_process(int ncpu, int proc_tab_size, int legacy_proc_tab) erts_ptab_init_table(&erts_proc, ERTS_ALC_T_PROC_TABLE, -#ifdef ERTS_SMP release_process, -#else - NULL, -#endif (ErtsPTabElementCommon *) &erts_invalid_process.common, proc_tab_size, sizeof(Process), "process_table", - legacy_proc_tab); + legacy_proc_tab, +#ifdef ERTS_SMP + 1 +#else + 0 +#endif + ); last_reductions = 0; last_exact_reductions = 0; - erts_default_process_flags = 0; } void @@ -698,8 +811,8 @@ init_sched_wall_time(ErtsSchedWallTime *swtp) static ERTS_INLINE Uint64 sched_wall_time_ts(void) { -#ifdef HAVE_GETHRTIME - return (Uint64) sys_gethrtime(); +#ifdef ERTS_HAVE_OS_MONOTONIC_TIME_SUPPORT + return (Uint64) erts_os_monotonic_time(); #else Uint64 res; SysTimeval tv; @@ -712,72 +825,24 @@ sched_wall_time_ts(void) #if ERTS_HAVE_SCHED_UTIL_BALANCING_SUPPORT -#ifdef ARCH_64 - -static ERTS_INLINE Uint64 -aschedtime_read(ErtsAtomicSchedTime *var) -{ - return (Uint64) erts_atomic_read_nob((erts_atomic_t *) var); -} - -static ERTS_INLINE void -aschedtime_set(ErtsAtomicSchedTime *var, Uint64 val) -{ - erts_atomic_set_nob((erts_atomic_t *) var, (erts_aint_t) val); -} - -static ERTS_INLINE void -aschedtime_init(ErtsAtomicSchedTime *var) -{ - erts_atomic_init_nob((erts_atomic_t *) var, (erts_aint_t) 0); -} - -#elif defined(ARCH_32) - static ERTS_INLINE Uint64 aschedtime_read(ErtsAtomicSchedTime *var) { - erts_dw_aint_t dw; - erts_dw_atomic_read_nob((erts_dw_atomic_t *) var, &dw); -#ifdef ETHR_SU_DW_NAINT_T__ - return (Uint64) dw.dw_sint; -#else - { - Uint64 res; - res = (Uint64) ((Uint32) dw.sint[ERTS_DW_AINT_HIGH_WORD]); - res <<= 32; - res |= (Uint64) ((Uint32) dw.sint[ERTS_DW_AINT_LOW_WORD]); - return res; - } -#endif + return (Uint64) erts_atomic64_read_nob((erts_atomic64_t *) var); } static ERTS_INLINE void aschedtime_set(ErtsAtomicSchedTime *var, Uint64 val) { - erts_dw_aint_t dw; -#ifdef ETHR_SU_DW_NAINT_T__ - dw.dw_sint = (ETHR_SU_DW_NAINT_T__) val; -#else - dw.sint[ERTS_DW_AINT_LOW_WORD] = (erts_aint_t) (val & 0xffffffff); - dw.sint[ERTS_DW_AINT_HIGH_WORD] = (erts_aint_t) ((val >> 32) & 0xffffffff); -#endif - erts_dw_atomic_set_nob((erts_dw_atomic_t *) var, &dw); + erts_atomic64_set_nob((erts_atomic64_t *) var, (erts_aint64_t) val); } static ERTS_INLINE void aschedtime_init(ErtsAtomicSchedTime *var) { - erts_dw_aint_t dw; - dw.sint[ERTS_DW_AINT_LOW_WORD] = (erts_aint_t) 0; - dw.sint[ERTS_DW_AINT_HIGH_WORD] = (erts_aint_t) 0; - erts_dw_atomic_init_nob((erts_dw_atomic_t *) var, &dw); + erts_atomic64_init_nob((erts_atomic64_t *) var, (erts_aint64_t) 0); } -#else -# error :-/ -#endif - #define ERTS_GET_AVG_MAX_UNLOCKED_TRY 50 #define ERTS_SCHED_AVG_UTIL_WRITE_MARKER (~((Uint64) 0)) @@ -959,6 +1024,12 @@ sched_wall_time_change(ErtsSchedulerData *esdp, int working) } } } + if (!working) { + ERTS_MSACC_SET_STATE_M_X(ERTS_MSACC_STATE_BUSY_WAIT); + } else { + ERTS_MSACC_SET_STATE_M_X(ERTS_MSACC_STATE_OTHER); + } + } typedef struct { @@ -971,25 +1042,24 @@ typedef struct { erts_smp_atomic32_t refc; } ErtsSchedWallTimeReq; -#if !HALFWORD_HEAP +typedef struct { + Process *proc; + Eterm ref; + Eterm ref_heap[REF_THING_SIZE]; + Uint req_sched; + erts_smp_atomic32_t refc; +} ErtsSystemCheckReq; + + ERTS_SCHED_PREF_QUICK_ALLOC_IMPL(swtreq, ErtsSchedWallTimeReq, 5, ERTS_ALC_T_SCHED_WTIME_REQ) -#else -static ERTS_INLINE ErtsSchedWallTimeReq * -swtreq_alloc(void) -{ - return erts_alloc(ERTS_ALC_T_SCHED_WTIME_REQ, - sizeof(ErtsSchedWallTimeReq)); -} -static ERTS_INLINE void -swtreq_free(ErtsSchedWallTimeReq *ptr) -{ - erts_free(ERTS_ALC_T_SCHED_WTIME_REQ, ptr); -} -#endif +ERTS_SCHED_PREF_QUICK_ALLOC_IMPL(screq, + ErtsSystemCheckReq, + 5, + ERTS_ALC_T_SYS_CHECK_REQ) static void reply_sched_wall_time(void *vswtrp) @@ -1006,7 +1076,7 @@ reply_sched_wall_time(void *vswtrp) Eterm **hpp; Uint sz, *szp; ErlOffHeap *ohp = NULL; - ErlHeapFragment *bp = NULL; + ErtsMessage *mp = NULL; ASSERT(esdp); #ifdef ERTS_DIRTY_SCHEDULERS @@ -1062,16 +1132,12 @@ reply_sched_wall_time(void *vswtrp) if (hpp) break; - hp = erts_alloc_message_heap(sz, &bp, &ohp, rp, &rp_locks); + mp = erts_alloc_message_heap(rp, &rp_locks, sz, &hp, &ohp); szp = NULL; hpp = &hp; } - erts_queue_message(rp, &rp_locks, bp, msg, NIL -#ifdef USE_VM_PROBES - , NIL -#endif - ); + erts_queue_message(rp, &rp_locks, mp, msg, NIL); if (swtrp->req_sched == esdp->no) rp_locks &= ~ERTS_PROC_LOCK_MAIN; @@ -1079,7 +1145,7 @@ reply_sched_wall_time(void *vswtrp) if (rp_locks) erts_smp_proc_unlock(rp, rp_locks); - erts_smp_proc_dec_refc(rp); + erts_proc_dec_refc(rp); if (erts_smp_atomic32_dec_read_nob(&swtrp->refc) == 0) swtreq_free(vswtrp); @@ -1111,7 +1177,7 @@ erts_sched_wall_time_request(Process *c_p, int set, int enable) erts_smp_atomic32_init_nob(&swtrp->refc, (erts_aint32_t) erts_no_schedulers); - erts_smp_proc_add_refc(c_p, (Sint32) erts_no_schedulers); + erts_proc_add_refc(c_p, (Sint32) erts_no_schedulers); #ifdef ERTS_SMP if (erts_no_schedulers > 1) @@ -1126,6 +1192,75 @@ erts_sched_wall_time_request(Process *c_p, int set, int enable) return ref; } +static void +reply_system_check(void *vscrp) +{ + ErtsSchedulerData *esdp = erts_get_scheduler_data(); + ErtsSystemCheckReq *scrp = (ErtsSystemCheckReq *) vscrp; + ErtsProcLocks rp_locks = (scrp->req_sched == esdp->no ? ERTS_PROC_LOCK_MAIN : 0); + Process *rp = scrp->proc; + Eterm msg; + Eterm *hp = NULL; + Eterm **hpp; + Uint sz; + ErlOffHeap *ohp = NULL; + ErtsMessage *mp = NULL; + + ASSERT(esdp); +#ifdef ERTS_DIRTY_SCHEDULERS + ASSERT(!ERTS_SCHEDULER_IS_DIRTY(esdp)); +#endif + + sz = REF_THING_SIZE; + mp = erts_alloc_message_heap(rp, &rp_locks, sz, &hp, &ohp); + hpp = &hp; + msg = STORE_NC(hpp, ohp, scrp->ref); + + erts_queue_message(rp, &rp_locks, mp, msg, NIL); + + if (scrp->req_sched == esdp->no) + rp_locks &= ~ERTS_PROC_LOCK_MAIN; + + if (rp_locks) + erts_smp_proc_unlock(rp, rp_locks); + + erts_proc_dec_refc(rp); + + if (erts_smp_atomic32_dec_read_nob(&scrp->refc) == 0) + screq_free(vscrp); +} + + +Eterm erts_system_check_request(Process *c_p) { + ErtsSchedulerData *esdp = ERTS_PROC_GET_SCHDATA(c_p); + Eterm ref; + ErtsSystemCheckReq *scrp; + Eterm *hp; + + scrp = screq_alloc(); + ref = erts_make_ref(c_p); + hp = &scrp->ref_heap[0]; + + scrp->proc = c_p; + scrp->ref = STORE_NC(&hp, NULL, ref); + scrp->req_sched = esdp->no; + erts_smp_atomic32_init_nob(&scrp->refc, (erts_aint32_t) erts_no_schedulers); + + erts_proc_add_refc(c_p, (Sint) erts_no_schedulers); + +#ifdef ERTS_SMP + if (erts_no_schedulers > 1) + erts_schedule_multi_misc_aux_work(1, + erts_no_schedulers, + reply_system_check, + (void *) scrp); +#endif + + reply_system_check((void *) scrp); + + return ref; +} + static ERTS_INLINE ErtsProcList * proclist_create(Process *p) { @@ -1136,6 +1271,15 @@ proclist_create(Process *p) return plp; } +static ERTS_INLINE ErtsProcList * +proclist_copy(ErtsProcList *plp0) +{ + ErtsProcList *plp1 = proclist_alloc(); + plp1->pid = plp0->pid; + plp1->started_interval = plp0->started_interval; + return plp1; +} + static ERTS_INLINE void proclist_destroy(ErtsProcList *plp) { @@ -1143,6 +1287,12 @@ proclist_destroy(ErtsProcList *plp) } ErtsProcList * +erts_proclist_copy(ErtsProcList *plp) +{ + return proclist_copy(plp); +} + +ErtsProcList * erts_proclist_create(Process *p) { return proclist_create(p); @@ -1155,46 +1305,25 @@ erts_proclist_destroy(ErtsProcList *plp) } void * -erts_psd_set_init(Process *p, ErtsProcLocks plocks, int ix, void *data) +erts_psd_set_init(Process *p, int ix, void *data) { void *old; - ErtsProcLocks xplocks; - int refc = 0; - ErtsPSD *psd = erts_alloc(ERTS_ALC_T_PSD, sizeof(ErtsPSD)); + ErtsPSD *psd, *new_psd; int i; - for (i = 0; i < ERTS_PSD_SIZE; i++) - psd->data[i] = NULL; - ERTS_SMP_LC_ASSERT(plocks); - ERTS_SMP_LC_ASSERT(plocks == erts_proc_lc_my_proc_locks(p)); + new_psd = erts_alloc(ERTS_ALC_T_PSD, sizeof(ErtsPSD)); + for (i = 0; i < ERTS_PSD_SIZE; i++) + new_psd->data[i] = NULL; - xplocks = ERTS_PROC_LOCKS_ALL; - xplocks &= ~plocks; - if (xplocks && erts_smp_proc_trylock(p, xplocks) == EBUSY) { - if (xplocks & ERTS_PROC_LOCK_MAIN) { - erts_smp_proc_inc_refc(p); - erts_smp_proc_unlock(p, plocks); - erts_smp_proc_lock(p, ERTS_PROC_LOCKS_ALL); - refc = 1; - } - else { - if (plocks & ERTS_PROC_LOCKS_ALL_MINOR) - erts_smp_proc_unlock(p, plocks & ERTS_PROC_LOCKS_ALL_MINOR); - erts_smp_proc_lock(p, ERTS_PROC_LOCKS_ALL_MINOR); - } - } - if (!p->psd) - p->psd = psd; - if (xplocks) - erts_smp_proc_unlock(p, xplocks); - if (refc) - erts_smp_proc_dec_refc(p); - ASSERT(p->psd); - if (p->psd != psd) - erts_free(ERTS_ALC_T_PSD, psd); - old = p->psd->data[ix]; - p->psd->data[ix] = data; - ERTS_SMP_LC_ASSERT(plocks == erts_proc_lc_my_proc_locks(p)); + psd = (ErtsPSD *) erts_smp_atomic_cmpxchg_mb(&p->psd, + (erts_aint_t) new_psd, + (erts_aint_t) NULL); + if (psd) + erts_free(ERTS_ALC_T_PSD, new_psd); + else + psd = new_psd; + old = psd->data[ix]; + psd->data[ix] = data; return old; } @@ -1220,7 +1349,7 @@ erts_sched_finish_poke(ErtsSchedulerSleepInfo *ssi, erts_aint32_t flags) case 0: break; default: - erl_exit(ERTS_ABORT_EXIT, "%s:%d: Internal error\n", + erts_exit(ERTS_ABORT_EXIT, "%s:%d: Internal error\n", __FILE__, __LINE__); break; } @@ -1237,11 +1366,11 @@ set_aux_work_flags_wakeup_nob(ErtsSchedulerSleepInfo *ssi, ERTS_DBG_CHK_SSI_AUX_WORK(ssi); old_flgs = erts_atomic32_read_nob(&ssi->aux_work); - if ((old_flgs & flgs) == 0) { + if ((old_flgs & flgs) != flgs) { old_flgs = erts_atomic32_read_bor_nob(&ssi->aux_work, flgs); - if ((old_flgs & flgs) == 0) { + if ((old_flgs & flgs) != flgs) { #ifdef ERTS_SMP erts_sched_poke(ssi); #else @@ -1261,7 +1390,7 @@ set_aux_work_flags_wakeup_relb(ErtsSchedulerSleepInfo *ssi, old_flgs = erts_atomic32_read_bor_relb(&ssi->aux_work, flgs); - if ((old_flgs & flgs) == 0) { + if ((old_flgs & flgs) != flgs) { #ifdef ERTS_SMP erts_sched_poke(ssi); #else @@ -1733,15 +1862,17 @@ handle_delayed_dealloc(ErtsAuxWorkData *awdp, erts_aint32_t aux_work, int waitin int need_thr_progress = 0; ErtsThrPrgrVal wakeup = ERTS_THR_PRGR_INVALID; int more_work = 0; - + ERTS_MSACC_PUSH_STATE_M_X(); #ifdef ERTS_DIRTY_SCHEDULERS ASSERT(!awdp->esdp || !ERTS_SCHEDULER_IS_DIRTY(awdp->esdp)); #endif unset_aux_work_flags(ssi, ERTS_SSI_AUX_WORK_DD); + ERTS_MSACC_SET_STATE_CACHED_M_X(ERTS_MSACC_STATE_ALLOC); erts_alloc_scheduler_handle_delayed_dealloc((void *) awdp->esdp, &need_thr_progress, &wakeup, &more_work); + ERTS_MSACC_POP_STATE_M_X(); if (more_work) { if (set_aux_work_flags(ssi, ERTS_SSI_AUX_WORK_DD) & ERTS_SSI_AUX_WORK_DD_THR_PRGR) { @@ -1759,11 +1890,6 @@ handle_delayed_dealloc(ErtsAuxWorkData *awdp, erts_aint32_t aux_work, int waitin awdp->dd.thr_prgr = wakeup; haw_thr_prgr_soft_wakeup(awdp, wakeup); } - else if (awdp->dd.completed_callback) { - awdp->dd.completed_callback(awdp->dd.completed_arg); - awdp->dd.completed_callback = NULL; - awdp->dd.completed_arg = NULL; - } return aux_work & ~ERTS_SSI_AUX_WORK_DD; } @@ -1805,17 +1931,107 @@ handle_delayed_dealloc_thr_prgr(ErtsAuxWorkData *awdp, erts_aint32_t aux_work, i } else { unset_aux_work_flags(ssi, ERTS_SSI_AUX_WORK_DD_THR_PRGR); - if (awdp->dd.completed_callback) { - awdp->dd.completed_callback(awdp->dd.completed_arg); - awdp->dd.completed_callback = NULL; - awdp->dd.completed_arg = NULL; - } } return aux_work & ~ERTS_SSI_AUX_WORK_DD_THR_PRGR; } /* + * Canceled timers + */ + +void +erts_notify_canceled_timer(ErtsSchedulerData *esdp, int rsid) +{ + ASSERT(esdp && esdp == erts_get_scheduler_data()); + if (esdp && !ERTS_SCHEDULER_IS_DIRTY(esdp)) + schedule_aux_work_wakeup(&esdp->aux_work_data, + rsid, + ERTS_SSI_AUX_WORK_CNCLD_TMRS); + else + set_aux_work_flags_wakeup_relb(ERTS_SCHED_SLEEP_INFO_IX(rsid-1), + ERTS_SSI_AUX_WORK_CNCLD_TMRS); +} + +static ERTS_INLINE erts_aint32_t +handle_canceled_timers(ErtsAuxWorkData *awdp, erts_aint32_t aux_work, int waiting) +{ + ErtsSchedulerSleepInfo *ssi = awdp->ssi; + int need_thr_progress = 0; + ErtsThrPrgrVal wakeup = ERTS_THR_PRGR_INVALID; + int more_work = 0; + +#ifdef ERTS_DIRTY_SCHEDULERS + ASSERT(!awdp->esdp || !ERTS_SCHEDULER_IS_DIRTY(awdp->esdp)); +#endif + unset_aux_work_flags(ssi, ERTS_SSI_AUX_WORK_CNCLD_TMRS); + erts_handle_canceled_timers((void *) awdp->esdp, + &need_thr_progress, + &wakeup, + &more_work); + if (more_work) { + if (set_aux_work_flags(ssi, ERTS_SSI_AUX_WORK_CNCLD_TMRS) + & ERTS_SSI_AUX_WORK_CNCLD_TMRS_THR_PRGR) { + unset_aux_work_flags(ssi, ERTS_SSI_AUX_WORK_CNCLD_TMRS_THR_PRGR); + aux_work &= ~ERTS_SSI_AUX_WORK_CNCLD_TMRS_THR_PRGR; + } + return aux_work; + } + + if (need_thr_progress) { + if (wakeup == ERTS_THR_PRGR_INVALID) + wakeup = erts_thr_progress_later(awdp->esdp); + awdp->cncld_tmrs.thr_prgr = wakeup; + set_aux_work_flags(ssi, ERTS_SSI_AUX_WORK_CNCLD_TMRS_THR_PRGR); + haw_thr_prgr_soft_wakeup(awdp, wakeup); + } + return aux_work & ~ERTS_SSI_AUX_WORK_CNCLD_TMRS; +} + +static ERTS_INLINE erts_aint32_t +handle_canceled_timers_thr_prgr(ErtsAuxWorkData *awdp, erts_aint32_t aux_work, int waiting) +{ + ErtsSchedulerSleepInfo *ssi; + int need_thr_progress; + int more_work; + ErtsThrPrgrVal wakeup = ERTS_THR_PRGR_INVALID; + ErtsThrPrgrVal current = haw_thr_prgr_current(awdp); + +#ifdef ERTS_DIRTY_SCHEDULERS + ASSERT(!awdp->esdp || !ERTS_SCHEDULER_IS_DIRTY(awdp->esdp)); +#endif + if (!erts_thr_progress_has_reached_this(current, awdp->cncld_tmrs.thr_prgr)) + return aux_work & ~ERTS_SSI_AUX_WORK_CNCLD_TMRS_THR_PRGR; + + ssi = awdp->ssi; + need_thr_progress = 0; + more_work = 0; + + erts_handle_canceled_timers((void *) awdp->esdp, + &need_thr_progress, + &wakeup, + &more_work); + if (more_work) { + set_aux_work_flags(ssi, ERTS_SSI_AUX_WORK_CNCLD_TMRS); + unset_aux_work_flags(ssi, ERTS_SSI_AUX_WORK_CNCLD_TMRS_THR_PRGR); + return ((aux_work & ~ERTS_SSI_AUX_WORK_CNCLD_TMRS_THR_PRGR) + | ERTS_SSI_AUX_WORK_CNCLD_TMRS); + } + + if (need_thr_progress) { + if (wakeup == ERTS_THR_PRGR_INVALID) + wakeup = erts_thr_progress_later(awdp->esdp); + awdp->cncld_tmrs.thr_prgr = wakeup; + haw_thr_prgr_soft_wakeup(awdp, wakeup); + } + else { + unset_aux_work_flags(ssi, ERTS_SSI_AUX_WORK_CNCLD_TMRS_THR_PRGR); + } + + return aux_work & ~ERTS_SSI_AUX_WORK_CNCLD_TMRS_THR_PRGR; +} + +/* * Handle scheduled thread progress later operations. */ #define ERTS_MAX_THR_PRGR_LATER_OPS 50 @@ -1904,109 +2120,154 @@ erts_schedule_thr_prgr_later_cleanup_op(void (*later_func)(void *), #endif } -#ifdef ERTS_SMP +static ERTS_INLINE erts_aint32_t +handle_debug_wait_completed(ErtsAuxWorkData *awdp, erts_aint32_t aux_work, int waiting) +{ + ErtsSchedulerSleepInfo *ssi = awdp->ssi; + erts_aint32_t saved_aux_work, flags; + +#ifdef ERTS_DIRTY_SCHEDULERS + ASSERT(!awdp->esdp || !ERTS_SCHEDULER_IS_DIRTY(awdp->esdp)); +#endif + + flags = awdp->debug.wait_completed.flags; + + if (aux_work & flags) + return aux_work; -static erts_atomic32_t completed_dealloc_count; + saved_aux_work = erts_atomic32_read_acqb(&ssi->aux_work); + + if (saved_aux_work & flags) + return aux_work & ~ERTS_SSI_AUX_WORK_DEBUG_WAIT_COMPLETED; + + awdp->debug.wait_completed.callback(awdp->debug.wait_completed.arg); + + awdp->debug.wait_completed.flags = 0; + awdp->debug.wait_completed.callback = NULL; + awdp->debug.wait_completed.arg = NULL; + + unset_aux_work_flags(ssi, ERTS_SSI_AUX_WORK_DEBUG_WAIT_COMPLETED); + + return aux_work & ~ERTS_SSI_AUX_WORK_DEBUG_WAIT_COMPLETED; +} + +static erts_atomic32_t debug_wait_completed_count; +static int debug_wait_completed_flags; static void -completed_dealloc(void *vproc) +thr_debug_wait_completed(void *vproc) { - if (erts_atomic32_dec_read_mb(&completed_dealloc_count) == 0) { + if (erts_atomic32_dec_read_mb(&debug_wait_completed_count) == 0) { erts_resume((Process *) vproc, (ErtsProcLocks) 0); - erts_smp_proc_dec_refc((Process *) vproc); + erts_proc_dec_refc((Process *) vproc); } } static void -setup_completed_dealloc(void *vproc) +setup_thr_debug_wait_completed(void *vproc) { ErtsSchedulerData *esdp = erts_get_scheduler_data(); - ErtsAuxWorkData *awdp = (esdp - ? &esdp->aux_work_data - : aux_thread_aux_work_data); - erts_alloc_fix_alloc_shrink(awdp->sched_id, 0); - set_aux_work_flags_wakeup_nob(awdp->ssi, ERTS_SSI_AUX_WORK_DD); - awdp->dd.completed_callback = completed_dealloc; - awdp->dd.completed_arg = vproc; + ErtsAuxWorkData *awdp; + erts_aint32_t wait_flags, aux_work_flags; +#ifdef ERTS_SMP + awdp = esdp ? &esdp->aux_work_data : aux_thread_aux_work_data; +#else + awdp = &esdp->aux_work_data; +#endif + + wait_flags = 0; + aux_work_flags = ERTS_SSI_AUX_WORK_DEBUG_WAIT_COMPLETED; + + if (debug_wait_completed_flags & ERTS_DEBUG_WAIT_COMPLETED_DEALLOCATIONS) { + erts_alloc_fix_alloc_shrink(awdp->sched_id, 0); + wait_flags |= (ERTS_SSI_AUX_WORK_DD + | ERTS_SSI_AUX_WORK_DD_THR_PRGR); +#ifdef ERTS_SMP + aux_work_flags |= ERTS_SSI_AUX_WORK_DD; +#endif + } + + if (debug_wait_completed_flags & ERTS_DEBUG_WAIT_COMPLETED_TIMER_CANCELLATIONS) { + wait_flags |= (ERTS_SSI_AUX_WORK_CNCLD_TMRS + | ERTS_SSI_AUX_WORK_CNCLD_TMRS_THR_PRGR); +#ifdef ERTS_SMP + if (awdp->esdp && !ERTS_SCHEDULER_IS_DIRTY(awdp->esdp)) + aux_work_flags |= ERTS_SSI_AUX_WORK_CNCLD_TMRS; +#endif + } + + set_aux_work_flags_wakeup_nob(awdp->ssi, aux_work_flags); + + awdp->debug.wait_completed.flags = wait_flags; + awdp->debug.wait_completed.callback = thr_debug_wait_completed; + awdp->debug.wait_completed.arg = vproc; } -static void -prep_setup_completed_dealloc(void *vproc) +struct debug_lop { + ErtsThrPrgrLaterOp lop; + Process *proc; +}; + +static void later_thr_debug_wait_completed(void *vlop) { - erts_aint32_t count = (erts_aint32_t) (erts_no_schedulers+1); - if (erts_atomic32_dec_read_mb(&completed_dealloc_count) == count) { - /* scheduler threads */ - erts_schedule_multi_misc_aux_work(0, - erts_no_schedulers, - setup_completed_dealloc, - vproc); - /* aux_thread */ - erts_schedule_misc_aux_work(0, - setup_completed_dealloc, - vproc); + struct debug_lop *lop = vlop; + erts_aint32_t count = (erts_aint32_t) erts_no_schedulers; +#ifdef ERTS_SMP + count += 1; /* aux thread */ +#endif + if (erts_atomic32_dec_read_mb(&debug_wait_completed_count) == count) { + /* scheduler threads */ + erts_schedule_multi_misc_aux_work(0, + erts_no_schedulers, + setup_thr_debug_wait_completed, + lop->proc); +#ifdef ERTS_SMP + /* aux_thread */ + erts_schedule_misc_aux_work(0, + setup_thr_debug_wait_completed, + lop->proc); +#endif } + erts_free(ERTS_ALC_T_DEBUG, lop); +} + + +static void +init_thr_debug_wait_completed(void *vproc) +{ + struct debug_lop* lop = erts_alloc(ERTS_ALC_T_DEBUG, + sizeof(struct debug_lop)); + lop->proc = vproc; + erts_schedule_thr_prgr_later_op(later_thr_debug_wait_completed, lop, &lop->lop); } -#endif /* ERTS_SMP */ int -erts_debug_wait_deallocations(Process *c_p) +erts_debug_wait_completed(Process *c_p, int flags) { -#ifndef ERTS_SMP - erts_alloc_fix_alloc_shrink(1, 0); - return 1; -#else /* Only one process at a time can do this */ - erts_aint32_t count = (erts_aint32_t) (2*(erts_no_schedulers+1)); - if (0 == erts_atomic32_cmpxchg_mb(&completed_dealloc_count, + erts_aint32_t count = (erts_aint32_t) (2*erts_no_schedulers); +#ifdef ERTS_SMP + count += 1; /* aux thread */ +#endif + if (0 == erts_atomic32_cmpxchg_mb(&debug_wait_completed_count, count, 0)) { + debug_wait_completed_flags = flags; erts_suspend(c_p, ERTS_PROC_LOCK_MAIN, NULL); - erts_smp_proc_inc_refc(c_p); - /* scheduler threads */ + erts_proc_inc_refc(c_p); + + /* First flush later-ops on all scheduler threads */ erts_schedule_multi_misc_aux_work(0, erts_no_schedulers, - prep_setup_completed_dealloc, + init_thr_debug_wait_completed, (void *) c_p); - /* aux_thread */ - erts_schedule_misc_aux_work(0, - prep_setup_completed_dealloc, - (void *) c_p); return 1; } return 0; -#endif } -#ifdef ERTS_SMP_SCHEDULERS_NEED_TO_CHECK_CHILDREN -void -erts_smp_notify_check_children_needed(void) -{ - int i; - for (i = 0; i < erts_no_schedulers; i++) - set_aux_work_flags_wakeup_nob(ERTS_SCHED_SLEEP_INFO_IX(i), - ERTS_SSI_AUX_WORK_CHECK_CHILDREN); -#ifdef ERTS_DIRTY_SCHEDULERS - for (i = 0; i < erts_no_dirty_cpu_schedulers; i++) - set_aux_work_flags_wakeup_nob(ERTS_DIRTY_CPU_SCHED_SLEEP_INFO_IX(i), - ERTS_SSI_AUX_WORK_CHECK_CHILDREN); - for (i = 0; i < erts_no_dirty_io_schedulers; i++) - set_aux_work_flags_wakeup_nob(ERTS_DIRTY_IO_SCHED_SLEEP_INFO_IX(i), - ERTS_SSI_AUX_WORK_CHECK_CHILDREN); -#endif -} - -static ERTS_INLINE erts_aint32_t -handle_check_children(ErtsAuxWorkData *awdp, erts_aint32_t aux_work, int waiting) -{ - unset_aux_work_flags(awdp->ssi, ERTS_SSI_AUX_WORK_CHECK_CHILDREN); - erts_check_children(); - return aux_work & ~ERTS_SSI_AUX_WORK_CHECK_CHILDREN; -} - -#endif - static void notify_reap_ports_relb(void) { @@ -2055,7 +2316,7 @@ handle_reap_ports(ErtsAuxWorkData *awdp, erts_aint32_t aux_work, int waiting) erts_port_release(prt); } if (erts_smp_atomic32_dec_read_nob(&erts_halt_progress) == 0) { - erl_exit_flush_async(erts_halt_code, ""); + erts_flush_async_exit(erts_halt_code, ""); } } return aux_work & ~ERTS_SSI_AUX_WORK_REAP_PORTS; @@ -2077,7 +2338,7 @@ static ERTS_INLINE erts_aint32_t handle_setup_aux_work_timer(ErtsAuxWorkData *awdp, erts_aint32_t aux_work, int waiting) { unset_aux_work_flags(awdp->ssi, ERTS_SSI_AUX_WORK_SET_TMO); - setup_aux_work_timer(); + setup_aux_work_timer(awdp->esdp); return aux_work & ~ERTS_SSI_AUX_WORK_SET_TMO; } @@ -2092,13 +2353,17 @@ handle_aux_work(ErtsAuxWorkData *awdp, erts_aint32_t orig_aux_work, int waiting) ERTS_DBG_CHK_AUX_WORK_VAL(aux_work); \ if (!(aux_work & ~ignore)) { \ ERTS_DBG_CHK_AUX_WORK_VAL(aux_work); \ + ERTS_MSACC_UPDATE_CACHE(); \ + ERTS_MSACC_POP_STATE_M(); \ return aux_work; \ } \ } erts_aint32_t aux_work = orig_aux_work; erts_aint32_t ignore = 0; + ERTS_MSACC_PUSH_AND_SET_STATE_M(ERTS_MSACC_STATE_AUX); + ASSERT(!awdp->esdp || !ERTS_SCHEDULER_IS_DIRTY(awdp->esdp)); #ifdef ERTS_SMP haw_thr_prgr_current_reset(awdp); #endif @@ -2137,6 +2402,11 @@ handle_aux_work(ErtsAuxWorkData *awdp, erts_aint32_t orig_aux_work, int waiting) #ifdef ERTS_SMP HANDLE_AUX_WORK(ERTS_SSI_AUX_WORK_THR_PRGR_LATER_OP, handle_thr_prgr_later_op); + HANDLE_AUX_WORK(ERTS_SSI_AUX_WORK_CNCLD_TMRS, + handle_canceled_timers); + /* CNCLD_TMRS must be before CNCLD_TMRS_THR_PRGR */ + HANDLE_AUX_WORK(ERTS_SSI_AUX_WORK_CNCLD_TMRS_THR_PRGR, + handle_canceled_timers_thr_prgr); #endif #if ERTS_USE_ASYNC_READY_Q @@ -2155,10 +2425,6 @@ handle_aux_work(ErtsAuxWorkData *awdp, erts_aint32_t orig_aux_work, int waiting) HANDLE_AUX_WORK(ERTS_SSI_AUX_WORK_MISC, handle_misc_aux_work); -#ifdef ERTS_SMP_SCHEDULERS_NEED_TO_CHECK_CHILDREN - HANDLE_AUX_WORK(ERTS_SSI_AUX_WORK_CHECK_CHILDREN, - handle_check_children); -#endif HANDLE_AUX_WORK(ERTS_SSI_AUX_WORK_SET_TMO, handle_setup_aux_work_timer); @@ -2171,6 +2437,14 @@ handle_aux_work(ErtsAuxWorkData *awdp, erts_aint32_t orig_aux_work, int waiting) HANDLE_AUX_WORK(ERTS_SSI_AUX_WORK_REAP_PORTS, handle_reap_ports); + /* + * ERTS_SSI_AUX_WORK_DEBUG_WAIT_COMPLETED *need* to be + * the last flag checked! + */ + + HANDLE_AUX_WORK(ERTS_SSI_AUX_WORK_DEBUG_WAIT_COMPLETED, + handle_debug_wait_completed); + ERTS_DBG_CHK_AUX_WORK_VAL(aux_work); #ifdef ERTS_SMP @@ -2178,6 +2452,8 @@ handle_aux_work(ErtsAuxWorkData *awdp, erts_aint32_t orig_aux_work, int waiting) haw_thr_prgr_current_check_progress(awdp); #endif + ERTS_MSACC_UPDATE_CACHE(); + ERTS_MSACC_POP_STATE_M(); return aux_work; #undef HANDLE_AUX_WORK @@ -2186,8 +2462,8 @@ handle_aux_work(ErtsAuxWorkData *awdp, erts_aint32_t orig_aux_work, int waiting) typedef struct { union { - ErlTimer data; - char align__[ERTS_ALC_CACHE_LINE_ALIGN_SIZE(sizeof(ErlTimer))]; + ErtsTWheelTimer data; + char align__[ERTS_ALC_CACHE_LINE_ALIGN_SIZE(sizeof(ErtsTWheelTimer))]; } timer; int initialized; @@ -2197,6 +2473,22 @@ typedef struct { static ErtsAuxWorkTmo *aux_work_tmo; +static ERTS_INLINE void +start_aux_work_timer(ErtsSchedulerData *esdp) +{ + ErtsMonotonicTime tmo = erts_get_monotonic_time(esdp); + tmo = ERTS_MONOTONIC_TO_CLKTCKS(tmo-1); + tmo += ERTS_MSEC_TO_CLKTCKS(1000) + 1; + erts_twheel_init_timer(&aux_work_tmo->timer.data); + ASSERT(esdp); + erts_twheel_set_timer(esdp->timer_wheel, + &aux_work_tmo->timer.data, + aux_work_timeout, + NULL, + (void *) esdp, + tmo); +} + static void aux_work_timeout_early_init(int no_schedulers) { @@ -2211,6 +2503,9 @@ aux_work_timeout_early_init(int no_schedulers) p = (UWord) malloc((sizeof(ErtsAuxWorkTmo) + sizeof(erts_atomic32_t)*(no_schedulers+1)) + ERTS_CACHE_LINE_SIZE-1); + if (!p) { + ERTS_INTERNAL_ERROR("malloc failed to allocate memory!"); + } if (p & ERTS_CACHE_LINE_MASK) p = (p & ~ERTS_CACHE_LINE_MASK) + ERTS_CACHE_LINE_SIZE; ASSERT((p & ERTS_CACHE_LINE_MASK) == 0); @@ -2226,18 +2521,12 @@ void aux_work_timeout_late_init(void) { aux_work_tmo->initialized = 1; - if (erts_atomic32_read_nob(&aux_work_tmo->refc)) { - aux_work_tmo->timer.data.active = 0; - erts_set_timer(&aux_work_tmo->timer.data, - aux_work_timeout, - NULL, - NULL, - 1000); - } + if (erts_atomic32_read_nob(&aux_work_tmo->refc)) + start_aux_work_timer(erts_get_scheduler_data()); } static void -aux_work_timeout(void *unused) +aux_work_timeout(void *vesdp) { erts_aint32_t refc; int i; @@ -2260,32 +2549,18 @@ aux_work_timeout(void *unused) if (refc != 1 || 1 != erts_atomic32_cmpxchg_relb(&aux_work_tmo->refc, 0, 1)) { /* Setup next timeout... */ - aux_work_tmo->timer.data.active = 0; - erts_set_timer(&aux_work_tmo->timer.data, - aux_work_timeout, - NULL, - NULL, - 1000); + start_aux_work_timer((ErtsSchedulerData *) vesdp); } } static void -setup_aux_work_timer(void) +setup_aux_work_timer(ErtsSchedulerData *esdp) { -#ifndef ERTS_SMP - if (!erts_get_scheduler_data()) + if (!esdp || !esdp->timer_wheel) set_aux_work_flags_wakeup_nob(ERTS_SCHED_SLEEP_INFO_IX(0), ERTS_SSI_AUX_WORK_SET_TMO); else -#endif - { - aux_work_tmo->timer.data.active = 0; - erts_set_timer(&aux_work_tmo->timer.data, - aux_work_timeout, - NULL, - NULL, - 1000); - } + start_aux_work_timer(esdp); } erts_aint32_t @@ -2316,7 +2591,7 @@ erts_set_aux_work_timeout(int ix, erts_aint32_t type, int enable) if (refc == 1) { erts_atomic32_inc_acqb(&aux_work_tmo->refc); if (aux_work_tmo->initialized) - setup_aux_work_timer(); + setup_aux_work_timer(erts_get_scheduler_data()); } } return old; @@ -2359,7 +2634,6 @@ erts_active_schedulers(void) ERTS_ATOMIC_FOREACH_RUNQ(rq, as -= abs(rq->waiting)); - ASSERT(as >= 0); return as; } @@ -2380,29 +2654,28 @@ try_set_sys_scheduling(void) #endif static ERTS_INLINE int -prepare_for_sys_schedule(ErtsSchedulerData *esdp) +prepare_for_sys_schedule(int non_blocking) { + if (non_blocking && erts_eager_check_io) { #ifdef ERTS_SMP - while (!erts_port_task_have_outstanding_io_tasks() - && try_set_sys_scheduling()) { -#ifdef ERTS_SCHED_ONLY_POLL_SCHED_1 - if (esdp->no != 1) { - /* If we are not scheduler 1 and ERTS_SCHED_ONLY_POLL_SCHED_1 is used - then we make sure to wake scheduler 1 */ - ErtsRunQueue *rq = ERTS_RUNQ_IX(0); - clear_sys_scheduling(); - wake_scheduler(rq); - return 0; - } -#endif - if (!erts_port_task_have_outstanding_io_tasks()) + return try_set_sys_scheduling(); +#else return 1; - clear_sys_scheduling(); +#endif } - return 0; + else { +#ifdef ERTS_SMP + while (!erts_port_task_have_outstanding_io_tasks() + && try_set_sys_scheduling()) { + if (!erts_port_task_have_outstanding_io_tasks()) + return 1; + clear_sys_scheduling(); + } + return 0; #else - return !erts_port_task_have_outstanding_io_tasks(); + return !erts_port_task_have_outstanding_io_tasks(); #endif + } } #ifdef ERTS_SMP @@ -2445,13 +2718,6 @@ sched_active(Uint no, ErtsRunQueue *rq) profile_scheduler(make_small(no), am_active); } -static int ERTS_INLINE -ongoing_multi_scheduling_block(void) -{ - ERTS_SMP_LC_ASSERT(erts_lc_mtx_is_locked(&schdlr_sspnd.mtx)); - return schdlr_sspnd.msb.ongoing; -} - static ERTS_INLINE void empty_runq_aux(ErtsRunQueue *rq, Uint32 old_flags) { @@ -2663,6 +2929,13 @@ thr_prgr_fin_wait(void *vssi) static void init_aux_work_data(ErtsAuxWorkData *awdp, ErtsSchedulerData *esdp, char *dawwp); +void +erts_interupt_aux_thread_timed(ErtsMonotonicTime timeout_time) +{ + /* TODO only poke when needed (based on timeout_time) */ + erts_sched_poke(ERTS_SCHED_SLEEP_INFO_IX(-1)); +} + static void * aux_thread(void *unused) { @@ -2672,8 +2945,17 @@ aux_thread(void *unused) ErtsThrPrgrCallbacks callbacks; int thr_prgr_active = 1; +#ifdef ERTS_ENABLE_LOCK_CHECK + { + char buf[] = "aux_thread"; + erts_lc_set_thread_name(buf); + } +#endif + ssi->event = erts_tse_fetch(); + erts_msacc_init_thread("aux", 1, 1); + callbacks.arg = (void *) ssi; callbacks.wakeup = thr_prgr_wakeup; callbacks.prepare_wait = thr_prgr_prep_wait; @@ -2684,6 +2966,7 @@ aux_thread(void *unused) init_aux_work_data(awdp, NULL, NULL); awdp->ssi = ssi; + sched_prep_spin_wait(ssi); while (1) { @@ -2703,8 +2986,6 @@ aux_thread(void *unused) erts_thr_progress_active(NULL, thr_prgr_active = 0); erts_thr_progress_prepare_wait(NULL); - ERTS_SCHED_FAIR_YIELD(); - flgs = sched_spin_wait(ssi, 0); if (flgs & ERTS_SSI_FLG_SLEEPING) { @@ -2739,6 +3020,9 @@ scheduler_wait(int *fcalls, ErtsSchedulerData *esdp, ErtsRunQueue *rq) #ifdef ERTS_SMP int thr_prgr_active = 1; erts_aint32_t flgs; +#endif + ERTS_MSACC_PUSH_STATE_M(); +#ifdef ERTS_SMP ERTS_SMP_LC_ASSERT(erts_smp_lc_runq_is_locked(rq)); @@ -2772,7 +3056,7 @@ scheduler_wait(int *fcalls, ErtsSchedulerData *esdp, ErtsRunQueue *rq) * be waiting in erl_sys_schedule() */ - if (ERTS_SCHEDULER_IS_DIRTY(esdp) || !prepare_for_sys_schedule(esdp)) { + if (ERTS_SCHEDULER_IS_DIRTY(esdp) || !prepare_for_sys_schedule(0)) { sched_waiting(esdp->no, rq); @@ -2786,47 +3070,92 @@ scheduler_wait(int *fcalls, ErtsSchedulerData *esdp, ErtsRunQueue *rq) sched_wall_time_change(esdp, thr_prgr_active); while (1) { + ErtsMonotonicTime current_time = 0; aux_work = erts_atomic32_read_acqb(&ssi->aux_work); - if (aux_work) { - if (!ERTS_SCHEDULER_IS_DIRTY(esdp) && !thr_prgr_active) { + if (aux_work && !ERTS_SCHEDULER_IS_DIRTY(esdp)) { + if (!thr_prgr_active) { erts_thr_progress_active(esdp, thr_prgr_active = 1); sched_wall_time_change(esdp, 1); } aux_work = handle_aux_work(&esdp->aux_work_data, aux_work, 1); - if (aux_work && !ERTS_SCHEDULER_IS_DIRTY(esdp) - && erts_thr_progress_update(esdp)) + ERTS_MSACC_UPDATE_CACHE(); + if (aux_work && erts_thr_progress_update(esdp)) erts_thr_progress_leader_update(esdp); } - if (aux_work) - flgs = erts_smp_atomic32_read_acqb(&ssi->flags); + if (aux_work) { + if (!ERTS_SCHEDULER_IS_DIRTY(esdp)) { + flgs = erts_smp_atomic32_read_acqb(&ssi->flags); + current_time = erts_get_monotonic_time(esdp); + if (current_time >= erts_next_timeout_time(esdp->next_tmo_ref)) { + if (!thr_prgr_active) { + erts_thr_progress_active(esdp, thr_prgr_active = 1); + sched_wall_time_change(esdp, 1); + } + erts_bump_timers(esdp->timer_wheel, current_time); + } + } + } else { + ErtsMonotonicTime timeout_time; + int do_timeout = 0; if (!ERTS_SCHEDULER_IS_DIRTY(esdp)) { - if (thr_prgr_active) { - erts_thr_progress_active(esdp, thr_prgr_active = 0); - sched_wall_time_change(esdp, 0); + timeout_time = erts_check_next_timeout_time(esdp); + current_time = erts_get_monotonic_time(esdp); + do_timeout = (current_time >= timeout_time); + } else { + current_time = 0; + timeout_time = ERTS_MONOTONIC_TIME_MAX; + } + if (do_timeout) { + if (!thr_prgr_active) { + erts_thr_progress_active(esdp, thr_prgr_active = 1); + sched_wall_time_change(esdp, 1); } - erts_thr_progress_prepare_wait(esdp); } + else { + if (!ERTS_SCHEDULER_IS_DIRTY(esdp)) { + if (thr_prgr_active) { + erts_thr_progress_active(esdp, thr_prgr_active = 0); + sched_wall_time_change(esdp, 0); + } + erts_thr_progress_prepare_wait(esdp); + } - ERTS_SCHED_FAIR_YIELD(); - - flgs = sched_spin_wait(ssi, spincount); - if (flgs & ERTS_SSI_FLG_SLEEPING) { - ASSERT(flgs & ERTS_SSI_FLG_WAITING); - flgs = sched_set_sleeptype(ssi, ERTS_SSI_FLG_TSE_SLEEPING); + flgs = sched_spin_wait(ssi, spincount); if (flgs & ERTS_SSI_FLG_SLEEPING) { - int res; - ASSERT(flgs & ERTS_SSI_FLG_TSE_SLEEPING); ASSERT(flgs & ERTS_SSI_FLG_WAITING); - do { - res = erts_tse_wait(ssi->event); - } while (res == EINTR); + flgs = sched_set_sleeptype(ssi, ERTS_SSI_FLG_TSE_SLEEPING); + if (flgs & ERTS_SSI_FLG_SLEEPING) { + int res; + ASSERT(flgs & ERTS_SSI_FLG_TSE_SLEEPING); + ASSERT(flgs & ERTS_SSI_FLG_WAITING); + current_time = ERTS_SCHEDULER_IS_DIRTY(esdp) ? 0 : + erts_get_monotonic_time(esdp); + do { + Sint64 timeout; + if (current_time >= timeout_time) + break; + if (!ERTS_SCHEDULER_IS_DIRTY(esdp)) { + timeout = ERTS_MONOTONIC_TO_NSEC(timeout_time + - current_time + - 1) + 1; + } else + timeout = -1; + ERTS_MSACC_SET_STATE_CACHED_M(ERTS_MSACC_STATE_SLEEP); + res = erts_tse_twait(ssi->event, timeout); + ERTS_MSACC_POP_STATE_M(); + current_time = ERTS_SCHEDULER_IS_DIRTY(esdp) ? 0 : + erts_get_monotonic_time(esdp); + } while (res == EINTR); + } } + if (!ERTS_SCHEDULER_IS_DIRTY(esdp)) + erts_thr_progress_finalize_wait(esdp); } - if (!ERTS_SCHEDULER_IS_DIRTY(esdp)) - erts_thr_progress_finalize_wait(esdp); + if (!ERTS_SCHEDULER_IS_DIRTY(esdp) && current_time >= timeout_time) + erts_bump_timers(esdp->timer_wheel, current_time); } if (!(flgs & ERTS_SSI_FLG_WAITING)) { @@ -2859,7 +3188,6 @@ scheduler_wait(int *fcalls, ErtsSchedulerData *esdp, ErtsRunQueue *rq) else #endif { - erts_aint_t dt; erts_smp_atomic32_set_relb(&function_calls, 0); *fcalls = 0; @@ -2867,13 +3195,8 @@ scheduler_wait(int *fcalls, ErtsSchedulerData *esdp, ErtsRunQueue *rq) #ifdef ERTS_DIRTY_SCHEDULERS ASSERT(!ERTS_SCHEDULER_IS_DIRTY(esdp)); #endif - -#ifdef ERTS_SCHED_ONLY_POLL_SCHED_1 - ASSERT(esdp->no == 1); -#endif sched_waiting_sys(esdp->no, rq); - erts_smp_runq_unlock(rq); ASSERT(working); @@ -2884,17 +3207,25 @@ scheduler_wait(int *fcalls, ErtsSchedulerData *esdp, ErtsRunQueue *rq) goto sys_aux_work; while (spincount-- > 0) { + ErtsMonotonicTime current_time; sys_poll_aux_work: if (working) sched_wall_time_change(esdp, working = 0); + ERTS_MSACC_SET_STATE_CACHED_M(ERTS_MSACC_STATE_CHECK_IO); + ASSERT(!erts_port_task_have_outstanding_io_tasks()); erl_sys_schedule(1); /* Might give us something to do */ - dt = erts_do_time_read_and_reset(); - if (dt) erts_bump_timer(dt); + ERTS_MSACC_POP_STATE_M(); + + if (!ERTS_SCHEDULER_IS_DIRTY(esdp)) { + current_time = erts_get_monotonic_time(esdp); + if (current_time >= erts_next_timeout_time(esdp->next_tmo_ref)) + erts_bump_timers(esdp->timer_wheel, current_time); + } sys_aux_work: #ifndef ERTS_SMP @@ -2902,7 +3233,7 @@ scheduler_wait(int *fcalls, ErtsSchedulerData *esdp, ErtsRunQueue *rq) #endif aux_work = erts_atomic32_read_acqb(&ssi->aux_work); - if (aux_work) { + if (aux_work && !ERTS_SCHEDULER_IS_DIRTY(esdp)) { if (!working) sched_wall_time_change(esdp, working = 1); #ifdef ERTS_SMP @@ -2910,6 +3241,7 @@ scheduler_wait(int *fcalls, ErtsSchedulerData *esdp, ErtsRunQueue *rq) erts_thr_progress_active(esdp, thr_prgr_active = 1); #endif aux_work = handle_aux_work(&esdp->aux_work_data, aux_work, 1); + ERTS_MSACC_UPDATE_CACHE(); #ifdef ERTS_SMP if (aux_work && erts_thr_progress_update(esdp)) erts_thr_progress_leader_update(esdp); @@ -2917,7 +3249,7 @@ scheduler_wait(int *fcalls, ErtsSchedulerData *esdp, ErtsRunQueue *rq) } #ifndef ERTS_SMP - if (rq->len != 0 || rq->misc.start) + if (erts_smp_atomic32_read_dirty(&rq->len) != 0 || rq->misc.start) goto sys_woken; #else flgs = erts_smp_atomic32_read_acqb(&ssi->flags); @@ -2936,7 +3268,7 @@ scheduler_wait(int *fcalls, ErtsSchedulerData *esdp, ErtsRunQueue *rq) * Got to check that we still got I/O tasks; otherwise * we have to continue checking for I/O... */ - if (!prepare_for_sys_schedule(esdp)) { + if (!prepare_for_sys_schedule(0)) { spincount *= ERTS_SCHED_TSE_SLEEP_SPINCOUNT_FACT; goto tse_wait; } @@ -2958,7 +3290,7 @@ scheduler_wait(int *fcalls, ErtsSchedulerData *esdp, ErtsRunQueue *rq) * Got to check that we still got I/O tasks; otherwise * we have to wait in erl_sys_schedule() after all... */ - if (!prepare_for_sys_schedule(esdp)) { + if (!prepare_for_sys_schedule(0)) { /* * Not allowed to wait in erl_sys_schedule; * do tse wait instead... @@ -3007,13 +3339,20 @@ scheduler_wait(int *fcalls, ErtsSchedulerData *esdp, ErtsRunQueue *rq) ASSERT(!erts_port_task_have_outstanding_io_tasks()); + ERTS_MSACC_SET_STATE_CACHED_M(ERTS_MSACC_STATE_CHECK_IO); + erl_sys_schedule(0); - dt = erts_do_time_read_and_reset(); - if (dt) erts_bump_timer(dt); + ERTS_MSACC_POP_STATE_M(); + + if (!ERTS_SCHEDULER_IS_DIRTY(esdp)) { + ErtsMonotonicTime current_time = erts_get_monotonic_time(esdp); + if (current_time >= erts_next_timeout_time(esdp->next_tmo_ref)) + erts_bump_timers(esdp->timer_wheel, current_time); + } #ifndef ERTS_SMP - if (rq->len == 0 && !rq->misc.start) + if (erts_smp_atomic32_read_dirty(&rq->len) == 0 && !rq->misc.start) goto sys_aux_work; sys_woken: #else @@ -3075,7 +3414,8 @@ wake_scheduler(ErtsRunQueue *rq) * so all code *should* handle this without having * the lock on the run queue. */ - ERTS_SMP_LC_ASSERT(!erts_smp_lc_runq_is_locked(rq)); + ERTS_SMP_LC_ASSERT(!erts_smp_lc_runq_is_locked(rq) + || ERTS_RUNQ_IX_IS_DIRTY(rq->ix)); ssi = rq->scheduler->ssi; @@ -3212,11 +3552,11 @@ chk_wake_sched(ErtsRunQueue *crq, int ix, int activate) return 0; wrq = ERTS_RUNQ_IX(ix); flags = ERTS_RUNQ_FLGS_GET(wrq); + if (activate && !(flags & ERTS_RUNQ_FLG_SUSPENDED)) { + if (try_inc_no_active_runqs(ix+1)) + (void) ERTS_RUNQ_FLGS_UNSET(wrq, ERTS_RUNQ_FLG_INACTIVE); + } if (!(flags & (ERTS_RUNQ_FLG_SUSPENDED|ERTS_RUNQ_FLG_NONEMPTY))) { - if (activate) { - if (try_inc_no_active_runqs(ix+1)) - (void) ERTS_RUNQ_FLGS_UNSET(wrq, ERTS_RUNQ_FLG_INACTIVE); - } wake_scheduler(wrq); return 1; } @@ -3400,6 +3740,13 @@ check_requeue_process(ErtsRunQueue *rq, int prio_q) return 0; } +static ERTS_INLINE void +free_proxy_proc(Process *proxy) +{ + ASSERT(erts_smp_atomic32_read_nob(&proxy->state) & ERTS_PSFLG_PROXY); + erts_free(ERTS_ALC_T_PROC, proxy); +} + #ifdef ERTS_SMP static ErtsRunQueue * @@ -3482,7 +3829,7 @@ immigrate(ErtsRunQueue *c_rq, ErtsMigrationPath *mp) prio = ERTS_PORT_PRIO_LEVEL; break; default: - erl_exit(ERTS_ABORT_EXIT, + erts_exit(ERTS_ABORT_EXIT, "%s:%d:%s(): Invalid immigrate queue mask", __FILE__, __LINE__, __func__); prio = 0; @@ -3506,7 +3853,7 @@ immigrate(ErtsRunQueue *c_rq, ErtsMigrationPath *mp) rq = erts_port_runq(prt); if (rq) { if (rq != c_rq) - erl_exit(ERTS_ABORT_EXIT, + erts_exit(ERTS_ABORT_EXIT, "%s:%d:%s(): Internal error", __FILE__, __LINE__, __func__); erts_enqueue_port(c_rq, prt); @@ -3571,24 +3918,33 @@ static ERTS_INLINE void resume_run_queue(ErtsRunQueue *rq) { int pix; + Uint32 oflgs; erts_smp_runq_lock(rq); - (void) ERTS_RUNQ_FLGS_READ_BSET(rq, - (ERTS_RUNQ_FLG_OUT_OF_WORK - | ERTS_RUNQ_FLG_HALFTIME_OUT_OF_WORK - | ERTS_RUNQ_FLG_SUSPENDED), - (ERTS_RUNQ_FLG_OUT_OF_WORK - | ERTS_RUNQ_FLG_HALFTIME_OUT_OF_WORK)); + oflgs = ERTS_RUNQ_FLGS_READ_BSET(rq, + (ERTS_RUNQ_FLG_OUT_OF_WORK + | ERTS_RUNQ_FLG_HALFTIME_OUT_OF_WORK + | ERTS_RUNQ_FLG_SUSPENDED), + (ERTS_RUNQ_FLG_OUT_OF_WORK + | ERTS_RUNQ_FLG_HALFTIME_OUT_OF_WORK)); + + if (oflgs & ERTS_RUNQ_FLG_SUSPENDED) { + erts_aint32_t len; + + rq->check_balance_reds = ERTS_RUNQ_CALL_CHECK_BALANCE_REDS; + for (pix = 0; pix < ERTS_NO_PROC_PRIO_LEVELS; pix++) { + len = erts_smp_atomic32_read_dirty(&rq->procs.prio_info[pix].len); + rq->procs.prio_info[pix].max_len = len; + rq->procs.prio_info[pix].reds = 0; + } + len = erts_smp_atomic32_read_dirty(&rq->ports.info.len); + rq->ports.info.max_len = len; + rq->ports.info.reds = 0; + len = erts_smp_atomic32_read_dirty(&rq->len); + rq->max_len = len; - rq->check_balance_reds = ERTS_RUNQ_CALL_CHECK_BALANCE_REDS; - for (pix = 0; pix < ERTS_NO_PROC_PRIO_LEVELS; pix++) { - rq->procs.prio_info[pix].max_len = 0; - rq->procs.prio_info[pix].reds = 0; } - rq->ports.info.max_len = 0; - rq->ports.info.reds = 0; - rq->max_len = 0; erts_smp_runq_unlock(rq); @@ -3697,7 +4053,7 @@ evacuate_run_queue(ErtsRunQueue *rq, prt_rq = erts_port_runq(prt); if (prt_rq) { if (prt_rq != to_rq) - erl_exit(ERTS_ABORT_EXIT, + erts_exit(ERTS_ABORT_EXIT, "%s:%d:%s() internal error\n", __FILE__, __LINE__, __func__); erts_enqueue_port(to_rq, prt); @@ -3714,9 +4070,6 @@ evacuate_run_queue(ErtsRunQueue *rq, erts_aint32_t state; Process *proc; int notify = 0; -#ifdef ERTS_DIRTY_SCHEDULERS - int requeue; -#endif to_rq = NULL; #ifdef ERTS_DIRTY_SCHEDULERS @@ -3731,10 +4084,85 @@ evacuate_run_queue(ErtsRunQueue *rq, proc = dequeue_process(rq, prio_q, &state); while (proc) { + Process *real_proc; + int prio; + erts_aint32_t max_qbit, qbit, real_state; + + prio = ERTS_PSFLGS_GET_PRQ_PRIO(state); + qbit = ((erts_aint32_t) 1) << prio; + + if (!(state & ERTS_PSFLG_PROXY)) { + real_proc = proc; + real_state = state; + } + else { + real_proc = erts_proc_lookup_raw(proc->common.id); + if (!real_proc) { + free_proxy_proc(proc); + goto handle_next_proc; + } + real_state = erts_smp_atomic32_read_acqb(&real_proc->state); + } + + max_qbit = (state >> ERTS_PSFLGS_IN_PRQ_MASK_OFFSET); + max_qbit &= ERTS_PSFLGS_QMASK; + max_qbit |= 1 << ERTS_PSFLGS_QMASK_BITS; + max_qbit &= -max_qbit; + + if (qbit > max_qbit) { + /* Process already queued with higher prio; drop it... */ + if (real_proc != proc) + free_proxy_proc(proc); + else { + erts_aint32_t clr_bits; +#ifdef DEBUG + erts_aint32_t old; +#endif + + clr_bits = ERTS_PSFLG_IN_RUNQ; + clr_bits |= qbit << ERTS_PSFLGS_IN_PRQ_MASK_OFFSET; + +#ifdef DEBUG + old = +#else + (void) +#endif + erts_smp_atomic32_read_band_mb(&proc->state, + ~clr_bits); + ASSERT((old & clr_bits) == clr_bits); + + } + + goto handle_next_proc; + } + #ifdef ERTS_DIRTY_SCHEDULERS - requeue = 1; + + if (ERTS_RUNQ_IX_IS_DIRTY(rq->ix)) { + erts_aint32_t dqbit = qbit; +#ifdef DEBUG + erts_aint32_t old_dqbit; #endif - if (ERTS_PSFLG_BOUND & state) { + + if (rq == ERTS_DIRTY_CPU_RUNQ) + dqbit <<= ERTS_PDSFLGS_IN_CPU_PRQ_MASK_OFFSET; + else { + ASSERT(rq == ERTS_DIRTY_IO_RUNQ); + dqbit <<= ERTS_PDSFLGS_IN_IO_PRQ_MASK_OFFSET; + } + +#ifdef DEBUG + old_dqbit = (int) +#else + (void) +#endif + erts_smp_atomic32_read_band_mb(&real_proc->dirty_state, + ~dqbit); + ASSERT(old_dqbit & dqbit); + } +#endif + + if (ERTS_PSFLG_BOUND & real_state) { /* Bound processes get stuck here... */ proc->next = NULL; if (sbpp->last) @@ -3742,30 +4170,8 @@ evacuate_run_queue(ErtsRunQueue *rq, else sbpp->first = proc; sbpp->last = proc; -#ifdef ERTS_DIRTY_SCHEDULERS - requeue = 0; -#endif } -#ifdef ERTS_DIRTY_SCHEDULERS - else if (state & ERTS_PSFLG_DIRTY_CPU_PROC_IN_Q) { - erts_aint32_t old; - old = erts_smp_atomic32_read_band_nob(&proc->state, - ~(ERTS_PSFLG_DIRTY_CPU_PROC - | ERTS_PSFLG_DIRTY_CPU_PROC_IN_Q)); - /* assert that no other dirty flags are set */ - ASSERT(!(old & (ERTS_PSFLG_DIRTY_IO_PROC|ERTS_PSFLG_DIRTY_IO_PROC_IN_Q))); - } else if (state & ERTS_PSFLG_DIRTY_IO_PROC_IN_Q) { - erts_aint32_t old; - old = erts_smp_atomic32_read_band_nob(&proc->state, - ~(ERTS_PSFLG_DIRTY_IO_PROC - | ERTS_PSFLG_DIRTY_IO_PROC_IN_Q)); - /* assert that no other dirty flags are set */ - ASSERT(!(old & (ERTS_PSFLG_DIRTY_CPU_PROC|ERTS_PSFLG_DIRTY_CPU_PROC_IN_Q))); - } - if (requeue) { -#else else { -#endif int prio = (int) ERTS_PSFLGS_GET_PRQ_PRIO(state); erts_smp_runq_unlock(rq); @@ -3788,6 +4194,8 @@ evacuate_run_queue(ErtsRunQueue *rq, erts_smp_runq_lock(rq); } + + handle_next_proc: proc = dequeue_process(rq, prio_q, &state); } if (notify) @@ -3890,7 +4298,7 @@ no_procs: return 0; else { if (prt_rq != rq) - erl_exit(ERTS_ABORT_EXIT, + erts_exit(ERTS_ABORT_EXIT, "%s:%d:%s() internal error\n", __FILE__, __LINE__, __func__); *rq_lockedp = 1; @@ -4722,7 +5130,7 @@ erts_fprintf(stderr, "--------------------------------\n"); rq->out_of_work_count = 0; (void) ERTS_RUNQ_FLGS_READ_BSET(rq, ERTS_RUNQ_FLGS_MIGRATION_INFO, flags); - rq->max_len = rq->len; + rq->max_len = erts_smp_atomic32_read_dirty(&rq->len); for (pix = 0; pix < ERTS_NO_PRIO_LEVELS; pix++) { ErtsRunQueueInfo *rqi; rqi = (pix == ERTS_PORT_PRIO_LEVEL @@ -4880,7 +5288,7 @@ wakeup_other_check(ErtsRunQueue *rq, Uint32 flags) { int wo_reds = rq->wakeup_other_reds; if (wo_reds) { - int left_len = rq->len - 1; + int left_len = erts_smp_atomic32_read_dirty(&rq->len) - 1; if (left_len < 1) { int wo_reduce = wo_reds << wakeup_other.dec_shift; wo_reduce &= wakeup_other.dec_mask; @@ -4893,9 +5301,11 @@ wakeup_other_check(ErtsRunQueue *rq, Uint32 flags) + ERTS_WAKEUP_OTHER_FIXED_INC); if (rq->wakeup_other > wakeup_other.limit) { #ifdef ERTS_DIRTY_SCHEDULERS - if (ERTS_RUNQ_IX_IS_DIRTY(rq->ix) && rq->waiting) - wake_dirty_schedulers(rq, 1); - else + if (ERTS_RUNQ_IX_IS_DIRTY(rq->ix)) { + if (rq->waiting) { + wake_dirty_schedulers(rq, 1); + } + } else #endif { int empty_rqs = @@ -4951,7 +5361,7 @@ wakeup_other_check_legacy(ErtsRunQueue *rq, Uint32 flags) { int wo_reds = rq->wakeup_other_reds; if (wo_reds) { - erts_aint32_t len = rq->len; + erts_aint32_t len = erts_smp_atomic32_read_dirty(&rq->len); if (len < 2) { rq->wakeup_other -= ERTS_WAKEUP_OTHER_DEC_LEGACY*wo_reds; if (rq->wakeup_other < 0) @@ -5047,7 +5457,7 @@ runq_supervisor(void *unused) ErtsRunQueue *rq = ERTS_RUNQ_IX(ix); if (ERTS_RUNQ_FLGS_GET(rq) & ERTS_RUNQ_FLG_NONEMPTY) { erts_smp_runq_lock(rq); - if (rq->len != 0) + if (erts_smp_atomic32_read_dirty(&rq->len) != 0) wake_scheduler_on_empty_runq(rq); /* forced wakeup... */ erts_smp_runq_unlock(rq); } @@ -5066,17 +5476,11 @@ erts_early_init_scheduling(int no_schedulers) wakeup_other.threshold = ERTS_SCHED_WAKEUP_OTHER_THRESHOLD_MEDIUM; wakeup_other.type = ERTS_SCHED_WAKEUP_OTHER_TYPE_DEFAULT; #endif -#ifndef ERTS_SCHED_MIN_SPIN sched_busy_wait.sys_schedule = ERTS_SCHED_SYS_SLEEP_SPINCOUNT_MEDIUM; sched_busy_wait.tse = (ERTS_SCHED_SYS_SLEEP_SPINCOUNT_MEDIUM * ERTS_SCHED_TSE_SLEEP_SPINCOUNT_FACT); sched_busy_wait.aux_work = (ERTS_SCHED_SYS_SLEEP_SPINCOUNT_MEDIUM * ERTS_SCHED_AUX_WORK_SLEEP_SPINCOUNT_FACT_MEDIUM); -#else - sched_busy_wait.sys_schedule = ERTS_SCHED_SYS_SLEEP_SPINCOUNT_NONE; - sched_busy_wait.tse = ERTS_SCHED_SYS_SLEEP_SPINCOUNT_NONE; - sched_busy_wait.aux_work = ERTS_SCHED_SYS_SLEEP_SPINCOUNT_NONE; -#endif } int @@ -5206,8 +5610,7 @@ init_aux_work_data(ErtsAuxWorkData *awdp, ErtsSchedulerData *esdp, char *dawwp) awdp->latest_wakeup = ERTS_THR_PRGR_VAL_FIRST; awdp->misc.thr_prgr = ERTS_THR_PRGR_VAL_WAITING; awdp->dd.thr_prgr = ERTS_THR_PRGR_VAL_WAITING; - awdp->dd.completed_callback = NULL; - awdp->dd.completed_arg = NULL; + awdp->cncld_tmrs.thr_prgr = ERTS_THR_PRGR_VAL_WAITING; awdp->later_op.thr_prgr = ERTS_THR_PRGR_VAL_FIRST; awdp->later_op.size = 0; awdp->later_op.first = NULL; @@ -5237,6 +5640,9 @@ init_aux_work_data(ErtsAuxWorkData *awdp, ErtsSchedulerData *esdp, char *dawwp) awdp->delayed_wakeup.sched2jix[i] = -1; } #endif + awdp->debug.wait_completed.flags = 0; + awdp->debug.wait_completed.callback = NULL; + awdp->debug.wait_completed.arg = NULL; } static void @@ -5245,6 +5651,7 @@ init_scheduler_data(ErtsSchedulerData* esdp, int num, ErtsRunQueue* runq, char** daww_ptr, size_t daww_sz) { + esdp->timer_wheel = NULL; #ifdef ERTS_SMP erts_bits_init_state(&esdp->erl_bits_state); esdp->match_pseudo_process = NULL; @@ -5257,9 +5664,6 @@ init_scheduler_data(ErtsSchedulerData* esdp, int num, esdp->f_reg_array = erts_alloc_permanent_cache_aligned(ERTS_ALC_T_BEAM_REGISTER, MAX_REG * sizeof(FloatDef)); -#if !HEAP_ON_C_STACK - esdp->num_tmp_heap_used = 0; -#endif #ifdef ERTS_DIRTY_SCHEDULERS if (ERTS_RUNQ_IX_IS_DIRTY(runq->ix)) { esdp->no = 0; @@ -5272,6 +5676,7 @@ init_scheduler_data(ErtsSchedulerData* esdp, int num, #else esdp->no = (Uint) num; #endif + esdp->ssi = ssi; esdp->current_process = NULL; esdp->current_port = NULL; @@ -5284,6 +5689,15 @@ init_scheduler_data(ErtsSchedulerData* esdp, int num, esdp->run_queue = runq; esdp->run_queue->scheduler = esdp; + esdp->last_monotonic_time = 0; + esdp->check_time_reds = 0; + + esdp->thr_id = (Uint32) num; + erts_sched_bif_unique_init(esdp); + + esdp->io.out = (Uint64) 0; + esdp->io.in = (Uint64) 0; + if (daww_ptr) { init_aux_work_data(&esdp->aux_work_data, esdp, *daww_ptr); #ifdef ERTS_SMP @@ -5309,6 +5723,9 @@ erts_init_scheduling(int no_schedulers, int no_schedulers_online char *daww_ptr; size_t daww_sz; size_t size_runqs; +#ifdef ERTS_SMP + erts_aint32_t set_schdlr_sspnd_change_flags; +#endif init_misc_op_list_alloc(); init_proc_sys_task_queues_alloc(); @@ -5384,7 +5801,7 @@ erts_init_scheduling(int no_schedulers, int no_schedulers_online } rq->out_of_work_count = 0; rq->max_len = 0; - rq->len = 0; + erts_smp_atomic32_set_nob(&rq->len, 0); rq->wakeup_other = 0; rq->wakeup_other_reds = 0; rq->halt_in_progress = 0; @@ -5534,38 +5951,18 @@ erts_init_scheduling(int no_schedulers, int no_schedulers_online #endif init_misc_aux_work(); -#if !HALFWORD_HEAP init_swtreq_alloc(); -#endif + init_screq_alloc(); + erts_atomic32_init_nob(&debug_wait_completed_count, 0); /* debug only */ + debug_wait_completed_flags = 0; #ifdef ERTS_SMP - erts_atomic32_init_nob(&completed_dealloc_count, 0); /* debug only */ - aux_thread_aux_work_data = erts_alloc_permanent_cache_aligned(ERTS_ALC_T_SCHDLR_DATA, sizeof(ErtsAuxWorkData)); - erts_smp_mtx_init(&schdlr_sspnd.mtx, "schdlr_sspnd"); - erts_smp_cnd_init(&schdlr_sspnd.cnd); - - erts_smp_atomic32_init_nob(&schdlr_sspnd.changing, 0); - schdlr_sspnd.online = no_schedulers_online; - schdlr_sspnd.curr_online = no_schedulers; - schdlr_sspnd.msb.ongoing = 0; - erts_smp_atomic32_init_nob(&schdlr_sspnd.active, no_schedulers); -#ifdef ERTS_DIRTY_SCHEDULERS - erts_smp_atomic32_init_nob(&schdlr_sspnd.dirty_cpu_changing, 0); - schdlr_sspnd.dirty_cpu_online = no_dirty_cpu_schedulers_online; - schdlr_sspnd.dirty_cpu_curr_online = no_dirty_cpu_schedulers; - erts_smp_atomic32_init_nob(&schdlr_sspnd.dirty_cpu_active, no_dirty_cpu_schedulers); - erts_smp_atomic32_init_nob(&schdlr_sspnd.dirty_io_changing, 0); - schdlr_sspnd.dirty_io_online = no_dirty_io_schedulers; - schdlr_sspnd.dirty_io_curr_online = no_dirty_io_schedulers; - erts_smp_atomic32_init_nob(&schdlr_sspnd.dirty_io_active, no_dirty_io_schedulers); -#endif - schdlr_sspnd.msb.procs = NULL; init_no_runqs(no_schedulers_online, no_schedulers_online); balance_info.last_active_runqs = no_schedulers; erts_smp_mtx_init(&balance_info.update_mtx, "migration_info_update"); @@ -5580,32 +5977,66 @@ erts_init_scheduling(int no_schedulers, int no_schedulers_online init_migration_paths(); - if (no_schedulers_online < no_schedulers) { + init_scheduler_suspend(); + + set_schdlr_sspnd_change_flags = 0; + + schdlr_sspnd_set_nscheds(&schdlr_sspnd.online, + ERTS_SCHED_NORMAL, + no_schedulers_online); + schdlr_sspnd_set_nscheds(&schdlr_sspnd.curr_online, + ERTS_SCHED_NORMAL, + no_schedulers); + schdlr_sspnd_set_nscheds(&schdlr_sspnd.active, + ERTS_SCHED_NORMAL, + no_schedulers); + + if (no_schedulers_online != no_schedulers) { + ASSERT(no_schedulers_online < no_schedulers); + set_schdlr_sspnd_change_flags |= ERTS_SCHDLR_SSPND_CHNG_ONLN; + schdlr_sspnd.changer = am_init; change_no_used_runqs(no_schedulers_online); for (ix = no_schedulers_online; ix < erts_no_run_queues; ix++) suspend_run_queue(ERTS_RUNQ_IX(ix)); } - schdlr_sspnd.wait_curr_online = no_schedulers_online; - schdlr_sspnd.curr_online *= 2; /* Boot strapping... */ - ERTS_SCHDLR_SSPND_CHNG_SET((ERTS_SCHDLR_SSPND_CHNG_ONLN - | ERTS_SCHDLR_SSPND_CHNG_WAITER), 0); #ifdef ERTS_DIRTY_SCHEDULERS - schdlr_sspnd.dirty_cpu_wait_curr_online = no_dirty_cpu_schedulers_online; - schdlr_sspnd.dirty_cpu_curr_online *= 2; - ERTS_SCHDLR_SSPND_DIRTY_CPU_CHNG_SET((ERTS_SCHDLR_SSPND_CHNG_ONLN - | ERTS_SCHDLR_SSPND_CHNG_WAITER), 0); - for (ix = no_dirty_cpu_schedulers_online; ix < no_dirty_cpu_schedulers; ix++) { - ErtsSchedulerData* esdp = ERTS_DIRTY_CPU_SCHEDULER_IX(ix); - erts_smp_atomic32_read_bor_nob(&esdp->ssi->flags, ERTS_SSI_FLG_SUSPENDED); + + schdlr_sspnd_set_nscheds(&schdlr_sspnd.online, + ERTS_SCHED_DIRTY_CPU, + no_dirty_cpu_schedulers_online); + schdlr_sspnd_set_nscheds(&schdlr_sspnd.curr_online, + ERTS_SCHED_DIRTY_CPU, + no_dirty_cpu_schedulers); + schdlr_sspnd_set_nscheds(&schdlr_sspnd.active, + ERTS_SCHED_DIRTY_CPU, + no_dirty_cpu_schedulers); + + if (no_dirty_cpu_schedulers_online != no_dirty_cpu_schedulers) { + ASSERT(no_dirty_cpu_schedulers_online < no_dirty_cpu_schedulers); + set_schdlr_sspnd_change_flags |= ERTS_SCHDLR_SSPND_CHNG_DCPU_ONLN; + for (ix = no_dirty_cpu_schedulers_online; ix < no_dirty_cpu_schedulers; ix++) { + ErtsSchedulerData* esdp = ERTS_DIRTY_CPU_SCHEDULER_IX(ix); + erts_smp_atomic32_read_bor_nob(&esdp->ssi->flags, ERTS_SSI_FLG_SUSPENDED); + } } - schdlr_sspnd.dirty_io_wait_curr_online = no_dirty_io_schedulers; - schdlr_sspnd.dirty_io_curr_online *= 2; - ERTS_SCHDLR_SSPND_DIRTY_IO_CHNG_SET((ERTS_SCHDLR_SSPND_CHNG_ONLN - | ERTS_SCHDLR_SSPND_CHNG_WAITER), 0); + schdlr_sspnd_set_nscheds(&schdlr_sspnd.online, + ERTS_SCHED_DIRTY_IO, + no_dirty_io_schedulers); + schdlr_sspnd_set_nscheds(&schdlr_sspnd.curr_online, + ERTS_SCHED_DIRTY_IO, + no_dirty_io_schedulers); + schdlr_sspnd_set_nscheds(&schdlr_sspnd.active, + ERTS_SCHED_DIRTY_IO, + no_dirty_io_schedulers); + #endif + if (set_schdlr_sspnd_change_flags) + erts_smp_atomic32_set_nob(&schdlr_sspnd.changing, + set_schdlr_sspnd_change_flags); + erts_smp_atomic32_init_nob(&doing_sys_schedule, 0); init_misc_aux_work(); @@ -5620,11 +6051,9 @@ erts_init_scheduling(int no_schedulers, int no_schedulers_online #endif } erts_no_schedulers = 1; -#ifdef ERTS_DIRTY_SCHEDULERS erts_no_dirty_cpu_schedulers = 0; erts_no_dirty_io_schedulers = 0; #endif -#endif erts_smp_atomic32_init_nob(&function_calls, 0); @@ -5716,19 +6145,96 @@ make_proxy_proc(Process *prev_proxy, Process *proc, erts_aint32_t prio) return proxy; } -static ERTS_INLINE void -free_proxy_proc(Process *proxy) -{ - ASSERT(erts_smp_atomic32_read_nob(&proxy->state) & ERTS_PSFLG_PROXY); - erts_free(ERTS_ALC_T_PROC, proxy); -} - #define ERTS_ENQUEUE_NOT 0 #define ERTS_ENQUEUE_NORMAL_QUEUE 1 -#ifdef ERTS_DIRTY_SCHEDULERS #define ERTS_ENQUEUE_DIRTY_CPU_QUEUE 2 #define ERTS_ENQUEUE_DIRTY_IO_QUEUE 3 + +#ifdef ERTS_DIRTY_SCHEDULERS + +static int +check_dirty_enqueue_in_prio_queue(Process *c_p, + erts_aint32_t *newp, + erts_aint32_t actual, + erts_aint32_t aprio, + erts_aint32_t qbit) +{ + int queue; + erts_aint32_t dact, max_qbit; + + /* Termination should be done on an ordinary scheduler */ + if (actual & ERTS_PSFLG_EXITING) { + *newp &= ~ERTS_PSFLGS_DIRTY_WORK; + return ERTS_ENQUEUE_NORMAL_QUEUE; + } + + /* + * If we have system tasks, we enqueue on ordinary run-queue + * and take care of those system tasks first. + */ + if (actual & ERTS_PSFLG_ACTIVE_SYS) + return ERTS_ENQUEUE_NORMAL_QUEUE; + + dact = erts_smp_atomic32_read_mb(&c_p->dirty_state); + if (actual & (ERTS_PSFLG_DIRTY_ACTIVE_SYS + | ERTS_PSFLG_DIRTY_CPU_PROC)) { + max_qbit = ((dact >> ERTS_PDSFLGS_IN_CPU_PRQ_MASK_OFFSET) + & ERTS_PDSFLGS_QMASK); + queue = ERTS_ENQUEUE_DIRTY_CPU_QUEUE; + } + else { + ASSERT(actual & ERTS_PSFLG_DIRTY_IO_PROC); + max_qbit = ((dact >> ERTS_PDSFLGS_IN_IO_PRQ_MASK_OFFSET) + & ERTS_PDSFLGS_QMASK); + queue = ERTS_ENQUEUE_DIRTY_IO_QUEUE; + } + + max_qbit |= 1 << ERTS_PSFLGS_QMASK_BITS; + max_qbit &= -max_qbit; + + if (qbit >= max_qbit) + return ERTS_ENQUEUE_NOT; /* Already queued in higher or equal prio */ + if ((actual & (ERTS_PSFLG_IN_RUNQ|ERTS_PSFLGS_USR_PRIO_MASK)) + != (aprio << ERTS_PSFLGS_USR_PRIO_OFFSET)) { + /* + * Process struct already enqueued, or actual prio not + * equal to user prio, i.e., enqueue using proxy. + */ + return -1*queue; + } + + *newp |= ERTS_PSFLG_IN_RUNQ; + return queue; +} + +static ERTS_INLINE int +fin_dirty_enq_s_change(Process *p, + int pstruct_reserved, + erts_aint32_t enq_prio, + int qmask_offset) +{ + erts_aint32_t qbit = 1 << enq_prio; + qbit <<= qmask_offset; + + if (qbit & erts_smp_atomic32_read_bor_mb(&p->dirty_state, qbit)) { + /* Already enqueue by someone else... */ + if (pstruct_reserved) { + /* We reserved process struct for enqueue; clear it... */ +#ifdef DEBUG + erts_aint32_t old = +#else + (void) #endif + erts_smp_atomic32_read_band_nob(&p->state, ~ERTS_PSFLG_IN_RUNQ); + ASSERT(old & ERTS_PSFLG_IN_RUNQ); + } + return 0; + } + + return !0; +} + +#endif /* ERTS_DIRTY_SCHEDULERS */ static ERTS_INLINE int check_enqueue_in_prio_queue(Process *c_p, @@ -5744,61 +6250,14 @@ check_enqueue_in_prio_queue(Process *c_p, *prq_prio_p = aprio; #ifdef ERTS_DIRTY_SCHEDULERS - if (actual & (ERTS_PSFLG_DIRTY_CPU_PROC|ERTS_PSFLG_DIRTY_IO_PROC)) { - /* - * If we have system tasks of a priority higher - * or equal to the user priority, we enqueue - * on ordinary run-queue and take care of - * those system tasks first. - */ - if (actual & ERTS_PSFLG_ACTIVE_SYS) { - erts_aint32_t uprio, stprio, qmask; - uprio = (actual >> ERTS_PSFLGS_USR_PRIO_OFFSET) & ERTS_PSFLGS_PRIO_MASK; - if (aprio < uprio) - goto enqueue_normal_runq; /* system tasks with higher prio */ - erts_smp_proc_lock(c_p, ERTS_PROC_LOCK_STATUS); - qmask = c_p->sys_task_qs->qmask; - erts_smp_proc_unlock(c_p, ERTS_PROC_LOCK_STATUS); - switch (qmask & -qmask) { - case MAX_BIT: - stprio = PRIORITY_MAX; - break; - case HIGH_BIT: - stprio = PRIORITY_HIGH; - break; - case NORMAL_BIT: - stprio = PRIORITY_NORMAL; - break; - case LOW_BIT: - stprio = PRIORITY_LOW; - break; - default: - stprio = PRIORITY_LOW+1; - break; - } - if (stprio <= uprio) - goto enqueue_normal_runq; /* system tasks with higher prio */ - } - - /* Enqueue in dirty run queue if not already enqueued */ - if (actual & (ERTS_PSFLG_DIRTY_CPU_PROC_IN_Q|ERTS_PSFLG_DIRTY_IO_PROC_IN_Q)) - return ERTS_ENQUEUE_NOT; /* already in queue */ - if (actual & ERTS_PSFLG_DIRTY_CPU_PROC) { - *newp |= ERTS_PSFLG_DIRTY_CPU_PROC_IN_Q; - if (actual & ERTS_PSFLG_IN_RUNQ) - return -ERTS_ENQUEUE_DIRTY_CPU_QUEUE; /* use proxy */ - *newp |= ERTS_PSFLG_IN_RUNQ; - return ERTS_ENQUEUE_DIRTY_CPU_QUEUE; - } - *newp |= ERTS_PSFLG_DIRTY_IO_PROC_IN_Q; - if (actual & ERTS_PSFLG_IN_RUNQ) - return -ERTS_ENQUEUE_DIRTY_IO_QUEUE; /* use proxy */ - *newp |= ERTS_PSFLG_IN_RUNQ; - return ERTS_ENQUEUE_DIRTY_IO_QUEUE; + if (actual & ERTS_PSFLGS_DIRTY_WORK) { + int res = check_dirty_enqueue_in_prio_queue(c_p, newp, actual, + aprio, qbit); + if (res != ERTS_ENQUEUE_NORMAL_QUEUE) + return res; } - - enqueue_normal_runq: #endif + max_qbit = (actual >> ERTS_PSFLGS_IN_PRQ_MASK_OFFSET) & ERTS_PSFLGS_QMASK; max_qbit |= 1 << ERTS_PSFLGS_QMASK_BITS; max_qbit &= -max_qbit; @@ -5830,6 +6289,65 @@ check_enqueue_in_prio_queue(Process *c_p, return ERTS_ENQUEUE_NORMAL_QUEUE; } +static ERTS_INLINE ErtsRunQueue * +select_enqueue_run_queue(int enqueue, int enq_prio, Process *p, erts_aint32_t state) +{ + + switch (enqueue) { + + case ERTS_ENQUEUE_NOT: + + return NULL; + +#ifdef ERTS_DIRTY_SCHEDULERS + + case ERTS_ENQUEUE_DIRTY_CPU_QUEUE: + case -ERTS_ENQUEUE_DIRTY_CPU_QUEUE: + + if (fin_dirty_enq_s_change(p, enqueue > 0, enq_prio, + ERTS_PDSFLGS_IN_CPU_PRQ_MASK_OFFSET)) + return ERTS_DIRTY_CPU_RUNQ; + + return NULL; + + + case ERTS_ENQUEUE_DIRTY_IO_QUEUE: + case -ERTS_ENQUEUE_DIRTY_IO_QUEUE: + + if (fin_dirty_enq_s_change(p, enqueue > 0, enq_prio, + ERTS_PDSFLGS_IN_IO_PRQ_MASK_OFFSET)) + return ERTS_DIRTY_IO_RUNQ; + + return NULL; + +#endif + + default: { + ErtsRunQueue* runq; + + ASSERT(enqueue == ERTS_ENQUEUE_NORMAL_QUEUE + || enqueue == -ERTS_ENQUEUE_NORMAL_QUEUE); + + runq = erts_get_runq_proc(p); + +#ifdef ERTS_SMP + if (!(ERTS_PSFLG_BOUND & state)) { + ErtsRunQueue *new_runq = erts_check_emigration_need(runq, enq_prio); + if (new_runq) { + RUNQ_SET_RQ(&p->run_queue, new_runq); + runq = new_runq; + } + } +#endif + + ASSERT(runq); + + return runq; + } + } +} + + /* * schedule_out_process() return with c_rq locked. */ @@ -5838,11 +6356,7 @@ schedule_out_process(ErtsRunQueue *c_rq, erts_aint32_t state, Process *p, Proces { erts_aint32_t a, e, n, enq_prio = -1; int enqueue; /* < 0 -> use proxy */ - Process* sched_p; ErtsRunQueue* runq; -#ifdef ERTS_SMP - int check_emigration_need; -#endif a = state; @@ -5854,7 +6368,7 @@ schedule_out_process(ErtsRunQueue *c_rq, erts_aint32_t state, Process *p, Proces enqueue = ERTS_ENQUEUE_NOT; n &= ~(ERTS_PSFLG_RUNNING|ERTS_PSFLG_RUNNING_SYS); - if (a & ERTS_PSFLG_ACTIVE_SYS + if (a & (ERTS_PSFLG_ACTIVE_SYS|ERTS_PSFLG_DIRTY_ACTIVE_SYS) || (a & (ERTS_PSFLG_ACTIVE|ERTS_PSFLG_SUSPENDED)) == ERTS_PSFLG_ACTIVE) { enqueue = check_enqueue_in_prio_queue(p, &enq_prio, &n, a); } @@ -5863,13 +6377,17 @@ schedule_out_process(ErtsRunQueue *c_rq, erts_aint32_t state, Process *p, Proces break; } - switch (enqueue) { - case ERTS_ENQUEUE_NOT: + runq = select_enqueue_run_queue(enqueue, enq_prio, p, n); + + if (!runq) { + if (erts_system_profile_flags.runnable_procs) { - if (!(a & ERTS_PSFLG_ACTIVE_SYS) - && (!(a & ERTS_PSFLG_ACTIVE) - || (a & ERTS_PSFLG_SUSPENDED))) { + /* Status lock prevents out of order "runnable proc" trace msgs */ + ERTS_SMP_LC_ASSERT(ERTS_PROC_LOCK_STATUS & erts_proc_lc_my_proc_locks(p)); + + if (!(a & (ERTS_PSFLG_ACTIVE_SYS|ERTS_PSFLG_DIRTY_ACTIVE_SYS)) + && (!(a & ERTS_PSFLG_ACTIVE) || (a & ERTS_PSFLG_SUSPENDED))) { /* Process inactive */ profile_runnable_proc(p, am_inactive); } @@ -5879,100 +6397,79 @@ schedule_out_process(ErtsRunQueue *c_rq, erts_aint32_t state, Process *p, Proces free_proxy_proc(proxy); erts_smp_runq_lock(c_rq); + return 0; -#ifdef ERTS_DIRTY_SCHEDULERS -#ifdef ERTS_SMP - case ERTS_ENQUEUE_DIRTY_CPU_QUEUE: - case -ERTS_ENQUEUE_DIRTY_CPU_QUEUE: - runq = ERTS_DIRTY_CPU_RUNQ; - ASSERT(ERTS_SCHEDULER_IS_DIRTY_CPU(runq->scheduler)); -#ifdef ERTS_SMP - check_emigration_need = 0; -#endif - break; + } + else { + Process* sched_p; - case ERTS_ENQUEUE_DIRTY_IO_QUEUE: - case -ERTS_ENQUEUE_DIRTY_IO_QUEUE: - runq = ERTS_DIRTY_IO_RUNQ; - ASSERT(ERTS_SCHEDULER_IS_DIRTY_IO(runq->scheduler)); -#ifdef ERTS_SMP - check_emigration_need = 0; -#endif - break; -#endif -#endif + ASSERT(!(n & ERTS_PSFLG_SUSPENDED) || (n & (ERTS_PSFLG_ACTIVE_SYS + | ERTS_PSFLG_DIRTY_ACTIVE_SYS))); - default: - ASSERT(enqueue == ERTS_ENQUEUE_NORMAL_QUEUE - || enqueue == -ERTS_ENQUEUE_NORMAL_QUEUE); + if (enqueue < 0) + sched_p = make_proxy_proc(proxy, p, enq_prio); + else { + sched_p = p; + if (proxy) + free_proxy_proc(proxy); + } - runq = erts_get_runq_proc(p); -#ifdef ERTS_SMP - check_emigration_need = !(ERTS_PSFLG_BOUND & n); -#endif - break; - } + ASSERT(runq); - ASSERT(!(n & ERTS_PSFLG_SUSPENDED) || (n & ERTS_PSFLG_ACTIVE_SYS)); + erts_smp_runq_lock(runq); - if (enqueue < 0) - sched_p = make_proxy_proc(proxy, p, enq_prio); - else { - sched_p = p; - if (proxy) - free_proxy_proc(proxy); - } + /* Enqueue the process */ + enqueue_process(runq, (int) enq_prio, sched_p); -#ifdef ERTS_SMP - if (check_emigration_need) { - ErtsRunQueue *new_runq = erts_check_emigration_need(runq, enq_prio); - if (new_runq) { - RUNQ_SET_RQ(&sched_p->run_queue, new_runq); - runq = new_runq; - } - } -#endif + if (runq == c_rq) + return 1; - ASSERT(runq); + erts_smp_runq_unlock(runq); - erts_smp_runq_lock(runq); + smp_notify_inc_runq(runq); - /* Enqueue the process */ - enqueue_process(runq, (int) enq_prio, sched_p); + erts_smp_runq_lock(c_rq); - if (runq == c_rq) return 1; - erts_smp_runq_unlock(runq); - smp_notify_inc_runq(runq); - erts_smp_runq_lock(c_rq); - return 1; + } + } static ERTS_INLINE void -add2runq(Process *p, erts_aint32_t state, erts_aint32_t prio) +add2runq(int enqueue, erts_aint32_t prio, + Process *proc, erts_aint32_t state, + Process **proxy) { - ErtsRunQueue *runq = erts_get_runq_proc(p); + ErtsRunQueue *runq; -#ifdef ERTS_SMP - if (!(ERTS_PSFLG_BOUND & state)) { - ErtsRunQueue *new_runq = erts_check_emigration_need(runq, (int) prio); - if (new_runq) { - RUNQ_SET_RQ(&p->run_queue, new_runq); - runq = new_runq; - } - } -#endif - ASSERT(runq); + runq = select_enqueue_run_queue(enqueue, prio, proc, state); - erts_smp_runq_lock(runq); + if (runq) { + Process *sched_p; - /* Enqueue the process */ - enqueue_process(runq, (int) prio, p); + if (enqueue > 0) + sched_p = proc; + else { + Process *pxy; - erts_smp_runq_unlock(runq); - smp_notify_inc_runq(runq); + if (!proxy) + pxy = NULL; + else { + pxy = *proxy; + *proxy = NULL; + } + sched_p = make_proxy_proc(pxy, proc, prio); + } + + erts_smp_runq_lock(runq); + /* Enqueue the process */ + enqueue_process(runq, (int) prio, sched_p); + + erts_smp_runq_unlock(runq); + smp_notify_inc_runq(runq); + } } static ERTS_INLINE int @@ -5980,7 +6477,8 @@ change_proc_schedule_state(Process *p, erts_aint32_t clear_state_flags, erts_aint32_t set_state_flags, erts_aint32_t *statep, - erts_aint32_t *enq_prio_p) + erts_aint32_t *enq_prio_p, + ErtsProcLocks locks) { /* * NOTE: ERTS_PSFLG_RUNNING, ERTS_PSFLG_RUNNING_SYS and @@ -5989,6 +6487,11 @@ change_proc_schedule_state(Process *p, */ erts_aint32_t a = *statep, n; int enqueue; /* < 0 -> use proxy */ + unsigned int prof_runnable_procs = erts_system_profile_flags.runnable_procs; + unsigned int lock_status = (prof_runnable_procs + && !(locks & ERTS_PROC_LOCK_STATUS)); + + ERTS_SMP_LC_ASSERT(locks == erts_proc_lc_my_proc_locks(p)); ASSERT(!(a & ERTS_PSFLG_PROXY)); ASSERT((clear_state_flags & (ERTS_PSFLG_RUNNING @@ -5998,6 +6501,9 @@ change_proc_schedule_state(Process *p, | ERTS_PSFLG_RUNNING_SYS | ERTS_PSFLG_ACTIVE_SYS)) == 0); + if (lock_status) + erts_smp_proc_lock(p, ERTS_PROC_LOCK_STATUS); + while (1) { erts_aint32_t e; n = e = a; @@ -6033,7 +6539,9 @@ change_proc_schedule_state(Process *p, break; } - if (erts_system_profile_flags.runnable_procs) { + if (prof_runnable_procs) { + + /* Status lock prevents out of order "runnable proc" trace msgs */ if (((n & (ERTS_PSFLG_SUSPENDED | ERTS_PSFLG_ACTIVE)) == ERTS_PSFLG_ACTIVE) @@ -6046,15 +6554,18 @@ change_proc_schedule_state(Process *p, profile_runnable_proc(p, am_active); } + if (lock_status) + erts_smp_proc_unlock(p, ERTS_PROC_LOCK_STATUS); } + *statep = a; return enqueue; } static ERTS_INLINE void -schedule_process(Process *p, erts_aint32_t in_state) +schedule_process(Process *p, erts_aint32_t in_state, ErtsProcLocks locks) { erts_aint32_t enq_prio = -1; erts_aint32_t state = in_state; @@ -6062,24 +6573,110 @@ schedule_process(Process *p, erts_aint32_t in_state) 0, ERTS_PSFLG_ACTIVE, &state, - &enq_prio); - if (enqueue != ERTS_ENQUEUE_NOT) - add2runq(enqueue > 0 ? p : make_proxy_proc(NULL, p, enq_prio), - state, - enq_prio); + &enq_prio, + locks); + add2runq(enqueue, enq_prio, p, state, NULL); } void -erts_schedule_process(Process *p, erts_aint32_t state) +erts_schedule_process(Process *p, erts_aint32_t state, ErtsProcLocks locks) { - schedule_process(p, state); + schedule_process(p, state, locks); } -static void -schedule_process_sys_task(Process *p, erts_aint32_t state, Process *proxy) +static int +schedule_process_sys_task(Process *p, erts_aint32_t prio, ErtsProcSysTask *st) { - erts_aint32_t a = state, n, enq_prio = -1; + int res; + int locked; + ErtsProcSysTaskQs *stqs, *free_stqs; + erts_aint32_t state, a, n, enq_prio; int enqueue; /* < 0 -> use proxy */ + unsigned int prof_runnable_procs; + + res = 1; /* prepare for success */ + st->next = st->prev = st; /* Prep for empty prio queue */ + state = erts_smp_atomic32_read_nob(&p->state); + prof_runnable_procs = erts_system_profile_flags.runnable_procs; + locked = 0; + free_stqs = NULL; + if (state & ERTS_PSFLG_ACTIVE_SYS) + stqs = NULL; + else { + alloc_qs: + stqs = proc_sys_task_queues_alloc(); + stqs->qmask = 1 << prio; + stqs->ncount = 0; + stqs->q[PRIORITY_MAX] = NULL; + stqs->q[PRIORITY_HIGH] = NULL; + stqs->q[PRIORITY_NORMAL] = NULL; + stqs->q[PRIORITY_LOW] = NULL; + stqs->q[prio] = st; + } + + if (!locked) { + locked = 1; + erts_smp_proc_lock(p, ERTS_PROC_LOCK_STATUS); + + state = erts_smp_atomic32_read_nob(&p->state); + if (state & ERTS_PSFLG_EXITING) { + free_stqs = stqs; + res = 0; + goto cleanup; + } + } + + if (!p->sys_task_qs) { + if (stqs) + p->sys_task_qs = stqs; + else + goto alloc_qs; + } + else { + free_stqs = stqs; + stqs = p->sys_task_qs; + if (!stqs->q[prio]) { + stqs->q[prio] = st; + stqs->qmask |= 1 << prio; + } + else { + st->next = stqs->q[prio]; + st->prev = stqs->q[prio]->prev; + st->next->prev = st; + st->prev->next = st; + ASSERT(stqs->qmask & (1 << prio)); + } + } + + if (ERTS_PSFLGS_GET_ACT_PRIO(state) > prio) { + erts_aint32_t n, a, e; + /* Need to elevate actual prio */ + + a = state; + do { + if (ERTS_PSFLGS_GET_ACT_PRIO(a) <= prio) { + n = a; + break; + } + n = e = a; + n &= ~ERTS_PSFLGS_ACT_PRIO_MASK; + n |= (prio << ERTS_PSFLGS_ACT_PRIO_OFFSET); + a = erts_smp_atomic32_cmpxchg_nob(&p->state, n, e); + } while (a != e); + state = n; + } + + + a = state; + enq_prio = -1; + + /* Status lock prevents out of order "runnable proc" trace msgs */ + ERTS_SMP_LC_ASSERT(ERTS_PROC_LOCK_STATUS & erts_proc_lc_my_proc_locks(p)); + + if (!prof_runnable_procs) { + erts_smp_proc_unlock(p, ERTS_PROC_LOCK_STATUS); + locked = 0; + } ASSERT(!(state & ERTS_PSFLG_PROXY)); @@ -6087,8 +6684,10 @@ schedule_process_sys_task(Process *p, erts_aint32_t state, Process *proxy) erts_aint32_t e; n = e = a; - if (a & ERTS_PSFLG_FREE) - return; /* We don't want to schedule free processes... */ + if (a & ERTS_PSFLG_FREE) { + res = 0; + goto cleanup; /* We don't want to schedule free processes... */ + } enqueue = ERTS_ENQUEUE_NOT; n |= ERTS_PSFLG_ACTIVE_SYS; @@ -6101,7 +6700,7 @@ schedule_process_sys_task(Process *p, erts_aint32_t state, Process *proxy) goto cleanup; } - if (erts_system_profile_flags.runnable_procs) { + if (prof_runnable_procs) { if (!(a & (ERTS_PSFLG_ACTIVE_SYS | ERTS_PSFLG_RUNNING @@ -6111,22 +6710,23 @@ schedule_process_sys_task(Process *p, erts_aint32_t state, Process *proxy) profile_runnable_proc(p, am_active); } + erts_smp_proc_unlock(p, ERTS_PROC_LOCK_STATUS); + locked = 0; } - if (enqueue != ERTS_ENQUEUE_NOT) { - Process *sched_p; - if (enqueue > 0) - sched_p = p; - else { - sched_p = make_proxy_proc(proxy, p, enq_prio); - proxy = NULL; - } - add2runq(sched_p, n, enq_prio); - } + add2runq(enqueue, enq_prio, p, n, NULL); cleanup: - if (proxy) - free_proxy_proc(proxy); + + if (locked) + erts_smp_proc_unlock(p, ERTS_PROC_LOCK_STATUS); + + if (free_stqs) + proc_sys_task_queues_free(free_stqs); + + ERTS_SMP_LC_ASSERT(!(ERTS_PROC_LOCK_STATUS & erts_proc_lc_my_proc_locks(p))); + + return res; } static ERTS_INLINE int @@ -6144,7 +6744,7 @@ suspend_process(Process *c_p, Process *p) if (c_p == p) { state = erts_smp_atomic32_read_bor_relb(&p->state, ERTS_PSFLG_SUSPENDED); - ASSERT(state & ERTS_PSFLG_RUNNING); + ASSERT(state & (ERTS_PSFLG_RUNNING|ERTS_PSFLG_RUNNING_SYS)); suspended = (state & ERTS_PSFLG_SUSPENDED) ? -1: 1; } else { @@ -6168,9 +6768,6 @@ suspend_process(Process *c_p, Process *p) if (suspended) { - ASSERT(!(ERTS_PSFLG_RUNNING & state) - || p == erts_get_current_process()); - if (suspended > 0 && erts_system_profile_flags.runnable_procs) { /* 'state' is before our change... */ @@ -6193,7 +6790,7 @@ suspend_process(Process *c_p, Process *p) } static ERTS_INLINE void -resume_process(Process *p) +resume_process(Process *p, ErtsProcLocks locks) { erts_aint32_t state, enq_prio = -1; int enqueue; @@ -6210,24 +6807,9 @@ resume_process(Process *p) ERTS_PSFLG_SUSPENDED, 0, &state, - &enq_prio); - if (enqueue) - add2runq(enqueue > 0 ? p : make_proxy_proc(NULL, p, enq_prio), - state, - enq_prio); -} - -int -erts_get_max_no_executing_schedulers(void) -{ -#ifdef ERTS_SMP - if (erts_smp_atomic32_read_nob(&schdlr_sspnd.changing)) - return (int) erts_no_schedulers; - ERTS_THR_MEMORY_BARRIER; - return (int) erts_smp_atomic32_read_nob(&schdlr_sspnd.active); -#else - return 1; -#endif + &enq_prio, + locks); + add2runq(enqueue, enq_prio, p, state, NULL); } #ifdef ERTS_SMP @@ -6332,42 +6914,82 @@ sched_set_suspended_sleeptype(ErtsSchedulerSleepInfo *ssi) } } -#ifdef ERTS_DIRTY_SCHEDULERS +static void +init_scheduler_suspend(void) +{ + erts_smp_mtx_init(&schdlr_sspnd.mtx, "schdlr_sspnd"); + schdlr_sspnd.online = ERTS_SCHDLR_SSPND_MAKE_NSCHEDS_VAL(1, 0, 0); + schdlr_sspnd.curr_online = ERTS_SCHDLR_SSPND_MAKE_NSCHEDS_VAL(1, 0, 0); + schdlr_sspnd.active = ERTS_SCHDLR_SSPND_MAKE_NSCHEDS_VAL(1, 0, 0); + erts_smp_atomic32_init_nob(&schdlr_sspnd.changing, 0); + schdlr_sspnd.chngq = NULL; + schdlr_sspnd.changer = am_false; + schdlr_sspnd.nmsb.ongoing = 0; + schdlr_sspnd.nmsb.blckrs = NULL; + schdlr_sspnd.nmsb.chngq = NULL; + schdlr_sspnd.msb.ongoing = 0; + schdlr_sspnd.msb.blckrs = NULL; + schdlr_sspnd.msb.chngq = NULL; +} + +typedef struct { + struct { + Eterm chngr; + Eterm nxt; + } onln; + struct { + ErtsProcList *chngrs; + } msb; +} ErtsSchdlrSspndResume; + +static void +schdlr_sspnd_resume_proc(Eterm pid) +{ + Process *p = erts_pid2proc(NULL, 0, pid, ERTS_PROC_LOCK_STATUS); + if (p) { + resume_process(p, ERTS_PROC_LOCK_STATUS); + erts_smp_proc_unlock(p, ERTS_PROC_LOCK_STATUS); + } +} + +static ERTS_INLINE void +schdlr_sspnd_resume_procs(ErtsSchedType sched_type, + ErtsSchdlrSspndResume *resume) +{ + if (is_internal_pid(resume->onln.chngr)) { + schdlr_sspnd_resume_proc(resume->onln.chngr); + resume->onln.chngr = NIL; + } + if (is_internal_pid(resume->onln.nxt)) { + schdlr_sspnd_resume_proc(resume->onln.nxt); + resume->onln.nxt = NIL; + } + while (resume->msb.chngrs) { + ErtsProcList *plp = resume->msb.chngrs; + resume->msb.chngrs = plp->next; + schdlr_sspnd_resume_proc(plp->pid); + proclist_destroy(plp); + } +} static void suspend_scheduler(ErtsSchedulerData *esdp) { erts_aint32_t flgs; erts_aint32_t changing; -#ifdef ERTS_DIRTY_SCHEDULERS - long no = (long) (ERTS_SCHEDULER_IS_DIRTY(esdp) - ? ERTS_DIRTY_SCHEDULER_NO(esdp) - : esdp->no); -#else - long no = (long) esdp->no; -#endif + long no; ErtsSchedulerSleepInfo *ssi = esdp->ssi; - long active_schedulers; int curr_online = 1; - int wake = 0; + ErtsSchdlrSspndResume resume = {{NIL, NIL}, {NULL}}; erts_aint32_t aux_work; int thr_prgr_active = 1; ErtsStuckBoundProcesses sbp = {NULL, NULL}; - int* ss_onlinep; - int* ss_curr_onlinep; - int* ss_wait_curr_onlinep; - long* ss_wait_activep; - long ss_wait_active_target; - erts_smp_atomic32_t* ss_changingp; - erts_smp_atomic32_t* ss_activep; + ErtsSchedType sched_type; + erts_aint32_t online_flag; /* * Schedulers may be suspended in two different ways: * - A scheduler may be suspended since it is not online. - * All schedulers with scheduler ids greater than - * schdlr_sspnd.online are suspended; same for dirty - * schedulers and schdlr_sspnd.dirty_cpu_online and - * schdlr_sspnd.dirty_io_online. * - Multi scheduling is blocked. All schedulers except the * scheduler with scheduler id 1 are suspended, and all * dirty CPU and dirty I/O schedulers are suspended. @@ -6375,27 +6997,43 @@ suspend_scheduler(ErtsSchedulerData *esdp) * Regardless of why a scheduler is suspended, it ends up here. */ - ASSERT(ERTS_SCHEDULER_IS_DIRTY(esdp) || no != 1); - #ifdef ERTS_DIRTY_SCHEDULERS if (ERTS_SCHEDULER_IS_DIRTY(esdp)) { + no = ERTS_DIRTY_SCHEDULER_NO(esdp); + if (ERTS_RUNQ_IS_DIRTY_CPU_RUNQ(esdp->run_queue)) { + online_flag = ERTS_SCHDLR_SSPND_CHNG_DCPU_ONLN; + sched_type = ERTS_SCHED_DIRTY_CPU; + } + else { + online_flag = 0; + sched_type = ERTS_SCHED_DIRTY_IO; + } + } + else +#endif + { + online_flag = ERTS_SCHDLR_SSPND_CHNG_ONLN; + no = esdp->no; + sched_type = ERTS_SCHED_NORMAL; + } + + ASSERT(sched_type != ERTS_SCHED_NORMAL || no != 1); + + if (sched_type != ERTS_SCHED_NORMAL) { if (erts_smp_mtx_trylock(&schdlr_sspnd.mtx) == EBUSY) { erts_smp_runq_unlock(esdp->run_queue); erts_smp_mtx_lock(&schdlr_sspnd.mtx); erts_smp_runq_lock(esdp->run_queue); } - if (ongoing_multi_scheduling_block()) + if (schdlr_sspnd.msb.ongoing) evacuate_run_queue(esdp->run_queue, &sbp); - } else -#endif + erts_smp_runq_unlock(esdp->run_queue); + } + else { evacuate_run_queue(esdp->run_queue, &sbp); - erts_smp_runq_unlock(esdp->run_queue); + erts_smp_runq_unlock(esdp->run_queue); -#ifdef ERTS_DIRTY_SCHEDULERS - if (!ERTS_SCHEDULER_IS_DIRTY(esdp)) -#endif - { erts_sched_check_cpu_bind_prep_suspend(esdp); if (erts_system_profile_flags.scheduler) @@ -6409,364 +7047,245 @@ suspend_scheduler(ErtsSchedulerData *esdp) flgs = sched_prep_spin_suspended(ssi, ERTS_SSI_FLG_SUSPENDED); if (flgs & ERTS_SSI_FLG_SUSPENDED) { -#ifdef ERTS_DIRTY_SCHEDULERS - if (ERTS_SCHEDULER_IS_DIRTY(esdp)) { - if (ERTS_RUNQ_IS_DIRTY_CPU_RUNQ(esdp->run_queue)) { - active_schedulers = erts_smp_atomic32_dec_read_nob(&schdlr_sspnd.dirty_cpu_active); - ASSERT(active_schedulers >= 0); - changing = erts_smp_atomic32_read_nob(&schdlr_sspnd.dirty_cpu_changing); - ss_onlinep = &schdlr_sspnd.dirty_cpu_online; - ss_curr_onlinep = &schdlr_sspnd.dirty_cpu_curr_online; - ss_wait_curr_onlinep = &schdlr_sspnd.dirty_cpu_wait_curr_online; - ss_changingp = &schdlr_sspnd.dirty_cpu_changing; - ss_wait_activep = &schdlr_sspnd.msb.dirty_cpu_wait_active; - ss_activep = &schdlr_sspnd.dirty_cpu_active; - } else { - active_schedulers = erts_smp_atomic32_dec_read_nob(&schdlr_sspnd.dirty_io_active); - ASSERT(active_schedulers >= 0); - changing = erts_smp_atomic32_read_nob(&schdlr_sspnd.dirty_io_changing); - ss_onlinep = &schdlr_sspnd.dirty_io_online; - ss_curr_onlinep = &schdlr_sspnd.dirty_io_curr_online; - ss_wait_curr_onlinep = &schdlr_sspnd.dirty_io_wait_curr_online; - ss_changingp = &schdlr_sspnd.dirty_io_changing; - ss_wait_activep = &schdlr_sspnd.msb.dirty_io_wait_active; - ss_activep = &schdlr_sspnd.dirty_io_active; - } - ss_wait_active_target = 0; - } - else -#endif - { - active_schedulers = erts_smp_atomic32_dec_read_nob(&schdlr_sspnd.active); - ASSERT(active_schedulers >= 1); - changing = erts_smp_atomic32_read_nob(&schdlr_sspnd.changing); - ss_onlinep = &schdlr_sspnd.online; - ss_curr_onlinep = &schdlr_sspnd.curr_online; - ss_wait_curr_onlinep = &schdlr_sspnd.wait_curr_online; - ss_changingp = &schdlr_sspnd.changing; - ss_wait_activep = &schdlr_sspnd.msb.wait_active; - ss_activep = &schdlr_sspnd.active; - ss_wait_active_target = 1; - } - if (changing & ERTS_SCHDLR_SSPND_CHNG_MSB) { - if (active_schedulers == *ss_wait_activep) - wake = 1; - if (active_schedulers == ss_wait_active_target) { - changing = erts_smp_atomic32_read_band_nob(ss_changingp, - ~ERTS_SCHDLR_SSPND_CHNG_MSB); - changing &= ~ERTS_SCHDLR_SSPND_CHNG_MSB; - } - } + schdlr_sspnd_dec_nscheds(&schdlr_sspnd.active, sched_type); - while (1) { - if (changing & ERTS_SCHDLR_SSPND_CHNG_ONLN) { - int changed = 0; - if (no > *ss_onlinep && curr_online) { - (*ss_curr_onlinep)--; - curr_online = 0; - changed = 1; - } - else if (no <= *ss_onlinep && !curr_online) { - (*ss_curr_onlinep)++; - curr_online = 1; - changed = 1; - } - if (changed - && *ss_curr_onlinep == *ss_wait_curr_onlinep) - wake = 1; - if (*ss_onlinep == *ss_curr_onlinep) { - changing = erts_smp_atomic32_read_band_nob(ss_changingp, - ~ERTS_SCHDLR_SSPND_CHNG_ONLN); - changing &= ~ERTS_SCHDLR_SSPND_CHNG_ONLN; - } - } + ASSERT(schdlr_sspnd_get_nscheds(&schdlr_sspnd.active, + ERTS_SCHED_NORMAL) >= 1); - if (wake) { - erts_smp_cnd_signal(&schdlr_sspnd.cnd); - wake = 0; - } - - if (curr_online && !ongoing_multi_scheduling_block()) { - flgs = erts_smp_atomic32_read_acqb(&ssi->flags); - if (!(flgs & ERTS_SSI_FLG_SUSPENDED)) - break; - } - erts_smp_mtx_unlock(&schdlr_sspnd.mtx); + changing = erts_smp_atomic32_read_nob(&schdlr_sspnd.changing); - while (1) { - erts_aint32_t qmask; - erts_aint32_t flgs; + while (1) { - qmask = (ERTS_RUNQ_FLGS_GET(esdp->run_queue) - & ERTS_RUNQ_FLGS_QMASK); - aux_work = erts_atomic32_read_acqb(&ssi->aux_work); - if (aux_work|qmask) { - if (!ERTS_SCHEDULER_IS_DIRTY(esdp) && !thr_prgr_active) { - erts_thr_progress_active(esdp, thr_prgr_active = 1); - sched_wall_time_change(esdp, 1); + if (changing & (ERTS_SCHDLR_SSPND_CHNG_NMSB + | ERTS_SCHDLR_SSPND_CHNG_MSB)) { + int i = 0; + ErtsMultiSchedulingBlock *msb[3] = {0}; + if (changing & ERTS_SCHDLR_SSPND_CHNG_NMSB) + msb[i++] = &schdlr_sspnd.nmsb; + if (changing & ERTS_SCHDLR_SSPND_CHNG_MSB) + msb[i++] = &schdlr_sspnd.msb; + + for (i = 0; msb[i]; i++) { + erts_aint32_t clr_flg = 0; + + if (msb[i] == &schdlr_sspnd.nmsb + && schdlr_sspnd_get_nscheds(&schdlr_sspnd.active, + ERTS_SCHED_NORMAL) == 1) { + clr_flg = ERTS_SCHDLR_SSPND_CHNG_NMSB; } - if (aux_work) - aux_work = handle_aux_work(&esdp->aux_work_data, - aux_work, - 1); - - if (!ERTS_SCHEDULER_IS_DIRTY(esdp) && - (aux_work && erts_thr_progress_update(esdp))) - erts_thr_progress_leader_update(esdp); - if (qmask) { -#ifdef ERTS_DIRTY_SCHEDULERS - if (ERTS_SCHEDULER_IS_DIRTY(esdp)) { - erts_smp_mtx_lock(&schdlr_sspnd.mtx); - erts_smp_runq_lock(esdp->run_queue); - if (ongoing_multi_scheduling_block()) - evacuate_run_queue(esdp->run_queue, &sbp); - erts_smp_runq_unlock(esdp->run_queue); - erts_smp_mtx_unlock(&schdlr_sspnd.mtx); - } else -#endif - { - erts_smp_runq_lock(esdp->run_queue); - evacuate_run_queue(esdp->run_queue, &sbp); - erts_smp_runq_unlock(esdp->run_queue); - } - } - } - - if (!aux_work) { -#ifdef ERTS_DIRTY_SCHEDULERS - if (!ERTS_SCHEDULER_IS_DIRTY(esdp)) -#endif - { - if (thr_prgr_active) { - erts_thr_progress_active(esdp, thr_prgr_active = 0); - sched_wall_time_change(esdp, 0); - } - erts_thr_progress_prepare_wait(esdp); + else if (schdlr_sspnd.active + == ERTS_SCHDLR_SSPND_MAKE_NSCHEDS_VAL(1, 0, 0)) { + clr_flg = ERTS_SCHDLR_SSPND_CHNG_MSB; } - flgs = sched_spin_suspended(ssi, - ERTS_SCHED_SUSPEND_SLEEP_SPINCOUNT); - if (flgs == (ERTS_SSI_FLG_SLEEPING - | ERTS_SSI_FLG_WAITING - | ERTS_SSI_FLG_SUSPENDED)) { - flgs = sched_set_suspended_sleeptype(ssi); - if (flgs == (ERTS_SSI_FLG_SLEEPING - | ERTS_SSI_FLG_TSE_SLEEPING - | ERTS_SSI_FLG_WAITING - | ERTS_SSI_FLG_SUSPENDED)) { - int res; - do { - res = erts_tse_wait(ssi->event); - } while (res == EINTR); + if (clr_flg) { + ErtsProcList *plp, *end_plp; + changing = erts_smp_atomic32_read_band_nob(&schdlr_sspnd.changing, + ~clr_flg); + changing &= ~clr_flg; + (void) erts_proclist_fetch(&msb[i]->chngq, &end_plp); + /* resume processes that initiated the multi scheduling block... */ + plp = msb[i]->chngq; + while (plp) { + erts_proclist_store_last(&msb[i]->blckrs, + proclist_copy(plp)); + plp = plp->next; } + if (end_plp) + end_plp->next = resume.msb.chngrs; + resume.msb.chngrs = msb[i]->chngq; + msb[i]->chngq = NULL; } -#ifdef ERTS_DIRTY_SCHEDULERS - if (!ERTS_SCHEDULER_IS_DIRTY(esdp)) -#endif - erts_thr_progress_finalize_wait(esdp); } - - flgs = sched_prep_spin_suspended(ssi, (ERTS_SSI_FLG_WAITING - | ERTS_SSI_FLG_SUSPENDED)); - if (!(flgs & ERTS_SSI_FLG_SUSPENDED)) - break; - changing = erts_smp_atomic32_read_nob(ss_changingp); - if (changing & ~ERTS_SCHDLR_SSPND_CHNG_WAITER) - break; } - erts_smp_mtx_lock(&schdlr_sspnd.mtx); - changing = erts_smp_atomic32_read_nob(ss_changingp); - } - - active_schedulers = erts_smp_atomic32_inc_read_nob(ss_activep); - changing = erts_smp_atomic32_read_nob(ss_changingp); - if ((changing & ERTS_SCHDLR_SSPND_CHNG_MSB) - && *ss_onlinep == active_schedulers) { - erts_smp_atomic32_read_band_nob(ss_changingp, - ~ERTS_SCHDLR_SSPND_CHNG_MSB); - } - -#ifdef ERTS_DIRTY_SCHEDULERS - if (!ERTS_SCHEDULER_IS_DIRTY(esdp)) -#endif - ASSERT(no <= *ss_onlinep); - ASSERT(!ongoing_multi_scheduling_block()); - - } - - erts_smp_mtx_unlock(&schdlr_sspnd.mtx); - - ASSERT(curr_online); - -#ifdef ERTS_DIRTY_SCHEDULERS - if (!ERTS_SCHEDULER_IS_DIRTY(esdp)) -#endif - { - if (erts_system_profile_flags.scheduler) - profile_scheduler(make_small(esdp->no), am_active); - - if (!thr_prgr_active) { - erts_thr_progress_active(esdp, thr_prgr_active = 1); - sched_wall_time_change(esdp, 1); - } - } - - erts_smp_runq_lock(esdp->run_queue); - non_empty_runq(esdp->run_queue); - -#ifdef ERTS_DIRTY_SCHEDULERS - if (!ERTS_SCHEDULER_IS_DIRTY(esdp)) -#endif - { - schedule_bound_processes(esdp->run_queue, &sbp); - - erts_sched_check_cpu_bind_post_suspend(esdp); - } -} - -#else /* !ERTS_DIRTY_SCHEDULERS */ - -static void -suspend_scheduler(ErtsSchedulerData *esdp) -{ - erts_aint32_t flgs; - erts_aint32_t changing; - long no = (long) esdp->no; - ErtsSchedulerSleepInfo *ssi = esdp->ssi; - long active_schedulers; - int curr_online = 1; - int wake = 0; - erts_aint32_t aux_work; - int thr_prgr_active = 1; - ErtsStuckBoundProcesses sbp = {NULL, NULL}; - - /* - * Schedulers may be suspended in two different ways: - * - A scheduler may be suspended since it is not online. - * All schedulers with scheduler ids greater than - * schdlr_sspnd.online are suspended. - * - Multi scheduling is blocked. All schedulers except the - * scheduler with scheduler id 1 are suspended. - * - * Regardless of why a scheduler is suspended, it ends up here. - */ - - ASSERT(no != 1); - - evacuate_run_queue(esdp->run_queue, &sbp); - - erts_smp_runq_unlock(esdp->run_queue); - - erts_sched_check_cpu_bind_prep_suspend(esdp); - - if (erts_system_profile_flags.scheduler) - profile_scheduler(make_small(esdp->no), am_inactive); - - sched_wall_time_change(esdp, 0); - - erts_smp_mtx_lock(&schdlr_sspnd.mtx); - - flgs = sched_prep_spin_suspended(ssi, ERTS_SSI_FLG_SUSPENDED); - if (flgs & ERTS_SSI_FLG_SUSPENDED) { - - active_schedulers = erts_smp_atomic32_dec_read_nob(&schdlr_sspnd.active); - ASSERT(active_schedulers >= 1); - changing = erts_smp_atomic32_read_nob(&schdlr_sspnd.changing); - if (changing & ERTS_SCHDLR_SSPND_CHNG_MSB) { - if (active_schedulers == schdlr_sspnd.msb.wait_active) - wake = 1; - if (active_schedulers == 1) { - changing = erts_smp_atomic32_read_band_nob(&schdlr_sspnd.changing, - ~ERTS_SCHDLR_SSPND_CHNG_MSB); - changing &= ~ERTS_SCHDLR_SSPND_CHNG_MSB; - } - } - - while (1) { - if (changing & ERTS_SCHDLR_SSPND_CHNG_ONLN) { + if (changing & online_flag) { int changed = 0; - if (no > schdlr_sspnd.online && curr_online) { - schdlr_sspnd.curr_online--; + Uint32 st_online; + + st_online = schdlr_sspnd_get_nscheds(&schdlr_sspnd.online, + sched_type); + if (no > st_online && curr_online) { + schdlr_sspnd_dec_nscheds(&schdlr_sspnd.curr_online, + sched_type); curr_online = 0; changed = 1; } - else if (no <= schdlr_sspnd.online && !curr_online) { - schdlr_sspnd.curr_online++; + else if (no <= st_online && !curr_online) { + schdlr_sspnd_inc_nscheds(&schdlr_sspnd.curr_online, + sched_type); curr_online = 1; changed = 1; } if (changed - && schdlr_sspnd.curr_online == schdlr_sspnd.wait_curr_online) - wake = 1; - if (schdlr_sspnd.online == schdlr_sspnd.curr_online) { + && (schdlr_sspnd_get_nscheds(&schdlr_sspnd.online, + sched_type) + == schdlr_sspnd_get_nscheds(&schdlr_sspnd.curr_online, + sched_type))) { + ErtsProcList *plp; changing = erts_smp_atomic32_read_band_nob(&schdlr_sspnd.changing, - ~ERTS_SCHDLR_SSPND_CHNG_ONLN); - changing &= ~ERTS_SCHDLR_SSPND_CHNG_ONLN; + ~online_flag); + changing &= ~online_flag; + if (sched_type == ERTS_SCHED_NORMAL) { + ASSERT(is_internal_pid(schdlr_sspnd.changer) + || schdlr_sspnd.changer == am_init); + /* resume process that initiated this change... */ + resume.onln.chngr = schdlr_sspnd.changer; + plp = erts_proclist_peek_first(schdlr_sspnd.chngq); + if (!plp) + schdlr_sspnd.changer = am_false; + else { + schdlr_sspnd.changer = am_true; /* change right in transit */ + /* resume process that is queued for next change... */ + resume.onln.nxt = plp->pid; + ASSERT(is_internal_pid(resume.onln.nxt)); + } + } } } - if (wake) { - erts_smp_cnd_signal(&schdlr_sspnd.cnd); - wake = 0; - } - - if (curr_online && !ongoing_multi_scheduling_block()) { + if (curr_online + && (sched_type == ERTS_SCHED_NORMAL + ? !(schdlr_sspnd.msb.ongoing|schdlr_sspnd.nmsb.ongoing) + : !schdlr_sspnd.msb.ongoing)) { flgs = erts_smp_atomic32_read_acqb(&ssi->flags); if (!(flgs & ERTS_SSI_FLG_SUSPENDED)) break; } erts_smp_mtx_unlock(&schdlr_sspnd.mtx); + schdlr_sspnd_resume_procs(sched_type, &resume); + while (1) { + ErtsMonotonicTime current_time; erts_aint32_t qmask; erts_aint32_t flgs; qmask = (ERTS_RUNQ_FLGS_GET(esdp->run_queue) & ERTS_RUNQ_FLGS_QMASK); - aux_work = erts_atomic32_read_acqb(&ssi->aux_work); - if (aux_work|qmask) { - if (!thr_prgr_active) { - erts_thr_progress_active(esdp, thr_prgr_active = 1); - sched_wall_time_change(esdp, 1); - } - if (aux_work) - aux_work = handle_aux_work(&esdp->aux_work_data, - aux_work, - 1); - if (aux_work && erts_thr_progress_update(esdp)) - erts_thr_progress_leader_update(esdp); + + if (sched_type != ERTS_SCHED_NORMAL) { if (qmask) { + erts_smp_mtx_lock(&schdlr_sspnd.mtx); erts_smp_runq_lock(esdp->run_queue); - evacuate_run_queue(esdp->run_queue, &sbp); + if (schdlr_sspnd.msb.ongoing) + evacuate_run_queue(esdp->run_queue, &sbp); erts_smp_runq_unlock(esdp->run_queue); + erts_smp_mtx_unlock(&schdlr_sspnd.mtx); } + aux_work = 0; } + else { - if (!aux_work) { - if (thr_prgr_active) { - erts_thr_progress_active(esdp, thr_prgr_active = 0); - sched_wall_time_change(esdp, 0); + aux_work = erts_atomic32_read_acqb(&ssi->aux_work); + + if (aux_work|qmask) { + if (!thr_prgr_active) { + erts_thr_progress_active(esdp, thr_prgr_active = 1); + sched_wall_time_change(esdp, 1); + } + if (aux_work) + aux_work = handle_aux_work(&esdp->aux_work_data, + aux_work, + 1); + + if (aux_work && erts_thr_progress_update(esdp)) + erts_thr_progress_leader_update(esdp); + if (qmask) { + erts_smp_runq_lock(esdp->run_queue); + evacuate_run_queue(esdp->run_queue, &sbp); + erts_smp_runq_unlock(esdp->run_queue); + } } - erts_thr_progress_prepare_wait(esdp); - flgs = sched_spin_suspended(ssi, - ERTS_SCHED_SUSPEND_SLEEP_SPINCOUNT); - if (flgs == (ERTS_SSI_FLG_SLEEPING - | ERTS_SSI_FLG_WAITING - | ERTS_SSI_FLG_SUSPENDED)) { - flgs = sched_set_suspended_sleeptype(ssi); + + } + + if (aux_work) { + ASSERT(sched_type == ERTS_SCHED_NORMAL); + current_time = erts_get_monotonic_time(esdp); + if (current_time >= erts_next_timeout_time(esdp->next_tmo_ref)) { + if (!thr_prgr_active) { + erts_thr_progress_active(esdp, thr_prgr_active = 1); + sched_wall_time_change(esdp, 1); + } + erts_bump_timers(esdp->timer_wheel, current_time); + } + } + else { + ErtsMonotonicTime timeout_time; + int do_timeout; + + if (sched_type == ERTS_SCHED_NORMAL) { + timeout_time = erts_check_next_timeout_time(esdp); + current_time = erts_get_monotonic_time(esdp); + do_timeout = (current_time >= timeout_time); + } + else { + timeout_time = ERTS_MONOTONIC_TIME_MAX; + current_time = 0; + do_timeout = 0; + } + + if (do_timeout) { + ASSERT(sched_type == ERTS_SCHED_NORMAL); + if (!thr_prgr_active) { + erts_thr_progress_active(esdp, thr_prgr_active = 1); + sched_wall_time_change(esdp, 1); + } + } + else { + if (sched_type == ERTS_SCHED_NORMAL) { + if (thr_prgr_active) { + erts_thr_progress_active(esdp, thr_prgr_active = 0); + sched_wall_time_change(esdp, 0); + } + erts_thr_progress_prepare_wait(esdp); + } + flgs = sched_spin_suspended(ssi, + ERTS_SCHED_SUSPEND_SLEEP_SPINCOUNT); if (flgs == (ERTS_SSI_FLG_SLEEPING - | ERTS_SSI_FLG_TSE_SLEEPING | ERTS_SSI_FLG_WAITING | ERTS_SSI_FLG_SUSPENDED)) { - int res; - - do { - res = erts_tse_wait(ssi->event); - } while (res == EINTR); + flgs = sched_set_suspended_sleeptype(ssi); + if (flgs == (ERTS_SSI_FLG_SLEEPING + | ERTS_SSI_FLG_TSE_SLEEPING + | ERTS_SSI_FLG_WAITING + | ERTS_SSI_FLG_SUSPENDED)) { + int res; + + if (sched_type == ERTS_SCHED_NORMAL) + current_time = erts_get_monotonic_time(esdp); + else + current_time = 0; + + do { + Sint64 timeout; + if (current_time >= timeout_time) + break; + if (sched_type != ERTS_SCHED_NORMAL) + timeout = -1; + else + timeout = ERTS_MONOTONIC_TO_NSEC(timeout_time + - current_time + - 1) + 1; + res = erts_tse_twait(ssi->event, timeout); + + if (sched_type == ERTS_SCHED_NORMAL) + current_time = erts_get_monotonic_time(esdp); + else + current_time = 0; + + } while (res == EINTR); + } } + if (sched_type == ERTS_SCHED_NORMAL) + erts_thr_progress_finalize_wait(esdp); + } + + if (current_time >= timeout_time) { + ASSERT(sched_type == ERTS_SCHED_NORMAL); + erts_bump_timers(esdp->timer_wheel, current_time); } - erts_thr_progress_finalize_wait(esdp); } flgs = sched_prep_spin_suspended(ssi, (ERTS_SSI_FLG_WAITING @@ -6774,7 +7293,7 @@ suspend_scheduler(ErtsSchedulerData *esdp) if (!(flgs & ERTS_SSI_FLG_SUSPENDED)) break; changing = erts_smp_atomic32_read_nob(&schdlr_sspnd.changing); - if (changing & ~ERTS_SCHDLR_SSPND_CHNG_WAITER) + if (changing) break; } @@ -6782,612 +7301,494 @@ suspend_scheduler(ErtsSchedulerData *esdp) changing = erts_smp_atomic32_read_nob(&schdlr_sspnd.changing); } - active_schedulers = erts_smp_atomic32_inc_read_nob(&schdlr_sspnd.active); + schdlr_sspnd_inc_nscheds(&schdlr_sspnd.active, sched_type); changing = erts_smp_atomic32_read_nob(&schdlr_sspnd.changing); if ((changing & ERTS_SCHDLR_SSPND_CHNG_MSB) - && schdlr_sspnd.online == active_schedulers) { + && schdlr_sspnd.online == schdlr_sspnd.active) { erts_smp_atomic32_read_band_nob(&schdlr_sspnd.changing, ~ERTS_SCHDLR_SSPND_CHNG_MSB); } - ASSERT(no <= schdlr_sspnd.online); - ASSERT(!ongoing_multi_scheduling_block()); - + ASSERT(no <= schdlr_sspnd_get_nscheds(&schdlr_sspnd.online, sched_type)); + ASSERT((sched_type == ERTS_SCHED_NORMAL + ? !(schdlr_sspnd.msb.ongoing|schdlr_sspnd.nmsb.ongoing) + : !schdlr_sspnd.msb.ongoing)); } erts_smp_mtx_unlock(&schdlr_sspnd.mtx); + ASSERT(!resume.msb.chngrs); + schdlr_sspnd_resume_procs(sched_type, &resume); + ASSERT(curr_online); - if (erts_system_profile_flags.scheduler) - profile_scheduler(make_small(esdp->no), am_active); + if (!ERTS_SCHEDULER_IS_DIRTY(esdp)) { + if (erts_system_profile_flags.scheduler) + profile_scheduler(make_small(esdp->no), am_active); - if (!thr_prgr_active) { - erts_thr_progress_active(esdp, thr_prgr_active = 1); - sched_wall_time_change(esdp, 1); + if (!thr_prgr_active) { + erts_thr_progress_active(esdp, thr_prgr_active = 1); + sched_wall_time_change(esdp, 1); + } } + if (sched_type == ERTS_SCHED_NORMAL) + (void) erts_get_monotonic_time(esdp); erts_smp_runq_lock(esdp->run_queue); non_empty_runq(esdp->run_queue); - schedule_bound_processes(esdp->run_queue, &sbp); + if (sched_type == ERTS_SCHED_NORMAL) { + schedule_bound_processes(esdp->run_queue, &sbp); - erts_sched_check_cpu_bind_post_suspend(esdp); + erts_sched_check_cpu_bind_post_suspend(esdp); + } } -#endif - -ErtsSchedSuspendResult +void erts_schedulers_state(Uint *total, Uint *online, Uint *active, Uint *dirty_cpu, Uint *dirty_cpu_online, + Uint *dirty_cpu_active, Uint *dirty_io, - int yield_allowed) + Uint *dirty_io_active) { - int res = ERTS_SCHDLR_SSPND_EINVAL; - erts_aint32_t changing; - erts_smp_mtx_lock(&schdlr_sspnd.mtx); - changing = erts_smp_atomic32_read_nob(&schdlr_sspnd.changing); -#ifdef ERTS_DIRTY_SCHEDULERS - changing |= (erts_smp_atomic32_read_nob(&schdlr_sspnd.dirty_cpu_changing) - | erts_smp_atomic32_read_nob(&schdlr_sspnd.dirty_io_changing)); -#endif - if (yield_allowed && (changing & ~ERTS_SCHDLR_SSPND_CHNG_WAITER)) - res = ERTS_SCHDLR_SSPND_YIELD_RESTART; - else { + if (active || online || dirty_cpu_online + || dirty_cpu_active || dirty_io_active) { + erts_smp_mtx_lock(&schdlr_sspnd.mtx); if (active) - *active = schdlr_sspnd.online; + *active = schdlr_sspnd_get_nscheds(&schdlr_sspnd.active, + ERTS_SCHED_NORMAL); if (online) - *online = schdlr_sspnd.online; - if (ongoing_multi_scheduling_block() && active) - *active = 1; -#ifdef ERTS_DIRTY_SCHEDULERS + *online = schdlr_sspnd_get_nscheds(&schdlr_sspnd.curr_online, + ERTS_SCHED_NORMAL); + if (dirty_cpu_active) + *dirty_cpu_active = schdlr_sspnd_get_nscheds(&schdlr_sspnd.active, + ERTS_SCHED_DIRTY_CPU); if (dirty_cpu_online) - *dirty_cpu_online = schdlr_sspnd.dirty_cpu_online; -#endif - res = ERTS_SCHDLR_SSPND_DONE; + *dirty_cpu_online = schdlr_sspnd_get_nscheds(&schdlr_sspnd.curr_online, + ERTS_SCHED_DIRTY_CPU); + if (dirty_io_active) + *dirty_io_active = schdlr_sspnd_get_nscheds(&schdlr_sspnd.active, + ERTS_SCHED_DIRTY_IO); + erts_smp_mtx_unlock(&schdlr_sspnd.mtx); } - erts_smp_mtx_unlock(&schdlr_sspnd.mtx); + if (total) *total = erts_no_schedulers; -#ifdef ERTS_DIRTY_SCHEDULERS if (dirty_cpu) *dirty_cpu = erts_no_dirty_cpu_schedulers; if (dirty_io) *dirty_io = erts_no_dirty_io_schedulers; -#endif - return res; } -#ifdef ERTS_DIRTY_SCHEDULERS +static void +abort_sched_onln_chng_waitq(Process *p) +{ + Eterm resume = NIL; + + erts_smp_mtx_lock(&schdlr_sspnd.mtx); + +#ifdef DEBUG + { + int found_it = 0; + ErtsProcList *plp = erts_proclist_peek_first(schdlr_sspnd.chngq); + while (plp) { + if (erts_proclist_same(plp, p)) + found_it++; + plp = erts_proclist_peek_next(schdlr_sspnd.chngq, plp); + } + ASSERT(found_it == !!(p->flags & F_SCHDLR_ONLN_WAITQ)); + } +#endif + + if (p->flags & F_SCHDLR_ONLN_WAITQ) { + ErtsProcList *plp = NULL; + + plp = erts_proclist_peek_first(schdlr_sspnd.chngq); + if (plp) { + if (erts_proclist_same(plp, p) + && schdlr_sspnd.changer == am_true) { + p->flags &= ~F_SCHDLR_ONLN_WAITQ; + /* + * Change right was in transit to us; + * transfer it to the next process by + * resuming it... + */ + erts_proclist_remove(&schdlr_sspnd.chngq, plp); + proclist_destroy(plp); + plp = erts_proclist_peek_first(schdlr_sspnd.chngq); + if (plp) + resume = plp->pid; + else + schdlr_sspnd.changer = am_false; + } + else { + do { + if (erts_proclist_same(plp, p)) { + p->flags &= ~F_SCHDLR_ONLN_WAITQ; + erts_proclist_remove(&schdlr_sspnd.chngq, plp); + proclist_destroy(plp); + break; + } + plp = erts_proclist_peek_next(schdlr_sspnd.chngq, plp); + } while (plp); + } + } + } + + erts_smp_mtx_unlock(&schdlr_sspnd.mtx); + + if (is_internal_pid(resume)) + schdlr_sspnd_resume_proc(resume); +} ErtsSchedSuspendResult erts_set_schedulers_online(Process *p, ErtsProcLocks plocks, Sint new_no, - Sint *old_no -#ifdef ERTS_DIRTY_SCHEDULERS - , int dirty_only -#endif - ) + Sint *old_no, + int dirty_only) { - ErtsSchedulerData *esdp; - int ix, res = -1, no, have_unlocked_plocks, end_wait; - erts_aint32_t changing = 0; + int resume_proc, ix, res = -1, no, have_unlocked_plocks; + erts_aint32_t changing = 0, change_flags; + int online, increase; + ErtsProcList *plp; #ifdef ERTS_DIRTY_SCHEDULERS - ErtsSchedulerSleepInfo* ssi; - int dirty_no, change_dirty; + int dirty_no, change_dirty, dirty_online; +#else + ASSERT(!dirty_only); #endif if (new_no < 1) return ERTS_SCHDLR_SSPND_EINVAL; -#ifdef ERTS_DIRTY_SCHEDULERS else if (dirty_only && erts_no_dirty_cpu_schedulers < new_no) return ERTS_SCHDLR_SSPND_EINVAL; -#endif else if (erts_no_schedulers < new_no) return ERTS_SCHDLR_SSPND_EINVAL; - esdp = ERTS_PROC_GET_SCHDATA(p); - end_wait = 0; +#ifdef ERTS_DIRTY_SCHEDULERS + if (dirty_only) + resume_proc = 0; + else +#endif + { + resume_proc = 1; + /* + * If we suspend current process we need to suspend before + * requesting the change; otherwise, we got a resume/suspend + * race... + */ + if (!(plocks & ERTS_PROC_LOCK_STATUS)) + erts_smp_proc_lock(p, ERTS_PROC_LOCK_STATUS); + suspend_process(p, p); + if (!(plocks & ERTS_PROC_LOCK_STATUS)) + erts_smp_proc_unlock(p, ERTS_PROC_LOCK_STATUS); + } erts_smp_mtx_lock(&schdlr_sspnd.mtx); + change_flags = 0; have_unlocked_plocks = 0; no = (int) new_no; #ifdef ERTS_DIRTY_SCHEDULERS - ASSERT(schdlr_sspnd.dirty_cpu_online <= erts_no_dirty_cpu_schedulers); + if (!dirty_only) +#endif + { + changing = erts_smp_atomic32_read_nob(&schdlr_sspnd.changing); + if (changing & ERTS_SCHDLR_SSPND_CHNG_ONLN) { + enqueue_wait: + p->flags |= F_SCHDLR_ONLN_WAITQ; + plp = proclist_create(p); + erts_proclist_store_last(&schdlr_sspnd.chngq, plp); + resume_proc = 0; + res = ERTS_SCHDLR_SSPND_YIELD_RESTART; + goto done; + } + plp = erts_proclist_peek_first(schdlr_sspnd.chngq); + if (!plp) { + ASSERT(schdlr_sspnd.changer == am_false); + } + else { + ASSERT(schdlr_sspnd.changer == am_true); + if (!erts_proclist_same(plp, p)) + goto enqueue_wait; + p->flags &= ~F_SCHDLR_ONLN_WAITQ; + erts_proclist_remove(&schdlr_sspnd.chngq, plp); + proclist_destroy(plp); + } + } + + *old_no = online = schdlr_sspnd_get_nscheds(&schdlr_sspnd.online, + ERTS_SCHED_NORMAL); +#ifndef ERTS_DIRTY_SCHEDULERS + if (no == online) { + res = ERTS_SCHDLR_SSPND_DONE; + goto done; + } +#else /* ERTS_DIRTY_SCHEDULERS */ + dirty_online = schdlr_sspnd_get_nscheds(&schdlr_sspnd.online, + ERTS_SCHED_DIRTY_CPU); + if (dirty_only) + *old_no = dirty_online; + + ASSERT(dirty_online <= erts_no_dirty_cpu_schedulers); + if (dirty_only) { - if (no > schdlr_sspnd.online) { - erts_smp_mtx_unlock(&schdlr_sspnd.mtx); - return ERTS_SCHDLR_SSPND_EINVAL; + if (no > online) { + res = ERTS_SCHDLR_SSPND_EINVAL; + goto done; } dirty_no = no; + if (dirty_no == dirty_online) { + res = ERTS_SCHDLR_SSPND_DONE; + goto done; + } + change_dirty = 1; } else { /* * Adjust the number of dirty CPU schedulers online relative to the * adjustment made to the number of normal schedulers online. */ int total_pct = erts_no_dirty_cpu_schedulers*100/erts_no_schedulers; - int onln_pct = no*total_pct/schdlr_sspnd.online; - dirty_no = schdlr_sspnd.dirty_cpu_online*onln_pct/100; + int onln_pct = no*total_pct/online; + dirty_no = dirty_online*onln_pct/100; if (dirty_no == 0) dirty_no = 1; ASSERT(dirty_no <= erts_no_dirty_cpu_schedulers); - } -#endif - changing = erts_smp_atomic32_read_nob(&schdlr_sspnd.changing); -#ifdef ERTS_DIRTY_SCHEDULERS - changing |= erts_smp_atomic32_read_nob(&schdlr_sspnd.dirty_cpu_changing); -#endif - if (changing) { - res = ERTS_SCHDLR_SSPND_YIELD_RESTART; - } - else { - int online = *old_no = schdlr_sspnd.online; -#ifdef ERTS_DIRTY_SCHEDULERS - int dirty_online = schdlr_sspnd.dirty_cpu_online; - if (dirty_only) { - *old_no = schdlr_sspnd.dirty_cpu_online; - if (dirty_no == schdlr_sspnd.dirty_cpu_online) { + if (no != online) + change_dirty = (dirty_no != dirty_online); + else { + dirty_only = 1; + if (dirty_no == dirty_online) { res = ERTS_SCHDLR_SSPND_DONE; + goto done; } change_dirty = 1; - } else { -#endif - if (no == schdlr_sspnd.online) { -#ifdef ERTS_DIRTY_SCHEDULERS - dirty_only = 1; - if (dirty_no == schdlr_sspnd.dirty_cpu_online) -#endif - res = ERTS_SCHDLR_SSPND_DONE; -#ifdef ERTS_DIRTY_SCHEDULERS - else - change_dirty = 1; -#endif - } -#ifdef ERTS_DIRTY_SCHEDULERS - else - change_dirty = (dirty_no != schdlr_sspnd.dirty_cpu_online); } -#endif - if (res == -1) - { - int increase = (no > online); -#ifdef ERTS_DIRTY_SCHEDULERS - if (!dirty_only) { -#endif - ERTS_SCHDLR_SSPND_CHNG_SET((ERTS_SCHDLR_SSPND_CHNG_ONLN - | ERTS_SCHDLR_SSPND_CHNG_WAITER), 0); - schdlr_sspnd.online = no; -#ifdef ERTS_DIRTY_SCHEDULERS - } else - increase = (dirty_no > dirty_online); - if (change_dirty) { - ERTS_SCHDLR_SSPND_DIRTY_CPU_CHNG_SET((ERTS_SCHDLR_SSPND_CHNG_ONLN - | ERTS_SCHDLR_SSPND_CHNG_WAITER), 0); - schdlr_sspnd.dirty_cpu_online = dirty_no; - } -#endif - if (increase) { - int ix; -#ifdef ERTS_DIRTY_SCHEDULERS - if (!dirty_only) { -#endif - schdlr_sspnd.wait_curr_online = no; - if (ongoing_multi_scheduling_block()) { - for (ix = online; ix < no; ix++) - erts_sched_poke(ERTS_SCHED_SLEEP_INFO_IX(ix)); - } - else { - if (plocks) { - have_unlocked_plocks = 1; - erts_smp_proc_unlock(p, plocks); - } - change_no_used_runqs(no); + } + if (change_dirty) { + change_flags |= ERTS_SCHDLR_SSPND_CHNG_DCPU_ONLN; + schdlr_sspnd_set_nscheds(&schdlr_sspnd.online, + ERTS_SCHED_DIRTY_CPU, + dirty_no); + } - for (ix = online; ix < no; ix++) - resume_run_queue(ERTS_RUNQ_IX(ix)); + if (dirty_only) + increase = (dirty_no > dirty_online); + else +#endif /* ERTS_DIRTY_SCHEDULERS */ + { + change_flags |= ERTS_SCHDLR_SSPND_CHNG_ONLN; + schdlr_sspnd_set_nscheds(&schdlr_sspnd.online, + ERTS_SCHED_NORMAL, + no); + increase = (no > online); + } - for (ix = no; ix < erts_no_run_queues; ix++) - suspend_run_queue(ERTS_RUNQ_IX(ix)); - } + erts_smp_atomic32_read_bor_nob(&schdlr_sspnd.changing, change_flags); + + res = ERTS_SCHDLR_SSPND_DONE; + if (increase) { + int ix; #ifdef ERTS_DIRTY_SCHEDULERS + if (change_dirty) { + ErtsSchedulerSleepInfo* ssi; + if (schdlr_sspnd.msb.ongoing) { + for (ix = dirty_online; ix < dirty_no; ix++) { + ssi = ERTS_DIRTY_CPU_SCHED_SLEEP_INFO_IX(ix); + erts_sched_poke(ssi); } - if (change_dirty) { - schdlr_sspnd.dirty_cpu_wait_curr_online = dirty_no; - ASSERT(schdlr_sspnd.dirty_cpu_curr_online != - schdlr_sspnd.dirty_cpu_wait_curr_online); - if (ongoing_multi_scheduling_block()) { - for (ix = dirty_online; ix < dirty_no; ix++) { - ssi = ERTS_DIRTY_CPU_SCHED_SLEEP_INFO_IX(ix); - erts_sched_poke(ssi); - } - } else { - for (ix = dirty_online; ix < dirty_no; ix++) { - ssi = ERTS_DIRTY_CPU_SCHED_SLEEP_INFO_IX(ix); - scheduler_ssi_resume_wake(ssi); - erts_smp_atomic32_read_band_nob(&ssi->flags, - ~ERTS_SSI_FLG_SUSPENDED); - } - wake_dirty_schedulers(ERTS_DIRTY_CPU_RUNQ, 0); - } + } else { + for (ix = dirty_online; ix < dirty_no; ix++) { + ssi = ERTS_DIRTY_CPU_SCHED_SLEEP_INFO_IX(ix); + scheduler_ssi_resume_wake(ssi); } -#endif - res = ERTS_SCHDLR_SSPND_DONE; } - else /* if (no < online) */ { -#ifdef ERTS_DIRTY_SCHEDULERS - if (change_dirty) { - schdlr_sspnd.dirty_cpu_wait_curr_online = dirty_no; - ASSERT(schdlr_sspnd.dirty_cpu_curr_online != - schdlr_sspnd.dirty_cpu_wait_curr_online); - if (ongoing_multi_scheduling_block()) { - for (ix = dirty_no; ix < dirty_online; ix++) { - ssi = ERTS_DIRTY_CPU_SCHED_SLEEP_INFO_IX(ix); - erts_sched_poke(ssi); - } - } else { - for (ix = dirty_no; ix < dirty_online; ix++) { - ssi = ERTS_DIRTY_CPU_SCHED_SLEEP_INFO_IX(ix); - erts_smp_atomic32_read_bor_nob(&ssi->flags, - ERTS_SSI_FLG_SUSPENDED); - } - wake_dirty_schedulers(ERTS_DIRTY_CPU_RUNQ, 0); - } - } - if (dirty_only) { - res = ERTS_SCHDLR_SSPND_DONE; - } - else + } + if (!dirty_only) #endif - { - if (p->scheduler_data->no <= no) { - res = ERTS_SCHDLR_SSPND_DONE; - schdlr_sspnd.wait_curr_online = no; - } - else { - /* - * Yield! Current process needs to migrate - * before bif returns. - */ - res = ERTS_SCHDLR_SSPND_YIELD_DONE; - schdlr_sspnd.wait_curr_online = no+1; - } - - if (ongoing_multi_scheduling_block()) { - for (ix = no; ix < online; ix++) - erts_sched_poke(ERTS_SCHED_SLEEP_INFO_IX(ix)); - } - else { - if (plocks) { - have_unlocked_plocks = 1; - erts_smp_proc_unlock(p, plocks); - } - - change_no_used_runqs(no); - for (ix = no; ix < erts_no_run_queues; ix++) - suspend_run_queue(ERTS_RUNQ_IX(ix)); - - for (ix = no; ix < online; ix++) { - ErtsRunQueue *rq = ERTS_RUNQ_IX(ix); - wake_scheduler(rq); - } - } - } + { + if (schdlr_sspnd.msb.ongoing|schdlr_sspnd.nmsb.ongoing) { + for (ix = online; ix < no; ix++) + erts_sched_poke(ERTS_SCHED_SLEEP_INFO_IX(ix)); } - -#ifdef ERTS_DIRTY_SCHEDULERS - if (change_dirty) { - while (schdlr_sspnd.dirty_cpu_curr_online != schdlr_sspnd.dirty_cpu_wait_curr_online) - erts_smp_cnd_wait(&schdlr_sspnd.cnd, &schdlr_sspnd.mtx); - ERTS_SCHDLR_SSPND_DIRTY_CPU_CHNG_SET(0, ERTS_SCHDLR_SSPND_CHNG_WAITER); - erts_smp_atomic32_read_band_nob(&schdlr_sspnd.dirty_cpu_changing, - ~ERTS_SCHDLR_SSPND_CHNG_WAITER); - } - if (!dirty_only) -#endif - { - if (schdlr_sspnd.curr_online != schdlr_sspnd.wait_curr_online) { - erts_smp_mtx_unlock(&schdlr_sspnd.mtx); - if (plocks && !have_unlocked_plocks) { - have_unlocked_plocks = 1; - erts_smp_proc_unlock(p, plocks); - } - erts_thr_progress_active(esdp, 0); - erts_thr_progress_prepare_wait(esdp); - end_wait = 1; - erts_smp_mtx_lock(&schdlr_sspnd.mtx); + else { + if (plocks) { + have_unlocked_plocks = 1; + erts_smp_proc_unlock(p, plocks); } + change_no_used_runqs(no); - while (schdlr_sspnd.curr_online != schdlr_sspnd.wait_curr_online) - erts_smp_cnd_wait(&schdlr_sspnd.cnd, &schdlr_sspnd.mtx); + for (ix = online; ix < no; ix++) + resume_run_queue(ERTS_RUNQ_IX(ix)); - ASSERT(res != ERTS_SCHDLR_SSPND_DONE - ? (ERTS_SCHDLR_SSPND_CHNG_WAITER - & erts_smp_atomic32_read_nob(&schdlr_sspnd.changing)) - : (ERTS_SCHDLR_SSPND_CHNG_WAITER - == erts_smp_atomic32_read_nob(&schdlr_sspnd.changing))); - erts_smp_atomic32_read_band_nob(&schdlr_sspnd.changing, - ~ERTS_SCHDLR_SSPND_CHNG_WAITER); + for (ix = no; ix < erts_no_run_queues; ix++) + suspend_run_queue(ERTS_RUNQ_IX(ix)); } } } - - erts_smp_mtx_unlock(&schdlr_sspnd.mtx); + else /* if decrease */ { #ifdef ERTS_DIRTY_SCHEDULERS - ASSERT(schdlr_sspnd.dirty_cpu_online <= schdlr_sspnd.online); - if (!dirty_only) -#endif - { - if (end_wait) { - erts_thr_progress_finalize_wait(esdp); - erts_thr_progress_active(esdp, 1); - } - if (have_unlocked_plocks) - erts_smp_proc_lock(p, plocks); - } - - return res; -} - -#else /* !ERTS_DIRTY_SCHEDULERS */ - -ErtsSchedSuspendResult -erts_set_schedulers_online(Process *p, - ErtsProcLocks plocks, - Sint new_no, - Sint *old_no) -{ - ErtsSchedulerData *esdp; - int ix, res, no, have_unlocked_plocks, end_wait; - erts_aint32_t changing; - - if (new_no < 1 || erts_no_schedulers < new_no) - return ERTS_SCHDLR_SSPND_EINVAL; - - esdp = ERTS_PROC_GET_SCHDATA(p); - end_wait = 0; - - erts_smp_mtx_lock(&schdlr_sspnd.mtx); - - have_unlocked_plocks = 0; - no = (int) new_no; - - changing = erts_smp_atomic32_read_nob(&schdlr_sspnd.changing); - if (changing) { - res = ERTS_SCHDLR_SSPND_YIELD_RESTART; - } - else { - int online = *old_no = schdlr_sspnd.online; - if (no == schdlr_sspnd.online) { - res = ERTS_SCHDLR_SSPND_DONE; - } - else { - ERTS_SCHDLR_SSPND_CHNG_SET((ERTS_SCHDLR_SSPND_CHNG_ONLN - | ERTS_SCHDLR_SSPND_CHNG_WAITER), 0); - schdlr_sspnd.online = no; - if (no > online) { - int ix; - schdlr_sspnd.wait_curr_online = no; - if (ongoing_multi_scheduling_block()) { - for (ix = online; ix < no; ix++) - erts_sched_poke(ERTS_SCHED_SLEEP_INFO_IX(ix)); + if (change_dirty) { + ErtsSchedulerSleepInfo* ssi; + if (schdlr_sspnd.msb.ongoing) { + for (ix = dirty_no; ix < dirty_online; ix++) { + ssi = ERTS_DIRTY_CPU_SCHED_SLEEP_INFO_IX(ix); + erts_sched_poke(ssi); } - else { - if (plocks) { - have_unlocked_plocks = 1; - erts_smp_proc_unlock(p, plocks); - } - change_no_used_runqs(no); - - for (ix = online; ix < no; ix++) - resume_run_queue(ERTS_RUNQ_IX(ix)); - - for (ix = no; ix < erts_no_run_queues; ix++) - suspend_run_queue(ERTS_RUNQ_IX(ix)); + } else { + for (ix = dirty_no; ix < dirty_online; ix++) { + ssi = ERTS_DIRTY_CPU_SCHED_SLEEP_INFO_IX(ix); + erts_smp_atomic32_read_bor_nob(&ssi->flags, + ERTS_SSI_FLG_SUSPENDED); } - res = ERTS_SCHDLR_SSPND_DONE; + wake_dirty_schedulers(ERTS_DIRTY_CPU_RUNQ, 0); } - else /* if (no < online) */ { - if (p->scheduler_data->no <= no) { - res = ERTS_SCHDLR_SSPND_DONE; - schdlr_sspnd.wait_curr_online = no; - } - else { - /* - * Yield! Current process needs to migrate - * before bif returns. - */ - res = ERTS_SCHDLR_SSPND_YIELD_DONE; - schdlr_sspnd.wait_curr_online = no+1; - } - - if (ongoing_multi_scheduling_block()) { - for (ix = no; ix < online; ix++) - erts_sched_poke(ERTS_SCHED_SLEEP_INFO_IX(ix)); - } - else { - if (plocks) { - have_unlocked_plocks = 1; - erts_smp_proc_unlock(p, plocks); - } - - change_no_used_runqs(no); - for (ix = no; ix < erts_no_run_queues; ix++) - suspend_run_queue(ERTS_RUNQ_IX(ix)); - - for (ix = no; ix < online; ix++) { - ErtsRunQueue *rq = ERTS_RUNQ_IX(ix); - wake_scheduler(rq); - } - } + } + if (!dirty_only) +#endif + { + if (schdlr_sspnd.msb.ongoing|schdlr_sspnd.nmsb.ongoing) { + for (ix = no; ix < online; ix++) + erts_sched_poke(ERTS_SCHED_SLEEP_INFO_IX(ix)); } - - if (schdlr_sspnd.curr_online != schdlr_sspnd.wait_curr_online) { - erts_smp_mtx_unlock(&schdlr_sspnd.mtx); - if (plocks && !have_unlocked_plocks) { + else { + if (plocks) { have_unlocked_plocks = 1; erts_smp_proc_unlock(p, plocks); } - erts_thr_progress_active(esdp, 0); - erts_thr_progress_prepare_wait(esdp); - end_wait = 1; - erts_smp_mtx_lock(&schdlr_sspnd.mtx); - } - while (schdlr_sspnd.curr_online != schdlr_sspnd.wait_curr_online) - erts_smp_cnd_wait(&schdlr_sspnd.cnd, &schdlr_sspnd.mtx); - - ASSERT(res != ERTS_SCHDLR_SSPND_DONE - ? (ERTS_SCHDLR_SSPND_CHNG_WAITER - & erts_smp_atomic32_read_nob(&schdlr_sspnd.changing)) - : (ERTS_SCHDLR_SSPND_CHNG_WAITER - == erts_smp_atomic32_read_nob(&schdlr_sspnd.changing))); - erts_smp_atomic32_read_band_nob(&schdlr_sspnd.changing, - ~ERTS_SCHDLR_SSPND_CHNG_WAITER); + change_no_used_runqs(no); + for (ix = no; ix < erts_no_run_queues; ix++) + suspend_run_queue(ERTS_RUNQ_IX(ix)); + for (ix = no; ix < online; ix++) { + ErtsRunQueue *rq = ERTS_RUNQ_IX(ix); + wake_scheduler(rq); + } + } } } - erts_smp_mtx_unlock(&schdlr_sspnd.mtx); - if (end_wait) { - erts_thr_progress_finalize_wait(esdp); - erts_thr_progress_active(esdp, 1); + if (change_flags & ERTS_SCHDLR_SSPND_CHNG_ONLN) { + /* Suspend and wait for requested change to complete... */ + schdlr_sspnd.changer = p->common.id; + resume_proc = 0; + res = ERTS_SCHDLR_SSPND_YIELD_DONE; } + +done: + + ASSERT(schdlr_sspnd_get_nscheds(&schdlr_sspnd.online, + ERTS_SCHED_DIRTY_CPU) + <= schdlr_sspnd_get_nscheds(&schdlr_sspnd.online, + ERTS_SCHED_NORMAL)); + + erts_smp_mtx_unlock(&schdlr_sspnd.mtx); + if (have_unlocked_plocks) erts_smp_proc_lock(p, plocks); + if (resume_proc) { + if (!(plocks & ERTS_PROC_LOCK_STATUS)) + erts_smp_proc_lock(p, ERTS_PROC_LOCK_STATUS); + resume_process(p, plocks|ERTS_PROC_LOCK_STATUS); + if (!(plocks & ERTS_PROC_LOCK_STATUS)) + erts_smp_proc_unlock(p, ERTS_PROC_LOCK_STATUS); + } + return res; } -#endif - ErtsSchedSuspendResult -erts_block_multi_scheduling(Process *p, ErtsProcLocks plocks, int on, int all) +erts_block_multi_scheduling(Process *p, ErtsProcLocks plocks, int on, int normal, int all) { - int ix, res, have_unlocked_plocks = 0, online; - erts_aint32_t changing; + int resume_proc, ix, res, have_unlocked_plocks = 0; ErtsProcList *plp; #ifdef ERTS_DIRTY_SCHEDULERS ErtsSchedulerSleepInfo* ssi; #endif + ErtsMultiSchedulingBlock *msbp; + erts_aint32_t chng_flg; + int have_blckd_flg; - erts_smp_mtx_lock(&schdlr_sspnd.mtx); - changing = erts_smp_atomic32_read_nob(&schdlr_sspnd.changing); -#ifdef ERTS_DIRTY_SCHEDULERS - changing |= (erts_smp_atomic32_read_nob(&schdlr_sspnd.dirty_cpu_changing) - | erts_smp_atomic32_read_nob(&schdlr_sspnd.dirty_io_changing)); -#endif - if (changing) { - res = ERTS_SCHDLR_SSPND_YIELD_RESTART; /* Yield */ + if (normal) { + chng_flg = ERTS_SCHDLR_SSPND_CHNG_NMSB; + have_blckd_flg = F_HAVE_BLCKD_NMSCHED; + msbp = &schdlr_sspnd.nmsb; } - else if (on) { /* ------ BLOCK ------ */ - if (schdlr_sspnd.msb.procs) { + else { + chng_flg = ERTS_SCHDLR_SSPND_CHNG_MSB; + have_blckd_flg = F_HAVE_BLCKD_MSCHED; + msbp = &schdlr_sspnd.msb; + } + + /* + * If we suspend current process we need to suspend before + * requesting the change; otherwise, we got a resume/suspend + * race... + */ + if (!on) { + /* We never suspend current process when unblocking... */ + resume_proc = 0; + } + else { + resume_proc = 1; + if (!(plocks & ERTS_PROC_LOCK_STATUS)) + erts_smp_proc_lock(p, ERTS_PROC_LOCK_STATUS); + suspend_process(p, p); + if (!(plocks & ERTS_PROC_LOCK_STATUS)) + erts_smp_proc_unlock(p, ERTS_PROC_LOCK_STATUS); + } + + erts_smp_mtx_lock(&schdlr_sspnd.mtx); + if (on) { /* ------ BLOCK ------ */ + if (msbp->chngq) { + ASSERT(msbp->ongoing); + p->flags |= have_blckd_flg; + goto wait_until_msb; + } + else if (msbp->blckrs) { + ASSERT(msbp->ongoing); plp = proclist_create(p); - erts_proclist_store_last(&schdlr_sspnd.msb.procs, plp); - p->flags |= F_HAVE_BLCKD_MSCHED; - ASSERT(erts_smp_atomic32_read_nob(&schdlr_sspnd.active) == 1); -#ifdef ERTS_DIRTY_SCHEDULERS - ASSERT(erts_smp_atomic32_read_nob(&schdlr_sspnd.dirty_cpu_active) == 0); - ASSERT(erts_smp_atomic32_read_nob(&schdlr_sspnd.dirty_io_active) == 0); -#endif + erts_proclist_store_last(&msbp->blckrs, plp); + p->flags |= have_blckd_flg; + ASSERT(schdlr_sspnd.active == ERTS_SCHDLR_SSPND_MAKE_NSCHEDS_VAL(1, 0, 0)); ASSERT(p->scheduler_data->no == 1); - res = ERTS_SCHDLR_SSPND_DONE_MSCHED_BLOCKED; - } else { - int online = schdlr_sspnd.online; - p->flags |= F_HAVE_BLCKD_MSCHED; + if (schdlr_sspnd.msb.ongoing) + res = ERTS_SCHDLR_SSPND_DONE_MSCHED_BLOCKED; + else + res = ERTS_SCHDLR_SSPND_DONE_NMSCHED_BLOCKED; + } + else { + int online = (int) schdlr_sspnd_get_nscheds(&schdlr_sspnd.online, + ERTS_SCHED_NORMAL); + ASSERT(!msbp->ongoing); + p->flags |= have_blckd_flg; if (plocks) { have_unlocked_plocks = 1; erts_smp_proc_unlock(p, plocks); } - ASSERT(!ongoing_multi_scheduling_block()); - schdlr_sspnd.msb.ongoing = 1; - if (online == 1) { - res = ERTS_SCHDLR_SSPND_DONE_MSCHED_BLOCKED; - ASSERT(erts_smp_atomic32_read_nob(&schdlr_sspnd.active) == 1); -#ifdef ERTS_DIRTY_SCHEDULERS - ASSERT(erts_smp_atomic32_read_nob(&schdlr_sspnd.dirty_cpu_active) == 1); - ASSERT(!(erts_smp_atomic32_read_nob(&ERTS_DIRTY_CPU_SCHED_SLEEP_INFO_IX(0)->flags) - & ERTS_SSI_FLG_SUSPENDED)); - schdlr_sspnd.msb.dirty_cpu_wait_active = 0; - ERTS_SCHDLR_SSPND_DIRTY_CPU_CHNG_SET((ERTS_SCHDLR_SSPND_CHNG_MSB - | ERTS_SCHDLR_SSPND_CHNG_WAITER), 0); - ssi = ERTS_DIRTY_CPU_SCHED_SLEEP_INFO_IX(0); - erts_smp_atomic32_read_bor_nob(&ssi->flags, ERTS_SSI_FLG_SUSPENDED); - wake_dirty_schedulers(ERTS_DIRTY_CPU_RUNQ, 0); - while (erts_smp_atomic32_read_nob(&schdlr_sspnd.dirty_cpu_active) - != schdlr_sspnd.msb.dirty_cpu_wait_active) - erts_smp_cnd_wait(&schdlr_sspnd.cnd, &schdlr_sspnd.mtx); - ERTS_SCHDLR_SSPND_DIRTY_CPU_CHNG_SET(0, ERTS_SCHDLR_SSPND_CHNG_WAITER); - - schdlr_sspnd.msb.dirty_io_wait_active = 0; - ERTS_SCHDLR_SSPND_DIRTY_IO_CHNG_SET((ERTS_SCHDLR_SSPND_CHNG_MSB - | ERTS_SCHDLR_SSPND_CHNG_WAITER), 0); - for (ix = 0; ix < erts_no_dirty_io_schedulers; ix++) { - ssi = ERTS_DIRTY_IO_SCHED_SLEEP_INFO_IX(ix); - erts_smp_atomic32_read_bor_nob(&ssi->flags, - ERTS_SSI_FLG_SUSPENDED); - } - wake_dirty_schedulers(ERTS_DIRTY_IO_RUNQ, 0); - while (erts_smp_atomic32_read_nob(&schdlr_sspnd.dirty_io_active) - != schdlr_sspnd.msb.dirty_io_wait_active) - erts_smp_cnd_wait(&schdlr_sspnd.cnd, &schdlr_sspnd.mtx); - ERTS_SCHDLR_SSPND_DIRTY_IO_CHNG_SET(0, ERTS_SCHDLR_SSPND_CHNG_WAITER); -#endif + ASSERT(!msbp->ongoing); + msbp->ongoing = 1; + if (schdlr_sspnd.active == ERTS_SCHDLR_SSPND_MAKE_NSCHEDS_VAL(1, 0, 0) + || (normal && schdlr_sspnd_get_nscheds(&schdlr_sspnd.active, + ERTS_SCHED_NORMAL) == 1)) { ASSERT(p->scheduler_data->no == 1); + plp = proclist_create(p); + erts_proclist_store_last(&msbp->blckrs, plp); + if (schdlr_sspnd.msb.ongoing) + res = ERTS_SCHDLR_SSPND_DONE_MSCHED_BLOCKED; + else + res = ERTS_SCHDLR_SSPND_DONE_NMSCHED_BLOCKED; } else { - ERTS_SCHDLR_SSPND_CHNG_SET((ERTS_SCHDLR_SSPND_CHNG_MSB - | ERTS_SCHDLR_SSPND_CHNG_WAITER), 0); - if (p->scheduler_data->no == 1) { - res = ERTS_SCHDLR_SSPND_DONE_MSCHED_BLOCKED; - schdlr_sspnd.msb.wait_active = 1; - } - else { - /* - * Yield! Current process needs to migrate - * before bif returns. - */ - res = ERTS_SCHDLR_SSPND_YIELD_DONE_MSCHED_BLOCKED; - schdlr_sspnd.msb.wait_active = 2; - } - -#ifdef ERTS_DIRTY_SCHEDULERS - schdlr_sspnd.msb.dirty_cpu_wait_active = 0; - ERTS_SCHDLR_SSPND_DIRTY_CPU_CHNG_SET((ERTS_SCHDLR_SSPND_CHNG_MSB - | ERTS_SCHDLR_SSPND_CHNG_WAITER), 0); - for (ix = 0; ix < erts_no_dirty_cpu_schedulers; ix++) { - ssi = ERTS_DIRTY_CPU_SCHED_SLEEP_INFO_IX(ix); - erts_smp_atomic32_read_bor_nob(&ssi->flags, - ERTS_SSI_FLG_SUSPENDED); - } - wake_dirty_schedulers(ERTS_DIRTY_CPU_RUNQ, 0); - while (erts_smp_atomic32_read_nob(&schdlr_sspnd.dirty_cpu_active) - != schdlr_sspnd.msb.dirty_cpu_wait_active) - erts_smp_cnd_wait(&schdlr_sspnd.cnd, &schdlr_sspnd.mtx); - ERTS_SCHDLR_SSPND_DIRTY_CPU_CHNG_SET(0, ERTS_SCHDLR_SSPND_CHNG_WAITER); - ASSERT(schdlr_sspnd.dirty_cpu_curr_online == schdlr_sspnd.dirty_cpu_online); - - schdlr_sspnd.msb.dirty_io_wait_active = 0; - ERTS_SCHDLR_SSPND_DIRTY_IO_CHNG_SET((ERTS_SCHDLR_SSPND_CHNG_MSB - | ERTS_SCHDLR_SSPND_CHNG_WAITER), 0); - for (ix = 0; ix < erts_no_dirty_io_schedulers; ix++) { - ssi = ERTS_DIRTY_IO_SCHED_SLEEP_INFO_IX(ix); - erts_smp_atomic32_read_bor_nob(&ssi->flags, - ERTS_SSI_FLG_SUSPENDED); - } - wake_dirty_schedulers(ERTS_DIRTY_IO_RUNQ, 0); - while (erts_smp_atomic32_read_nob(&schdlr_sspnd.dirty_io_active) - != schdlr_sspnd.msb.dirty_io_wait_active) - erts_smp_cnd_wait(&schdlr_sspnd.cnd, &schdlr_sspnd.mtx); - ERTS_SCHDLR_SSPND_DIRTY_IO_CHNG_SET(0, ERTS_SCHDLR_SSPND_CHNG_WAITER); - ASSERT(schdlr_sspnd.dirty_io_curr_online == schdlr_sspnd.dirty_io_online); -#endif + erts_smp_atomic32_read_bor_nob(&schdlr_sspnd.changing, + chng_flg); change_no_used_runqs(1); for (ix = 1; ix < erts_no_run_queues; ix++) suspend_run_queue(ERTS_RUNQ_IX(ix)); @@ -7397,84 +7798,84 @@ erts_block_multi_scheduling(Process *p, ErtsProcLocks plocks, int on, int all) wake_scheduler(rq); } - if (erts_smp_atomic32_read_nob(&schdlr_sspnd.active) - != schdlr_sspnd.msb.wait_active) { - ErtsSchedulerData *esdp; - - erts_smp_mtx_unlock(&schdlr_sspnd.mtx); - - if (plocks && !have_unlocked_plocks) { - have_unlocked_plocks = 1; - erts_smp_proc_unlock(p, plocks); +#ifdef ERTS_DIRTY_SCHEDULERS + if (!normal) { + for (ix = 0; ix < erts_no_dirty_cpu_schedulers; ix++) { + ssi = ERTS_DIRTY_CPU_SCHED_SLEEP_INFO_IX(ix); + erts_smp_atomic32_read_bor_nob(&ssi->flags, + ERTS_SSI_FLG_SUSPENDED); } + wake_dirty_schedulers(ERTS_DIRTY_CPU_RUNQ, 0); - esdp = ERTS_PROC_GET_SCHDATA(p); - - erts_thr_progress_active(esdp, 0); - erts_thr_progress_prepare_wait(esdp); - - erts_smp_mtx_lock(&schdlr_sspnd.mtx); - - while (erts_smp_atomic32_read_nob(&schdlr_sspnd.active) - != schdlr_sspnd.msb.wait_active) - erts_smp_cnd_wait(&schdlr_sspnd.cnd, - &schdlr_sspnd.mtx); - - erts_smp_mtx_unlock(&schdlr_sspnd.mtx); - - erts_thr_progress_active(esdp, 1); - erts_thr_progress_finalize_wait(esdp); + for (ix = 0; ix < erts_no_dirty_io_schedulers; ix++) { + ssi = ERTS_DIRTY_IO_SCHED_SLEEP_INFO_IX(ix); + erts_smp_atomic32_read_bor_nob(&ssi->flags, + ERTS_SSI_FLG_SUSPENDED); + } + wake_dirty_schedulers(ERTS_DIRTY_IO_RUNQ, 0); + } +#endif - erts_smp_mtx_lock(&schdlr_sspnd.mtx); + wait_until_msb: - } + ASSERT(chng_flg & erts_smp_atomic32_read_nob(&schdlr_sspnd.changing)); - ASSERT(res != ERTS_SCHDLR_SSPND_DONE_MSCHED_BLOCKED - ? (ERTS_SCHDLR_SSPND_CHNG_WAITER - & erts_smp_atomic32_read_nob(&schdlr_sspnd.changing)) - : (ERTS_SCHDLR_SSPND_CHNG_WAITER - == erts_smp_atomic32_read_nob(&schdlr_sspnd.changing))); - erts_smp_atomic32_read_band_nob(&schdlr_sspnd.changing, - ~ERTS_SCHDLR_SSPND_CHNG_WAITER); + plp = proclist_create(p); + erts_proclist_store_last(&msbp->chngq, plp); + resume_proc = 0; + if (schdlr_sspnd.msb.ongoing) + res = ERTS_SCHDLR_SSPND_YIELD_DONE_MSCHED_BLOCKED; + else + res = ERTS_SCHDLR_SSPND_YIELD_DONE_NMSCHED_BLOCKED; } - plp = proclist_create(p); - erts_proclist_store_last(&schdlr_sspnd.msb.procs, plp); ASSERT(p->scheduler_data); } } - else if (!ongoing_multi_scheduling_block()) { - /* unblock not ongoing */ - ASSERT(!schdlr_sspnd.msb.procs); - res = ERTS_SCHDLR_SSPND_DONE; + else if (!msbp->ongoing) { + ASSERT(!msbp->blckrs); + goto unblock_res; } else { /* ------ UNBLOCK ------ */ - if (p->flags & F_HAVE_BLCKD_MSCHED) { - ErtsProcList *plp = erts_proclist_peek_first(schdlr_sspnd.msb.procs); - - while (plp) { - ErtsProcList *tmp_plp = plp; - plp = erts_proclist_peek_next(schdlr_sspnd.msb.procs, plp); - if (erts_proclist_same(tmp_plp, p)) { - erts_proclist_remove(&schdlr_sspnd.msb.procs, tmp_plp); - proclist_destroy(tmp_plp); - if (!all) - break; + if (p->flags & have_blckd_flg) { + ErtsProcList *plps[2]; + ErtsProcList *plp; + int limit = 0; + + plps[limit++] = erts_proclist_peek_first(msbp->blckrs); + if (all) + plps[limit++] = erts_proclist_peek_first(msbp->chngq); + + for (ix = 0; ix < limit; ix++) { + plp = plps[ix]; + while (plp) { + ErtsProcList *tmp_plp = plp; + plp = erts_proclist_peek_next(msbp->blckrs, plp); + if (erts_proclist_same(tmp_plp, p)) { + erts_proclist_remove(&msbp->blckrs, tmp_plp); + proclist_destroy(tmp_plp); + if (!all) + break; + } } } } - if (schdlr_sspnd.msb.procs) - res = ERTS_SCHDLR_SSPND_DONE_MSCHED_BLOCKED; - else { - ERTS_SCHDLR_SSPND_CHNG_SET(ERTS_SCHDLR_SSPND_CHNG_MSB, 0); - p->flags &= ~F_HAVE_BLCKD_MSCHED; - schdlr_sspnd.msb.ongoing = 0; - if (schdlr_sspnd.online == 1) { + if (!msbp->blckrs && !msbp->chngq) { + int online = (int) schdlr_sspnd_get_nscheds(&schdlr_sspnd.online, + ERTS_SCHED_NORMAL); + erts_smp_atomic32_read_bor_nob(&schdlr_sspnd.changing, + chng_flg); + p->flags &= ~have_blckd_flg; + msbp->ongoing = 0; + if (online == 1) { /* No normal schedulers to resume */ - ASSERT(erts_smp_atomic32_read_nob(&schdlr_sspnd.active) == 1); - ERTS_SCHDLR_SSPND_CHNG_SET(0, ERTS_SCHDLR_SSPND_CHNG_MSB); + ASSERT(schdlr_sspnd_get_nscheds(&schdlr_sspnd.active, + ERTS_SCHED_NORMAL) == 1); +#ifndef ERTS_DIRTY_SCHEDULERS + erts_smp_atomic32_read_band_nob(&schdlr_sspnd.changing, + ~chng_flg); +#endif } - else { - online = schdlr_sspnd.online; + else if (!(schdlr_sspnd.msb.ongoing|schdlr_sspnd.nmsb.ongoing)) { if (plocks) { have_unlocked_plocks = 1; erts_smp_proc_unlock(p, plocks); @@ -7490,83 +7891,91 @@ erts_block_multi_scheduling(Process *p, ErtsProcLocks plocks, int on, int all) suspend_run_queue(ERTS_RUNQ_IX(ix)); } #ifdef ERTS_DIRTY_SCHEDULERS - ERTS_SCHDLR_SSPND_DIRTY_CPU_CHNG_SET(ERTS_SCHDLR_SSPND_CHNG_MSB, 0); - schdlr_sspnd.msb.dirty_cpu_wait_active = schdlr_sspnd.dirty_cpu_online; - for (ix = 0; ix < schdlr_sspnd.dirty_cpu_online; ix++) { - ssi = ERTS_DIRTY_CPU_SCHED_SLEEP_INFO_IX(ix); - scheduler_ssi_resume_wake(ssi); - erts_smp_atomic32_read_band_nob(&ssi->flags, - ~ERTS_SSI_FLG_SUSPENDED); - } - wake_dirty_schedulers(ERTS_DIRTY_CPU_RUNQ, 0); - - ERTS_SCHDLR_SSPND_DIRTY_IO_CHNG_SET(ERTS_SCHDLR_SSPND_CHNG_MSB, 0); - schdlr_sspnd.msb.dirty_io_wait_active = erts_no_dirty_io_schedulers; - for (ix = 0; ix < erts_no_dirty_io_schedulers; ix++) { - ssi = ERTS_DIRTY_IO_SCHED_SLEEP_INFO_IX(ix); - scheduler_ssi_resume_wake(ssi); - erts_smp_atomic32_read_band_nob(&ssi->flags, - ~ERTS_SSI_FLG_SUSPENDED); - } - wake_dirty_schedulers(ERTS_DIRTY_IO_RUNQ, 0); + if (!normal) { + ASSERT(!schdlr_sspnd.msb.ongoing); + online = (int) schdlr_sspnd_get_nscheds(&schdlr_sspnd.online, + ERTS_SCHED_DIRTY_CPU); + for (ix = 0; ix < online; ix++) { + ssi = ERTS_DIRTY_CPU_SCHED_SLEEP_INFO_IX(ix); + scheduler_ssi_resume_wake(ssi); + } + + for (ix = 0; ix < erts_no_dirty_io_schedulers; ix++) { + ssi = ERTS_DIRTY_IO_SCHED_SLEEP_INFO_IX(ix); + scheduler_ssi_resume_wake(ssi); + } + } #endif - res = ERTS_SCHDLR_SSPND_DONE; } + + unblock_res: + if (schdlr_sspnd.msb.ongoing) + res = ERTS_SCHDLR_SSPND_DONE_MSCHED_BLOCKED; + else if (schdlr_sspnd.nmsb.ongoing) + res = ERTS_SCHDLR_SSPND_DONE_NMSCHED_BLOCKED; + else + res = ERTS_SCHDLR_SSPND_DONE; } erts_smp_mtx_unlock(&schdlr_sspnd.mtx); + if (have_unlocked_plocks) erts_smp_proc_lock(p, plocks); - return res; -} -#ifdef DEBUG -void -erts_dbg_multi_scheduling_return_trap(Process *p, Eterm return_value) -{ - if (return_value == am_blocked) { - erts_aint32_t active = erts_smp_atomic32_read_nob(&schdlr_sspnd.active); - ASSERT(1 <= active && active <= 2); - ASSERT(ERTS_PROC_GET_SCHDATA(p)->no == 1); + if (resume_proc) { + if (!(plocks & ERTS_PROC_LOCK_STATUS)) + erts_smp_proc_lock(p, ERTS_PROC_LOCK_STATUS); + resume_process(p, plocks|ERTS_PROC_LOCK_STATUS); + if (!(plocks & ERTS_PROC_LOCK_STATUS)) + erts_smp_proc_unlock(p, ERTS_PROC_LOCK_STATUS); } + + return res; } -#endif int erts_is_multi_scheduling_blocked(void) { int res; erts_smp_mtx_lock(&schdlr_sspnd.mtx); - res = schdlr_sspnd.msb.procs != NULL; + if (schdlr_sspnd.msb.blckrs) + res = 1; + else if (schdlr_sspnd.nmsb.blckrs) + res = -1; + else + res = 0; erts_smp_mtx_unlock(&schdlr_sspnd.mtx); return res; } Eterm -erts_multi_scheduling_blockers(Process *p) +erts_multi_scheduling_blockers(Process *p, int normal) { Eterm res = NIL; + ErtsMultiSchedulingBlock *msbp; + + msbp = normal ? &schdlr_sspnd.nmsb : &schdlr_sspnd.msb; erts_smp_mtx_lock(&schdlr_sspnd.mtx); - if (!erts_proclist_is_empty(schdlr_sspnd.msb.procs)) { + if (!erts_proclist_is_empty(msbp->blckrs)) { Eterm *hp, *hp_end; ErtsProcList *plp1, *plp2; Uint max_size = 0; - for (plp1 = erts_proclist_peek_first(schdlr_sspnd.msb.procs); + for (plp1 = erts_proclist_peek_first(msbp->blckrs); plp1; - plp1 = erts_proclist_peek_next(schdlr_sspnd.msb.procs, plp1)) { + plp1 = erts_proclist_peek_next(msbp->blckrs, plp1)) { max_size += 2; } ASSERT(max_size); hp = HAlloc(p, max_size); hp_end = hp + max_size; - for (plp1 = erts_proclist_peek_first(schdlr_sspnd.msb.procs); + for (plp1 = erts_proclist_peek_first(msbp->blckrs); plp1; - plp1 = erts_proclist_peek_next(schdlr_sspnd.msb.procs, plp1)) { - for (plp2 = erts_proclist_peek_first(schdlr_sspnd.msb.procs); + plp1 = erts_proclist_peek_next(msbp->blckrs, plp1)) { + for (plp2 = erts_proclist_peek_first(msbp->blckrs); plp2->pid != plp1->pid; - plp2 = erts_proclist_peek_next(schdlr_sspnd.msb.procs, plp2)); + plp2 = erts_proclist_peek_next(msbp->blckrs, plp2)); if (plp2 == plp1) { res = CONS(hp, plp1->pid, res); hp += 2; @@ -7586,13 +7995,26 @@ sched_thread_func(void *vesdp) ErtsSchedulerData *esdp = vesdp; Uint no = esdp->no; #ifdef ERTS_SMP - ERTS_SCHED_SLEEP_INFO_IX(no - 1)->event = erts_tse_fetch(); + erts_tse_t *tse; +#endif + + erts_sched_init_time_sup(esdp); + + (void) ERTS_RUNQ_FLGS_SET_NOB(esdp->run_queue, + ERTS_RUNQ_FLG_EXEC); + +#ifdef ERTS_SMP + tse = erts_tse_fetch(); + erts_tse_prepare_timed(tse); + ERTS_SCHED_SLEEP_INFO_IX(no - 1)->event = tse; callbacks.arg = (void *) esdp->ssi; callbacks.wakeup = thr_prgr_wakeup; callbacks.prepare_wait = thr_prgr_prep_wait; callbacks.wait = thr_prgr_wait; callbacks.finalize_wait = thr_prgr_fin_wait; + erts_msacc_init_thread("scheduler", no, 1); + erts_thr_progress_register_managed_thread(esdp, &callbacks, 0); erts_alloc_register_scheduler(vesdp); #endif @@ -7622,39 +8044,6 @@ sched_thread_func(void *vesdp) #endif erts_thread_init_float(); - if (no == 1) { - erts_thr_progress_active(esdp, 0); - erts_thr_progress_prepare_wait(esdp); - } - - erts_smp_mtx_lock(&schdlr_sspnd.mtx); - - ASSERT(erts_smp_atomic32_read_nob(&schdlr_sspnd.changing) - & ERTS_SCHDLR_SSPND_CHNG_ONLN); - - if (--schdlr_sspnd.curr_online == schdlr_sspnd.wait_curr_online) { - erts_smp_atomic32_read_band_nob(&schdlr_sspnd.changing, - ~ERTS_SCHDLR_SSPND_CHNG_ONLN); - if (no != 1) -#ifdef ERTS_DIRTY_SCHEDULERS - erts_smp_cnd_broadcast(&schdlr_sspnd.cnd); -#else - erts_smp_cnd_signal(&schdlr_sspnd.cnd); -#endif - } - - if (no == 1) { - while (schdlr_sspnd.curr_online != schdlr_sspnd.wait_curr_online) - erts_smp_cnd_wait(&schdlr_sspnd.cnd, &schdlr_sspnd.mtx); - ERTS_SCHDLR_SSPND_CHNG_SET(0, ERTS_SCHDLR_SSPND_CHNG_WAITER); - } - erts_smp_mtx_unlock(&schdlr_sspnd.mtx); - - if (no == 1) { - erts_thr_progress_finalize_wait(esdp); - erts_thr_progress_active(esdp, 1); - } - #ifdef ERTS_DO_VERIFY_UNUSED_TEMP_ALLOC esdp->verify_unused_temp_alloc = erts_alloc_get_verify_unused_temp_alloc( @@ -7664,7 +8053,7 @@ sched_thread_func(void *vesdp) process_main(); /* No schedulers should *ever* terminate */ - erl_exit(ERTS_ABORT_EXIT, + erts_exit(ERTS_ABORT_EXIT, "Scheduler thread number %beu terminated\n", no); return NULL; @@ -7687,6 +8076,8 @@ sched_dirty_cpu_thread_func(void *vesdp) callbacks.wait = NULL; callbacks.finalize_wait = NULL; + esdp->thr_id += erts_no_schedulers; + erts_thr_progress_register_unmanaged_thread(&callbacks); #ifdef ERTS_ENABLE_LOCK_CHECK { @@ -7707,27 +8098,9 @@ sched_dirty_cpu_thread_func(void *vesdp) #endif erts_thread_init_float(); - erts_smp_mtx_lock(&schdlr_sspnd.mtx); - ASSERT(erts_smp_atomic32_read_nob(&schdlr_sspnd.dirty_cpu_changing) - & ERTS_SCHDLR_SSPND_CHNG_ONLN); - - if (--schdlr_sspnd.dirty_cpu_curr_online == schdlr_sspnd.dirty_cpu_wait_curr_online) { - erts_smp_atomic32_read_band_nob(&schdlr_sspnd.dirty_cpu_changing, - ~ERTS_SCHDLR_SSPND_CHNG_ONLN); - if (no != 1) - erts_smp_cnd_broadcast(&schdlr_sspnd.cnd); - } - - if (no == 1) { - while (schdlr_sspnd.dirty_cpu_curr_online != schdlr_sspnd.dirty_cpu_wait_curr_online) - erts_smp_cnd_wait(&schdlr_sspnd.cnd, &schdlr_sspnd.mtx); - ERTS_SCHDLR_SSPND_DIRTY_CPU_CHNG_SET(0, ERTS_SCHDLR_SSPND_CHNG_WAITER); - } - erts_smp_mtx_unlock(&schdlr_sspnd.mtx); - process_main(); /* No schedulers should *ever* terminate */ - erl_exit(ERTS_ABORT_EXIT, + erts_exit(ERTS_ABORT_EXIT, "Dirty CPU scheduler thread number %beu terminated\n", no); return NULL; @@ -7748,6 +8121,8 @@ sched_dirty_io_thread_func(void *vesdp) callbacks.wait = NULL; callbacks.finalize_wait = NULL; + esdp->thr_id += erts_no_schedulers + erts_no_dirty_cpu_schedulers; + erts_thr_progress_register_unmanaged_thread(&callbacks); #ifdef ERTS_ENABLE_LOCK_CHECK { @@ -7768,27 +8143,9 @@ sched_dirty_io_thread_func(void *vesdp) #endif erts_thread_init_float(); - erts_smp_mtx_lock(&schdlr_sspnd.mtx); - ASSERT(erts_smp_atomic32_read_nob(&schdlr_sspnd.dirty_io_changing) - & ERTS_SCHDLR_SSPND_CHNG_ONLN); - - if (--schdlr_sspnd.dirty_io_curr_online == schdlr_sspnd.dirty_io_wait_curr_online) { - erts_smp_atomic32_read_band_nob(&schdlr_sspnd.dirty_io_changing, - ~ERTS_SCHDLR_SSPND_CHNG_ONLN); - if (no != 1) - erts_smp_cnd_broadcast(&schdlr_sspnd.cnd); - } - - if (no == 1) { - while (schdlr_sspnd.dirty_io_curr_online != schdlr_sspnd.dirty_io_wait_curr_online) - erts_smp_cnd_wait(&schdlr_sspnd.cnd, &schdlr_sspnd.mtx); - ERTS_SCHDLR_SSPND_DIRTY_IO_CHNG_SET(0, ERTS_SCHDLR_SSPND_CHNG_WAITER); - } - erts_smp_mtx_unlock(&schdlr_sspnd.mtx); - process_main(); /* No schedulers should *ever* terminate */ - erl_exit(ERTS_ABORT_EXIT, + erts_exit(ERTS_ABORT_EXIT, "Dirty I/O scheduler thread number %beu terminated\n", no); return NULL; @@ -7805,28 +8162,25 @@ erts_start_schedulers(void) Uint actual; Uint wanted = erts_no_schedulers; Uint wanted_no_schedulers = erts_no_schedulers; + char name[16]; ethr_thr_opts opts = ETHR_THR_OPTS_DEFAULT_INITER; opts.detached = 1; -#ifdef ETHR_HAVE_THREAD_NAMES - opts.name = malloc(80); -#endif + opts.name = name; #ifdef ERTS_SMP if (erts_runq_supervision_interval) { opts.suggested_stack_size = 16; -#ifdef ETHR_HAVE_THREAD_NAMES - sprintf(opts.name, "runq_supervisor"); -#endif + erts_snprintf(opts.name, 16, "runq_supervisor"); erts_atomic_init_nob(&runq_supervisor_sleeping, 0); if (0 != ethr_event_init(&runq_supervision_event)) - erl_exit(1, "Failed to create run-queue supervision event\n"); + erts_exit(ERTS_ERROR_EXIT, "Failed to create run-queue supervision event\n"); if (0 != ethr_thr_create(&runq_supervisor_tid, runq_supervisor, NULL, &opts)) - erl_exit(1, "Failed to create run-queue supervision thread\n"); + erts_exit(ERTS_ERROR_EXIT, "Failed to create run-queue supervision thread\n"); } #endif @@ -7845,14 +8199,7 @@ erts_start_schedulers(void) ASSERT(actual == esdp->no - 1); -#ifdef ETHR_HAVE_THREAD_NAMES - sprintf(opts.name, "scheduler_%d", actual + 1); -#endif - -#ifdef __OSE__ - /* This should be done in the bind strategy */ - opts.coreNo = (actual+1) % ose_num_cpus(); -#endif + erts_snprintf(opts.name, 16, "%lu_scheduler", actual + 1); res = ethr_thr_create(&esdp->tid, sched_thread_func, (void*)esdp, &opts); @@ -7860,7 +8207,6 @@ erts_start_schedulers(void) break; } } - erts_no_schedulers = actual; #ifdef ERTS_DIRTY_SCHEDULERS @@ -7869,21 +8215,17 @@ erts_start_schedulers(void) int ix; for (ix = 0; ix < erts_no_dirty_cpu_schedulers; ix++) { ErtsSchedulerData *esdp = ERTS_DIRTY_CPU_SCHEDULER_IX(ix); -#ifdef ETHR_HAVE_THREAD_NAMES - sprintf(opts.name,"dirty_cpu_scheduler_%d", ix + 1); -#endif + erts_snprintf(opts.name, 16, "%d_dirty_cpu_scheduler", ix + 1); res = ethr_thr_create(&esdp->tid,sched_dirty_cpu_thread_func,(void*)esdp,&opts); if (res != 0) - erl_exit(1, "Failed to create dirty cpu scheduler thread %d\n", ix); + erts_exit(ERTS_ERROR_EXIT, "Failed to create dirty cpu scheduler thread %d\n", ix); } for (ix = 0; ix < erts_no_dirty_io_schedulers; ix++) { ErtsSchedulerData *esdp = ERTS_DIRTY_IO_SCHEDULER_IX(ix); -#ifdef ETHR_HAVE_THREAD_NAMES - sprintf(opts.name,"dirty_io_scheduler_%d", ix + 1); -#endif + erts_snprintf(opts.name, 16, "%d_dirty_io_scheduler", ix + 1); res = ethr_thr_create(&esdp->tid,sched_dirty_io_thread_func,(void*)esdp,&opts); if (res != 0) - erl_exit(1, "Failed to create dirty io scheduler thread %d\n", ix); + erts_exit(ERTS_ERROR_EXIT, "Failed to create dirty io scheduler thread %d\n", ix); } } #endif @@ -7891,20 +8233,14 @@ erts_start_schedulers(void) ERTS_THR_MEMORY_BARRIER; -#ifdef ETHR_HAVE_THREAD_NAMES - sprintf(opts.name, "aux"); -#endif - -#ifdef __OSE__ - opts.coreNo = 0; -#endif /* __OSE__ */ + erts_snprintf(opts.name, 16, "aux"); res = ethr_thr_create(&aux_tid, aux_thread, NULL, &opts); if (res != 0) - erl_exit(1, "Failed to create aux thread\n"); + erts_exit(ERTS_ERROR_EXIT, "Failed to create aux thread\n"); if (actual < 1) - erl_exit(1, + erts_exit(ERTS_ERROR_EXIT, "Failed to create any scheduler-threads: %s (%d)\n", erl_errno_id(res), res); @@ -7918,10 +8254,6 @@ erts_start_schedulers(void) actual, actual == 1 ? " was" : "s were"); erts_send_error_to_logger_nogl(dsbufp); } - -#ifdef ETHR_HAVE_THREAD_NAMES - free(opts.name); -#endif } #endif /* ERTS_SMP */ @@ -7940,7 +8272,7 @@ add_pend_suspend(Process *suspendee, sizeof(ErtsPendingSuspend)); psp->next = NULL; #ifdef DEBUG -#if defined(ARCH_64) && !HALFWORD_HEAP +#if defined(ARCH_64) psp->end = (ErtsPendingSuspend *) 0xdeaddeaddeaddead; #else psp->end = (ErtsPendingSuspend *) 0xdeaddead; @@ -8023,7 +8355,8 @@ handle_pend_sync_suspend(Process *suspendee, } /* suspender is suspended waiting for suspendee to suspend; resume suspender */ - resume_process(suspender); + ASSERT(suspendee != suspender); + resume_process(suspender, ERTS_PROC_LOCK_STATUS); erts_smp_proc_unlock(suspender, ERTS_PROC_LOCK_STATUS); } } @@ -8058,12 +8391,11 @@ pid2proc_not_running(Process *c_p, ErtsProcLocks c_p_locks, ASSERT(c_p->flags & F_P2PNR_RESCHED); c_p->flags &= ~F_P2PNR_RESCHED; if (!suspend && rp) - resume_process(rp); + resume_process(rp, rp_locks); } else { - rp = erts_pid2proc(c_p, c_p_locks|ERTS_PROC_LOCK_STATUS, - pid, pid_locks|ERTS_PROC_LOCK_STATUS); + pid, ERTS_PROC_LOCK_STATUS); if (!rp) { c_p->flags &= ~F_P2PNR_RESCHED; @@ -8072,40 +8404,84 @@ pid2proc_not_running(Process *c_p, ErtsProcLocks c_p_locks, ASSERT(!(c_p->flags & F_P2PNR_RESCHED)); - if (suspend) { - if (suspend_process(c_p, rp)) - goto done; - } - else { - if (!((ERTS_PSFLG_RUNNING|ERTS_PSFLG_RUNNING_SYS) - & erts_smp_atomic32_read_acqb(&rp->state))) - goto done; + /* + * Suspend the other process in order to prevent + * it from being selected for normal execution. + * This will however not prevent it from being + * selected for execution of a system task. If + * it is selected for execution of a system task + * we might be blocked for quite a while if the + * try-lock below fails. That is, there is room + * for improvement here... + */ - } + if (!suspend_process(c_p, rp)) { + /* Other process running */ - /* Other process running */ + ASSERT(ERTS_PSFLG_RUNNING + & erts_smp_atomic32_read_nob(&rp->state)); - /* - * If we got pending suspenders and suspend ourselves waiting - * to suspend another process we might deadlock. - * In this case we have to yield, be suspended by - * someone else and then do it all over again. - */ - if (!c_p->pending_suspenders) { - /* Mark rp pending for suspend by c_p */ - add_pend_suspend(rp, c_p->common.id, handle_pend_sync_suspend); - ASSERT(is_nil(c_p->suspendee)); + running: - /* Suspend c_p; when rp is suspended c_p will be resumed. */ - suspend_process(c_p, c_p); - c_p->flags |= F_P2PNR_RESCHED; + /* + * If we got pending suspenders and suspend ourselves waiting + * to suspend another process we might deadlock. + * In this case we have to yield, be suspended by + * someone else and then do it all over again. + */ + if (!c_p->pending_suspenders) { + /* Mark rp pending for suspend by c_p */ + add_pend_suspend(rp, c_p->common.id, handle_pend_sync_suspend); + ASSERT(is_nil(c_p->suspendee)); + + /* Suspend c_p; when rp is suspended c_p will be resumed. */ + suspend_process(c_p, c_p); + c_p->flags |= F_P2PNR_RESCHED; + } + /* Yield (caller is assumed to yield immediately in bif). */ + erts_smp_proc_unlock(rp, ERTS_PROC_LOCK_STATUS); + rp = ERTS_PROC_LOCK_BUSY; + } + else { + ErtsProcLocks need_locks = pid_locks & ~ERTS_PROC_LOCK_STATUS; + if (need_locks && erts_smp_proc_trylock(rp, need_locks) == EBUSY) { + if (ERTS_PSFLG_RUNNING_SYS + & erts_smp_atomic32_read_nob(&rp->state)) { + /* Executing system task... */ + resume_process(rp, ERTS_PROC_LOCK_STATUS); + goto running; + } + erts_smp_proc_unlock(rp, ERTS_PROC_LOCK_STATUS); + /* + * If we are unlucky, the process just got selected for + * execution of a system task. In this case we may be + * blocked here for quite a while... Execution of system + * tasks are fortunately quite rare events. We try to + * avoid this by checking if it is in a state executing + * system tasks (above), but it will not prevent all + * scenarios for a long block here... + */ + rp = erts_pid2proc(c_p, c_p_locks|ERTS_PROC_LOCK_STATUS, + pid, pid_locks|ERTS_PROC_LOCK_STATUS); + if (!rp) + goto done; + } + + /* + * The previous suspend has prevented the process + * from being selected for normal execution regardless + * of locks held or not held on it... + */ + ASSERT(!(ERTS_PSFLG_RUNNING + & erts_smp_atomic32_read_nob(&rp->state))); + + if (!suspend) + resume_process(rp, pid_locks|ERTS_PROC_LOCK_STATUS); } - /* Yield (caller is assumed to yield immediately in bif). */ - erts_smp_proc_unlock(rp, pid_locks|ERTS_PROC_LOCK_STATUS); - rp = ERTS_PROC_LOCK_BUSY; } done: + if (rp && rp != ERTS_PROC_LOCK_BUSY && !(pid_locks & ERTS_PROC_LOCK_STATUS)) erts_smp_proc_unlock(rp, ERTS_PROC_LOCK_STATUS); if (unlock_c_p_status) @@ -8216,7 +8592,8 @@ handle_pend_bif_sync_suspend(Process *suspendee, } /* suspender is suspended waiting for suspendee to suspend; resume suspender */ - resume_process(suspender); + ASSERT(suspender != suspendee); + resume_process(suspender, ERTS_PROC_LOCK_LINK|ERTS_PROC_LOCK_STATUS); erts_smp_proc_unlock(suspender, ERTS_PROC_LOCK_LINK|ERTS_PROC_LOCK_STATUS); } @@ -8576,7 +8953,8 @@ resume_process_1(BIF_ALIST_1) ASSERT(ERTS_PSFLG_SUSPENDED & erts_smp_atomic32_read_nob(&suspendee->state)); - resume_process(suspendee); + ASSERT(BIF_P != suspendee); + resume_process(suspendee, ERTS_PROC_LOCK_STATUS); erts_smp_proc_unlock(suspendee, ERTS_PROC_LOCK_STATUS); } @@ -8598,24 +8976,39 @@ resume_process_1(BIF_ALIST_1) } Uint -erts_run_queues_len(Uint *qlen) +erts_run_queues_len(Uint *qlen, int atomic_queues_read, int incl_active_sched) { int i = 0; Uint len = 0; - ERTS_ATOMIC_FOREACH_RUNQ(rq, - { - Sint pqlen = 0; - int pix; - for (pix = 0; pix < ERTS_NO_PROC_PRIO_LEVELS; pix++) - pqlen += RUNQ_READ_LEN(&rq->procs.prio_info[pix].len); + if (atomic_queues_read) + ERTS_ATOMIC_FOREACH_RUNQ(rq, + { + Sint rq_len = (Sint) erts_smp_atomic32_read_dirty(&rq->len); + ASSERT(rq_len >= 0); + if (incl_active_sched + && (ERTS_RUNQ_FLGS_GET_NOB(rq) & ERTS_RUNQ_FLG_EXEC)) { + rq_len++; + } + if (qlen) + qlen[i++] = rq_len; + len += (Uint) rq_len; + } + ); + else { + for (i = 0; i < erts_no_run_queues; i++) { + ErtsRunQueue *rq = ERTS_RUNQ_IX(i); + Sint rq_len = (Sint) erts_smp_atomic32_read_nob(&rq->len); + ASSERT(rq_len >= 0); + if (incl_active_sched + && (ERTS_RUNQ_FLGS_GET_NOB(rq) & ERTS_RUNQ_FLG_EXEC)) { + rq_len++; + } + if (qlen) + qlen[i] = rq_len; + len += (Uint) rq_len; + } - if (pqlen < 0) - pqlen = 0; - if (qlen) - qlen[i++] = pqlen; - len += pqlen; } - ); return len; } @@ -8706,7 +9099,7 @@ erts_resume(Process* process, ErtsProcLocks process_locks) ERTS_SMP_LC_ASSERT(process_locks == erts_proc_lc_my_proc_locks(process)); if (!(process_locks & ERTS_PROC_LOCK_STATUS)) erts_smp_proc_lock(process, ERTS_PROC_LOCK_STATUS); - resume_process(process); + resume_process(process, process_locks|ERTS_PROC_LOCK_STATUS); if (!(process_locks & ERTS_PROC_LOCK_STATUS)) erts_smp_proc_unlock(process, ERTS_PROC_LOCK_STATUS); } @@ -8725,7 +9118,7 @@ erts_resume_processes(ErtsProcList *list) proc = erts_pid2proc(NULL, 0, plp->pid, ERTS_PROC_LOCK_STATUS); if (proc) { if (erts_proclist_same(plp, proc)) { - resume_process(proc); + resume_process(proc, ERTS_PROC_LOCK_STATUS); nresumed++; } erts_smp_proc_unlock(proc, ERTS_PROC_LOCK_STATUS); @@ -8828,6 +9221,10 @@ erts_set_process_priority(Process *p, Eterm value) a = erts_smp_atomic32_cmpxchg_mb(&p->state, n, e); } while (a != e); + + if (slocked) + erts_smp_proc_unlock(p, ERTS_PROC_LOCK_STATUS); + } switch (oprio) { @@ -8861,7 +9258,6 @@ Process *schedule(Process *p, int calls) { Process *proxy_p = NULL; ErtsRunQueue *rq; - erts_aint_t dt; ErtsSchedulerData *esdp; int context_reds; int fcalls; @@ -8871,6 +9267,8 @@ Process *schedule(Process *p, int calls) Uint32 flags; erts_aint32_t state = 0; /* Supress warning... */ + ERTS_MSACC_DECLARE_CACHE(); + #ifdef USE_VM_PROBES if (p != NULL && DTRACE_ENABLED(process_unscheduled)) { DTRACE_CHARBUF(process_buf, DTRACE_TERM_BUF_SIZE); @@ -8961,7 +9359,7 @@ Process *schedule(Process *p, int calls) schedule_out_process(rq, state, p, proxy_p); /* Returns with rq locked! */ proxy_p = NULL; - ERTS_PROC_REDUCTIONS_EXECUTED(rq, + ERTS_PROC_REDUCTIONS_EXECUTED(esdp, rq, (int) ERTS_PSFLGS_GET_USR_PRIO(state), reds, actual_reds); @@ -8973,17 +9371,16 @@ Process *schedule(Process *p, int calls) erts_smp_proc_unlock(p, ERTS_PROC_LOCK_MAIN|ERTS_PROC_LOCK_STATUS); + ERTS_MSACC_SET_STATE_CACHED_M(ERTS_MSACC_STATE_OTHER); + if (state & ERTS_PSFLG_FREE) { #ifdef ERTS_SMP ASSERT(esdp->free_process == p); esdp->free_process = NULL; #else - state = erts_smp_atomic32_read_nob(&p->state); - if (!(state & ERTS_PSFLG_IN_RUNQ)) - erts_free_proc(p); + erts_proc_dec_refc(p); #endif } - #ifdef ERTS_SMP ASSERT(!esdp->free_process); #endif @@ -8991,11 +9388,15 @@ Process *schedule(Process *p, int calls) ERTS_SMP_CHK_NO_PROC_LOCKS; - dt = erts_do_time_read_and_reset(); - if (dt) { - erts_smp_runq_unlock(rq); - erts_bump_timer(dt); - erts_smp_runq_lock(rq); + if (!ERTS_SCHEDULER_IS_DIRTY(esdp)) { + if (esdp->check_time_reds >= ERTS_CHECK_TIME_REDS) + (void) erts_get_monotonic_time(esdp); + + if (esdp->last_monotonic_time >= erts_next_timeout_time(esdp->next_tmo_ref)) { + erts_smp_runq_unlock(rq); + erts_bump_timers(esdp->timer_wheel, esdp->last_monotonic_time); + erts_smp_runq_lock(rq); + } } BM_STOP_TIMER(system); @@ -9008,15 +9409,16 @@ Process *schedule(Process *p, int calls) #ifdef ERTS_SMP ErtsMigrationPaths *mps; ErtsMigrationPath *mp; - ErtsProcList *pnd_xtrs = rq->procs.pending_exiters; - if (erts_proclist_fetch(&pnd_xtrs, NULL)) { - rq->procs.pending_exiters = NULL; - erts_smp_runq_unlock(rq); - handle_pending_exiters(pnd_xtrs); - erts_smp_runq_lock(rq); - } if (!ERTS_SCHEDULER_IS_DIRTY(esdp)) { + ErtsProcList *pnd_xtrs = rq->procs.pending_exiters; + if (erts_proclist_fetch(&pnd_xtrs, NULL)) { + rq->procs.pending_exiters = NULL; + erts_smp_runq_unlock(rq); + handle_pending_exiters(pnd_xtrs); + erts_smp_runq_lock(rq); + } + if (rq->check_balance_reds <= 0) check_balance(rq); @@ -9038,8 +9440,10 @@ Process *schedule(Process *p, int calls) if (flags & (ERTS_RUNQ_FLG_CHK_CPU_BIND|ERTS_RUNQ_FLG_SUSPENDED)) { if (flags & ERTS_RUNQ_FLG_SUSPENDED) { + (void) ERTS_RUNQ_FLGS_UNSET_NOB(rq, ERTS_RUNQ_FLG_EXEC); suspend_scheduler(esdp); - flags = ERTS_RUNQ_FLGS_GET_NOB(rq); + flags = ERTS_RUNQ_FLGS_SET_NOB(rq, ERTS_RUNQ_FLG_EXEC); + flags |= ERTS_RUNQ_FLG_EXEC; } if (flags & ERTS_RUNQ_FLG_CHK_CPU_BIND) { flags = ERTS_RUNQ_FLGS_UNSET(rq, ERTS_RUNQ_FLG_CHK_CPU_BIND); @@ -9054,24 +9458,20 @@ Process *schedule(Process *p, int calls) suspend_scheduler(esdp); #endif - { + if (!ERTS_SCHEDULER_IS_DIRTY(esdp)) { erts_aint32_t aux_work; - int leader_update = ERTS_SCHEDULER_IS_DIRTY(esdp) ? 0 - : erts_thr_progress_update(esdp); + int leader_update = erts_thr_progress_update(esdp); aux_work = erts_atomic32_read_acqb(&esdp->ssi->aux_work); - if (aux_work | leader_update | ERTS_SCHED_FAIR) { + if (aux_work | leader_update) { erts_smp_runq_unlock(rq); if (leader_update) erts_thr_progress_leader_update(esdp); - else if (ERTS_SCHED_FAIR) - ERTS_SCHED_FAIR_YIELD(); if (aux_work) handle_aux_work(&esdp->aux_work_data, aux_work, 0); erts_smp_runq_lock(rq); } - ERTS_SMP_LC_ASSERT(ERTS_SCHEDULER_IS_DIRTY(esdp) - || !erts_thr_progress_is_blocking()); + ERTS_SMP_LC_ASSERT(!erts_thr_progress_is_blocking()); } ERTS_SMP_LC_ASSERT(erts_smp_lc_runq_is_locked(rq)); @@ -9130,8 +9530,11 @@ Process *schedule(Process *p, int calls) } #endif + (void) ERTS_RUNQ_FLGS_UNSET(rq, ERTS_RUNQ_FLG_EXEC); scheduler_wait(&fcalls, esdp, rq); - + flags = ERTS_RUNQ_FLGS_SET_NOB(rq, ERTS_RUNQ_FLG_EXEC); + flags |= ERTS_RUNQ_FLG_EXEC; + ERTS_MSACC_UPDATE_CACHE(); #ifdef ERTS_SMP non_empty_runq(rq); #endif @@ -9140,23 +9543,28 @@ Process *schedule(Process *p, int calls) } else if (!ERTS_SCHEDULER_IS_DIRTY(esdp) && (fcalls > input_reductions && - prepare_for_sys_schedule(esdp))) { + prepare_for_sys_schedule(!0))) { + ErtsMonotonicTime current_time; /* * Schedule system-level activities. */ + ERTS_MSACC_PUSH_STATE_CACHED_M(); + erts_smp_atomic32_set_relb(&function_calls, 0); fcalls = 0; - ASSERT(!erts_port_task_have_outstanding_io_tasks()); - #if 0 /* Not needed since we wont wait in sys schedule */ erts_sys_schedule_interrupt(0); #endif erts_smp_runq_unlock(rq); + ERTS_MSACC_SET_STATE_CACHED_M(ERTS_MSACC_STATE_CHECK_IO); erl_sys_schedule(1); - dt = erts_do_time_read_and_reset(); - if (dt) erts_bump_timer(dt); + ERTS_MSACC_POP_STATE_M(); + + current_time = erts_get_monotonic_time(esdp); + if (current_time >= erts_next_timeout_time(esdp->next_tmo_ref)) + erts_bump_timers(esdp->timer_wheel, current_time); #ifdef ERTS_SMP erts_smp_runq_lock(rq); @@ -9181,7 +9589,9 @@ Process *schedule(Process *p, int calls) if (RUNQ_READ_LEN(&rq->ports.info.len)) { int have_outstanding_io; have_outstanding_io = erts_port_task_execute(rq, &esdp->current_port); - if ((have_outstanding_io && fcalls > 2*input_reductions) + if ((!erts_eager_check_io + && have_outstanding_io + && fcalls > 2*input_reductions) || rq->halt_in_progress) { /* * If we have performed more than 2*INPUT_REDUCTIONS since @@ -9208,11 +9618,12 @@ Process *schedule(Process *p, int calls) pick_next_process: { erts_aint32_t psflg_band_mask; int prio_q; - int qmask; + int qmask, qbit; flags = ERTS_RUNQ_FLGS_GET_NOB(rq); qmask = (int) (flags & ERTS_RUNQ_FLGS_PROCS_QMASK); - switch (qmask & -qmask) { + qbit = qmask & -qmask; + switch (qbit) { case MAX_BIT: prio_q = PRIORITY_MAX; break; @@ -9228,9 +9639,12 @@ Process *schedule(Process *p, int calls) case 0: /* No process at all */ default: ASSERT(qmask == 0); + ERTS_MSACC_SET_STATE_CACHED_M(ERTS_MSACC_STATE_OTHER); goto check_activities_to_run; } + ERTS_MSACC_SET_STATE_CACHED_M(ERTS_MSACC_STATE_EMULATOR); + BM_START_TIMER(system); /* @@ -9240,20 +9654,11 @@ Process *schedule(Process *p, int calls) ASSERT(p); /* Wrong qmask in rq->flags? */ - psflg_band_mask = ~(((erts_aint32_t) 1) << (ERTS_PSFLGS_GET_PRQ_PRIO(state) - + ERTS_PSFLGS_IN_PRQ_MASK_OFFSET)); - -#ifdef ERTS_DIRTY_SCHEDULERS - ASSERT((state & (ERTS_PSFLG_DIRTY_CPU_PROC|ERTS_PSFLG_DIRTY_IO_PROC)) != - (ERTS_PSFLG_DIRTY_CPU_PROC|ERTS_PSFLG_DIRTY_IO_PROC)); - if (state & (ERTS_PSFLG_DIRTY_CPU_PROC|ERTS_PSFLG_DIRTY_IO_PROC)) { - ASSERT((ERTS_SCHEDULER_IS_DIRTY_CPU(esdp) && (state & ERTS_PSFLG_DIRTY_CPU_PROC)) || - (ERTS_SCHEDULER_IS_DIRTY_IO(esdp) && (state & ERTS_PSFLG_DIRTY_IO_PROC))); - if (!ERTS_SCHEDULER_IS_DIRTY(esdp) && !(state & ERTS_PSFLG_ACTIVE_SYS)) - goto pick_next_process; - state &= ~(ERTS_PSFLG_DIRTY_CPU_PROC_IN_Q|ERTS_PSFLG_DIRTY_IO_PROC_IN_Q); - } -#endif + if (ERTS_SCHEDULER_IS_DIRTY(esdp)) + psflg_band_mask = ~((erts_aint32_t) 0); + else + psflg_band_mask = ~(((erts_aint32_t) 1) << (ERTS_PSFLGS_GET_PRQ_PRIO(state) + + ERTS_PSFLGS_IN_PRQ_MASK_OFFSET)); if (!(state & ERTS_PSFLG_PROXY)) psflg_band_mask &= ~ERTS_PSFLG_IN_RUNQ; @@ -9276,9 +9681,11 @@ Process *schedule(Process *p, int calls) | ERTS_PSFLG_RUNNING_SYS))) { tmp = state & (ERTS_PSFLG_SUSPENDED | ERTS_PSFLG_PENDING_EXIT - | ERTS_PSFLG_ACTIVE_SYS); + | ERTS_PSFLG_ACTIVE_SYS + | ERTS_PSFLG_DIRTY_ACTIVE_SYS); if (tmp != ERTS_PSFLG_SUSPENDED) { - if (state & ERTS_PSFLG_ACTIVE_SYS) + if (state & (ERTS_PSFLG_ACTIVE_SYS + | ERTS_PSFLG_DIRTY_ACTIVE_SYS)) new |= ERTS_PSFLG_RUNNING_SYS; else new |= ERTS_PSFLG_RUNNING; @@ -9291,7 +9698,8 @@ Process *schedule(Process *p, int calls) | ERTS_PSFLG_FREE)) || ((state & (ERTS_PSFLG_SUSPENDED | ERTS_PSFLG_PENDING_EXIT - | ERTS_PSFLG_ACTIVE_SYS)) + | ERTS_PSFLG_ACTIVE_SYS + | ERTS_PSFLG_DIRTY_ACTIVE_SYS)) == ERTS_PSFLG_SUSPENDED)) { if (proxy_p) { free_proxy_proc(proxy_p); @@ -9299,11 +9707,7 @@ Process *schedule(Process *p, int calls) } else if (state & ERTS_PSFLG_FREE) { /* free and not queued by proxy */ -#ifdef ERTS_SMP - erts_smp_proc_dec_refc(p); -#else - erts_free_proc(p); -#endif + erts_proc_dec_refc(p); } goto pick_next_process; } @@ -9316,10 +9720,42 @@ Process *schedule(Process *p, int calls) esdp->current_process = p; + + reds = context_reds; + +#ifdef ERTS_SMP + + erts_smp_runq_unlock(rq); + +#ifdef ERTS_DIRTY_SCHEDULERS + if (ERTS_SCHEDULER_IS_DIRTY(esdp)) { +#ifdef DEBUG + int old_dqbit; +#endif + int dqbit = qbit; + + if (rq == ERTS_DIRTY_CPU_RUNQ) + dqbit <<= ERTS_PDSFLGS_IN_CPU_PRQ_MASK_OFFSET; + else { + ASSERT(rq == ERTS_DIRTY_IO_RUNQ); + dqbit <<= ERTS_PDSFLGS_IN_IO_PRQ_MASK_OFFSET; + } + +#ifdef DEBUG + old_dqbit = (int) +#else + (void) +#endif + erts_smp_atomic32_read_band_mb(&p->dirty_state, ~dqbit); + ASSERT(old_dqbit & dqbit); + } +#endif /* ERTS_DIRTY_SCHEDULERS */ + +#endif /* ERTS_SMP */ + } #ifdef ERTS_SMP - erts_smp_runq_unlock(rq); if (flags & ERTS_RUNQ_FLG_PROTECTED) (void) ERTS_RUNQ_FLGS_UNSET(rq, ERTS_RUNQ_FLG_PROTECTED); @@ -9328,12 +9764,11 @@ Process *schedule(Process *p, int calls) erts_smp_proc_lock(p, ERTS_PROC_LOCK_MAIN|ERTS_PROC_LOCK_STATUS); + state = erts_smp_atomic32_read_nob(&p->state); + if (erts_sched_stat.enabled) { int prio; - UWord old = ERTS_PROC_SCHED_ID(p, - (ERTS_PROC_LOCK_MAIN - | ERTS_PROC_LOCK_STATUS), - (UWord) esdp->no); + UWord old = ERTS_PROC_SCHED_ID(p, (UWord) esdp->no); int migrated = old && old != esdp->no; #ifdef ERTS_DIRTY_SCHEDULERS @@ -9352,15 +9787,56 @@ Process *schedule(Process *p, int calls) erts_smp_spin_unlock(&erts_sched_stat.lock); } - if (ERTS_PROC_PENDING_EXIT(p)) { + ASSERT(!p->scheduler_data); + p->scheduler_data = esdp; + + state = erts_smp_atomic32_read_nob(&p->state); + +#ifdef ERTS_DIRTY_SCHEDULERS + if (!ERTS_SCHEDULER_IS_DIRTY(esdp)) { + if (!!(state & ERTS_PSFLGS_DIRTY_WORK) + & !(state & ERTS_PSFLG_ACTIVE_SYS)) { + /* Migrate to dirty scheduler... */ + sunlock_sched_out_proc: + erts_smp_proc_unlock(p, ERTS_PROC_LOCK_STATUS); + p->fcalls = reds; + goto sched_out_proc; + + } + } + else { + if (state & (ERTS_PSFLG_ACTIVE_SYS + | ERTS_PSFLG_PENDING_EXIT + | ERTS_PSFLG_EXITING)) { + /* Migrate to normal scheduler... */ + goto sunlock_sched_out_proc; + } + if ((state & ERTS_PSFLG_DIRTY_ACTIVE_SYS) + && rq == ERTS_DIRTY_IO_RUNQ) { + /* Migrate to dirty cpu scheduler... */ + goto sunlock_sched_out_proc; + } + + ASSERT((state & ERTS_PSFLG_DIRTY_ACTIVE_SYS) + || *p->i == (BeamInstr) em_call_nif); + + ASSERT(rq == ERTS_DIRTY_CPU_RUNQ + ? (state & (ERTS_PSFLG_DIRTY_CPU_PROC + | ERTS_PSFLG_DIRTY_ACTIVE_SYS)) + : (rq == ERTS_DIRTY_IO_RUNQ + && (state & ERTS_PSFLG_DIRTY_IO_PROC))); + } +#endif + + if (state & ERTS_PSFLG_PENDING_EXIT) { erts_handle_pending_exit(p, ERTS_PROC_LOCK_MAIN|ERTS_PROC_LOCK_STATUS); state = erts_smp_atomic32_read_nob(&p->state); } - ASSERT(!p->scheduler_data); - p->scheduler_data = esdp; -#endif - reds = context_reds; + +#endif /* ERTS_SMP */ + + p->fcalls = reds; if (IS_TRACED(p)) { if (state & ERTS_PSFLG_EXITING) { @@ -9386,14 +9862,24 @@ Process *schedule(Process *p, int calls) #endif if (state & ERTS_PSFLG_RUNNING_SYS) { - reds -= execute_sys_tasks(p, &state, reds); - if (reds <= 0 + /* + * GC is normally never delayed when a process + * is scheduled out, but might be when executing + * hand written beam assembly in + * prim_eval:'receive'. If GC is delayed we are + * not allowed to execute system tasks. + */ + if (!(p->flags & F_DELAY_GC)) { + reds -= execute_sys_tasks(p, &state, reds); + if (reds <= 0 #ifdef ERTS_DIRTY_SCHEDULERS - || (state & (ERTS_PSFLG_DIRTY_CPU_PROC|ERTS_PSFLG_DIRTY_IO_PROC)) + || ERTS_SCHEDULER_IS_DIRTY(esdp) + || (state & ERTS_PSFLGS_DIRTY_WORK) #endif - ) { - p->fcalls = reds; - goto sched_out_proc; + ) { + p->fcalls = reds; + goto sched_out_proc; + } } ASSERT(state & ERTS_PSFLG_RUNNING_SYS); @@ -9422,13 +9908,13 @@ Process *schedule(Process *p, int calls) } } - if (!(state & ERTS_PSFLG_EXITING) - && ((FLAGS(p) & F_FORCE_GC) - || (MSO(p).overhead > BIN_VHEAP_SZ(p)))) { - reds -= erts_garbage_collect(p, 0, p->arg_reg, p->arity); - if (reds <= 0) { - p->fcalls = reds; - goto sched_out_proc; + if (ERTS_IS_GC_DESIRED(p)) { + if (!(state & ERTS_PSFLG_EXITING) && !(p->flags & (F_DELAY_GC|F_DISABLE_GC))) { + reds -= erts_garbage_collect_nobump(p, 0, p->arg_reg, p->arity); + if (reds <= 0) { + p->fcalls = reds; + goto sched_out_proc; + } } } @@ -9437,12 +9923,25 @@ Process *schedule(Process *p, int calls) proxy_p = NULL; } - p->fcalls = reds; ERTS_SMP_CHK_HAVE_ONLY_MAIN_PROC_LOCK(p); /* Never run a suspended process */ ASSERT(!(ERTS_PSFLG_SUSPENDED & erts_smp_atomic32_read_nob(&p->state))); + ASSERT(erts_proc_read_refc(p) > 0); + + if (!(state & ERTS_PSFLG_EXITING) && ERTS_PTMR_IS_TIMED_OUT(p)) { + BeamInstr** pi; +#ifdef ERTS_SMP + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore); +#endif + pi = (BeamInstr **) p->def_arg_reg; + p->i = *pi; + p->flags &= ~F_INSLPQUEUE; + p->flags |= F_TIMO; + ERTS_PTMR_CLEAR(p); + } + return p; } } @@ -9454,7 +9953,7 @@ notify_sys_task_executed(Process *c_p, ErtsProcSysTask *st, Eterm st_result) if (rp) { ErtsProcLocks rp_locks; ErlOffHeap *ohp; - ErlHeapFragment* bp; + ErtsMessage *mp; Eterm *hp, msg, req_id, result; Uint st_result_sz, hsz; #ifdef DEBUG @@ -9466,11 +9965,7 @@ notify_sys_task_executed(Process *c_p, ErtsProcSysTask *st, Eterm st_result) st_result_sz = is_immed(st_result) ? 0 : size_object(st_result); hsz = st->req_id_sz + st_result_sz + 4 /* 3-tuple */; - hp = erts_alloc_message_heap(hsz, - &bp, - &ohp, - rp, - &rp_locks); + mp = erts_alloc_message_heap(rp, &rp_locks, hsz, &hp, &ohp); #ifdef DEBUG hp_start = hp; @@ -9495,15 +9990,7 @@ notify_sys_task_executed(Process *c_p, ErtsProcSysTask *st, Eterm st_result) ASSERT(hp_start + hsz == hp); #endif - erts_queue_message(rp, - &rp_locks, - bp, - msg, - NIL -#ifdef USE_VM_PROBES - , NIL -#endif - ); + erts_queue_message(rp, &rp_locks, mp, msg, NIL); if (c_p == rp) rp_locks &= ~ERTS_PROC_LOCK_MAIN; @@ -9705,10 +10192,10 @@ execute_sys_tasks(Process *c_p, erts_aint32_t *statep, int in_reds) else { if (!garbage_collected) { FLAGS(c_p) |= F_NEED_FULLSWEEP; - reds += erts_garbage_collect(c_p, - 0, - c_p->arg_reg, - c_p->arity); + reds += erts_garbage_collect_nobump(c_p, + 0, + c_p->arg_reg, + c_p->arity); garbage_collected = 1; } st_res = am_true; @@ -9717,7 +10204,7 @@ execute_sys_tasks(Process *c_p, erts_aint32_t *statep, int in_reds) case ERTS_PSTT_CPC: st_res = erts_check_process_code(c_p, st->arg[0], - st->arg[1] == am_true, + unsigned_val(st->arg[1]), &reds); if (is_non_value(st_res)) { /* Needed gc, but gc was disabled */ @@ -9725,6 +10212,10 @@ execute_sys_tasks(Process *c_p, erts_aint32_t *statep, int in_reds) st = NULL; } break; + case ERTS_PSTT_COHMQ: + reds += erts_complete_off_heap_message_queue_change(c_p); + st_res = am_true; + break; default: ERTS_INTERNAL_ERROR("Invalid process sys task type"); st_res = am_false; @@ -9767,6 +10258,9 @@ cleanup_sys_tasks(Process *c_p, erts_aint32_t in_state, int in_reds) case ERTS_PSTT_CPC: st_res = am_false; break; + case ERTS_PSTT_COHMQ: + st_res = am_false; + break; default: ERTS_INTERNAL_ERROR("Invalid process sys task type"); st_res = am_false; @@ -9785,10 +10279,8 @@ BIF_RETTYPE erts_internal_request_system_task_3(BIF_ALIST_3) { Process *rp = erts_proc_lookup(BIF_ARG_1); - ErtsProcSysTaskQs *stqs, *free_stqs = NULL; ErtsProcSysTask *st = NULL; - erts_aint32_t prio, rp_state; - int rp_locked; + erts_aint32_t prio; Eterm noproc_res, req_type; if (!rp && !is_internal_pid(BIF_ARG_1)) { @@ -9845,7 +10337,6 @@ erts_internal_request_system_task_3(BIF_ALIST_3) } st = erts_alloc(ERTS_ALC_T_PROC_SYS_TSK, ERTS_PROC_SYS_TASK_SIZE(tot_sz)); - st->next = st->prev = st; /* Prep for empty prio queue */ ERTS_INIT_OFF_HEAP(&st->off_heap); hp = &st->heap[0]; @@ -9877,7 +10368,7 @@ erts_internal_request_system_task_3(BIF_ALIST_3) case am_check_process_code: if (is_not_atom(st->arg[0])) goto badarg; - if (st->arg[1] != am_true && st->arg[1] != am_false) + if (is_not_small(st->arg[1]) || (unsigned_val(st->arg[1]) & ~ERTS_CPC_ALL)) goto badarg; noproc_res = am_false; st->type = ERTS_PSTT_CPC; @@ -9889,93 +10380,11 @@ erts_internal_request_system_task_3(BIF_ALIST_3) goto badarg; } - rp_state = erts_smp_atomic32_read_nob(&rp->state); - - rp_locked = 0; - - free_stqs = NULL; - if (rp_state & ERTS_PSFLG_ACTIVE_SYS) - stqs = NULL; - else { - alloc_qs: - stqs = proc_sys_task_queues_alloc(); - stqs->qmask = 1 << prio; - stqs->ncount = 0; - stqs->q[PRIORITY_MAX] = NULL; - stqs->q[PRIORITY_HIGH] = NULL; - stqs->q[PRIORITY_NORMAL] = NULL; - stqs->q[PRIORITY_LOW] = NULL; - stqs->q[prio] = st; - } - - if (!rp_locked) { - rp_locked = 1; - erts_smp_proc_lock(rp, ERTS_PROC_LOCK_STATUS); - - rp_state = erts_smp_atomic32_read_nob(&rp->state); - if (rp_state & ERTS_PSFLG_EXITING) { - erts_smp_proc_unlock(rp, ERTS_PROC_LOCK_STATUS); - rp = NULL; - free_stqs = stqs; - goto noproc; - } - } - - if (!rp->sys_task_qs) { - if (stqs) - rp->sys_task_qs = stqs; - else - goto alloc_qs; - } - else { - if (stqs) - free_stqs = stqs; - stqs = rp->sys_task_qs; - if (!stqs->q[prio]) { - stqs->q[prio] = st; - stqs->qmask |= 1 << prio; - } - else { - st->next = stqs->q[prio]; - st->prev = stqs->q[prio]->prev; - st->next->prev = st; - st->prev->next = st; - ASSERT(stqs->qmask & (1 << prio)); - } - } - - if (ERTS_PSFLGS_GET_ACT_PRIO(rp_state) > prio) { - erts_aint32_t n, a, e; - /* Need to elevate actual prio */ - - a = rp_state; - do { - if (ERTS_PSFLGS_GET_ACT_PRIO(a) <= prio) { - n = a; - break; - } - n = e = a; - n &= ~ERTS_PSFLGS_ACT_PRIO_MASK; - n |= (prio << ERTS_PSFLGS_ACT_PRIO_OFFSET); - a = erts_smp_atomic32_cmpxchg_nob(&rp->state, n, e); - } while (a != e); - rp_state = n; + if (!schedule_process_sys_task(rp, prio, st)) { + noproc: + notify_sys_task_executed(BIF_P, st, noproc_res); } - erts_smp_proc_unlock(rp, ERTS_PROC_LOCK_STATUS); - - schedule_process_sys_task(rp, rp_state, NULL); - - if (free_stqs) - proc_sys_task_queues_free(free_stqs); - - BIF_RET(am_ok); - -noproc: - - notify_sys_task_executed(BIF_P, st, noproc_res); - if (free_stqs) - proc_sys_task_queues_free(free_stqs); BIF_RET(am_ok); badarg: @@ -9984,11 +10393,35 @@ badarg: erts_cleanup_offheap(&st->off_heap); erts_free(ERTS_ALC_T_PROC_SYS_TSK, st); } - if (free_stqs) - proc_sys_task_queues_free(free_stqs); BIF_ERROR(BIF_P, BADARG); } +void +erts_schedule_complete_off_heap_message_queue_change(Eterm pid) +{ + Process *rp = erts_proc_lookup(pid); + if (rp) { + ErtsProcSysTask *st; + erts_aint32_t state; + int i; + + st = erts_alloc(ERTS_ALC_T_PROC_SYS_TSK, + ERTS_PROC_SYS_TASK_SIZE(0)); + st->type = ERTS_PSTT_COHMQ; + st->requester = NIL; + st->reply_tag = NIL; + st->req_id = NIL; + st->req_id_sz = 0; + for (i = 0; i < ERTS_MAX_PROC_SYS_TASK_ARGS; i++) + st->arg[i] = NIL; + ERTS_INIT_OFF_HEAP(&st->off_heap); + state = erts_smp_atomic32_read_nob(&rp->state); + + if (!schedule_process_sys_task(rp, ERTS_PSFLGS_GET_USR_PRIO(state), st)) + erts_free(ERTS_ALC_T_PROC_SYS_TSK, st); + } +} + static void save_gc_task(Process *c_p, ErtsProcSysTask *st, int prio) { @@ -10008,7 +10441,7 @@ save_gc_task(Process *c_p, ErtsProcSysTask *st, int prio) qs->q[PRIORITY_NORMAL] = NULL; qs->q[PRIORITY_LOW] = NULL; qs->q[prio] = st; - (void) ERTS_PROC_SET_DELAYED_GC_TASK_QS(c_p, ERTS_PROC_LOCK_MAIN, qs); + (void) ERTS_PROC_SET_DELAYED_GC_TASK_QS(c_p, qs); } else { if (!qs->q[prio]) { @@ -10155,7 +10588,7 @@ erts_set_gc_state(Process *c_p, int enable) erts_smp_proc_unlock(c_p, ERTS_PROC_LOCK_STATUS); - (void) ERTS_PROC_SET_DELAYED_GC_TASK_QS(c_p, ERTS_PROC_LOCK_MAIN, NULL); + (void) ERTS_PROC_SET_DELAYED_GC_TASK_QS(c_p, NULL); if (dgc_tsk_qs) proc_sys_task_queues_free(dgc_tsk_qs); @@ -10357,6 +10790,8 @@ erts_free_proc(Process *p) #ifdef ERTS_SMP erts_proc_lock_fin(p); #endif + ASSERT(erts_smp_atomic32_read_nob(&p->state) & ERTS_PSFLG_FREE); + ASSERT(0 == erts_proc_read_refc(p)); erts_free(ERTS_ALC_T_PROC, (void *) p); } @@ -10372,6 +10807,9 @@ static void early_init_process_struct(void *varg, Eterm data) Process *proc = arg->proc; proc->common.id = make_internal_pid(data); +#ifdef ERTS_DIRTY_SCHEDULERS + erts_smp_atomic32_init_nob(&proc->dirty_state, 0); +#endif erts_smp_atomic32_init_relb(&proc->state, arg->state); #ifdef ERTS_SMP @@ -10399,7 +10837,7 @@ alloc_process(ErtsRunQueue *rq, erts_aint32_t state) init_arg.run_queue = rq; init_arg.state = state; - ASSERT(((char *) p) == ((char *) &p->common)); + ERTS_CT_ASSERT(offsetof(Process,common) == 0); if (!erts_ptab_new_element(&erts_proc, &p->common, @@ -10409,6 +10847,8 @@ alloc_process(ErtsRunQueue *rq, erts_aint32_t state) return NULL; } + ASSERT(erts_proc_read_refc(p) > 0); + ASSERT(internal_pid_serial(p->common.id) <= ERTS_MAX_PID_SERIAL); p->approx_started = erts_get_approx_time(); @@ -10430,6 +10870,7 @@ erl_create_process(Process* parent, /* Parent of process (default group leader). Eterm args, /* Arguments for function (must be well-formed list). */ ErlSpawnOpts* so) /* Options for spawn. */ { + Uint flags = 0; ErtsRunQueue *rq = NULL; Process *p; Sint arity; /* Number of arguments. */ @@ -10439,6 +10880,10 @@ erl_create_process(Process* parent, /* Parent of process (default group leader). Eterm res = THE_NON_VALUE; erts_aint32_t state = 0; erts_aint32_t prio = (erts_aint32_t) PRIORITY_NORMAL; +#ifdef SHCOPY_SPAWN + erts_shcopy_t info; + INITIALIZE_SHCOPY(info); +#endif #ifdef ERTS_SMP erts_smp_proc_lock(parent, ERTS_PROC_LOCKS_ALL_MINOR); @@ -10458,6 +10903,7 @@ erl_create_process(Process* parent, /* Parent of process (default group leader). int ix = so->scheduler-1; ASSERT(0 <= ix && ix < erts_no_run_queues); rq = ERTS_RUNQ_IX(ix); + /* Unsupported feature... */ state |= ERTS_PSFLG_BOUND; } prio = (erts_aint32_t) so->priority; @@ -10466,6 +10912,15 @@ erl_create_process(Process* parent, /* Parent of process (default group leader). state |= (((prio & ERTS_PSFLGS_PRIO_MASK) << ERTS_PSFLGS_ACT_PRIO_OFFSET) | ((prio & ERTS_PSFLGS_PRIO_MASK) << ERTS_PSFLGS_USR_PRIO_OFFSET)); + if (so->flags & SPO_OFF_HEAP_MSGQ) { + state |= ERTS_PSFLG_OFF_HEAP_MSGQ; + flags |= F_OFF_HEAP_MSGQ; + } + else if (so->flags & SPO_ON_HEAP_MSGQ) { + state |= ERTS_PSFLG_ON_HEAP_MSGQ; + flags |= F_ON_HEAP_MSGQ; + } + if (!rq) rq = erts_get_runq_proc(parent); @@ -10484,12 +10939,19 @@ erl_create_process(Process* parent, /* Parent of process (default group leader). BM_COUNT(processes_spawned); BM_SWAP_TIMER(system,size); +#ifdef SHCOPY_SPAWN + arg_size = copy_shared_calculate(args, &info); +#else arg_size = size_object(args); +#endif BM_SWAP_TIMER(size,system); heap_need = arg_size; - p->flags = erts_default_process_flags; + p->flags = flags; + p->static_flags = 0; + if (so->flags & SPO_SYSTEM_PROC) + p->static_flags |= ERTS_STC_FLG_SYSTEM_PROC; if (so->flags & SPO_USE_ARGS) { p->min_heap_size = so->min_heap_size; p->min_vheap_size = so->min_vheap_size; @@ -10502,9 +10964,9 @@ erl_create_process(Process* parent, /* Parent of process (default group leader). p->schedule_count = 0; ASSERT(p->min_heap_size == erts_next_heap_size(p->min_heap_size, 0)); - p->initial[INITIAL_MOD] = mod; - p->initial[INITIAL_FUN] = func; - p->initial[INITIAL_ARI] = (Uint) arity; + p->u.initial[INITIAL_MOD] = mod; + p->u.initial[INITIAL_FUN] = func; + p->u.initial[INITIAL_ARI] = (Uint) arity; /* * Must initialize binary lists here before copying binaries to process. @@ -10534,18 +10996,19 @@ erl_create_process(Process* parent, /* Parent of process (default group leader). p->stop = p->hend = p->heap + sz; p->htop = p->heap; p->heap_sz = sz; + p->abandoned_heap = NULL; + p->live_hf_end = ERTS_INVALID_HFRAG_PTR; p->catches = 0; p->bin_vheap_sz = p->min_vheap_size; p->bin_old_vheap_sz = p->min_vheap_size; p->bin_old_vheap = 0; - p->bin_vheap_mature = 0; p->sys_task_qs = NULL; /* No need to initialize p->fcalls. */ - p->current = p->initial+INITIAL_MOD; + p->current = p->u.initial+INITIAL_MOD; p->i = (BeamInstr *) beam_apply; p->cp = (BeamInstr *) beam_apply+1; @@ -10558,7 +11021,12 @@ erl_create_process(Process* parent, /* Parent of process (default group leader). BM_MESSAGE(args,p,parent); BM_START_TIMER(system); BM_SWAP_TIMER(system,copy); +#ifdef SHCOPY_SPAWN + p->arg_reg[2] = copy_shared_perform(args, arg_size, &info, &p->htop, &p->off_heap); + DESTROY_SHCOPY(info); +#else p->arg_reg[2] = copy_struct(args, arg_size, &p->htop, &p->off_heap); +#endif BM_MESSAGE_COPIED(arg_size); BM_SWAP_TIMER(copy,system); p->arity = 3; @@ -10568,11 +11036,7 @@ erl_create_process(Process* parent, /* Parent of process (default group leader). p->ftrace = NIL; p->reds = 0; -#ifdef ERTS_SMP - p->common.u.alive.ptimer = NULL; -#else - sys_memset(&p->common.u.alive.tm, 0, sizeof(ErlTimer)); -#endif + ERTS_PTMR_INIT(p); p->common.u.alive.reg = NULL; ERTS_P_LINKS(p) = NULL; @@ -10603,10 +11067,14 @@ erl_create_process(Process* parent, /* Parent of process (default group leader). p->msg_inq.last = &p->msg_inq.first; p->msg_inq.len = 0; #endif - p->u.bif_timers = NULL; + p->bif_timers = NULL; +#ifdef ERTS_BTM_ACCESSOR_SUPPORT + p->accessor_bif_timers = NULL; +#endif p->mbuf = NULL; + p->msg_frag = NULL; p->mbuf_sz = 0; - p->psd = NULL; + erts_smp_atomic_init_nob(&p->psd, (erts_aint_t) NULL); p->dictionary = NULL; p->seq_trace_lastcnt = 0; p->seq_trace_clock = 0; @@ -10708,7 +11176,7 @@ erl_create_process(Process* parent, /* Parent of process (default group leader). * Schedule process for execution. */ - schedule_process(p, state); + schedule_process(p, state, 0); VERBOSE(DEBUG_PROCESSES, ("Created a new process: %T\n",p->common.id)); @@ -10740,6 +11208,8 @@ void erts_init_empty_process(Process *p) p->stop = NULL; p->hend = NULL; p->heap = NULL; + p->abandoned_heap = NULL; + p->live_hf_end = ERTS_INVALID_HFRAG_PTR; p->gen_gcs = 0; p->max_gen_gcs = 0; p->min_heap_size = 0; @@ -10760,12 +11230,7 @@ void erts_init_empty_process(Process *p) p->bin_old_vheap_sz = BIN_VH_MIN_SIZE; p->bin_old_vheap = 0; p->sys_task_qs = NULL; - p->bin_vheap_mature = 0; -#ifdef ERTS_SMP - p->common.u.alive.ptimer = NULL; -#else - memset(&(p->common.u.alive.tm), 0, sizeof(ErlTimer)); -#endif + ERTS_PTMR_INIT(p); p->next = NULL; p->off_heap.first = NULL; p->off_heap.overhead = 0; @@ -10776,8 +11241,9 @@ void erts_init_empty_process(Process *p) p->old_htop = NULL; p->old_heap = NULL; p->mbuf = NULL; + p->msg_frag = NULL; p->mbuf_sz = 0; - p->psd = NULL; + erts_smp_atomic_init_nob(&p->psd, (erts_aint_t) NULL); ERTS_P_MONITORS(p) = NULL; ERTS_P_LINKS(p) = NULL; /* List of links */ p->nodes_monitors = NULL; @@ -10786,14 +11252,17 @@ void erts_init_empty_process(Process *p) p->msg.last = &p->msg.first; p->msg.save = &p->msg.first; p->msg.len = 0; - p->u.bif_timers = NULL; + p->bif_timers = NULL; +#ifdef ERTS_BTM_ACCESSOR_SUPPORT + p->accessor_bif_timers = NULL; +#endif p->dictionary = NULL; p->seq_trace_clock = 0; p->seq_trace_lastcnt = 0; p->seq_trace_token = NIL; - p->initial[0] = 0; - p->initial[1] = 0; - p->initial[2] = 0; + p->u.initial[0] = 0; + p->u.initial[1] = 0; + p->u.initial[2] = 0; p->catches = 0; p->cp = NULL; p->i = NULL; @@ -10814,6 +11283,8 @@ void erts_init_empty_process(Process *p) p->parent = NIL; p->approx_started = 0; + p->static_flags = 0; + p->common.u.alive.started_interval = 0; #ifdef HIPE @@ -10828,6 +11299,9 @@ void erts_init_empty_process(Process *p) p->last_old_htop = NULL; #endif +#ifdef ERTS_DIRTY_SCHEDULERS + erts_smp_atomic32_init_nob(&p->dirty_state, 0); +#endif erts_smp_atomic32_init_nob(&p->state, (erts_aint32_t) PRIORITY_NORMAL); #ifdef ERTS_SMP @@ -10859,6 +11333,8 @@ erts_debug_verify_clean_empty_process(Process* p) ASSERT(p->htop == NULL); ASSERT(p->stop == NULL); ASSERT(p->hend == NULL); + ASSERT(p->abandoned_heap == NULL); + ASSERT(p->live_hf_end == ERTS_INVALID_HFRAG_PTR); ASSERT(p->heap == NULL); ASSERT(p->common.id == ERTS_INVALID_PID); ASSERT(ERTS_TRACER_PROC(p) == NIL); @@ -10878,7 +11354,10 @@ erts_debug_verify_clean_empty_process(Process* p) ASSERT(p->suspend_monitors == NULL); ASSERT(p->msg.first == NULL); ASSERT(p->msg.len == 0); - ASSERT(p->u.bif_timers == NULL); + ASSERT(p->bif_timers == NULL); +#ifdef ERTS_BTM_ACCESSOR_SUPPORT + ASSERT(p->accessor_bif_timers == NULL); +#endif ASSERT(p->dictionary == NULL); ASSERT(p->catches == 0); ASSERT(p->cp == NULL); @@ -10933,14 +11412,18 @@ erts_cleanup_empty_process(Process* p) static void delete_process(Process* p) { - ErlMessage* mp; - + Eterm *heap; + ErtsPSD *psd; VERBOSE(DEBUG_PROCESSES, ("Removing process: %T\n",p->common.id)); + VERBOSE(DEBUG_SHCOPY, ("[pid=%T] delete process: %p %p %p %p\n", p->common.id, + HEAP_START(p), HEAP_END(p), OLD_HEAP(p), OLD_HEND(p))); /* Cleanup psd */ - if (p->psd) - erts_free(ERTS_ALC_T_PSD, p->psd); + psd = (ErtsPSD *) erts_smp_atomic_read_nob(&p->psd); + + if (psd) + erts_free(ERTS_ALC_T_PSD, psd); /* Clean binaries and funs */ erts_cleanup_offheap(&p->off_heap); @@ -10959,16 +11442,17 @@ delete_process(Process* p) * Release heaps. Clobber contents in DEBUG build. */ - -#ifdef DEBUG - sys_memset(p->heap, DEBUG_BAD_BYTE, p->heap_sz*sizeof(Eterm)); -#endif - #ifdef HIPE hipe_delete_process(&p->hipe); #endif - ERTS_HEAP_FREE(ERTS_ALC_T_HEAP, (void*) p->heap, p->heap_sz*sizeof(Eterm)); + heap = p->abandoned_heap ? p->abandoned_heap : p->heap; + +#ifdef DEBUG + sys_memset(heap, DEBUG_BAD_BYTE, p->heap_sz*sizeof(Eterm)); +#endif + + ERTS_HEAP_FREE(ERTS_ALC_T_HEAP, (void*) heap, p->heap_sz*sizeof(Eterm)); if (p->old_heap != NULL) { #ifdef DEBUG @@ -10987,27 +11471,14 @@ delete_process(Process* p) free_message_buffer(p->mbuf); } + if (p->msg_frag) + erts_cleanup_messages(p->msg_frag); + erts_erase_dicts(p); /* free all pending messages */ - mp = p->msg.first; - while(mp != NULL) { - ErlMessage* next_mp = mp->next; - if (mp->data.attached) { - if (is_value(mp->m[0])) - free_message_buffer(mp->data.heap_frag); - else { - if (is_not_nil(mp->m[1])) { - ErlHeapFragment *heap_frag; - heap_frag = (ErlHeapFragment *) mp->data.dist_ext->ext_endp; - erts_cleanup_offheap(&heap_frag->off_heap); - } - erts_free_dist_ext_copy(mp->data.dist_ext); - } - } - free_message(mp); - mp = next_mp; - } + erts_cleanup_messages(p->msg.first); + p->msg.first = NULL; ASSERT(!p->nodes_monitors); ASSERT(!p->suspend_monitors); @@ -11026,10 +11497,13 @@ set_proc_exiting(Process *p, ERTS_SMP_LC_ASSERT(erts_proc_lc_my_proc_locks(p) == ERTS_PROC_LOCKS_ALL); enqueue = change_proc_schedule_state(p, - ERTS_PSFLG_SUSPENDED|ERTS_PSFLG_PENDING_EXIT, + (ERTS_PSFLG_SUSPENDED + | ERTS_PSFLG_PENDING_EXIT + | ERTS_PSFLGS_DIRTY_WORK), ERTS_PSFLG_EXITING|ERTS_PSFLG_ACTIVE, &state, - &enq_prio); + &enq_prio, + ERTS_PROC_LOCKS_ALL); p->fvalue = reason; if (bp) @@ -11041,13 +11515,25 @@ set_proc_exiting(Process *p, */ p->freason = EXTAG_EXIT; KILL_CATCHES(p); - cancel_timer(p); p->i = (BeamInstr *) beam_exit; - if (enqueue) - add2runq(enqueue > 0 ? p : make_proxy_proc(NULL, p, enq_prio), - state, - enq_prio); +#ifndef ERTS_SMP + if (state & (ERTS_PSFLG_RUNNING|ERTS_PSFLG_RUNNING_SYS)) { + /* + * I non smp case: + * + * Currently executing process might be sent an exit + * signal if it is traced by a port that it also is + * linked to, and the port terminates during the + * trace. In this case we want schedule out the + * process as quickly as possible in order to detect + * the event as fast as possible. + */ + ERTS_VBUMP_ALL_REDS(p); + } +#endif + + add2runq(enqueue, enq_prio, p, state, NULL); } static ERTS_INLINE erts_aint32_t @@ -11070,7 +11556,8 @@ set_proc_self_exiting(Process *c_p) ERTS_PSFLG_SUSPENDED|ERTS_PSFLG_PENDING_EXIT, ERTS_PSFLG_EXITING|ERTS_PSFLG_ACTIVE, &state, - &enq_prio); + &enq_prio, + ERTS_PROC_LOCKS_ALL); ASSERT(!enqueue); return state; @@ -11140,10 +11627,19 @@ save_pending_exiter(Process *p) { ErtsProcList *plp; ErtsRunQueue *rq; + ErtsSchedulerData *esdp = erts_get_scheduler_data(); ERTS_SMP_LC_ASSERT(ERTS_PROC_LOCK_STATUS & erts_proc_lc_my_proc_locks(p)); - rq = erts_get_runq_current(NULL); + if (!esdp) + rq = RUNQ_READ_RQ(&p->run_queue); + else + rq = esdp->run_queue; + +#ifdef ERTS_DIRTY_SCHEDULERS + if (ERTS_RUNQ_IX_IS_DIRTY(rq->ix)) + rq = ERTS_RUNQ_IX(0); /* Handle on ordinary scheduler */ +#endif plp = proclist_create(p); @@ -11154,12 +11650,8 @@ save_pending_exiter(Process *p) non_empty_runq(rq); erts_smp_runq_unlock(rq); -#ifdef ERTS_DIRTY_SCHEDULERS - if (ERTS_RUNQ_IX_IS_DIRTY(rq->ix)) - wake_dirty_schedulers(rq, 0); - else -#endif - wake_scheduler(rq); + + wake_scheduler(rq); } #endif @@ -11173,43 +11665,46 @@ static ERTS_INLINE void send_exit_message(Process *to, ErtsProcLocks *to_locksp, Eterm exit_term, Uint term_size, Eterm token) { - if (token == NIL -#ifdef USE_VM_PROBES - || token == am_have_dt_utag -#endif - ) { - Eterm* hp; - Eterm mess; - ErlHeapFragment* bp; - ErlOffHeap *ohp; - - hp = erts_alloc_message_heap(term_size, &bp, &ohp, to, to_locksp); + ErtsMessage *mp; + ErlOffHeap *ohp; + Eterm* hp; + Eterm mess; +#ifdef SHCOPY_SEND + erts_shcopy_t info; +#endif + + if (!have_seqtrace(token)) { +#ifdef SHCOPY_SEND + INITIALIZE_SHCOPY(info); + term_size = copy_shared_calculate(exit_term, &info); + mp = erts_alloc_message_heap(to, to_locksp, term_size, &hp, &ohp); + mess = copy_shared_perform(exit_term, term_size, &info, &hp, ohp); + DESTROY_SHCOPY(info); +#else + mp = erts_alloc_message_heap(to, to_locksp, term_size, &hp, &ohp); mess = copy_struct(exit_term, term_size, &hp, ohp); - erts_queue_message(to, to_locksp, bp, mess, NIL -#ifdef USE_VM_PROBES - , NIL #endif - ); + erts_queue_message(to, to_locksp, mp, mess, NIL); } else { - ErlHeapFragment* bp; - Eterm* hp; - Eterm mess; Eterm temp_token; Uint sz_token; ASSERT(is_tuple(token)); sz_token = size_object(token); - bp = new_message_buffer(term_size+sz_token); - hp = bp->mem; - mess = copy_struct(exit_term, term_size, &hp, &bp->off_heap); +#ifdef SHCOPY_SEND + INITIALIZE_SHCOPY(info); + term_size = copy_shared_calculate(exit_term, &info); + mp = erts_alloc_message_heap(to, to_locksp, term_size+sz_token, &hp, &ohp); + mess = copy_shared_perform(exit_term, term_size, &info, &hp, ohp); + DESTROY_SHCOPY(info); +#else + mp = erts_alloc_message_heap(to, to_locksp, term_size+sz_token, &hp, &ohp); + mess = copy_struct(exit_term, term_size, &hp, ohp); +#endif /* the trace token must in this case be updated by the caller */ seq_trace_output(token, mess, SEQ_TRACE_SEND, to->common.id, NULL); - temp_token = copy_struct(token, sz_token, &hp, &bp->off_heap); - erts_queue_message(to, to_locksp, bp, mess, temp_token -#ifdef USE_VM_PROBES - , NIL -#endif - ); + temp_token = copy_struct(token, sz_token, &hp, ohp); + erts_queue_message(to, to_locksp, mp, mess, temp_token); } } @@ -11318,11 +11813,7 @@ send_exit_signal(Process *c_p, /* current process if and only if ((state & ERTS_PSFLG_TRAP_EXIT) && (reason != am_kill || (flags & ERTS_XSIG_FLG_IGN_KILL))) { - if (is_not_nil(token) -#ifdef USE_VM_PROBES - && token != am_have_dt_utag -#endif - && token_update) + if (have_seqtrace(token) && token_update) seq_trace_update_send(token_update); if (is_value(exit_tuple)) send_exit_message(rp, rp_locks, exit_tuple, exit_tuple_sz, token); @@ -11356,23 +11847,21 @@ send_exit_signal(Process *c_p, /* current process if and only if (need_locks && erts_smp_proc_trylock(rp, need_locks) == EBUSY) { /* ... but we havn't got all locks on it ... */ - save_pending_exiter(rp); + save_pending_exiter(rp); /* * The pending exit will be discovered when next * process is scheduled in */ - goto set_pending_exit; - } - else { - /* ...and we have all locks on it... */ - *rp_locks = ERTS_PROC_LOCKS_ALL; - set_proc_exiting(rp, - state, - (is_immed(rsn) - ? rsn - : copy_object(rsn, rp)), - NULL); + goto set_pending_exit; } + /* ...and we have all locks on it... */ + *rp_locks = ERTS_PROC_LOCKS_ALL; + set_proc_exiting(rp, + state, + (is_immed(rsn) + ? rsn + : copy_object(rsn, rp)), + NULL); } else { /* Process running... */ @@ -11462,7 +11951,8 @@ static void doit_exit_monitor(ErtsMonitor *mon, void *vpcontext) ErtsMonitor *rmon; Process *rp; - if (mon->type == MON_ORIGIN) { + switch (mon->type) { + case MON_ORIGIN: /* We are monitoring someone else, we need to demonitor that one.. */ if (is_atom(mon->pid)) { /* remote by name */ ASSERT(is_node_name_atom(mon->pid)); @@ -11525,7 +12015,8 @@ static void doit_exit_monitor(ErtsMonitor *mon, void *vpcontext) } } } - } else { /* type == MON_TARGET */ + break; + case MON_TARGET: ASSERT(mon->type == MON_TARGET); ASSERT(is_pid(mon->pid) || is_internal_port(mon->pid)); if (is_internal_port(mon->pid)) { @@ -11584,6 +12075,12 @@ static void doit_exit_monitor(ErtsMonitor *mon, void *vpcontext) } } } + break; + case MON_TIME_OFFSET: + erts_demonitor_time_offset(mon->ref); + break; + default: + ERTS_INTERNAL_ERROR("Invalid monitor type"); } done: /* As the monitors are previously removed from the process, @@ -11703,7 +12200,7 @@ static void doit_exit_link(ErtsLink *lnk, void *vpcontext) break; default: - erl_exit(1, "bad type in link list\n"); + erts_exit(ERTS_ERROR_EXIT, "bad type in link list\n"); break; } erts_destroy_link(lnk); @@ -11715,8 +12212,9 @@ resume_suspend_monitor(ErtsSuspendMonitor *smon, void *vc_p) Process *suspendee = erts_pid2proc((Process *) vc_p, ERTS_PROC_LOCK_MAIN, smon->pid, ERTS_PROC_LOCK_STATUS); if (suspendee) { + ASSERT(suspendee != vc_p); if (smon->active) - resume_process(suspendee); + resume_process(suspendee, ERTS_PROC_LOCK_STATUS); erts_smp_proc_unlock(suspendee, ERTS_PROC_LOCK_STATUS); } erts_destroy_suspend_monitor(smon); @@ -11729,6 +12227,7 @@ erts_do_exit_process(Process* p, Eterm reason) { p->arity = 0; /* No live registers */ p->fvalue = reason; + #ifdef USE_VM_PROBES if (DTRACE_ENABLED(process_exit)) { @@ -11741,6 +12240,10 @@ erts_do_exit_process(Process* p, Eterm reason) } #endif + if (p->static_flags & ERTS_STC_FLG_SYSTEM_PROC) + erts_exit(ERTS_DUMP_EXIT, "System process %T terminated: %T\n", + p->common.id, reason); + #ifdef ERTS_SMP ERTS_SMP_CHK_HAVE_ONLY_MAIN_PROC_LOCK(p); /* By locking all locks (main lock is already locked) when going @@ -11780,20 +12283,20 @@ erts_do_exit_process(Process* p, Eterm reason) ASSERT((ERTS_TRACE_FLAGS(p) & F_INITIAL_TRACE_FLAGS) == F_INITIAL_TRACE_FLAGS); - cancel_timer(p); /* Always cancel timer just in case */ - - if (p->u.bif_timers) - erts_cancel_bif_timers(p, ERTS_PROC_LOCKS_ALL); + ASSERT(erts_proc_read_refc(p) > 0); + if (ERTS_PTMR_IS_SET(p)) { + erts_cancel_proc_timer(p); + ASSERT(erts_proc_read_refc(p) > 0); + } erts_smp_proc_unlock(p, ERTS_PROC_LOCKS_ALL_MINOR); /* - * The p->u.bif_timers of this process can *not* be used anymore; + * p->u.initial of this process can *not* be used anymore; * will be overwritten by misc termination data. */ p->u.terminate = NULL; - erts_continue_exit_process(p); } @@ -11808,6 +12311,7 @@ erts_continue_exit_process(Process *p) struct saved_calls *scb; process_breakpoint_time_t *pbt; erts_aint32_t state; + void *nif_export; #ifdef DEBUG int yield_allowed = 1; @@ -11817,22 +12321,70 @@ erts_continue_exit_process(Process *p) ASSERT(ERTS_PROC_IS_EXITING(p)); + ASSERT(erts_proc_read_refc(p) > 0); + if (p->bif_timers) { + if (erts_cancel_bif_timers(p, p->bif_timers, &p->u.terminate)) { + ASSERT(erts_proc_read_refc(p) > 0); + goto yield; + } + ASSERT(erts_proc_read_refc(p) > 0); + p->bif_timers = NULL; + } + +#ifdef ERTS_BTM_ACCESSOR_SUPPORT + if (p->accessor_bif_timers) { + if (erts_detach_accessor_bif_timers(p, + p->accessor_bif_timers, + &p->u.terminate)) { + ASSERT(erts_proc_read_refc(p) > 0); + goto yield; + } + ASSERT(erts_proc_read_refc(p) > 0); + p->accessor_bif_timers = NULL; + } +#endif + #ifdef ERTS_SMP + if (p->flags & F_SCHDLR_ONLN_WAITQ) + abort_sched_onln_chng_waitq(p); + if (p->flags & F_HAVE_BLCKD_MSCHED) { ErtsSchedSuspendResult ssr; - ssr = erts_block_multi_scheduling(p, ERTS_PROC_LOCK_MAIN, 0, 1); + ssr = erts_block_multi_scheduling(p, ERTS_PROC_LOCK_MAIN, 0, 0, 1); + switch (ssr) { + case ERTS_SCHDLR_SSPND_YIELD_RESTART: + goto yield; + case ERTS_SCHDLR_SSPND_DONE_MSCHED_BLOCKED: + case ERTS_SCHDLR_SSPND_DONE_NMSCHED_BLOCKED: + case ERTS_SCHDLR_SSPND_YIELD_DONE_MSCHED_BLOCKED: + case ERTS_SCHDLR_SSPND_YIELD_DONE_NMSCHED_BLOCKED: + case ERTS_SCHDLR_SSPND_DONE: + case ERTS_SCHDLR_SSPND_YIELD_DONE: + p->flags &= ~F_HAVE_BLCKD_MSCHED; + break; + case ERTS_SCHDLR_SSPND_EINVAL: + default: + erts_exit(ERTS_ABORT_EXIT, "%s:%d: Internal error: %d\n", + __FILE__, __LINE__, (int) ssr); + } + } + if (p->flags & F_HAVE_BLCKD_NMSCHED) { + ErtsSchedSuspendResult ssr; + ssr = erts_block_multi_scheduling(p, ERTS_PROC_LOCK_MAIN, 0, 1, 1); switch (ssr) { case ERTS_SCHDLR_SSPND_YIELD_RESTART: goto yield; case ERTS_SCHDLR_SSPND_DONE_MSCHED_BLOCKED: + case ERTS_SCHDLR_SSPND_DONE_NMSCHED_BLOCKED: case ERTS_SCHDLR_SSPND_YIELD_DONE_MSCHED_BLOCKED: + case ERTS_SCHDLR_SSPND_YIELD_DONE_NMSCHED_BLOCKED: case ERTS_SCHDLR_SSPND_DONE: case ERTS_SCHDLR_SSPND_YIELD_DONE: p->flags &= ~F_HAVE_BLCKD_MSCHED; break; case ERTS_SCHDLR_SSPND_EINVAL: default: - erl_exit(ERTS_ABORT_EXIT, "%s:%d: Internal error: %d\n", + erts_exit(ERTS_ABORT_EXIT, "%s:%d: Internal error: %d\n", __FILE__, __LINE__, (int) ssr); } } @@ -11910,6 +12462,8 @@ erts_continue_exit_process(Process *p) p->scheduler_data->current_process = NULL; p->scheduler_data->free_process = p; +#else + erts_proc_inc_refc(p); /* Decremented in schedule() */ #endif /* Time of death! */ @@ -11928,36 +12482,29 @@ erts_continue_exit_process(Process *p) { /* Inactivate and notify free */ erts_aint32_t n, e, a = erts_smp_atomic32_read_nob(&p->state); -#ifdef ERTS_SMP int refc_inced = 0; -#endif while (1) { n = e = a; ASSERT(a & ERTS_PSFLG_EXITING); n |= ERTS_PSFLG_FREE; n &= ~ERTS_PSFLG_ACTIVE; -#ifdef ERTS_SMP if ((n & ERTS_PSFLG_IN_RUNQ) && !refc_inced) { - erts_smp_proc_inc_refc(p); + erts_proc_inc_refc(p); refc_inced = 1; } -#endif a = erts_smp_atomic32_cmpxchg_mb(&p->state, n, e); if (a == e) break; } -#ifdef ERTS_SMP if (refc_inced && !(n & ERTS_PSFLG_IN_RUNQ)) - erts_smp_proc_dec_refc(p); -#endif + erts_proc_dec_refc(p); } - dep = ((p->flags & F_DISTRIBUTION) - ? ERTS_PROC_SET_DIST_ENTRY(p, ERTS_PROC_LOCKS_ALL, NULL) - : NULL); - scb = ERTS_PROC_SET_SAVED_CALLS_BUF(p, ERTS_PROC_LOCKS_ALL, NULL); - pbt = ERTS_PROC_SET_CALL_TIME(p, ERTS_PROC_LOCKS_ALL, NULL); + dep = (p->flags & F_DISTRIBUTION) ? erts_this_dist_entry : NULL; + scb = ERTS_PROC_SET_SAVED_CALLS_BUF(p, NULL); + pbt = ERTS_PROC_SET_CALL_TIME(p, NULL); + nif_export = ERTS_PROC_SET_NIF_TRAP_EXPORT(p, NULL); erts_smp_proc_unlock(p, ERTS_PROC_LOCKS_ALL); #ifdef BM_COUNTERS @@ -11966,8 +12513,6 @@ erts_continue_exit_process(Process *p) if (dep) { erts_do_net_exits(dep, reason); - if(dep) - erts_deref_dist_entry(dep); } /* @@ -12005,6 +12550,9 @@ erts_continue_exit_process(Process *p) if (pbt) erts_free(ERTS_ALC_T_BPD, (void *) pbt); + if (nif_export) + erts_destroy_nif_export(nif_export); + delete_process(p); #ifdef ERTS_SMP @@ -12037,64 +12585,6 @@ erts_continue_exit_process(Process *p) } -/* Callback for process timeout */ -static void -timeout_proc(Process* p) -{ - erts_aint32_t state; - BeamInstr** pi = (BeamInstr **) p->def_arg_reg; - p->i = *pi; - p->flags |= F_TIMO; - p->flags &= ~F_INSLPQUEUE; - - state = erts_smp_atomic32_read_acqb(&p->state); - if (!(state & ERTS_PSFLG_ACTIVE)) - schedule_process(p, state); -} - - -void -cancel_timer(Process* p) -{ - ERTS_SMP_LC_ASSERT(ERTS_PROC_LOCK_MAIN & erts_proc_lc_my_proc_locks(p)); - p->flags &= ~(F_INSLPQUEUE|F_TIMO); -#ifdef ERTS_SMP - erts_cancel_smp_ptimer(p->common.u.alive.ptimer); -#else - erts_cancel_timer(&p->common.u.alive.tm); -#endif -} - -/* - * Insert a process into the time queue, with a timeout 'timeout' in ms. - */ -void -set_timer(Process* p, Uint timeout) -{ - ERTS_SMP_LC_ASSERT(ERTS_PROC_LOCK_MAIN & erts_proc_lc_my_proc_locks(p)); - - /* check for special case timeout=0 DONT ADD TO time queue */ - if (timeout == 0) { - p->flags |= F_TIMO; - return; - } - p->flags |= F_INSLPQUEUE; - p->flags &= ~F_TIMO; - -#ifdef ERTS_SMP - erts_create_smp_ptimer(&p->common.u.alive.ptimer, - p->common.id, - (ErlTimeoutProc) timeout_proc, - timeout); -#else - erts_set_timer(&p->common.u.alive.tm, - (ErlTimeoutProc) timeout_proc, - NULL, - (void*) p, - timeout); -#endif -} - /* * Stack dump functions follow. */ @@ -12110,7 +12600,7 @@ erts_stack_dump(int to, void *to_arg, Process *p) } erts_program_counter_info(to, to_arg, p); for (sp = p->stop; sp < STACK_START(p); sp++) { - yreg = stack_element_dump(to, to_arg, p, sp, yreg); + yreg = stack_element_dump(to, to_arg, sp, yreg); } } @@ -12167,7 +12657,7 @@ print_function_from_pc(int to, void *to_arg, BeamInstr* x) } static int -stack_element_dump(int to, void *to_arg, Process* p, Eterm* sp, int yreg) +stack_element_dump(int to, void *to_arg, Eterm* sp, int yreg) { Eterm x = *sp; @@ -12181,7 +12671,7 @@ stack_element_dump(int to, void *to_arg, Process* p, Eterm* sp, int yreg) } if (is_CP(x)) { - erts_print(to, to_arg, "Return addr %p (", (Eterm *) EXPAND_POINTER(x)); + erts_print(to, to_arg, "Return addr %p (", (Eterm *) x); print_function_from_pc(to, to_arg, cp_val(x)); erts_print(to, to_arg, ")\n"); yreg = 0; @@ -12196,6 +12686,188 @@ stack_element_dump(int to, void *to_arg, Process* p, Eterm* sp, int yreg) } /* + * Print scheduler information + */ +void +erts_print_scheduler_info(int to, void *to_arg, ErtsSchedulerData *esdp) { + int i; + erts_aint32_t flg; + Process *p; + + erts_print(to, to_arg, "=scheduler:%u\n", esdp->no); + +#ifdef ERTS_SMP + flg = erts_smp_atomic32_read_dirty(&esdp->ssi->flags); + erts_print(to, to_arg, "Scheduler Sleep Info Flags: "); + for (i = 0; i < ERTS_SSI_FLGS_MAX && flg; i++) { + erts_aint32_t chk = (1 << i); + if (flg & chk) { + switch (chk) { + case ERTS_SSI_FLG_SLEEPING: + erts_print(to, to_arg, "SLEEPING"); break; + case ERTS_SSI_FLG_POLL_SLEEPING: + erts_print(to, to_arg, "POLL_SLEEPING"); break; + case ERTS_SSI_FLG_TSE_SLEEPING: + erts_print(to, to_arg, "TSE_SLEEPING"); break; + case ERTS_SSI_FLG_WAITING: + erts_print(to, to_arg, "WAITING"); break; + case ERTS_SSI_FLG_SUSPENDED: + erts_print(to, to_arg, "SUSPENDED"); break; + default: + erts_print(to, to_arg, "UNKNOWN(%d)", flg); break; + } + if (flg > chk) + erts_print(to, to_arg, " | "); + flg -= chk; + } + } + erts_print(to, to_arg, "\n"); +#endif + + flg = erts_atomic32_read_dirty(&esdp->ssi->aux_work); + erts_print(to, to_arg, "Scheduler Sleep Info Aux Work: "); + for (i = 0; i < ERTS_SSI_AUX_WORK_NO_FLAGS && flg; i++) { + erts_aint32_t chk = (1 << i); + if (flg & chk) { + if (erts_aux_work_flag_descr[i]) + erts_print(to, to_arg, "%s", erts_aux_work_flag_descr[i]); + else + erts_print(to, to_arg, "1<<%d", i); + if (flg > chk) + erts_print(to, to_arg, " | "); + flg -= chk; + } + } + erts_print(to, to_arg, "\n"); + + erts_print(to, to_arg, "Current Port: "); + if (esdp->current_port) + erts_print(to, to_arg, "%T", esdp->current_port->common.id); + erts_print(to, to_arg, "\n"); + + for (i = 0; i < ERTS_NO_PROC_PRIO_LEVELS; i++) { + erts_print(to, to_arg, "Run Queue "); + switch (i) { + case PRIORITY_MAX: + erts_print(to, to_arg, "Max "); + break; + case PRIORITY_HIGH: + erts_print(to, to_arg, "High "); + break; + case PRIORITY_NORMAL: + erts_print(to, to_arg, "Normal "); + break; + case PRIORITY_LOW: + erts_print(to, to_arg, "Low "); + break; + default: + erts_print(to, to_arg, "Unknown "); + break; + } + erts_print(to, to_arg, "Length: %d\n", + erts_smp_atomic32_read_dirty(&esdp->run_queue->procs.prio_info[i].len)); + } + erts_print(to, to_arg, "Run Queue Port Length: %d\n", + erts_smp_atomic32_read_dirty(&esdp->run_queue->ports.info.len)); + + flg = erts_smp_atomic32_read_dirty(&esdp->run_queue->flags); + erts_print(to, to_arg, "Run Queue Flags: "); + for (i = 0; i < ERTS_RUNQ_FLG_MAX && flg; i++) { + erts_aint32_t chk = (1 << i); + if (flg & chk) { + switch (chk) { + case (1 << PRIORITY_MAX): + erts_print(to, to_arg, "NONEMPTY_MAX"); break; + case (1 << PRIORITY_HIGH): + erts_print(to, to_arg, "NONEMPTY_HIGH"); break; + case (1 << PRIORITY_NORMAL): + erts_print(to, to_arg, "NONEMPTY_NORMAL"); break; + case (1 << PRIORITY_LOW): + erts_print(to, to_arg, "NONEMPTY_LOW"); break; + case (1 << (PRIORITY_MAX + ERTS_RUNQ_FLGS_EMIGRATE_SHFT)): + erts_print(to, to_arg, "EMIGRATE_MAX"); break; + case (1 << (PRIORITY_HIGH + ERTS_RUNQ_FLGS_EMIGRATE_SHFT)): + erts_print(to, to_arg, "EMIGRATE_HIGH"); break; + case (1 << (PRIORITY_NORMAL + ERTS_RUNQ_FLGS_EMIGRATE_SHFT)): + erts_print(to, to_arg, "EMIGRATE_NORMAL"); break; + case (1 << (PRIORITY_LOW + ERTS_RUNQ_FLGS_EMIGRATE_SHFT)): + erts_print(to, to_arg, "EMIGRATE_LOW"); break; + case (1 << (PRIORITY_MAX + ERTS_RUNQ_FLGS_IMMIGRATE_SHFT)): + erts_print(to, to_arg, "IMMIGRATE_MAX"); break; + case (1 << (PRIORITY_HIGH + ERTS_RUNQ_FLGS_IMMIGRATE_SHFT)): + erts_print(to, to_arg, "IMMIGRATE_HIGH"); break; + case (1 << (PRIORITY_NORMAL + ERTS_RUNQ_FLGS_IMMIGRATE_SHFT)): + erts_print(to, to_arg, "IMMIGRATE_NORMAL"); break; + case (1 << (PRIORITY_LOW + ERTS_RUNQ_FLGS_IMMIGRATE_SHFT)): + erts_print(to, to_arg, "IMMIGRATE_LOW"); break; + case (1 << (PRIORITY_MAX + ERTS_RUNQ_FLGS_EVACUATE_SHFT)): + erts_print(to, to_arg, "EVACUATE_MAX"); break; + case (1 << (PRIORITY_HIGH + ERTS_RUNQ_FLGS_EVACUATE_SHFT)): + erts_print(to, to_arg, "EVACUATE_HIGH"); break; + case (1 << (PRIORITY_NORMAL + ERTS_RUNQ_FLGS_EVACUATE_SHFT)): + erts_print(to, to_arg, "EVACUATE_NORMAL"); break; + case (1 << (PRIORITY_LOW + ERTS_RUNQ_FLGS_EVACUATE_SHFT)): + erts_print(to, to_arg, "EVACUATE_LOW"); break; + case ERTS_RUNQ_FLG_OUT_OF_WORK: + erts_print(to, to_arg, "OUT_OF_WORK"); break; + case ERTS_RUNQ_FLG_HALFTIME_OUT_OF_WORK: + erts_print(to, to_arg, "HALFTIME_OUT_OF_WORK"); break; + case ERTS_RUNQ_FLG_SUSPENDED: + erts_print(to, to_arg, "SUSPENDED"); break; + case ERTS_RUNQ_FLG_CHK_CPU_BIND: + erts_print(to, to_arg, "CHK_CPU_BIND"); break; + case ERTS_RUNQ_FLG_INACTIVE: + erts_print(to, to_arg, "INACTIVE"); break; + case ERTS_RUNQ_FLG_NONEMPTY: + erts_print(to, to_arg, "NONEMPTY"); break; + case ERTS_RUNQ_FLG_PROTECTED: + erts_print(to, to_arg, "PROTECTED"); break; + default: + erts_print(to, to_arg, "UNKNOWN(%d)", flg); break; + } + if (flg > chk) + erts_print(to, to_arg, " | "); + flg -= chk; + } + } + erts_print(to, to_arg, "\n"); + + /* This *MUST* to be the last information in scheduler block */ + p = esdp->current_process; + erts_print(to, to_arg, "Current Process: "); + if (esdp->current_process && !(ERTS_TRACE_FLAGS(p) & F_SENSITIVE)) { + flg = erts_smp_atomic32_read_dirty(&p->state); + erts_print(to, to_arg, "%T\n", p->common.id); + + erts_print(to, to_arg, "Current Process State: "); + erts_dump_process_state(to, to_arg, flg); + + erts_print(to, to_arg, "Current Process Internal State: "); + erts_dump_extended_process_state(to, to_arg, flg); + + erts_print(to, to_arg, "Current Process Program counter: %p (", p->i); + print_function_from_pc(to, to_arg, p->i); + erts_print(to, to_arg, ")\n"); + erts_print(to, to_arg, "Current Process CP: %p (", p->cp); + print_function_from_pc(to, to_arg, p->cp); + erts_print(to, to_arg, ")\n"); + + /* Getting this stacktrace can segfault if we are very very + unlucky if called while a process is being garbage collected. + Therefore we only call this on other schedulers if we either + have protection against segfaults, or we know that the process + is not garbage collecting. It *should* always be safe to call + on a process owned by us, even if it is currently being garbage + collected. + */ + erts_print(to, to_arg, "Current Process Limited Stack Trace:\n"); + erts_limited_stack_trace(to, to_arg, p); + } else + erts_print(to, to_arg, "\n"); + +} + +/* * A nice system halt closing all open port goes as follows: * 1) This function schedules the aux work ERTS_SSI_AUX_WORK_REAP_PORTS * on all schedulers, then schedules itself out. @@ -12209,10 +12881,10 @@ stack_element_dump(int to, void *to_arg, Process* p, Eterm* sp, int yreg) * The same global atomic is used as refcount. * * A BIF that calls this should make sure to schedule out to never come back: - * erl_halt((int)(- code)); + * erts_halt(code); * ERTS_BIF_YIELD1(bif_export[BIF_erlang_halt_1], BIF_P, NIL); */ -void erl_halt(int code) +void erts_halt(int code) { if (-1 == erts_smp_atomic32_cmpxchg_acqb(&erts_halt_progress, erts_no_schedulers, diff --git a/erts/emulator/beam/erl_process.h b/erts/emulator/beam/erl_process.h index ed6dadbffa..c07337b325 100644 --- a/erts/emulator/beam/erl_process.h +++ b/erts/emulator/beam/erl_process.h @@ -3,16 +3,17 @@ * * Copyright Ericsson AB 1996-2014. All Rights Reserved. * - * The contents of this file are subject to the Erlang Public License, - * Version 1.1, (the "License"); you may not use this file except in - * compliance with the License. You should have received a copy of the - * Erlang Public License along with this software. If not, it can be - * retrieved online at http://www.erlang.org/. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Software distributed under the License is distributed on an "AS IS" - * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See - * the License for the specific language governing rights and limitations - * under the License. + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. * * %CopyrightEnd% */ @@ -52,12 +53,16 @@ typedef struct process Process; #include "erl_node_container_utils.h" #include "erl_node_tables.h" #include "erl_monitors.h" -#include "erl_bif_timer.h" +#include "erl_hl_timer.h" #include "erl_time.h" #include "erl_atom_table.h" #include "external.h" #include "erl_mseg.h" #include "erl_async.h" +#include "erl_gc.h" +#define ERTS_ONLY_INCLUDE_TRACE_FLAGS +#include "erl_trace.h" +#undef ERTS_ONLY_INCLUDE_TRACE_FLAGS #ifdef HIPE #include "hipe_process.h" @@ -74,10 +79,8 @@ struct ErtsNodesMonitor_; #define ERTS_HAVE_SCHED_UTIL_BALANCING_SUPPORT 0 #define ERTS_MAX_NO_OF_SCHEDULERS 1024 -#ifdef ERTS_DIRTY_SCHEDULERS #define ERTS_MAX_NO_OF_DIRTY_CPU_SCHEDULERS ERTS_MAX_NO_OF_SCHEDULERS #define ERTS_MAX_NO_OF_DIRTY_IO_SCHEDULERS ERTS_MAX_NO_OF_SCHEDULERS -#endif #define ERTS_DEFAULT_MAX_PROCESSES (1 << 18) @@ -104,6 +107,7 @@ struct saved_calls { }; extern Export exp_send, exp_receive, exp_timeout; +extern int erts_eager_check_io; extern int erts_sched_compact_load; extern int erts_sched_balance_util; extern Uint erts_no_schedulers; @@ -167,6 +171,10 @@ extern int erts_sched_thread_suggested_stack_size; (((Uint32) 1) << (ERTS_RUNQ_FLG_BASE2 + 5)) #define ERTS_RUNQ_FLG_PROTECTED \ (((Uint32) 1) << (ERTS_RUNQ_FLG_BASE2 + 6)) +#define ERTS_RUNQ_FLG_EXEC \ + (((Uint32) 1) << (ERTS_RUNQ_FLG_BASE2 + 7)) + +#define ERTS_RUNQ_FLG_MAX (ERTS_RUNQ_FLG_BASE2 + 8) #define ERTS_RUNQ_FLGS_MIGRATION_QMASKS \ (ERTS_RUNQ_FLGS_EMIGRATE_QMASK \ @@ -210,6 +218,9 @@ extern int erts_sched_thread_suggested_stack_size; #define ERTS_RUNQ_FLGS_SET(RQ, FLGS) \ ((Uint32) erts_smp_atomic32_read_bor_relb(&(RQ)->flags, \ (erts_aint32_t) (FLGS))) +#define ERTS_RUNQ_FLGS_SET_NOB(RQ, FLGS) \ + ((Uint32) erts_smp_atomic32_read_bor_nob(&(RQ)->flags, \ + (erts_aint32_t) (FLGS))) #define ERTS_RUNQ_FLGS_BSET(RQ, MSK, FLGS) \ ((Uint32) erts_smp_atomic32_read_bset_relb(&(RQ)->flags, \ (erts_aint32_t) (MSK), \ @@ -217,6 +228,9 @@ extern int erts_sched_thread_suggested_stack_size; #define ERTS_RUNQ_FLGS_UNSET(RQ, FLGS) \ ((Uint32) erts_smp_atomic32_read_band_relb(&(RQ)->flags, \ (erts_aint32_t) ~(FLGS))) +#define ERTS_RUNQ_FLGS_UNSET_NOB(RQ, FLGS) \ + ((Uint32) erts_smp_atomic32_read_band_nob(&(RQ)->flags, \ + (erts_aint32_t) ~(FLGS))) #define ERTS_RUNQ_FLGS_GET(RQ) \ ((Uint32) erts_smp_atomic32_read_acqb(&(RQ)->flags)) #define ERTS_RUNQ_FLGS_GET_NOB(RQ) \ @@ -230,7 +244,9 @@ extern int erts_sched_thread_suggested_stack_size; typedef enum { ERTS_SCHDLR_SSPND_DONE_MSCHED_BLOCKED, + ERTS_SCHDLR_SSPND_DONE_NMSCHED_BLOCKED, ERTS_SCHDLR_SSPND_YIELD_DONE_MSCHED_BLOCKED, + ERTS_SCHDLR_SSPND_YIELD_DONE_NMSCHED_BLOCKED, ERTS_SCHDLR_SSPND_DONE, ERTS_SCHDLR_SSPND_YIELD_RESTART, ERTS_SCHDLR_SSPND_YIELD_DONE, @@ -250,6 +266,8 @@ typedef enum { #define ERTS_SSI_FLG_WAITING (((erts_aint32_t) 1) << 3) #define ERTS_SSI_FLG_SUSPENDED (((erts_aint32_t) 1) << 4) +#define ERTS_SSI_FLGS_MAX 5 + #define ERTS_SSI_FLGS_SLEEP_TYPE \ (ERTS_SSI_FLG_TSE_SLEEPING|ERTS_SSI_FLG_POLL_SLEEPING) @@ -262,24 +280,70 @@ typedef enum { | ERTS_SSI_FLG_SUSPENDED) /* - * Keep ERTS_SSI_AUX_WORK flags in expected frequency order relative - * eachother. Most frequent - lowest bit number. + * Keep ERTS_SSI_AUX_WORK flags ordered in expected frequency + * order relative eachother. Most frequent at lowest at lowest + * index. + * + * ERTS_SSI_AUX_WORK_DEBUG_WAIT_COMPLETED_IX *need* to be + * highest index... + * + * Remember to update description in erts_pre_init_process() + * when adding new flags... */ -#define ERTS_SSI_AUX_WORK_DELAYED_AW_WAKEUP (((erts_aint32_t) 1) << 0) -#define ERTS_SSI_AUX_WORK_DD (((erts_aint32_t) 1) << 1) -#define ERTS_SSI_AUX_WORK_DD_THR_PRGR (((erts_aint32_t) 1) << 2) -#define ERTS_SSI_AUX_WORK_FIX_ALLOC_DEALLOC (((erts_aint32_t) 1) << 3) -#define ERTS_SSI_AUX_WORK_FIX_ALLOC_LOWER_LIM (((erts_aint32_t) 1) << 4) -#define ERTS_SSI_AUX_WORK_THR_PRGR_LATER_OP (((erts_aint32_t) 1) << 5) -#define ERTS_SSI_AUX_WORK_ASYNC_READY (((erts_aint32_t) 1) << 6) -#define ERTS_SSI_AUX_WORK_ASYNC_READY_CLEAN (((erts_aint32_t) 1) << 7) -#define ERTS_SSI_AUX_WORK_MISC_THR_PRGR (((erts_aint32_t) 1) << 8) -#define ERTS_SSI_AUX_WORK_MISC (((erts_aint32_t) 1) << 9) -#define ERTS_SSI_AUX_WORK_CHECK_CHILDREN (((erts_aint32_t) 1) << 10) -#define ERTS_SSI_AUX_WORK_SET_TMO (((erts_aint32_t) 1) << 11) -#define ERTS_SSI_AUX_WORK_MSEG_CACHE_CHECK (((erts_aint32_t) 1) << 12) -#define ERTS_SSI_AUX_WORK_REAP_PORTS (((erts_aint32_t) 1) << 13) +typedef enum { + ERTS_SSI_AUX_WORK_DELAYED_AW_WAKEUP_IX, + ERTS_SSI_AUX_WORK_DD_IX, + ERTS_SSI_AUX_WORK_DD_THR_PRGR_IX, + ERTS_SSI_AUX_WORK_FIX_ALLOC_DEALLOC_IX, + ERTS_SSI_AUX_WORK_FIX_ALLOC_LOWER_LIM_IX, + ERTS_SSI_AUX_WORK_THR_PRGR_LATER_OP_IX, + ERTS_SSI_AUX_WORK_CNCLD_TMRS_IX, + ERTS_SSI_AUX_WORK_CNCLD_TMRS_THR_PRGR_IX, + ERTS_SSI_AUX_WORK_ASYNC_READY_IX, + ERTS_SSI_AUX_WORK_ASYNC_READY_CLEAN_IX, + ERTS_SSI_AUX_WORK_MISC_THR_PRGR_IX, + ERTS_SSI_AUX_WORK_MISC_IX, + ERTS_SSI_AUX_WORK_SET_TMO_IX, + ERTS_SSI_AUX_WORK_MSEG_CACHE_CHECK_IX, + ERTS_SSI_AUX_WORK_REAP_PORTS_IX, + ERTS_SSI_AUX_WORK_DEBUG_WAIT_COMPLETED_IX, /* SHOULD be last flag index */ + + ERTS_SSI_AUX_WORK_NO_FLAGS /* Not a flag index... */ +} ErtsSsiAuxWorkFlagIndex; + +#define ERTS_SSI_AUX_WORK_DELAYED_AW_WAKEUP \ + (((erts_aint32_t) 1) << ERTS_SSI_AUX_WORK_DELAYED_AW_WAKEUP_IX) +#define ERTS_SSI_AUX_WORK_DD \ + (((erts_aint32_t) 1) << ERTS_SSI_AUX_WORK_DD_IX) +#define ERTS_SSI_AUX_WORK_DD_THR_PRGR \ + (((erts_aint32_t) 1) << ERTS_SSI_AUX_WORK_DD_THR_PRGR_IX) +#define ERTS_SSI_AUX_WORK_FIX_ALLOC_DEALLOC \ + (((erts_aint32_t) 1) << ERTS_SSI_AUX_WORK_FIX_ALLOC_DEALLOC_IX) +#define ERTS_SSI_AUX_WORK_FIX_ALLOC_LOWER_LIM \ + (((erts_aint32_t) 1) << ERTS_SSI_AUX_WORK_FIX_ALLOC_LOWER_LIM_IX) +#define ERTS_SSI_AUX_WORK_THR_PRGR_LATER_OP \ + (((erts_aint32_t) 1) << ERTS_SSI_AUX_WORK_THR_PRGR_LATER_OP_IX) +#define ERTS_SSI_AUX_WORK_CNCLD_TMRS \ + (((erts_aint32_t) 1) << ERTS_SSI_AUX_WORK_CNCLD_TMRS_IX) +#define ERTS_SSI_AUX_WORK_CNCLD_TMRS_THR_PRGR \ + (((erts_aint32_t) 1) << ERTS_SSI_AUX_WORK_CNCLD_TMRS_THR_PRGR_IX) +#define ERTS_SSI_AUX_WORK_ASYNC_READY \ + (((erts_aint32_t) 1) << ERTS_SSI_AUX_WORK_ASYNC_READY_IX) +#define ERTS_SSI_AUX_WORK_ASYNC_READY_CLEAN \ + (((erts_aint32_t) 1) << ERTS_SSI_AUX_WORK_ASYNC_READY_CLEAN_IX) +#define ERTS_SSI_AUX_WORK_MISC_THR_PRGR \ + (((erts_aint32_t) 1) << ERTS_SSI_AUX_WORK_MISC_THR_PRGR_IX) +#define ERTS_SSI_AUX_WORK_MISC \ + (((erts_aint32_t) 1) << ERTS_SSI_AUX_WORK_MISC_IX) +#define ERTS_SSI_AUX_WORK_SET_TMO \ + (((erts_aint32_t) 1) << ERTS_SSI_AUX_WORK_SET_TMO_IX) +#define ERTS_SSI_AUX_WORK_MSEG_CACHE_CHECK \ + (((erts_aint32_t) 1) << ERTS_SSI_AUX_WORK_MSEG_CACHE_CHECK_IX) +#define ERTS_SSI_AUX_WORK_REAP_PORTS \ + (((erts_aint32_t) 1) << ERTS_SSI_AUX_WORK_REAP_PORTS_IX) +#define ERTS_SSI_AUX_WORK_DEBUG_WAIT_COMPLETED \ + (((erts_aint32_t) 1) << ERTS_SSI_AUX_WORK_DEBUG_WAIT_COMPLETED_IX) typedef struct ErtsSchedulerSleepInfo_ ErtsSchedulerSleepInfo; @@ -340,7 +404,7 @@ typedef struct { } ErtsRunQueueInfo; -#ifdef HAVE_GETHRTIME +#ifdef ERTS_HAVE_OS_MONOTONIC_TIME_SUPPORT # undef ERTS_HAVE_SCHED_UTIL_BALANCING_SUPPORT_OPT # define ERTS_HAVE_SCHED_UTIL_BALANCING_SUPPORT_OPT 1 #endif @@ -350,13 +414,7 @@ typedef struct { #undef ERTS_HAVE_SCHED_UTIL_BALANCING_SUPPORT #define ERTS_HAVE_SCHED_UTIL_BALANCING_SUPPORT ERTS_HAVE_SCHED_UTIL_BALANCING_SUPPORT_OPT -#ifdef ARCH_64 -typedef erts_atomic_t ErtsAtomicSchedTime; -#elif defined(ARCH_32) -typedef erts_dw_atomic_t ErtsAtomicSchedTime; -#else -# error :-/ -#endif +typedef erts_atomic64_t ErtsAtomicSchedTime; #if ERTS_HAVE_SCHED_UTIL_BALANCING_SUPPORT typedef struct { @@ -417,7 +475,7 @@ struct ErtsRunQueue_ { int full_reds_history[ERTS_FULL_REDS_HISTORY_SIZE]; int out_of_work_count; erts_aint32_t max_len; - erts_aint32_t len; + erts_smp_atomic32_t len; int wakeup_other; int wakeup_other_reds; int halt_in_progress; @@ -461,19 +519,21 @@ typedef union { extern ErtsAlignedRunQueue *erts_aligned_run_queues; -#define ERTS_PROC_REDUCTIONS_EXECUTED(RQ, PRIO, REDS, AREDS) \ +#define ERTS_PROC_REDUCTIONS_EXECUTED(SD, RQ, PRIO, REDS, AREDS)\ do { \ (RQ)->procs.reductions += (AREDS); \ (RQ)->procs.prio_info[(PRIO)].reds += (REDS); \ (RQ)->check_balance_reds -= (REDS); \ (RQ)->wakeup_other_reds += (AREDS); \ + (SD)->check_time_reds += (AREDS); \ } while (0) -#define ERTS_PORT_REDUCTIONS_EXECUTED(RQ, REDS) \ +#define ERTS_PORT_REDUCTIONS_EXECUTED(SD, RQ, REDS) \ do { \ (RQ)->ports.info.reds += (REDS); \ (RQ)->check_balance_reds -= (REDS); \ (RQ)->wakeup_other_reds += (REDS); \ + (SD)->check_time_reds += (REDS); \ } while (0) typedef struct { @@ -488,11 +548,6 @@ typedef struct { } ErtsSchedWallTime; typedef struct { - Uint64 reclaimed; - Uint64 garbage_cols; -} ErtsGCInfo; - -typedef struct { int sched; erts_aint32_t aux_work; } ErtsDelayedAuxWorkWakeupJob; @@ -514,11 +569,12 @@ typedef struct { #ifdef ERTS_SMP struct { ErtsThrPrgrVal thr_prgr; - void (*completed_callback)(void *); - void (*completed_arg)(void *); } dd; struct { ErtsThrPrgrVal thr_prgr; + } cncld_tmrs; + struct { + ErtsThrPrgrVal thr_prgr; UWord size; ErtsThrPrgrLaterOp *first; ErtsThrPrgrLaterOp *last; @@ -541,6 +597,13 @@ typedef struct { ErtsDelayedAuxWorkWakeupJob *job; } delayed_wakeup; #endif + struct { + struct { + erts_aint32_t flags; + void (*callback)(void *); + void *arg; + } wait_completed; + } debug; } ErtsAuxWorkData; #ifdef ERTS_DIRTY_SCHEDULERS @@ -552,7 +615,7 @@ typedef enum { typedef union { struct { ErtsDirtySchedulerType type: 1; - unsigned num: 31; + Uint num: sizeof(Uint)*8 - 1; } s; Uint no; } ErtsDirtySchedId; @@ -567,6 +630,9 @@ struct ErtsSchedulerData_ { Eterm* x_reg_array; /* X registers */ FloatDef* f_reg_array; /* Floating point registers. */ + ErtsTimerWheel *timer_wheel; + ErtsNextTimeoutRef next_tmo_ref; + ErtsHLTimerService *timer_service; #ifdef ERTS_SMP ethr_tid tid; /* Thread id */ struct erl_bits_state erl_bits_state; /* erl_bits.c state */ @@ -574,12 +640,6 @@ struct ErtsSchedulerData_ { Process *free_process; ErtsThrPrgrData thr_progress_data; #endif -#if !HEAP_ON_C_STACK - Eterm tmp_heap[TMP_HEAP_SIZE]; - int num_tmp_heap_used; - Eterm beam_emu_tmp_heap[BEAM_EMU_TMP_HEAP_SIZE]; - Eterm erl_arith_tmp_heap[ERL_ARITH_TMP_HEAP_SIZE]; -#endif ErtsSchedulerSleepInfo *ssi; Process *current_process; Uint no; /* Scheduler number for normal schedulers */ @@ -593,8 +653,20 @@ struct ErtsSchedulerData_ { ErtsAuxWorkData aux_work_data; ErtsAtomCacheMap atom_cache_map; + ErtsMonotonicTime last_monotonic_time; + int check_time_reds; + + Uint32 thr_id; + Uint64 unique; + Uint64 ref; + ErtsSchedAllocData alloc_data; + struct { + Uint64 out; + Uint64 in; + } io; + Uint64 reductions; ErtsSchedWallTime sched_wall_time; ErtsGCInfo gc_info; @@ -621,13 +693,6 @@ extern ErtsAlignedSchedulerData *erts_aligned_dirty_io_scheduler_data; extern ErtsSchedulerData *erts_scheduler_data; #endif -#ifdef ERTS_SCHED_FAIR -#define ERTS_SCHED_FAIR_YIELD() ETHR_YIELD() -#else -#define ERTS_SCHED_FAIR 0 -#define ERTS_SCHED_FAIR_YIELD() -#endif - #if defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_CHECK) int erts_smp_lc_runq_is_locked(ErtsRunQueue *); #endif @@ -658,7 +723,19 @@ erts_smp_inc_runq_len(ErtsRunQueue *rq, ErtsRunQueueInfo *rqi, int prio) ERTS_SMP_LC_ASSERT(erts_smp_lc_runq_is_locked(rq)); - len = erts_smp_atomic32_read_nob(&rqi->len); + len = erts_smp_atomic32_read_dirty(&rq->len); + +#ifdef ERTS_SMP + if (len == 0) + erts_non_empty_runq(rq); +#endif + len++; + if (rq->max_len < len) + rq->max_len = len; + ASSERT(len > 0); + erts_smp_atomic32_set_nob(&rq->len, len); + + len = erts_smp_atomic32_read_dirty(&rqi->len); ASSERT(len >= 0); if (len == 0) { ASSERT((erts_smp_atomic32_read_nob(&rq->flags) @@ -671,15 +748,6 @@ erts_smp_inc_runq_len(ErtsRunQueue *rq, ErtsRunQueueInfo *rqi, int prio) rqi->max_len = len; erts_smp_atomic32_set_relb(&rqi->len, len); - -#ifdef ERTS_SMP - if (rq->len == 0) - erts_non_empty_runq(rq); -#endif - rq->len++; - if (rq->max_len < rq->len) - rq->max_len = len; - ASSERT(rq->len > 0); } ERTS_GLB_INLINE void @@ -689,7 +757,12 @@ erts_smp_dec_runq_len(ErtsRunQueue *rq, ErtsRunQueueInfo *rqi, int prio) ERTS_SMP_LC_ASSERT(erts_smp_lc_runq_is_locked(rq)); - len = erts_smp_atomic32_read_nob(&rqi->len); + len = erts_smp_atomic32_read_dirty(&rq->len); + len--; + ASSERT(len >= 0); + erts_smp_atomic32_set_nob(&rq->len, len); + + len = erts_smp_atomic32_read_dirty(&rqi->len); len--; ASSERT(len >= 0); if (len == 0) { @@ -700,8 +773,6 @@ erts_smp_dec_runq_len(ErtsRunQueue *rq, ErtsRunQueueInfo *rqi, int prio) } erts_smp_atomic32_set_relb(&rqi->len, len); - rq->len--; - ASSERT(rq->len >= 0); } ERTS_GLB_INLINE void @@ -711,7 +782,7 @@ erts_smp_reset_max_len(ErtsRunQueue *rq, ErtsRunQueueInfo *rqi) ERTS_SMP_LC_ASSERT(erts_smp_lc_runq_is_locked(rq)); - len = erts_smp_atomic32_read_nob(&rqi->len); + len = erts_smp_atomic32_read_dirty(&rqi->len); ASSERT(rqi->max_len >= len); rqi->max_len = len; } @@ -731,16 +802,11 @@ erts_smp_reset_max_len(ErtsRunQueue *rq, ErtsRunQueueInfo *rqi) #define ERTS_PSD_ERROR_HANDLER 0 #define ERTS_PSD_SAVED_CALLS_BUF 1 #define ERTS_PSD_SCHED_ID 2 -#define ERTS_PSD_DIST_ENTRY 3 -#define ERTS_PSD_CALL_TIME_BP 4 -#define ERTS_PSD_DELAYED_GC_TASK_QS 5 -#ifdef ERTS_DIRTY_SCHEDULERS -#define ERTS_PSD_DIRTY_SCHED_TRAP_EXPORT 6 +#define ERTS_PSD_CALL_TIME_BP 3 +#define ERTS_PSD_DELAYED_GC_TASK_QS 4 +#define ERTS_PSD_NIF_TRAP_EXPORT 5 -#define ERTS_PSD_SIZE 7 -#else #define ERTS_PSD_SIZE 6 -#endif typedef struct { void *data[ERTS_PSD_SIZE]; @@ -752,25 +818,20 @@ typedef struct { #define ERTS_PSD_ERROR_HANDLER_BUF_GET_LOCKS ERTS_PROC_LOCK_MAIN #define ERTS_PSD_ERROR_HANDLER_BUF_SET_LOCKS ERTS_PROC_LOCK_MAIN -#define ERTS_PSD_SAVED_CALLS_BUF_GET_LOCKS ERTS_PROC_LOCK_MAIN -#define ERTS_PSD_SAVED_CALLS_BUF_SET_LOCKS ERTS_PROC_LOCK_MAIN +#define ERTS_PSD_SAVED_CALLS_BUF_GET_LOCKS ((ErtsProcLocks) 0) +#define ERTS_PSD_SAVED_CALLS_BUF_SET_LOCKS ((ErtsProcLocks) 0) #define ERTS_PSD_SCHED_ID_GET_LOCKS ERTS_PROC_LOCK_STATUS #define ERTS_PSD_SCHED_ID_SET_LOCKS ERTS_PROC_LOCK_STATUS -#define ERTS_PSD_DIST_ENTRY_GET_LOCKS ERTS_PROC_LOCK_MAIN -#define ERTS_PSD_DIST_ENTRY_SET_LOCKS ERTS_PROC_LOCK_MAIN - #define ERTS_PSD_CALL_TIME_BP_GET_LOCKS ERTS_PROC_LOCK_MAIN #define ERTS_PSD_CALL_TIME_BP_SET_LOCKS ERTS_PROC_LOCK_MAIN #define ERTS_PSD_DELAYED_GC_TASK_QS_GET_LOCKS ERTS_PROC_LOCK_MAIN #define ERTS_PSD_DELAYED_GC_TASK_QS_SET_LOCKS ERTS_PROC_LOCK_MAIN -#ifdef ERTS_DIRTY_SCHEDULERS -#define ERTS_PSD_DIRTY_SCHED_TRAP_EXPORT_GET_LOCKS ERTS_PROC_LOCK_MAIN -#define ERTS_PSD_DIRTY_SCHED_TRAP_EXPORT_SET_LOCKS ERTS_PROC_LOCK_MAIN -#endif +#define ERTS_PSD_NIF_TRAP_EXPORT_GET_LOCKS ((ErtsProcLocks) 0) +#define ERTS_PSD_NIF_TRAP_EXPORT_SET_LOCKS ((ErtsProcLocks) 0) typedef struct { ErtsProcLocks get_locks; @@ -847,7 +908,6 @@ struct ErtsPendingSuspend_ { # define MIN_VHEAP_SIZE(p) (p)->min_vheap_size # define BIN_VHEAP_SZ(p) (p)->bin_vheap_sz -# define BIN_VHEAP_MATURE(p) (p)->bin_vheap_mature # define BIN_OLD_VHEAP_SZ(p) (p)->bin_old_vheap_sz # define BIN_OLD_VHEAP(p) (p)->bin_old_vheap @@ -865,6 +925,7 @@ struct process { Eterm* stop; /* Stack top */ Eterm* heap; /* Heap start */ Eterm* hend; /* Heap end */ + Eterm* abandoned_heap; Uint heap_sz; /* Size of heap in words */ Uint min_heap_size; /* Minimum size of heap (in words). */ Uint min_vheap_size; /* Minimum size of virtual heap (in words). */ @@ -916,10 +977,10 @@ struct process { ErlMessageQueue msg; /* Message queue */ - union { - ErtsBifTimer *bif_timers; /* Bif timers aiming at this process */ - void *terminate; - } u; + ErtsBifTimers *bif_timers; /* Bif timers aiming at this process */ +#ifdef ERTS_BTM_ACCESSOR_SUPPORT + ErtsBifTimers *accessor_bif_timers; /* Accessor bif timers */ +#endif ProcDict *dictionary; /* Process dictionary, may be NULL */ @@ -930,9 +991,12 @@ struct process { #ifdef USE_VM_PROBES Eterm dt_utag; /* Place to store the dynamc trace user tag */ Uint dt_utag_flags; /* flag field for the dt_utag */ -#endif - BeamInstr initial[3]; /* Initial module(0), function(1), arity(2), often used instead +#endif + union { + void *terminate; + BeamInstr initial[3]; /* Initial module(0), function(1), arity(2), often used instead of pointer to funcinfo instruction, hence the BeamInstr datatype */ + } u; BeamInstr* current; /* Current Erlang function, part of the funcinfo: * module(0), function(1), arity(2) * (module and functions are tagged atoms; @@ -945,6 +1009,8 @@ struct process { Eterm parent; /* Pid of process that created this process. */ erts_approx_time_t approx_started; /* Time when started. */ + Uint32 static_flags; /* Flags that do *not* change */ + /* This is the place, where all fields that differs between memory * architectures, have gone to. */ @@ -956,18 +1022,22 @@ struct process { Uint16 gen_gcs; /* Number of (minor) generational GCs. */ Uint16 max_gen_gcs; /* Max minor gen GCs before fullsweep. */ ErlOffHeap off_heap; /* Off-heap data updated by copy_struct(). */ - ErlHeapFragment* mbuf; /* Pointer to message buffer list */ - Uint mbuf_sz; /* Size of all message buffers */ - ErtsPSD *psd; /* Rarely used process specific data */ + ErlHeapFragment* mbuf; /* Pointer to heap fragment list */ + ErlHeapFragment* live_hf_end; + ErtsMessage *msg_frag; /* Pointer to message fragment list */ + Uint mbuf_sz; /* Total size of heap fragments and message fragments */ + erts_smp_atomic_t psd; /* Rarely used process specific data */ Uint64 bin_vheap_sz; /* Virtual heap block size for binaries */ - Uint64 bin_vheap_mature; /* Virtual heap block size for binaries */ Uint64 bin_old_vheap_sz; /* Virtual old heap block size for binaries */ Uint64 bin_old_vheap; /* Virtual old heap size for binaries */ ErtsProcSysTaskQs *sys_task_qs; erts_smp_atomic32_t state; /* Process state flags (see ERTS_PSFLG_*) */ +#ifdef ERTS_DIRTY_SCHEDULERS + erts_smp_atomic32_t dirty_state; /* Process dirty state flags (see ERTS_PDSFLG_*) */ +#endif #ifdef ERTS_SMP ErlMessageInQueue msg_inq; @@ -985,6 +1055,7 @@ struct process { #ifdef CHECK_FOR_HOLES Eterm* last_htop; /* No need to scan the heap below this point. */ ErlHeapFragment* last_mbuf; /* No need to scan beyond this mbuf. */ + ErlHeapFragment* heap_hfrag; /* Heap abandoned, htop now lives in this frag */ #endif #ifdef DEBUG @@ -1008,6 +1079,7 @@ extern const Process erts_invalid_process; do { \ (p)->last_htop = 0; \ (p)->last_mbuf = 0; \ + (p)->heap_hfrag = NULL; \ } while (0) # define ERTS_HOLE_CHECK(p) erts_check_for_holes((p)) @@ -1085,12 +1157,16 @@ void erts_check_for_holes(Process* p); #define ERTS_PSFLG_RUNNING_SYS ERTS_PSFLG_BIT(15) #define ERTS_PSFLG_PROXY ERTS_PSFLG_BIT(16) #define ERTS_PSFLG_DELAYED_SYS ERTS_PSFLG_BIT(17) -#ifdef ERTS_DIRTY_SCHEDULERS -#define ERTS_PSFLG_DIRTY_CPU_PROC ERTS_PSFLG_BIT(18) -#define ERTS_PSFLG_DIRTY_IO_PROC ERTS_PSFLG_BIT(19) -#define ERTS_PSFLG_DIRTY_CPU_PROC_IN_Q ERTS_PSFLG_BIT(20) -#define ERTS_PSFLG_DIRTY_IO_PROC_IN_Q ERTS_PSFLG_BIT(21) -#endif +#define ERTS_PSFLG_OFF_HEAP_MSGQ ERTS_PSFLG_BIT(18) +#define ERTS_PSFLG_ON_HEAP_MSGQ ERTS_PSFLG_BIT(19) +#define ERTS_PSFLG_DIRTY_CPU_PROC ERTS_PSFLG_BIT(20) +#define ERTS_PSFLG_DIRTY_IO_PROC ERTS_PSFLG_BIT(21) +#define ERTS_PSFLG_DIRTY_ACTIVE_SYS ERTS_PSFLG_BIT(22) +#define ERTS_PSFLG_MAX (ERTS_PSFLGS_ZERO_BIT_OFFSET + 22) + +#define ERTS_PSFLGS_DIRTY_WORK (ERTS_PSFLG_DIRTY_CPU_PROC \ + | ERTS_PSFLG_DIRTY_IO_PROC \ + | ERTS_PSFLG_DIRTY_ACTIVE_SYS) #define ERTS_PSFLGS_IN_PRQ_MASK (ERTS_PSFLG_IN_PRQ_MAX \ | ERTS_PSFLG_IN_PRQ_HIGH \ @@ -1102,7 +1178,42 @@ void erts_check_for_holes(Process* p); #define ERTS_PSFLGS_GET_USR_PRIO(PSFLGS) \ (((PSFLGS) >> ERTS_PSFLGS_USR_PRIO_OFFSET) & ERTS_PSFLGS_PRIO_MASK) #define ERTS_PSFLGS_GET_PRQ_PRIO(PSFLGS) \ - (((PSFLGS) >> ERTS_PSFLGS_USR_PRIO_OFFSET) & ERTS_PSFLGS_PRIO_MASK) + (((PSFLGS) >> ERTS_PSFLGS_PRQ_PRIO_OFFSET) & ERTS_PSFLGS_PRIO_MASK) + +#ifdef ERTS_DIRTY_SCHEDULERS + +/* + * Flags in the dirty_state field. + */ + +#define ERTS_PDSFLG_IN_CPU_PRQ_MAX (((erts_aint32_t) 1) << 0) +#define ERTS_PDSFLG_IN_CPU_PRQ_HIGH (((erts_aint32_t) 1) << 1) +#define ERTS_PDSFLG_IN_CPU_PRQ_NORMAL (((erts_aint32_t) 1) << 2) +#define ERTS_PDSFLG_IN_CPU_PRQ_LOW (((erts_aint32_t) 1) << 3) +#define ERTS_PDSFLG_IN_IO_PRQ_MAX (((erts_aint32_t) 1) << 4) +#define ERTS_PDSFLG_IN_IO_PRQ_HIGH (((erts_aint32_t) 1) << 5) +#define ERTS_PDSFLG_IN_IO_PRQ_NORMAL (((erts_aint32_t) 1) << 6) +#define ERTS_PDSFLG_IN_IO_PRQ_LOW (((erts_aint32_t) 1) << 7) + +#define ERTS_PDSFLGS_QMASK ERTS_PSFLGS_QMASK +#define ERTS_PDSFLGS_IN_CPU_PRQ_MASK_OFFSET 0 +#define ERTS_PDSFLGS_IN_IO_PRQ_MASK_OFFSET ERTS_PSFLGS_QMASK_BITS + +#define ERTS_PDSFLG_IN_CPU_PRQ_MASK (ERTS_PDSFLG_IN_CPU_PRQ_MAX \ + | ERTS_PDSFLG_IN_CPU_PRQ_HIGH \ + | ERTS_PDSFLG_IN_CPU_PRQ_NORMAL\ + | ERTS_PDSFLG_IN_CPU_PRQ_LOW) +#define ERTS_PDSFLG_IN_IO_PRQ_MASK (ERTS_PDSFLG_IN_CPU_PRQ_MAX \ + | ERTS_PDSFLG_IN_CPU_PRQ_HIGH \ + | ERTS_PDSFLG_IN_CPU_PRQ_NORMAL\ + | ERTS_PDSFLG_IN_CPU_PRQ_LOW) +#endif + + +/* + * Static flags that do not change after process creation. + */ +#define ERTS_STC_FLG_SYSTEM_PROC (((Uint32) 1) << 0) /* The sequential tracing token is a tuple of size 5: * @@ -1131,12 +1242,17 @@ void erts_check_for_holes(Process* p); #define SPO_LINK 1 #define SPO_USE_ARGS 2 #define SPO_MONITOR 4 +#define SPO_SYSTEM_PROC 8 +#define SPO_OFF_HEAP_MSGQ 16 +#define SPO_ON_HEAP_MSGQ 32 + +extern int erts_default_spo_flags; /* * The following struct contains options for a process to be spawned. */ typedef struct { - Uint flags; + int flags; int error_code; /* Error code returned from create_process(). */ Eterm mref; /* Monitor ref returned (if SPO_MONITOR was given). */ @@ -1176,7 +1292,6 @@ Eterm* erts_heap_alloc(Process* p, Uint need, Uint xtra); Eterm* erts_set_hole_marker(Eterm* ptr, Uint sz); #endif -extern Uint erts_default_process_flags; extern erts_smp_rwmtx_t erts_cpu_bind_rwmtx; /* If any of the erts_system_monitor_* variables are set (enabled), ** erts_system_monitor must be != NIL, to allow testing on just @@ -1204,6 +1319,7 @@ struct erts_system_profile_flags_t { unsigned int exclusive : 1; }; extern struct erts_system_profile_flags_t erts_system_profile_flags; +extern int erts_system_profile_ts_type; /* process flags */ #define F_HIBERNATE_SCHED (1 << 0) /* Schedule out after hibernate op */ @@ -1217,65 +1333,110 @@ extern struct erts_system_profile_flags_t erts_system_profile_flags; #define F_HAVE_BLCKD_MSCHED (1 << 8) /* Process has blocked multi-scheduling */ #define F_P2PNR_RESCHED (1 << 9) /* Process has been rescheduled via erts_pid2proc_not_running() */ #define F_FORCE_GC (1 << 10) /* Force gc at process in-scheduling */ -#define F_DISABLE_GC (1 << 11) /* Disable GC */ +#define F_DISABLE_GC (1 << 11) /* Disable GC (see below) */ +#define F_OFF_HEAP_MSGQ (1 << 12) /* Off heap msg queue */ +#define F_ON_HEAP_MSGQ (1 << 13) /* Off heap msg queue */ +#define F_OFF_HEAP_MSGQ_CHNG (1 << 14) /* Off heap msg queue changing */ +#define F_ABANDONED_HEAP_USE (1 << 15) /* Have usage of abandoned heap */ +#define F_DELAY_GC (1 << 16) /* Similar to disable GC (see below) */ +#define F_SCHDLR_ONLN_WAITQ (1 << 17) /* Process enqueued waiting to change schedulers online */ +#define F_HAVE_BLCKD_NMSCHED (1 << 18) /* Process has blocked normal multi-scheduling */ + +/* + * F_DISABLE_GC and F_DELAY_GC are similar. Both will prevent + * GC of the process, but it is important to use the right + * one: + * - F_DISABLE_GC should *only* be used by BIFs. This when + * the BIF needs to yield while preventig a GC. + * - F_DELAY_GC should only be used when GC is temporarily + * disabled while the process is scheduled. A process must + * not be scheduled out while F_DELAY_GC is set. + */ + +#define ERTS_TRACE_FLAGS_TS_TYPE_SHIFT 0 + +#define F_TRACE_FLAG(N) (1 << (ERTS_TRACE_TS_TYPE_BITS + (N))) /* process trace_flags */ -#define F_SENSITIVE (1 << 0) -#define F_TRACE_SEND (1 << 1) -#define F_TRACE_RECEIVE (1 << 2) -#define F_TRACE_SOS (1 << 3) /* Set on spawn */ -#define F_TRACE_SOS1 (1 << 4) /* Set on first spawn */ -#define F_TRACE_SOL (1 << 5) /* Set on link */ -#define F_TRACE_SOL1 (1 << 6) /* Set on first link */ -#define F_TRACE_CALLS (1 << 7) -#define F_TIMESTAMP (1 << 8) -#define F_TRACE_PROCS (1 << 9) -#define F_TRACE_FIRST_CHILD (1 << 10) -#define F_TRACE_SCHED (1 << 11) -#define F_TRACE_GC (1 << 12) -#define F_TRACE_ARITY_ONLY (1 << 13) -#define F_TRACE_RETURN_TO (1 << 14) /* Return_to trace when breakpoint tracing */ -#define F_TRACE_SILENT (1 << 15) /* No call trace msg suppress */ -#define F_TRACER (1 << 16) /* May be (has been) tracer */ -#define F_EXCEPTION_TRACE (1 << 17) /* May have exception trace on stack */ + +#define F_NOW_TS (ERTS_TRACE_FLG_NOW_TIMESTAMP \ + << ERTS_TRACE_FLAGS_TS_TYPE_SHIFT) +#define F_STRICT_MON_TS (ERTS_TRACE_FLG_STRICT_MONOTONIC_TIMESTAMP \ + << ERTS_TRACE_FLAGS_TS_TYPE_SHIFT) +#define F_MON_TS (ERTS_TRACE_FLG_MONOTONIC_TIMESTAMP \ + << ERTS_TRACE_FLAGS_TS_TYPE_SHIFT) +#define F_SENSITIVE F_TRACE_FLAG(0) +#define F_TRACE_SEND F_TRACE_FLAG(1) +#define F_TRACE_RECEIVE F_TRACE_FLAG(2) +#define F_TRACE_SOS F_TRACE_FLAG(3) /* Set on spawn */ +#define F_TRACE_SOS1 F_TRACE_FLAG(4) /* Set on first spawn */ +#define F_TRACE_SOL F_TRACE_FLAG(5) /* Set on link */ +#define F_TRACE_SOL1 F_TRACE_FLAG(6) /* Set on first link */ +#define F_TRACE_CALLS F_TRACE_FLAG(7) +#define F_TRACE_PROCS F_TRACE_FLAG(8) +#define F_TRACE_FIRST_CHILD F_TRACE_FLAG(9) +#define F_TRACE_SCHED F_TRACE_FLAG(10) +#define F_TRACE_GC F_TRACE_FLAG(11) +#define F_TRACE_ARITY_ONLY F_TRACE_FLAG(12) +#define F_TRACE_RETURN_TO F_TRACE_FLAG(13) /* Return_to trace when breakpoint tracing */ +#define F_TRACE_SILENT F_TRACE_FLAG(14) /* No call trace msg suppress */ +#define F_TRACER F_TRACE_FLAG(15) /* May be (has been) tracer */ +#define F_EXCEPTION_TRACE F_TRACE_FLAG(16) /* May have exception trace on stack */ /* port trace flags, currently the same as process trace flags */ -#define F_TRACE_SCHED_PORTS (1 << 18) /* Trace of port scheduling */ -#define F_TRACE_SCHED_PROCS (1 << 19) /* With virtual scheduling */ -#define F_TRACE_PORTS (1 << 20) /* Ports equivalent to F_TRACE_PROCS */ -#define F_TRACE_SCHED_NO (1 << 21) /* Trace with scheduler id */ -#define F_TRACE_SCHED_EXIT (1 << 22) +#define F_TRACE_SCHED_PORTS F_TRACE_FLAG(17) /* Trace of port scheduling */ +#define F_TRACE_SCHED_PROCS F_TRACE_FLAG(18) /* With virtual scheduling */ +#define F_TRACE_PORTS F_TRACE_FLAG(19) /* Ports equivalent to F_TRACE_PROCS */ +#define F_TRACE_SCHED_NO F_TRACE_FLAG(20) /* Trace with scheduler id */ +#define F_TRACE_SCHED_EXIT F_TRACE_FLAG(21) -#define F_NUM_FLAGS 23 +#define F_NUM_FLAGS (ERTS_TRACE_TS_TYPE_BITS + 22) #ifdef DEBUG # define F_INITIAL_TRACE_FLAGS (5 << F_NUM_FLAGS) #else # define F_INITIAL_TRACE_FLAGS 0 #endif - +/* F_TIMESTAMP_MASK is a bit-field of all all timestamp types */ +#define F_TIMESTAMP_MASK \ + (ERTS_TRACE_TS_TYPE_MASK << ERTS_TRACE_FLAGS_TS_TYPE_SHIFT) #define TRACEE_FLAGS ( F_TRACE_PROCS | F_TRACE_CALLS \ | F_TRACE_SOS | F_TRACE_SOS1| F_TRACE_RECEIVE \ | F_TRACE_SOL | F_TRACE_SOL1 | F_TRACE_SEND \ - | F_TRACE_SCHED | F_TIMESTAMP | F_TRACE_GC \ + | F_TRACE_SCHED | F_TIMESTAMP_MASK | F_TRACE_GC \ | F_TRACE_ARITY_ONLY | F_TRACE_RETURN_TO \ | F_TRACE_SILENT | F_TRACE_SCHED_PROCS | F_TRACE_PORTS \ | F_TRACE_SCHED_PORTS | F_TRACE_SCHED_NO \ | F_TRACE_SCHED_EXIT) #define ERTS_TRACEE_MODIFIER_FLAGS \ - (F_TRACE_SILENT | F_TIMESTAMP | F_TRACE_SCHED_NO) + (F_TRACE_SILENT | F_TIMESTAMP_MASK | F_TRACE_SCHED_NO) #define ERTS_PORT_TRACEE_FLAGS \ (ERTS_TRACEE_MODIFIER_FLAGS | F_TRACE_PORTS | F_TRACE_SCHED_PORTS) #define ERTS_PROC_TRACEE_FLAGS \ ((TRACEE_FLAGS & ~ERTS_PORT_TRACEE_FLAGS) | ERTS_TRACEE_MODIFIER_FLAGS) +#define SEQ_TRACE_FLAG(N) (1 << (ERTS_TRACE_TS_TYPE_BITS + (N))) + /* Sequential trace flags */ + +/* SEQ_TRACE_TIMESTAMP_MASK is a bit-field */ +#define SEQ_TRACE_TIMESTAMP_MASK \ + (ERTS_TRACE_TS_TYPE_MASK << ERTS_SEQ_TRACE_FLAGS_TS_TYPE_SHIFT) + #define SEQ_TRACE_SEND (1 << 0) #define SEQ_TRACE_RECEIVE (1 << 1) #define SEQ_TRACE_PRINT (1 << 2) -#define SEQ_TRACE_TIMESTAMP (1 << 3) + +#define ERTS_SEQ_TRACE_FLAGS_TS_TYPE_SHIFT 3 + +#define SEQ_TRACE_NOW_TS (ERTS_TRACE_FLG_NOW_TIMESTAMP \ + << ERTS_SEQ_TRACE_FLAGS_TS_TYPE_SHIFT) +#define SEQ_TRACE_STRICT_MON_TS (ERTS_TRACE_FLG_STRICT_MONOTONIC_TIMESTAMP \ + << ERTS_SEQ_TRACE_FLAGS_TS_TYPE_SHIFT) +#define SEQ_TRACE_MON_TS (ERTS_TRACE_FLG_MONOTONIC_TIMESTAMP \ + << ERTS_SEQ_TRACE_FLAGS_TS_TYPE_SHIFT) #ifdef USE_VM_PROBES #define DT_UTAG_PERMANENT (1 << 0) @@ -1288,12 +1449,14 @@ extern struct erts_system_profile_flags_t erts_system_profile_flags; #define ERTS_XSIG_FLG_IGN_KILL (((Uint32) 1) << 0) #define ERTS_XSIG_FLG_NO_IGN_NORMAL (((Uint32) 1) << 1) -#define CANCEL_TIMER(p) \ - do { \ - if ((p)->flags & (F_INSLPQUEUE)) \ - cancel_timer(p); \ - else \ - (p)->flags &= ~F_TIMO; \ +#define CANCEL_TIMER(P) \ + do { \ + if ((P)->flags & (F_INSLPQUEUE|F_TIMO)) { \ + if ((P)->flags & F_INSLPQUEUE) \ + erts_cancel_proc_timer((P)); \ + else \ + (P)->flags &= ~F_TIMO; \ + } \ } while (0) #if defined(ERTS_DIRTY_SCHEDULERS) && defined(ERTS_SMP) @@ -1362,12 +1525,17 @@ void erts_init_scheduling(int, int int erts_set_gc_state(Process *c_p, int enable); Eterm erts_sched_wall_time_request(Process *c_p, int set, int enable); +Eterm erts_system_check_request(Process *c_p); Eterm erts_gc_info_request(Process *c_p); Uint64 erts_get_proc_interval(void); Uint64 erts_ensure_later_proc_interval(Uint64); Uint64 erts_step_proc_interval(void); +int erts_setup_nif_gc(Process* proc, Eterm** objv, int* nobj); /* see erl_nif.c */ +void erts_destroy_nif_export(void *); /* see erl_nif.c */ + ErtsProcList *erts_proclist_create(Process *); +ErtsProcList *erts_proclist_copy(ErtsProcList *); void erts_proclist_destroy(ErtsProcList *); ERTS_GLB_INLINE int erts_proclist_same(ErtsProcList *, Process *); @@ -1548,36 +1716,30 @@ void erts_schedule_thr_prgr_later_cleanup_op(void (*)(void *), void *, ErtsThrPrgrLaterOp *, UWord); +void erts_schedule_complete_off_heap_message_queue_change(Eterm pid); #if defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_CHECK) int erts_dbg_check_halloc_lock(Process *p); #endif -#ifdef DEBUG -void erts_dbg_multi_scheduling_return_trap(Process *, Eterm); -#endif -int erts_get_max_no_executing_schedulers(void); #if defined(ERTS_SMP) || defined(ERTS_DIRTY_SCHEDULERS) -ErtsSchedSuspendResult -erts_schedulers_state(Uint *, Uint *, Uint *, Uint *, Uint *, Uint *, int); +void +erts_schedulers_state(Uint *, Uint *, Uint *, Uint *, Uint *, Uint *, Uint *, Uint *); #endif #ifdef ERTS_SMP ErtsSchedSuspendResult erts_set_schedulers_online(Process *p, ErtsProcLocks plocks, Sint new_no, - Sint *old_no -#ifdef ERTS_DIRTY_SCHEDULERS - , int dirty_only -#endif - ); + Sint *old_no, + int dirty_only); ErtsSchedSuspendResult -erts_block_multi_scheduling(Process *, ErtsProcLocks, int, int); +erts_block_multi_scheduling(Process *, ErtsProcLocks, int, int, int); int erts_is_multi_scheduling_blocked(void); -Eterm erts_multi_scheduling_blockers(Process *); +Eterm erts_multi_scheduling_blockers(Process *, int); void erts_start_schedulers(void); void erts_alloc_notify_delayed_dealloc(int); void erts_alloc_ensure_handle_delayed_dealloc_call(int); -void erts_smp_notify_check_children_needed(void); +void erts_notify_canceled_timer(ErtsSchedulerData *, int); #endif #if ERTS_USE_ASYNC_READY_Q void erts_notify_check_async_ready_queue(void *); @@ -1598,7 +1760,7 @@ void erts_sched_notify_check_cpu_bind(void); Uint erts_active_schedulers(void); void erts_init_process(int, int, int); Eterm erts_process_status(Process *, ErtsProcLocks, Process *, Eterm); -Uint erts_run_queues_len(Uint *); +Uint erts_run_queues_len(Uint *, int, int); void erts_add_to_runq(Process *); Eterm erts_bound_schedulers_term(Process *c_p); Eterm erts_get_cpu_topology_term(Process *c_p, Eterm which); @@ -1611,8 +1773,6 @@ void erts_schedule_misc_op(void (*)(void *), void *); Eterm erl_create_process(Process*, Eterm, Eterm, Eterm, ErlSpawnOpts*); void erts_do_exit_process(Process*, Eterm); void erts_continue_exit_process(Process *); -void set_timer(Process*, Uint); -void cancel_timer(Process*); /* Begin System profile */ Uint erts_runnable_process_count(void); /* End System profile */ @@ -1622,7 +1782,11 @@ void erts_cleanup_empty_process(Process* p); void erts_debug_verify_clean_empty_process(Process* p); #endif void erts_stack_dump(int to, void *to_arg, Process *); +void erts_limited_stack_trace(int to, void *to_arg, Process *); void erts_program_counter_info(int to, void *to_arg, Process *); +void erts_print_scheduler_info(int to, void *to_arg, ErtsSchedulerData *esdp); +void erts_dump_extended_process_state(int to, void *to_arg, erts_aint32_t psflg); +void erts_dump_process_state(int to, void *to_arg, erts_aint32_t psflg); Eterm erts_get_process_priority(Process *p); Eterm erts_set_process_priority(Process *p, Eterm prio); @@ -1664,9 +1828,13 @@ Eterm erts_get_reader_groups_map(Process *c_p); Eterm erts_debug_reader_groups_map(Process *c_p, int groups); Uint erts_debug_nbalance(void); -int erts_debug_wait_deallocations(Process *c_p); -Uint erts_process_memory(Process *c_p); +#define ERTS_DEBUG_WAIT_COMPLETED_DEALLOCATIONS (1 << 0) +#define ERTS_DEBUG_WAIT_COMPLETED_TIMER_CANCELLATIONS (1 << 1) + +int erts_debug_wait_completed(Process *c_p, int flags); + +Uint erts_process_memory(Process *c_p, int incl_msg_inq); #ifdef ERTS_SMP # define ERTS_GET_SCHEDULER_DATA_FROM_PROC(PROC) ((PROC)->scheduler_data) @@ -1704,17 +1872,17 @@ ErtsSchedulerData *erts_get_scheduler_data(void) #endif #endif -void erts_schedule_process(Process *, erts_aint32_t); +void erts_schedule_process(Process *, erts_aint32_t, ErtsProcLocks); -ERTS_GLB_INLINE void erts_proc_notify_new_message(Process *p); +ERTS_GLB_INLINE void erts_proc_notify_new_message(Process *p, ErtsProcLocks locks); #if ERTS_GLB_INLINE_INCL_FUNC_DEF ERTS_GLB_INLINE void -erts_proc_notify_new_message(Process *p) +erts_proc_notify_new_message(Process *p, ErtsProcLocks locks) { /* No barrier needed, due to msg lock */ erts_aint32_t state = erts_smp_atomic32_read_nob(&p->state); if (!(state & ERTS_PSFLG_ACTIVE)) - erts_schedule_process(p, state); + erts_schedule_process(p, state, locks); } #endif @@ -1735,18 +1903,19 @@ do { \ #define ERTS_SMP_LC_CHK_RUNQ_LOCK(RQ, L) #endif -void *erts_psd_set_init(Process *p, ErtsProcLocks plocks, int ix, void *data); +void *erts_psd_set_init(Process *p, int ix, void *data); ERTS_GLB_INLINE void * erts_psd_get(Process *p, int ix); ERTS_GLB_INLINE void * -erts_psd_set(Process *p, ErtsProcLocks plocks, int ix, void *new); +erts_psd_set(Process *p, int ix, void *new); #if ERTS_GLB_INLINE_INCL_FUNC_DEF ERTS_GLB_INLINE void * erts_psd_get(Process *p, int ix) { + ErtsPSD *psd; #if defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_CHECK) ErtsProcLocks locks = erts_proc_lc_my_proc_locks(p); if (ERTS_LC_PSD_ANY_LOCK == erts_psd_required_locks[ix].get_locks) @@ -1757,17 +1926,19 @@ erts_psd_get(Process *p, int ix) || erts_thr_progress_is_blocking()); } #endif + + psd = (ErtsPSD *) erts_smp_atomic_read_nob(&p->psd); ASSERT(0 <= ix && ix < ERTS_PSD_SIZE); - return p->psd ? p->psd->data[ix] : NULL; + if (!psd) + return NULL; + ERTS_SMP_DATA_DEPENDENCY_READ_MEMORY_BARRIER; + return psd->data[ix]; } - -/* - * NOTE: erts_psd_set() might release and reacquire locks on 'p'. - */ ERTS_GLB_INLINE void * -erts_psd_set(Process *p, ErtsProcLocks plocks, int ix, void *data) +erts_psd_set(Process *p, int ix, void *data) { + ErtsPSD *psd; #if defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_CHECK) ErtsProcLocks locks = erts_proc_lc_my_proc_locks(p); if (ERTS_LC_PSD_ANY_LOCK == erts_psd_required_locks[ix].set_locks) @@ -1778,57 +1949,56 @@ erts_psd_set(Process *p, ErtsProcLocks plocks, int ix, void *data) || erts_thr_progress_is_blocking()); } #endif + psd = (ErtsPSD *) erts_smp_atomic_read_nob(&p->psd); ASSERT(0 <= ix && ix < ERTS_PSD_SIZE); - if (p->psd) { - void *old = p->psd->data[ix]; - p->psd->data[ix] = data; + if (psd) { + void *old; +#ifdef ERTS_SMP +#ifdef ETHR_ORDERED_READ_DEPEND + ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreStore); +#else + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreStore); +#endif +#endif + old = psd->data[ix]; + psd->data[ix] = data; return old; } - else { - if (!data) - return NULL; - else - return erts_psd_set_init(p, plocks, ix, data); - } + + if (!data) + return NULL; + + return erts_psd_set_init(p, ix, data); } #endif -#define ERTS_PROC_SCHED_ID(P, L, ID) \ - ((UWord) erts_psd_set((P), (L), ERTS_PSD_SCHED_ID, (void *) (ID))) - -#define ERTS_PROC_GET_DIST_ENTRY(P) \ - ((DistEntry *) erts_psd_get((P), ERTS_PSD_DIST_ENTRY)) -#define ERTS_PROC_SET_DIST_ENTRY(P, L, D) \ - ((DistEntry *) erts_psd_set((P), (L), ERTS_PSD_DIST_ENTRY, (void *) (D))) +#define ERTS_PROC_SCHED_ID(P, ID) \ + ((UWord) erts_psd_set((P), ERTS_PSD_SCHED_ID, (void *) (ID))) #define ERTS_PROC_GET_SAVED_CALLS_BUF(P) \ ((struct saved_calls *) erts_psd_get((P), ERTS_PSD_SAVED_CALLS_BUF)) -#define ERTS_PROC_SET_SAVED_CALLS_BUF(P, L, SCB) \ - ((struct saved_calls *) erts_psd_set((P), (L), ERTS_PSD_SAVED_CALLS_BUF, (void *) (SCB))) +#define ERTS_PROC_SET_SAVED_CALLS_BUF(P, SCB) \ + ((struct saved_calls *) erts_psd_set((P), ERTS_PSD_SAVED_CALLS_BUF, (void *) (SCB))) #define ERTS_PROC_GET_CALL_TIME(P) \ ((process_breakpoint_time_t *) erts_psd_get((P), ERTS_PSD_CALL_TIME_BP)) -#define ERTS_PROC_SET_CALL_TIME(P, L, PBT) \ - ((process_breakpoint_time_t *) erts_psd_set((P), (L), ERTS_PSD_CALL_TIME_BP, (void *) (PBT))) +#define ERTS_PROC_SET_CALL_TIME(P, PBT) \ + ((process_breakpoint_time_t *) erts_psd_set((P), ERTS_PSD_CALL_TIME_BP, (void *) (PBT))) #define ERTS_PROC_GET_DELAYED_GC_TASK_QS(P) \ ((ErtsProcSysTaskQs *) erts_psd_get((P), ERTS_PSD_DELAYED_GC_TASK_QS)) -#define ERTS_PROC_SET_DELAYED_GC_TASK_QS(P, L, PBT) \ - ((ErtsProcSysTaskQs *) erts_psd_set((P), (L), ERTS_PSD_DELAYED_GC_TASK_QS, (void *) (PBT))) +#define ERTS_PROC_SET_DELAYED_GC_TASK_QS(P, PBT) \ + ((ErtsProcSysTaskQs *) erts_psd_set((P), ERTS_PSD_DELAYED_GC_TASK_QS, (void *) (PBT))) -#ifdef ERTS_DIRTY_SCHEDULERS -#define ERTS_PROC_GET_DIRTY_SCHED_TRAP_EXPORT(P) \ - ((Export *) erts_psd_get((P), ERTS_PSD_DIRTY_SCHED_TRAP_EXPORT)) -#define ERTS_PROC_SET_DIRTY_SCHED_TRAP_EXPORT(P, L, DSTE) \ - ((Export *) erts_psd_set((P), (L), ERTS_PSD_DIRTY_SCHED_TRAP_EXPORT, (void *) (DSTE))) -#endif +#define ERTS_PROC_GET_NIF_TRAP_EXPORT(P) \ + erts_psd_get((P), ERTS_PSD_NIF_TRAP_EXPORT) +#define ERTS_PROC_SET_NIF_TRAP_EXPORT(P, NTE) \ + erts_psd_set((P), ERTS_PSD_NIF_TRAP_EXPORT, (void *) (NTE)) ERTS_GLB_INLINE Eterm erts_proc_get_error_handler(Process *p); -ERTS_GLB_INLINE Eterm erts_proc_set_error_handler(Process *p, - ErtsProcLocks plocks, - Eterm handler); +ERTS_GLB_INLINE Eterm erts_proc_set_error_handler(Process *p, Eterm handler); #if ERTS_GLB_INLINE_INCL_FUNC_DEF ERTS_GLB_INLINE Eterm @@ -1844,13 +2014,13 @@ erts_proc_get_error_handler(Process *p) } ERTS_GLB_INLINE Eterm -erts_proc_set_error_handler(Process *p, ErtsProcLocks plocks, Eterm handler) +erts_proc_set_error_handler(Process *p, Eterm handler) { void *old_val; void *new_val; ASSERT(is_atom(handler)); new_val = (handler == am_error_handler) ? NULL : (void *) (UWord) handler; - old_val = erts_psd_set(p, plocks, ERTS_PSD_ERROR_HANDLER, new_val); + old_val = erts_psd_set(p, ERTS_PSD_ERROR_HANDLER, new_val); if (!old_val) return am_error_handler; else { @@ -1975,18 +2145,30 @@ ERTS_GLB_INLINE Eterm erts_get_current_pid(void); ERTS_GLB_INLINE Uint erts_get_scheduler_id(void); ERTS_GLB_INLINE ErtsRunQueue *erts_get_runq_proc(Process *p); ERTS_GLB_INLINE ErtsRunQueue *erts_get_runq_current(ErtsSchedulerData *esdp); -#ifndef ERTS_ENABLE_LOCK_COUNT ERTS_GLB_INLINE void erts_smp_runq_lock(ErtsRunQueue *rq); -#endif ERTS_GLB_INLINE int erts_smp_runq_trylock(ErtsRunQueue *rq); ERTS_GLB_INLINE void erts_smp_runq_unlock(ErtsRunQueue *rq); -#ifndef ERTS_ENABLE_LOCK_COUNT ERTS_GLB_INLINE void erts_smp_xrunq_lock(ErtsRunQueue *rq, ErtsRunQueue *xrq); -#endif ERTS_GLB_INLINE void erts_smp_xrunq_unlock(ErtsRunQueue *rq, ErtsRunQueue *xrq); ERTS_GLB_INLINE void erts_smp_runqs_lock(ErtsRunQueue *rq1, ErtsRunQueue *rq2); ERTS_GLB_INLINE void erts_smp_runqs_unlock(ErtsRunQueue *rq1, ErtsRunQueue *rq2); +ERTS_GLB_INLINE ErtsMessage *erts_alloc_message_heap_state(Process *pp, + erts_aint32_t *psp, + ErtsProcLocks *plp, + Uint sz, + Eterm **hpp, + ErlOffHeap **ohpp); +ERTS_GLB_INLINE ErtsMessage *erts_alloc_message_heap(Process *pp, + ErtsProcLocks *plp, + Uint sz, + Eterm **hpp, + ErlOffHeap **ohpp); + +ERTS_GLB_INLINE void erts_shrink_message_heap(ErtsMessage **msgpp, Process *pp, + Eterm *start_hp, Eterm *used_hp, Eterm *end_hp, + Eterm *brefs, Uint brefs_size); + #if ERTS_GLB_INLINE_INCL_FUNC_DEF ERTS_GLB_INLINE @@ -2060,12 +2242,6 @@ erts_smp_runq_lock(ErtsRunQueue *rq) #endif } -#ifdef ERTS_ENABLE_LOCK_COUNT - -#define erts_smp_runq_lock(rq) erts_smp_mtx_lock_x(&(rq)->mtx, __FILE__, __LINE__) - -#endif - ERTS_GLB_INLINE int erts_smp_runq_trylock(ErtsRunQueue *rq) { @@ -2084,31 +2260,6 @@ erts_smp_runq_unlock(ErtsRunQueue *rq) #endif } -#ifdef ERTS_ENABLE_LOCK_COUNT - -#define erts_smp_xrunq_lock(rq, xrq) erts_smp_xrunq_lock_x((rq), (xrq), __FILE__, __LINE__) - -ERTS_GLB_INLINE void -erts_smp_xrunq_lock_x(ErtsRunQueue *rq, ErtsRunQueue *xrq, char* file, int line) -{ -#ifdef ERTS_SMP - ERTS_SMP_LC_ASSERT(erts_smp_lc_mtx_is_locked(&rq->mtx)); - if (xrq != rq) { - if (erts_smp_mtx_trylock(&xrq->mtx) == EBUSY) { - if (rq < xrq) - erts_smp_mtx_lock_x(&xrq->mtx, file, line); - else { - erts_smp_mtx_unlock(&rq->mtx); - erts_smp_mtx_lock_x(&xrq->mtx, file, line); - erts_smp_mtx_lock_x(&rq->mtx, file, line); - } - } - } -#endif -} - -#else - ERTS_GLB_INLINE void erts_smp_xrunq_lock(ErtsRunQueue *rq, ErtsRunQueue *xrq) { @@ -2128,8 +2279,6 @@ erts_smp_xrunq_lock(ErtsRunQueue *rq, ErtsRunQueue *xrq) #endif } -#endif - ERTS_GLB_INLINE void erts_smp_xrunq_unlock(ErtsRunQueue *rq, ErtsRunQueue *xrq) { @@ -2168,6 +2317,63 @@ erts_smp_runqs_unlock(ErtsRunQueue *rq1, ErtsRunQueue *rq2) #endif } +ERTS_GLB_INLINE ErtsMessage * +erts_alloc_message_heap_state(Process *pp, + erts_aint32_t *psp, + ErtsProcLocks *plp, + Uint sz, + Eterm **hpp, + ErlOffHeap **ohpp) +{ + int on_heap; + + if ((*psp) & ERTS_PSFLG_OFF_HEAP_MSGQ) { + ErtsMessage *mp = erts_alloc_message(sz, hpp); + *ohpp = sz == 0 ? NULL : &mp->hfrag.off_heap; + return mp; + } + + return erts_try_alloc_message_on_heap(pp, psp, plp, sz, hpp, ohpp, &on_heap); +} + +ERTS_GLB_INLINE ErtsMessage * +erts_alloc_message_heap(Process *pp, + ErtsProcLocks *plp, + Uint sz, + Eterm **hpp, + ErlOffHeap **ohpp) +{ + erts_aint32_t state = erts_smp_atomic32_read_nob(&pp->state); + return erts_alloc_message_heap_state(pp, &state, plp, sz, hpp, ohpp); +} + +ERTS_GLB_INLINE void +erts_shrink_message_heap(ErtsMessage **msgpp, Process *pp, + Eterm *start_hp, Eterm *used_hp, Eterm *end_hp, + Eterm *brefs, Uint brefs_size) +{ + ASSERT(start_hp <= used_hp && used_hp <= end_hp); + if ((*msgpp)->data.attached == ERTS_MSG_COMBINED_HFRAG) + *msgpp = erts_shrink_message(*msgpp, used_hp - start_hp, + brefs, brefs_size); + else if (!(*msgpp)->data.attached) { + ERTS_SMP_LC_ASSERT(ERTS_PROC_LOCK_MAIN + & erts_proc_lc_my_proc_locks(pp)); + HRelease(pp, end_hp, used_hp); + } + else { + ErlHeapFragment *hfrag = (*msgpp)->data.heap_frag; + if (start_hp != used_hp) + hfrag = erts_resize_message_buffer(hfrag, used_hp - start_hp, + brefs, brefs_size); + else { + free_message_buffer(hfrag); + hfrag = NULL; + } + (*msgpp)->data.heap_frag = hfrag; + } +} + #endif /* #if ERTS_GLB_INLINE_INCL_FUNC_DEF */ ERTS_GLB_INLINE ErtsAtomCacheMap *erts_get_atom_cache_map(Process *c_p); @@ -2229,6 +2435,8 @@ extern int erts_disable_proc_not_running_opt; void erts_smp_notify_inc_runq(ErtsRunQueue *runq); +void erts_interupt_aux_thread_timed(ErtsMonotonicTime timeout_time); + #ifdef ERTS_SMP void erts_sched_finish_poke(ErtsSchedulerSleepInfo *, erts_aint32_t); ERTS_GLB_INLINE void erts_sched_poke(ErtsSchedulerSleepInfo *ssi); @@ -2259,6 +2467,6 @@ erts_sched_poke(ErtsSchedulerSleepInfo *ssi) #endif -void erl_halt(int code); +void erts_halt(int code); extern erts_smp_atomic32_t erts_halt_progress; extern int erts_halt_code; diff --git a/erts/emulator/beam/erl_process_dict.c b/erts/emulator/beam/erl_process_dict.c index 23e5bf737f..487b944fc0 100644 --- a/erts/emulator/beam/erl_process_dict.c +++ b/erts/emulator/beam/erl_process_dict.c @@ -3,16 +3,17 @@ * * Copyright Ericsson AB 1999-2013. All Rights Reserved. * - * The contents of this file are subject to the Erlang Public License, - * Version 1.1, (the "License"); you may not use this file except in - * compliance with the License. You should have received a copy of the - * Erlang Public License along with this software. If not, it can be - * retrieved online at http://www.erlang.org/. - * - * Software distributed under the License is distributed on an "AS IS" - * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See - * the License for the specific language governing rights and limitations - * under the License. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. * * %CopyrightEnd% */ @@ -47,19 +48,22 @@ /* Hash constant macros */ #define MAX_HASH 1342177280UL -#define INITIAL_SIZE 10 +#define INITIAL_SIZE (erts_pd_initial_size) /* Hash utility macros */ -#define HASH_RANGE(PDict) ((PDict)->homeSize + (PDict)->splitPosition) +#define HASH_RANGE(PDict) ((PDict)->usedSlots) -#define MAKE_HASH(Term) \ -((is_small(Term)) ? unsigned_val(Term) : \ - ((is_atom(Term)) ? \ - (atom_tab(atom_val(term))->slot.bucket.hvalue) : \ - make_hash2(Term))) +#define MAKE_HASH(Term) \ + ((is_small(Term)) ? unsigned_val(Term) : \ + ((is_atom(Term)) ? \ + (atom_tab(atom_val(Term))->slot.bucket.hvalue) : \ + make_internal_hash(Term))) #define PD_SZ2BYTES(Sz) (sizeof(ProcDict) + ((Sz) - 1)*sizeof(Eterm)) +#define pd_hash_value(Pdict, Key) \ + pd_hash_value_to_ix(Pdict, MAKE_HASH((Key))) + /* Memory allocation macros */ #define PD_ALLOC(Sz) \ erts_alloc(ERTS_ALC_T_PROC_DICT, (Sz)) @@ -73,15 +77,21 @@ #define TCDR(Term) CDR(list_val(Term)) /* Array access macro */ -#define ARRAY_GET(PDict, Index) (((PDict)->size > (Index)) ? \ - (PDict)->data[Index] : NIL) +#define ARRAY_GET(PDict, Index) (ASSERT((Index) < (PDict)->arraySize), \ + (PDict)->data[Index]) +#define ARRAY_PUT(PDict, Index, Val) (ASSERT((Index) < (PDict)->arraySize), \ + (PDict)->data[Index] = (Val)) + +#define IS_POW2(X) ((X) && !((X) & ((X)-1))) /* * Forward decalarations */ static void pd_hash_erase(Process *p, Eterm id, Eterm *ret); static void pd_hash_erase_all(Process *p); +static Eterm pd_hash_get_with_hval(Process *p, Eterm bucket, Eterm id); static Eterm pd_hash_get_keys(Process *p, Eterm value); +static Eterm pd_hash_get_all_keys(Process *p, ProcDict *pd); static Eterm pd_hash_get_all(Process *p, ProcDict *pd); static Eterm pd_hash_put(Process *p, Eterm id, Eterm value); @@ -89,9 +99,9 @@ static void shrink(Process *p, Eterm* ret); static void grow(Process *p); static void array_shrink(ProcDict **ppd, unsigned int need); -static Eterm array_put(ProcDict **ppdict, unsigned int ndx, Eterm term); +static void ensure_array_size(ProcDict**, unsigned int size); -static unsigned int pd_hash_value(ProcDict *pdict, Eterm term); +static unsigned int pd_hash_value_to_ix(ProcDict *pdict, Uint32 hx); static unsigned int next_array_size(unsigned int need); /* @@ -132,6 +142,16 @@ static void pd_check(ProcDict *pd); ** External interface */ +int +erts_pd_set_initial_size(int size) +{ + if (size <= 0) + return 0; + + erts_pd_initial_size = 1 << erts_fit_in_bits_uint(size-1); + return 1; +} + /* * Called from break handler */ @@ -144,9 +164,9 @@ erts_dictionary_dump(int to, void *to_arg, ProcDict *pd) /*PD_CHECK(pd);*/ if (pd == NULL) return; - erts_print(to, to_arg, "(size = %d, used = %d, homeSize = %d, " + erts_print(to, to_arg, "(size = %d, usedSlots = %d, " "splitPosition = %d, numElements = %d)\n", - pd->size, pd->used, pd->homeSize, + pd->arraySize, pd->usedSlots, pd->splitPosition, (unsigned int) pd->numElements); for (i = 0; i < HASH_RANGE(pd); ++i) { erts_print(to, to_arg, "%d: %T\n", i, ARRAY_GET(pd, i)); @@ -201,7 +221,7 @@ erts_dicts_mem_size(Process *p) { Uint size = 0; if (p->dictionary) - size += PD_SZ2BYTES(p->dictionary->size); + size += PD_SZ2BYTES(p->dictionary->arraySize); return size; } @@ -275,6 +295,16 @@ BIF_RETTYPE get_1(BIF_ALIST_1) BIF_RET(ret); } +BIF_RETTYPE get_keys_0(BIF_ALIST_0) +{ + Eterm ret; + + PD_CHECK(BIF_P->dictionary); + ret = pd_hash_get_all_keys(BIF_P,BIF_P->dictionary); + PD_CHECK(BIF_P->dictionary); + BIF_RET(ret); +} + BIF_RETTYPE get_keys_1(BIF_ALIST_1) { Eterm ret; @@ -333,7 +363,7 @@ static void pd_hash_erase(Process *p, Eterm id, Eterm *ret) if (is_boxed(old)) { /* Tuple */ ASSERT(is_tuple(old)); if (EQ(tuple_val(old)[1], id)) { - array_put(&(p->dictionary), hval, NIL); + ARRAY_PUT(p->dictionary, hval, NIL); --(p->dictionary->numElements); *ret = tuple_val(old)[2]; } @@ -353,7 +383,7 @@ static void pd_hash_erase(Process *p, Eterm id, Eterm *ret) old = ARRAY_GET(p->dictionary, hval); ASSERT(is_list(old)); if (is_nil(TCDR(old))) { - array_put(&p->dictionary, hval, TCAR(old)); + ARRAY_PUT(p->dictionary, hval, TCAR(old)); } } else if (is_not_nil(old)) { #ifdef DEBUG @@ -362,7 +392,7 @@ static void pd_hash_erase(Process *p, Eterm id, Eterm *ret) "display term found in line %d:\n" "%T\n", p->common.id, __LINE__, old); #endif - erl_exit(1, "Damaged process dictionary found during erase/1."); + erts_exit(ERTS_ERROR_EXIT, "Damaged process dictionary found during erase/1."); } if ((range = HASH_RANGE(p->dictionary)) > INITIAL_SIZE && range / 2 > (p->dictionary->numElements)) { @@ -378,40 +408,97 @@ static void pd_hash_erase_all(Process *p) } } +Eterm erts_pd_hash_get_with_hx(Process *p, Uint32 hx, Eterm id) +{ + unsigned int hval; + ProcDict *pd = p->dictionary; + + ASSERT(hx == MAKE_HASH(id)); + if (pd == NULL) + return am_undefined; + hval = pd_hash_value_to_ix(pd, hx); + return pd_hash_get_with_hval(p, ARRAY_GET(pd, hval), id); +} + Eterm erts_pd_hash_get(Process *p, Eterm id) { unsigned int hval; - Eterm tmp; ProcDict *pd = p->dictionary; if (pd == NULL) return am_undefined; hval = pd_hash_value(pd, id); - tmp = ARRAY_GET(pd, hval); - if (is_boxed(tmp)) { /* Tuple */ - ASSERT(is_tuple(tmp)); - if (EQ(tuple_val(tmp)[1], id)) { - return tuple_val(tmp)[2]; + return pd_hash_get_with_hval(p, ARRAY_GET(pd, hval), id); +} + +Eterm pd_hash_get_with_hval(Process *p, Eterm bucket, Eterm id) +{ + if (is_boxed(bucket)) { /* Tuple */ + ASSERT(is_tuple(bucket)); + if (EQ(tuple_val(bucket)[1], id)) { + return tuple_val(bucket)[2]; } - } else if (is_list(tmp)) { - for (; tmp != NIL && !EQ(tuple_val(TCAR(tmp))[1], id); tmp = TCDR(tmp)) { + } else if (is_list(bucket)) { + for (; bucket != NIL && !EQ(tuple_val(TCAR(bucket))[1], id); bucket = TCDR(bucket)) { ; } - if (tmp != NIL) { - return tuple_val(TCAR(tmp))[2]; + if (bucket != NIL) { + return tuple_val(TCAR(bucket))[2]; } - } else if (is_not_nil(tmp)) { + } else if (is_not_nil(bucket)) { #ifdef DEBUG erts_fprintf(stderr, "Process dictionary for process %T is broken, trying to " "display term found in line %d:\n" - "%T\n", p->common.id, __LINE__, tmp); + "%T\n", p->common.id, __LINE__, bucket); #endif - erl_exit(1, "Damaged process dictionary found during get/1."); + erts_exit(ERTS_ERROR_EXIT, "Damaged process dictionary found during get/1."); } return am_undefined; } + +#define PD_GET_TKEY(Dst,Src) \ +do { \ + ASSERT(is_tuple((Src))); \ + ASSERT(arityval(*((Eterm*)tuple_val((Src)))) == 2); \ + (Dst) = ((Eterm*)tuple_val((Src)))[1]; \ +} while(0) + +static Eterm pd_hash_get_all_keys(Process *p, ProcDict *pd) { + Eterm* hp; + Eterm res = NIL; + Eterm tmp, tmp2; + unsigned int i; + unsigned int num; + + if (pd == NULL) { + return res; + } + + num = HASH_RANGE(pd); + hp = HAlloc(p, pd->numElements * 2); + + for (i = 0; i < num; ++i) { + tmp = ARRAY_GET(pd, i); + if (is_boxed(tmp)) { + PD_GET_TKEY(tmp,tmp); + res = CONS(hp, tmp, res); + hp += 2; + } else if (is_list(tmp)) { + while (tmp != NIL) { + tmp2 = TCAR(tmp); + PD_GET_TKEY(tmp2,tmp2); + res = CONS(hp, tmp2, res); + hp += 2; + tmp = TCDR(tmp); + } + } + } + return res; +} +#undef PD_GET_TKEY + static Eterm pd_hash_get_keys(Process *p, Eterm value) { Eterm *hp; @@ -496,8 +583,11 @@ static Eterm pd_hash_put(Process *p, Eterm id, Eterm value) if (p->dictionary == NULL) { /* Create it */ - array_put(&(p->dictionary), INITIAL_SIZE - 1, NIL); - p->dictionary->homeSize = INITIAL_SIZE; + ensure_array_size(&p->dictionary, INITIAL_SIZE); + p->dictionary->usedSlots = INITIAL_SIZE; + p->dictionary->sizeMask = INITIAL_SIZE*2 - 1; + p->dictionary->splitPosition = 0; + p->dictionary->numElements = 0; } hval = pd_hash_value(p->dictionary, id); old = ARRAY_GET(p->dictionary, hval); @@ -530,7 +620,7 @@ static Eterm pd_hash_put(Process *p, Eterm id, Eterm value) root[0] = id; root[1] = value; root[2] = old; - BUMP_REDS(p, erts_garbage_collect(p, needed, root, 3)); + erts_garbage_collect(p, needed, root, 3); id = root[0]; value = root[1]; old = root[2]; @@ -549,19 +639,19 @@ static Eterm pd_hash_put(Process *p, Eterm id, Eterm value) * Update the dictionary. */ if (is_nil(old)) { - array_put(&(p->dictionary), hval, tpl); + ARRAY_PUT(p->dictionary, hval, tpl); ++(p->dictionary->numElements); } else if (is_boxed(old)) { ASSERT(is_tuple(old)); if (EQ(tuple_val(old)[1],id)) { - array_put(&(p->dictionary), hval, tpl); + ARRAY_PUT(p->dictionary, hval, tpl); return tuple_val(old)[2]; } else { hp = HeapOnlyAlloc(p, 4); tmp = CONS(hp, old, NIL); hp += 2; ++(p->dictionary->numElements); - array_put(&(p->dictionary), hval, CONS(hp, tpl, tmp)); + ARRAY_PUT(p->dictionary, hval, CONS(hp, tpl, tmp)); hp += 2; ASSERT(hp <= hp_limit); } @@ -571,7 +661,7 @@ static Eterm pd_hash_put(Process *p, Eterm id, Eterm value) * New key. Simply prepend the tuple to the beginning of the list. */ hp = HeapOnlyAlloc(p, 2); - array_put(&(p->dictionary), hval, CONS(hp, tpl, old)); + ARRAY_PUT(p->dictionary, hval, CONS(hp, tpl, old)); hp += 2; ASSERT(hp <= hp_limit); ++(p->dictionary->numElements); @@ -606,7 +696,7 @@ static Eterm pd_hash_put(Process *p, Eterm id, Eterm value) nlist = CONS(hp, tpl, nlist); hp += 2; ASSERT(hp <= hp_limit); - array_put(&(p->dictionary), hval, nlist); + ARRAY_PUT(p->dictionary, hval, nlist); return tuple_val(TCAR(tmp))[2]; } } else { @@ -617,7 +707,7 @@ static Eterm pd_hash_put(Process *p, Eterm id, Eterm value) "%T\n", p->common.id, __LINE__, old); #endif - erl_exit(1, "Damaged process dictionary found during put/2."); + erts_exit(ERTS_ERROR_EXIT, "Damaged process dictionary found during put/2."); } if (HASH_RANGE(p->dictionary) <= p->dictionary->numElements) { grow(p); @@ -631,6 +721,7 @@ static Eterm pd_hash_put(Process *p, Eterm id, Eterm value) static void shrink(Process *p, Eterm* ret) { + ProcDict *pd = p->dictionary; unsigned int range = HASH_RANGE(p->dictionary); unsigned int steps = (range*3) / 10; Eterm hi, lo, tmp; @@ -645,25 +736,26 @@ static void shrink(Process *p, Eterm* ret) } for (i = 0; i < steps; ++i) { - ProcDict *pd = p->dictionary; if (pd->splitPosition == 0) { - pd->homeSize /= 2; - pd->splitPosition = pd->homeSize; + ASSERT(IS_POW2(pd->usedSlots)); + pd->sizeMask = pd->usedSlots - 1; + pd->splitPosition = pd->usedSlots / 2; } --(pd->splitPosition); - hi = ARRAY_GET(pd, (pd->splitPosition + pd->homeSize)); + /* Must wait to decrement 'usedSlots' for GC rootset below */ + hi = ARRAY_GET(pd, pd->usedSlots - 1); lo = ARRAY_GET(pd, pd->splitPosition); if (hi != NIL) { if (lo == NIL) { - array_put(&(p->dictionary), pd->splitPosition, hi); + ARRAY_PUT(pd, pd->splitPosition, hi); } else { - int needed = 4; + Sint needed = 4; if (is_list(hi) && is_list(lo)) { needed = 2*erts_list_length(hi); } if (HeapWordsLeft(p) < needed) { - BUMP_REDS(p, erts_garbage_collect(p, needed, ret, 1)); - hi = pd->data[(pd->splitPosition + pd->homeSize)]; + erts_garbage_collect(p, needed, ret, 1); + hi = pd->data[pd->usedSlots - 1]; lo = pd->data[pd->splitPosition]; } #ifdef DEBUG @@ -674,13 +766,13 @@ static void shrink(Process *p, Eterm* ret) hp = HeapOnlyAlloc(p, 4); tmp = CONS(hp, hi, NIL); hp += 2; - array_put(&(p->dictionary), pd->splitPosition, + ARRAY_PUT(pd, pd->splitPosition, CONS(hp,lo,tmp)); hp += 2; ASSERT(hp <= hp_limit); } else { /* hi is a list */ hp = HeapOnlyAlloc(p, 2); - array_put(&(p->dictionary), pd->splitPosition, + ARRAY_PUT(pd, pd->splitPosition, CONS(hp, lo, hi)); hp += 2; ASSERT(hp <= hp_limit); @@ -688,7 +780,7 @@ static void shrink(Process *p, Eterm* ret) } else { /* lo is a list */ if (is_tuple(hi)) { hp = HeapOnlyAlloc(p, 2); - array_put(&(p->dictionary), pd->splitPosition, + ARRAY_PUT(pd, pd->splitPosition, CONS(hp, hi, lo)); hp += 2; ASSERT(hp <= hp_limit); @@ -700,14 +792,15 @@ static void shrink(Process *p, Eterm* ret) hp += 2; } ASSERT(hp <= hp_limit); - array_put(&(p->dictionary), pd->splitPosition, lo); + ARRAY_PUT(pd, pd->splitPosition, lo); } } } } - array_put(&(p->dictionary), (pd->splitPosition + pd->homeSize), NIL); + --pd->usedSlots; + ARRAY_PUT(pd, pd->usedSlots, NIL); } - if (HASH_RANGE(p->dictionary) <= (p->dictionary->size / 4)) { + if (HASH_RANGE(p->dictionary) <= (p->dictionary->arraySize / 4)) { array_shrink(&(p->dictionary), (HASH_RANGE(p->dictionary) * 3) / 2); } } @@ -715,14 +808,14 @@ static void shrink(Process *p, Eterm* ret) static void grow(Process *p) { unsigned int i,j; - unsigned int steps = p->dictionary->homeSize / 5; + unsigned int steps = (p->dictionary->usedSlots / 4) & 0xf; Eterm l1,l2; Eterm l; Eterm *hp; unsigned int pos; unsigned int homeSize; - int needed = 0; - ProcDict *pd; + Sint needed = 0; + ProcDict *pd = p->dictionary; #ifdef DEBUG Eterm *hp_limit; #endif @@ -731,18 +824,20 @@ static void grow(Process *p) if (steps == 0) steps = 1; /* Dont grow over MAX_HASH */ - if ((MAX_HASH - steps) <= HASH_RANGE(p->dictionary)) { + if ((MAX_HASH - steps) <= HASH_RANGE(pd)) { return; } + ensure_array_size(&p->dictionary, HASH_RANGE(pd) + steps); + pd = p->dictionary; + /* * Calculate total number of heap words needed, and garbage collect * if necessary. */ - pd = p->dictionary; pos = pd->splitPosition; - homeSize = pd->homeSize; + homeSize = pd->usedSlots - pd->splitPosition; for (i = 0; i < steps; ++i) { if (pos == homeSize) { homeSize *= 2; @@ -758,7 +853,7 @@ static void grow(Process *p) } } if (HeapWordsLeft(p) < needed) { - BUMP_REDS(p, erts_garbage_collect(p, needed, 0, 0)); + erts_garbage_collect(p, needed, 0, 0); } #ifdef DEBUG hp_limit = p->htop + needed; @@ -767,21 +862,22 @@ static void grow(Process *p) /* * Now grow. */ - + homeSize = pd->usedSlots - pd->splitPosition; for (i = 0; i < steps; ++i) { - ProcDict *pd = p->dictionary; - if (pd->splitPosition == pd->homeSize) { - pd->homeSize *= 2; - pd->splitPosition = 0; + if (pd->splitPosition == homeSize) { + homeSize *= 2; + pd->sizeMask = homeSize*2 - 1; + pd->splitPosition = 0; } pos = pd->splitPosition; ++pd->splitPosition; /* For the hashes */ + ++pd->usedSlots; + ASSERT(pos + homeSize == pd->usedSlots - 1); l = ARRAY_GET(pd, pos); if (is_tuple(l)) { if (pd_hash_value(pd, tuple_val(l)[1]) != pos) { - array_put(&(p->dictionary), pos + - p->dictionary->homeSize, l); - array_put(&(p->dictionary), pos, NIL); + ARRAY_PUT(pd, pos + homeSize, l); + ARRAY_PUT(pd, pos, NIL); } } else { l2 = NIL; @@ -803,10 +899,8 @@ static void grow(Process *p) if (l2 != NIL && TCDR(l2) == NIL) l2 = TCAR(l2); ASSERT(hp <= hp_limit); - /* After array_put pd is no longer valid */ - array_put(&(p->dictionary), pos, l1); - array_put(&(p->dictionary), pos + - p->dictionary->homeSize, l2); + ARRAY_PUT(pd, pos, l1); + ARRAY_PUT(pd, pos + homeSize, l2); } } @@ -823,73 +917,65 @@ static void array_shrink(ProcDict **ppd, unsigned int need) { unsigned int siz = next_array_size(need); - HDEBUGF(("array_shrink: size = %d, used = %d, need = %d", - (*ppd)->size, (*ppd)->used, need)); + HDEBUGF(("array_shrink: size = %d, need = %d", + (*ppd)->arraySize, need)); - if (siz > (*ppd)->size) + if (siz >= (*ppd)->arraySize) return; /* Only shrink */ *ppd = PD_REALLOC(((void *) *ppd), - PD_SZ2BYTES((*ppd)->size), + PD_SZ2BYTES((*ppd)->arraySize), PD_SZ2BYTES(siz)); - (*ppd)->size = siz; - if ((*ppd)->size < (*ppd)->used) - (*ppd)->used = (*ppd)->size; + (*ppd)->arraySize = siz; } -static Eterm array_put(ProcDict **ppdict, unsigned int ndx, Eterm term) +static void ensure_array_size(ProcDict **ppdict, unsigned int size) { + ProcDict *pd = *ppdict; unsigned int i; - Eterm ret; - if (*ppdict == NULL) { - Uint siz = next_array_size(ndx+1); - ProcDict *p; - p = PD_ALLOC(PD_SZ2BYTES(siz)); + if (pd == NULL) { + Uint siz = next_array_size(size); + + pd = PD_ALLOC(PD_SZ2BYTES(siz)); for (i = 0; i < siz; ++i) - p->data[i] = NIL; - p->size = siz; - p->homeSize = p->splitPosition = p->numElements = p->used = 0; - *ppdict = p; - } else if (ndx >= (*ppdict)->size) { - Uint osize = (*ppdict)->size; - Uint nsize = next_array_size(ndx+1); - *ppdict = PD_REALLOC(((void *) *ppdict), + pd->data[i] = NIL; + pd->arraySize = siz; + *ppdict = pd; + } else if (size > pd->arraySize) { + Uint osize = pd->arraySize; + Uint nsize = next_array_size(size); + pd = PD_REALLOC(((void *) pd), PD_SZ2BYTES(osize), PD_SZ2BYTES(nsize)); for (i = osize; i < nsize; ++i) - (*ppdict)->data[i] = NIL; - (*ppdict)->size = nsize; + pd->data[i] = NIL; + pd->arraySize = nsize; + *ppdict = pd; } - ret = (*ppdict)->data[ndx]; - (*ppdict)->data[ndx] = term; - if ((ndx + 1) > (*ppdict)->used) - (*ppdict)->used = ndx + 1; -#ifdef HARDDEBUG - HDEBUGF(("array_put: (*ppdict)->size = %d, (*ppdict)->used = %d, ndx = %d", - (*ppdict)->size, (*ppdict)->used, ndx)); - erts_fprintf(stderr, "%T", term); -#endif /* HARDDEBUG */ - return ret; } /* ** Basic utilities */ -static unsigned int pd_hash_value(ProcDict *pdict, Eterm term) +static unsigned int pd_hash_value_to_ix(ProcDict *pdict, Uint32 hx) { - Uint hash, high; + Uint high; - hash = MAKE_HASH(term); - high = hash % (pdict->homeSize*2); + ASSERT(IS_POW2(pdict->sizeMask+1)); + ASSERT(HASH_RANGE(pdict) >= (pdict->sizeMask >> 1)); + ASSERT(HASH_RANGE(pdict) <= (pdict->sizeMask + 1)); + + high = hx & pdict->sizeMask; if (high >= HASH_RANGE(pdict)) - return hash % pdict->homeSize; + return hx & (pdict->sizeMask >> 1); return high; } + static unsigned int next_array_size(unsigned int need) { static unsigned int tab[] = @@ -949,35 +1035,39 @@ static unsigned int next_array_size(unsigned int need) static void pd_check(ProcDict *pd) { unsigned int i; + unsigned int used; Uint num; if (pd == NULL) return; - ASSERT(pd->size >= pd->used); + used = HASH_RANGE(pd); + ASSERT(pd->arraySize >= used); ASSERT(HASH_RANGE(pd) <= MAX_HASH); - for (i = 0, num = 0; i < pd->used; ++i) { + for (i = 0, num = 0; i < used; ++i) { Eterm t = pd->data[i]; if (is_nil(t)) { continue; } else if (is_tuple(t)) { ++num; ASSERT(arityval(*tuple_val(t)) == 2); + ASSERT(pd_hash_value(pd, tuple_val(t)[1]) == i); continue; } else if (is_list(t)) { while (t != NIL) { ++num; ASSERT(is_tuple(TCAR(t))); ASSERT(arityval(*(tuple_val(TCAR(t)))) == 2); + ASSERT(pd_hash_value(pd, tuple_val(TCAR(t))[1]) == i); t = TCDR(t); } continue; } else { - erl_exit(1, + erts_exit(ERTS_ERROR_EXIT, "Found tag 0x%08x in process dictionary at position %d", (unsigned long) t, (int) i); } } ASSERT(num == pd->numElements); - ASSERT(pd->splitPosition <= pd->homeSize); + ASSERT(pd->usedSlots >= pd->splitPosition*2); } #endif /* DEBUG */ diff --git a/erts/emulator/beam/erl_process_dict.h b/erts/emulator/beam/erl_process_dict.h index 8fad2a67ab..dac214c8a1 100644 --- a/erts/emulator/beam/erl_process_dict.h +++ b/erts/emulator/beam/erl_process_dict.h @@ -3,16 +3,17 @@ * * Copyright Ericsson AB 1999-2009. All Rights Reserved. * - * The contents of this file are subject to the Erlang Public License, - * Version 1.1, (the "License"); you may not use this file except in - * compliance with the License. You should have received a copy of the - * Erlang Public License along with this software. If not, it can be - * retrieved online at http://www.erlang.org/. - * - * Software distributed under the License is distributed on an "AS IS" - * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See - * the License for the specific language governing rights and limitations - * under the License. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. * * %CopyrightEnd% */ @@ -22,14 +23,18 @@ #include "sys.h" typedef struct proc_dict { - unsigned int size; - unsigned int used; - unsigned int homeSize; + unsigned int sizeMask; + unsigned int usedSlots; + unsigned int arraySize; unsigned int splitPosition; Uint numElements; Eterm data[1]; /* The beginning of an array of erlang terms */ } ProcDict; +#define ERTS_PD_START(PD) ((PD)->data) +#define ERTS_PD_SIZE(PD) ((PD)->usedSlots) + +int erts_pd_set_initial_size(int size); Uint erts_dicts_mem_size(struct process *p); void erts_erase_dicts(struct process *p); void erts_dictionary_dump(int to, void *to_arg, ProcDict *pd); @@ -38,5 +43,6 @@ void erts_deep_dictionary_dump(int to, void *to_arg, Eterm erts_dictionary_copy(struct process *p, ProcDict *pd); Eterm erts_pd_hash_get(struct process *p, Eterm id); +Eterm erts_pd_hash_get_with_hx(Process *p, Uint32 hx, Eterm id); #endif diff --git a/erts/emulator/beam/erl_process_dump.c b/erts/emulator/beam/erl_process_dump.c index 2f3cf23b00..73552b28de 100644 --- a/erts/emulator/beam/erl_process_dump.c +++ b/erts/emulator/beam/erl_process_dump.c @@ -3,16 +3,17 @@ * * Copyright Ericsson AB 2003-2013. All Rights Reserved. * - * The contents of this file are subject to the Erlang Public License, - * Version 1.1, (the "License"); you may not use this file except in - * compliance with the License. You should have received a copy of the - * Erlang Public License along with this software. If not, it can be - * retrieved online at http://www.erlang.org/. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Software distributed under the License is distributed on an "AS IS" - * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See - * the License for the specific language governing rights and limitations - * under the License. + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. * * %CopyrightEnd% */ @@ -43,8 +44,9 @@ static void dump_process_info(int to, void *to_arg, Process *p); static void dump_element(int to, void *to_arg, Eterm x); static void dump_dist_ext(int to, void *to_arg, ErtsDistExternal *edep); static void dump_element_nl(int to, void *to_arg, Eterm x); -static int stack_element_dump(int to, void *to_arg, Process* p, Eterm* sp, +static int stack_element_dump(int to, void *to_arg, Eterm* sp, int yreg); +static void stack_trace_dump(int to, void *to_arg, Eterm* sp); static void print_function_from_pc(int to, void *to_arg, BeamInstr* x); static void heap_dump(int to, void *to_arg, Eterm x); static void dump_binaries(int to, void *to_arg, Binary* root); @@ -76,13 +78,14 @@ erts_deep_process_dump(int to, void *to_arg) dump_binaries(to, to_arg, all_binaries); } -Uint erts_process_memory(Process *p) { - ErlMessage *mp; +Uint erts_process_memory(Process *p, int incl_msg_inq) { + ErtsMessage *mp; Uint size = 0; struct saved_calls *scb; size += sizeof(Process); - ERTS_SMP_MSGQ_MV_INQ2PRIVQ(p); + if (incl_msg_inq) + ERTS_SMP_MSGQ_MV_INQ2PRIVQ(p); erts_doforall_links(ERTS_P_LINKS(p), &erts_one_link_size, &size); erts_doforall_monitors(ERTS_P_MONITORS(p), &erts_one_mon_size, &size); @@ -90,7 +93,7 @@ Uint erts_process_memory(Process *p) { if (p->old_hend && p->old_heap) size += (p->old_hend - p->old_heap) * sizeof(Eterm); - size += p->msg.len * sizeof(ErlMessage); + size += p->msg.len * sizeof(ErtsMessage); for (mp = p->msg.first; mp; mp = mp->next) if (mp->data.attached) @@ -100,7 +103,7 @@ Uint erts_process_memory(Process *p) { size += p->arity * sizeof(p->arg_reg[0]); } - if (p->psd) + if (erts_smp_atomic_read_nob(&p->psd) != (erts_aint_t) NULL) size += sizeof(ErtsPSD); scb = ERTS_PROC_GET_SAVED_CALLS_BUF(p); @@ -117,7 +120,7 @@ static void dump_process_info(int to, void *to_arg, Process *p) { Eterm* sp; - ErlMessage* mp; + ErtsMessage* mp; int yreg = -1; ERTS_SMP_MSGQ_MV_INQ2PRIVQ(p); @@ -148,7 +151,7 @@ dump_process_info(int to, void *to_arg, Process *p) if ((ERTS_TRACE_FLAGS(p) & F_SENSITIVE) == 0) { erts_print(to, to_arg, "=proc_stack:%T\n", p->common.id); for (sp = p->stop; sp < STACK_START(p); sp++) { - yreg = stack_element_dump(to, to_arg, p, sp, yreg); + yreg = stack_element_dump(to, to_arg, sp, yreg); } erts_print(to, to_arg, "=proc_heap:%T\n", p->common.id); @@ -243,9 +246,65 @@ dump_element_nl(int to, void *to_arg, Eterm x) erts_putc(to, to_arg, '\n'); } +static void +stack_trace_dump(int to, void *to_arg, Eterm *sp) { + Eterm x = *sp; + if (is_CP(x)) { + erts_print(to, to_arg, "%p:", sp); + erts_print(to, to_arg, "SReturn addr 0x%X (", cp_val(x)); + print_function_from_pc(to, to_arg, cp_val(x)); + erts_print(to, to_arg, ")\n"); + } +} + +void +erts_limited_stack_trace(int to, void *to_arg, Process *p) +{ + Eterm* sp; + + + if (ERTS_TRACE_FLAGS(p) & F_SENSITIVE) { + return; + } + + if (STACK_START(p) < STACK_TOP(p)) { + return; + } + + if ((STACK_START(p) - STACK_TOP(p)) < 512) { + if (erts_sys_is_area_readable((char*)STACK_TOP(p), + (char*)STACK_START(p))) + for (sp = STACK_TOP(p); sp < STACK_START(p); sp++) + stack_trace_dump(to, to_arg, sp); + else + erts_print(to, to_arg, "Could not read from stack memory: %p - %p\n", + STACK_TOP(p), STACK_START(p)); + } else { + sp = STACK_TOP(p); + if (erts_sys_is_area_readable((char*)STACK_TOP(p), + (char*)(STACK_TOP(p) + 25))) + for (; sp < (STACK_TOP(p) + 256); sp++) + stack_trace_dump(to, to_arg, sp); + else + erts_print(to, to_arg, "Could not read from stack memory: %p - %p\n", + STACK_TOP(p), STACK_TOP(p) + 256); + + erts_print(to, to_arg, "%p: skipping %d frames\n", + sp, STACK_START(p) - STACK_TOP(p) - 512); + + if (erts_sys_is_area_readable((char*)(STACK_START(p) - 256), + (char*)STACK_START(p))) + for (sp = STACK_START(p) - 256; sp < STACK_START(p); sp++) + stack_trace_dump(to, to_arg, sp); + else + erts_print(to, to_arg, "Could not read from stack memory: %p - %p\n", + STACK_START(p) - 256, STACK_START(p)); + } + +} static int -stack_element_dump(int to, void *to_arg, Process* p, Eterm* sp, int yreg) +stack_element_dump(int to, void *to_arg, Eterm* sp, int yreg) { Eterm x = *sp; @@ -307,7 +366,7 @@ heap_dump(int to, void *to_arg, Eterm x) while (x != OUR_NIL) { if (is_CP(x)) { - next = (Eterm *) EXPAND_POINTER(x); + next = (Eterm *) x; } else if (is_list(x)) { ptr = list_val(x); if (ptr[0] != OUR_NIL) { @@ -320,7 +379,7 @@ heap_dump(int to, void *to_arg, Eterm x) ptr[1] = make_small(0); } x = ptr[0]; - ptr[0] = (Eterm) COMPRESS_POINTER(next); + ptr[0] = (Eterm) next; next = ptr + 1; continue; } @@ -350,7 +409,7 @@ heap_dump(int to, void *to_arg, Eterm x) ptr[0] = OUR_NIL; } else { x = ptr[arity]; - ptr[0] = (Eterm) COMPRESS_POINTER(next); + ptr[0] = (Eterm) next; next = ptr + arity - 1; continue; } @@ -508,3 +567,114 @@ dump_externally(int to, void *to_arg, Eterm term) erts_print(to, to_arg, "%02X", *s++); } } + +void erts_dump_process_state(int to, void *to_arg, erts_aint32_t psflg) { + if (psflg & ERTS_PSFLG_FREE) + erts_print(to, to_arg, "Non Existing\n"); /* Should never happen */ + else if (psflg & ERTS_PSFLG_EXITING) + erts_print(to, to_arg, "Exiting\n"); + else if (psflg & ERTS_PSFLG_GC) { + erts_print(to, to_arg, "Garbing\n"); + } + else if (psflg & ERTS_PSFLG_SUSPENDED) + erts_print(to, to_arg, "Suspended\n"); + else if (psflg & ERTS_PSFLG_RUNNING) { + erts_print(to, to_arg, "Running\n"); + } + else if (psflg & ERTS_PSFLG_ACTIVE) + erts_print(to, to_arg, "Scheduled\n"); + else + erts_print(to, to_arg, "Waiting\n"); +} + +void +erts_dump_extended_process_state(int to, void *to_arg, erts_aint32_t psflg) { + + int i; + + switch (ERTS_PSFLGS_GET_ACT_PRIO(psflg)) { + case PRIORITY_MAX: erts_print(to, to_arg, "ACT_PRIO_MAX | "); break; + case PRIORITY_HIGH: erts_print(to, to_arg, "ACT_PRIO_HIGH | "); break; + case PRIORITY_NORMAL: erts_print(to, to_arg, "ACT_PRIO_NORMAL | "); break; + case PRIORITY_LOW: erts_print(to, to_arg, "ACT_PRIO_LOW | "); break; + } + switch (ERTS_PSFLGS_GET_USR_PRIO(psflg)) { + case PRIORITY_MAX: erts_print(to, to_arg, "USR_PRIO_MAX | "); break; + case PRIORITY_HIGH: erts_print(to, to_arg, "USR_PRIO_HIGH | "); break; + case PRIORITY_NORMAL: erts_print(to, to_arg, "USR_PRIO_NORMAL | "); break; + case PRIORITY_LOW: erts_print(to, to_arg, "USR_PRIO_LOW | "); break; + } + switch (ERTS_PSFLGS_GET_PRQ_PRIO(psflg)) { + case PRIORITY_MAX: erts_print(to, to_arg, "PRQ_PRIO_MAX"); break; + case PRIORITY_HIGH: erts_print(to, to_arg, "PRQ_PRIO_HIGH"); break; + case PRIORITY_NORMAL: erts_print(to, to_arg, "PRQ_PRIO_NORMAL"); break; + case PRIORITY_LOW: erts_print(to, to_arg, "PRQ_PRIO_LOW"); break; + } + + psflg &= ~(ERTS_PSFLGS_ACT_PRIO_MASK | + ERTS_PSFLGS_USR_PRIO_MASK | + ERTS_PSFLGS_PRQ_PRIO_MASK); + + if (psflg) + erts_print(to, to_arg, " | "); + + for (i = 0; i <= ERTS_PSFLG_MAX && psflg; i++) { + erts_aint32_t chk = (1 << i); + if (psflg & chk) { + switch (chk) { + case ERTS_PSFLG_IN_PRQ_MAX: + erts_print(to, to_arg, "IN_PRQ_MAX"); break; + case ERTS_PSFLG_IN_PRQ_HIGH: + erts_print(to, to_arg, "IN_PRQ_HIGH"); break; + case ERTS_PSFLG_IN_PRQ_NORMAL: + erts_print(to, to_arg, "IN_PRQ_NORMAL"); break; + case ERTS_PSFLG_IN_PRQ_LOW: + erts_print(to, to_arg, "IN_PRQ_LOW"); break; + case ERTS_PSFLG_FREE: + erts_print(to, to_arg, "FREE"); break; + case ERTS_PSFLG_EXITING: + erts_print(to, to_arg, "EXITING"); break; + case ERTS_PSFLG_PENDING_EXIT: + erts_print(to, to_arg, "PENDING_EXIT"); break; + case ERTS_PSFLG_ACTIVE: + erts_print(to, to_arg, "ACTIVE"); break; + case ERTS_PSFLG_IN_RUNQ: + erts_print(to, to_arg, "IN_RUNQ"); break; + case ERTS_PSFLG_RUNNING: + erts_print(to, to_arg, "RUNNING"); break; + case ERTS_PSFLG_SUSPENDED: + erts_print(to, to_arg, "SUSPENDED"); break; + case ERTS_PSFLG_GC: + erts_print(to, to_arg, "GC"); break; + case ERTS_PSFLG_BOUND: + erts_print(to, to_arg, "BOUND"); break; + case ERTS_PSFLG_TRAP_EXIT: + erts_print(to, to_arg, "TRAP_EXIT"); break; + case ERTS_PSFLG_ACTIVE_SYS: + erts_print(to, to_arg, "ACTIVE_SYS"); break; + case ERTS_PSFLG_RUNNING_SYS: + erts_print(to, to_arg, "RUNNING_SYS"); break; + case ERTS_PSFLG_PROXY: + erts_print(to, to_arg, "PROXY"); break; + case ERTS_PSFLG_DELAYED_SYS: + erts_print(to, to_arg, "DELAYED_SYS"); break; + case ERTS_PSFLG_OFF_HEAP_MSGQ: + erts_print(to, to_arg, "OFF_HEAP_MSGQ"); break; + case ERTS_PSFLG_ON_HEAP_MSGQ: + erts_print(to, to_arg, "ON_HEAP_MSGQ"); break; + case ERTS_PSFLG_DIRTY_CPU_PROC: + erts_print(to, to_arg, "DIRTY_CPU_PROC"); break; + case ERTS_PSFLG_DIRTY_IO_PROC: + erts_print(to, to_arg, "DIRTY_IO_PROC"); break; + case ERTS_PSFLG_DIRTY_ACTIVE_SYS: + erts_print(to, to_arg, "DIRTY_ACTIVE_SYS"); break; + default: + erts_print(to, to_arg, "UNKNOWN(%d)", chk); break; + } + if (psflg > chk) + erts_print(to, to_arg, " | "); + psflg -= chk; + } + } + erts_print(to, to_arg, "\n"); +} diff --git a/erts/emulator/beam/erl_process_lock.c b/erts/emulator/beam/erl_process_lock.c index 82cc68222d..0bee2c848c 100644 --- a/erts/emulator/beam/erl_process_lock.c +++ b/erts/emulator/beam/erl_process_lock.c @@ -3,16 +3,17 @@ * * Copyright Ericsson AB 2007-2012. All Rights Reserved. * - * The contents of this file are subject to the Erlang Public License, - * Version 1.1, (the "License"); you may not use this file except in - * compliance with the License. You should have received a copy of the - * Erlang Public License along with this software. If not, it can be - * retrieved online at http://www.erlang.org/. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Software distributed under the License is distributed on an "AS IS" - * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See - * the License for the specific language governing rights and limitations - * under the License. + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. * * %CopyrightEnd% */ @@ -103,12 +104,16 @@ static struct { Sint16 proc_lock_main; Sint16 proc_lock_link; Sint16 proc_lock_msgq; + Sint16 proc_lock_btm; Sint16 proc_lock_status; } lc_id; #endif erts_pix_lock_t erts_pix_locks[ERTS_NO_OF_PIX_LOCKS]; +#ifdef ERTS_ENABLE_LOCK_COUNT +static void lcnt_enable_proc_lock_count(Process *proc, int enable); +#endif void erts_init_proc_lock(int cpus) @@ -145,6 +150,7 @@ erts_init_proc_lock(int cpus) lc_id.proc_lock_main = erts_lc_get_lock_order_id("proc_main"); lc_id.proc_lock_link = erts_lc_get_lock_order_id("proc_link"); lc_id.proc_lock_msgq = erts_lc_get_lock_order_id("proc_msgq"); + lc_id.proc_lock_btm = erts_lc_get_lock_order_id("proc_btm"); lc_id.proc_lock_status = erts_lc_get_lock_order_id("proc_status"); #endif } @@ -707,7 +713,7 @@ proc_safelock(int is_managed, need_locks1 |= unlock_locks; if (!is_managed && !have_locks1) { refc1 = 1; - erts_smp_proc_inc_refc(p1); + erts_proc_inc_refc(p1); } erts_smp_proc_unlock(p1, unlock_locks); } @@ -717,7 +723,7 @@ proc_safelock(int is_managed, need_locks2 |= unlock_locks; if (!is_managed && !have_locks2) { refc2 = 1; - erts_smp_proc_inc_refc(p2); + erts_proc_inc_refc(p2); } erts_smp_proc_unlock(p2, unlock_locks); } @@ -798,9 +804,9 @@ proc_safelock(int is_managed, if (!is_managed) { if (refc1) - erts_smp_proc_dec_refc(p1); + erts_proc_dec_refc(p1); if (refc2) - erts_smp_proc_dec_refc(p2); + erts_proc_dec_refc(p2); } } @@ -861,8 +867,8 @@ erts_pid2proc_opt(Process *c_p, return NULL; need_locks &= ~c_p_have_locks; if (!need_locks) { - if (flags & ERTS_P2P_FLG_SMP_INC_REFC) - erts_smp_proc_inc_refc(c_p); + if (flags & ERTS_P2P_FLG_INC_REFC) + erts_proc_inc_refc(c_p); return c_p; } } @@ -875,8 +881,8 @@ erts_pid2proc_opt(Process *c_p, if (proc->common.id != pid) proc = NULL; else if (!need_locks) { - if (flags & ERTS_P2P_FLG_SMP_INC_REFC) - erts_smp_proc_inc_refc(proc); + if (flags & ERTS_P2P_FLG_INC_REFC) + erts_proc_inc_refc(proc); } else { int busy; @@ -916,8 +922,8 @@ erts_pid2proc_opt(Process *c_p, #endif if (!busy) { - if (flags & ERTS_P2P_FLG_SMP_INC_REFC) - erts_smp_proc_inc_refc(proc); + if (flags & ERTS_P2P_FLG_INC_REFC) + erts_proc_inc_refc(proc); #if ERTS_PROC_LOCK_OWN_IMPL && defined(ERTS_ENABLE_LOCK_COUNT) /* all is great */ @@ -932,8 +938,8 @@ erts_pid2proc_opt(Process *c_p, proc = ERTS_PROC_LOCK_BUSY; else { int managed; - if (flags & ERTS_P2P_FLG_SMP_INC_REFC) - erts_smp_proc_inc_refc(proc); + if (flags & ERTS_P2P_FLG_INC_REFC) + erts_proc_inc_refc(proc); #if ERTS_PROC_LOCK_OWN_IMPL && defined(ERTS_ENABLE_LOCK_COUNT) erts_lcnt_proc_lock_unaquire(&proc->lock, lcnt_locks); @@ -941,7 +947,7 @@ erts_pid2proc_opt(Process *c_p, managed = dhndl == ERTS_THR_PRGR_DHANDLE_MANAGED; if (!managed) { - erts_smp_proc_inc_refc(proc); + erts_proc_inc_refc(proc); erts_thr_progress_unmanaged_continue(dhndl); dec_refc_proc = proc; @@ -978,14 +984,14 @@ erts_pid2proc_opt(Process *c_p, erts_smp_proc_unlock(proc, need_locks); - if (flags & ERTS_P2P_FLG_SMP_INC_REFC) + if (flags & ERTS_P2P_FLG_INC_REFC) dec_refc_proc = proc; proc = NULL; } if (dec_refc_proc) - erts_smp_proc_dec_refc(dec_refc_proc); + erts_proc_dec_refc(dec_refc_proc); #if ERTS_PROC_LOCK_OWN_IMPL && defined(ERTS_PROC_LOCK_DEBUG) ERTS_LC_ASSERT(!proc @@ -1001,7 +1007,9 @@ erts_pid2proc_opt(Process *c_p, void erts_proc_lock_init(Process *p) { +#if ERTS_PROC_LOCK_OWN_IMPL || defined(ERTS_PROC_LOCK_DEBUG) int i; +#endif #if ERTS_PROC_LOCK_OWN_IMPL /* We always start with all locks locked */ #if ERTS_PROC_LOCK_ATOMIC_IMPL @@ -1038,6 +1046,11 @@ erts_proc_lock_init(Process *p) #ifdef ERTS_ENABLE_LOCK_CHECK erts_lc_trylock(1, &p->lock.msgq.lc); #endif + erts_mtx_init_x(&p->lock.btm, "proc_btm", p->common.id, do_lock_count); + ethr_mutex_lock(&p->lock.btm.mtx); +#ifdef ERTS_ENABLE_LOCK_CHECK + erts_lc_trylock(1, &p->lock.btm.lc); +#endif erts_mtx_init_x(&p->lock.status, "proc_status", p->common.id, do_lock_count); ethr_mutex_lock(&p->lock.status.mtx); @@ -1045,7 +1058,6 @@ erts_proc_lock_init(Process *p) erts_lc_trylock(1, &p->lock.status.lc); #endif #endif - erts_atomic32_init_nob(&p->lock.refc, 1); #ifdef ERTS_PROC_LOCK_DEBUG for (i = 0; i <= ERTS_PROC_LOCK_MAX_BIT; i++) erts_smp_atomic32_init_nob(&p->lock.locked[i], (erts_aint32_t) 1); @@ -1064,6 +1076,7 @@ erts_proc_lock_fin(Process *p) erts_mtx_destroy(&p->lock.main); erts_mtx_destroy(&p->lock.link); erts_mtx_destroy(&p->lock.msgq); + erts_mtx_destroy(&p->lock.btm); erts_mtx_destroy(&p->lock.status); #endif #if defined(ERTS_ENABLE_LOCK_COUNT) && defined(ERTS_SMP) @@ -1074,141 +1087,109 @@ erts_proc_lock_fin(Process *p) /* --- Process lock counting ----------------------------------------------- */ #if ERTS_PROC_LOCK_OWN_IMPL && defined(ERTS_ENABLE_LOCK_COUNT) -void erts_lcnt_proc_lock_init(Process *p) { - if (erts_lcnt_rt_options & ERTS_LCNT_OPT_PROCLOCK) { - if (p->common.id != ERTS_INVALID_PID) { - erts_lcnt_init_lock_x(&(p->lock.lcnt_main), "proc_main", ERTS_LCNT_LT_PROCLOCK, p->common.id); - erts_lcnt_init_lock_x(&(p->lock.lcnt_msgq), "proc_msgq", ERTS_LCNT_LT_PROCLOCK, p->common.id); - erts_lcnt_init_lock_x(&(p->lock.lcnt_link), "proc_link", ERTS_LCNT_LT_PROCLOCK, p->common.id); - erts_lcnt_init_lock_x(&(p->lock.lcnt_status), "proc_status", ERTS_LCNT_LT_PROCLOCK, p->common.id); - } else { - erts_lcnt_init_lock(&(p->lock.lcnt_main), "proc_main", ERTS_LCNT_LT_PROCLOCK); - erts_lcnt_init_lock(&(p->lock.lcnt_msgq), "proc_msgq", ERTS_LCNT_LT_PROCLOCK); - erts_lcnt_init_lock(&(p->lock.lcnt_link), "proc_link", ERTS_LCNT_LT_PROCLOCK); - erts_lcnt_init_lock(&(p->lock.lcnt_status), "proc_status", ERTS_LCNT_LT_PROCLOCK); - } - } else { - sys_memzero(&(p->lock.lcnt_main), sizeof(p->lock.lcnt_main)); - sys_memzero(&(p->lock.lcnt_msgq), sizeof(p->lock.lcnt_msgq)); - sys_memzero(&(p->lock.lcnt_link), sizeof(p->lock.lcnt_link)); - sys_memzero(&(p->lock.lcnt_status), sizeof(p->lock.lcnt_status)); - } + +void erts_lcnt_enable_proc_lock_count(int enable) { + int ix, max = erts_ptab_max(&erts_proc); + Process *proc = NULL; + for (ix = 0; ix < max; ++ix) { + if ((proc = erts_pix2proc(ix)) != NULL) + lcnt_enable_proc_lock_count(proc, enable); + } /* for all processes */ } - + +void erts_lcnt_proc_lock_init(Process *p) { + if (!(erts_lcnt_rt_options & ERTS_LCNT_OPT_PROCLOCK)) { + erts_lcnt_init_lock_empty(&(p->lock.lcnt_main)); + erts_lcnt_init_lock_empty(&(p->lock.lcnt_msgq)); + erts_lcnt_init_lock_empty(&(p->lock.lcnt_btm)); + erts_lcnt_init_lock_empty(&(p->lock.lcnt_link)); + erts_lcnt_init_lock_empty(&(p->lock.lcnt_status)); + } else { /* now the common case */ + Eterm pid = (p->common.id != ERTS_INVALID_PID) ? p->common.id : NIL; + erts_lcnt_init_lock_x(&(p->lock.lcnt_main), "proc_main", ERTS_LCNT_LT_PROCLOCK, pid); + erts_lcnt_init_lock_x(&(p->lock.lcnt_msgq), "proc_msgq", ERTS_LCNT_LT_PROCLOCK, pid); + erts_lcnt_init_lock_x(&(p->lock.lcnt_btm), "proc_btm", ERTS_LCNT_LT_PROCLOCK, pid); + erts_lcnt_init_lock_x(&(p->lock.lcnt_link), "proc_link", ERTS_LCNT_LT_PROCLOCK, pid); + erts_lcnt_init_lock_x(&(p->lock.lcnt_status),"proc_status",ERTS_LCNT_LT_PROCLOCK, pid); + } /* the lock names should really be aligned to four characters */ +} /* logic reversed */ void erts_lcnt_proc_lock_destroy(Process *p) { erts_lcnt_destroy_lock(&(p->lock.lcnt_main)); erts_lcnt_destroy_lock(&(p->lock.lcnt_msgq)); + erts_lcnt_destroy_lock(&(p->lock.lcnt_btm)); erts_lcnt_destroy_lock(&(p->lock.lcnt_link)); erts_lcnt_destroy_lock(&(p->lock.lcnt_status)); } -void erts_lcnt_proc_lock(erts_proc_lock_t *lock, ErtsProcLocks locks) { - if (erts_lcnt_rt_options & ERTS_LCNT_OPT_PROCLOCK) { - if (locks & ERTS_PROC_LOCK_MAIN) { - erts_lcnt_lock(&(lock->lcnt_main)); - } - if (locks & ERTS_PROC_LOCK_MSGQ) { - erts_lcnt_lock(&(lock->lcnt_msgq)); - } - if (locks & ERTS_PROC_LOCK_LINK) { - erts_lcnt_lock(&(lock->lcnt_link)); - } - if (locks & ERTS_PROC_LOCK_STATUS) { - erts_lcnt_lock(&(lock->lcnt_status)); +static void lcnt_enable_proc_lock_count(Process *proc, int enable) { + if (enable) { + if (!ERTS_LCNT_LOCK_TYPE(&(proc->lock.lcnt_main))) { + erts_lcnt_proc_lock_init(proc); + } } + else { + if (ERTS_LCNT_LOCK_TYPE(&(proc->lock.lcnt_main))) { + erts_lcnt_proc_lock_destroy(proc); + } } } -void erts_lcnt_proc_lock_post_x(erts_proc_lock_t *lock, ErtsProcLocks locks, char *file, unsigned int line) { - if (erts_lcnt_rt_options & ERTS_LCNT_OPT_PROCLOCK) { +void erts_lcnt_proc_lock(erts_proc_lock_t *lock, ErtsProcLocks locks) { + if (!(erts_lcnt_rt_options & ERTS_LCNT_OPT_PROCLOCK)) return; + if (locks & ERTS_PROC_LOCK_MAIN) { erts_lcnt_lock(&(lock->lcnt_main)); } + if (locks & ERTS_PROC_LOCK_MSGQ) { erts_lcnt_lock(&(lock->lcnt_msgq)); } + if (locks & ERTS_PROC_LOCK_BTM) { erts_lcnt_lock(&(lock->lcnt_btm)); } + if (locks & ERTS_PROC_LOCK_LINK) { erts_lcnt_lock(&(lock->lcnt_link)); } + if (locks & ERTS_PROC_LOCK_STATUS) { erts_lcnt_lock(&(lock->lcnt_status)); } +} + +void erts_lcnt_proc_lock_post_x(erts_proc_lock_t *lock, ErtsProcLocks locks, + char *file, unsigned int line) { + if (!(erts_lcnt_rt_options & ERTS_LCNT_OPT_PROCLOCK)) return; if (locks & ERTS_PROC_LOCK_MAIN) { erts_lcnt_lock_post_x(&(lock->lcnt_main), file, line); } if (locks & ERTS_PROC_LOCK_MSGQ) { erts_lcnt_lock_post_x(&(lock->lcnt_msgq), file, line); } + if (locks & ERTS_PROC_LOCK_BTM) { + erts_lcnt_lock_post_x(&(lock->lcnt_btm), file, line); + } if (locks & ERTS_PROC_LOCK_LINK) { erts_lcnt_lock_post_x(&(lock->lcnt_link), file, line); } if (locks & ERTS_PROC_LOCK_STATUS) { erts_lcnt_lock_post_x(&(lock->lcnt_status), file, line); } - } } void erts_lcnt_proc_lock_unaquire(erts_proc_lock_t *lock, ErtsProcLocks locks) { - if (erts_lcnt_rt_options & ERTS_LCNT_OPT_PROCLOCK) { - if (locks & ERTS_PROC_LOCK_MAIN) { - erts_lcnt_lock_unaquire(&(lock->lcnt_main)); - } - if (locks & ERTS_PROC_LOCK_MSGQ) { - erts_lcnt_lock_unaquire(&(lock->lcnt_msgq)); - } - if (locks & ERTS_PROC_LOCK_LINK) { - erts_lcnt_lock_unaquire(&(lock->lcnt_link)); - } - if (locks & ERTS_PROC_LOCK_STATUS) { - erts_lcnt_lock_unaquire(&(lock->lcnt_status)); - } - } + if (!(erts_lcnt_rt_options & ERTS_LCNT_OPT_PROCLOCK)) return; + if (locks & ERTS_PROC_LOCK_MAIN) { erts_lcnt_lock_unaquire(&(lock->lcnt_main)); } + if (locks & ERTS_PROC_LOCK_MSGQ) { erts_lcnt_lock_unaquire(&(lock->lcnt_msgq)); } + if (locks & ERTS_PROC_LOCK_BTM) { erts_lcnt_lock_unaquire(&(lock->lcnt_btm)); } + if (locks & ERTS_PROC_LOCK_LINK) { erts_lcnt_lock_unaquire(&(lock->lcnt_link)); } + if (locks & ERTS_PROC_LOCK_STATUS) { erts_lcnt_lock_unaquire(&(lock->lcnt_status)); } } void erts_lcnt_proc_unlock(erts_proc_lock_t *lock, ErtsProcLocks locks) { - if (erts_lcnt_rt_options & ERTS_LCNT_OPT_PROCLOCK) { - if (locks & ERTS_PROC_LOCK_MAIN) { - erts_lcnt_unlock(&(lock->lcnt_main)); - } - if (locks & ERTS_PROC_LOCK_MSGQ) { - erts_lcnt_unlock(&(lock->lcnt_msgq)); - } - if (locks & ERTS_PROC_LOCK_LINK) { - erts_lcnt_unlock(&(lock->lcnt_link)); - } - if (locks & ERTS_PROC_LOCK_STATUS) { - erts_lcnt_unlock(&(lock->lcnt_status)); - } - } + if (!(erts_lcnt_rt_options & ERTS_LCNT_OPT_PROCLOCK)) return; + if (locks & ERTS_PROC_LOCK_MAIN) { erts_lcnt_unlock(&(lock->lcnt_main)); } + if (locks & ERTS_PROC_LOCK_MSGQ) { erts_lcnt_unlock(&(lock->lcnt_msgq)); } + if (locks & ERTS_PROC_LOCK_BTM) { erts_lcnt_unlock(&(lock->lcnt_btm)); } + if (locks & ERTS_PROC_LOCK_LINK) { erts_lcnt_unlock(&(lock->lcnt_link)); } + if (locks & ERTS_PROC_LOCK_STATUS) { erts_lcnt_unlock(&(lock->lcnt_status)); } } void erts_lcnt_proc_trylock(erts_proc_lock_t *lock, ErtsProcLocks locks, int res) { - if (erts_lcnt_rt_options & ERTS_LCNT_OPT_PROCLOCK) { - if (locks & ERTS_PROC_LOCK_MAIN) { - erts_lcnt_trylock(&(lock->lcnt_main), res); - } - if (locks & ERTS_PROC_LOCK_MSGQ) { - erts_lcnt_trylock(&(lock->lcnt_msgq), res); - } - if (locks & ERTS_PROC_LOCK_LINK) { - erts_lcnt_trylock(&(lock->lcnt_link), res); - } - if (locks & ERTS_PROC_LOCK_STATUS) { - erts_lcnt_trylock(&(lock->lcnt_status), res); - } - } -} - - -void erts_lcnt_enable_proc_lock_count(int enable) -{ - int i, max = erts_ptab_max(&erts_proc); - - for (i = 0; i < max; ++i) { - Process* p = erts_pix2proc(i); - if (p) { - if (enable) { - if (!ERTS_LCNT_LOCK_TYPE(&(p->lock.lcnt_main))) { - erts_lcnt_proc_lock_init(p); - } - } else { - if (ERTS_LCNT_LOCK_TYPE(&(p->lock.lcnt_main))) { - erts_lcnt_proc_lock_destroy(p); - } - } - } - } -} - -#endif /* ifdef ERTS_ENABLE_LOCK_COUNT */ + if (!(erts_lcnt_rt_options & ERTS_LCNT_OPT_PROCLOCK)) return; + if (locks & ERTS_PROC_LOCK_MAIN) { erts_lcnt_trylock(&(lock->lcnt_main), res); } + if (locks & ERTS_PROC_LOCK_MSGQ) { erts_lcnt_trylock(&(lock->lcnt_msgq), res); } + if (locks & ERTS_PROC_LOCK_BTM) { erts_lcnt_trylock(&(lock->lcnt_btm), res); } + if (locks & ERTS_PROC_LOCK_LINK) { erts_lcnt_trylock(&(lock->lcnt_link), res); } + if (locks & ERTS_PROC_LOCK_STATUS) { erts_lcnt_trylock(&(lock->lcnt_status), res); } +} /* reversed logic */ +#endif /* ERTS_ENABLE_LOCK_COUNT */ /* --- Process lock checking ----------------------------------------------- */ @@ -1235,6 +1216,10 @@ erts_proc_lc_lock(Process *p, ErtsProcLocks locks, char *file, unsigned int line lck.id = lc_id.proc_lock_msgq; erts_lc_lock_x(&lck,file,line); } + if (locks & ERTS_PROC_LOCK_BTM) { + lck.id = lc_id.proc_lock_btm; + erts_lc_lock_x(&lck,file,line); + } if (locks & ERTS_PROC_LOCK_STATUS) { lck.id = lc_id.proc_lock_status; erts_lc_lock_x(&lck,file,line); @@ -1260,6 +1245,10 @@ erts_proc_lc_trylock(Process *p, ErtsProcLocks locks, int locked, lck.id = lc_id.proc_lock_msgq; erts_lc_trylock_x(locked, &lck, file, line); } + if (locks & ERTS_PROC_LOCK_BTM) { + lck.id = lc_id.proc_lock_btm; + erts_lc_trylock_x(locked, &lck, file, line); + } if (locks & ERTS_PROC_LOCK_STATUS) { lck.id = lc_id.proc_lock_status; erts_lc_trylock_x(locked, &lck, file, line); @@ -1276,6 +1265,10 @@ erts_proc_lc_unlock(Process *p, ErtsProcLocks locks) lck.id = lc_id.proc_lock_status; erts_lc_unlock(&lck); } + if (locks & ERTS_PROC_LOCK_BTM) { + lck.id = lc_id.proc_lock_btm; + erts_lc_unlock(&lck); + } if (locks & ERTS_PROC_LOCK_MSGQ) { lck.id = lc_id.proc_lock_msgq; erts_lc_unlock(&lck); @@ -1303,6 +1296,10 @@ erts_proc_lc_might_unlock(Process *p, ErtsProcLocks locks) lck.id = lc_id.proc_lock_status; erts_lc_might_unlock(&lck); } + if (locks & ERTS_PROC_LOCK_BTM) { + lck.id = lc_id.proc_lock_btm; + erts_lc_might_unlock(&lck); + } if (locks & ERTS_PROC_LOCK_MSGQ) { lck.id = lc_id.proc_lock_msgq; erts_lc_might_unlock(&lck); @@ -1322,6 +1319,8 @@ erts_proc_lc_might_unlock(Process *p, ErtsProcLocks locks) erts_lc_might_unlock(&p->lock.link.lc); if (locks & ERTS_PROC_LOCK_MSGQ) erts_lc_might_unlock(&p->lock.msgq.lc); + if (locks & ERTS_PROC_LOCK_BTM) + erts_lc_might_unlock(&p->lock.btm.lc); if (locks & ERTS_PROC_LOCK_STATUS) erts_lc_might_unlock(&p->lock.status.lc); #endif @@ -1347,6 +1346,10 @@ erts_proc_lc_require_lock(Process *p, ErtsProcLocks locks, char *file, lck.id = lc_id.proc_lock_msgq; erts_lc_require_lock(&lck, file, line); } + if (locks & ERTS_PROC_LOCK_BTM) { + lck.id = lc_id.proc_lock_btm; + erts_lc_require_lock(&lck, file, line); + } if (locks & ERTS_PROC_LOCK_STATUS) { lck.id = lc_id.proc_lock_status; erts_lc_require_lock(&lck, file, line); @@ -1358,6 +1361,8 @@ erts_proc_lc_require_lock(Process *p, ErtsProcLocks locks, char *file, erts_lc_require_lock(&p->lock.link.lc, file, line); if (locks & ERTS_PROC_LOCK_MSGQ) erts_lc_require_lock(&p->lock.msgq.lc, file, line); + if (locks & ERTS_PROC_LOCK_BTM) + erts_lc_require_lock(&p->lock.btm.lc, file, line); if (locks & ERTS_PROC_LOCK_STATUS) erts_lc_require_lock(&p->lock.status.lc, file, line); #endif @@ -1374,6 +1379,10 @@ erts_proc_lc_unrequire_lock(Process *p, ErtsProcLocks locks) lck.id = lc_id.proc_lock_status; erts_lc_unrequire_lock(&lck); } + if (locks & ERTS_PROC_LOCK_BTM) { + lck.id = lc_id.proc_lock_btm; + erts_lc_unrequire_lock(&lck); + } if (locks & ERTS_PROC_LOCK_MSGQ) { lck.id = lc_id.proc_lock_msgq; erts_lc_unrequire_lock(&lck); @@ -1393,6 +1402,8 @@ erts_proc_lc_unrequire_lock(Process *p, ErtsProcLocks locks) erts_lc_unrequire_lock(&p->lock.link.lc); if (locks & ERTS_PROC_LOCK_MSGQ) erts_lc_unrequire_lock(&p->lock.msgq.lc); + if (locks & ERTS_PROC_LOCK_BTM) + erts_lc_unrequire_lock(&p->lock.btm.lc); if (locks & ERTS_PROC_LOCK_STATUS) erts_lc_unrequire_lock(&p->lock.status.lc); #endif @@ -1414,6 +1425,8 @@ erts_proc_lc_trylock_force_busy(Process *p, ErtsProcLocks locks) lck.id = lc_id.proc_lock_link; else if (locks & ERTS_PROC_LOCK_MSGQ) lck.id = lc_id.proc_lock_msgq; + else if (locks & ERTS_PROC_LOCK_BTM) + lck.id = lc_id.proc_lock_btm; else if (locks & ERTS_PROC_LOCK_STATUS) lck.id = lc_id.proc_lock_status; else @@ -1448,7 +1461,8 @@ erts_proc_lc_chk_have_proc_locks(Process *p, ErtsProcLocks locks) { int have_locks_len = 0; #if ERTS_PROC_LOCK_OWN_IMPL - erts_lc_lock_t have_locks[4] = {ERTS_PROC_LC_EMPTY_LOCK_INIT, + erts_lc_lock_t have_locks[5] = {ERTS_PROC_LC_EMPTY_LOCK_INIT, + ERTS_PROC_LC_EMPTY_LOCK_INIT, ERTS_PROC_LC_EMPTY_LOCK_INIT, ERTS_PROC_LC_EMPTY_LOCK_INIT, ERTS_PROC_LC_EMPTY_LOCK_INIT}; @@ -1464,18 +1478,24 @@ erts_proc_lc_chk_have_proc_locks(Process *p, ErtsProcLocks locks) have_locks[have_locks_len].id = lc_id.proc_lock_msgq; have_locks[have_locks_len++].extra = p->common.id; } + if (locks & ERTS_PROC_LOCK_BTM) { + have_locks[have_locks_len].id = lc_id.proc_lock_btm; + have_locks[have_locks_len++].extra = p->common.id; + } if (locks & ERTS_PROC_LOCK_STATUS) { have_locks[have_locks_len].id = lc_id.proc_lock_status; have_locks[have_locks_len++].extra = p->common.id; } #elif ERTS_PROC_LOCK_RAW_MUTEX_IMPL - erts_lc_lock_t have_locks[4]; + erts_lc_lock_t have_locks[5]; if (locks & ERTS_PROC_LOCK_MAIN) have_locks[have_locks_len++] = p->lock.main.lc; if (locks & ERTS_PROC_LOCK_LINK) have_locks[have_locks_len++] = p->lock.link.lc; if (locks & ERTS_PROC_LOCK_MSGQ) have_locks[have_locks_len++] = p->lock.msgq.lc; + if (locks & ERTS_PROC_LOCK_BTM) + have_locks[have_locks_len++] = p->lock.btm.lc; if (locks & ERTS_PROC_LOCK_STATUS) have_locks[have_locks_len++] = p->lock.status.lc; #endif @@ -1488,11 +1508,11 @@ erts_proc_lc_chk_proc_locks(Process *p, ErtsProcLocks locks) int have_locks_len = 0; int have_not_locks_len = 0; #if ERTS_PROC_LOCK_OWN_IMPL - erts_lc_lock_t have_locks[4] = {ERTS_PROC_LC_EMPTY_LOCK_INIT, + erts_lc_lock_t have_locks[5] = {ERTS_PROC_LC_EMPTY_LOCK_INIT, ERTS_PROC_LC_EMPTY_LOCK_INIT, ERTS_PROC_LC_EMPTY_LOCK_INIT, ERTS_PROC_LC_EMPTY_LOCK_INIT}; - erts_lc_lock_t have_not_locks[4] = {ERTS_PROC_LC_EMPTY_LOCK_INIT, + erts_lc_lock_t have_not_locks[5] = {ERTS_PROC_LC_EMPTY_LOCK_INIT, ERTS_PROC_LC_EMPTY_LOCK_INIT, ERTS_PROC_LC_EMPTY_LOCK_INIT, ERTS_PROC_LC_EMPTY_LOCK_INIT}; @@ -1521,6 +1541,14 @@ erts_proc_lc_chk_proc_locks(Process *p, ErtsProcLocks locks) have_not_locks[have_not_locks_len].id = lc_id.proc_lock_msgq; have_not_locks[have_not_locks_len++].extra = p->common.id; } + if (locks & ERTS_PROC_LOCK_BTM) { + have_locks[have_locks_len].id = lc_id.proc_lock_btm; + have_locks[have_locks_len++].extra = p->common.id; + } + else { + have_not_locks[have_not_locks_len].id = lc_id.proc_lock_btm; + have_not_locks[have_not_locks_len++].extra = p->common.id; + } if (locks & ERTS_PROC_LOCK_STATUS) { have_locks[have_locks_len].id = lc_id.proc_lock_status; have_locks[have_locks_len++].extra = p->common.id; @@ -1530,8 +1558,8 @@ erts_proc_lc_chk_proc_locks(Process *p, ErtsProcLocks locks) have_not_locks[have_not_locks_len++].extra = p->common.id; } #elif ERTS_PROC_LOCK_RAW_MUTEX_IMPL - erts_lc_lock_t have_locks[4]; - erts_lc_lock_t have_not_locks[4]; + erts_lc_lock_t have_locks[5]; + erts_lc_lock_t have_not_locks[5]; if (locks & ERTS_PROC_LOCK_MAIN) have_locks[have_locks_len++] = p->lock.main.lc; @@ -1545,6 +1573,10 @@ erts_proc_lc_chk_proc_locks(Process *p, ErtsProcLocks locks) have_locks[have_locks_len++] = p->lock.msgq.lc; else have_not_locks[have_not_locks_len++] = p->lock.msgq.lc; + if (locks & ERTS_PROC_LOCK_BTM) + have_locks[have_locks_len++] = p->lock.btm.lc; + else + have_not_locks[have_not_locks_len++] = p->lock.btm.lc; if (locks & ERTS_PROC_LOCK_STATUS) have_locks[have_locks_len++] = p->lock.status.lc; else @@ -1558,10 +1590,10 @@ erts_proc_lc_chk_proc_locks(Process *p, ErtsProcLocks locks) ErtsProcLocks erts_proc_lc_my_proc_locks(Process *p) { - int resv[4]; + int resv[5]; ErtsProcLocks res = 0; #if ERTS_PROC_LOCK_OWN_IMPL - erts_lc_lock_t locks[4] = {ERTS_LC_LOCK_INIT(lc_id.proc_lock_main, + erts_lc_lock_t locks[5] = {ERTS_LC_LOCK_INIT(lc_id.proc_lock_main, p->common.id, ERTS_LC_FLG_LT_PROCLOCK), ERTS_LC_LOCK_INIT(lc_id.proc_lock_link, @@ -1570,17 +1602,21 @@ erts_proc_lc_my_proc_locks(Process *p) ERTS_LC_LOCK_INIT(lc_id.proc_lock_msgq, p->common.id, ERTS_LC_FLG_LT_PROCLOCK), + ERTS_LC_LOCK_INIT(lc_id.proc_lock_btm, + p->common.id, + ERTS_LC_FLG_LT_PROCLOCK), ERTS_LC_LOCK_INIT(lc_id.proc_lock_status, p->common.id, ERTS_LC_FLG_LT_PROCLOCK)}; #elif ERTS_PROC_LOCK_RAW_MUTEX_IMPL - erts_lc_lock_t locks[4] = {p->lock.main.lc, + erts_lc_lock_t locks[5] = {p->lock.main.lc, p->lock.link.lc, p->lock.msgq.lc, + p->lock.btm.lc, p->lock.status.lc}; #endif - erts_lc_have_locks(resv, locks, 4); + erts_lc_have_locks(resv, locks, 5); if (resv[0]) res |= ERTS_PROC_LOCK_MAIN; if (resv[1]) @@ -1588,6 +1624,8 @@ erts_proc_lc_my_proc_locks(Process *p) if (resv[2]) res |= ERTS_PROC_LOCK_MSGQ; if (resv[3]) + res |= ERTS_PROC_LOCK_BTM; + if (resv[4]) res |= ERTS_PROC_LOCK_STATUS; return res; @@ -1596,13 +1634,14 @@ erts_proc_lc_my_proc_locks(Process *p) void erts_proc_lc_chk_no_proc_locks(char *file, int line) { - int resv[4]; - int ids[4] = {lc_id.proc_lock_main, + int resv[5]; + int ids[5] = {lc_id.proc_lock_main, lc_id.proc_lock_link, lc_id.proc_lock_msgq, + lc_id.proc_lock_btm, lc_id.proc_lock_status}; - erts_lc_have_lock_ids(resv, ids, 4); - if (!ERTS_IS_CRASH_DUMPING && (resv[0] || resv[1] || resv[2] || resv[3])) { + erts_lc_have_lock_ids(resv, ids, 5); + if (!ERTS_IS_CRASH_DUMPING && (resv[0] || resv[1] || resv[2] || resv[3] || resv[4])) { erts_lc_fail("%s:%d: Thread has process locks locked when expected " "not to have any process locks locked", file, line); diff --git a/erts/emulator/beam/erl_process_lock.h b/erts/emulator/beam/erl_process_lock.h index 052d992d3f..9c59301086 100644 --- a/erts/emulator/beam/erl_process_lock.h +++ b/erts/emulator/beam/erl_process_lock.h @@ -3,16 +3,17 @@ * * Copyright Ericsson AB 2007-2012. All Rights Reserved. * - * The contents of this file are subject to the Erlang Public License, - * Version 1.1, (the "License"); you may not use this file except in - * compliance with the License. You should have received a copy of the - * Erlang Public License along with this software. If not, it can be - * retrieved online at http://www.erlang.org/. - * - * Software distributed under the License is distributed on an "AS IS" - * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See - * the License for the specific language governing rights and limitations - * under the License. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. * * %CopyrightEnd% */ @@ -65,7 +66,7 @@ #endif -#define ERTS_PROC_LOCK_MAX_BIT 3 +#define ERTS_PROC_LOCK_MAX_BIT 4 typedef erts_aint32_t ErtsProcLocks; @@ -81,17 +82,18 @@ typedef struct erts_proc_lock_t_ { erts_lcnt_lock_t lcnt_main; erts_lcnt_lock_t lcnt_link; erts_lcnt_lock_t lcnt_msgq; + erts_lcnt_lock_t lcnt_btm; erts_lcnt_lock_t lcnt_status; #endif #elif ERTS_PROC_LOCK_RAW_MUTEX_IMPL erts_mtx_t main; erts_mtx_t link; erts_mtx_t msgq; + erts_mtx_t btm; erts_mtx_t status; #else # error "no implementation" #endif - erts_atomic32_t refc; #ifdef ERTS_PROC_LOCK_DEBUG erts_smp_atomic32_t locked[ERTS_PROC_LOCK_MAX_BIT+1]; #endif @@ -120,11 +122,17 @@ typedef struct erts_proc_lock_t_ { * Message queue lock: * Protects the following fields in the process structure: * * msg_inq - * * bif_timers */ #define ERTS_PROC_LOCK_MSGQ (((ErtsProcLocks) 1) << 2) /* + * Bif timer lock: + * Protects the following fields in the process structure: + * * bif_timers + */ +#define ERTS_PROC_LOCK_BTM (((ErtsProcLocks) 1) << 3) + +/* * Status lock: * Protects the following fields in the process structure: * * pending_suspenders @@ -463,6 +471,9 @@ erts_smp_proc_raw_trylock__(Process *p, ErtsProcLocks locks) if (locks & ERTS_PROC_LOCK_MSGQ) if (erts_mtx_trylock(&p->lock.msgq) == EBUSY) goto busy_msgq; + if (locks & ERTS_PROC_LOCK_BTM) + if (erts_mtx_trylock(&p->lock.btm) == EBUSY) + goto busy_btm; if (locks & ERTS_PROC_LOCK_STATUS) if (erts_mtx_trylock(&p->lock.status) == EBUSY) goto busy_status; @@ -470,6 +481,9 @@ erts_smp_proc_raw_trylock__(Process *p, ErtsProcLocks locks) return 0; busy_status: + if (locks & ERTS_PROC_LOCK_BTM) + erts_mtx_unlock(&p->lock.btm); +busy_btm: if (locks & ERTS_PROC_LOCK_MSGQ) erts_mtx_unlock(&p->lock.msgq); busy_msgq: @@ -509,6 +523,10 @@ erts_smp_proc_lock__(Process *p, ERTS_LC_ASSERT((locks & ~ERTS_PROC_LOCKS_ALL) == 0); +#ifdef ERTS_ENABLE_LOCK_CHECK + erts_proc_lc_lock(p, locks, file, line); +#endif + old_lflgs = erts_smp_proc_raw_trylock__(p, locks); if (old_lflgs != 0) { @@ -530,9 +548,6 @@ erts_smp_proc_lock__(Process *p, #ifdef ERTS_ENABLE_LOCK_COUNT erts_lcnt_proc_lock_post_x(&(p->lock), locks, file, line); #endif -#ifdef ERTS_ENABLE_LOCK_CHECK - erts_proc_lc_lock(p, locks, file, line); -#endif #ifdef ERTS_PROC_LOCK_DEBUG erts_proc_lock_op_debug(p, locks, 1); @@ -549,6 +564,8 @@ erts_smp_proc_lock__(Process *p, erts_mtx_lock(&p->lock.link); if (locks & ERTS_PROC_LOCK_MSGQ) erts_mtx_lock(&p->lock.msgq); + if (locks & ERTS_PROC_LOCK_BTM) + erts_mtx_lock(&p->lock.btm); if (locks & ERTS_PROC_LOCK_STATUS) erts_mtx_lock(&p->lock.status); @@ -638,6 +655,8 @@ erts_smp_proc_unlock__(Process *p, if (locks & ERTS_PROC_LOCK_STATUS) erts_mtx_unlock(&p->lock.status); + if (locks & ERTS_PROC_LOCK_BTM) + erts_mtx_unlock(&p->lock.btm); if (locks & ERTS_PROC_LOCK_MSGQ) erts_mtx_unlock(&p->lock.msgq); if (locks & ERTS_PROC_LOCK_LINK) @@ -752,9 +771,10 @@ ERTS_GLB_INLINE void erts_smp_proc_lock(Process *, ErtsProcLocks); ERTS_GLB_INLINE void erts_smp_proc_unlock(Process *, ErtsProcLocks); ERTS_GLB_INLINE int erts_smp_proc_trylock(Process *, ErtsProcLocks); -ERTS_GLB_INLINE void erts_smp_proc_inc_refc(Process *); -ERTS_GLB_INLINE void erts_smp_proc_dec_refc(Process *); -ERTS_GLB_INLINE void erts_smp_proc_add_refc(Process *, Sint32); +ERTS_GLB_INLINE void erts_proc_inc_refc(Process *); +ERTS_GLB_INLINE void erts_proc_dec_refc(Process *); +ERTS_GLB_INLINE void erts_proc_add_refc(Process *, Sint); +ERTS_GLB_INLINE Sint erts_proc_read_refc(Process *); #if ERTS_GLB_INLINE_INCL_FUNC_DEF @@ -814,28 +834,53 @@ erts_smp_proc_trylock(Process *p, ErtsProcLocks locks) #endif } -ERTS_GLB_INLINE void erts_smp_proc_inc_refc(Process *p) +ERTS_GLB_INLINE void erts_proc_inc_refc(Process *p) { + ASSERT(!(erts_smp_atomic32_read_nob(&p->state) & ERTS_PSFLG_PROXY)); #ifdef ERTS_SMP + erts_ptab_atmc_inc_refc(&p->common); +#else erts_ptab_inc_refc(&p->common); #endif } -ERTS_GLB_INLINE void erts_smp_proc_dec_refc(Process *p) +ERTS_GLB_INLINE void erts_proc_dec_refc(Process *p) { + Sint referred; + ASSERT(!(erts_smp_atomic32_read_nob(&p->state) & ERTS_PSFLG_PROXY)); #ifdef ERTS_SMP - int referred = erts_ptab_dec_test_refc(&p->common); - if (!referred) - erts_free_proc(p); + referred = erts_ptab_atmc_dec_test_refc(&p->common); +#else + referred = erts_ptab_dec_test_refc(&p->common); #endif + if (!referred) { + ASSERT(ERTS_PROC_IS_EXITING(p)); + erts_free_proc(p); + } } -ERTS_GLB_INLINE void erts_smp_proc_add_refc(Process *p, Sint32 add_refc) +ERTS_GLB_INLINE void erts_proc_add_refc(Process *p, Sint add_refc) { + Sint referred; + ASSERT(!(erts_smp_atomic32_read_nob(&p->state) & ERTS_PSFLG_PROXY)); #ifdef ERTS_SMP - int referred = erts_ptab_add_test_refc(&p->common, add_refc); - if (!referred) + referred = erts_ptab_atmc_add_test_refc(&p->common, add_refc); +#else + referred = erts_ptab_add_test_refc(&p->common, add_refc); +#endif + if (!referred) { + ASSERT(ERTS_PROC_IS_EXITING(p)); erts_free_proc(p); + } +} + +ERTS_GLB_INLINE Sint erts_proc_read_refc(Process *p) +{ + ASSERT(!(erts_smp_atomic32_read_nob(&p->state) & ERTS_PSFLG_PROXY)); +#ifdef ERTS_SMP + return erts_ptab_atmc_read_refc(&p->common); +#else + return erts_ptab_read_refc(&p->common); #endif } @@ -868,7 +913,7 @@ void erts_proc_safelock(Process *a_proc, #define ERTS_P2P_FLG_ALLOW_OTHER_X (1 << 0) #define ERTS_P2P_FLG_TRY_LOCK (1 << 1) -#define ERTS_P2P_FLG_SMP_INC_REFC (1 << 2) +#define ERTS_P2P_FLG_INC_REFC (1 << 2) #define ERTS_PROC_LOCK_BUSY ((Process *) &erts_invalid_process) @@ -928,11 +973,14 @@ erts_pid2proc_opt(Process *c_p_unused, int flags) { Process *proc = erts_proc_lookup_raw(pid); - return ((!(flags & ERTS_P2P_FLG_ALLOW_OTHER_X) - && proc - && ERTS_PROC_IS_EXITING(proc)) - ? NULL - : proc); + if (!proc) + return NULL; + if (!(flags & ERTS_P2P_FLG_ALLOW_OTHER_X) + && ERTS_PROC_IS_EXITING(proc)) + return NULL; + if (flags & ERTS_P2P_FLG_INC_REFC) + erts_proc_inc_refc(proc); + return proc; } #endif /* !ERTS_SMP */ diff --git a/erts/emulator/beam/erl_ptab.c b/erts/emulator/beam/erl_ptab.c index eabf016081..9ed175fe01 100644 --- a/erts/emulator/beam/erl_ptab.c +++ b/erts/emulator/beam/erl_ptab.c @@ -3,16 +3,17 @@ * * Copyright Ericsson AB 2012-2013. All Rights Reserved. * - * The contents of this file are subject to the Erlang Public License, - * Version 1.1, (the "License"); you may not use this file except in - * compliance with the License. You should have received a copy of the - * Erlang Public License along with this software. If not, it can be - * retrieved online at http://www.erlang.org/. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Software distributed under the License is distributed on an "AS IS" - * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See - * the License for the specific language governing rights and limitations - * under the License. + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. * * %CopyrightEnd% */ @@ -280,124 +281,38 @@ struct ErtsPTabListBifData_ { }; -#ifdef ARCH_32 - -static ERTS_INLINE Uint64 -dw_aint_to_uint64(erts_dw_aint_t *dw) -{ -#ifdef ETHR_SU_DW_NAINT_T__ - return (Uint64) dw->dw_sint; -#else - Uint64 res; - res = (Uint64) ((Uint32) dw->sint[ERTS_DW_AINT_HIGH_WORD]); - res <<= 32; - res |= (Uint64) ((Uint32) dw->sint[ERTS_DW_AINT_LOW_WORD]); - return res; -#endif -} - -static void -unint64_to_dw_aint(erts_dw_aint_t *dw, Uint64 val) -{ -#ifdef ETHR_SU_DW_NAINT_T__ - dw->dw_sint = (ETHR_SU_DW_NAINT_T__) val; -#else - dw->sint[ERTS_DW_AINT_LOW_WORD] = (erts_aint_t) (val & 0xffffffff); - dw->sint[ERTS_DW_AINT_HIGH_WORD] = (erts_aint_t) ((val >> 32) & 0xffffffff); -#endif -} - static ERTS_INLINE void last_data_init_nob(ErtsPTab *ptab, Uint64 val) { - erts_dw_aint_t dw; - unint64_to_dw_aint(&dw, val); - erts_smp_dw_atomic_init_nob(&ptab->vola.tile.last_data, &dw); + erts_smp_atomic64_init_nob(&ptab->vola.tile.last_data, (erts_aint64_t) val); } static ERTS_INLINE void last_data_set_relb(ErtsPTab *ptab, Uint64 val) { - erts_dw_aint_t dw; - unint64_to_dw_aint(&dw, val); - erts_smp_dw_atomic_set_relb(&ptab->vola.tile.last_data, &dw); + erts_smp_atomic64_set_relb(&ptab->vola.tile.last_data, (erts_aint64_t) val); } static ERTS_INLINE Uint64 last_data_read_nob(ErtsPTab *ptab) { - erts_dw_aint_t dw; - erts_smp_dw_atomic_read_nob(&ptab->vola.tile.last_data, &dw); - return dw_aint_to_uint64(&dw); + return (Uint64) erts_smp_atomic64_read_nob(&ptab->vola.tile.last_data); } static ERTS_INLINE Uint64 last_data_read_acqb(ErtsPTab *ptab) { - erts_dw_aint_t dw; - erts_smp_dw_atomic_read_acqb(&ptab->vola.tile.last_data, &dw); - return dw_aint_to_uint64(&dw); + return (Uint64) erts_smp_atomic64_read_acqb(&ptab->vola.tile.last_data); } static ERTS_INLINE Uint64 last_data_cmpxchg_relb(ErtsPTab *ptab, Uint64 new, Uint64 exp) { - erts_dw_aint_t dw_new, dw_xchg; - - unint64_to_dw_aint(&dw_new, new); - unint64_to_dw_aint(&dw_xchg, exp); - - if (erts_smp_dw_atomic_cmpxchg_relb(&ptab->vola.tile.last_data, - &dw_new, - &dw_xchg)) - return exp; - else - return dw_aint_to_uint64(&dw_xchg); + return (Uint64) erts_smp_atomic64_cmpxchg_relb(&ptab->vola.tile.last_data, + (erts_aint64_t) new, + (erts_aint64_t) exp); } -#elif defined(ARCH_64) - -union { - erts_smp_atomic_t pid_data; - char align[ERTS_CACHE_LINE_SIZE]; -} last erts_align_attribute(ERTS_CACHE_LINE_SIZE); - -static ERTS_INLINE void -last_data_init_nob(ErtsPTab *ptab, Uint64 val) -{ - erts_smp_atomic_init_nob(&ptab->vola.tile.last_data, (erts_aint_t) val); -} - -static ERTS_INLINE void -last_data_set_relb(ErtsPTab *ptab, Uint64 val) -{ - erts_smp_atomic_set_relb(&ptab->vola.tile.last_data, (erts_aint_t) val); -} - -static ERTS_INLINE Uint64 -last_data_read_nob(ErtsPTab *ptab) -{ - return (Uint64) erts_smp_atomic_read_nob(&ptab->vola.tile.last_data); -} - -static ERTS_INLINE Uint64 -last_data_read_acqb(ErtsPTab *ptab) -{ - return (Uint64) erts_smp_atomic_read_acqb(&ptab->vola.tile.last_data); -} - -static ERTS_INLINE Uint64 -last_data_cmpxchg_relb(ErtsPTab *ptab, Uint64 new, Uint64 exp) -{ - return (Uint64) erts_smp_atomic_cmpxchg_relb(&ptab->vola.tile.last_data, - (erts_aint_t) new, - (erts_aint_t) exp); -} - -#else -# error "Not 64-bit, nor 32-bit architecture..." -#endif - static ERTS_INLINE int last_data_cmp(Uint64 ld1, Uint64 ld2) { @@ -446,7 +361,8 @@ erts_ptab_init_table(ErtsPTab *ptab, int size, UWord element_size, char *name, - int legacy) + int legacy, + int atomic_refc) { size_t tab_sz, alloc_sz; Uint32 bits, cl, cli, ix, ix_per_cache_line, tab_cache_lines; @@ -501,6 +417,8 @@ erts_ptab_init_table(ErtsPTab *ptab, ptab->r.o.invalid_data = erts_ptab_id2data(ptab, invalid_element->id); ptab->r.o.release_element = release_element; + ptab->r.o.atomic_refc = atomic_refc; + if (legacy) { ptab->r.o.free_id_data = NULL; ptab->r.o.dix_cl_mask = 0; @@ -619,9 +537,10 @@ erts_ptab_new_element(ErtsPTab *ptab, init_ptab_el(init_arg, (Eterm) data); -#ifdef ERTS_SMP - erts_smp_atomic32_init_nob(&ptab_el->refc, 1); -#endif + if (ptab->r.o.atomic_refc) + erts_atomic_init_nob(&ptab_el->refc.atmc, 1); + else + ptab_el->refc.sint = 1; pix = erts_ptab_data2pix(ptab, (Eterm) data); @@ -694,9 +613,10 @@ erts_ptab_new_element(ErtsPTab *ptab, init_ptab_el(init_arg, data); -#ifdef ERTS_SMP - erts_smp_atomic32_init_nob(&ptab_el->refc, 1); -#endif + if (ptab->r.o.atomic_refc) + erts_atomic_init_nob(&ptab_el->refc.atmc, 1); + else + ptab_el->refc.sint = 1; /* Move into slot reserved */ #ifdef DEBUG @@ -1335,7 +1255,7 @@ ptab_list_bif_engine(Process *c_p, Eterm *res_accp, Binary *mbp) return 1; default: - erl_exit(ERTS_ABORT_EXIT, + erts_exit(ERTS_ABORT_EXIT, "%s:%d:ptab_list_bif_engine(): Invalid state: %d\n", __FILE__, __LINE__, (int) ptlbdp->state); } diff --git a/erts/emulator/beam/erl_ptab.h b/erts/emulator/beam/erl_ptab.h index e3e05f14af..8fd961e3ce 100644 --- a/erts/emulator/beam/erl_ptab.h +++ b/erts/emulator/beam/erl_ptab.h @@ -3,16 +3,17 @@ * * Copyright Ericsson AB 2012-2013. All Rights Reserved. * - * The contents of this file are subject to the Erlang Public License, - * Version 1.1, (the "License"); you may not use this file except in - * compliance with the License. You should have received a copy of the - * Erlang Public License along with this software. If not, it can be - * retrieved online at http://www.erlang.org/. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Software distributed under the License is distributed on an "AS IS" - * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See - * the License for the specific language governing rights and limitations - * under the License. + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. * * %CopyrightEnd% */ @@ -51,11 +52,13 @@ typedef struct { Eterm id; -#ifdef ERTS_SMP - erts_atomic32_t refc; -#endif + union { + erts_atomic_t atmc; + Sint sint; + } refc; Eterm tracer_proc; Uint trace_flags; + erts_smp_atomic_t timer; union { /* --- While being alive --- */ struct { @@ -63,11 +66,6 @@ typedef struct { struct reg_proc *reg; ErtsLink *links; ErtsMonitor *monitors; -#ifdef ERTS_SMP - ErtsSmpPTimer *ptimer; -#else - ErlTimer tm; -#endif } alive; /* --- While being released --- */ @@ -88,11 +86,7 @@ typedef struct { } ErtsPTabListData; typedef struct { -#ifdef ARCH_32 - erts_smp_dw_atomic_t last_data; -#else - erts_smp_atomic_t last_data; -#endif + erts_smp_atomic64_t last_data; erts_smp_atomic32_t count; erts_smp_atomic32_t aid_ix; erts_smp_atomic32_t fid_ix; @@ -115,6 +109,7 @@ typedef struct { Eterm invalid_data; void (*release_element)(void *); UWord element_size; + int atomic_refc; } ErtsPTabReadOnlyData; typedef struct { @@ -185,7 +180,8 @@ void erts_ptab_init_table(ErtsPTab *ptab, int size, UWord element_size, char *name, - int legacy); + int legacy, + int atomic_refc); int erts_ptab_new_element(ErtsPTab *ptab, ErtsPTabElementCommon *ptab_el, void *init_arg, @@ -210,9 +206,15 @@ ERTS_GLB_INLINE erts_aint_t erts_ptab_pix2intptr_ddrb(ErtsPTab *ptab, int ix); ERTS_GLB_INLINE erts_aint_t erts_ptab_pix2intptr_rb(ErtsPTab *ptab, int ix); ERTS_GLB_INLINE erts_aint_t erts_ptab_pix2intptr_acqb(ErtsPTab *ptab, int ix); ERTS_GLB_INLINE void erts_ptab_inc_refc(ErtsPTabElementCommon *ptab_el); -ERTS_GLB_INLINE int erts_ptab_dec_test_refc(ErtsPTabElementCommon *ptab_el); -ERTS_GLB_INLINE int erts_ptab_add_test_refc(ErtsPTabElementCommon *ptab_el, - Sint32 add_refc); +ERTS_GLB_INLINE Sint erts_ptab_dec_test_refc(ErtsPTabElementCommon *ptab_el); +ERTS_GLB_INLINE Sint erts_ptab_add_test_refc(ErtsPTabElementCommon *ptab_el, + Sint add_refc); +ERTS_GLB_INLINE Sint erts_ptab_read_refc(ErtsPTabElementCommon *ptab_el); +ERTS_GLB_INLINE void erts_ptab_atmc_inc_refc(ErtsPTabElementCommon *ptab_el); +ERTS_GLB_INLINE Sint erts_ptab_atmc_dec_test_refc(ErtsPTabElementCommon *ptab_el); +ERTS_GLB_INLINE Sint erts_ptab_atmc_add_test_refc(ErtsPTabElementCommon *ptab_el, + Sint add_refc); +ERTS_GLB_INLINE Sint erts_ptab_atmc_read_refc(ErtsPTabElementCommon *ptab_el); ERTS_GLB_INLINE void erts_ptab_rlock(ErtsPTab *ptab); ERTS_GLB_INLINE int erts_ptab_tryrlock(ErtsPTab *ptab); ERTS_GLB_INLINE void erts_ptab_runlock(ErtsPTab *ptab); @@ -369,50 +371,65 @@ ERTS_GLB_INLINE erts_aint_t erts_ptab_pix2intptr_acqb(ErtsPTab *ptab, int ix) return erts_smp_atomic_read_acqb(&ptab->r.o.tab[ix]); } -ERTS_GLB_INLINE void erts_ptab_inc_refc(ErtsPTabElementCommon *ptab_el) +ERTS_GLB_INLINE void erts_ptab_atmc_inc_refc(ErtsPTabElementCommon *ptab_el) { -#ifdef ERTS_SMP #ifdef ERTS_ENABLE_LOCK_CHECK - erts_aint32_t refc = erts_atomic32_inc_read_nob(&ptab_el->refc); - ERTS_SMP_LC_ASSERT(refc > 1); + erts_aint_t refc = erts_atomic_inc_read_nob(&ptab_el->refc.atmc); + ERTS_LC_ASSERT(refc > 1); #else - erts_atomic32_inc_nob(&ptab_el->refc); -#endif + erts_atomic_inc_nob(&ptab_el->refc.atmc); #endif } -ERTS_GLB_INLINE int erts_ptab_dec_test_refc(ErtsPTabElementCommon *ptab_el) +ERTS_GLB_INLINE Sint erts_ptab_atmc_dec_test_refc(ErtsPTabElementCommon *ptab_el) { -#ifdef ERTS_SMP - erts_aint32_t refc = erts_atomic32_dec_read_nob(&ptab_el->refc); + erts_aint_t refc = erts_atomic_dec_read_relb(&ptab_el->refc.atmc); ERTS_SMP_LC_ASSERT(refc >= 0); - return (int) refc; -#else - return 0; +#ifdef ERTS_SMP + if (refc == 0) + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore); #endif + return (Sint) refc; } -ERTS_GLB_INLINE int erts_ptab_add_test_refc(ErtsPTabElementCommon *ptab_el, - Sint32 add_refc) +ERTS_GLB_INLINE Sint erts_ptab_atmc_add_test_refc(ErtsPTabElementCommon *ptab_el, + Sint add_refc) { -#ifdef ERTS_SMP - erts_aint32_t refc; + erts_aint_t refc = erts_atomic_add_read_mb(&ptab_el->refc.atmc, + (erts_aint_t) add_refc); + ERTS_SMP_LC_ASSERT(refc >= 0); + return (Sint) refc; +} -#ifndef ERTS_ENABLE_LOCK_CHECK - if (add_refc >= 0) { - erts_atomic32_add_nob(&ptab_el->refc, - (erts_aint32_t) add_refc); - return 1; - } -#endif +ERTS_GLB_INLINE Sint erts_ptab_atmc_read_refc(ErtsPTabElementCommon *ptab_el) +{ + return (Sint) erts_atomic_read_nob(&ptab_el->refc.atmc); +} - refc = erts_atomic32_add_read_nob(&ptab_el->refc, - (erts_aint32_t) add_refc); +ERTS_GLB_INLINE void erts_ptab_inc_refc(ErtsPTabElementCommon *ptab_el) +{ + ptab_el->refc.sint++; + ASSERT(ptab_el->refc.sint > 1); +} + +ERTS_GLB_INLINE Sint erts_ptab_dec_test_refc(ErtsPTabElementCommon *ptab_el) +{ + Sint refc = --ptab_el->refc.sint; ERTS_SMP_LC_ASSERT(refc >= 0); - return (int) refc; -#else - return 0; -#endif + return refc; +} + +ERTS_GLB_INLINE Sint erts_ptab_add_test_refc(ErtsPTabElementCommon *ptab_el, + Sint add_refc) +{ + ptab_el->refc.sint += add_refc; + ERTS_SMP_LC_ASSERT(ptab_el->refc.sint >= 0); + return (Sint) ptab_el->refc.sint; +} + +ERTS_GLB_INLINE Sint erts_ptab_read_refc(ErtsPTabElementCommon *ptab_el) +{ + return ptab_el->refc.sint; } ERTS_GLB_INLINE void erts_ptab_rlock(ErtsPTab *ptab) diff --git a/erts/emulator/beam/erl_rbtree.h b/erts/emulator/beam/erl_rbtree.h new file mode 100644 index 0000000000..5fefaea978 --- /dev/null +++ b/erts/emulator/beam/erl_rbtree.h @@ -0,0 +1,1741 @@ +/* + * %CopyrightBegin% + * + * Copyright Ericsson AB 2015. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * %CopyrightEnd% + */ + +/* + * Description: A Red-Black (binary search) Tree implementation. The search, + * insert, and delete operations are all O(log n) operations + * on a Red-Black Tree. Red-Black Trees are described in + * "Introduction to Algorithms", by Thomas H. Cormen, Charles + * E. Leiserson, and Ronald L. Riverest. + * + * Use by defining mandatory defines as well as defines for + * API functions wanted, and include this header. + * + * Author: Rickard Green + * + * + * Mandatory defines: + * - ERTS_RBT_PREFIX - Prefix to use on functions. + * - ERTS_RBT_T - Type of a tree node. + * - ERTS_RBT_KEY_T - Type of key for a tree node. + * - ERTS_RBT_FLAGS_T - Type of flags for a tree node. + * - ERTS_RBT_INIT_EMPTY_TNODE(T) -Initialize an empty tree node. + * - ERTS_RBT_IS_RED(T) - Is tree node red? + * - ERTS_RBT_SET_RED(T) - Set tree node red. + * - ERTS_RBT_IS_BLACK(T) - Is tree node back? + * - ERTS_RBT_SET_BLACK(T) - Set tree node black. + * - ERTS_RBT_GET_FLAGS(T) - Get flags of tree node (incl colors). + * - ERTS_RBT_SET_FLAGS(T, F) - Set flags of tree note. + * - ERTS_RBT_GET_PARENT(T) - Get parent node. + * - ERTS_RBT_SET_PARENT(T, P) - Set parent node. + * - ERTS_RBT_GET_RIGHT(T) - Get right child node. + * - ERTS_RBT_SET_RIGHT(T, R) - Set right child node. + * - ERTS_RBT_GET_LEFT(T) - Get left child node. + * - ERTS_RBT_SET_LEFT(T, L) - Set left child node. + * - ERTS_RBT_GET_KEY(T) - Get key of node. + * - ERTS_RBT_IS_LT(KX, KY) - Is key KX less than key KY? + * - ERTS_RBT_IS_EQ(KX, KY) - Is key KX equal to key KY? + * + * Optional defines: + * + * - ERTS_RBT_UNDEF - Undefine all user defined ERTS_RBT_* + * defines after use. + * + * - ERTS_RBT_NO_API_INLINE - Do not inline API functions. + * + * Attached data management: + * - ERTS_RBT_UPDATE_ATTACHED_DATA_ROTATE(L, OP, NP) - Called + * when a rotate operation has been performed. If L (in int) + * is a non zero, a left rotation was performed; otherwise, + * a right rotation was performed. OR points to the old + * parent node and NP points to the new parent node. + * - ERTS_RBT_UPDATE_ATTACHED_DATA_DMOD(F, T) - Called when + * a delete operation modifies a tree node. A delete + * modification is either a removal or replacement of a + * node. F points to the parent of the tree node that was + * modified. T points to the next ancestor that will be + * modified. If T is NULL, no more removal and/or + * replacements will be made. One typically wants to update + * the attached data of each node between F and T. If T is + * NULL all the way up to the root. + * - ERTS_RBT_UPDATE_ATTACHED_DATA_CHGROOT(OR, NR) - Called + * when the root node changes. OR points to the old + * root node and NP points to the new root node. + * + * Request implementation of API functions: + * - ERTS_RBT_WANT_DELETE + * - ERTS_RBT_WANT_INSERT + * - ERTS_RBT_WANT_LOOKUP_INSERT + * - ERTS_RBT_WANT_REPLACE + * - ERTS_RBT_WANT_LOOKUP + * - ERTS_RBT_WANT_SMALLEST + * - ERTS_RBT_WANT_LARGEST + * - ERTS_RBT_WANT_FOREACH + * - ERTS_RBT_WANT_FOREACH_DESTROY + * - ERTS_RBT_WANT_FOREACH_YIELDING + * - ERTS_RBT_WANT_FOREACH_DESTROY_YIELDING + * - ERTS_RBT_WANT_FOREACH_SMALL + * - ERTS_RBT_WANT_FOREACH_LARGE + * - ERTS_RBT_WANT_FOREACH_SMALL_DESTROY + * - ERTS_RBT_WANT_FOREACH_LARGE_DESTROY + * - ERTS_RBT_WANT_FOREACH_SMALL_YIELDING + * - ERTS_RBT_WANT_FOREACH_LARGE_YIELDING + * - ERTS_RBT_WANT_FOREACH_SMALL_DESTROY_YIELDING + * - ERTS_RBT_WANT_FOREACH_LARGE_DESTROY_YIELDING + * - ERTS_RBT_WANT_DEBUG_PRINT + * + * The yield state data type will equal + * <ERTS_RBT_PREFIX>_rbt_yield_state_t. + * + * The yield state should be statically initialized by + * ERTS_RBT_YIELD_STAT_INITER. + * + * + * The following API functions are implemented if corresponding + * ERTS_RBT_WANT_<OPERATION> is defined: + * + * - void <ERTS_RBT_PREFIX>_rbt_delete( + * ERTS_RBT_T **tree, + * ERTS_RBT_T *element); + * Delete element from tree. + * + * - void <ERTS_RBT_PREFIX>_rbt_insert( + * ERTS_RBT_T **tree, + * ERTS_RBT_T *element); + * Insert element into tree. + * + * - ERTS_RBT_T * <ERTS_RBT_PREFIX>_rbt_lookup_insert( + * ERTS_RBT_T **tree, + * ERTS_RBT_T *element); + * Look up an element in the tree that compares as equal to the + * element passed as argument, and return the looked up element. + * If no element compared as equal, insert the element passed as + * argument into the tree, and return NULL. + * + * - void <ERTS_RBT_PREFIX>_rbt_replace( + * ERTS_RBT_T **tree, + * ERTS_RBT_T *old_element, + * ERTS_RBT_T *new_element); + * Replace old_element in the tree with new_element. Both elements + * *should* compare as equal. + * + * - ERTS_RBT_T * <ERTS_RBT_PREFIX>_rbt_lookup( + * ERTS_RBT_T *tree, + * ERTS_RBT_KEY_T key); + * Look up an element with a key that compares as equal to + * the key passed as argument. + * + * - ERTS_RBT_T * <ERTS_RBT_PREFIX>_rbt_smallest( + * ERTS_RBT_T *tree); + * Look up the element with the smallest key. + * + * - ERTS_RBT_T * <ERTS_RBT_PREFIX>_rbt_largest( + * ERTS_RBT_T *tree); + * Look up the element with the largest key. + * + * - void <ERTS_RBT_PREFIX>_rbt_foreach( + * ERTS_RBT_T *tree, + * void (*op)(ERTS_RBT_T *, void *), + * void *arg); + * Operate by calling the operator 'op' on each element. + * Order is undefined. + * + * 'arg' is passed as argument to 'op'. + * + * - void <ERTS_RBT_PREFIX>_rbt_foreach_destroy( + * ERTS_RBT_T *tree, + * void (*op)(ERTS_RBT_T *, void *), + * void *arg); + * Operate by calling the operator 'op' on each element. + * Order is undefined. Each element should be destroyed + * by 'op'. + * + * 'arg' is passed as argument to 'op'. + * + * - int <ERTS_RBT_PREFIX>_rbt_foreach_yielding( + * ERTS_RBT_T *tree, + * void (*op)(ERTS_RBT_T *, void *), + * void *arg, + * <ERTS_RBT_PREFIX>_rbt_yield_state_t *ystate, + * Sint ylimit); + * Operate by calling the operator 'op' on each element. + * Order is undefined. + * + * Yield when 'ylimit' elements has been processed. Zero is + * returned when yielding, and a non-zero value is returned when + * the whole tree has been processed. The tree should not be + * modified until all of it has been processed. + * + * 'arg' is passed as argument to 'op'. + * + * - int <ERTS_RBT_PREFIX>_rbt_foreach_destroy_yielding( + * ERTS_RBT_T *tree, + * void (*op)(ERTS_RBT_T *, void *), + * void *arg, + * <ERTS_RBT_PREFIX>_rbt_yield_state_t *ystate, + * Sint ylimit); + * Operate by calling the operator 'op' on each element. + * Order is undefined. Each element should be destroyed + * by 'op'. + * + * Yield when 'ylimit' elements has been processed. Zero is + * returned when yielding, and a non-zero value is returned when + * the whole tree has been processed. + * + * 'arg' is passed as argument to 'op'. + * + * - void <ERTS_RBT_PREFIX>_rbt_foreach_small( + * ERTS_RBT_T *tree, + * void (*op)(ERTS_RBT_T *, void *), + * void *arg); + * Operate by calling the operator 'op' on each element from + * smallest towards larger elements. + * + * 'arg' is passed as argument to 'op'. + * + * - void <ERTS_RBT_PREFIX>_rbt_foreach_large( + * ERTS_RBT_T *tree, + * void (*op)(ERTS_RBT_T *, void *), + * void *arg); + * Operate by calling the operator 'op' on each element from + * largest towards smaller elements. + * + * 'arg' is passed as argument to 'op'. + * + * - int <ERTS_RBT_PREFIX>_rbt_foreach_small_yielding( + * ERTS_RBT_T *tree, + * void (*op)(ERTS_RBT_T *, void *), + * void *arg, + * <ERTS_RBT_PREFIX>_rbt_yield_state_t *ystate, + * Sint ylimit); + * Operate by calling the operator 'op' on each element from + * smallest towards larger elements. + * + * Yield when 'ylimit' elements has been processed. Zero is + * returned when yielding, and a non-zero value is returned when + * the whole tree has been processed. The tree should not be + * modified until all of it has been processed. + * + * 'arg' is passed as argument to 'op'. + * + * - int <ERTS_RBT_PREFIX>_rbt_foreach_large_yielding( + * ERTS_RBT_T *tree, + * void (*op)(ERTS_RBT_T *, void *), + * void *arg, + * <ERTS_RBT_PREFIX>_rbt_yield_state_t *ystate, + * Sint ylimit); + * Operate by calling the operator 'op' on each element from + * largest towards smaller elements. + * + * Yield when 'ylimit' elements has been processed. Zero is + * returned when yielding, and a non-zero value is returned when + * the whole tree has been processed. The tree should not be + * modified until all of it has been processed. + * + * 'arg' is passed as argument to 'op'. + * + * - void <ERTS_RBT_PREFIX>_rbt_foreach_small_destroy( + * ERTS_RBT_T **tree, + * void (*op)(ERTS_RBT_T *, void *), + * void (*destr)(ERTS_RBT_T *, void *), + * void *arg); + * Operate by calling the operator 'op' on each element from + * smallest towards larger elements. + * + * Destroy elements by calling the destructor 'destr'. Elements + * are destroyed when not needed by the tree structure anymore. + * Note that elements are often *not* destroyed in another order + * than the order that the elements are operated on. + * + * 'arg' is passed as argument to 'op' and 'destroy'. + * + * - void <ERTS_RBT_PREFIX>_rbt_foreach_large_destroy( + * ERTS_RBT_T **tree, + * void (*op)(ERTS_RBT_T *, void *), + * void (*destr)(ERTS_RBT_T *, void *), + * void *arg); + * Operate by calling the operator 'op' on each element from + * largest towards smaller elements. + * + * Destroy elements by calling the destructor 'destr'. Elements + * are destroyed when not needed by the tree structure anymore. + * Note that elements are often destroyed in another order + * than the order that the elements are operated on. + * + * 'arg' is passed as argument to 'op' and 'destroy'. + * + * - int <ERTS_RBT_PREFIX>_rbt_foreach_small_destroy_yielding( + * ERTS_RBT_T **tree, + * void (*op)(ERTS_RBT_T *, void *), + * void (*destr)(ERTS_RBT_T *, void *), + * void *arg, + * <ERTS_RBT_PREFIX>_rbt_yield_state_t *ystate, + * Sint ylimit); + * Operate by calling the operator 'op' on each element from + * smallest towards larger elements. + * + * Destroy elements by calling the destructor 'destr'. Elements + * are destroyed when not needed by the tree structure anymore. + * Note that elements are often destroyed in another order + * than the order that the elements are operated on. + * + * Yield when 'ylimit' elements has been processed. Zero is + * returned when yielding, and a non-zero value is returned when + * the whole tree has been processed. The tree should not be + * modified until all of it has been processed. + * + * 'arg' is passed as argument to 'op' and 'destroy'. + * + * - int <ERTS_RBT_PREFIX>_rbt_foreach_large_destroy_yielding( + * ERTS_RBT_T **tree, + * void (*op)(ERTS_RBT_T *, void *), + * void (*destr)(ERTS_RBT_T *, void *), + * void *arg, + * <ERTS_RBT_PREFIX>_rbt_yield_state_t *ystate, + * Sint ylimit); + * Operate by calling the operator 'op' on each element from + * largest towards smaller elements. + * + * Destroy elements by calling the destructor 'destr'. Elements + * are destroyed when not needed by the tree structure anymore. + * Note that elements are often destroyed in another order + * than the order that the elements are operated on. + * + * Yield when 'ylimit' elements has been processed. Zero is + * returned when yielding, and a non-zero value is returned when + * the whole tree has been processed. The tree should not be + * modified until all of it has been processed. + * + * 'arg' is passed as argument to 'op' and 'destroy'. + * + * - void <ERTS_RBT_PREFIX>_rbt_debug_print( + * FILE *filep, + * ERTS_RBT_T *x, + * int indent, + * (void)(*print_node)(ERTS_RBT_T *)); + * Prints the tree. Note that this function is recursive. + * Should only be used for debuging. + */ + + +/* + * Check that we have all mandatory defines + */ +#ifndef ERTS_RBT_PREFIX +# error Missing definition of ERTS_RBT_PREFIX +#endif +#ifndef ERTS_RBT_T +# error Missing definition of ERTS_RBT_T +#endif +#ifndef ERTS_RBT_KEY_T +# error Missing definition of ERTS_RBT_KEY_T +#endif +#ifndef ERTS_RBT_FLAGS_T +# error Missing definition of ERTS_RBT_FLAGS_T +#endif +#ifndef ERTS_RBT_INIT_EMPTY_TNODE +# error Missing definition of ERTS_RBT_INIT_EMPTY_TNODE +#endif +#ifndef ERTS_RBT_IS_RED +# error Missing definition of ERTS_RBT_IS_RED +#endif +#ifndef ERTS_RBT_SET_RED +# error Missing definition of ERTS_RBT_SET_RED +#endif +#ifndef ERTS_RBT_IS_BLACK +# error Missing definition of ERTS_RBT_IS_BLACK +#endif +#ifndef ERTS_RBT_SET_BLACK +# error Missing definition of ERTS_RBT_SET_BLACK +#endif +#ifndef ERTS_RBT_GET_FLAGS +# error Missing definition of ERTS_RBT_GET_FLAGS +#endif +#ifndef ERTS_RBT_SET_FLAGS +# error Missing definition of ERTS_RBT_SET_FLAGS +#endif +#ifndef ERTS_RBT_GET_PARENT +# error Missing definition of ERTS_RBT_GET_PARENT +#endif +#ifndef ERTS_RBT_SET_PARENT +# error Missing definition of ERTS_RBT_SET_PARENT +#endif +#ifndef ERTS_RBT_GET_RIGHT +# error Missing definition of ERTS_RBT_GET_RIGHT +#endif +#ifndef ERTS_RBT_GET_LEFT +# error Missing definition of ERTS_RBT_GET_LEFT +#endif +#ifndef ERTS_RBT_IS_LT +# error Missing definition of ERTS_RBT_IS_LT +#endif +#ifndef ERTS_RBT_GET_KEY +# error Missing definition of ERTS_RBT_GET_KEY +#endif +#ifndef ERTS_RBT_IS_EQ +# error Missing definition of ERTS_RBT_IS_EQ +#endif + +#if defined(ERTS_RBT_HARD_DEBUG) || defined(DEBUG) +# ifndef ERTS_RBT_DEBUG +# define ERTS_RBT_DEBUG 1 +# endif +#endif + +#if defined(ERTS_RBT_HARD_DEBUG) && defined(__GNUC__) +#warning "* * * * * * * * * * * * * * * * * *" +#warning "* ERTS_RBT_HARD_DEBUG IS ENABLED! *" +#warning "* * * * * * * * * * * * * * * * * *" +#endif + +#undef ERTS_RBT_ASSERT +#if defined(ERTS_RBT_DEBUG) +#define ERTS_RBT_ASSERT(E) ERTS_ASSERT(E) +#else +#define ERTS_RBT_ASSERT(E) ((void) 1) +#endif + +#undef ERTS_RBT_API_INLINE__ +#if defined(ERTS_RBT_NO_API_INLINE) || defined(ERTS_RBT_DEBUG) +# define ERTS_RBT_API_INLINE__ +#else +# define ERTS_RBT_API_INLINE__ ERTS_INLINE +#endif + +#ifndef ERTS_RBT_YIELD_STAT_INITER +# define ERTS_RBT_YIELD_STAT_INITER {NULL, 0} +#endif + +#define ERTS_RBT_CONCAT_MACRO_VALUES___(X, Y) \ + X ## Y +#define ERTS_RBT_CONCAT_MACRO_VALUES__(X, Y) \ + ERTS_RBT_CONCAT_MACRO_VALUES___(X, Y) + +#undef ERTS_RBT_YIELD_STATE_T__ +#define ERTS_RBT_YIELD_STATE_T__ \ + ERTS_RBT_CONCAT_MACRO_VALUES__(ERTS_RBT_PREFIX, _rbt_yield_state_t) + +typedef struct { + ERTS_RBT_T *x; + int up; +} ERTS_RBT_YIELD_STATE_T__; + +#define ERTS_RBT_FUNC__(Name) \ + ERTS_RBT_CONCAT_MACRO_VALUES__(ERTS_RBT_PREFIX, _rbt_ ## Name) + +#undef ERTS_RBT_NEED_REPLACE__ +#undef ERTS_RBT_NEED_INSERT__ +#undef ERTS_RBT_NEED_ROTATE__ +#undef ERTS_RBT_NEED_FOREACH_UNORDERED__ +#undef ERTS_RBT_NEED_FOREACH_ORDERED__ +#undef ERTS_RBT_NEED_HDBG_CHECK_TREE__ +#undef ERTS_RBT_HDBG_CHECK_TREE__ + +#if defined(ERTS_RBT_WANT_REPLACE) || defined(ERTS_RBT_WANT_DELETE) +# define ERTS_RBT_NEED_REPLACE__ +#endif +#if defined(ERTS_RBT_WANT_INSERT) || defined(ERTS_RBT_WANT_LOOKUP_INSERT) +# define ERTS_RBT_NEED_INSERT__ +#endif +#if defined(ERTS_RBT_WANT_DELETE) || defined(ERTS_RBT_NEED_INSERT__) +# define ERTS_RBT_NEED_ROTATE__ +#endif +#if defined(ERTS_RBT_WANT_FOREACH) \ + || defined(ERTS_RBT_WANT_FOREACH_YIELDING) \ + || defined(ERTS_RBT_WANT_FOREACH_DESTROY) \ + || defined(ERTS_RBT_WANT_FOREACH_DESTROY_YIELDING) +# define ERTS_RBT_NEED_FOREACH_UNORDERED__ +#endif +#if defined(ERTS_RBT_WANT_FOREACH_SMALL) \ + || defined(ERTS_RBT_WANT_FOREACH_LARGE) \ + || defined(ERTS_RBT_WANT_FOREACH_SMALL_YIELDING) \ + || defined(ERTS_RBT_WANT_FOREACH_LARGE_YIELDING) \ + || defined(ERTS_RBT_WANT_FOREACH_SMALL_DESTROY) \ + || defined(ERTS_RBT_WANT_FOREACH_LARGE_DESTROY) \ + || defined(ERTS_RBT_WANT_FOREACH_SMALL_DESTROY_YIELDING) \ + || defined(ERTS_RBT_WANT_FOREACH_LARGE_DESTROY_YIELDING) +# define ERTS_RBT_NEED_FOREACH_ORDERED__ +#endif +#if defined(ERTS_RBT_HARD_DEBUG) \ + && (defined(ERTS_RBT_WANT_DELETE) \ + || defined(ERTS_RBT_NEED_INSERT__)) +static void ERTS_RBT_FUNC__(hdbg_check_tree)(ERTS_RBT_T *root); +# define ERTS_RBT_NEED_HDBG_CHECK_TREE__ +# define ERTS_RBT_HDBG_CHECK_TREE__(R) \ + ERTS_RBT_FUNC__(hdbg_check_tree)((R)) +#else +# define ERTS_RBT_HDBG_CHECK_TREE__(R) ((void) 1) +#endif + +#ifdef ERTS_RBT_NEED_ROTATE__ + +static ERTS_INLINE void +ERTS_RBT_FUNC__(left_rotate__)(ERTS_RBT_T **root, ERTS_RBT_T *x) +{ + ERTS_RBT_T *y, *l, *p; + + y = ERTS_RBT_GET_RIGHT(x); + l = ERTS_RBT_GET_LEFT(y); + ERTS_RBT_SET_RIGHT(x, l); + + if (l) + ERTS_RBT_SET_PARENT(l, x); + + p = ERTS_RBT_GET_PARENT(x); + ERTS_RBT_SET_PARENT(y, p); + + if (!p) { + ERTS_RBT_ASSERT(*root == x); + *root = y; +#ifdef ERTS_RBT_UPDATE_ATTACHED_DATA_CHGROOT + ERTS_RBT_UPDATE_ATTACHED_DATA_CHGROOT(x, y); +#endif + } + else if (x == ERTS_RBT_GET_LEFT(p)) + ERTS_RBT_SET_LEFT(p, y); + else { + ERTS_RBT_ASSERT(x == ERTS_RBT_GET_RIGHT(p)); + ERTS_RBT_SET_RIGHT(p, y); + } + ERTS_RBT_SET_LEFT(y, x); + ERTS_RBT_SET_PARENT(x, y); + +#ifdef ERTS_RBT_UPDATE_ATTACHED_DATA_ROTATE + ERTS_RBT_UPDATE_ATTACHED_DATA_ROTATE(!0, x, y); +#endif + +} + +static ERTS_INLINE void +ERTS_RBT_FUNC__(right_rotate__)(ERTS_RBT_T **root, ERTS_RBT_T *x) +{ + ERTS_RBT_T *y, *r, *p; + + y = ERTS_RBT_GET_LEFT(x); + r = ERTS_RBT_GET_RIGHT(y); + ERTS_RBT_SET_LEFT(x, r); + + if (r) + ERTS_RBT_SET_PARENT(r, x); + + p = ERTS_RBT_GET_PARENT(x); + ERTS_RBT_SET_PARENT(y, p); + + if (!p) { + ERTS_RBT_ASSERT(*root == x); + *root = y; +#ifdef ERTS_RBT_UPDATE_ATTACHED_DATA_CHGROOT + ERTS_RBT_UPDATE_ATTACHED_DATA_CHGROOT(x, y); +#endif + } + else if (x == ERTS_RBT_GET_RIGHT(p)) + ERTS_RBT_SET_RIGHT(p, y); + else { + ERTS_RBT_ASSERT(x == ERTS_RBT_GET_LEFT(p)); + ERTS_RBT_SET_LEFT(p, y); + } + + ERTS_RBT_SET_RIGHT(y, x); + ERTS_RBT_SET_PARENT(x, y); + +#ifdef ERTS_RBT_UPDATE_ATTACHED_DATA_ROTATE + ERTS_RBT_UPDATE_ATTACHED_DATA_ROTATE(0, x, y); +#endif + +} + +#endif /* ERTS_RBT_NEED_ROTATE__ */ + +#ifdef ERTS_RBT_NEED_REPLACE__ + +/* + * Replace node x with node y + */ +static ERTS_INLINE void +ERTS_RBT_FUNC__(replace__)(ERTS_RBT_T **root, ERTS_RBT_T *x, ERTS_RBT_T *y) +{ + ERTS_RBT_T *p, *r, *l; + ERTS_RBT_FLAGS_T f; + + p = ERTS_RBT_GET_PARENT(x); + if (!p) { + ERTS_RBT_ASSERT(*root == x); + *root = y; +#ifdef ERTS_RBT_UPDATE_ATTACHED_DATA_CHGROOT + ERTS_RBT_UPDATE_ATTACHED_DATA_CHGROOT(x, y); +#endif + } + else if (x == ERTS_RBT_GET_LEFT(p)) + ERTS_RBT_SET_LEFT(p, y); + else { + ERTS_RBT_ASSERT(x == ERTS_RBT_GET_RIGHT(p)); + ERTS_RBT_SET_RIGHT(p, y); + } + l = ERTS_RBT_GET_LEFT(x); + if (l) { + ERTS_RBT_ASSERT(ERTS_RBT_GET_PARENT(l) == x); + ERTS_RBT_SET_PARENT(l, y); + } + r = ERTS_RBT_GET_RIGHT(x); + if (r) { + ERTS_RBT_ASSERT(ERTS_RBT_GET_PARENT(r) == x); + ERTS_RBT_SET_PARENT(r, y); + } + + f = ERTS_RBT_GET_FLAGS(x); + ERTS_RBT_SET_FLAGS(y, f); + ERTS_RBT_SET_PARENT(y, p); + ERTS_RBT_SET_RIGHT(y, r); + ERTS_RBT_SET_LEFT(y, l); +} + +#endif /* ERTS_RBT_NEED_REPLACE__ */ + +#ifdef ERTS_RBT_WANT_REPLACE + +static ERTS_RBT_API_INLINE__ void +ERTS_RBT_FUNC__(replace)(ERTS_RBT_T **root, ERTS_RBT_T *x, ERTS_RBT_T *y) +{ + ERTS_RBT_ASSERT(ERTS_RBT_IS_EQ(ERTS_RBT_GET_KEY(x), + ERTS_RBT_GET_KEY(y))); + + ERTS_RBT_FUNC__(replace__)(root, x, y); +} + +#endif /* ERTS_RBT_WANT_REPLACE */ + +#ifdef ERTS_RBT_WANT_DELETE + +/* + * Delete a node. + */ +static ERTS_RBT_API_INLINE__ void +ERTS_RBT_FUNC__(delete)(ERTS_RBT_T **root, ERTS_RBT_T *n) +{ + int spliced_is_black; + ERTS_RBT_T *p, *x, *y, *z = n; + ERTS_RBT_T null_x; /* null_x is used to get the fixup started when we + splice out a node without children. */ + + ERTS_RBT_HDBG_CHECK_TREE__(*root); + + ERTS_RBT_INIT_EMPTY_TNODE(&null_x); + + /* Remove node from tree... */ + + /* Find node to splice out */ + if (!ERTS_RBT_GET_LEFT(z) || !ERTS_RBT_GET_RIGHT(z)) + y = z; + else { + /* Set y to z:s successor */ + y = ERTS_RBT_GET_RIGHT(z); + while (1) { + ERTS_RBT_T *t = ERTS_RBT_GET_LEFT(y); + if (!t) + break; + y = t; + } + } + /* splice out y */ + x = ERTS_RBT_GET_LEFT(y); + if (!x) + x = ERTS_RBT_GET_RIGHT(y); + spliced_is_black = ERTS_RBT_IS_BLACK(y); + p = ERTS_RBT_GET_PARENT(y); + if (x) + ERTS_RBT_SET_PARENT(x, p); + else if (spliced_is_black) { + x = &null_x; + ERTS_RBT_SET_BLACK(x); + ERTS_RBT_SET_PARENT(x, p); + ERTS_RBT_SET_LEFT(y, x); + } + + if (!p) { + ERTS_RBT_ASSERT(*root == y); + *root = x; +#ifdef ERTS_RBT_UPDATE_ATTACHED_DATA_CHGROOT + ERTS_RBT_UPDATE_ATTACHED_DATA_CHGROOT(y, x); +#endif + } + else { + if (y == ERTS_RBT_GET_LEFT(p)) + ERTS_RBT_SET_LEFT(p, x); + else { + ERTS_RBT_ASSERT(y == ERTS_RBT_GET_RIGHT(p)); + ERTS_RBT_SET_RIGHT(p, x); + } +#ifdef ERTS_RBT_UPDATE_ATTACHED_DATA_DMOD + if (p != z) + ERTS_RBT_UPDATE_ATTACHED_DATA_DMOD(p, y == z ? NULL : z); +#endif + } + if (y != z) { + /* We spliced out the successor of z; replace z by the successor */ + ERTS_RBT_FUNC__(replace__)(root, z, y); +#ifdef ERTS_RBT_UPDATE_ATTACHED_DATA_DMOD + ERTS_RBT_UPDATE_ATTACHED_DATA_DMOD(y, NULL); +#endif + } + + if (spliced_is_black) { + /* We removed a black node which makes the resulting tree + violate the Red-Black Tree properties. Fixup tree... */ + + p = ERTS_RBT_GET_PARENT(x); + while (ERTS_RBT_IS_BLACK(x) && p) { + ERTS_RBT_T *r, *l; + + /* + * x has an "extra black" which we move up the tree + * until we reach the root or until we can get rid of it. + * + * y is the sibbling of x, and p is their parent + */ + + if (x == ERTS_RBT_GET_LEFT(p)) { + y = ERTS_RBT_GET_RIGHT(p); + + ERTS_RBT_ASSERT(y); + + if (ERTS_RBT_IS_RED(y)) { + ERTS_RBT_ASSERT(ERTS_RBT_GET_RIGHT(y)); + ERTS_RBT_ASSERT(ERTS_RBT_GET_LEFT(y)); + + ERTS_RBT_SET_BLACK(y); + + ERTS_RBT_ASSERT(ERTS_RBT_IS_BLACK(p)); + + ERTS_RBT_SET_RED(p); + ERTS_RBT_FUNC__(left_rotate__)(root, p); + p = ERTS_RBT_GET_PARENT(x); + y = ERTS_RBT_GET_RIGHT(p); + } + + ERTS_RBT_ASSERT(y); + ERTS_RBT_ASSERT(ERTS_RBT_IS_BLACK(y)); + + l = ERTS_RBT_GET_LEFT(y); + r = ERTS_RBT_GET_RIGHT(y); + if ((!l || ERTS_RBT_IS_BLACK(l)) + && (!r || ERTS_RBT_IS_BLACK(r))) { + ERTS_RBT_SET_RED(y); + x = p; + p = ERTS_RBT_GET_PARENT(x); + } + else { + if (!r || ERTS_RBT_IS_BLACK(r)) { + ERTS_RBT_SET_BLACK(l); + ERTS_RBT_SET_RED(y); + ERTS_RBT_FUNC__(right_rotate__)(root, y); + p = ERTS_RBT_GET_PARENT(x); + y = ERTS_RBT_GET_RIGHT(p); + } + + ERTS_RBT_ASSERT(y); + + if (p && ERTS_RBT_IS_RED(p)) { + + ERTS_RBT_SET_BLACK(p); + ERTS_RBT_SET_RED(y); + } + + ERTS_RBT_ASSERT(ERTS_RBT_GET_RIGHT(y)); + + ERTS_RBT_SET_BLACK(ERTS_RBT_GET_RIGHT(y)); + ERTS_RBT_FUNC__(left_rotate__)(root, p); + x = *root; + break; + } + } + else { + ERTS_RBT_ASSERT(x == ERTS_RBT_GET_RIGHT(p)); + + y = ERTS_RBT_GET_LEFT(p); + + ERTS_RBT_ASSERT(y); + + if (ERTS_RBT_IS_RED(y)) { + ERTS_RBT_ASSERT(ERTS_RBT_GET_RIGHT(y)); + ERTS_RBT_ASSERT(ERTS_RBT_GET_LEFT(y)); + + ERTS_RBT_SET_BLACK(y); + ERTS_RBT_ASSERT(ERTS_RBT_IS_BLACK(p)); + ERTS_RBT_SET_RED(p); + ERTS_RBT_FUNC__(right_rotate__)(root, p); + + p = ERTS_RBT_GET_PARENT(x); + y = ERTS_RBT_GET_LEFT(p); + } + + ERTS_RBT_ASSERT(y); + ERTS_RBT_ASSERT(ERTS_RBT_IS_BLACK(y)); + + l = ERTS_RBT_GET_LEFT(y); + r = ERTS_RBT_GET_RIGHT(y); + + if ((!r || ERTS_RBT_IS_BLACK(r)) + && (!l || ERTS_RBT_IS_BLACK(l))) { + ERTS_RBT_SET_RED(y); + x = p; + p = ERTS_RBT_GET_PARENT(x); + } + else { + if (!l || ERTS_RBT_IS_BLACK(l)) { + ERTS_RBT_SET_BLACK(r); + ERTS_RBT_SET_RED(y); + ERTS_RBT_FUNC__(left_rotate__)(root, y); + + p = ERTS_RBT_GET_PARENT(x); + y = ERTS_RBT_GET_LEFT(p); + } + + ERTS_RBT_ASSERT(y); + + if (p && ERTS_RBT_IS_RED(p)) { + ERTS_RBT_SET_BLACK(p); + ERTS_RBT_SET_RED(y); + } + + ERTS_RBT_ASSERT(ERTS_RBT_GET_LEFT(y)); + + ERTS_RBT_SET_BLACK(ERTS_RBT_GET_LEFT(y)); + ERTS_RBT_FUNC__(right_rotate__)(root, p); + x = *root; + break; + } + } + } + + ERTS_RBT_SET_BLACK(x); + + x = &null_x; + p = ERTS_RBT_GET_PARENT(x); + + if (p) { + if (ERTS_RBT_GET_LEFT(p) == x) + ERTS_RBT_SET_LEFT(p, NULL); + else { + ERTS_RBT_ASSERT(ERTS_RBT_GET_RIGHT(p) == x); + ERTS_RBT_SET_RIGHT(p, NULL); + } + + ERTS_RBT_ASSERT(!ERTS_RBT_GET_LEFT(x)); + ERTS_RBT_ASSERT(!ERTS_RBT_GET_RIGHT(x)); + } + else if (*root == x) { + *root = NULL; + +#ifdef ERTS_RBT_UPDATE_ATTACHED_DATA_CHGROOT + ERTS_RBT_UPDATE_ATTACHED_DATA_CHGROOT(x, NULL); +#endif + + ERTS_RBT_ASSERT(!ERTS_RBT_GET_LEFT(x)); + ERTS_RBT_ASSERT(!ERTS_RBT_GET_RIGHT(x)); + } + } + + ERTS_RBT_HDBG_CHECK_TREE__(*root); + +} + +#endif /* ERTS_RBT_WANT_DELETE */ + +#ifdef ERTS_RBT_NEED_INSERT__ + +static void +ERTS_RBT_FUNC__(insert_fixup__)(ERTS_RBT_T **root, ERTS_RBT_T *n) +{ + ERTS_RBT_T *x, *y; + + x = n; + + /* + * Rearrange the tree so that it satisfies the Red-Black Tree properties + */ + + ERTS_RBT_ASSERT(x != *root && ERTS_RBT_IS_RED(ERTS_RBT_GET_PARENT(x))); + do { + ERTS_RBT_T *p, *pp; + + /* + * x and its parent are both red. Move the red pair up the tree + * until we get to the root or until we can separate them. + */ + + p = ERTS_RBT_GET_PARENT(x); + pp = ERTS_RBT_GET_PARENT(p); + + ERTS_RBT_ASSERT(p && pp); + ERTS_RBT_ASSERT(ERTS_RBT_IS_RED(x)); + ERTS_RBT_ASSERT(ERTS_RBT_IS_BLACK(pp)); + + if (p == ERTS_RBT_GET_LEFT(pp)) { + y = ERTS_RBT_GET_RIGHT(pp); + if (y && ERTS_RBT_IS_RED(y)) { + ERTS_RBT_SET_BLACK(y); + ERTS_RBT_SET_BLACK(p); + ERTS_RBT_SET_RED(pp); + x = pp; + } + else { + + if (x == ERTS_RBT_GET_RIGHT(p)) { + x = p; + ERTS_RBT_FUNC__(left_rotate__)(root, x); + p = ERTS_RBT_GET_PARENT(x); + pp = ERTS_RBT_GET_PARENT(p); + + ERTS_RBT_ASSERT(p && pp); + } + + ERTS_RBT_ASSERT(x == ERTS_RBT_GET_LEFT(ERTS_RBT_GET_LEFT(pp))); + ERTS_RBT_ASSERT(ERTS_RBT_IS_RED(x)); + ERTS_RBT_ASSERT(ERTS_RBT_IS_RED(p)); + ERTS_RBT_ASSERT(ERTS_RBT_IS_BLACK(pp)); + ERTS_RBT_ASSERT(!y || ERTS_RBT_IS_BLACK(y)); + + + ERTS_RBT_SET_BLACK(p); + ERTS_RBT_SET_RED(pp); + ERTS_RBT_FUNC__(right_rotate__)(root, pp); + + + ERTS_RBT_ASSERT(ERTS_RBT_GET_LEFT(ERTS_RBT_GET_PARENT(x)) == x); + ERTS_RBT_ASSERT(ERTS_RBT_IS_RED(x)); + ERTS_RBT_ASSERT(ERTS_RBT_IS_RED( + ERTS_RBT_GET_RIGHT( + ERTS_RBT_GET_PARENT(x)))); + ERTS_RBT_ASSERT(!ERTS_RBT_GET_PARENT(x) + || ERTS_RBT_IS_BLACK(ERTS_RBT_GET_PARENT(x))); + break; + } + } + else { + ERTS_RBT_ASSERT(p == ERTS_RBT_GET_RIGHT(pp)); + + y = ERTS_RBT_GET_LEFT(pp); + if (y && ERTS_RBT_IS_RED(y)) { + ERTS_RBT_SET_BLACK(y); + ERTS_RBT_SET_BLACK(p); + ERTS_RBT_SET_RED(pp); + x = pp; + } + else { + + if (x == ERTS_RBT_GET_LEFT(p)) { + x = p; + ERTS_RBT_FUNC__(right_rotate__)(root, x); + p = ERTS_RBT_GET_PARENT(x); + pp = ERTS_RBT_GET_PARENT(p); + + ERTS_RBT_ASSERT(p && pp); + } + + ERTS_RBT_ASSERT(x == ERTS_RBT_GET_RIGHT(ERTS_RBT_GET_RIGHT(pp))); + ERTS_RBT_ASSERT(ERTS_RBT_IS_RED(x)); + ERTS_RBT_ASSERT(ERTS_RBT_IS_RED(p)); + ERTS_RBT_ASSERT(ERTS_RBT_IS_BLACK(pp)); + ERTS_RBT_ASSERT(!y || ERTS_RBT_IS_BLACK(y)); + + + ERTS_RBT_SET_BLACK(p); + ERTS_RBT_SET_RED(pp); + ERTS_RBT_FUNC__(left_rotate__)(root, pp); + + + ERTS_RBT_ASSERT(ERTS_RBT_GET_RIGHT(ERTS_RBT_GET_PARENT(x)) == x); + ERTS_RBT_ASSERT(ERTS_RBT_IS_RED(x)); + ERTS_RBT_ASSERT(ERTS_RBT_IS_RED( + ERTS_RBT_GET_LEFT( + ERTS_RBT_GET_PARENT(x)))); + ERTS_RBT_ASSERT(!ERTS_RBT_GET_PARENT(x) + || ERTS_RBT_IS_BLACK(ERTS_RBT_GET_PARENT(x))); + break; + } + } + } while (x != *root && ERTS_RBT_IS_RED(ERTS_RBT_GET_PARENT(x))); + + ERTS_RBT_SET_BLACK(*root); + +} + +static ERTS_INLINE ERTS_RBT_T * +ERTS_RBT_FUNC__(insert_aux__)(ERTS_RBT_T **root, ERTS_RBT_T *n, int lookup) +{ + ERTS_RBT_KEY_T kn = ERTS_RBT_GET_KEY(n); + + ERTS_RBT_HDBG_CHECK_TREE__(*root); + + ERTS_RBT_INIT_EMPTY_TNODE(n); + + if (!*root) { + ERTS_RBT_SET_BLACK(n); + *root = n; +#ifdef ERTS_RBT_UPDATE_ATTACHED_DATA_CHGROOT + ERTS_RBT_UPDATE_ATTACHED_DATA_CHGROOT(NULL, n); +#endif + } + else { + ERTS_RBT_T *p, *x = *root; + + while (1) { + ERTS_RBT_KEY_T kx; + ERTS_RBT_T *c; + + kx = ERTS_RBT_GET_KEY(x); + + if (lookup && ERTS_RBT_IS_EQ(kn, kx)) { + + ERTS_RBT_HDBG_CHECK_TREE__(*root); + + return x; + } + + if (ERTS_RBT_IS_LT(kn, kx)) { + c = ERTS_RBT_GET_LEFT(x); + if (!c) { + ERTS_RBT_SET_PARENT(n, x); + ERTS_RBT_SET_LEFT(x, n); + p = x; + break; + } + } + else { + c = ERTS_RBT_GET_RIGHT(x); + if (!c) { + ERTS_RBT_SET_PARENT(n, x); + ERTS_RBT_SET_RIGHT(x, n); + p = x; + break; + } + } + + x = c; + } + + ERTS_RBT_ASSERT(p); + + ERTS_RBT_SET_RED(n); + if (ERTS_RBT_IS_RED(p)) + ERTS_RBT_FUNC__(insert_fixup__)(root, n); + } + + ERTS_RBT_HDBG_CHECK_TREE__(*root); + + return NULL; +} + +#endif /* ERTS_RBT_NEED_INSERT__ */ + +#ifdef ERTS_RBT_WANT_LOOKUP_INSERT + +static ERTS_RBT_API_INLINE__ ERTS_RBT_T * +ERTS_RBT_FUNC__(lookup_insert)(ERTS_RBT_T **root, ERTS_RBT_T *n) +{ + return ERTS_RBT_FUNC__(insert_aux__)(root, n, !0); +} + +#endif /* ERTS_RBT_WANT_LOOKUP_INSERT */ + +#ifdef ERTS_RBT_WANT_INSERT + +static ERTS_RBT_API_INLINE__ void +ERTS_RBT_FUNC__(insert)(ERTS_RBT_T **root, ERTS_RBT_T *n) +{ + (void) ERTS_RBT_FUNC__(insert_aux__)(root, n, 0); +} + +#endif /* ERTS_RBT_WANT_INSERT */ + +#ifdef ERTS_RBT_WANT_LOOKUP + +static ERTS_RBT_API_INLINE__ ERTS_RBT_T * +ERTS_RBT_FUNC__(lookup)(ERTS_RBT_T *root, ERTS_RBT_KEY_T key) +{ + ERTS_RBT_T *x = root; + + if (!x) + return NULL; + + while (1) { + ERTS_RBT_KEY_T kx = ERTS_RBT_GET_KEY(x); + ERTS_RBT_T *c; + + if (ERTS_RBT_IS_EQ(key, kx)) + return x; + + if (ERTS_RBT_IS_LT(key, kx)) { + c = ERTS_RBT_GET_LEFT(x); + if (!c) + return NULL; + } + else { + c = ERTS_RBT_GET_RIGHT(x); + if (!c) + return NULL; + } + + x = c; + } +} + +#endif /* ERTS_RBT_WANT_LOOKUP */ + +#ifdef ERTS_RBT_WANT_SMALLEST + +static ERTS_RBT_API_INLINE__ ERTS_RBT_T * +ERTS_RBT_FUNC__(smallest)(ERTS_RBT_T *root) +{ + ERTS_RBT_T *x = root; + + if (!x) + return NULL; + + while (1) { + ERTS_RBT_T *c = ERTS_RBT_GET_LEFT(x); + if (!c) + break; + x = c; + } + + return x; +} + +#endif /* ERTS_RBT_WANT_SMALLEST */ + +#ifdef ERTS_RBT_WANT_LARGEST + +static ERTS_RBT_API_INLINE__ ERTS_RBT_T * +ERTS_RBT_FUNC__(largest)(ERTS_RBT_T *root) +{ + ERTS_RBT_T *x = root; + + if (!x) + return NULL; + + while (1) { + ERTS_RBT_T *c = ERTS_RBT_GET_RIGHT(x); + if (!c) + break; + x = c; + } + + return x; +} + +#endif /* ERTS_RBT_WANT_LARGEST */ + +#ifdef ERTS_RBT_NEED_FOREACH_UNORDERED__ + +static ERTS_INLINE int +ERTS_RBT_FUNC__(foreach_unordered__)(ERTS_RBT_T **root, + int destroying, + void (*op)(ERTS_RBT_T *, void *), + void *arg, + int yielding, + ERTS_RBT_YIELD_STATE_T__ *ystate, + Sint ylimit) +{ + ERTS_RBT_T *c, *p, *x; + + ERTS_RBT_ASSERT(!yielding || ystate); + + if (yielding && ystate->x) { + x = ystate->x; + ERTS_RBT_ASSERT(ystate->up); + goto restart_up; + } + else { + x = *root; + if (!x) + return 0; + if (destroying) + *root = NULL; + } + + while (1) { + + while (1) { + + while (1) { + c = ERTS_RBT_GET_LEFT(x); + if (!c) + break; + x = c; + } + + c = ERTS_RBT_GET_RIGHT(x); + if (!c) + break; + x = c; + } + + while (1) { +#ifdef ERTS_RBT_DEBUG + int cdir; +#endif + if (yielding && ylimit-- <= 0) { + ystate->x = x; + ystate->up = 1; + return 1; + } + + restart_up: + + p = ERTS_RBT_GET_PARENT(x); + +#ifdef ERTS_RBT_DEBUG + ERTS_RBT_ASSERT(!destroying || !ERTS_RBT_GET_LEFT(x)); + ERTS_RBT_ASSERT(!destroying || !ERTS_RBT_GET_RIGHT(x)); + + if (p) { + if (x == ERTS_RBT_GET_LEFT(p)) { + cdir = -1; + if (destroying) + ERTS_RBT_SET_LEFT(p, NULL); + } + else { + ERTS_RBT_ASSERT(x == ERTS_RBT_GET_RIGHT(p)); + cdir = 1; + if (destroying) + ERTS_RBT_SET_RIGHT(p, NULL); + } + } +#endif + + (*op)(x, arg); + + if (!p) { + if (yielding) { + ystate->x = NULL; + ystate->up = 0; + } + return 0; /* Done */ + } + + c = ERTS_RBT_GET_RIGHT(p); + if (c && c != x) { + ERTS_RBT_ASSERT(cdir < 0); + + /* Go down tree of x's sibling... */ + x = c; + break; + } + + x = p; + } + } +} + +#endif /* ERTS_RBT_NEED_FOREACH_UNORDERED__ */ + +#ifdef ERTS_RBT_NEED_FOREACH_ORDERED__ + +static ERTS_INLINE int +ERTS_RBT_FUNC__(foreach_ordered__)(ERTS_RBT_T **root, + int from_small, + int destroying, + void (*op)(ERTS_RBT_T *, void *), + void (*destroy)(ERTS_RBT_T *, void *), + void *arg, + int yielding, + ERTS_RBT_YIELD_STATE_T__ *ystate, + Sint ylimit) +{ + ERTS_RBT_T *c, *p, *x; + + ERTS_RBT_ASSERT(!yielding || ystate); + ERTS_RBT_ASSERT(!destroying || destroy); + + if (yielding && ystate->x) { + x = ystate->x; + if (ystate->up) + goto restart_up; + else + goto restart_down; + } + else { + x = *root; + if (!x) + return 0; + if (destroying) + *root = NULL; + } + + while (1) { + + while (1) { + + while (1) { + c = from_small ? ERTS_RBT_GET_LEFT(x) : ERTS_RBT_GET_RIGHT(x); + if (!c) + break; + x = c; + } + + (*op)(x, arg); + + if (yielding && --ylimit <= 0) { + ystate->x = x; + ystate->up = 0; + return 1; + } + + restart_down: + + c = from_small ? ERTS_RBT_GET_RIGHT(x) : ERTS_RBT_GET_LEFT(x); + if (!c) + break; + x = c; + } + + while (1) { + p = ERTS_RBT_GET_PARENT(x); + + if (p) { + + c = from_small ? ERTS_RBT_GET_RIGHT(p) : ERTS_RBT_GET_LEFT(p); + if (!c || c != x) { + ERTS_RBT_ASSERT((from_small + ? ERTS_RBT_GET_LEFT(p) + : ERTS_RBT_GET_RIGHT(p)) == x); + + (*op)(p, arg); + + if (yielding && --ylimit <= 0) { + ystate->x = x; + ystate->up = 1; + return 1; + restart_up: + p = ERTS_RBT_GET_PARENT(x); + } + } + + if (c && c != x) { + ERTS_RBT_ASSERT((from_small + ? ERTS_RBT_GET_LEFT(p) + : ERTS_RBT_GET_RIGHT(p)) == x); + + /* Go down tree of x's sibling... */ + x = c; + break; + } + } + + if (destroying) { + +#ifdef ERTS_RBT_DEBUG + ERTS_RBT_ASSERT(!ERTS_RBT_GET_LEFT(x) + && !ERTS_RBT_GET_RIGHT(x)); + + if (p) { + if (x == ERTS_RBT_GET_LEFT(p)) + ERTS_RBT_SET_LEFT(p, NULL); + else { + ERTS_RBT_ASSERT(x == ERTS_RBT_GET_RIGHT(p)); + ERTS_RBT_SET_RIGHT(p, NULL); + } + } +#endif + + (*destroy)(x, arg); + } + + if (!p) { + if (yielding) { + ystate->x = NULL; + ystate->up = 0; + } + return 1; /* Done */ + } + x = p; + } + } +} + +#endif /* ERTS_RBT_NEED_FOREACH_ORDERED__ */ + +#ifdef ERTS_RBT_WANT_FOREACH + +static ERTS_RBT_API_INLINE__ void +ERTS_RBT_FUNC__(foreach)(ERTS_RBT_T *root, + void (*op)(ERTS_RBT_T *, void *), + void *arg) +{ + (void) ERTS_RBT_FUNC__(foreach_unordered__)(&root, 0, op, arg, + 0, NULL, 0); +} + +#endif /* ERTS_RBT_WANT_FOREACH */ + +#ifdef ERTS_RBT_WANT_FOREACH_SMALL + +static ERTS_RBT_API_INLINE__ void +ERTS_RBT_FUNC__(foreach_small)(ERTS_RBT_T *root, + void (*op)(ERTS_RBT_T *, void *), + void *arg) +{ + (void) ERTS_RBT_FUNC__(foreach_ordered__)(&root, 1, 0, + op, NULL, arg, + 0, NULL, 0); +} + +#endif /* ERTS_RBT_WANT_FOREACH_SMALL */ + +#ifdef ERTS_RBT_WANT_FOREACH_LARGE + +static ERTS_RBT_API_INLINE__ void +ERTS_RBT_FUNC__(foreach_large)(ERTS_RBT_T *root, + void (*op)(ERTS_RBT_T *, void *), + void *arg) +{ + (void) ERTS_RBT_FUNC__(foreach_ordered__)(&root, 0, 0, + op, NULL, arg, + 0, NULL, 0); +} + +#endif /* ERTS_RBT_WANT_FOREACH_LARGE */ + +#ifdef ERTS_RBT_WANT_FOREACH_YIELDING + +static ERTS_RBT_API_INLINE__ void +ERTS_RBT_FUNC__(foreach_yielding)(ERTS_RBT_T *root, + void (*op)(ERTS_RBT_T *, void *), + void *arg, + ERTS_RBT_YIELD_STATE_T__ *ystate, + Sint ylimit) +{ + (void) ERTS_RBT_FUNC__(foreach_unordered__)(*root, 0, op, arg, + 1, ystate, ylimit); +} + +#endif /* ERTS_RBT_WANT_FOREACH_YIELDING */ + +#ifdef ERTS_RBT_WANT_FOREACH_SMALL_YIELDING + +static ERTS_RBT_API_INLINE__ int +ERTS_RBT_FUNC__(foreach_small_yielding)(ERTS_RBT_T *root, + void (*op)(ERTS_RBT_T *, void *), + void *arg, + ERTS_RBT_YIELD_STATE_T__ *ystate, + Sint ylimit) +{ + return ERTS_RBT_FUNC__(foreach_ordered__)(&root, 1, 0, + op, NULL, arg, + 1, ystate, ylimit); +} + +#endif /* ERTS_RBT_WANT_FOREACH_SMALL_YIELDING */ + +#ifdef ERTS_RBT_WANT_FOREACH_LARGE_YIELDING + +static ERTS_RBT_API_INLINE__ int +ERTS_RBT_FUNC__(foreach_large_yielding)(ERTS_RBT_T *root, + void (*op)(ERTS_RBT_T *, void *), + void *arg, + ERTS_RBT_YIELD_STATE_T__ *ystate, + Sint ylimit) +{ + return ERTS_RBT_FUNC__(foreach_ordered__)(&root, 0, 0, + op, NULL, arg, + 1, ystate, ylimit); +} + +#endif /* ERTS_RBT_WANT_FOREACH_LARGE_YIELDING */ + +#ifdef ERTS_RBT_WANT_FOREACH_DESTROY + +static ERTS_RBT_API_INLINE__ void +ERTS_RBT_FUNC__(foreach_destroy)(ERTS_RBT_T **root, + void (*op)(ERTS_RBT_T *, void *), + void *arg) +{ + (void) ERTS_RBT_FUNC__(foreach_unordered__)(root, 1, op, arg, + 0, NULL, 0); +} + +#endif /* ERTS_RBT_WANT_FOREACH_DESTROY */ + +#ifdef ERTS_RBT_WANT_FOREACH_SMALL_DESTROY + +static ERTS_RBT_API_INLINE__ void +ERTS_RBT_FUNC__(foreach_small_destroy)(ERTS_RBT_T **root, + void (*op)(ERTS_RBT_T *, void *), + void (*destr)(ERTS_RBT_T *, void *), + void *arg) +{ + (void) ERTS_RBT_FUNC__(foreach_ordered__)(root, 1, 1, + op, destr, arg, + 0, NULL, 0); +} + +#endif /* ERTS_RBT_WANT_FOREACH_SMALL_DESTROY */ + +#ifdef ERTS_RBT_WANT_FOREACH_LARGE_DESTROY + +static ERTS_RBT_API_INLINE__ void +ERTS_RBT_FUNC__(foreach_large_destroy)(ERTS_RBT_T **root, + void (*op)(ERTS_RBT_T *, void *), + void (*destr)(ERTS_RBT_T *, void *), + void *arg) +{ + (void) ERTS_RBT_FUNC__(foreach_ordered__)(root, 0, 1, + op, destr, arg, + 0, NULL, 0); +} + +#endif /* ERTS_RBT_WANT_FOREACH_LARGE_DESTROY */ + +#ifdef ERTS_RBT_WANT_FOREACH_DESTROY_YIELDING + +static ERTS_RBT_API_INLINE__ int +ERTS_RBT_FUNC__(foreach_destroy_yielding)(ERTS_RBT_T **root, + void (*op)(ERTS_RBT_T *, void *), + void *arg, + ERTS_RBT_YIELD_STATE_T__ *ystate, + Sint ylimit) +{ + return ERTS_RBT_FUNC__(foreach_unordered__)(root, 1, op, arg, + 1, ystate, ylimit); +} + +#endif /* ERTS_RBT_WANT_FOREACH_DESTROY_YIELDING */ + +#ifdef ERTS_RBT_WANT_FOREACH_SMALL_DESTROY_YIELDING + +static ERTS_RBT_API_INLINE__ int +ERTS_RBT_FUNC__(foreach_small_destroy_yielding)(ERTS_RBT_T **root, + void (*op)(ERTS_RBT_T *, void *), + void (*destr)(ERTS_RBT_T *, void *), + void *arg, + ERTS_RBT_YIELD_STATE_T__ *ystate, + Sint ylimit) +{ + return ERTS_RBT_FUNC__(foreach_ordered__)(root, 1, 1, + op, destr, arg, + 1, ystate, ylimit); +} + +#endif /* ERTS_RBT_WANT_FOREACH_SMALL_DESTROY_YIELDING */ + +#ifdef ERTS_RBT_WANT_FOREACH_LARGE_DESTROY_YIELDING + +static ERTS_RBT_API_INLINE__ int +ERTS_RBT_FUNC__(foreach_large_destroy_yielding)(ERTS_RBT_T **root, + void (*op)(ERTS_RBT_T *, void *), + void (*destr)(ERTS_RBT_T *, void *), + void *arg, + ERTS_RBT_YIELD_STATE_T__ *ystate, + Sint ylimit) +{ + return ERTS_RBT_FUNC__(foreach_ordered__)(root, 0, 1, + op, destr, arg, + 1, ystate, ylimit); +} + +#endif /* ERTS_RBT_WANT_FOREACH_LARGE_DESTROY_YIELDING */ + +#ifdef ERTS_RBT_WANT_DEBUG_PRINT + +static void +ERTS_RBT_FUNC__(debug_print)(FILE *filep, ERTS_RBT_T *x, int indent, + void (*print_node)(ERTS_RBT_T *)) +{ + if (x) { + ERTS_RBT_FUNC__(debug_print)(filep, ERTS_RBT_GET_RIGHT(x), + indent+2, print_node); + erts_fprintf(filep, + "%*s[%s:%p:", + indent, "", + ERTS_RBT_IS_BLACK(x) ? "Black" : "Red", + x); + (*print_node)(x); + erts_fprintf(filep, "]\n"); + ERTS_RBT_FUNC__(debug_print)(filep, ERTS_RBT_GET_LEFT(x), + indent+2, print_node); + } +} + +#endif /* ERTS_RBT_WANT_DEBUG_PRINT */ + +#ifdef ERTS_RBT_NEED_HDBG_CHECK_TREE__ + +static void +ERTS_RBT_FUNC__(hdbg_check_tree)(ERTS_RBT_T *root) +{ + int black_depth = -1, no_black = 0; + ERTS_RBT_T *c, *p, *x = root; + ERTS_RBT_KEY_T kx; + ERTS_RBT_KEY_T kc; + + if (!x) + return; + + ERTS_RBT_ASSERT(!ERTS_RBT_GET_PARENT(x)); + + while (1) { + + while (1) { + + while (1) { + + if (ERTS_RBT_IS_BLACK(x)) + no_black++; + else { + c = ERTS_RBT_GET_RIGHT(x); + ERTS_RBT_ASSERT(!c || ERTS_RBT_IS_BLACK(c)); + c = ERTS_RBT_GET_LEFT(x); + ERTS_RBT_ASSERT(!c || ERTS_RBT_IS_BLACK(c)); + } + + c = ERTS_RBT_GET_LEFT(x); + if (!c) + break; + + ERTS_RBT_ASSERT(x == ERTS_RBT_GET_PARENT(c)); + + kx = ERTS_RBT_GET_KEY(x); + kc = ERTS_RBT_GET_KEY(c); + + ERTS_RBT_ASSERT(ERTS_RBT_IS_LT(kc, kx) + || ERTS_RBT_IS_EQ(kc, kx)); + + x = c; + } + + c = ERTS_RBT_GET_RIGHT(x); + if (!c) { + if (black_depth < 0) + black_depth = no_black; + ERTS_RBT_ASSERT(black_depth == no_black); + break; + } + + ERTS_RBT_ASSERT(x == ERTS_RBT_GET_PARENT(c)); + + kx = ERTS_RBT_GET_KEY(x); + kc = ERTS_RBT_GET_KEY(c); + + ERTS_RBT_ASSERT(ERTS_RBT_IS_LT(kx, kc) + || ERTS_RBT_IS_EQ(kx, kc)); + x = c; + } + + while (1) { + p = ERTS_RBT_GET_PARENT(x); + + if (ERTS_RBT_IS_BLACK(x)) + no_black--; + + if (p) { + + ERTS_RBT_ASSERT(x == ERTS_RBT_GET_LEFT(p) + || x == ERTS_RBT_GET_RIGHT(p)); + + c = ERTS_RBT_GET_RIGHT(p); + if (c && c != x) { + ERTS_RBT_ASSERT(ERTS_RBT_GET_LEFT(p) == x); + + kx = ERTS_RBT_GET_KEY(x); + kc = ERTS_RBT_GET_KEY(c); + + ERTS_RBT_ASSERT(ERTS_RBT_IS_LT(kx, kc) + || ERTS_RBT_IS_EQ(kx, kc)); + /* Go down tree of x's sibling... */ + x = c; + break; + } + } + + if (!p) { + ERTS_RBT_ASSERT(root == x); + ERTS_RBT_ASSERT(no_black == 0); + return; /* Done */ + } + + x = p; + } + } +} + +#undef ERTS_RBT_PRINT_TREE__ + +#endif /* ERTS_RBT_NEED_HDBG_CHECK_TREE__ */ + +#undef ERTS_RBT_ASSERT +#undef ERTS_RBT_DEBUG +#undef ERTS_RBT_API_INLINE__ +#undef ERTS_RBT_YIELD_STATE_T__ +#undef ERTS_RBT_NEED_REPLACE__ +#undef ERTS_RBT_NEED_INSERT__ +#undef ERTS_RBT_NEED_ROTATE__ +#undef ERTS_RBT_NEED_FOREACH_UNORDERED__ +#undef ERTS_RBT_NEED_FOREACH_ORDERED__ +#undef ERTS_RBT_NEED_HDBG_CHECK_TREE__ +#undef ERTS_RBT_HDBG_CHECK_TREE__ + +#ifdef ERTS_RBT_UNDEF +# undef ERTS_RBT_PREFIX +# undef ERTS_RBT_T +# undef ERTS_RBT_KEY_T +# undef ERTS_RBT_FLAGS_T +# undef ERTS_RBT_INIT_EMPTY_TNODE +# undef ERTS_RBT_IS_RED +# undef ERTS_RBT_SET_RED +# undef ERTS_RBT_IS_BLACK +# undef ERTS_RBT_SET_BLACK +# undef ERTS_RBT_GET_FLAGS +# undef ERTS_RBT_SET_FLAGS +# undef ERTS_RBT_GET_PARENT +# undef ERTS_RBT_SET_PARENT +# undef ERTS_RBT_GET_RIGHT +# undef ERTS_RBT_SET_RIGHT +# undef ERTS_RBT_GET_LEFT +# undef ERTS_RBT_SET_LEFT +# undef ERTS_RBT_GET_KEY +# undef ERTS_RBT_IS_LT +# undef ERTS_RBT_IS_EQ +# undef ERTS_RBT_UNDEF +# undef ERTS_RBT_NO_API_INLINE +# undef ERTS_RBT_UPDATE_ATTACHED_DATA_ROTATE +# undef ERTS_RBT_UPDATE_ATTACHED_DATA_DMOD +# undef ERTS_RBT_UPDATE_ATTACHED_DATA_CHGROOT +# undef ERTS_RBT_WANT_DELETE +# undef ERTS_RBT_WANT_INSERT +# undef ERTS_RBT_WANT_LOOKUP_INSERT +# undef ERTS_RBT_WANT_REPLACE +# undef ERTS_RBT_WANT_LOOKUP +# undef ERTS_RBT_WANT_SMALLEST +# undef ERTS_RBT_WANT_LARGEST +# undef ERTS_RBT_WANT_FOREACH +# undef ERTS_RBT_WANT_FOREACH_DESTROY +# undef ERTS_RBT_WANT_FOREACH_YIELDING +# undef ERTS_RBT_WANT_FOREACH_DESTROY_YIELDING +# undef ERTS_RBT_WANT_FOREACH_SMALL +# undef ERTS_RBT_WANT_FOREACH_LARGE +# undef ERTS_RBT_WANT_FOREACH_SMALL_DESTROY +# undef ERTS_RBT_WANT_FOREACH_LARGE_DESTROY +# undef ERTS_RBT_WANT_FOREACH_SMALL_YIELDING +# undef ERTS_RBT_WANT_FOREACH_LARGE_YIELDING +# undef ERTS_RBT_WANT_FOREACH_SMALL_DESTROY_YIELDING +# undef ERTS_RBT_WANT_FOREACH_LARGE_DESTROY_YIELDING +# undef ERTS_RBT_WANT_DEBUG_PRINT +#endif diff --git a/erts/emulator/beam/erl_sched_spec_pre_alloc.c b/erts/emulator/beam/erl_sched_spec_pre_alloc.c index a490aec734..caec24bc03 100644 --- a/erts/emulator/beam/erl_sched_spec_pre_alloc.c +++ b/erts/emulator/beam/erl_sched_spec_pre_alloc.c @@ -3,16 +3,17 @@ * * Copyright Ericsson AB 2011-2012. All Rights Reserved. * - * The contents of this file are subject to the Erlang Public License, - * Version 1.1, (the "License"); you may not use this file except in - * compliance with the License. You should have received a copy of the - * Erlang Public License along with this software. If not, it can be - * retrieved online at http://www.erlang.org/. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Software distributed under the License is distributed on an "AS IS" - * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See - * the License for the specific language governing rights and limitations - * under the License. + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. * * %CopyrightEnd% */ diff --git a/erts/emulator/beam/erl_sched_spec_pre_alloc.h b/erts/emulator/beam/erl_sched_spec_pre_alloc.h index 9144c73acd..4d07b0f674 100644 --- a/erts/emulator/beam/erl_sched_spec_pre_alloc.h +++ b/erts/emulator/beam/erl_sched_spec_pre_alloc.h @@ -3,16 +3,17 @@ * * Copyright Ericsson AB 2011-2012. All Rights Reserved. * - * The contents of this file are subject to the Erlang Public License, - * Version 1.1, (the "License"); you may not use this file except in - * compliance with the License. You should have received a copy of the - * Erlang Public License along with this software. If not, it can be - * retrieved online at http://www.erlang.org/. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Software distributed under the License is distributed on an "AS IS" - * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See - * the License for the specific language governing rights and limitations - * under the License. + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. * * %CopyrightEnd% */ diff --git a/erts/emulator/beam/erl_smp.h b/erts/emulator/beam/erl_smp.h index c38ef47d87..5fc5e989a6 100644 --- a/erts/emulator/beam/erl_smp.h +++ b/erts/emulator/beam/erl_smp.h @@ -3,16 +3,17 @@ * * Copyright Ericsson AB 2005-2013. All Rights Reserved. * - * The contents of this file are subject to the Erlang Public License, - * Version 1.1, (the "License"); you may not use this file except in - * compliance with the License. You should have received a copy of the - * Erlang Public License along with this software. If not, it can be - * retrieved online at http://www.erlang.org/. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Software distributed under the License is distributed on an "AS IS" - * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See - * the License for the specific language governing rights and limitations - * under the License. + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. * * %CopyrightEnd% */ @@ -60,6 +61,7 @@ typedef erts_tsd_key_t erts_smp_tsd_key_t; #define erts_smp_dw_atomic_t erts_dw_atomic_t #define erts_smp_atomic_t erts_atomic_t #define erts_smp_atomic32_t erts_atomic32_t +#define erts_smp_atomic64_t erts_atomic64_t typedef erts_spinlock_t erts_smp_spinlock_t; typedef erts_rwlock_t erts_smp_rwlock_t; void erts_thr_fatal_error(int, char *); /* implemented in erl_init.c */ @@ -95,6 +97,7 @@ typedef int erts_smp_tsd_key_t; #define erts_smp_dw_atomic_t erts_no_dw_atomic_t #define erts_smp_atomic_t erts_no_atomic_t #define erts_smp_atomic32_t erts_no_atomic32_t +#define erts_smp_atomic64_t erts_no_atomic64_t #if __GNUC__ > 2 typedef struct { } erts_smp_spinlock_t; typedef struct { } erts_smp_rwlock_t; @@ -489,6 +492,116 @@ ERTS_GLB_INLINE void erts_smp_thr_sigwait(const sigset_t *set, int *sig); #define erts_smp_atomic32_set_dirty erts_atomic32_set_dirty #define erts_smp_atomic32_read_dirty erts_atomic32_read_dirty +/* 64-bit atomics */ + +#define erts_smp_atomic64_init_nob erts_atomic64_init_nob +#define erts_smp_atomic64_set_nob erts_atomic64_set_nob +#define erts_smp_atomic64_read_nob erts_atomic64_read_nob +#define erts_smp_atomic64_inc_read_nob erts_atomic64_inc_read_nob +#define erts_smp_atomic64_dec_read_nob erts_atomic64_dec_read_nob +#define erts_smp_atomic64_inc_nob erts_atomic64_inc_nob +#define erts_smp_atomic64_dec_nob erts_atomic64_dec_nob +#define erts_smp_atomic64_add_read_nob erts_atomic64_add_read_nob +#define erts_smp_atomic64_add_nob erts_atomic64_add_nob +#define erts_smp_atomic64_read_bor_nob erts_atomic64_read_bor_nob +#define erts_smp_atomic64_read_band_nob erts_atomic64_read_band_nob +#define erts_smp_atomic64_xchg_nob erts_atomic64_xchg_nob +#define erts_smp_atomic64_cmpxchg_nob erts_atomic64_cmpxchg_nob +#define erts_smp_atomic64_read_bset_nob erts_atomic64_read_bset_nob + +#define erts_smp_atomic64_init_mb erts_atomic64_init_mb +#define erts_smp_atomic64_set_mb erts_atomic64_set_mb +#define erts_smp_atomic64_read_mb erts_atomic64_read_mb +#define erts_smp_atomic64_inc_read_mb erts_atomic64_inc_read_mb +#define erts_smp_atomic64_dec_read_mb erts_atomic64_dec_read_mb +#define erts_smp_atomic64_inc_mb erts_atomic64_inc_mb +#define erts_smp_atomic64_dec_mb erts_atomic64_dec_mb +#define erts_smp_atomic64_add_read_mb erts_atomic64_add_read_mb +#define erts_smp_atomic64_add_mb erts_atomic64_add_mb +#define erts_smp_atomic64_read_bor_mb erts_atomic64_read_bor_mb +#define erts_smp_atomic64_read_band_mb erts_atomic64_read_band_mb +#define erts_smp_atomic64_xchg_mb erts_atomic64_xchg_mb +#define erts_smp_atomic64_cmpxchg_mb erts_atomic64_cmpxchg_mb +#define erts_smp_atomic64_read_bset_mb erts_atomic64_read_bset_mb + +#define erts_smp_atomic64_init_acqb erts_atomic64_init_acqb +#define erts_smp_atomic64_set_acqb erts_atomic64_set_acqb +#define erts_smp_atomic64_read_acqb erts_atomic64_read_acqb +#define erts_smp_atomic64_inc_read_acqb erts_atomic64_inc_read_acqb +#define erts_smp_atomic64_dec_read_acqb erts_atomic64_dec_read_acqb +#define erts_smp_atomic64_inc_acqb erts_atomic64_inc_acqb +#define erts_smp_atomic64_dec_acqb erts_atomic64_dec_acqb +#define erts_smp_atomic64_add_read_acqb erts_atomic64_add_read_acqb +#define erts_smp_atomic64_add_acqb erts_atomic64_add_acqb +#define erts_smp_atomic64_read_bor_acqb erts_atomic64_read_bor_acqb +#define erts_smp_atomic64_read_band_acqb erts_atomic64_read_band_acqb +#define erts_smp_atomic64_xchg_acqb erts_atomic64_xchg_acqb +#define erts_smp_atomic64_cmpxchg_acqb erts_atomic64_cmpxchg_acqb +#define erts_smp_atomic64_read_bset_acqb erts_atomic64_read_bset_acqb + +#define erts_smp_atomic64_init_relb erts_atomic64_init_relb +#define erts_smp_atomic64_set_relb erts_atomic64_set_relb +#define erts_smp_atomic64_read_relb erts_atomic64_read_relb +#define erts_smp_atomic64_inc_read_relb erts_atomic64_inc_read_relb +#define erts_smp_atomic64_dec_read_relb erts_atomic64_dec_read_relb +#define erts_smp_atomic64_inc_relb erts_atomic64_inc_relb +#define erts_smp_atomic64_dec_relb erts_atomic64_dec_relb +#define erts_smp_atomic64_add_read_relb erts_atomic64_add_read_relb +#define erts_smp_atomic64_add_relb erts_atomic64_add_relb +#define erts_smp_atomic64_read_bor_relb erts_atomic64_read_bor_relb +#define erts_smp_atomic64_read_band_relb erts_atomic64_read_band_relb +#define erts_smp_atomic64_xchg_relb erts_atomic64_xchg_relb +#define erts_smp_atomic64_cmpxchg_relb erts_atomic64_cmpxchg_relb +#define erts_smp_atomic64_read_bset_relb erts_atomic64_read_bset_relb + +#define erts_smp_atomic64_init_ddrb erts_atomic64_init_ddrb +#define erts_smp_atomic64_set_ddrb erts_atomic64_set_ddrb +#define erts_smp_atomic64_read_ddrb erts_atomic64_read_ddrb +#define erts_smp_atomic64_inc_read_ddrb erts_atomic64_inc_read_ddrb +#define erts_smp_atomic64_dec_read_ddrb erts_atomic64_dec_read_ddrb +#define erts_smp_atomic64_inc_ddrb erts_atomic64_inc_ddrb +#define erts_smp_atomic64_dec_ddrb erts_atomic64_dec_ddrb +#define erts_smp_atomic64_add_read_ddrb erts_atomic64_add_read_ddrb +#define erts_smp_atomic64_add_ddrb erts_atomic64_add_ddrb +#define erts_smp_atomic64_read_bor_ddrb erts_atomic64_read_bor_ddrb +#define erts_smp_atomic64_read_band_ddrb erts_atomic64_read_band_ddrb +#define erts_smp_atomic64_xchg_ddrb erts_atomic64_xchg_ddrb +#define erts_smp_atomic64_cmpxchg_ddrb erts_atomic64_cmpxchg_ddrb +#define erts_smp_atomic64_read_bset_ddrb erts_atomic64_read_bset_ddrb + +#define erts_smp_atomic64_init_rb erts_atomic64_init_rb +#define erts_smp_atomic64_set_rb erts_atomic64_set_rb +#define erts_smp_atomic64_read_rb erts_atomic64_read_rb +#define erts_smp_atomic64_inc_read_rb erts_atomic64_inc_read_rb +#define erts_smp_atomic64_dec_read_rb erts_atomic64_dec_read_rb +#define erts_smp_atomic64_inc_rb erts_atomic64_inc_rb +#define erts_smp_atomic64_dec_rb erts_atomic64_dec_rb +#define erts_smp_atomic64_add_read_rb erts_atomic64_add_read_rb +#define erts_smp_atomic64_add_rb erts_atomic64_add_rb +#define erts_smp_atomic64_read_bor_rb erts_atomic64_read_bor_rb +#define erts_smp_atomic64_read_band_rb erts_atomic64_read_band_rb +#define erts_smp_atomic64_xchg_rb erts_atomic64_xchg_rb +#define erts_smp_atomic64_cmpxchg_rb erts_atomic64_cmpxchg_rb +#define erts_smp_atomic64_read_bset_rb erts_atomic64_read_bset_rb + +#define erts_smp_atomic64_init_wb erts_atomic64_init_wb +#define erts_smp_atomic64_set_wb erts_atomic64_set_wb +#define erts_smp_atomic64_read_wb erts_atomic64_read_wb +#define erts_smp_atomic64_inc_read_wb erts_atomic64_inc_read_wb +#define erts_smp_atomic64_dec_read_wb erts_atomic64_dec_read_wb +#define erts_smp_atomic64_inc_wb erts_atomic64_inc_wb +#define erts_smp_atomic64_dec_wb erts_atomic64_dec_wb +#define erts_smp_atomic64_add_read_wb erts_atomic64_add_read_wb +#define erts_smp_atomic64_add_wb erts_atomic64_add_wb +#define erts_smp_atomic64_read_bor_wb erts_atomic64_read_bor_wb +#define erts_smp_atomic64_read_band_wb erts_atomic64_read_band_wb +#define erts_smp_atomic64_xchg_wb erts_atomic64_xchg_wb +#define erts_smp_atomic64_cmpxchg_wb erts_atomic64_cmpxchg_wb +#define erts_smp_atomic64_read_bset_wb erts_atomic64_read_bset_wb + +#define erts_smp_atomic64_set_dirty erts_atomic64_set_dirty +#define erts_smp_atomic64_read_dirty erts_atomic64_read_dirty + #else /* !ERTS_SMP */ /* Double word size atomics */ @@ -751,6 +864,116 @@ ERTS_GLB_INLINE void erts_smp_thr_sigwait(const sigset_t *set, int *sig); #define erts_smp_atomic32_set_dirty erts_no_atomic32_set #define erts_smp_atomic32_read_dirty erts_no_atomic32_read +/* 64-bit atomics */ + +#define erts_smp_atomic64_init_nob erts_no_atomic64_set +#define erts_smp_atomic64_set_nob erts_no_atomic64_set +#define erts_smp_atomic64_read_nob erts_no_atomic64_read +#define erts_smp_atomic64_inc_read_nob erts_no_atomic64_inc_read +#define erts_smp_atomic64_dec_read_nob erts_no_atomic64_dec_read +#define erts_smp_atomic64_inc_nob erts_no_atomic64_inc +#define erts_smp_atomic64_dec_nob erts_no_atomic64_dec +#define erts_smp_atomic64_add_read_nob erts_no_atomic64_add_read +#define erts_smp_atomic64_add_nob erts_no_atomic64_add +#define erts_smp_atomic64_read_bor_nob erts_no_atomic64_read_bor +#define erts_smp_atomic64_read_band_nob erts_no_atomic64_read_band +#define erts_smp_atomic64_xchg_nob erts_no_atomic64_xchg +#define erts_smp_atomic64_cmpxchg_nob erts_no_atomic64_cmpxchg +#define erts_smp_atomic64_read_bset_nob erts_no_atomic64_read_bset + +#define erts_smp_atomic64_init_mb erts_no_atomic64_set +#define erts_smp_atomic64_set_mb erts_no_atomic64_set +#define erts_smp_atomic64_read_mb erts_no_atomic64_read +#define erts_smp_atomic64_inc_read_mb erts_no_atomic64_inc_read +#define erts_smp_atomic64_dec_read_mb erts_no_atomic64_dec_read +#define erts_smp_atomic64_inc_mb erts_no_atomic64_inc +#define erts_smp_atomic64_dec_mb erts_no_atomic64_dec +#define erts_smp_atomic64_add_read_mb erts_no_atomic64_add_read +#define erts_smp_atomic64_add_mb erts_no_atomic64_add +#define erts_smp_atomic64_read_bor_mb erts_no_atomic64_read_bor +#define erts_smp_atomic64_read_band_mb erts_no_atomic64_read_band +#define erts_smp_atomic64_xchg_mb erts_no_atomic64_xchg +#define erts_smp_atomic64_cmpxchg_mb erts_no_atomic64_cmpxchg +#define erts_smp_atomic64_read_bset_mb erts_no_atomic64_read_bset + +#define erts_smp_atomic64_init_acqb erts_no_atomic64_set +#define erts_smp_atomic64_set_acqb erts_no_atomic64_set +#define erts_smp_atomic64_read_acqb erts_no_atomic64_read +#define erts_smp_atomic64_inc_read_acqb erts_no_atomic64_inc_read +#define erts_smp_atomic64_dec_read_acqb erts_no_atomic64_dec_read +#define erts_smp_atomic64_inc_acqb erts_no_atomic64_inc +#define erts_smp_atomic64_dec_acqb erts_no_atomic64_dec +#define erts_smp_atomic64_add_read_acqb erts_no_atomic64_add_read +#define erts_smp_atomic64_add_acqb erts_no_atomic64_add +#define erts_smp_atomic64_read_bor_acqb erts_no_atomic64_read_bor +#define erts_smp_atomic64_read_band_acqb erts_no_atomic64_read_band +#define erts_smp_atomic64_xchg_acqb erts_no_atomic64_xchg +#define erts_smp_atomic64_cmpxchg_acqb erts_no_atomic64_cmpxchg +#define erts_smp_atomic64_read_bset_acqb erts_no_atomic64_read_bset + +#define erts_smp_atomic64_init_relb erts_no_atomic64_set +#define erts_smp_atomic64_set_relb erts_no_atomic64_set +#define erts_smp_atomic64_read_relb erts_no_atomic64_read +#define erts_smp_atomic64_inc_read_relb erts_no_atomic64_inc_read +#define erts_smp_atomic64_dec_read_relb erts_no_atomic64_dec_read +#define erts_smp_atomic64_inc_relb erts_no_atomic64_inc +#define erts_smp_atomic64_dec_relb erts_no_atomic64_dec +#define erts_smp_atomic64_add_read_relb erts_no_atomic64_add_read +#define erts_smp_atomic64_add_relb erts_no_atomic64_add +#define erts_smp_atomic64_read_bor_relb erts_no_atomic64_read_bor +#define erts_smp_atomic64_read_band_relb erts_no_atomic64_read_band +#define erts_smp_atomic64_xchg_relb erts_no_atomic64_xchg +#define erts_smp_atomic64_cmpxchg_relb erts_no_atomic64_cmpxchg +#define erts_smp_atomic64_read_bset_relb erts_no_atomic64_read_bset + +#define erts_smp_atomic64_init_ddrb erts_no_atomic64_set +#define erts_smp_atomic64_set_ddrb erts_no_atomic64_set +#define erts_smp_atomic64_read_ddrb erts_no_atomic64_read +#define erts_smp_atomic64_inc_read_ddrb erts_no_atomic64_inc_read +#define erts_smp_atomic64_dec_read_ddrb erts_no_atomic64_dec_read +#define erts_smp_atomic64_inc_ddrb erts_no_atomic64_inc +#define erts_smp_atomic64_dec_ddrb erts_no_atomic64_dec +#define erts_smp_atomic64_add_read_ddrb erts_no_atomic64_add_read +#define erts_smp_atomic64_add_ddrb erts_no_atomic64_add +#define erts_smp_atomic64_read_bor_ddrb erts_no_atomic64_read_bor +#define erts_smp_atomic64_read_band_ddrb erts_no_atomic64_read_band +#define erts_smp_atomic64_xchg_ddrb erts_no_atomic64_xchg +#define erts_smp_atomic64_cmpxchg_ddrb erts_no_atomic64_cmpxchg +#define erts_smp_atomic64_read_bset_ddrb erts_no_atomic64_read_bset + +#define erts_smp_atomic64_init_rb erts_no_atomic64_set +#define erts_smp_atomic64_set_rb erts_no_atomic64_set +#define erts_smp_atomic64_read_rb erts_no_atomic64_read +#define erts_smp_atomic64_inc_read_rb erts_no_atomic64_inc_read +#define erts_smp_atomic64_dec_read_rb erts_no_atomic64_dec_read +#define erts_smp_atomic64_inc_rb erts_no_atomic64_inc +#define erts_smp_atomic64_dec_rb erts_no_atomic64_dec +#define erts_smp_atomic64_add_read_rb erts_no_atomic64_add_read +#define erts_smp_atomic64_add_rb erts_no_atomic64_add +#define erts_smp_atomic64_read_bor_rb erts_no_atomic64_read_bor +#define erts_smp_atomic64_read_band_rb erts_no_atomic64_read_band +#define erts_smp_atomic64_xchg_rb erts_no_atomic64_xchg +#define erts_smp_atomic64_cmpxchg_rb erts_no_atomic64_cmpxchg +#define erts_smp_atomic64_read_bset_rb erts_no_atomic64_read_bset + +#define erts_smp_atomic64_init_wb erts_no_atomic64_set +#define erts_smp_atomic64_set_wb erts_no_atomic64_set +#define erts_smp_atomic64_read_wb erts_no_atomic64_read +#define erts_smp_atomic64_inc_read_wb erts_no_atomic64_inc_read +#define erts_smp_atomic64_dec_read_wb erts_no_atomic64_dec_read +#define erts_smp_atomic64_inc_wb erts_no_atomic64_inc +#define erts_smp_atomic64_dec_wb erts_no_atomic64_dec +#define erts_smp_atomic64_add_read_wb erts_no_atomic64_add_read +#define erts_smp_atomic64_add_wb erts_no_atomic64_add +#define erts_smp_atomic64_read_bor_wb erts_no_atomic64_read_bor +#define erts_smp_atomic64_read_band_wb erts_no_atomic64_read_band +#define erts_smp_atomic64_xchg_wb erts_no_atomic64_xchg +#define erts_smp_atomic64_cmpxchg_wb erts_no_atomic64_cmpxchg +#define erts_smp_atomic64_read_bset_wb erts_no_atomic64_read_bset + +#define erts_smp_atomic64_set_dirty erts_no_atomic64_set +#define erts_smp_atomic64_read_dirty erts_no_atomic64_read + #endif /* !ERTS_SMP */ #if ERTS_GLB_INLINE_INCL_FUNC_DEF diff --git a/erts/emulator/beam/erl_sock.h b/erts/emulator/beam/erl_sock.h index 7ae6116dc5..7be6062115 100644 --- a/erts/emulator/beam/erl_sock.h +++ b/erts/emulator/beam/erl_sock.h @@ -3,16 +3,17 @@ * * Copyright Ericsson AB 2003-2009. All Rights Reserved. * - * The contents of this file are subject to the Erlang Public License, - * Version 1.1, (the "License"); you may not use this file except in - * compliance with the License. You should have received a copy of the - * Erlang Public License along with this software. If not, it can be - * retrieved online at http://www.erlang.org/. - * - * Software distributed under the License is distributed on an "AS IS" - * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See - * the License for the specific language governing rights and limitations - * under the License. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. * * %CopyrightEnd% */ diff --git a/erts/emulator/beam/erl_sys_driver.h b/erts/emulator/beam/erl_sys_driver.h index dab4a94a9b..4031eef0aa 100644 --- a/erts/emulator/beam/erl_sys_driver.h +++ b/erts/emulator/beam/erl_sys_driver.h @@ -3,16 +3,17 @@ * * Copyright Ericsson AB 2001-2013. All Rights Reserved. * - * The contents of this file are subject to the Erlang Public License, - * Version 1.1, (the "License"); you may not use this file except in - * compliance with the License. You should have received a copy of the - * Erlang Public License along with this software. If not, it can be - * retrieved online at http://www.erlang.org/. - * - * Software distributed under the License is distributed on an "AS IS" - * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See - * the License for the specific language governing rights and limitations - * under the License. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. * * %CopyrightEnd% */ diff --git a/erts/emulator/beam/erl_term.c b/erts/emulator/beam/erl_term.c index 28cbe7004f..72a1e0b6ec 100644 --- a/erts/emulator/beam/erl_term.c +++ b/erts/emulator/beam/erl_term.c @@ -3,16 +3,17 @@ * * Copyright Ericsson AB 2000-2013. All Rights Reserved. * - * The contents of this file are subject to the Erlang Public License, - * Version 1.1, (the "License"); you may not use this file except in - * compliance with the License. You should have received a copy of the - * Erlang Public License along with this software. If not, it can be - * retrieved online at http://www.erlang.org/. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Software distributed under the License is distributed on an "AS IS" - * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See - * the License for the specific language governing rights and limitations - * under the License. + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. * * %CopyrightEnd% */ @@ -27,92 +28,35 @@ #include <stdlib.h> #include <stdio.h> -__decl_noreturn static void __noreturn -et_abort(const char *expr, const char *file, unsigned line) +void +erts_set_literal_tag(Eterm *term, Eterm *hp_start, Eterm hsz) { -#ifdef EXIT_ON_ET_ABORT - static int have_been_called = 0; +#ifdef TAG_LITERAL_PTR + Eterm *hp_end, *hp; + + hp_end = hp_start + hsz; + hp = hp_start; - if (have_been_called) { - abort(); - } else { - /* - * Prevent infinite loop. - */ - have_been_called = 1; - erl_exit(1, "TYPE ASSERTION FAILED, file %s, line %u: %s\n", file, line, expr); - } -#else - erts_fprintf(stderr, "TYPE ASSERTION FAILED, file %s, line %u: %s\n", file, line, expr); - abort(); -#endif -} - -#if ET_DEBUG -#define ET_ASSERT(expr,file,line) \ -do { \ - if (!(expr)) \ - et_abort(#expr, file, line); \ -} while(0) -#else -#define ET_ASSERT(expr,file,line) do { } while(0) -#endif - -#if ET_DEBUG -unsigned tag_val_def_debug(Wterm x, const char *file, unsigned line) -#else -unsigned tag_val_def(Wterm x) -#define file __FILE__ -#define line __LINE__ -#endif -{ - static char msg[32]; - - switch (x & _TAG_PRIMARY_MASK) { - case TAG_PRIMARY_LIST: - ET_ASSERT(_list_precond(x),file,line); - return LIST_DEF; - case TAG_PRIMARY_BOXED: { - Eterm hdr = *boxed_val(x); - ET_ASSERT(is_header(hdr),file,line); - switch ((hdr & _TAG_HEADER_MASK) >> _TAG_PRIMARY_SIZE) { - case (_TAG_HEADER_ARITYVAL >> _TAG_PRIMARY_SIZE): return TUPLE_DEF; - case (_TAG_HEADER_POS_BIG >> _TAG_PRIMARY_SIZE): return BIG_DEF; - case (_TAG_HEADER_NEG_BIG >> _TAG_PRIMARY_SIZE): return BIG_DEF; - case (_TAG_HEADER_REF >> _TAG_PRIMARY_SIZE): return REF_DEF; - case (_TAG_HEADER_FLOAT >> _TAG_PRIMARY_SIZE): return FLOAT_DEF; - case (_TAG_HEADER_EXPORT >> _TAG_PRIMARY_SIZE): return EXPORT_DEF; - case (_TAG_HEADER_FUN >> _TAG_PRIMARY_SIZE): return FUN_DEF; - case (_TAG_HEADER_EXTERNAL_PID >> _TAG_PRIMARY_SIZE): return EXTERNAL_PID_DEF; - case (_TAG_HEADER_EXTERNAL_PORT >> _TAG_PRIMARY_SIZE): return EXTERNAL_PORT_DEF; - case (_TAG_HEADER_EXTERNAL_REF >> _TAG_PRIMARY_SIZE): return EXTERNAL_REF_DEF; - case (_TAG_HEADER_REFC_BIN >> _TAG_PRIMARY_SIZE): return BINARY_DEF; - case (_TAG_HEADER_HEAP_BIN >> _TAG_PRIMARY_SIZE): return BINARY_DEF; - case (_TAG_HEADER_SUB_BIN >> _TAG_PRIMARY_SIZE): return BINARY_DEF; - case (_TAG_HEADER_MAP >> _TAG_PRIMARY_SIZE): return MAP_DEF; - } - break; - } - case TAG_PRIMARY_IMMED1: { - switch ((x & _TAG_IMMED1_MASK) >> _TAG_PRIMARY_SIZE) { - case (_TAG_IMMED1_PID >> _TAG_PRIMARY_SIZE): return PID_DEF; - case (_TAG_IMMED1_PORT >> _TAG_PRIMARY_SIZE): return PORT_DEF; - case (_TAG_IMMED1_IMMED2 >> _TAG_PRIMARY_SIZE): { - switch ((x & _TAG_IMMED2_MASK) >> _TAG_IMMED1_SIZE) { - case (_TAG_IMMED2_ATOM >> _TAG_IMMED1_SIZE): return ATOM_DEF; - case (_TAG_IMMED2_NIL >> _TAG_IMMED1_SIZE): return NIL_DEF; - } - break; + while (hp < hp_end) { + switch (primary_tag(*hp)) { + case TAG_PRIMARY_BOXED: + case TAG_PRIMARY_LIST: + *hp |= TAG_LITERAL_PTR; + break; + case TAG_PRIMARY_HEADER: + if (header_is_thing(*hp)) { + hp += thing_arityval(*hp); } - case (_TAG_IMMED1_SMALL >> _TAG_PRIMARY_SIZE): return SMALL_DEF; - } - break; - } + break; + default: + break; + } + + hp++; } - erts_snprintf(msg, sizeof(msg), "tag_val_def: %#lx", (unsigned long) x); - et_abort(msg, file, line); -#undef file -#undef line + if (is_boxed(*term) || is_list(*term)) + *term |= TAG_LITERAL_PTR; +#endif } /* @@ -127,10 +71,10 @@ FUNTY checked_##FUN(ARGTY x, const char *file, unsigned line) \ return _unchecked_##FUN(x); \ } -ET_DEFINE_CHECKED(Eterm,make_boxed,Eterm*,_is_taggable_pointer); +ET_DEFINE_CHECKED(Eterm,make_boxed,const Eterm*,_is_taggable_pointer); ET_DEFINE_CHECKED(int,is_boxed,Eterm,!is_header); ET_DEFINE_CHECKED(Eterm*,boxed_val,Wterm,_boxed_precond); -ET_DEFINE_CHECKED(Eterm,make_list,Eterm*,_is_taggable_pointer); +ET_DEFINE_CHECKED(Eterm,make_list,const Eterm*,_is_taggable_pointer); ET_DEFINE_CHECKED(int,is_not_list,Eterm,!is_header); ET_DEFINE_CHECKED(Eterm*,list_val,Wterm,_list_precond); ET_DEFINE_CHECKED(Uint,unsigned_val,Eterm,is_small); @@ -171,9 +115,7 @@ ET_DEFINE_CHECKED(Uint,external_thing_data_words,ExternalThing*,is_thing_ptr); ET_DEFINE_CHECKED(Eterm,make_cp,UWord *,_is_taggable_pointer); ET_DEFINE_CHECKED(UWord *,cp_val,Eterm,is_CP); ET_DEFINE_CHECKED(Uint,catch_val,Eterm,is_catch); -ET_DEFINE_CHECKED(Uint,x_reg_offset,Uint,_is_xreg); -ET_DEFINE_CHECKED(Uint,y_reg_offset,Uint,_is_yreg); -ET_DEFINE_CHECKED(Uint,x_reg_index,Uint,_is_xreg); -ET_DEFINE_CHECKED(Uint,y_reg_index,Uint,_is_yreg); +ET_DEFINE_CHECKED(Uint,loader_x_reg_index,Uint,_is_loader_x_reg); +ET_DEFINE_CHECKED(Uint,loader_y_reg_index,Uint,_is_loader_y_reg); #endif /* ET_DEBUG */ diff --git a/erts/emulator/beam/erl_term.h b/erts/emulator/beam/erl_term.h index 37014ccf94..0a71534790 100644 --- a/erts/emulator/beam/erl_term.h +++ b/erts/emulator/beam/erl_term.h @@ -3,16 +3,17 @@ * * Copyright Ericsson AB 2000-2014. All Rights Reserved. * - * The contents of this file are subject to the Erlang Public License, - * Version 1.1, (the "License"); you may not use this file except in - * compliance with the License. You should have received a copy of the - * Erlang Public License along with this software. If not, it can be - * retrieved online at http://www.erlang.org/. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Software distributed under the License is distributed on an "AS IS" - * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See - * the License for the specific language governing rights and limitations - * under the License. + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. * * %CopyrightEnd% */ @@ -20,34 +21,10 @@ #ifndef __ERL_TERM_H #define __ERL_TERM_H -#include "sys.h" /* defines HALFWORD_HEAP */ +#include "erl_mmap.h" typedef UWord Wterm; /* Full word terms */ -#if HALFWORD_HEAP -# define HEAP_ON_C_STACK 0 -# if HALFWORD_ASSERT -# ifdef ET_DEBUG -# undef ET_DEBUG -# endif -# define ET_DEBUG 1 -# endif -# if 1 -# define CHECK_POINTER_MASK 0xFFFFFFFF00000000UL -# define COMPRESS_POINTER(APointer) ((Eterm) (UWord) (APointer)) -# define EXPAND_POINTER(AnEterm) ((UWord) (AnEterm)) -# else -# define CHECK_POINTER_MASK 0x0UL -# define COMPRESS_POINTER(AnUint) (AnUint) -# define EXPAND_POINTER(APointer) (APointer) -# endif -#else -# define HEAP_ON_C_STACK 1 -# define CHECK_POINTER_MASK 0x0UL -# define COMPRESS_POINTER(AnUint) (AnUint) -# define EXPAND_POINTER(APointer) (APointer) -#endif - struct erl_node_; /* Declared in erl_node_tables.h */ /* @@ -73,6 +50,24 @@ struct erl_node_; /* Declared in erl_node_tables.h */ #define _ET_APPLY(F,X) _unchecked_##F(X) #endif +#if defined(ARCH_64) +# define TAG_PTR_MASK__ 0x7 +# if !defined(ERTS_HAVE_OS_PHYSICAL_MEMORY_RESERVATION) +# ifdef HIPE +# error Hipe on 64-bit needs a real mmap as it does not support the literal tag +# endif +# define TAG_LITERAL_PTR 0x4 +# else +# undef TAG_LITERAL_PTR +# endif +#elif defined(ARCH_32) +# define TAG_PTR_MASK__ 0x3 +# undef TAG_LITERAL_PTR +#else +# error Not supported arch +#endif + + #define _TAG_PRIMARY_SIZE 2 #define _TAG_PRIMARY_MASK 0x3 #define TAG_PRIMARY_HEADER 0x0 @@ -147,21 +142,21 @@ struct erl_node_; /* Declared in erl_node_tables.h */ #define MAP_SUBTAG (0xF << _TAG_PRIMARY_SIZE) /* MAP */ -#define _TAG_HEADER_ARITYVAL (TAG_PRIMARY_HEADER|ARITYVAL_SUBTAG) -#define _TAG_HEADER_FUN (TAG_PRIMARY_HEADER|FUN_SUBTAG) -#define _TAG_HEADER_POS_BIG (TAG_PRIMARY_HEADER|POS_BIG_SUBTAG) -#define _TAG_HEADER_NEG_BIG (TAG_PRIMARY_HEADER|NEG_BIG_SUBTAG) -#define _TAG_HEADER_FLOAT (TAG_PRIMARY_HEADER|FLOAT_SUBTAG) -#define _TAG_HEADER_EXPORT (TAG_PRIMARY_HEADER|EXPORT_SUBTAG) -#define _TAG_HEADER_REF (TAG_PRIMARY_HEADER|REF_SUBTAG) -#define _TAG_HEADER_REFC_BIN (TAG_PRIMARY_HEADER|REFC_BINARY_SUBTAG) -#define _TAG_HEADER_HEAP_BIN (TAG_PRIMARY_HEADER|HEAP_BINARY_SUBTAG) -#define _TAG_HEADER_SUB_BIN (TAG_PRIMARY_HEADER|SUB_BINARY_SUBTAG) -#define _TAG_HEADER_EXTERNAL_PID (TAG_PRIMARY_HEADER|EXTERNAL_PID_SUBTAG) -#define _TAG_HEADER_EXTERNAL_PORT (TAG_PRIMARY_HEADER|EXTERNAL_PORT_SUBTAG) -#define _TAG_HEADER_EXTERNAL_REF (TAG_PRIMARY_HEADER|EXTERNAL_REF_SUBTAG) +#define _TAG_HEADER_ARITYVAL (TAG_PRIMARY_HEADER|ARITYVAL_SUBTAG) +#define _TAG_HEADER_FUN (TAG_PRIMARY_HEADER|FUN_SUBTAG) +#define _TAG_HEADER_POS_BIG (TAG_PRIMARY_HEADER|POS_BIG_SUBTAG) +#define _TAG_HEADER_NEG_BIG (TAG_PRIMARY_HEADER|NEG_BIG_SUBTAG) +#define _TAG_HEADER_FLOAT (TAG_PRIMARY_HEADER|FLOAT_SUBTAG) +#define _TAG_HEADER_EXPORT (TAG_PRIMARY_HEADER|EXPORT_SUBTAG) +#define _TAG_HEADER_REF (TAG_PRIMARY_HEADER|REF_SUBTAG) +#define _TAG_HEADER_REFC_BIN (TAG_PRIMARY_HEADER|REFC_BINARY_SUBTAG) +#define _TAG_HEADER_HEAP_BIN (TAG_PRIMARY_HEADER|HEAP_BINARY_SUBTAG) +#define _TAG_HEADER_SUB_BIN (TAG_PRIMARY_HEADER|SUB_BINARY_SUBTAG) +#define _TAG_HEADER_EXTERNAL_PID (TAG_PRIMARY_HEADER|EXTERNAL_PID_SUBTAG) +#define _TAG_HEADER_EXTERNAL_PORT (TAG_PRIMARY_HEADER|EXTERNAL_PORT_SUBTAG) +#define _TAG_HEADER_EXTERNAL_REF (TAG_PRIMARY_HEADER|EXTERNAL_REF_SUBTAG) #define _TAG_HEADER_BIN_MATCHSTATE (TAG_PRIMARY_HEADER|BIN_MATCHSTATE_SUBTAG) -#define _TAG_HEADER_MAP (TAG_PRIMARY_HEADER|MAP_SUBTAG) +#define _TAG_HEADER_MAP (TAG_PRIMARY_HEADER|MAP_SUBTAG) #define _TAG_HEADER_MASK 0x3F @@ -189,16 +184,14 @@ struct erl_node_; /* Declared in erl_node_tables.h */ /* boxed object access methods */ -#if HALFWORD_HEAP -#define _is_taggable_pointer(x) (((UWord)(x) & (CHECK_POINTER_MASK | 0x3)) == 0) -#define _boxed_precond(x) (is_boxed(x)) -#else -#define _is_taggable_pointer(x) (((Uint)(x) & 0x3) == 0) + +#define _is_taggable_pointer(x) (((Uint)(x) & TAG_PTR_MASK__) == 0) + #define _boxed_precond(x) (is_boxed(x)) -#endif -#define _is_aligned(x) (((Uint)(x) & 0x3) == 0) -#define _unchecked_make_boxed(x) ((Uint) COMPRESS_POINTER(x) + TAG_PRIMARY_BOXED) -_ET_DECLARE_CHECKED(Eterm,make_boxed,Eterm*) + +#define _is_aligned(x) (((Uint)(x) & TAG_PTR_MASK__) == 0) +#define _unchecked_make_boxed(x) ((Uint)(x) + TAG_PRIMARY_BOXED) +_ET_DECLARE_CHECKED(Eterm,make_boxed,const Eterm*) #define make_boxed(x) _ET_APPLY(make_boxed,(x)) #if 1 #define _is_not_boxed(x) ((x) & (_TAG_PRIMARY_MASK-TAG_PRIMARY_BOXED)) @@ -208,13 +201,17 @@ _ET_DECLARE_CHECKED(int,is_boxed,Eterm) #else #define is_boxed(x) (((x) & _TAG_PRIMARY_MASK) == TAG_PRIMARY_BOXED) #endif -#define _unchecked_boxed_val(x) ((Eterm*) EXPAND_POINTER(((x) - TAG_PRIMARY_BOXED))) +#ifdef TAG_LITERAL_PTR +#define _unchecked_boxed_val(x) _unchecked_ptr_val(x) +#else +#define _unchecked_boxed_val(x) ((Eterm*) ((x) - TAG_PRIMARY_BOXED)) +#endif _ET_DECLARE_CHECKED(Eterm*,boxed_val,Wterm) #define boxed_val(x) _ET_APPLY(boxed_val,(x)) /* cons cell ("list") access methods */ -#define _unchecked_make_list(x) ((Uint) COMPRESS_POINTER(x) + TAG_PRIMARY_LIST) -_ET_DECLARE_CHECKED(Eterm,make_list,Eterm*) +#define _unchecked_make_list(x) ((Uint)(x) + TAG_PRIMARY_LIST) +_ET_DECLARE_CHECKED(Eterm,make_list,const Eterm*) #define make_list(x) _ET_APPLY(make_list,(x)) #if 1 #define _unchecked_is_not_list(x) ((x) & (_TAG_PRIMARY_MASK-TAG_PRIMARY_LIST)) @@ -225,12 +222,12 @@ _ET_DECLARE_CHECKED(int,is_not_list,Eterm) #define is_list(x) (((x) & _TAG_PRIMARY_MASK) == TAG_PRIMARY_LIST) #define is_not_list(x) (!is_list((x))) #endif -#if HALFWORD_HEAP #define _list_precond(x) (is_list(x)) +#ifdef TAG_LITERAL_PTR +#define _unchecked_list_val(x) _unchecked_ptr_val(x) #else -#define _list_precond(x) (is_list(x)) +#define _unchecked_list_val(x) ((Eterm*) ((x) - TAG_PRIMARY_LIST)) #endif -#define _unchecked_list_val(x) ((Eterm*) EXPAND_POINTER((x) - TAG_PRIMARY_LIST)) _ET_DECLARE_CHECKED(Eterm*,list_val,Wterm) #define list_val(x) _ET_APPLY(list_val,(x)) @@ -241,15 +238,22 @@ _ET_DECLARE_CHECKED(Eterm*,list_val,Wterm) #define CDR(x) ((x)[1]) /* generic tagged pointer (boxed or list) access methods */ -#define _unchecked_ptr_val(x) ((Eterm*) EXPAND_POINTER((x) & ~((Uint) 0x3))) +#define _unchecked_ptr_val(x) ((Eterm*) ((x) & ~((Uint) TAG_PTR_MASK__))) #define ptr_val(x) _unchecked_ptr_val((x)) /*XXX*/ #define _unchecked_offset_ptr(x,offs) ((x)+((offs)*sizeof(Eterm))) #define offset_ptr(x,offs) _unchecked_offset_ptr(x,offs) /*XXX*/ #define _unchecked_byte_offset_ptr(x,byte_offs) ((x)+(offs)) #define byte_offset_ptr(x,offs) _unchecked_byte_offset_ptr(x,offs) /*XXX*/ +#ifdef TAG_LITERAL_PTR +#define _unchecked_is_not_literal_ptr(x) (!((x) & TAG_LITERAL_PTR)) +#define is_not_literal_ptr(x) _unchecked_is_not_literal_ptr((x)) /*XXX*/ +#define is_literal_ptr(x) (!is_not_literal_ptr((x))) /*XXX*/ +#endif + + /* fixnum ("small") access methods */ -#if defined(ARCH_64) && !HALFWORD_HEAP +#if defined(ARCH_64) #define SMALL_BITS (64-4) #define SMALL_DIGITS (17) #else @@ -296,9 +300,10 @@ _ET_DECLARE_CHECKED(Uint,atom_val,Eterm) #define atom_val(x) _ET_APPLY(atom_val,(x)) /* header (arityval or thing) access methods */ -#define _make_header(sz,tag) ((Uint)(((sz) << _HEADER_ARITY_OFFS) + (tag))) +#define _make_header(sz,tag) ((Uint)(((Uint)(sz) << _HEADER_ARITY_OFFS) + (tag))) #define is_header(x) (((x) & _TAG_PRIMARY_MASK) == TAG_PRIMARY_HEADER) -#define _unchecked_header_arity(x) ((x) >> _HEADER_ARITY_OFFS) +#define _unchecked_header_arity(x) \ + (is_map_header(x) ? MAP_HEADER_ARITY(x) : ((x) >> _HEADER_ARITY_OFFS)) _ET_DECLARE_CHECKED(Uint,header_arity,Eterm) #define header_arity(x) _ET_APPLY(header_arity,(x)) @@ -361,6 +366,7 @@ _ET_DECLARE_CHECKED(Uint,thing_subtag,Eterm) ((((x) & (_TAG_HEADER_MASK)) == _TAG_HEADER_REFC_BIN) || \ (((x) & (_TAG_HEADER_MASK)) == _TAG_HEADER_HEAP_BIN) || \ (((x) & (_TAG_HEADER_MASK)) == _TAG_HEADER_SUB_BIN)) + #define make_binary(x) make_boxed((Eterm*)(x)) #define is_binary(x) (is_boxed((x)) && is_binary_header(*boxed_val((x)))) #define is_not_binary(x) (!is_binary((x))) @@ -393,11 +399,7 @@ _ET_DECLARE_CHECKED(Eterm*,fun_val,Wterm) _ET_DECLARE_CHECKED(Eterm*,export_val,Wterm) #define export_val(x) _ET_APPLY(export_val,(x)) #define is_export_header(x) ((x) == HEADER_EXPORT) -#if HALFWORD_HEAP -#define HEADER_EXPORT _make_header(2,_TAG_HEADER_EXPORT) -#else #define HEADER_EXPORT _make_header(1,_TAG_HEADER_EXPORT) -#endif /* bignum access methods */ #define make_pos_bignum_header(sz) _make_header((sz),_TAG_HEADER_POS_BIG) @@ -421,7 +423,7 @@ _ET_DECLARE_CHECKED(Eterm*,big_val,Wterm) #define big_val(x) _ET_APPLY(big_val,(x)) /* flonum ("float") access methods */ -#if defined(ARCH_64) && !HALFWORD_HEAP +#if defined(ARCH_64) #define HEADER_FLONUM _make_header(1,_TAG_HEADER_FLOAT) #else #define HEADER_FLONUM _make_header(2,_TAG_HEADER_FLOAT) @@ -442,12 +444,12 @@ typedef union float_def byte fb[sizeof(ieee754_8)]; Uint16 fs[sizeof(ieee754_8) / sizeof(Uint16)]; Uint32 fw[sizeof(ieee754_8) / sizeof(Uint32)]; -#if defined(ARCH_64) && !HALFWORD_HEAP +#if defined(ARCH_64) Uint fdw; #endif } FloatDef; -#if defined(ARCH_64) && !HALFWORD_HEAP +#if defined(ARCH_64) #define FLOAT_VAL_GET_DOUBLE(fval, f) (f).fdw = *((fval)+1) @@ -724,7 +726,7 @@ _ET_DECLARE_CHECKED(struct erl_node_*,internal_port_node,Eterm) #define ERTS_MAX_REF_NUMBERS 3 #define ERTS_REF_NUMBERS ERTS_MAX_REF_NUMBERS -#if defined(ARCH_64) && !HALFWORD_HEAP +#if defined(ARCH_64) # define ERTS_REF_WORDS (ERTS_REF_NUMBERS/2 + 1) # define ERTS_REF_32BIT_WORDS (ERTS_REF_NUMBERS+1) #else @@ -746,7 +748,7 @@ typedef struct { #define make_ref_thing_header(DW) \ _make_header((DW)+REF_THING_HEAD_SIZE-1,_TAG_HEADER_REF) -#if defined(ARCH_64) && !HALFWORD_HEAP +#if defined(ARCH_64) /* * Ref layout on a 64-bit little endian machine: @@ -990,6 +992,37 @@ _ET_DECLARE_CHECKED(Uint32*,external_ref_data,Wterm) _ET_DECLARE_CHECKED(struct erl_node_*,external_ref_node,Eterm) #define external_ref_node(x) _ET_APPLY(external_ref_node,(x)) +/* maps */ + +#define MAP_HEADER_TAG_SZ (2) +#define MAP_HEADER_ARITY_SZ (8) +#define MAP_HEADER_VAL_SZ (16) + +#define MAP_HEADER_TAG_FLATMAP_HEAD (0x0) +#define MAP_HEADER_TAG_HAMT_NODE_BITMAP (0x1) +#define MAP_HEADER_TAG_HAMT_HEAD_ARRAY (0x2) +#define MAP_HEADER_TAG_HAMT_HEAD_BITMAP (0x3) + +#define MAP_HEADER_TYPE(Hdr) (((Hdr) >> (_HEADER_ARITY_OFFS)) & (0x3)) +#define MAP_HEADER_ARITY(Hdr) (((Hdr) >> (_HEADER_ARITY_OFFS + MAP_HEADER_TAG_SZ)) & (0xff)) +#define MAP_HEADER_VAL(Hdr) (((Hdr) >> (_HEADER_ARITY_OFFS + MAP_HEADER_TAG_SZ + MAP_HEADER_ARITY_SZ)) & (0xffff)) + +#define make_hashmap(x) make_boxed((Eterm*)(x)) +#define is_hashmap(x) (is_boxed((x)) && is_hashmap_header(*boxed_val((x)))) +#define is_not_hashmap(x) (!is_hashmap(x)) +#define is_hashmap_header(x) (((x) & (_HEADER_MAP_HASHMAP_HEAD_MASK)) == HAMT_SUBTAG_HEAD_ARRAY) +#define hashmap_val(x) _unchecked_boxed_val((x)) + +#define make_flatmap(x) make_boxed((Eterm*)(x)) +#define is_flatmap(x) (is_boxed((x)) && is_flatmap_header(*boxed_val((x)))) +#define is_not_flatmap(x) (!is_flatmap((x))) +#define is_flatmap_header(x) (((x) & (_HEADER_MAP_SUBTAG_MASK)) == HAMT_SUBTAG_HEAD_FLATMAP) +#define flatmap_val(x) (_unchecked_boxed_val((x))) + +#define is_map_header(x) (((x) & (_TAG_HEADER_MASK)) == _TAG_HEADER_MAP) +#define is_map(x) (is_boxed((x)) && is_map_header(*boxed_val(x))) +#define is_not_map(x) (!is_map(x)) + /* number tests */ #define is_integer(x) (is_small(x) || is_big(x)) @@ -1007,14 +1040,14 @@ _ET_DECLARE_CHECKED(struct erl_node_*,external_ref_node,Eterm) #error "fix yer arch, like" #endif -#define _unchecked_make_cp(x) ((Eterm) COMPRESS_POINTER(x)) +#define _unchecked_make_cp(x) ((Eterm)(x)) _ET_DECLARE_CHECKED(Eterm,make_cp,BeamInstr*) #define make_cp(x) _ET_APPLY(make_cp,(x)) #define is_not_CP(x) ((x) & _CPMASK) #define is_CP(x) (!is_not_CP(x)) -#define _unchecked_cp_val(x) ((BeamInstr*) EXPAND_POINTER(x)) +#define _unchecked_cp_val(x) ((BeamInstr*) (x)) _ET_DECLARE_CHECKED(BeamInstr*,cp_val,Eterm) #define cp_val(x) _ET_APPLY(cp_val,(x)) @@ -1030,44 +1063,40 @@ _ET_DECLARE_CHECKED(Uint,catch_val,Eterm) /* * Overloaded tags. * - * SMALL = 15 - * ATOM/NIL=7 + * In the loader, we want to tag a term in a way so that it can + * be any literal (atom/integer/float/tuple/list/binary) or a + * register. * - * Note that the two least significant bits in SMALL/ATOM/NIL always are 3; - * thus, we can distinguish register from literals by looking at only these - * two bits. + * We can achive that by overloading the PID and PORT tags to + * mean X and Y registers. That works because there are no + * pid or port literals. */ -#define X_REG_DEF 0 -#define Y_REG_DEF 1 -#define R_REG_DEF 2 +#define _LOADER_TAG_XREG _TAG_IMMED1_PID +#define _LOADER_TAG_YREG _TAG_IMMED1_PORT +#define _LOADER_TAG_SIZE _TAG_IMMED1_SIZE +#define _LOADER_MASK _TAG_IMMED1_MASK -#define beam_reg_tag(x) ((x) & 3) +#define LOADER_X_REG _LOADER_TAG_XREG +#define LOADER_Y_REG _LOADER_TAG_YREG -#define make_rreg() R_REG_DEF -#define make_xreg(ix) (((ix) * sizeof(Eterm)) | X_REG_DEF) -#define make_yreg(ix) (((ix) * sizeof(Eterm)) | Y_REG_DEF) +#define make_loader_x_reg(R) (((R) << _LOADER_TAG_SIZE) | _LOADER_TAG_XREG) +#define make_loader_y_reg(R) (((R) << _LOADER_TAG_SIZE) | _LOADER_TAG_YREG) -#define _is_xreg(x) (beam_reg_tag(x) == X_REG_DEF) -#define _is_yreg(x) (beam_reg_tag(x) == Y_REG_DEF) +#define loader_reg_index(R) ((R) >> _LOADER_TAG_SIZE) -#define _unchecked_x_reg_offset(R) ((R) - X_REG_DEF) -_ET_DECLARE_CHECKED(Uint,x_reg_offset,Uint) -#define x_reg_offset(R) _ET_APPLY(x_reg_offset,(R)) +#define loader_tag(T) ((T) & _LOADER_MASK) -#define _unchecked_y_reg_offset(R) ((R) - Y_REG_DEF) -_ET_DECLARE_CHECKED(Uint,y_reg_offset,Uint) -#define y_reg_offset(R) _ET_APPLY(y_reg_offset,(R)) +#define _is_loader_x_reg(x) (loader_tag(x) == _LOADER_TAG_XREG) +#define _is_loader_y_reg(x) (loader_tag(x) == _LOADER_TAG_YREG) -#define reg_index(R) ((R) / sizeof(Eterm)) +#define _unchecked_loader_x_reg_index(R) ((R) >> _LOADER_TAG_SIZE) +_ET_DECLARE_CHECKED(Uint,loader_x_reg_index,Uint) +#define loader_x_reg_index(R) _ET_APPLY(loader_x_reg_index,(R)) -#define _unchecked_x_reg_index(R) ((R) >> 2) -_ET_DECLARE_CHECKED(Uint,x_reg_index,Uint) -#define x_reg_index(R) _ET_APPLY(x_reg_index,(R)) - -#define _unchecked_y_reg_index(R) ((R) >> 2) -_ET_DECLARE_CHECKED(Uint,y_reg_index,Uint) -#define y_reg_index(R) _ET_APPLY(y_reg_index,(R)) +#define _unchecked_loader_y_reg_index(R) ((R) >> _LOADER_TAG_SIZE) +_ET_DECLARE_CHECKED(Uint,loader_y_reg_index,Uint) +#define loader_y_reg_index(R) _ET_APPLY(loader_y_reg_index,(R)) /* * Backwards compatibility definitions: @@ -1095,13 +1124,16 @@ _ET_DECLARE_CHECKED(Uint,y_reg_index,Uint) #define FLOAT_DEF 0xe #define BIG_DEF 0xf #define SMALL_DEF 0x10 +#define MATCHSTATE_DEF 0x11 /* not a "real" term */ + +#define FIRST_VACANT_TAG_DEF 0x12 #if ET_DEBUG -extern unsigned tag_val_def_debug(Wterm, const char*, unsigned); -#define tag_val_def(x) tag_val_def_debug((x),__FILE__,__LINE__) +ERTS_GLB_INLINE unsigned tag_val_def(Wterm, const char*, unsigned); #else -extern unsigned tag_val_def(Wterm); +ERTS_GLB_INLINE unsigned tag_val_def(Wterm); #endif + #define not_eq_tags(X,Y) (tag_val_def((X)) ^ tag_val_def((Y))) #define NUMBER_CODE(x,y) ((tag_val_def(x) << 5) | tag_val_def(y)) @@ -1116,81 +1148,83 @@ extern unsigned tag_val_def(Wterm); #define FLOAT_BIG _NUMBER_CODE(FLOAT_DEF,BIG_DEF) #define FLOAT_FLOAT _NUMBER_CODE(FLOAT_DEF,FLOAT_DEF) -#if HALFWORD_HEAP -#define ptr2rel(PTR,BASE) ((Eterm*)((char*)(PTR) - (char*)(BASE))) -#define rterm2wterm(REL,BASE) ((Wterm)(REL) + (Wterm)(BASE)) - -#else /* HALFWORD_HEAP */ - -#define ptr2rel(PTR,BASE) (PTR) -#define rterm2wterm(REL,BASE) (REL) - -#endif /* !HALFWORD_HEAP */ - -#define make_list_rel(PTR, BASE) make_list(ptr2rel(PTR,BASE)) -#define make_boxed_rel(PTR, BASE) make_boxed(ptr2rel(PTR,BASE)) -#define make_fun_rel make_boxed_rel -#define make_binary_rel make_boxed_rel -#define make_tuple_rel make_boxed_rel -#define make_external_rel make_boxed_rel -#define make_internal_ref_rel make_boxed_rel -#define make_big_rel make_boxed_rel - -#define binary_val_rel(RTERM, BASE) binary_val(rterm2wterm(RTERM, BASE)) -#define list_val_rel(RTERM, BASE) list_val(rterm2wterm(RTERM, BASE)) -#define boxed_val_rel(RTERM, BASE) boxed_val(rterm2wterm(RTERM, BASE)) -#define tuple_val_rel(RTERM, BASE) tuple_val(rterm2wterm(RTERM, BASE)) -#define export_val_rel(RTERM, BASE) export_val(rterm2wterm(RTERM, BASE)) -#define fun_val_rel(RTERM, BASE) fun_val(rterm2wterm(RTERM, BASE)) -#define big_val_rel(RTERM,BASE) big_val(rterm2wterm(RTERM,BASE)) -#define float_val_rel(RTERM,BASE) float_val(rterm2wterm(RTERM,BASE)) -#define internal_ref_val_rel(RTERM,BASE) internal_ref_val(rterm2wterm(RTERM,BASE)) - -#define external_thing_ptr_rel(RTERM, BASE) external_thing_ptr(rterm2wterm(RTERM, BASE)) -#define external_data_words_rel(RTERM,BASE) external_data_words(rterm2wterm(RTERM,BASE)) +#define is_same(A,B) ((A)==(B)) -#define external_port_node_rel(RTERM,BASE) external_port_node(rterm2wterm(RTERM,BASE)) -#define external_port_data_rel(RTERM,BASE) external_port_data(rterm2wterm(RTERM,BASE)) +void erts_set_literal_tag(Eterm *term, Eterm *hp_start, Eterm hsz); -#define is_external_pid_rel(RTERM,BASE) is_external_pid(rterm2wterm(RTERM,BASE)) -#define external_pid_node_rel(RTERM,BASE) external_pid_node(rterm2wterm(RTERM,BASE)) -#define external_pid_data_rel(RTERM,BASE) external_pid_data(rterm2wterm(RTERM,BASE)) - -#define is_binary_rel(RTERM,BASE) is_binary(rterm2wterm(RTERM,BASE)) -#define is_float_rel(RTERM,BASE) is_float(rterm2wterm(RTERM,BASE)) -#define is_fun_rel(RTERM,BASE) is_fun(rterm2wterm(RTERM,BASE)) -#define is_big_rel(RTERM,BASE) is_big(rterm2wterm(RTERM,BASE)) -#define is_export_rel(RTERM,BASE) is_export(rterm2wterm(RTERM,BASE)) -#define is_tuple_rel(RTERM,BASE) is_tuple(rterm2wterm(RTERM,BASE)) - -#define GET_DOUBLE_REL(RTERM, f, BASE) GET_DOUBLE(rterm2wterm(RTERM,BASE), f) - -#define ref_thing_ptr_rel(RTERM,BASE) ref_thing_ptr(rterm2wterm(RTERM,BASE)) -#define is_internal_ref_rel(RTERM,BASE) is_internal_ref(rterm2wterm(RTERM,BASE)) -#define is_external_rel(RTERM,BASE) is_external(rterm2wterm(RTERM,BASE)) -#define is_external_port_rel(RTERM,BASE) is_external_port(rterm2wterm(RTERM,BASE)) -#define is_external_ref_rel(RTERM,BASE) is_external_ref(rterm2wterm(RTERM,BASE)) - -#define external_node_rel(RTERM,BASE) external_node(rterm2wterm(RTERM,BASE)) - - -#if HALFWORD_HEAP -ERTS_GLB_INLINE int is_same(Eterm a, Eterm* a_base, Eterm b, Eterm* b_base); +#if ET_DEBUG +#define ET_ASSERT(expr,file,line) \ +do { \ + if (!(expr)) \ + erl_assert_error("TYPE ASSERTION: " #expr, __FUNCTION__, file, line); \ +} while(0) +#else +#define ET_ASSERT(expr,file,line) do { } while(0) +#endif #if ERTS_GLB_INLINE_INCL_FUNC_DEF -ERTS_GLB_INLINE int is_same(Eterm a, Eterm* a_base, Eterm b, Eterm* b_base) -{ - /* If bases differ, assume a and b are on different "heaps", - ie can only be same if immed */ - ASSERT(a_base == b_base || is_immed(a) || is_immed(b) - || rterm2wterm(a,a_base) != rterm2wterm(b,b_base)); - return a == b && (a_base == b_base || is_immed(a)); +#if ET_DEBUG +ERTS_GLB_INLINE unsigned tag_val_def(Wterm x, const char *file, unsigned line) +#else +ERTS_GLB_INLINE unsigned tag_val_def(Wterm x) +#define file __FILE__ +#define line __LINE__ +#endif +{ + static char *msg = "tag_val_def error"; + + switch (x & _TAG_PRIMARY_MASK) { + case TAG_PRIMARY_LIST: + ET_ASSERT(_list_precond(x),file,line); + return LIST_DEF; + case TAG_PRIMARY_BOXED: { + Eterm hdr = *boxed_val(x); + ET_ASSERT(is_header(hdr),file,line); + switch ((hdr & _TAG_HEADER_MASK) >> _TAG_PRIMARY_SIZE) { + case (_TAG_HEADER_ARITYVAL >> _TAG_PRIMARY_SIZE): return TUPLE_DEF; + case (_TAG_HEADER_POS_BIG >> _TAG_PRIMARY_SIZE): return BIG_DEF; + case (_TAG_HEADER_NEG_BIG >> _TAG_PRIMARY_SIZE): return BIG_DEF; + case (_TAG_HEADER_REF >> _TAG_PRIMARY_SIZE): return REF_DEF; + case (_TAG_HEADER_FLOAT >> _TAG_PRIMARY_SIZE): return FLOAT_DEF; + case (_TAG_HEADER_EXPORT >> _TAG_PRIMARY_SIZE): return EXPORT_DEF; + case (_TAG_HEADER_FUN >> _TAG_PRIMARY_SIZE): return FUN_DEF; + case (_TAG_HEADER_EXTERNAL_PID >> _TAG_PRIMARY_SIZE): return EXTERNAL_PID_DEF; + case (_TAG_HEADER_EXTERNAL_PORT >> _TAG_PRIMARY_SIZE): return EXTERNAL_PORT_DEF; + case (_TAG_HEADER_EXTERNAL_REF >> _TAG_PRIMARY_SIZE): return EXTERNAL_REF_DEF; + case (_TAG_HEADER_MAP >> _TAG_PRIMARY_SIZE): return MAP_DEF; + case (_TAG_HEADER_REFC_BIN >> _TAG_PRIMARY_SIZE): return BINARY_DEF; + case (_TAG_HEADER_HEAP_BIN >> _TAG_PRIMARY_SIZE): return BINARY_DEF; + case (_TAG_HEADER_SUB_BIN >> _TAG_PRIMARY_SIZE): return BINARY_DEF; + case (_TAG_HEADER_BIN_MATCHSTATE >> _TAG_PRIMARY_SIZE): return MATCHSTATE_DEF; + } + + break; + } + case TAG_PRIMARY_IMMED1: { + switch ((x & _TAG_IMMED1_MASK) >> _TAG_PRIMARY_SIZE) { + case (_TAG_IMMED1_PID >> _TAG_PRIMARY_SIZE): return PID_DEF; + case (_TAG_IMMED1_PORT >> _TAG_PRIMARY_SIZE): return PORT_DEF; + case (_TAG_IMMED1_IMMED2 >> _TAG_PRIMARY_SIZE): { + switch ((x & _TAG_IMMED2_MASK) >> _TAG_IMMED1_SIZE) { + case (_TAG_IMMED2_ATOM >> _TAG_IMMED1_SIZE): return ATOM_DEF; + case (_TAG_IMMED2_NIL >> _TAG_IMMED1_SIZE): return NIL_DEF; + } + break; + } + case (_TAG_IMMED1_SMALL >> _TAG_PRIMARY_SIZE): return SMALL_DEF; + } + break; + } + } + erl_assert_error(msg, __FUNCTION__, file, line); +#undef file +#undef line } #endif -#else /* !HALFWORD_HEAP */ -#define is_same(A,A_BASE,B,B_BASE) ((A)==(B)) +#if ET_DEBUG +#define tag_val_def(X) tag_val_def(X, __FILE__, __LINE__) #endif #endif /* __ERL_TERM_H */ diff --git a/erts/emulator/beam/erl_thr_progress.c b/erts/emulator/beam/erl_thr_progress.c index 545a0343d0..7b06fd840f 100644 --- a/erts/emulator/beam/erl_thr_progress.c +++ b/erts/emulator/beam/erl_thr_progress.c @@ -3,16 +3,17 @@ * * Copyright Ericsson AB 2011-2013. All Rights Reserved. * - * The contents of this file are subject to the Erlang Public License, - * Version 1.1, (the "License"); you may not use this file except in - * compliance with the License. You should have received a copy of the - * Erlang Public License along with this software. If not, it can be - * retrieved online at http://www.erlang.org/. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Software distributed under the License is distributed on an "AS IS" - * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See - * the License for the specific language governing rights and limitations - * under the License. + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. * * %CopyrightEnd% */ @@ -115,70 +116,24 @@ #undef read_nob #define read_nob erts_thr_prgr_read_nob__ -#ifdef ARCH_64 - -static ERTS_INLINE void -set_mb(ERTS_THR_PRGR_ATOMIC *atmc, ErtsThrPrgrVal val) -{ - erts_atomic_set_mb(atmc, val); -} - -static ERTS_INLINE void -set_nob(ERTS_THR_PRGR_ATOMIC *atmc, ErtsThrPrgrVal val) -{ - erts_atomic_set_nob(atmc, val); -} - -static ERTS_INLINE void -init_nob(ERTS_THR_PRGR_ATOMIC *atmc, ErtsThrPrgrVal val) -{ - erts_atomic_init_nob(atmc, val); -} - -#else - -#undef dw_aint_to_val -#define dw_aint_to_val erts_thr_prgr_dw_aint_to_val__ - -static void -val_to_dw_aint(erts_dw_aint_t *dw_aint, ErtsThrPrgrVal val) -{ -#ifdef ETHR_SU_DW_NAINT_T__ - dw_aint->dw_sint = (ETHR_SU_DW_NAINT_T__) val; -#else - dw_aint->sint[ERTS_DW_AINT_LOW_WORD] - = (erts_aint_t) (val & 0xffffffff); - dw_aint->sint[ERTS_DW_AINT_HIGH_WORD] - = (erts_aint_t) ((val >> 32) & 0xffffffff); -#endif -} - static ERTS_INLINE void set_mb(ERTS_THR_PRGR_ATOMIC *atmc, ErtsThrPrgrVal val) { - erts_dw_aint_t dw_aint; - val_to_dw_aint(&dw_aint, val); - erts_dw_atomic_set_mb(atmc, &dw_aint); + erts_atomic64_set_mb(atmc, (erts_aint64_t) val); } static ERTS_INLINE void set_nob(ERTS_THR_PRGR_ATOMIC *atmc, ErtsThrPrgrVal val) { - erts_dw_aint_t dw_aint; - val_to_dw_aint(&dw_aint, val); - erts_dw_atomic_set_nob(atmc, &dw_aint); + erts_atomic64_set_nob(atmc, (erts_aint64_t) val); } static ERTS_INLINE void init_nob(ERTS_THR_PRGR_ATOMIC *atmc, ErtsThrPrgrVal val) { - erts_dw_aint_t dw_aint; - val_to_dw_aint(&dw_aint, val); - erts_dw_atomic_init_nob(atmc, &dw_aint); + erts_atomic64_init_nob(atmc, (erts_aint64_t) val); } -#endif - /* #define ERTS_THR_PROGRESS_STATE_DEBUG */ #ifdef ERTS_THR_PROGRESS_STATE_DEBUG @@ -547,7 +502,7 @@ erts_thr_progress_register_unmanaged_thread(ErtsThrPrgrCallbacks *callbacks) if (tpd) { if (!tpd->is_temporary) - erl_exit(ERTS_ABORT_EXIT, + erts_exit(ERTS_ABORT_EXIT, "%s:%d:%s(): Double register of thread\n", __FILE__, __LINE__, __func__); is_blocking = tpd->is_blocking; @@ -569,7 +524,7 @@ erts_thr_progress_register_unmanaged_thread(ErtsThrPrgrCallbacks *callbacks) #endif ASSERT(tpd->id >= 0); if (tpd->id >= intrnl->unmanaged.no) - erl_exit(ERTS_ABORT_EXIT, + erts_exit(ERTS_ABORT_EXIT, "%s:%d:%s(): Too many unmanaged registered threads\n", __FILE__, __LINE__, __func__); @@ -592,7 +547,7 @@ erts_thr_progress_register_managed_thread(ErtsSchedulerData *esdp, if (tpd) { if (!tpd->is_temporary) - erl_exit(ERTS_ABORT_EXIT, + erts_exit(ERTS_ABORT_EXIT, "%s:%d:%s(): Double register of thread\n", __FILE__, __LINE__, __func__); is_blocking = tpd->is_blocking; @@ -613,7 +568,7 @@ erts_thr_progress_register_managed_thread(ErtsSchedulerData *esdp, tpd->id = erts_atomic32_inc_read_nob(&intrnl->misc.data.managed_id); ASSERT(tpd->id >= 0); if (tpd->id >= intrnl->managed.no) - erl_exit(ERTS_ABORT_EXIT, + erts_exit(ERTS_ABORT_EXIT, "%s:%d:%s(): Too many managed registered threads\n", __FILE__, __LINE__, __func__); @@ -1078,7 +1033,7 @@ has_reached_wakeup(ErtsThrPrgrVal wakeup) limit += 1; if (!erts_thr_progress_has_passed__(limit, wakeup)) - erl_exit(ERTS_ABORT_EXIT, + erts_exit(ERTS_ABORT_EXIT, "Invalid wakeup request value found:" " current=%b64u, wakeup=%b64u, limit=%b64u", current, wakeup, limit); @@ -1147,7 +1102,7 @@ request_wakeup_managed(ErtsThrPrgrData *tpd, ErtsThrPrgrVal value) ix = erts_atomic32_inc_read_nob(&mwd->len) - 1; #if ERTS_THR_PRGR_DBG_CHK_WAKEUP_REQUEST_VALUE if (ix >= intrnl->managed.no) - erl_exit(ERTS_ABORT_EXIT, "Internal error: Too many wakeup requests\n"); + erts_exit(ERTS_ABORT_EXIT, "Internal error: Too many wakeup requests\n"); #endif mwd->id[ix] = tpd->id; @@ -1381,25 +1336,10 @@ erts_thr_progress_block(void) thr_progress_block(tmp_thr_prgr_data(NULL), 1); } -void -erts_thr_progress_fatal_error_block(SWord timeout, - ErtsThrPrgrData *tmp_tpd_bufp) +int +erts_thr_progress_fatal_error_block(ErtsThrPrgrData *tmp_tpd_bufp) { ErtsThrPrgrData *tpd = perhaps_thr_prgr_data(NULL); - erts_aint32_t bc; - SWord time_left = timeout; - SysTimeval to; - - /* - * Counting poll intervals may give us a too long timeout - * if cpu is busy. If we got tolerant time of day we use it - * to prevent this. - */ - if (!erts_disable_tolerant_timeofday) { - erts_get_timeval(&to); - to.tv_sec += timeout / 1000; - to.tv_sec += timeout % 1000; - } if (!tpd) { /* @@ -1412,9 +1352,25 @@ erts_thr_progress_fatal_error_block(SWord timeout, init_tmp_thr_prgr_data(tpd); } - bc = thr_progress_block(tpd, 0); - if (bc == 0) - return; /* Succefully blocked all managed threads */ + /* Returns number of threads that have not yes been blocked */ + return thr_progress_block(tpd, 0); +} + +void +erts_thr_progress_fatal_error_wait(SWord timeout) { + erts_aint32_t bc; + SWord time_left = timeout; + ErtsMonotonicTime timeout_time; + ErtsSchedulerData *esdp = erts_get_scheduler_data(); + + /* + * Counting poll intervals may give us a too long timeout + * if cpu is busy. We use timeout time to try to prevent + * this. In case we havn't got time correction this may + * however fail too... + */ + timeout_time = erts_get_monotonic_time(esdp); + timeout_time += ERTS_MSEC_TO_MONOTONIC((ErtsMonotonicTime) timeout); while (1) { if (erts_milli_sleep(ERTS_THR_PRGR_FTL_ERR_BLCK_POLL_INTERVAL) == 0) @@ -1424,14 +1380,8 @@ erts_thr_progress_fatal_error_block(SWord timeout, break; /* Succefully blocked all managed threads */ if (time_left <= 0) break; /* Timeout */ - if (!erts_disable_tolerant_timeofday) { - SysTimeval now; - erts_get_timeval(&now); - if (now.tv_sec > to.tv_sec) - break; /* Timeout */ - if (now.tv_sec == to.tv_sec && now.tv_usec >= to.tv_usec) - break; /* Timeout */ - } + if (timeout_time <= erts_get_monotonic_time(esdp)) + break; /* Timeout */ } } diff --git a/erts/emulator/beam/erl_thr_progress.h b/erts/emulator/beam/erl_thr_progress.h index 5f392944c2..b89cc4c267 100644 --- a/erts/emulator/beam/erl_thr_progress.h +++ b/erts/emulator/beam/erl_thr_progress.h @@ -3,16 +3,17 @@ * * Copyright Ericsson AB 2011-2013. All Rights Reserved. * - * The contents of this file are subject to the Erlang Public License, - * Version 1.1, (the "License"); you may not use this file except in - * compliance with the License. You should have received a copy of the - * Erlang Public License along with this software. If not, it can be - * retrieved online at http://www.erlang.org/. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Software distributed under the License is distributed on an "AS IS" - * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See - * the License for the specific language governing rights and limitations - * under the License. + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. * * %CopyrightEnd% */ @@ -83,8 +84,8 @@ typedef struct { ErtsThrPrgrLeaderState leader_state; } ErtsThrPrgrData; -void erts_thr_progress_fatal_error_block(SWord timeout, - ErtsThrPrgrData *tmp_tpd_bufp); +int erts_thr_progress_fatal_error_block(ErtsThrPrgrData *tmp_tpd_bufp); +void erts_thr_progress_fatal_error_wait(SWord timeout); #endif /* ERTS_SMP */ @@ -115,11 +116,7 @@ struct ErtsThrPrgrLaterOp_ { extern erts_tsd_key_t erts_thr_prgr_data_key__; -#ifdef ARCH_64 -# define ERTS_THR_PRGR_ATOMIC erts_atomic_t -#else /* ARCH_32 */ -# define ERTS_THR_PRGR_ATOMIC erts_dw_atomic_t -#endif +#define ERTS_THR_PRGR_ATOMIC erts_atomic64_t typedef struct { void *arg; @@ -158,10 +155,6 @@ void erts_thr_progress_unmanaged_continue__(int umrefc_ix); void erts_thr_progress_dbg_print_state(void); -#ifdef ARCH_32 -#define ERTS_THR_PRGR_ATOMIC erts_dw_atomic_t -ERTS_GLB_INLINE ErtsThrPrgrVal erts_thr_prgr_dw_aint_to_val__(erts_dw_aint_t *dw_aint); -#endif ERTS_GLB_INLINE ErtsThrPrgrVal erts_thr_prgr_read_nob__(ERTS_THR_PRGR_ATOMIC *atmc); ERTS_GLB_INLINE ErtsThrPrgrVal erts_thr_prgr_read_acqb__(ERTS_THR_PRGR_ATOMIC *atmc); ERTS_GLB_INLINE ErtsThrPrgrVal erts_thr_prgr_read_mb__(ERTS_THR_PRGR_ATOMIC *atmc); @@ -184,68 +177,24 @@ ERTS_GLB_INLINE int erts_thr_progress_has_reached(ErtsThrPrgrVal val); #if ERTS_GLB_INLINE_INCL_FUNC_DEF -#ifdef ARCH_64 - ERTS_GLB_INLINE ErtsThrPrgrVal erts_thr_prgr_read_nob__(ERTS_THR_PRGR_ATOMIC *atmc) { - return (ErtsThrPrgrVal) erts_atomic_read_nob(atmc); + return (ErtsThrPrgrVal) erts_atomic64_read_nob(atmc); } ERTS_GLB_INLINE ErtsThrPrgrVal erts_thr_prgr_read_acqb__(ERTS_THR_PRGR_ATOMIC *atmc) { - return (ErtsThrPrgrVal) erts_atomic_read_acqb(atmc); + return (ErtsThrPrgrVal) erts_atomic64_read_acqb(atmc); } ERTS_GLB_INLINE ErtsThrPrgrVal erts_thr_prgr_read_mb__(ERTS_THR_PRGR_ATOMIC *atmc) { - return (ErtsThrPrgrVal) erts_atomic_read_mb(atmc); -} - -#else /* ARCH_32 */ - -ERTS_GLB_INLINE ErtsThrPrgrVal -erts_thr_prgr_dw_aint_to_val__(erts_dw_aint_t *dw_aint) -{ -#ifdef ETHR_SU_DW_NAINT_T__ - return (ErtsThrPrgrVal) dw_aint->dw_sint; -#else - ErtsThrPrgrVal res; - res = (ErtsThrPrgrVal) ((Uint32) dw_aint->sint[ERTS_DW_AINT_HIGH_WORD]); - res <<= 32; - res |= (ErtsThrPrgrVal) ((Uint32) dw_aint->sint[ERTS_DW_AINT_LOW_WORD]); - return res; -#endif + return (ErtsThrPrgrVal) erts_atomic64_read_mb(atmc); } -ERTS_GLB_INLINE ErtsThrPrgrVal -erts_thr_prgr_read_nob__(ERTS_THR_PRGR_ATOMIC *atmc) -{ - erts_dw_aint_t dw_aint; - erts_dw_atomic_read_nob(atmc, &dw_aint); - return erts_thr_prgr_dw_aint_to_val__(&dw_aint); -} - -ERTS_GLB_INLINE ErtsThrPrgrVal -erts_thr_prgr_read_acqb__(ERTS_THR_PRGR_ATOMIC *atmc) -{ - erts_dw_aint_t dw_aint; - erts_dw_atomic_read_acqb(atmc, &dw_aint); - return erts_thr_prgr_dw_aint_to_val__(&dw_aint); -} - -ERTS_GLB_INLINE ErtsThrPrgrVal -erts_thr_prgr_read_mb__(ERTS_THR_PRGR_ATOMIC *atmc) -{ - erts_dw_aint_t dw_aint; - erts_dw_atomic_read_mb(atmc, &dw_aint); - return erts_thr_prgr_dw_aint_to_val__(&dw_aint); -} - -#endif - ERTS_GLB_INLINE int erts_thr_progress_is_managed_thread(void) { diff --git a/erts/emulator/beam/erl_thr_queue.c b/erts/emulator/beam/erl_thr_queue.c index f8ca87ddcc..7ff456b915 100644 --- a/erts/emulator/beam/erl_thr_queue.c +++ b/erts/emulator/beam/erl_thr_queue.c @@ -3,16 +3,17 @@ * * Copyright Ericsson AB 2011-2013. All Rights Reserved. * - * The contents of this file are subject to the Erlang Public License, - * Version 1.1, (the "License"); you may not use this file except in - * compliance with the License. You should have received a copy of the - * Erlang Public License along with this software. If not, it can be - * retrieved online at http://www.erlang.org/. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Software distributed under the License is distributed on an "AS IS" - * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See - * the License for the specific language governing rights and limitations - * under the License. + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. * * %CopyrightEnd% */ @@ -223,7 +224,7 @@ ErtsThrQCleanState_t erts_thr_q_destroy(ErtsThrQ_t *q) { if (!q->q.blk) - erl_exit(ERTS_ABORT_EXIT, + erts_exit(ERTS_ABORT_EXIT, "Trying to destroy not created thread queue\n"); return erts_thr_q_finalize(q); } @@ -588,7 +589,7 @@ enqueue(ErtsThrQ_t *q, void *data, ErtsThrQElement_t *this) #if ERTS_THR_Q_DBG_CHK_DATA if (!data) - erl_exit(ERTS_ABORT_EXIT, "Missing data in enqueue\n"); + erts_exit(ERTS_ABORT_EXIT, "Missing data in enqueue\n"); #endif ASSERT(!q->q.finalizing); @@ -770,7 +771,7 @@ erts_thr_q_dequeue(ErtsThrQ_t *q) #if ERTS_THR_Q_DBG_CHK_DATA head->data.ptr = NULL; if (!res) - erl_exit(ERTS_ABORT_EXIT, "Missing data in dequeue\n"); + erts_exit(ERTS_ABORT_EXIT, "Missing data in dequeue\n"); #endif clean(q, (q->head.deq_fini.automatic diff --git a/erts/emulator/beam/erl_thr_queue.h b/erts/emulator/beam/erl_thr_queue.h index 13af758b3f..27a6d03224 100644 --- a/erts/emulator/beam/erl_thr_queue.h +++ b/erts/emulator/beam/erl_thr_queue.h @@ -3,16 +3,17 @@ * * Copyright Ericsson AB 2011-2013. All Rights Reserved. * - * The contents of this file are subject to the Erlang Public License, - * Version 1.1, (the "License"); you may not use this file except in - * compliance with the License. You should have received a copy of the - * Erlang Public License along with this software. If not, it can be - * retrieved online at http://www.erlang.org/. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Software distributed under the License is distributed on an "AS IS" - * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See - * the License for the specific language governing rights and limitations - * under the License. + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. * * %CopyrightEnd% */ diff --git a/erts/emulator/beam/erl_threads.h b/erts/emulator/beam/erl_threads.h index 80026104db..e689547d1d 100644 --- a/erts/emulator/beam/erl_threads.h +++ b/erts/emulator/beam/erl_threads.h @@ -3,16 +3,17 @@ * * Copyright Ericsson AB 2001-2013. All Rights Reserved. * - * The contents of this file are subject to the Erlang Public License, - * Version 1.1, (the "License"); you may not use this file except in - * compliance with the License. You should have received a copy of the - * Erlang Public License along with this software. If not, it can be - * retrieved online at http://www.erlang.org/. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Software distributed under the License is distributed on an "AS IS" - * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See - * the License for the specific language governing rights and limitations - * under the License. + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. * * %CopyrightEnd% */ @@ -344,6 +345,16 @@ typedef ethr_ts_event erts_tse_t; #define erts_aint32_t ethr_sint32_t #define erts_atomic32_t ethr_atomic32_t +#if defined(ARCH_32) +# define erts_atomic64_t ethr_dw_atomic_t +# define erts_aint64_t ethr_sint64_t +#elif defined(ARCH_64) +# define erts_atomic64_t ethr_atomic_t +# define erts_aint64_t ethr_sint_t +#else +# error "Not supported architecture" +#endif + #define ERTS_DW_AINT_HIGH_WORD ETHR_DW_SINT_HIGH_WORD #define ERTS_DW_AINT_LOW_WORD ETHR_DW_SINT_LOW_WORD @@ -414,10 +425,12 @@ typedef int erts_tse_t; typedef struct { SWord sint[2]; } erts_dw_aint_t; typedef SWord erts_aint_t; typedef Sint32 erts_aint32_t; +typedef Sint64 erts_aint64_t; #define erts_dw_atomic_t erts_dw_aint_t #define erts_atomic_t erts_aint_t #define erts_atomic32_t erts_aint32_t +#define erts_atomic64_t erts_aint64_t #if __GNUC__ > 2 typedef struct { } erts_spinlock_t; @@ -446,6 +459,7 @@ typedef struct { int gcc_is_buggy; } erts_rwlock_t; #define erts_no_dw_atomic_t erts_dw_aint_t #define erts_no_atomic_t erts_aint_t #define erts_no_atomic32_t erts_aint32_t +#define erts_no_atomic64_t erts_aint64_t #define ERTS_AINT_NULL ((erts_aint_t) NULL) @@ -463,6 +477,7 @@ ERTS_GLB_INLINE void erts_thr_detach(erts_tid_t tid); ERTS_GLB_INLINE void erts_thr_exit(void *res); ERTS_GLB_INLINE void erts_thr_install_exit_handler(void (*exit_handler)(void)); ERTS_GLB_INLINE erts_tid_t erts_thr_self(void); +ERTS_GLB_INLINE int erts_thr_getname(erts_tid_t tid, char *buf, size_t len); ERTS_GLB_INLINE int erts_equal_tids(erts_tid_t x, erts_tid_t y); ERTS_GLB_INLINE void erts_mtx_init_x(erts_mtx_t *mtx, char *name, Eterm extra, int enable_lcnt); @@ -570,6 +585,29 @@ ERTS_GLB_INLINE erts_aint32_t erts_no_atomic32_cmpxchg(erts_no_atomic32_t *xchgp ERTS_GLB_INLINE erts_aint32_t erts_no_atomic32_read_bset(erts_no_atomic32_t *var, erts_aint32_t mask, erts_aint32_t set); +ERTS_GLB_INLINE void erts_no_atomic64_set(erts_no_atomic64_t *var, + erts_aint64_t i); +ERTS_GLB_INLINE erts_aint64_t erts_no_atomic64_read(erts_no_atomic64_t *var); +ERTS_GLB_INLINE erts_aint64_t erts_no_atomic64_inc_read(erts_no_atomic64_t *incp); +ERTS_GLB_INLINE erts_aint64_t erts_no_atomic64_dec_read(erts_no_atomic64_t *decp); +ERTS_GLB_INLINE void erts_no_atomic64_inc(erts_no_atomic64_t *incp); +ERTS_GLB_INLINE void erts_no_atomic64_dec(erts_no_atomic64_t *decp); +ERTS_GLB_INLINE erts_aint64_t erts_no_atomic64_add_read(erts_no_atomic64_t *addp, + erts_aint64_t i); +ERTS_GLB_INLINE void erts_no_atomic64_add(erts_no_atomic64_t *addp, + erts_aint64_t i); +ERTS_GLB_INLINE erts_aint64_t erts_no_atomic64_read_bor(erts_no_atomic64_t *var, + erts_aint64_t mask); +ERTS_GLB_INLINE erts_aint64_t erts_no_atomic64_read_band(erts_no_atomic64_t *var, + erts_aint64_t mask); +ERTS_GLB_INLINE erts_aint64_t erts_no_atomic64_xchg(erts_no_atomic64_t *xchgp, + erts_aint64_t new); +ERTS_GLB_INLINE erts_aint64_t erts_no_atomic64_cmpxchg(erts_no_atomic64_t *xchgp, + erts_aint64_t new, + erts_aint64_t expected); +ERTS_GLB_INLINE erts_aint64_t erts_no_atomic64_read_bset(erts_no_atomic64_t *var, + erts_aint64_t mask, + erts_aint64_t set); ERTS_GLB_INLINE void erts_spinlock_init_x_opt(erts_spinlock_t *lock, char *name, @@ -611,20 +649,29 @@ ERTS_GLB_INLINE void erts_tsd_set(erts_tsd_key_t key, void *value); ERTS_GLB_INLINE void * erts_tsd_get(erts_tsd_key_t key); ERTS_GLB_INLINE erts_tse_t *erts_tse_fetch(void); ERTS_GLB_INLINE void erts_tse_return(erts_tse_t *ep); +ERTS_GLB_INLINE void erts_tse_prepare_timed(erts_tse_t *ep); ERTS_GLB_INLINE void erts_tse_set(erts_tse_t *ep); ERTS_GLB_INLINE void erts_tse_reset(erts_tse_t *ep); ERTS_GLB_INLINE int erts_tse_wait(erts_tse_t *ep); ERTS_GLB_INLINE int erts_tse_swait(erts_tse_t *ep, int spincount); +ERTS_GLB_INLINE int erts_tse_twait(erts_tse_t *ep, Sint64 tmo); +ERTS_GLB_INLINE int erts_tse_stwait(erts_tse_t *ep, int spincount, Sint64 tmo); ERTS_GLB_INLINE int erts_tse_is_tmp(erts_tse_t *ep); ERTS_GLB_INLINE void erts_thr_set_main_status(int, int); ERTS_GLB_INLINE int erts_thr_get_main_status(void); ERTS_GLB_INLINE void erts_thr_yield(void); + #ifdef ETHR_HAVE_ETHR_SIG_FUNCS #define ERTS_THR_HAVE_SIG_FUNCS 1 ERTS_GLB_INLINE void erts_thr_sigmask(int how, const sigset_t *set, sigset_t *oset); ERTS_GLB_INLINE void erts_thr_sigwait(const sigset_t *set, int *sig); + +#ifdef USE_THREADS +ERTS_GLB_INLINE void erts_thr_kill(erts_tid_t tid, int sig); +#endif + #endif /* #ifdef HAVE_ETHR_SIG_FUNCS */ #ifdef USE_THREADS @@ -1200,6 +1247,441 @@ erts_atomic32_read_dirty(erts_atomic32_t *var) #endif +/* 64-bit atomics */ + +#if defined(ARCH_64) + +#define erts_atomic64_init_nob ethr_atomic_init +#define erts_atomic64_set_nob ethr_atomic_set +#define erts_atomic64_read_nob ethr_atomic_read +#define erts_atomic64_inc_read_nob ethr_atomic_inc_read +#define erts_atomic64_dec_read_nob ethr_atomic_dec_read +#define erts_atomic64_inc_nob ethr_atomic_inc +#define erts_atomic64_dec_nob ethr_atomic_dec +#define erts_atomic64_add_read_nob ethr_atomic_add_read +#define erts_atomic64_add_nob ethr_atomic_add +#define erts_atomic64_read_bor_nob ethr_atomic_read_bor +#define erts_atomic64_read_band_nob ethr_atomic_read_band +#define erts_atomic64_xchg_nob ethr_atomic_xchg +#define erts_atomic64_cmpxchg_nob ethr_atomic_cmpxchg +#define erts_atomic64_read_bset_nob erts_atomic_read_bset_nob + +#define erts_atomic64_init_mb ethr_atomic_init_mb +#define erts_atomic64_set_mb ethr_atomic_set_mb +#define erts_atomic64_read_mb ethr_atomic_read_mb +#define erts_atomic64_inc_read_mb ethr_atomic_inc_read_mb +#define erts_atomic64_dec_read_mb ethr_atomic_dec_read_mb +#define erts_atomic64_inc_mb ethr_atomic_inc_mb +#define erts_atomic64_dec_mb ethr_atomic_dec_mb +#define erts_atomic64_add_read_mb ethr_atomic_add_read_mb +#define erts_atomic64_add_mb ethr_atomic_add_mb +#define erts_atomic64_read_bor_mb ethr_atomic_read_bor_mb +#define erts_atomic64_read_band_mb ethr_atomic_read_band_mb +#define erts_atomic64_xchg_mb ethr_atomic_xchg_mb +#define erts_atomic64_cmpxchg_mb ethr_atomic_cmpxchg_mb +#define erts_atomic64_read_bset_mb erts_atomic_read_bset_mb + +#define erts_atomic64_init_acqb ethr_atomic_init_acqb +#define erts_atomic64_set_acqb ethr_atomic_set_acqb +#define erts_atomic64_read_acqb ethr_atomic_read_acqb +#define erts_atomic64_inc_read_acqb ethr_atomic_inc_read_acqb +#define erts_atomic64_dec_read_acqb ethr_atomic_dec_read_acqb +#define erts_atomic64_inc_acqb ethr_atomic_inc_acqb +#define erts_atomic64_dec_acqb ethr_atomic_dec_acqb +#define erts_atomic64_add_read_acqb ethr_atomic_add_read_acqb +#define erts_atomic64_add_acqb ethr_atomic_add_acqb +#define erts_atomic64_read_bor_acqb ethr_atomic_read_bor_acqb +#define erts_atomic64_read_band_acqb ethr_atomic_read_band_acqb +#define erts_atomic64_xchg_acqb ethr_atomic_xchg_acqb +#define erts_atomic64_cmpxchg_acqb ethr_atomic_cmpxchg_acqb +#define erts_atomic64_read_bset_acqb erts_atomic_read_bset_acqb + +#define erts_atomic64_init_relb ethr_atomic_init_relb +#define erts_atomic64_set_relb ethr_atomic_set_relb +#define erts_atomic64_read_relb ethr_atomic_read_relb +#define erts_atomic64_inc_read_relb ethr_atomic_inc_read_relb +#define erts_atomic64_dec_read_relb ethr_atomic_dec_read_relb +#define erts_atomic64_inc_relb ethr_atomic_inc_relb +#define erts_atomic64_dec_relb ethr_atomic_dec_relb +#define erts_atomic64_add_read_relb ethr_atomic_add_read_relb +#define erts_atomic64_add_relb ethr_atomic_add_relb +#define erts_atomic64_read_bor_relb ethr_atomic_read_bor_relb +#define erts_atomic64_read_band_relb ethr_atomic_read_band_relb +#define erts_atomic64_xchg_relb ethr_atomic_xchg_relb +#define erts_atomic64_cmpxchg_relb ethr_atomic_cmpxchg_relb +#define erts_atomic64_read_bset_relb erts_atomic_read_bset_relb + +#define erts_atomic64_init_ddrb ethr_atomic_init_ddrb +#define erts_atomic64_set_ddrb ethr_atomic_set_ddrb +#define erts_atomic64_read_ddrb ethr_atomic_read_ddrb +#define erts_atomic64_inc_read_ddrb ethr_atomic_inc_read_ddrb +#define erts_atomic64_dec_read_ddrb ethr_atomic_dec_read_ddrb +#define erts_atomic64_inc_ddrb ethr_atomic_inc_ddrb +#define erts_atomic64_dec_ddrb ethr_atomic_dec_ddrb +#define erts_atomic64_add_read_ddrb ethr_atomic_add_read_ddrb +#define erts_atomic64_add_ddrb ethr_atomic_add_ddrb +#define erts_atomic64_read_bor_ddrb ethr_atomic_read_bor_ddrb +#define erts_atomic64_read_band_ddrb ethr_atomic_read_band_ddrb +#define erts_atomic64_xchg_ddrb ethr_atomic_xchg_ddrb +#define erts_atomic64_cmpxchg_ddrb ethr_atomic_cmpxchg_ddrb +#define erts_atomic64_read_bset_ddrb erts_atomic_read_bset_ddrb + +#define erts_atomic64_init_rb ethr_atomic_init_rb +#define erts_atomic64_set_rb ethr_atomic_set_rb +#define erts_atomic64_read_rb ethr_atomic_read_rb +#define erts_atomic64_inc_read_rb ethr_atomic_inc_read_rb +#define erts_atomic64_dec_read_rb ethr_atomic_dec_read_rb +#define erts_atomic64_inc_rb ethr_atomic_inc_rb +#define erts_atomic64_dec_rb ethr_atomic_dec_rb +#define erts_atomic64_add_read_rb ethr_atomic_add_read_rb +#define erts_atomic64_add_rb ethr_atomic_add_rb +#define erts_atomic64_read_bor_rb ethr_atomic_read_bor_rb +#define erts_atomic64_read_band_rb ethr_atomic_read_band_rb +#define erts_atomic64_xchg_rb ethr_atomic_xchg_rb +#define erts_atomic64_cmpxchg_rb ethr_atomic_cmpxchg_rb +#define erts_atomic64_read_bset_rb erts_atomic_read_bset_rb + +#define erts_atomic64_init_wb ethr_atomic_init_wb +#define erts_atomic64_set_wb ethr_atomic_set_wb +#define erts_atomic64_read_wb ethr_atomic_read_wb +#define erts_atomic64_inc_read_wb ethr_atomic_inc_read_wb +#define erts_atomic64_dec_read_wb ethr_atomic_dec_read_wb +#define erts_atomic64_inc_wb ethr_atomic_inc_wb +#define erts_atomic64_dec_wb ethr_atomic_dec_wb +#define erts_atomic64_add_read_wb ethr_atomic_add_read_wb +#define erts_atomic64_add_wb ethr_atomic_add_wb +#define erts_atomic64_read_bor_wb ethr_atomic_read_bor_wb +#define erts_atomic64_read_band_wb ethr_atomic_read_band_wb +#define erts_atomic64_xchg_wb ethr_atomic_xchg_wb +#define erts_atomic64_cmpxchg_wb ethr_atomic_cmpxchg_wb +#define erts_atomic64_read_bset_wb erts_atomic_read_bset_wb + +#define erts_atomic64_set_dirty erts_atomic_set_dirty +#define erts_atomic64_read_dirty erts_atomic_read_dirty + +#elif defined(ARCH_32) + +#undef ERTS_ATOMIC64_OPS_DECL__ + +#define ERTS_ATOMIC64_OPS_DECL__(BARRIER) \ +ERTS_GLB_INLINE void \ +erts_atomic64_init_ ## BARRIER(erts_atomic64_t *var, \ + erts_aint64_t val); \ +ERTS_GLB_INLINE void \ +erts_atomic64_set_ ## BARRIER(erts_atomic64_t *var, \ + erts_aint64_t val); \ +ERTS_GLB_INLINE erts_aint64_t \ +erts_atomic64_read_ ## BARRIER(erts_atomic64_t *var); \ +ERTS_GLB_INLINE erts_aint64_t \ +erts_atomic64_inc_read_ ## BARRIER(erts_atomic64_t *var); \ +ERTS_GLB_INLINE erts_aint64_t \ +erts_atomic64_dec_read_ ## BARRIER(erts_atomic64_t *var); \ +ERTS_GLB_INLINE void \ +erts_atomic64_inc_ ## BARRIER(erts_atomic64_t *var); \ +ERTS_GLB_INLINE void \ +erts_atomic64_dec_ ## BARRIER(erts_atomic64_t *var); \ +ERTS_GLB_INLINE erts_aint64_t \ +erts_atomic64_add_read_ ## BARRIER(erts_atomic64_t *var, \ + erts_aint64_t val); \ +ERTS_GLB_INLINE void \ +erts_atomic64_add_ ## BARRIER(erts_atomic64_t *var, \ + erts_aint64_t val); \ +ERTS_GLB_INLINE erts_aint64_t \ +erts_atomic64_read_bor_ ## BARRIER(erts_atomic64_t *var, \ + erts_aint64_t val); \ +ERTS_GLB_INLINE erts_aint64_t \ +erts_atomic64_read_band_ ## BARRIER(erts_atomic64_t *var, \ + erts_aint64_t val); \ +ERTS_GLB_INLINE erts_aint64_t \ +erts_atomic64_xchg_ ## BARRIER(erts_atomic64_t *var, \ + erts_aint64_t val); \ +ERTS_GLB_INLINE erts_aint64_t \ +erts_atomic64_cmpxchg_ ## BARRIER(erts_atomic64_t *var, \ + erts_aint64_t new, \ + erts_aint64_t exp); \ +ERTS_GLB_INLINE erts_aint64_t \ +erts_atomic64_read_bset_ ## BARRIER(erts_atomic64_t *var, \ + erts_aint64_t mask, \ + erts_aint64_t set) + +ERTS_ATOMIC64_OPS_DECL__(nob); +ERTS_ATOMIC64_OPS_DECL__(mb); +ERTS_ATOMIC64_OPS_DECL__(acqb); +ERTS_ATOMIC64_OPS_DECL__(relb); +ERTS_ATOMIC64_OPS_DECL__(ddrb); +ERTS_ATOMIC64_OPS_DECL__(rb); +ERTS_ATOMIC64_OPS_DECL__(wb); + +#undef ERTS_ATOMIC64_OPS_DECL__ + +ERTS_GLB_INLINE void +erts_atomic64_set_dirty(erts_atomic64_t *var, erts_aint64_t val); +ERTS_GLB_INLINE erts_aint64_t +erts_atomic64_read_dirty(erts_atomic64_t *var); + +#if ERTS_GLB_INLINE_INCL_FUNC_DEF + +/* + * The ethr_dw_atomic_*_nob() functions below + * are here to make it possible for the + * ERTS_ATOMIC64_OPS_IMPL__() to map erts + * barriers to ethread barriers... + */ +static ERTS_INLINE void +ethr_dw_atomic_init_nob(ethr_dw_atomic_t *var, + ethr_dw_sint_t *val) +{ + ethr_dw_atomic_init(var, val); +} + +static ERTS_INLINE void +ethr_dw_atomic_set_nob(ethr_dw_atomic_t *var, + ethr_dw_sint_t *val) +{ + ethr_dw_atomic_set(var, val); +} + +static ERTS_INLINE void +ethr_dw_atomic_read_nob(ethr_dw_atomic_t *var, + ethr_dw_sint_t *val) +{ + ethr_dw_atomic_read(var, val); +} + +static ERTS_INLINE int +ethr_dw_atomic_cmpxchg_nob(ethr_dw_atomic_t *var, + ethr_dw_sint_t *new, + ethr_dw_sint_t *xchg) +{ + return ethr_dw_atomic_cmpxchg(var, new, xchg); +} + +#undef ERTS_ATOMIC64_OPS_IMPL__ +#undef ERTS_ATOMIC64_DW_CMPXCHG_IMPL__ +#undef ERTS_DW_SINT_TO_AINT64__ +#undef ERTS_AINT64_TO_DW_SINT__ + +#ifdef ETHR_SU_DW_NAINT_T__ +#define ERTS_DW_SINT_TO_AINT64__(DW) \ + ((erts_aint64_t) DW.dw_sint) +#define ERTS_AINT64_TO_DW_SINT__(DW, AINT64) \ + (DW.dw_sint = (ETHR_SU_DW_NAINT_T__) AINT64) +#else /* !ETHR_SU_DW_NAINT_T__ */ +#define ERTS_DW_SINT_TO_AINT64__(DW) \ + ((((erts_aint64_t) DW.sint[ETHR_DW_SINT_HIGH_WORD]) << 32) \ + | (((erts_aint64_t) DW.sint[ETHR_DW_SINT_LOW_WORD]) \ + & ((erts_aint64_t) 0xffffffff))) +#define ERTS_AINT64_TO_DW_SINT__(DW, AINT64) \ + do { \ + DW.sint[ETHR_DW_SINT_LOW_WORD] = \ + (ethr_sint_t) (AINT64 & 0xffffffff); \ + DW.sint[ETHR_DW_SINT_HIGH_WORD] = \ + (ethr_sint_t) ((AINT64 >> 32) & 0xffffffff); \ + } while (0) +#endif /* !ETHR_SU_DW_NAINT_T__ */ + +#define ERTS_ATOMIC64_DW_CMPXCHG_IMPL__(CmpXchgOp, \ + AVarP, XchgVar, NewVar, \ + ModificationCode) \ +do { \ + ethr_dw_sint_t dw_xchg__, dw_new__; \ + ethr_dw_atomic_read(AVarP, &dw_xchg__); \ + do { \ + XchgVar = ERTS_DW_SINT_TO_AINT64__(dw_xchg__); \ + { \ + ModificationCode; \ + } \ + ERTS_AINT64_TO_DW_SINT__(dw_new__, NewVar); \ + } while (!CmpXchgOp((AVarP), &dw_new__, &dw_xchg__)); \ +} while (0) + +#define ERTS_ATOMIC64_OPS_IMPL__(BARRIER) \ + \ +ERTS_GLB_INLINE void \ +erts_atomic64_init_ ## BARRIER(erts_atomic64_t *var, \ + erts_aint64_t val) \ +{ \ + ethr_dw_sint_t dw; \ + ERTS_AINT64_TO_DW_SINT__(dw, val); \ + ethr_dw_atomic_init_ ## BARRIER(var, &dw); \ +} \ + \ +ERTS_GLB_INLINE void \ +erts_atomic64_set_ ## BARRIER(erts_atomic64_t *var, \ + erts_aint64_t val) \ +{ \ + ethr_dw_sint_t dw; \ + ERTS_AINT64_TO_DW_SINT__(dw, val); \ + ethr_dw_atomic_set_ ## BARRIER(var, &dw); \ +} \ + \ +ERTS_GLB_INLINE erts_aint64_t \ +erts_atomic64_read_ ## BARRIER(erts_atomic64_t *var) \ +{ \ + ethr_dw_sint_t dw; \ + ethr_dw_atomic_read_ ## BARRIER(var, &dw); \ + return ERTS_DW_SINT_TO_AINT64__(dw); \ +} \ + \ +ERTS_GLB_INLINE erts_aint64_t \ +erts_atomic64_inc_read_ ## BARRIER(erts_atomic64_t *var) \ +{ \ + erts_aint64_t xchg, new; \ + ERTS_ATOMIC64_DW_CMPXCHG_IMPL__(ethr_dw_atomic_cmpxchg_ ## BARRIER, \ + var, xchg, new, \ + new = xchg + 1); \ + return new; \ +} \ + \ +ERTS_GLB_INLINE erts_aint64_t \ +erts_atomic64_dec_read_ ## BARRIER(erts_atomic64_t *var) \ +{ \ + erts_aint64_t xchg, new; \ + ERTS_ATOMIC64_DW_CMPXCHG_IMPL__(ethr_dw_atomic_cmpxchg_ ## BARRIER, \ + var, xchg, new, \ + new = xchg - 1); \ + return new; \ +} \ + \ +ERTS_GLB_INLINE void \ +erts_atomic64_inc_ ## BARRIER(erts_atomic64_t *var) \ +{ \ + erts_aint64_t xchg, new; \ + ERTS_ATOMIC64_DW_CMPXCHG_IMPL__(ethr_dw_atomic_cmpxchg_ ## BARRIER, \ + var, xchg, new, \ + new = xchg + 1); \ +} \ + \ +ERTS_GLB_INLINE void \ +erts_atomic64_dec_ ## BARRIER(erts_atomic64_t *var) \ +{ \ + erts_aint64_t xchg, new; \ + ERTS_ATOMIC64_DW_CMPXCHG_IMPL__(ethr_dw_atomic_cmpxchg_ ## BARRIER, \ + var, xchg, new, \ + new = xchg - 1); \ +} \ + \ +ERTS_GLB_INLINE erts_aint64_t \ +erts_atomic64_add_read_ ## BARRIER(erts_atomic64_t *var, \ + erts_aint64_t val) \ +{ \ + erts_aint64_t xchg, new; \ + ERTS_ATOMIC64_DW_CMPXCHG_IMPL__(ethr_dw_atomic_cmpxchg_ ## BARRIER, \ + var, xchg, new, \ + new = xchg + val); \ + return new; \ +} \ + \ +ERTS_GLB_INLINE void \ +erts_atomic64_add_ ## BARRIER(erts_atomic64_t *var, \ + erts_aint64_t val) \ +{ \ + erts_aint64_t xchg, new; \ + ERTS_ATOMIC64_DW_CMPXCHG_IMPL__(ethr_dw_atomic_cmpxchg_ ## BARRIER, \ + var, xchg, new, \ + new = xchg + val); \ +} \ + \ +ERTS_GLB_INLINE erts_aint64_t \ +erts_atomic64_read_bor_ ## BARRIER(erts_atomic64_t *var, \ + erts_aint64_t val) \ +{ \ + erts_aint64_t xchg, new; \ + ERTS_ATOMIC64_DW_CMPXCHG_IMPL__(ethr_dw_atomic_cmpxchg_ ## BARRIER, \ + var, xchg, new, \ + new = xchg | val); \ + return xchg; \ +} \ + \ +ERTS_GLB_INLINE erts_aint64_t \ +erts_atomic64_read_band_ ## BARRIER(erts_atomic64_t *var, \ + erts_aint64_t val) \ +{ \ + erts_aint64_t xchg, new; \ + ERTS_ATOMIC64_DW_CMPXCHG_IMPL__(ethr_dw_atomic_cmpxchg_ ## BARRIER, \ + var, xchg, new, \ + new = xchg & val); \ + return xchg; \ +} \ + \ +ERTS_GLB_INLINE erts_aint64_t \ +erts_atomic64_xchg_ ## BARRIER(erts_atomic64_t *var, \ + erts_aint64_t val) \ +{ \ + erts_aint64_t xchg, new; \ + ERTS_ATOMIC64_DW_CMPXCHG_IMPL__(ethr_dw_atomic_cmpxchg_ ## BARRIER, \ + var, xchg, new, \ + new = val); \ + return xchg; \ +} \ + \ +ERTS_GLB_INLINE erts_aint64_t \ +erts_atomic64_cmpxchg_ ## BARRIER(erts_atomic64_t *var, \ + erts_aint64_t new, \ + erts_aint64_t exp) \ +{ \ + ethr_dw_sint_t dw_xchg, dw_new; \ + ERTS_AINT64_TO_DW_SINT__(dw_xchg, exp); \ + ERTS_AINT64_TO_DW_SINT__(dw_new, new); \ + if (ethr_dw_atomic_cmpxchg_ ## BARRIER(var, &dw_new, &dw_xchg)) \ + return exp; \ + return ERTS_DW_SINT_TO_AINT64__(dw_xchg); \ +} \ + \ +ERTS_GLB_INLINE erts_aint64_t \ +erts_atomic64_read_bset_ ## BARRIER(erts_atomic64_t *var, \ + erts_aint64_t mask, \ + erts_aint64_t set) \ +{ \ + erts_aint64_t xchg, new; \ + ERTS_ATOMIC64_DW_CMPXCHG_IMPL__(ethr_dw_atomic_cmpxchg_ ## BARRIER, \ + var, xchg, new, \ + { \ + new = xchg & ~mask; \ + new |= mask & set; \ + }); \ + return xchg; \ +} + +ERTS_ATOMIC64_OPS_IMPL__(nob) +ERTS_ATOMIC64_OPS_IMPL__(mb) +ERTS_ATOMIC64_OPS_IMPL__(acqb) +ERTS_ATOMIC64_OPS_IMPL__(relb) +ERTS_ATOMIC64_OPS_IMPL__(ddrb) +ERTS_ATOMIC64_OPS_IMPL__(rb) +ERTS_ATOMIC64_OPS_IMPL__(wb) + +#undef ERTS_ATOMIC64_OPS_IMPL__ +#undef ERTS_ATOMIC64_DW_CMPXCHG_IMPL__ + +ERTS_GLB_INLINE void +erts_atomic64_set_dirty(erts_atomic64_t *var, erts_aint64_t val) +{ + ethr_sint_t *sint = ethr_dw_atomic_addr(var); + ethr_dw_sint_t dw; + ERTS_AINT64_TO_DW_SINT__(dw, val); + sint[0] = dw.sint[0]; + sint[1] = dw.sint[1]; +} + +ERTS_GLB_INLINE erts_aint64_t +erts_atomic64_read_dirty(erts_atomic64_t *var) +{ + ethr_sint_t *sint; + ethr_dw_sint_t dw; + sint = ethr_dw_atomic_addr(var); + dw.sint[0] = sint[0]; + dw.sint[1] = sint[1]; + return ERTS_DW_SINT_TO_AINT64__(dw); +} + +#undef ERTS_DW_SINT_TO_AINT64__ +#undef ERTS_AINT64_TO_DW_SINT__ + +#endif /* ERTS_GLB_INLINE_INCL_FUNC_DEF */ + +#endif /* ARCH_32 */ + #else /* !USE_THREADS */ /* Double word size atomics */ @@ -1462,8 +1944,120 @@ erts_atomic32_read_dirty(erts_atomic32_t *var) #define erts_atomic32_set_dirty erts_no_atomic32_set #define erts_atomic32_read_dirty erts_no_atomic32_read +/* 64-bit atomics */ + +#define erts_atomic64_init_nob erts_no_atomic64_set +#define erts_atomic64_set_nob erts_no_atomic64_set +#define erts_atomic64_read_nob erts_no_atomic64_read +#define erts_atomic64_inc_read_nob erts_no_atomic64_inc_read +#define erts_atomic64_dec_read_nob erts_no_atomic64_dec_read +#define erts_atomic64_inc_nob erts_no_atomic64_inc +#define erts_atomic64_dec_nob erts_no_atomic64_dec +#define erts_atomic64_add_read_nob erts_no_atomic64_add_read +#define erts_atomic64_add_nob erts_no_atomic64_add +#define erts_atomic64_read_bor_nob erts_no_atomic64_read_bor +#define erts_atomic64_read_band_nob erts_no_atomic64_read_band +#define erts_atomic64_xchg_nob erts_no_atomic64_xchg +#define erts_atomic64_cmpxchg_nob erts_no_atomic64_cmpxchg +#define erts_atomic64_read_bset_nob erts_no_atomic64_read_bset + +#define erts_atomic64_init_mb erts_no_atomic64_set +#define erts_atomic64_set_mb erts_no_atomic64_set +#define erts_atomic64_read_mb erts_no_atomic64_read +#define erts_atomic64_inc_read_mb erts_no_atomic64_inc_read +#define erts_atomic64_dec_read_mb erts_no_atomic64_dec_read +#define erts_atomic64_inc_mb erts_no_atomic64_inc +#define erts_atomic64_dec_mb erts_no_atomic64_dec +#define erts_atomic64_add_read_mb erts_no_atomic64_add_read +#define erts_atomic64_add_mb erts_no_atomic64_add +#define erts_atomic64_read_bor_mb erts_no_atomic64_read_bor +#define erts_atomic64_read_band_mb erts_no_atomic64_read_band +#define erts_atomic64_xchg_mb erts_no_atomic64_xchg +#define erts_atomic64_cmpxchg_mb erts_no_atomic64_cmpxchg +#define erts_atomic64_read_bset_mb erts_no_atomic64_read_bset + +#define erts_atomic64_init_acqb erts_no_atomic64_set +#define erts_atomic64_set_acqb erts_no_atomic64_set +#define erts_atomic64_read_acqb erts_no_atomic64_read +#define erts_atomic64_inc_read_acqb erts_no_atomic64_inc_read +#define erts_atomic64_dec_read_acqb erts_no_atomic64_dec_read +#define erts_atomic64_inc_acqb erts_no_atomic64_inc +#define erts_atomic64_dec_acqb erts_no_atomic64_dec +#define erts_atomic64_add_read_acqb erts_no_atomic64_add_read +#define erts_atomic64_add_acqb erts_no_atomic64_add +#define erts_atomic64_read_bor_acqb erts_no_atomic64_read_bor +#define erts_atomic64_read_band_acqb erts_no_atomic64_read_band +#define erts_atomic64_xchg_acqb erts_no_atomic64_xchg +#define erts_atomic64_cmpxchg_acqb erts_no_atomic64_cmpxchg +#define erts_atomic64_read_bset_acqb erts_no_atomic64_read_bset + +#define erts_atomic64_init_relb erts_no_atomic64_set +#define erts_atomic64_set_relb erts_no_atomic64_set +#define erts_atomic64_read_relb erts_no_atomic64_read +#define erts_atomic64_inc_read_relb erts_no_atomic64_inc_read +#define erts_atomic64_dec_read_relb erts_no_atomic64_dec_read +#define erts_atomic64_inc_relb erts_no_atomic64_inc +#define erts_atomic64_dec_relb erts_no_atomic64_dec +#define erts_atomic64_add_read_relb erts_no_atomic64_add_read +#define erts_atomic64_add_relb erts_no_atomic64_add +#define erts_atomic64_read_bor_relb erts_no_atomic64_read_bor +#define erts_atomic64_read_band_relb erts_no_atomic64_read_band +#define erts_atomic64_xchg_relb erts_no_atomic64_xchg +#define erts_atomic64_cmpxchg_relb erts_no_atomic64_cmpxchg +#define erts_atomic64_read_bset_relb erts_no_atomic64_read_bset + +#define erts_atomic64_init_ddrb erts_no_atomic64_set +#define erts_atomic64_set_ddrb erts_no_atomic64_set +#define erts_atomic64_read_ddrb erts_no_atomic64_read +#define erts_atomic64_inc_read_ddrb erts_no_atomic64_inc_read +#define erts_atomic64_dec_read_ddrb erts_no_atomic64_dec_read +#define erts_atomic64_inc_ddrb erts_no_atomic64_inc +#define erts_atomic64_dec_ddrb erts_no_atomic64_dec +#define erts_atomic64_add_read_ddrb erts_no_atomic64_add_read +#define erts_atomic64_add_ddrb erts_no_atomic64_add +#define erts_atomic64_read_bor_ddrb erts_no_atomic64_read_bor +#define erts_atomic64_read_band_ddrb erts_no_atomic64_read_band +#define erts_atomic64_xchg_ddrb erts_no_atomic64_xchg +#define erts_atomic64_cmpxchg_ddrb erts_no_atomic64_cmpxchg +#define erts_atomic64_read_bset_ddrb erts_no_atomic64_read_bset + +#define erts_atomic64_init_rb erts_no_atomic64_set +#define erts_atomic64_set_rb erts_no_atomic64_set +#define erts_atomic64_read_rb erts_no_atomic64_read +#define erts_atomic64_inc_read_rb erts_no_atomic64_inc_read +#define erts_atomic64_dec_read_rb erts_no_atomic64_dec_read +#define erts_atomic64_inc_rb erts_no_atomic64_inc +#define erts_atomic64_dec_rb erts_no_atomic64_dec +#define erts_atomic64_add_read_rb erts_no_atomic64_add_read +#define erts_atomic64_add_rb erts_no_atomic64_add +#define erts_atomic64_read_bor_rb erts_no_atomic64_read_bor +#define erts_atomic64_read_band_rb erts_no_atomic64_read_band +#define erts_atomic64_xchg_rb erts_no_atomic64_xchg +#define erts_atomic64_cmpxchg_rb erts_no_atomic64_cmpxchg +#define erts_atomic64_read_bset_rb erts_no_atomic64_read_bset + +#define erts_atomic64_init_wb erts_no_atomic64_set +#define erts_atomic64_set_wb erts_no_atomic64_set +#define erts_atomic64_read_wb erts_no_atomic64_read +#define erts_atomic64_inc_read_wb erts_no_atomic64_inc_read +#define erts_atomic64_dec_read_wb erts_no_atomic64_dec_read +#define erts_atomic64_inc_wb erts_no_atomic64_inc +#define erts_atomic64_dec_wb erts_no_atomic64_dec +#define erts_atomic64_add_read_wb erts_no_atomic64_add_read +#define erts_atomic64_add_wb erts_no_atomic64_add +#define erts_atomic64_read_bor_wb erts_no_atomic64_read_bor +#define erts_atomic64_read_band_wb erts_no_atomic64_read_band +#define erts_atomic64_xchg_wb erts_no_atomic64_xchg +#define erts_atomic64_cmpxchg_wb erts_no_atomic64_cmpxchg +#define erts_atomic64_read_bset_wb erts_no_atomic64_read_bset + +#define erts_atomic64_set_dirty erts_no_atomic64_set +#define erts_atomic64_read_dirty erts_no_atomic64_read + #endif /* !USE_THREADS */ +#include "erl_msacc.h" + #if ERTS_GLB_INLINE_INCL_FUNC_DEF ERTS_GLB_INLINE void @@ -1548,6 +2142,16 @@ erts_thr_self(void) #endif } +ERTS_GLB_INLINE int +erts_thr_getname(erts_tid_t tid, char *buf, size_t len) +{ +#ifdef USE_THREADS + return ethr_getname(tid, buf, len); +#else + return -1; +#endif +} + ERTS_GLB_INLINE int erts_equal_tids(erts_tid_t x, erts_tid_t y) @@ -1812,6 +2416,7 @@ erts_cnd_wait(erts_cnd_t *cnd, erts_mtx_t *mtx) { #ifdef USE_THREADS int res; + ERTS_MSACC_PUSH_AND_SET_STATE(ERTS_MSACC_STATE_SLEEP); #ifdef ERTS_ENABLE_LOCK_CHECK erts_lc_unlock(&mtx->lc); #endif @@ -1830,6 +2435,7 @@ erts_cnd_wait(erts_cnd_t *cnd, erts_mtx_t *mtx) #endif if (res != 0 && res != EINTR) erts_thr_fatal_error(res, "wait on condition variable"); + ERTS_MSACC_POP_STATE(); #endif } @@ -2383,6 +2989,104 @@ erts_no_atomic32_read_bset(erts_no_atomic32_t *var, return old; } +/* atomic64 */ + +ERTS_GLB_INLINE void +erts_no_atomic64_set(erts_no_atomic64_t *var, erts_aint64_t i) +{ + *var = i; +} + +ERTS_GLB_INLINE erts_aint64_t +erts_no_atomic64_read(erts_no_atomic64_t *var) +{ + return *var; +} + +ERTS_GLB_INLINE erts_aint64_t +erts_no_atomic64_inc_read(erts_no_atomic64_t *incp) +{ + return ++(*incp); +} + +ERTS_GLB_INLINE erts_aint64_t +erts_no_atomic64_dec_read(erts_no_atomic64_t *decp) +{ + return --(*decp); +} + +ERTS_GLB_INLINE void +erts_no_atomic64_inc(erts_no_atomic64_t *incp) +{ + ++(*incp); +} + +ERTS_GLB_INLINE void +erts_no_atomic64_dec(erts_no_atomic64_t *decp) +{ + --(*decp); +} + +ERTS_GLB_INLINE erts_aint64_t +erts_no_atomic64_add_read(erts_no_atomic64_t *addp, erts_aint64_t i) +{ + return *addp += i; +} + +ERTS_GLB_INLINE void +erts_no_atomic64_add(erts_no_atomic64_t *addp, erts_aint64_t i) +{ + *addp += i; +} + +ERTS_GLB_INLINE erts_aint64_t +erts_no_atomic64_read_bor(erts_no_atomic64_t *var, erts_aint64_t mask) +{ + erts_aint64_t old; + old = *var; + *var |= mask; + return old; +} + +ERTS_GLB_INLINE erts_aint64_t +erts_no_atomic64_read_band(erts_no_atomic64_t *var, erts_aint64_t mask) +{ + erts_aint64_t old; + old = *var; + *var &= mask; + return old; +} + +ERTS_GLB_INLINE erts_aint64_t +erts_no_atomic64_xchg(erts_no_atomic64_t *xchgp, erts_aint64_t new) +{ + erts_aint64_t old = *xchgp; + *xchgp = new; + return old; +} + +ERTS_GLB_INLINE erts_aint64_t +erts_no_atomic64_cmpxchg(erts_no_atomic64_t *xchgp, + erts_aint64_t new, + erts_aint64_t expected) +{ + erts_aint64_t old = *xchgp; + if (old == expected) + *xchgp = new; + return old; +} + +ERTS_GLB_INLINE erts_aint64_t +erts_no_atomic64_read_bset(erts_no_atomic64_t *var, + erts_aint64_t mask, + erts_aint64_t set) +{ + erts_aint64_t old = *var; + *var &= ~mask; + *var |= (mask & set); + return old; +} + /* spinlock */ ERTS_GLB_INLINE void @@ -2762,6 +3466,15 @@ ERTS_GLB_INLINE void erts_tse_return(erts_tse_t *ep) #endif } +ERTS_GLB_INLINE void erts_tse_prepare_timed(erts_tse_t *ep) +{ +#ifdef USE_THREADS + int res = ethr_event_prepare_timed(&((ethr_ts_event *) ep)->event); + if (res != 0) + erts_thr_fatal_error(res, "prepare timed"); +#endif +} + ERTS_GLB_INLINE void erts_tse_set(erts_tse_t *ep) { #ifdef USE_THREADS @@ -2779,7 +3492,11 @@ ERTS_GLB_INLINE void erts_tse_reset(erts_tse_t *ep) ERTS_GLB_INLINE int erts_tse_wait(erts_tse_t *ep) { #ifdef USE_THREADS - return ethr_event_wait(&((ethr_ts_event *) ep)->event); + int res; + ERTS_MSACC_PUSH_AND_SET_STATE(ERTS_MSACC_STATE_SLEEP); + res = ethr_event_wait(&((ethr_ts_event *) ep)->event); + ERTS_MSACC_POP_STATE(); + return res; #else return ENOTSUP; #endif @@ -2788,7 +3505,40 @@ ERTS_GLB_INLINE int erts_tse_wait(erts_tse_t *ep) ERTS_GLB_INLINE int erts_tse_swait(erts_tse_t *ep, int spincount) { #ifdef USE_THREADS - return ethr_event_swait(&((ethr_ts_event *) ep)->event, spincount); + int res; + ERTS_MSACC_PUSH_AND_SET_STATE(ERTS_MSACC_STATE_SLEEP); + res = ethr_event_swait(&((ethr_ts_event *) ep)->event, spincount); + ERTS_MSACC_POP_STATE(); + return res; +#else + return ENOTSUP; +#endif +} + +ERTS_GLB_INLINE int erts_tse_twait(erts_tse_t *ep, Sint64 tmo) +{ +#ifdef USE_THREADS + int res; + ERTS_MSACC_PUSH_AND_SET_STATE(ERTS_MSACC_STATE_SLEEP); + res = ethr_event_twait(&((ethr_ts_event *) ep)->event, + (ethr_sint64_t) tmo); + ERTS_MSACC_POP_STATE(); + return res; +#else + return ENOTSUP; +#endif +} + +ERTS_GLB_INLINE int erts_tse_stwait(erts_tse_t *ep, int spincount, Sint64 tmo) +{ +#ifdef USE_THREADS + int res; + ERTS_MSACC_PUSH_AND_SET_STATE(ERTS_MSACC_STATE_SLEEP); + res = ethr_event_stwait(&((ethr_ts_event *) ep)->event, + spincount, + (ethr_sint64_t) tmo); + ERTS_MSACC_POP_STATE(); + return res; #else return ENOTSUP; #endif @@ -2838,6 +3588,15 @@ ERTS_GLB_INLINE void erts_thr_yield(void) #ifdef ETHR_HAVE_ETHR_SIG_FUNCS ERTS_GLB_INLINE void +erts_thr_kill(erts_tid_t tid, int sig) { +#ifdef USE_THREADS + int res = ethr_kill((ethr_tid)tid, sig); + if (res) + erts_thr_fatal_error(res, "killing thread"); +#endif +} + +ERTS_GLB_INLINE void erts_thr_sigmask(int how, const sigset_t *set, sigset_t *oset) { #ifdef USE_THREADS diff --git a/erts/emulator/beam/erl_time.h b/erts/emulator/beam/erl_time.h index 4bbdcaa3e3..5242063550 100644 --- a/erts/emulator/beam/erl_time.h +++ b/erts/emulator/beam/erl_time.h @@ -3,16 +3,17 @@ * * Copyright Ericsson AB 2006-2011. All Rights Reserved. * - * The contents of this file are subject to the Erlang Public License, - * Version 1.1, (the "License"); you may not use this file except in - * compliance with the License. You should have received a copy of the - * Erlang Public License along with this software. If not, it can be - * retrieved online at http://www.erlang.org/. - * - * Software distributed under the License is distributed on an "AS IS" - * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See - * the License for the specific language governing rights and limitations - * under the License. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. * * %CopyrightEnd% */ @@ -20,94 +21,49 @@ #ifndef ERL_TIME_H__ #define ERL_TIME_H__ -#define ERTS_SHORT_TIME_T_MAX ERTS_AINT32_T_MAX -#define ERTS_SHORT_TIME_T_MIN ERTS_AINT32_T_MIN -typedef erts_aint32_t erts_short_time_t; +/* timer wheel size NEED to be a power of 2 */ +#ifdef SMALL_MEMORY +#define ERTS_TIW_SIZE (1 << 13) +#else +#define ERTS_TIW_SIZE (1 << 16) +#endif -extern erts_smp_atomic32_t do_time; /* set at clock interrupt */ -extern SysTimeval erts_first_emu_time; +#if defined(DEBUG) || 0 +#define ERTS_TIME_ASSERT(B) ERTS_ASSERT(B) +#else +#define ERTS_TIME_ASSERT(B) ((void) 1) +#endif -/* -** Timer entry: -*/ -typedef struct erl_timer { - struct erl_timer* next; /* next entry tiw slot or chain */ - struct erl_timer* prev; /* prev entry tiw slot or chain */ - Uint slot; /* slot in timer wheel */ - Uint count; /* number of loops remaining */ - int active; /* 1=activated, 0=deactivated */ - /* called when timeout */ - void (*timeout)(void*); - /* called when cancel (may be NULL) */ - void (*cancel)(void*); - void* arg; /* argument to timeout/cancel procs */ -} ErlTimer; +typedef enum { + ERTS_NO_TIME_WARP_MODE, + ERTS_SINGLE_TIME_WARP_MODE, + ERTS_MULTI_TIME_WARP_MODE +} ErtsTimeWarpMode; -typedef void (*ErlTimeoutProc)(void*); -typedef void (*ErlCancelProc)(void*); +typedef struct ErtsTimerWheel_ ErtsTimerWheel; +typedef ErtsMonotonicTime * ErtsNextTimeoutRef; -#ifdef ERTS_SMP -/* - * Process and port timer - */ -typedef union ErtsSmpPTimer_ ErtsSmpPTimer; -union ErtsSmpPTimer_ { - struct { - ErlTimer tm; - Eterm id; - void (*timeout_func)(void*); - ErtsSmpPTimer **timer_ref; - Uint32 flags; - } timer; - ErtsSmpPTimer *next; -}; +extern SysTimeval erts_first_emu_time; -void erts_create_smp_ptimer(ErtsSmpPTimer **timer_ref, - Eterm id, - ErlTimeoutProc timeout_func, - Uint timeout); -void erts_cancel_smp_ptimer(ErtsSmpPTimer *ptimer); -#endif +void erts_monitor_time_offset(Eterm id, Eterm ref); +int erts_demonitor_time_offset(Eterm ref); -/* timer-wheel api */ +int erts_init_time_sup(int, ErtsTimeWarpMode); +void erts_late_init_time_sup(void); -void erts_init_time(void); -void erts_set_timer(ErlTimer*, ErlTimeoutProc, ErlCancelProc, void*, Uint); -void erts_cancel_timer(ErlTimer*); -void erts_bump_timer(erts_short_time_t); +ErtsNextTimeoutRef erts_get_next_timeout_reference(ErtsTimerWheel *); +void erts_init_time(int time_correction, ErtsTimeWarpMode time_warp_mode); +void erts_bump_timers(ErtsTimerWheel *, ErtsMonotonicTime); Uint erts_timer_wheel_memory_size(void); -Uint erts_time_left(ErlTimer *); -erts_short_time_t erts_next_time(void); #ifdef DEBUG void erts_p_slpq(void); #endif -ERTS_GLB_INLINE erts_short_time_t erts_do_time_read_and_reset(void); -ERTS_GLB_INLINE void erts_do_time_add(erts_short_time_t); - -#if ERTS_GLB_INLINE_INCL_FUNC_DEF - -ERTS_GLB_INLINE erts_short_time_t erts_do_time_read_and_reset(void) -{ - erts_short_time_t time = erts_smp_atomic32_xchg_acqb(&do_time, 0); - if (time < 0) - erl_exit(ERTS_ABORT_EXIT, "Internal time management error\n"); - return time; -} - -ERTS_GLB_INLINE void erts_do_time_add(erts_short_time_t elapsed) -{ - erts_smp_atomic32_add_relb(&do_time, elapsed); -} - -#endif /* #if ERTS_GLB_INLINE_INCL_FUNC_DEF */ - - /* time_sup */ -#if (defined(HAVE_GETHRVTIME) || defined(HAVE_CLOCK_GETTIME)) +#if (defined(HAVE_GETHRVTIME) || defined(HAVE_CLOCK_GETTIME_CPU_TIME)) # ifndef HAVE_ERTS_NOW_CPU # define HAVE_ERTS_NOW_CPU # ifdef HAVE_GETHRVTIME @@ -121,25 +77,397 @@ void erts_get_now_cpu(Uint* megasec, Uint* sec, Uint* microsec); typedef UWord erts_approx_time_t; erts_approx_time_t erts_get_approx_time(void); -void erts_get_timeval(SysTimeval *tv); -erts_time_t erts_get_time(void); +int erts_has_time_correction(void); +int erts_check_time_adj_support(int time_correction, + ErtsTimeWarpMode time_warp_mode); + +ErtsTimeWarpMode erts_time_warp_mode(void); + +typedef enum { + ERTS_TIME_OFFSET_PRELIMINARY, + ERTS_TIME_OFFSET_FINAL, + ERTS_TIME_OFFSET_VOLATILE +} ErtsTimeOffsetState; -ERTS_GLB_INLINE int erts_cmp_timeval(SysTimeval *t1p, SysTimeval *t2p); +ErtsTimeOffsetState erts_time_offset_state(void); +ErtsTimeOffsetState erts_finalize_time_offset(void); +struct process; +Eterm erts_get_monotonic_start_time(struct process *c_p); +Eterm erts_get_monotonic_end_time(struct process *c_p); +Eterm erts_monotonic_time_source(struct process*c_p); +Eterm erts_system_time_source(struct process*c_p); + +#ifdef SYS_CLOCK_RESOLUTION +#define ERTS_CLKTCK_RESOLUTION ((ErtsMonotonicTime) (SYS_CLOCK_RESOLUTION*1000)) +#else +#define ERTS_CLKTCK_RESOLUTION (erts_time_sup__.r.o.clktck_resolution) +#endif + +#define ERTS_TIMER_WHEEL_MSEC (ERTS_TIW_SIZE/(ERTS_CLKTCK_RESOLUTION/1000)) + +struct erts_time_sup_read_only__ { + ErtsMonotonicTime monotonic_time_unit; +#if !ERTS_COMPILE_TIME_MONOTONIC_TIME_UNIT + ErtsMonotonicTime start; + struct { + ErtsMonotonicTime native; + ErtsMonotonicTime nsec; + ErtsMonotonicTime usec; + ErtsMonotonicTime msec; + ErtsMonotonicTime sec; + } start_offset; +#endif +#ifndef SYS_CLOCK_RESOLUTION + ErtsMonotonicTime clktck_resolution; +#endif +}; + +typedef struct { + union { + struct erts_time_sup_read_only__ o; + char align__[(((sizeof(struct erts_time_sup_read_only__) - 1) + / ASSUMED_CACHE_LINE_SIZE) + 1) + * ASSUMED_CACHE_LINE_SIZE]; + } r; +} ErtsTimeSupData; + +extern ErtsTimeSupData erts_time_sup__; + +ErtsMonotonicTime erts_napi_monotonic_time(int time_unit); +ErtsMonotonicTime erts_napi_time_offset(int time_unit); +ErtsMonotonicTime erts_napi_convert_time_unit(ErtsMonotonicTime val, int from, int to); + +ERTS_GLB_INLINE Uint64 +erts_time_unit_conversion(Uint64 value, + Uint32 from_time_unit, + Uint32 to_time_unit); + +ErtsSysPerfCounter erts_perf_counter_unit(void); #if ERTS_GLB_INLINE_INCL_FUNC_DEF -ERTS_GLB_INLINE int -erts_cmp_timeval(SysTimeval *t1p, SysTimeval *t2p) +ERTS_GLB_INLINE Uint64 +erts_time_unit_conversion(Uint64 value, + Uint32 from_time_unit, + Uint32 to_time_unit) { - if (t1p->tv_sec == t2p->tv_sec) { - if (t1p->tv_usec < t2p->tv_usec) - return -1; - else if (t1p->tv_usec > t2p->tv_usec) - return 1; - return 0; - } - return t1p->tv_sec < t2p->tv_sec ? -1 : 1; + Uint64 high, low, result; + if (value <= ~((Uint64) 0)/to_time_unit) + return (value*to_time_unit)/from_time_unit; + + low = value & ((Uint64) 0xffffffff); + high = (value >> 32) & ((Uint64) 0xffffffff); + + low *= to_time_unit; + high *= to_time_unit; + + high += (low >> 32) & ((Uint64) 0xffffffff); + low &= ((Uint64) 0xffffffff); + + result = high % from_time_unit; + high /= from_time_unit; + high <<= 32; + + result <<= 32; + result += low; + result /= from_time_unit; + result += high; + + return result; } -#endif /* #if ERTS_GLB_INLINE_INCL_FUNC_DEF */ +#endif /* ERTS_GLB_INLINE_INCL_FUNC_DEF */ + +/* + * Range of monotonic time internally + */ + +#define ERTS_MONOTONIC_BEGIN \ + ERTS_MONOTONIC_TIME_UNIT +#define ERTS_MONOTONIC_END \ + ((ERTS_MONOTONIC_TIME_MAX / ERTS_MONOTONIC_TIME_UNIT) \ + * ERTS_MONOTONIC_TIME_UNIT) + +#if ERTS_COMPILE_TIME_MONOTONIC_TIME_UNIT + +/* + * If the monotonic time unit is a compile time constant, + * it is assumed (and need) to be a power of 10. + */ + +#if ERTS_COMPILE_TIME_MONOTONIC_TIME_UNIT < 1000*1000 +# error Compile time time unit needs to be at least 1000000 +#endif + +#if ERTS_COMPILE_TIME_MONOTONIC_TIME_UNIT == 1000*1000*1000 +/* Nano-second time unit */ + +#define ERTS_MONOTONIC_TO_SEC__(NSEC) ((NSEC) / (1000*1000*1000)) +#define ERTS_MONOTONIC_TO_MSEC__(NSEC) ((NSEC) / (1000*1000)) +#define ERTS_MONOTONIC_TO_USEC__(NSEC) ((NSEC) / 1000) +#define ERTS_MONOTONIC_TO_NSEC__(NSEC) (NSEC) + +#define ERTS_SEC_TO_MONOTONIC__(SEC) (((ErtsMonotonicTime) (SEC))*(1000*1000*1000)) +#define ERTS_MSEC_TO_MONOTONIC__(MSEC) (((ErtsMonotonicTime) (MSEC))*(1000*1000)) +#define ERTS_USEC_TO_MONOTONIC__(USEC) (((ErtsMonotonicTime) (USEC))*1000) +#define ERTS_NSEC_TO_MONOTONIC__(NSEC) ((ErtsMonotonicTime) (NSEC)) + +#elif ERTS_COMPILE_TIME_MONOTONIC_TIME_UNIT == 1000*1000 +/* Micro-second time unit */ + +#define ERTS_MONOTONIC_TO_SEC__(USEC) ((USEC) / (1000*1000)) +#define ERTS_MONOTONIC_TO_MSEC__(USEC) ((USEC) / 1000) +#define ERTS_MONOTONIC_TO_USEC__(USEC) (USEC) +#define ERTS_MONOTONIC_TO_NSEC__(USEC) ((USEC)*1000) + +#define ERTS_SEC_TO_MONOTONIC__(SEC) (((ErtsMonotonicTime) (SEC))*(1000*1000)) +#define ERTS_MSEC_TO_MONOTONIC__(MSEC) (((ErtsMonotonicTime) (MSEC))*1000) +#define ERTS_USEC_TO_MONOTONIC__(USEC) ((ErtsMonotonicTime) (USEC)) +#define ERTS_NSEC_TO_MONOTONIC__(NSEC) (((ErtsMonotonicTime) (NSEC))/1000) + +#else +#error Missing implementation for monotonic time unit +#endif + +#define ERTS_MONOTONIC_TIME_UNIT \ + ((ErtsMonotonicTime) ERTS_COMPILE_TIME_MONOTONIC_TIME_UNIT) + +/* + * NOTE! ERTS_MONOTONIC_TIME_START_EXTERNAL *need* to be a multiple + * of ERTS_MONOTONIC_TIME_UNIT. + */ + +#ifdef ARCH_32 +/* + * Want to use a big-num of arity 2 as long as possible (584 years + * in the nano-second time unit case). + */ +#define ERTS_MONOTONIC_TIME_START_EXTERNAL \ + (((((((ErtsMonotonicTime) 1) << 32)-1) \ + / ERTS_MONOTONIC_TIME_UNIT) \ + * ERTS_MONOTONIC_TIME_UNIT) \ + + ERTS_MONOTONIC_TIME_UNIT) + +#else /* ARCH_64 */ + +#if ERTS_COMPILE_TIME_MONOTONIC_TIME_UNIT <= 10*1000*1000 + +/* + * Using micro second time unit or lower. Start at zero since + * time will remain an immediate for a very long time anyway + * (1827 years in the 10 micro second case)... + */ +#define ERTS_MONOTONIC_TIME_START_EXTERNAL ((ErtsMonotonicTime) 0) + +#else /* ERTS_COMPILE_TIME_MONOTONIC_TIME_UNIT > 10*1000*1000 */ + +/* + * Want to use an immediate as long as possible (36 years in the + * nano-second time unit case). +*/ +#define ERTS_MONOTONIC_TIME_START_EXTERNAL \ + ((((ErtsMonotonicTime) MIN_SMALL) \ + / ERTS_MONOTONIC_TIME_UNIT) \ + * ERTS_MONOTONIC_TIME_UNIT) + +#endif /* ERTS_COMPILE_TIME_MONOTONIC_TIME_UNIT > 1000*1000 */ + +#endif /* ARCH_64 */ + +/* + * Offsets from internal monotonic time to external monotonic time + */ + +#define ERTS_MONOTONIC_OFFSET_NATIVE \ + (ERTS_MONOTONIC_TIME_START_EXTERNAL - ERTS_MONOTONIC_BEGIN) +#define ERTS_MONOTONIC_OFFSET_NSEC \ + ERTS_MONOTONIC_TO_NSEC__(ERTS_MONOTONIC_OFFSET_NATIVE) +#define ERTS_MONOTONIC_OFFSET_USEC \ + ERTS_MONOTONIC_TO_USEC__(ERTS_MONOTONIC_OFFSET_NATIVE) +#define ERTS_MONOTONIC_OFFSET_MSEC \ + ERTS_MONOTONIC_TO_MSEC__(ERTS_MONOTONIC_OFFSET_NATIVE) +#define ERTS_MONOTONIC_OFFSET_SEC \ + ERTS_MONOTONIC_TO_SEC__(ERTS_MONOTONIC_OFFSET_NATIVE) + +#define ERTS_MONOTONIC_TO_CLKTCKS__(MON) \ + ((MON) / (ERTS_MONOTONIC_TIME_UNIT/ERTS_CLKTCK_RESOLUTION)) +#define ERTS_CLKTCKS_TO_MONOTONIC__(TCKS) \ + ((TCKS) * (ERTS_MONOTONIC_TIME_UNIT/ERTS_CLKTCK_RESOLUTION)) + +#else /* !ERTS_COMPILE_TIME_MONOTONIC_TIME_UNIT */ + +/* + * Initialized in erts_init_sys_time_sup() + */ +#define ERTS_MONOTONIC_TIME_UNIT (erts_time_sup__.r.o.monotonic_time_unit) + +/* + * Offsets from internal monotonic time to external monotonic time + * + * Initialized in erts_init_time_sup()... + */ +#define ERTS_MONOTONIC_TIME_START_EXTERNAL (erts_time_sup__.r.o.start) +#define ERTS_MONOTONIC_OFFSET_NATIVE (erts_time_sup__.r.o.start_offset.native) +#define ERTS_MONOTONIC_OFFSET_NSEC (erts_time_sup__.r.o.start_offset.nsec) +#define ERTS_MONOTONIC_OFFSET_USEC (erts_time_sup__.r.o.start_offset.usec) +#define ERTS_MONOTONIC_OFFSET_MSEC (erts_time_sup__.r.o.start_offset.msec) +#define ERTS_MONOTONIC_OFFSET_SEC (erts_time_sup__.r.o.start_offset.sec) + +#define ERTS_CONV_FROM_MON_UNIT___(M, TO) \ + ((ErtsMonotonicTime) \ + erts_time_unit_conversion((Uint64) (M), \ + (Uint32) ERTS_MONOTONIC_TIME_UNIT, \ + (Uint32) (TO))) + +#define ERTS_CONV_TO_MON_UNIT___(M, FROM) \ + ((ErtsMonotonicTime) \ + erts_time_unit_conversion((Uint64) (M), \ + (Uint32) (FROM), \ + (Uint32) ERTS_MONOTONIC_TIME_UNIT)) \ + +#define ERTS_MONOTONIC_TO_SEC__(M) \ + ERTS_CONV_FROM_MON_UNIT___((M), 1) +#define ERTS_MONOTONIC_TO_MSEC__(M) \ + ERTS_CONV_FROM_MON_UNIT___((M), 1000) +#define ERTS_MONOTONIC_TO_USEC__(M) \ + ERTS_CONV_FROM_MON_UNIT___((M), 1000*1000) +#define ERTS_MONOTONIC_TO_NSEC__(M) \ + ERTS_CONV_FROM_MON_UNIT___((M), 1000*1000*1000) + +#define ERTS_SEC_TO_MONOTONIC__(SEC) \ + ERTS_CONV_TO_MON_UNIT___((SEC), 1) +#define ERTS_MSEC_TO_MONOTONIC__(MSEC) \ + ERTS_CONV_TO_MON_UNIT___((MSEC), 1000) +#define ERTS_USEC_TO_MONOTONIC__(USEC) \ + ERTS_CONV_TO_MON_UNIT___((USEC), 1000*1000) +#define ERTS_NSEC_TO_MONOTONIC__(NSEC) \ + ERTS_CONV_TO_MON_UNIT___((NSEC), 1000*1000*1000) + +#define ERTS_MONOTONIC_TO_CLKTCKS__(MON) \ + ERTS_CONV_FROM_MON_UNIT___((MON), ERTS_CLKTCK_RESOLUTION) +#define ERTS_CLKTCKS_TO_MONOTONIC__(TCKS) \ + ERTS_CONV_TO_MON_UNIT___((TCKS), ERTS_CLKTCK_RESOLUTION) + +#endif /* !ERTS_COMPILE_TIME_MONOTONIC_TIME_UNIT */ + +#define ERTS_MONOTONIC_TIME_END_EXTERNAL \ + (ERTS_MONOTONIC_TIME_START_EXTERNAL < 0 \ + ? (ERTS_MONOTONIC_TIME_START_EXTERNAL \ + + (ERTS_MONOTONIC_END - ERTS_MONOTONIC_BEGIN)) \ + : (ERTS_MONOTONIC_END - ERTS_MONOTONIC_TIME_START_EXTERNAL)) + +#define ERTS_MSEC_TO_CLKTCKS__(MON) \ + ((MON) * (ERTS_CLKTCK_RESOLUTION/1000)) +#define ERTS_CLKTCKS_TO_MSEC__(TCKS) \ + ((TCKS) / (ERTS_CLKTCK_RESOLUTION/1000)) + +#define ERTS_MONOTONIC_TO_SEC(X) \ + (ERTS_TIME_ASSERT((X) >= 0), \ + ERTS_MONOTONIC_TO_SEC__((X))) +#define ERTS_MONOTONIC_TO_MSEC(X) \ + (ERTS_TIME_ASSERT((X) >= 0), \ + ERTS_MONOTONIC_TO_MSEC__((X))) +#define ERTS_MONOTONIC_TO_USEC(X) \ + (ERTS_TIME_ASSERT((X) >= 0), \ + ERTS_MONOTONIC_TO_USEC__((X))) +#define ERTS_MONOTONIC_TO_NSEC(X) \ + (ERTS_TIME_ASSERT((X) >= 0), \ + ERTS_MONOTONIC_TO_NSEC__((X))) +#define ERTS_SEC_TO_MONOTONIC(X) \ + (ERTS_TIME_ASSERT((X) >= 0), \ + ERTS_SEC_TO_MONOTONIC__((X))) +#define ERTS_MSEC_TO_MONOTONIC(X) \ + (ERTS_TIME_ASSERT((X) >= 0), \ + ERTS_MSEC_TO_MONOTONIC__((X))) +#define ERTS_USEC_TO_MONOTONIC(X) \ + (ERTS_TIME_ASSERT((X) >= 0), \ + ERTS_USEC_TO_MONOTONIC__((X))) +#define ERTS_NSEC_TO_MONOTONIC(X) \ + (ERTS_TIME_ASSERT((X) >= 0), \ + ERTS_NSEC_TO_MONOTONIC__((X))) + +#define ERTS_MONOTONIC_TO_CLKTCKS(X) \ + (ERTS_TIME_ASSERT((X) >= 0), \ + ERTS_MONOTONIC_TO_CLKTCKS__((X))) +#define ERTS_CLKTCKS_TO_MONOTONIC(X) \ + (ERTS_TIME_ASSERT((X) >= 0), \ + ERTS_CLKTCKS_TO_MONOTONIC__((X))) + +#define ERTS_MSEC_TO_CLKTCKS(X) \ + (ERTS_TIME_ASSERT((X) >= 0), \ + ERTS_MSEC_TO_CLKTCKS__((X))) +#define ERTS_CLKTCKS_TO_MSEC(X) \ + (ERTS_TIME_ASSERT((X) >= 0), \ + ERTS_CLKTCKS_TO_MSEC__((X))) + #endif /* ERL_TIME_H__ */ + +/* timer-wheel api */ +#if defined(ERTS_WANT_TIMER_WHEEL_API) && !defined(ERTS_GOT_TIMER_WHEEL_API) +#define ERTS_GOT_TIMER_WHEEL_API + +#include "erl_thr_progress.h" +#include "erl_process.h" + +void erts_sched_init_time_sup(ErtsSchedulerData *esdp); + + +#define ERTS_TWHEEL_SLOT_AT_ONCE -1 +#define ERTS_TWHEEL_SLOT_INACTIVE -2 + +/* +** Timer entry: +*/ +typedef struct erl_timer { + struct erl_timer* next; /* next entry tiw slot or chain */ + struct erl_timer* prev; /* prev entry tiw slot or chain */ + union { + struct { + void (*timeout)(void*); /* called when timeout */ + void (*cancel)(void*); /* called when cancel (may be NULL) */ + void* arg; /* argument to timeout/cancel procs */ + } func; + ErtsThrPrgrLaterOp cleanup; + } u; + ErtsMonotonicTime timeout_pos; /* Timeout in absolute clock ticks */ + int slot; +} ErtsTWheelTimer; + +typedef void (*ErlTimeoutProc)(void*); +typedef void (*ErlCancelProc)(void*); + +void erts_twheel_set_timer(ErtsTimerWheel *tiw, + ErtsTWheelTimer *p, ErlTimeoutProc timeout, + ErlCancelProc cancel, void *arg, + ErtsMonotonicTime timeout_pos); +void erts_twheel_cancel_timer(ErtsTimerWheel *tiw, ErtsTWheelTimer *p); +ErtsTimerWheel *erts_create_timer_wheel(ErtsSchedulerData *esdp); + +ErtsMonotonicTime erts_check_next_timeout_time(ErtsSchedulerData *); + +ERTS_GLB_INLINE void erts_twheel_init_timer(ErtsTWheelTimer *p); +ERTS_GLB_INLINE ErtsMonotonicTime erts_next_timeout_time(ErtsNextTimeoutRef); + +#if ERTS_GLB_INLINE_INCL_FUNC_DEF + +ERTS_GLB_INLINE void erts_twheel_init_timer(ErtsTWheelTimer *p) +{ + p->slot = ERTS_TWHEEL_SLOT_INACTIVE; +} + +ERTS_GLB_INLINE ErtsMonotonicTime erts_next_timeout_time(ErtsNextTimeoutRef nxt_tmo_ref) +{ + return *((ErtsMonotonicTime *) nxt_tmo_ref); +} + +#endif /* ERTS_GLB_INLINE_INCL_FUNC_DEF */ + +void +erts_twheel_debug_foreach(ErtsTimerWheel *tiw, + void (*tclbk)(void *), + void (*func)(void *, + ErtsMonotonicTime, + void *), + void *arg); + +#endif /* timer wheel api */ diff --git a/erts/emulator/beam/erl_time_sup.c b/erts/emulator/beam/erl_time_sup.c index 3272a5326d..f3f528eaf6 100644 --- a/erts/emulator/beam/erl_time_sup.c +++ b/erts/emulator/beam/erl_time_sup.c @@ -1,77 +1,28 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 1999-2012. All Rights Reserved. + * Copyright Ericsson AB 1999-2015. All Rights Reserved. * - * The contents of this file are subject to the Erlang Public License, - * Version 1.1, (the "License"); you may not use this file except in - * compliance with the License. You should have received a copy of the - * Erlang Public License along with this software. If not, it can be - * retrieved online at http://www.erlang.org/. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Software distributed under the License is distributed on an "AS IS" - * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See - * the License for the specific language governing rights and limitations - * under the License. + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. * * %CopyrightEnd% */ /* -** Support routines for the timer wheel -** -** This code contains two strategies for dealing with -** date/time changes in the system. -** If the system has some kind of high resolution timer (HAVE_GETHRTIME), -** the high resolution timer is used to correct the time-of-day and the -** timeouts, the base source is the hrtimer, but at certain intervals the -** OS time-of-day is checked and if it is not within certain bounds, the -** delivered time gets slowly adjusted for each call until -** it corresponds to the system time (built-in adjtime...). -** The call gethrtime() is detected by autoconf on Unix, but other -** platforms may define it in erl_*_sys.h and implement -** their own high resolution timer. The high resolution timer -** strategy is (probably) best on all systems where the timer have -** a resolution higher or equal to gettimeofday (or what's implemented -** is sys_gettimeofday()). The actual resolution is the interesting thing, -** not the unit's thats used (i.e. on VxWorks, nanoseconds can be -** retrieved in terms of units, but the actual resolution is the same as -** for the clock ticks). -** If the systems best timer routine is kernel ticks returned from -** sys_times(), and the actual resolution of sys_gettimeofday() is -** better (like most unixes that does not have any realtime extensions), -** another strategy is used. The tolerant gettimeofday() corrects -** the value with respect to uptime (sys_times() return value) and checks -** for correction both when delivering timeticks and delivering nowtime. -** this strategy is slower, but accurate on systems without better timer -** routines. The kernel tick resolution is not enough to implement -** a gethrtime routine. On Linux and other non solaris unix-boxes the second -** strategy is used, on all other platforms we use the first. -** -** The following is expected (from sys.[ch] and erl_*_sys.h): -** -** 64 bit integers. So it is, and so it will be. -** -** sys_init_time(), will return the clock resolution in MS and -** that's about it. More could be added of course -** If the clock-rate is constant (i.e. 1 ms) one can define -** SYS_CLOCK_RESOLUTION (to 1), -** which makes erts_deliver_time/erts_time_remaining a bit faster. -** -** if HAVE_GETHRTIME is defined: -** sys_gethrtime() will return a SysHrTime (long long) representing -** nanoseconds, sys_init_hrtime() will do any initialization. -** else -** a long (64bit) integer type called Sint64 should be defined. -** -** sys_times() will return clock_ticks since start and -** fill in a SysTimes structure (struct tms). Instead of CLK_TCK, -** SYS_CLK_TCK is used to determine the resolution of kernel ticks. -** -** sys_gettimeofday() will take a SysTimeval (a struct timeval) as parameter -** and fill it in as gettimeofday(X,NULL). -** -*/ + * Support routines for the time + */ + +/* #define ERTS_TIME_CORRECTION_PRINT */ #ifdef HAVE_CONFIG_H # include "config.h" @@ -80,384 +31,1261 @@ #include "sys.h" #include "erl_vm.h" #include "global.h" - +#define ERTS_WANT_TIMER_WHEEL_API +#include "erl_time.h" +#include "erl_driver.h" +#include "erl_nif.h" + static erts_smp_mtx_t erts_timeofday_mtx; - -static SysTimeval inittv; /* Used everywhere, the initial time-of-day */ +static erts_smp_mtx_t erts_get_time_mtx; static SysTimes t_start; /* Used in elapsed_time_both */ -static SysTimeval gtv; /* Used in wall_clock_elapsed_time_both */ -static SysTimeval then; /* Used in get_now */ -static SysTimeval last_emu_time; /* Used in erts_get_emu_time() */ -SysTimeval erts_first_emu_time; /* Used in erts_get_emu_time() */ +static ErtsMonotonicTime prev_wall_clock_elapsed; /* Used in wall_clock_elapsed_time_both */ +static ErtsMonotonicTime previous_now; /* Used in get_now */ + +static ErtsMonitor *time_offset_monitors = NULL; +static Uint no_time_offset_monitors = 0; + +#ifdef DEBUG +static int time_sup_initialized = 0; +#endif -union { - erts_smp_atomic_t time; - char align[ERTS_CACHE_LINE_SIZE]; -} approx erts_align_attribute(ERTS_CACHE_LINE_SIZE); +#define ERTS_MONOTONIC_TIME_KILO \ + ((ErtsMonotonicTime) 1000) +#define ERTS_MONOTONIC_TIME_MEGA \ + (ERTS_MONOTONIC_TIME_KILO*ERTS_MONOTONIC_TIME_KILO) +#define ERTS_MONOTONIC_TIME_GIGA \ + (ERTS_MONOTONIC_TIME_MEGA*ERTS_MONOTONIC_TIME_KILO) +#define ERTS_MONOTONIC_TIME_TERA \ + (ERTS_MONOTONIC_TIME_GIGA*ERTS_MONOTONIC_TIME_KILO) +static void init_time_napi(void); static void -init_approx_time(void) +schedule_send_time_offset_changed_notifications(ErtsMonotonicTime new_offset); + +struct time_sup_read_only__ { + ErtsMonotonicTime (*get_time)(void); + int correction; + ErtsTimeWarpMode warp_mode; +#ifdef ERTS_HAVE_OS_MONOTONIC_TIME_SUPPORT + ErtsMonotonicTime moffset; + int os_corrected_monotonic_time; + int os_monotonic_time_disable; + char *os_monotonic_time_func; + char *os_monotonic_time_clock_id; + int os_monotonic_time_locked; + Uint64 os_monotonic_time_resolution; + Uint64 os_monotonic_time_extended; +#endif + char *os_system_time_func; + char *os_system_time_clock_id; + int os_system_time_locked; + Uint64 os_system_time_resolution; + Uint64 os_system_time_extended; + struct { + ErtsMonotonicTime large_diff; + ErtsMonotonicTime small_diff; + } adj; + struct { + ErtsMonotonicTime error; + ErtsMonotonicTime resolution; + int intervals; + int use_avg; + } drift_adj; +}; + +typedef struct { + ErtsMonotonicTime drift; /* Correction for os monotonic drift */ + ErtsMonotonicTime error; /* Correction for error between system times */ +} ErtsMonotonicCorrection; + +typedef struct { + ErtsMonotonicTime erl_mtime; + ErtsMonotonicTime os_mtime; + ErtsMonotonicCorrection correction; +} ErtsMonotonicCorrectionInstance; + +#define ERTS_MAX_DRIFT_INTERVALS 50 +typedef struct { + struct { + struct { + ErtsMonotonicTime sys; + ErtsMonotonicTime mon; + } diff; + struct { + ErtsMonotonicTime sys; + ErtsMonotonicTime mon; + } time; + } intervals[ERTS_MAX_DRIFT_INTERVALS]; + struct { + ErtsMonotonicTime sys; + ErtsMonotonicTime mon; + } acc; + int ix; + int dirty_counter; +} ErtsMonotonicDriftData; + +typedef struct { + ErtsMonotonicCorrectionInstance prev; + ErtsMonotonicCorrectionInstance curr; +} ErtsMonotonicCorrectionInstances; + +typedef struct { + ErtsMonotonicCorrectionInstances insts; + ErtsMonotonicDriftData drift; + ErtsMonotonicTime last_check; + int short_check_interval; +} ErtsMonotonicCorrectionData; + +struct time_sup_infrequently_changed__ { +#ifdef ERTS_HAVE_OS_MONOTONIC_TIME_SUPPORT + struct { + erts_smp_rwmtx_t rwmtx; + ErtsTWheelTimer timer; + ErtsMonotonicCorrectionData cdata; + } parmon; + ErtsMonotonicTime minit; +#endif + ErtsSystemTime sinit; + ErtsMonotonicTime not_corrected_moffset; + erts_smp_atomic64_t offset; + ErtsMonotonicTime shadow_offset; + erts_smp_atomic32_t preliminary_offset; +}; + +struct time_sup_frequently_changed__ { + ErtsMonotonicTime last_not_corrected_time; +}; + +static struct { + union { + struct time_sup_read_only__ o; + char align__[ERTS_ALC_CACHE_LINE_ALIGN_SIZE(sizeof(struct time_sup_read_only__))]; + } r; + union { + struct time_sup_infrequently_changed__ c; + char align__[ERTS_ALC_CACHE_LINE_ALIGN_SIZE(sizeof(struct time_sup_infrequently_changed__))]; + } inf; + union { + struct time_sup_frequently_changed__ c; + char align__[ERTS_ALC_CACHE_LINE_ALIGN_SIZE(sizeof(struct time_sup_frequently_changed__))]; + } f; +} time_sup erts_align_attribute(ERTS_CACHE_LINE_SIZE); + +ErtsTimeSupData erts_time_sup__ erts_align_attribute(ERTS_CACHE_LINE_SIZE); + +/* + * erts_get_approx_time() returns an *approximate* time + * in seconds. NOTE that this time may jump backwards!!! + */ +erts_approx_time_t +erts_get_approx_time(void) { - erts_smp_atomic_init_nob(&approx.time, 0); + ErtsSystemTime stime = erts_os_system_time(); + return (erts_approx_time_t) ERTS_MONOTONIC_TO_SEC(stime); } -static ERTS_INLINE erts_approx_time_t -get_approx_time(void) +static ERTS_INLINE void +init_time_offset(ErtsMonotonicTime offset) { - return (erts_approx_time_t) erts_smp_atomic_read_nob(&approx.time); + erts_smp_atomic64_init_nob(&time_sup.inf.c.offset, (erts_aint64_t) offset); } static ERTS_INLINE void -update_approx_time(SysTimeval *tv) +set_time_offset(ErtsMonotonicTime offset) +{ + erts_smp_atomic64_set_relb(&time_sup.inf.c.offset, (erts_aint64_t) offset); +} + +static ERTS_INLINE ErtsMonotonicTime +get_time_offset(void) { - erts_approx_time_t new_secs = (erts_approx_time_t) tv->tv_sec; - erts_approx_time_t old_secs = get_approx_time(); - if (old_secs != new_secs) - erts_smp_atomic_set_nob(&approx.time, new_secs); + return (ErtsMonotonicTime) erts_smp_atomic64_read_acqb(&time_sup.inf.c.offset); } +static ERTS_INLINE void +update_last_mtime(ErtsSchedulerData *esdp, ErtsMonotonicTime mtime) +{ + if (!esdp) + esdp = erts_get_scheduler_data(); + if (esdp) { + ASSERT(mtime >= esdp->last_monotonic_time); + esdp->last_monotonic_time = mtime; + esdp->check_time_reds = 0; + } +} + +#ifdef ERTS_HAVE_OS_MONOTONIC_TIME_SUPPORT + /* - * erts_get_approx_time() returns an *approximate* time - * in seconds. NOTE that this time may jump backwards!!! + * Time correction adjustments made due to + * error between Erlang system time and OS + * system time: + * - Large adjustment ~1% + * - Small adjustment ~0.05% */ -erts_approx_time_t -erts_get_approx_time(void) +#define ERTS_TCORR_ERR_UNIT 2048 +#define ERTS_TCORR_ERR_LARGE_ADJ 20 +#define ERTS_TCORR_ERR_SMALL_ADJ 1 + +#define ERTS_INIT_SHORT_INTERVAL_COUNTER 10 +#define ERTS_LONG_TIME_CORRECTION_CHECK ERTS_SEC_TO_MONOTONIC(60) +#define ERTS_SHORT_TIME_CORRECTION_CHECK ERTS_SEC_TO_MONOTONIC(15) + +#define ERTS_TIME_DRIFT_MAX_ADJ_DIFF ERTS_USEC_TO_MONOTONIC(50) +#define ERTS_TIME_DRIFT_MIN_ADJ_DIFF ERTS_USEC_TO_MONOTONIC(5) + +/* + * Maximum drift of the OS monotonic clock expected. + * + * We use 1 milli second per second. If the monotonic + * clock drifts more than this we will fail to adjust for + * drift, and error correction will kick in instead. + * If it is larger than this, one could argue that the + * primitive is to poor to be used... + */ +#define ERTS_MAX_MONOTONIC_DRIFT ERTS_MSEC_TO_MONOTONIC(1) + +/* + * We assume that precision is 32 times worse than the + * resolution. This is a wild guess, but there are no + * practical way to determine actual precision. + */ +#define ERTS_ASSUMED_PRECISION_DROP 32 + +#define ERTS_MIN_MONOTONIC_DRIFT_MEASUREMENT \ + (ERTS_SHORT_TIME_CORRECTION_CHECK - 2*ERTS_MAX_MONOTONIC_DRIFT) + + +static ERTS_INLINE ErtsMonotonicTime +calc_corrected_erl_mtime(ErtsMonotonicTime os_mtime, + ErtsMonotonicCorrectionInstance *cip, + ErtsMonotonicTime *os_mdiff_p, + int os_drift_corrected) { - return get_approx_time(); + ErtsMonotonicTime erl_mtime, diff = os_mtime - cip->os_mtime; + ERTS_TIME_ASSERT(diff >= 0); + if (!os_drift_corrected) + diff += (cip->correction.drift*diff)/ERTS_MONOTONIC_TIME_UNIT; + erl_mtime = cip->erl_mtime; + erl_mtime += diff; + erl_mtime += cip->correction.error*(diff/ERTS_TCORR_ERR_UNIT); + if (os_mdiff_p) + *os_mdiff_p = diff; + return erl_mtime; } -#ifdef HAVE_GETHRTIME +static ERTS_INLINE ErtsMonotonicTime +read_corrected_time(int os_drift_corrected) +{ + ErtsMonotonicTime os_mtime; + ErtsMonotonicCorrectionInstance ci; -int erts_disable_tolerant_timeofday; + erts_smp_rwmtx_rlock(&time_sup.inf.c.parmon.rwmtx); -static SysHrTime hr_init_time, hr_last_correction_check, - hr_correction, hr_last_time; + os_mtime = erts_os_monotonic_time(); -static void init_tolerant_timeofday(void) -{ - /* Should be in sys.c */ -#if defined(HAVE_SYSCONF) && defined(_SC_NPROCESSORS_CONF) - if (sysconf(_SC_NPROCESSORS_CONF) > 1) { - char b[1024]; - int maj,min,build; - os_flavor(b,1024); - os_version(&maj,&min,&build); - if (!strcmp(b,"sunos") && maj <= 5 && min <= 7) { - erts_disable_tolerant_timeofday = 1; - } + if (os_mtime >= time_sup.inf.c.parmon.cdata.insts.curr.os_mtime) + ci = time_sup.inf.c.parmon.cdata.insts.curr; + else { + if (os_mtime < time_sup.inf.c.parmon.cdata.insts.prev.os_mtime) + erts_exit(ERTS_ABORT_EXIT, + "OS monotonic time stepped backwards\n"); + ci = time_sup.inf.c.parmon.cdata.insts.prev; } + + erts_smp_rwmtx_runlock(&time_sup.inf.c.parmon.rwmtx); + + return calc_corrected_erl_mtime(os_mtime, &ci, NULL, + os_drift_corrected); +} + +static ErtsMonotonicTime get_os_drift_corrected_time(void) +{ + return read_corrected_time(!0); +} + +static ErtsMonotonicTime get_corrected_time(void) +{ + return read_corrected_time(0); +} + +#ifdef ERTS_TIME_CORRECTION_PRINT + +static ERTS_INLINE void +print_correction(int change, + ErtsMonotonicTime sdiff, + ErtsMonotonicTime old_ecorr, + ErtsMonotonicTime old_dcorr, + ErtsMonotonicTime new_ecorr, + ErtsMonotonicTime new_dcorr, + Uint tmo) +{ + ErtsMonotonicTime usec_sdiff; + if (sdiff < 0) + usec_sdiff = -1*ERTS_MONOTONIC_TO_USEC(-1*sdiff); + else + usec_sdiff = ERTS_MONOTONIC_TO_USEC(sdiff); + + if (!change) + erts_fprintf(stderr, + "sdiff = %b64d usec : [ec=%b64d ppm, dc=%b64d ppb] : " + "tmo = %bpu msec\r\n", + usec_sdiff, + (1000000*old_ecorr) / ERTS_TCORR_ERR_UNIT, + (1000000000*old_dcorr) / ERTS_MONOTONIC_TIME_UNIT, + tmo); + else + erts_fprintf(stderr, + "sdiff = %b64d usec : [ec=%b64d ppm, dc=%b64d ppb] " + "-> [ec=%b64d ppm, dc=%b64d ppb] : tmo = %bpu msec\r\n", + usec_sdiff, + (1000000*old_ecorr) / ERTS_TCORR_ERR_UNIT, + (1000000000*old_dcorr) / ERTS_MONOTONIC_TIME_UNIT, + (1000000*new_ecorr) / ERTS_TCORR_ERR_UNIT, + (1000000000*new_dcorr) / ERTS_MONOTONIC_TIME_UNIT, + tmo); +} + #endif - hr_init_time = sys_gethrtime(); - hr_last_correction_check = hr_last_time = hr_init_time; - hr_correction = 0; + +static ERTS_INLINE ErtsMonotonicTime +get_timeout_pos(ErtsMonotonicTime now, ErtsMonotonicTime tmo) +{ + ErtsMonotonicTime tpos; + tpos = ERTS_MONOTONIC_TO_CLKTCKS(now - 1); + tpos += ERTS_MSEC_TO_CLKTCKS(tmo); + tpos += 1; + return tpos; } -static void get_tolerant_timeofday(SysTimeval *tv) +static void +check_time_correction(void *vesdp) { - SysHrTime diff_time, curr; + int init_drift_adj = !vesdp; + ErtsSchedulerData *esdp = (ErtsSchedulerData *) vesdp; + ErtsMonotonicCorrection new_correction; + ErtsMonotonicCorrectionInstance ci; + ErtsMonotonicTime mdiff, sdiff, os_mtime, erl_mtime, os_stime, + erl_stime, time_offset, timeout_pos; + Uint timeout; + int os_drift_corrected = time_sup.r.o.os_corrected_monotonic_time; + int set_new_correction = 0, begin_short_intervals = 0; + + erts_smp_rwmtx_rlock(&time_sup.inf.c.parmon.rwmtx); + + erts_os_times(&os_mtime, &os_stime); + + ci = time_sup.inf.c.parmon.cdata.insts.curr; + + erts_smp_rwmtx_runlock(&time_sup.inf.c.parmon.rwmtx); - if (erts_disable_tolerant_timeofday) { - sys_gettimeofday(tv); - return; + if (os_mtime < ci.os_mtime) + erts_exit(ERTS_ABORT_EXIT, + "OS monotonic time stepped backwards\n"); + + erl_mtime = calc_corrected_erl_mtime(os_mtime, &ci, &mdiff, + os_drift_corrected); + time_offset = get_time_offset(); + erl_stime = erl_mtime + time_offset; + + sdiff = erl_stime - os_stime; + + if (time_sup.inf.c.shadow_offset) { + ERTS_TIME_ASSERT(time_sup.r.o.warp_mode == ERTS_SINGLE_TIME_WARP_MODE); + if (erts_smp_atomic32_read_nob(&time_sup.inf.c.preliminary_offset)) + sdiff += time_sup.inf.c.shadow_offset; + else + time_sup.inf.c.shadow_offset = 0; } - *tv = inittv; - diff_time = ((curr = sys_gethrtime()) + hr_correction - hr_init_time) / 1000; - if (curr < hr_init_time) { - erl_exit(1,"Unexpected behaviour from operating system high " - "resolution timer"); + new_correction = ci.correction; + + if (time_sup.r.o.warp_mode == ERTS_MULTI_TIME_WARP_MODE + && (sdiff < -2*time_sup.r.o.adj.small_diff + || 2*time_sup.r.o.adj.small_diff < sdiff)) { + /* System time diff exeeded limits; change time offset... */ + time_offset -= sdiff; + sdiff = 0; + set_time_offset(time_offset); + schedule_send_time_offset_changed_notifications(time_offset); + begin_short_intervals = 1; + if (ci.correction.error != 0) { + set_new_correction = 1; + new_correction.error = 0; + } + } + else if ((time_sup.r.o.warp_mode == ERTS_SINGLE_TIME_WARP_MODE + && erts_smp_atomic32_read_nob(&time_sup.inf.c.preliminary_offset)) + && (sdiff < -2*time_sup.r.o.adj.small_diff + || 2*time_sup.r.o.adj.small_diff < sdiff)) { + /* + * System time diff exeeded limits; change shadow offset + * and let OS system time leap away from Erlang system + * time. + */ + time_sup.inf.c.shadow_offset -= sdiff; + sdiff = 0; + begin_short_intervals = 1; + if (ci.correction.error != 0) { + set_new_correction = 1; + new_correction.error = 0; + } + } + else if (ci.correction.error == 0) { + if (sdiff < -time_sup.r.o.adj.small_diff) { + set_new_correction = 1; + if (sdiff < -time_sup.r.o.adj.large_diff) + new_correction.error = ERTS_TCORR_ERR_LARGE_ADJ; + else + new_correction.error = ERTS_TCORR_ERR_SMALL_ADJ; + } + else if (sdiff > time_sup.r.o.adj.small_diff) { + set_new_correction = 1; + if (sdiff > time_sup.r.o.adj.large_diff) + new_correction.error = -ERTS_TCORR_ERR_LARGE_ADJ; + else + new_correction.error = -ERTS_TCORR_ERR_SMALL_ADJ; + } + } + else if (ci.correction.error > 0) { + if (sdiff < 0) { + if (ci.correction.error != ERTS_TCORR_ERR_LARGE_ADJ + && sdiff < -time_sup.r.o.adj.large_diff) { + new_correction.error = ERTS_TCORR_ERR_LARGE_ADJ; + set_new_correction = 1; + } + } + else if (sdiff > time_sup.r.o.adj.small_diff) { + set_new_correction = 1; + if (sdiff > time_sup.r.o.adj.large_diff) + new_correction.error = -ERTS_TCORR_ERR_LARGE_ADJ; + else + new_correction.error = -ERTS_TCORR_ERR_SMALL_ADJ; + } + else { + set_new_correction = 1; + new_correction.error = 0; + } + } + else /* if (ci.correction.error < 0) */ { + if (0 < sdiff) { + if (ci.correction.error != -ERTS_TCORR_ERR_LARGE_ADJ + && time_sup.r.o.adj.large_diff < sdiff) { + new_correction.error = -ERTS_TCORR_ERR_LARGE_ADJ; + set_new_correction = 1; + } + } + else if (sdiff < -time_sup.r.o.adj.small_diff) { + set_new_correction = 1; + if (sdiff < -time_sup.r.o.adj.large_diff) + new_correction.error = ERTS_TCORR_ERR_LARGE_ADJ; + else + new_correction.error = ERTS_TCORR_ERR_SMALL_ADJ; + } + else { + set_new_correction = 1; + new_correction.error = 0; + } } - if ((curr - hr_last_correction_check) / 1000 > 1000000) { - /* Check the correction need */ - SysHrTime tv_diff, diffdiff; - SysTimeval tmp; - int done = 0; - - sys_gettimeofday(&tmp); - tv_diff = ((SysHrTime) tmp.tv_sec) * 1000000 + tmp.tv_usec; - tv_diff -= ((SysHrTime) inittv.tv_sec) * 1000000 + inittv.tv_usec; - diffdiff = diff_time - tv_diff; - if (diffdiff > 10000) { - SysHrTime corr = (curr - hr_last_time) / 100; - if (corr / 1000 >= diffdiff) { - ++done; - hr_correction -= ((SysHrTime)diffdiff) * 1000; - } else { - hr_correction -= corr; + if (!os_drift_corrected) { + ErtsMonotonicDriftData *ddp = &time_sup.inf.c.parmon.cdata.drift; + int ix = ddp->ix; + ErtsMonotonicTime mtime_diff, old_os_mtime; + + old_os_mtime = ddp->intervals[ix].time.mon; + mtime_diff = os_mtime - old_os_mtime; + + if ((mtime_diff >= ERTS_MIN_MONOTONIC_DRIFT_MEASUREMENT) + | init_drift_adj) { + ErtsMonotonicTime drift_adj, drift_adj_diff, old_os_stime, + smtime_diff, stime_diff, mtime_acc, stime_acc, + avg_drift_adj, max_drift; + + old_os_stime = ddp->intervals[ix].time.sys; + + mtime_acc = ddp->acc.mon; + stime_acc = ddp->acc.sys; + + avg_drift_adj = (((stime_acc - mtime_acc) + * ERTS_MONOTONIC_TIME_UNIT) + / mtime_acc); + + mtime_diff = os_mtime - old_os_mtime; + stime_diff = os_stime - old_os_stime; + smtime_diff = stime_diff - mtime_diff; + ix++; + if (ix >= time_sup.r.o.drift_adj.intervals) + ix = 0; + mtime_acc -= ddp->intervals[ix].diff.mon; + mtime_acc += mtime_diff; + stime_acc -= ddp->intervals[ix].diff.sys; + stime_acc += stime_diff; + + ddp->intervals[ix].diff.mon = mtime_diff; + ddp->intervals[ix].diff.sys = stime_diff; + ddp->intervals[ix].time.mon = os_mtime; + ddp->intervals[ix].time.sys = os_stime; + + ddp->ix = ix; + ddp->acc.mon = mtime_acc; + ddp->acc.sys = stime_acc; + + max_drift = ERTS_MAX_MONOTONIC_DRIFT; + max_drift *= ERTS_MONOTONIC_TO_SEC(mtime_diff); + + if (smtime_diff > time_sup.r.o.drift_adj.error + max_drift + || smtime_diff < -1*time_sup.r.o.drift_adj.error - max_drift) { + dirty_intervals: + /* + * We had a leap in system time. Mark array as + * dirty to ensure that dirty values are rotated + * out before we use it again... + */ + ddp->dirty_counter = time_sup.r.o.drift_adj.intervals; + begin_short_intervals = 1; } - diff_time = (curr + hr_correction - hr_init_time) / 1000; - } else if (diffdiff < -10000) { - SysHrTime corr = (curr - hr_last_time) / 100; - if (corr / 1000 >= -diffdiff) { - ++done; - hr_correction -= ((SysHrTime)diffdiff) * 1000; - } else { - hr_correction += corr; + else if (ddp->dirty_counter > 0) { + if (init_drift_adj) { + new_correction.drift = ((smtime_diff + * ERTS_MONOTONIC_TIME_UNIT) + / mtime_diff); + set_new_correction = 1; + } + ddp->dirty_counter--; + } + else { + if (ddp->dirty_counter == 0) { + /* Force set new drift correction... */ + set_new_correction = 1; + ddp->dirty_counter--; + } + + if (time_sup.r.o.drift_adj.use_avg) + drift_adj = (((stime_acc - mtime_acc) + * ERTS_MONOTONIC_TIME_UNIT) + / mtime_acc); + else + drift_adj = ((smtime_diff + * ERTS_MONOTONIC_TIME_UNIT) + / mtime_diff); + + drift_adj_diff = avg_drift_adj - drift_adj; + if (drift_adj_diff < -ERTS_TIME_DRIFT_MAX_ADJ_DIFF + || ERTS_TIME_DRIFT_MAX_ADJ_DIFF < drift_adj_diff) + goto dirty_intervals; + + drift_adj_diff = drift_adj - new_correction.drift; + if (drift_adj_diff) { + if (drift_adj_diff > ERTS_TIME_DRIFT_MAX_ADJ_DIFF) + drift_adj_diff = ERTS_TIME_DRIFT_MAX_ADJ_DIFF; + else if (drift_adj_diff < -ERTS_TIME_DRIFT_MAX_ADJ_DIFF) + drift_adj_diff = -ERTS_TIME_DRIFT_MAX_ADJ_DIFF; + new_correction.drift += drift_adj_diff; + if (drift_adj_diff < -ERTS_TIME_DRIFT_MIN_ADJ_DIFF + || ERTS_TIME_DRIFT_MIN_ADJ_DIFF < drift_adj_diff) { + set_new_correction = 1; + } + } } - diff_time = (curr + hr_correction - hr_init_time) / 1000; - } else { - ++done; } - if (done) { - hr_last_correction_check = curr; + } + + begin_short_intervals |= set_new_correction; + + if (begin_short_intervals) { + time_sup.inf.c.parmon.cdata.short_check_interval + = ERTS_INIT_SHORT_INTERVAL_COUNTER; + } + else if ((os_mtime - time_sup.inf.c.parmon.cdata.last_check + >= ERTS_SHORT_TIME_CORRECTION_CHECK - ERTS_MONOTONIC_TIME_UNIT) + && time_sup.inf.c.parmon.cdata.short_check_interval > 0) { + time_sup.inf.c.parmon.cdata.short_check_interval--; + } + time_sup.inf.c.parmon.cdata.last_check = os_mtime; + + if (new_correction.error == 0) + timeout = ERTS_MONOTONIC_TO_MSEC(ERTS_LONG_TIME_CORRECTION_CHECK); + else { + ErtsMonotonicTime ecorr = new_correction.error; + ErtsMonotonicTime abs_sdiff; + abs_sdiff = (sdiff < 0) ? -1*sdiff : sdiff; + if (ecorr < 0) + ecorr = -1*ecorr; + if (abs_sdiff > ecorr*(ERTS_LONG_TIME_CORRECTION_CHECK/ERTS_TCORR_ERR_UNIT)) + timeout = ERTS_MONOTONIC_TO_MSEC(ERTS_LONG_TIME_CORRECTION_CHECK); + else { + timeout = ERTS_MONOTONIC_TO_MSEC((ERTS_TCORR_ERR_UNIT*abs_sdiff)/ecorr); + if (timeout < 10) + timeout = 10; } } - tv->tv_sec += (int) (diff_time / ((SysHrTime) 1000000)); - tv->tv_usec += (int) (diff_time % ((SysHrTime) 1000000)); - if (tv->tv_usec >= 1000000) { - tv->tv_usec -= 1000000; - tv->tv_sec += 1; + + if (timeout > ERTS_MONOTONIC_TO_MSEC(ERTS_SHORT_TIME_CORRECTION_CHECK) + && (time_sup.inf.c.parmon.cdata.short_check_interval + || time_sup.inf.c.parmon.cdata.drift.dirty_counter >= 0)) { + timeout = ERTS_MONOTONIC_TO_MSEC(ERTS_SHORT_TIME_CORRECTION_CHECK); + } + + timeout_pos = get_timeout_pos(erl_mtime, timeout); + +#ifdef ERTS_TIME_CORRECTION_PRINT + print_correction(set_new_correction, + sdiff, + ci.correction.error, + ci.correction.drift, + new_correction.error, + new_correction.drift, + timeout); +#endif + + if (set_new_correction) { + erts_smp_rwmtx_rwlock(&time_sup.inf.c.parmon.rwmtx); + + os_mtime = erts_os_monotonic_time(); + + /* Save previous correction instance */ + time_sup.inf.c.parmon.cdata.insts.prev = ci; + + /* + * Current correction instance begin when + * OS monotonic time has increased two units. + */ + os_mtime += 2; + + /* + * Erlang monotonic time corresponding to + * next OS monotonic time using previous + * correction. + */ + erl_mtime = calc_corrected_erl_mtime(os_mtime, &ci, NULL, + os_drift_corrected); + + /* + * Save new current correction instance. + */ + time_sup.inf.c.parmon.cdata.insts.curr.erl_mtime = erl_mtime; + time_sup.inf.c.parmon.cdata.insts.curr.os_mtime = os_mtime; + time_sup.inf.c.parmon.cdata.insts.curr.correction = new_correction; + + erts_smp_rwmtx_rwunlock(&time_sup.inf.c.parmon.rwmtx); } - hr_last_time = curr; + + if (!esdp) + esdp = erts_get_scheduler_data(); + + erts_twheel_set_timer(esdp->timer_wheel, + &time_sup.inf.c.parmon.timer, + check_time_correction, + NULL, + (void *) esdp, + timeout_pos); } -#define correction (hr_correction/1000000) +static ErtsMonotonicTime get_os_corrected_time(void) +{ + ASSERT(time_sup.r.o.warp_mode == ERTS_MULTI_TIME_WARP_MODE); + return erts_os_monotonic_time() + time_sup.r.o.moffset; +} -#else /* !HAVE_GETHRTIME */ -#if !defined(CORRECT_USING_TIMES) -#define init_tolerant_timeofday() -#define get_tolerant_timeofday(tvp) sys_gettimeofday(tvp) -#else +static void +check_time_offset(void *vesdp) +{ + ErtsSchedulerData *esdp = (ErtsSchedulerData *) vesdp; + ErtsMonotonicTime sdiff, os_mtime, erl_mtime, os_stime, + erl_stime, time_offset, timeout, timeout_pos; -typedef Sint64 Milli; - -static clock_t init_ct; -static Sint64 ct_wrap; -static Milli init_tv_m; -static Milli correction_supress; -static Milli last_ct_diff; -static Milli last_cc; -static clock_t last_ct; - -/* sys_times() might need to be wrapped and the values shifted (right) - a bit to cope with newer linux (2.5.*) kernels, this has to be taken care - of dynamically to start with, a special version that uses - the times() return value as a high resolution timer can be made - to fully utilize the faster ticks, like on windows, but for now, we'll - settle with this silly workaround */ -#ifdef ERTS_WRAP_SYS_TIMES -#define KERNEL_TICKS() (sys_times_wrap() & \ - ((1UL << ((sizeof(clock_t) * 8) - 1)) - 1)) -#else -SysTimes dummy_tms; + ASSERT(time_sup.r.o.warp_mode == ERTS_MULTI_TIME_WARP_MODE); + + erts_os_times(&os_mtime, &os_stime); -#define KERNEL_TICKS() (sys_times(&dummy_tms) & \ - ((1UL << ((sizeof(clock_t) * 8) - 1)) - 1)) + erl_mtime = os_mtime + time_sup.r.o.moffset; + time_offset = get_time_offset(); + erl_stime = erl_mtime + time_offset; + sdiff = erl_stime - os_stime; + + if ((sdiff < -2*time_sup.r.o.adj.small_diff + || 2*time_sup.r.o.adj.small_diff < sdiff)) { + /* System time diff exeeded limits; change time offset... */ +#ifdef ERTS_TIME_CORRECTION_PRINT + erts_fprintf(stderr, "sdiff = %b64d nsec -> 0 nsec\n", + ERTS_MONOTONIC_TO_NSEC(sdiff)); +#endif + time_offset -= sdiff; + sdiff = 0; + set_time_offset(time_offset); + schedule_send_time_offset_changed_notifications(time_offset); + } +#ifdef ERTS_TIME_CORRECTION_PRINT + else erts_fprintf(stderr, "sdiff = %b64d nsec\n", + ERTS_MONOTONIC_TO_NSEC(sdiff)); #endif -static void init_tolerant_timeofday(void) + timeout = ERTS_MONOTONIC_TO_MSEC(ERTS_LONG_TIME_CORRECTION_CHECK); + timeout_pos = get_timeout_pos(erl_mtime, timeout); + + erts_twheel_set_timer(esdp->timer_wheel, + &time_sup.inf.c.parmon.timer, + check_time_offset, + NULL, + vesdp, + timeout_pos); +} + +static void +init_check_time_correction(void *vesdp) { - last_ct = init_ct = KERNEL_TICKS(); - last_cc = 0; - init_tv_m = (((Milli) inittv.tv_sec) * 1000) + - (inittv.tv_usec / 1000); - ct_wrap = 0; - correction_supress = 0; + ErtsMonotonicDriftData *ddp; + ErtsMonotonicTime old_mtime, old_stime, mtime, stime, mtime_diff, + stime_diff, smtime_diff, max_drift; + int ix; + + ddp = &time_sup.inf.c.parmon.cdata.drift; + ix = ddp->ix; + old_mtime = ddp->intervals[0].time.mon; + old_stime = ddp->intervals[0].time.sys; + + erts_os_times(&mtime, &stime); + + mtime_diff = mtime - old_mtime; + stime_diff = stime - old_stime; + smtime_diff = stime_diff - mtime_diff; + + max_drift = ERTS_MAX_MONOTONIC_DRIFT; + max_drift *= ERTS_MONOTONIC_TO_SEC(mtime_diff); + + if (smtime_diff > time_sup.r.o.drift_adj.error + max_drift + || smtime_diff < -1*time_sup.r.o.drift_adj.error - max_drift) { + /* Had a system time leap... pretend no drift... */ + stime_diff = mtime_diff; + } + + /* + * We use old time values in order to trigger + * a drift adjustment, and repeat this interval + * in all slots... + */ + for (ix = 0; ix < time_sup.r.o.drift_adj.intervals; ix++) { + ddp->intervals[ix].diff.mon = mtime_diff; + ddp->intervals[ix].diff.sys = stime_diff; + ddp->intervals[ix].time.mon = old_mtime; + ddp->intervals[ix].time.sys = old_stime; + } + + ddp->acc.sys = stime_diff*time_sup.r.o.drift_adj.intervals; + ddp->acc.mon = mtime_diff*time_sup.r.o.drift_adj.intervals; + ddp->ix = 0; + ddp->dirty_counter = time_sup.r.o.drift_adj.intervals; + + check_time_correction(vesdp); } +static ErtsMonotonicTime +finalize_corrected_time_offset(ErtsSystemTime *stimep) +{ + ErtsMonotonicTime os_mtime; + ErtsMonotonicCorrectionInstance ci; + int os_drift_corrected = time_sup.r.o.os_corrected_monotonic_time; + + erts_smp_rwmtx_rlock(&time_sup.inf.c.parmon.rwmtx); + + erts_os_times(&os_mtime, stimep); -static void get_tolerant_timeofday(SysTimeval *tvp) + ci = time_sup.inf.c.parmon.cdata.insts.curr; + + erts_smp_rwmtx_runlock(&time_sup.inf.c.parmon.rwmtx); + + if (os_mtime < ci.os_mtime) + erts_exit(ERTS_ABORT_EXIT, + "OS monotonic time stepped backwards\n"); + + return calc_corrected_erl_mtime(os_mtime, &ci, NULL, + os_drift_corrected); +} + +static void +late_init_time_correction(ErtsSchedulerData *esdp) { - clock_t current_ct; - SysTimeval current_tv; - Milli ct_diff; - Milli tv_diff; - Milli current_correction; - Milli act_correction; /* long shown to be too small */ - Milli max_adjust; + int quick_init_drift_adj; + void (*check_func)(void *); + ErtsMonotonicTime timeout, timeout_pos; + + quick_init_drift_adj = + ERTS_MONOTONIC_TO_USEC(time_sup.r.o.drift_adj.error) == 0; + + if (quick_init_drift_adj) + timeout = ERTS_MONOTONIC_TO_MSEC(ERTS_SHORT_TIME_CORRECTION_CHECK/10); + else + timeout = ERTS_MONOTONIC_TO_MSEC(ERTS_SHORT_TIME_CORRECTION_CHECK); + + if (!time_sup.r.o.os_corrected_monotonic_time) + check_func = init_check_time_correction; + else if (time_sup.r.o.get_time == get_os_corrected_time) { + quick_init_drift_adj = 0; + check_func = check_time_offset; + } + else + check_func = check_time_correction; + + timeout_pos = get_timeout_pos(erts_get_monotonic_time(esdp), + timeout); + + erts_twheel_init_timer(&time_sup.inf.c.parmon.timer); + erts_twheel_set_timer(esdp->timer_wheel, + &time_sup.inf.c.parmon.timer, + check_func, + NULL, + (quick_init_drift_adj + ? NULL + : esdp), + timeout_pos); +} + +#endif /* ERTS_HAVE_OS_MONOTONIC_TIME_SUPPORT */ + +static ErtsMonotonicTime get_not_corrected_time(void) +{ + ErtsMonotonicTime stime, mtime; + + erts_smp_mtx_lock(&erts_get_time_mtx); + + stime = erts_os_system_time(); + + mtime = stime - time_sup.inf.c.not_corrected_moffset; + + if (mtime >= time_sup.f.c.last_not_corrected_time) + time_sup.f.c.last_not_corrected_time = mtime; + else { + mtime = time_sup.f.c.last_not_corrected_time; + + if (time_sup.r.o.warp_mode == ERTS_MULTI_TIME_WARP_MODE) { + ErtsMonotonicTime new_offset = stime - mtime; + new_offset = ERTS_MONOTONIC_TO_USEC(new_offset); + new_offset = ERTS_USEC_TO_MONOTONIC(new_offset); + if (time_sup.inf.c.not_corrected_moffset != new_offset) { + time_sup.inf.c.not_corrected_moffset = new_offset; + set_time_offset(new_offset); + schedule_send_time_offset_changed_notifications(new_offset); + } + } - if (erts_disable_tolerant_timeofday) { - sys_gettimeofday(tvp); - return; } -#ifdef ERTS_WRAP_SYS_TIMES -#define TICK_MS (1000 / SYS_CLK_TCK_WRAP) + ASSERT(stime == mtime + time_sup.inf.c.not_corrected_moffset); + + erts_smp_mtx_unlock(&erts_get_time_mtx); + + return mtime; +} + +int erts_check_time_adj_support(int time_correction, + ErtsTimeWarpMode time_warp_mode) +{ + if (!time_correction) + return 1; + + /* User wants time correction */ + +#ifdef ERTS_HAVE_OS_MONOTONIC_TIME_SUPPORT + return !time_sup.r.o.os_monotonic_time_disable; #else -#define TICK_MS (1000 / SYS_CLK_TCK) + return 0; #endif - current_ct = KERNEL_TICKS(); - sys_gettimeofday(¤t_tv); - - /* I dont know if uptime can move some units backwards - on some systems, but I allow for small backward - jumps to avoid such problems if they exist...*/ - if (last_ct > 100 && current_ct < (last_ct - 100)) { - ct_wrap += ((Sint64) 1) << ((sizeof(clock_t) * 8) - 1); - } - last_ct = current_ct; - ct_diff = ((ct_wrap + current_ct) - init_ct) * TICK_MS; +} + +int +erts_has_time_correction(void) +{ + return time_sup.r.o.correction; +} + +void erts_init_sys_time_sup(void) +{ + ErtsSysInitTimeResult sys_init_time_res + = ERTS_SYS_INIT_TIME_RESULT_INITER; + + sys_init_time(&sys_init_time_res); + + erts_time_sup__.r.o.monotonic_time_unit + = sys_init_time_res.os_monotonic_time_unit; + +#ifndef SYS_CLOCK_RESOLUTION + erts_time_sup__.r.o.clktck_resolution + = sys_init_time_res.sys_clock_resolution; + erts_time_sup__.r.o.clktck_resolution *= 1000; +#endif + +#ifdef ERTS_HAVE_OS_MONOTONIC_TIME_SUPPORT + time_sup.r.o.os_monotonic_time_disable + = !sys_init_time_res.have_os_monotonic_time; + time_sup.r.o.os_corrected_monotonic_time = + sys_init_time_res.have_corrected_os_monotonic_time; + time_sup.r.o.os_monotonic_time_func + = sys_init_time_res.os_monotonic_time_info.func; + time_sup.r.o.os_monotonic_time_clock_id + = sys_init_time_res.os_monotonic_time_info.clock_id; + time_sup.r.o.os_monotonic_time_locked + = sys_init_time_res.os_monotonic_time_info.locked_use; + time_sup.r.o.os_monotonic_time_resolution + = sys_init_time_res.os_monotonic_time_info.resolution; + time_sup.r.o.os_monotonic_time_extended + = sys_init_time_res.os_monotonic_time_info.extended; +#endif + time_sup.r.o.os_system_time_func + = sys_init_time_res.os_system_time_info.func; + time_sup.r.o.os_system_time_clock_id + = sys_init_time_res.os_system_time_info.clock_id; + time_sup.r.o.os_system_time_locked + = sys_init_time_res.os_system_time_info.locked_use; + time_sup.r.o.os_system_time_resolution + = sys_init_time_res.os_system_time_info.resolution; +} + +int +erts_init_time_sup(int time_correction, ErtsTimeWarpMode time_warp_mode) +{ + ErtsMonotonicTime resolution, ilength, intervals, short_isecs; +#if !ERTS_COMPILE_TIME_MONOTONIC_TIME_UNIT + ErtsMonotonicTime abs_native_offset, native_offset; +#endif + + init_time_napi(); + + erts_hl_timer_init(); + + ASSERT(ERTS_MONOTONIC_TIME_MIN < ERTS_MONOTONIC_TIME_MAX); + + erts_smp_mtx_init(&erts_timeofday_mtx, "timeofday"); + erts_smp_mtx_init(&erts_get_time_mtx, "get_time"); + + time_sup.r.o.correction = time_correction; + time_sup.r.o.warp_mode = time_warp_mode; + + if (time_warp_mode == ERTS_SINGLE_TIME_WARP_MODE) + erts_smp_atomic32_init_nob(&time_sup.inf.c.preliminary_offset, 1); + else + erts_smp_atomic32_init_nob(&time_sup.inf.c.preliminary_offset, 0); + time_sup.inf.c.shadow_offset = 0; + +#if !ERTS_COMPILE_TIME_MONOTONIC_TIME_UNIT /* - * We will adjust the time in milliseconds and we allow for 1% - * adjustments, but if this function is called more often then every 100 - * millisecond (which is obviously possible), we will never adjust, so - * we accumulate small times by setting last_ct_diff iff max_adjust > 0 + * NOTE! erts_time_sup__.r.o.start *need* to be a multiple + * of ERTS_MONOTONIC_TIME_UNIT. */ - if ((max_adjust = (ct_diff - last_ct_diff)/100) > 0) - last_ct_diff = ct_diff; - tv_diff = ((((Milli) current_tv.tv_sec) * 1000) + - (current_tv.tv_usec / 1000)) - init_tv_m; +#ifdef ARCH_32 + erts_time_sup__.r.o.start = ((((ErtsMonotonicTime) 1) << 32)-1); + erts_time_sup__.r.o.start /= ERTS_MONOTONIC_TIME_UNIT; + erts_time_sup__.r.o.start *= ERTS_MONOTONIC_TIME_UNIT; + erts_time_sup__.r.o.start += ERTS_MONOTONIC_TIME_UNIT; + native_offset = erts_time_sup__.r.o.start - ERTS_MONOTONIC_BEGIN; + abs_native_offset = native_offset; +#else /* ARCH_64 */ + if (ERTS_MONOTONIC_TIME_UNIT <= 10*1000*1000) { + erts_time_sup__.r.o.start = 0; + native_offset = -ERTS_MONOTONIC_BEGIN; + abs_native_offset = ERTS_MONOTONIC_BEGIN; + } + else { + erts_time_sup__.r.o.start = ((ErtsMonotonicTime) MIN_SMALL); + erts_time_sup__.r.o.start /= ERTS_MONOTONIC_TIME_UNIT; + erts_time_sup__.r.o.start *= ERTS_MONOTONIC_TIME_UNIT; + native_offset = erts_time_sup__.r.o.start - ERTS_MONOTONIC_BEGIN; + abs_native_offset = -1*native_offset; + } +#endif + + erts_time_sup__.r.o.start_offset.native = native_offset; + erts_time_sup__.r.o.start_offset.nsec = (ErtsMonotonicTime) + erts_time_unit_conversion((Uint64) abs_native_offset, + (Uint32) ERTS_MONOTONIC_TIME_UNIT, + (Uint32) 1000*1000*1000); + erts_time_sup__.r.o.start_offset.usec = (ErtsMonotonicTime) + erts_time_unit_conversion((Uint64) abs_native_offset, + (Uint32) ERTS_MONOTONIC_TIME_UNIT, + (Uint32) 1000*1000); + erts_time_sup__.r.o.start_offset.msec = (ErtsMonotonicTime) + erts_time_unit_conversion((Uint64) abs_native_offset, + (Uint32) ERTS_MONOTONIC_TIME_UNIT, + (Uint32) 1000); + erts_time_sup__.r.o.start_offset.sec = (ErtsMonotonicTime) + erts_time_unit_conversion((Uint64) abs_native_offset, + (Uint32) ERTS_MONOTONIC_TIME_UNIT, + (Uint32) 1); + if (native_offset < 0) { + erts_time_sup__.r.o.start_offset.nsec *= -1; + erts_time_sup__.r.o.start_offset.usec *= -1; + erts_time_sup__.r.o.start_offset.msec *= -1; + erts_time_sup__.r.o.start_offset.sec *= -1; + } - current_correction = ((ct_diff - tv_diff) / TICK_MS) * TICK_MS; /* trunc */ +#endif - /* - * We allow the current_correction value to wobble a little, as it - * suffers from the low resolution of the kernel ticks. - * if it hasn't changed more than one tick in either direction, - * we will keep the old value. - */ - if ((last_cc > current_correction + TICK_MS) || - (last_cc < current_correction - TICK_MS)) { - last_cc = current_correction; - } else { - current_correction = last_cc; + resolution = time_sup.r.o.os_system_time_resolution; +#ifdef ERTS_HAVE_OS_MONOTONIC_TIME_SUPPORT + if (resolution > time_sup.r.o.os_monotonic_time_resolution) + resolution = time_sup.r.o.os_monotonic_time_resolution; +#endif + + time_sup.r.o.adj.large_diff = erts_time_sup__.r.o.monotonic_time_unit; + time_sup.r.o.adj.large_diff *= 50; + time_sup.r.o.adj.large_diff /= resolution; + if (time_sup.r.o.adj.large_diff < ERTS_USEC_TO_MONOTONIC(500)) + time_sup.r.o.adj.large_diff = ERTS_USEC_TO_MONOTONIC(500); + time_sup.r.o.adj.small_diff = time_sup.r.o.adj.large_diff/10; + + time_sup.r.o.drift_adj.resolution = resolution; + + if (time_sup.r.o.os_corrected_monotonic_time) { + time_sup.r.o.drift_adj.use_avg = 0; + time_sup.r.o.drift_adj.intervals = 0; + time_sup.r.o.drift_adj.error = 0; + time_sup.inf.c.parmon.cdata.drift.dirty_counter = -1; } - - /* - * As time goes, we try to get the actual correction to 0, - * that is, make erlangs time correspond to the systems dito. - * The act correction is what we seem to need (current_correction) - * minus the correction suppression. The correction supression - * will change slowly (max 1% of elapsed time) but in millisecond steps. - */ - act_correction = current_correction - correction_supress; - if (max_adjust > 0) { + else { /* - * Here we slowly adjust erlangs time to correspond with the - * system time by changing the correction_supress variable. - * It can change max_adjust milliseconds which is 1% of elapsed time + * Calculate length of the interval in seconds needed + * in order to get an error that is at most 1 micro second. + * If this interval is longer than the short time correction + * check interval we use the average of all values instead + * of the latest value. */ - if (act_correction > 0) { - if (current_correction - correction_supress > max_adjust) { - correction_supress += max_adjust; - } else { - correction_supress = current_correction; - } - act_correction = current_correction - correction_supress; - } else if (act_correction < 0) { - if (correction_supress - current_correction > max_adjust) { - correction_supress -= max_adjust; - } else { - correction_supress = current_correction; - } - act_correction = current_correction - correction_supress; + short_isecs = ERTS_MONOTONIC_TO_SEC(ERTS_SHORT_TIME_CORRECTION_CHECK); + ilength = ERTS_ASSUMED_PRECISION_DROP * ERTS_MONOTONIC_TIME_UNIT; + ilength /= (resolution * ERTS_USEC_TO_MONOTONIC(1)); + time_sup.r.o.drift_adj.use_avg = ilength > short_isecs; + + if (ilength == 0) + intervals = 5; + else { + intervals = ilength / short_isecs; + if (intervals > ERTS_MAX_DRIFT_INTERVALS) + intervals = ERTS_MAX_DRIFT_INTERVALS; + else if (intervals < 5) + intervals = 5; } + time_sup.r.o.drift_adj.intervals = (int) intervals; + + /* + * drift_adj.error equals maximum assumed error + * over a short time interval. We use this value also + * when examining a large interval. In this case the + * error will be smaller, but we do not want to + * recalculate this over and over again. + */ + + time_sup.r.o.drift_adj.error = ERTS_MONOTONIC_TIME_UNIT; + time_sup.r.o.drift_adj.error *= ERTS_ASSUMED_PRECISION_DROP; + time_sup.r.o.drift_adj.error /= resolution * short_isecs; } - /* - * The actual correction will correct the timeval so that system - * time warps gets smothed down. - */ - current_tv.tv_sec += act_correction / 1000; - current_tv.tv_usec += (act_correction % 1000) * 1000; - - if (current_tv.tv_usec >= 1000000) { - ++current_tv.tv_sec ; - current_tv.tv_usec -= 1000000; - } else if (current_tv.tv_usec < 0) { - --current_tv.tv_sec; - current_tv.tv_usec += 1000000; +#ifdef ERTS_TIME_CORRECTION_PRINT + erts_fprintf(stderr, "resolution = %b64d\n", resolution); + erts_fprintf(stderr, "adj large diff = %b64d usec\n", + ERTS_MONOTONIC_TO_USEC(time_sup.r.o.adj.large_diff)); + erts_fprintf(stderr, "adj small diff = %b64d usec\n", + ERTS_MONOTONIC_TO_USEC(time_sup.r.o.adj.small_diff)); + if (!time_sup.r.o.os_corrected_monotonic_time) { + erts_fprintf(stderr, "drift intervals = %d\n", + time_sup.r.o.drift_adj.intervals); + erts_fprintf(stderr, "drift adj error = %b64d usec\n", + ERTS_MONOTONIC_TO_USEC(time_sup.r.o.drift_adj.error)); + erts_fprintf(stderr, "drift adj max diff = %b64d nsec\n", + ERTS_MONOTONIC_TO_NSEC(ERTS_TIME_DRIFT_MAX_ADJ_DIFF)); + erts_fprintf(stderr, "drift adj min diff = %b64d nsec\n", + ERTS_MONOTONIC_TO_NSEC(ERTS_TIME_DRIFT_MIN_ADJ_DIFF)); } - *tvp = current_tv; -#undef TICK_MS -} +#endif -#endif /* CORRECT_USING_TIMES */ -#endif /* !HAVE_GETHRTIME */ + if (ERTS_MONOTONIC_TIME_UNIT < ERTS_CLKTCK_RESOLUTION) + ERTS_INTERNAL_ERROR("Too small monotonic time time unit"); -/* -** Why this? Well, most platforms have a constant clock resolution of 1, -** we dont want the deliver_time/time_remaining routines to waste -** time dividing and multiplying by/with a variable that's always one. -** so the return value of sys_init_time is ignored on those platforms. -*/ - -#ifndef SYS_CLOCK_RESOLUTION -static int clock_resolution; -#define CLOCK_RESOLUTION clock_resolution +#ifndef ERTS_HAVE_OS_MONOTONIC_TIME_SUPPORT + time_sup.r.o.correction = 0; #else -#define CLOCK_RESOLUTION SYS_CLOCK_RESOLUTION + if (time_sup.r.o.os_monotonic_time_disable) + time_sup.r.o.correction = 0; + + if (time_sup.r.o.correction) { + ErtsMonotonicCorrectionData *cdatap; + erts_smp_rwmtx_opt_t rwmtx_opts = ERTS_SMP_RWMTX_OPT_DEFAULT_INITER; + ErtsMonotonicTime offset; + erts_os_times(&time_sup.inf.c.minit, + &time_sup.inf.c.sinit); + time_sup.r.o.moffset = -1*time_sup.inf.c.minit; + time_sup.r.o.moffset += ERTS_MONOTONIC_BEGIN; + offset = time_sup.inf.c.sinit; + offset -= ERTS_MONOTONIC_BEGIN; + init_time_offset(offset); + + rwmtx_opts.type = ERTS_SMP_RWMTX_TYPE_EXTREMELY_FREQUENT_READ; + rwmtx_opts.lived = ERTS_SMP_RWMTX_LONG_LIVED; + + erts_smp_rwmtx_init_opt(&time_sup.inf.c.parmon.rwmtx, + &rwmtx_opts, "get_corrected_time"); + + cdatap = &time_sup.inf.c.parmon.cdata; + + cdatap->drift.intervals[0].time.sys = time_sup.inf.c.sinit; + cdatap->drift.intervals[0].time.mon = time_sup.inf.c.minit; + cdatap->insts.curr.correction.drift = 0; + cdatap->insts.curr.correction.error = 0; + cdatap->insts.curr.erl_mtime = ERTS_MONOTONIC_BEGIN; + cdatap->insts.curr.os_mtime = time_sup.inf.c.minit; + cdatap->last_check = time_sup.inf.c.minit; + cdatap->short_check_interval = ERTS_INIT_SHORT_INTERVAL_COUNTER; + cdatap->insts.prev = cdatap->insts.curr; + + if (!time_sup.r.o.os_corrected_monotonic_time) + time_sup.r.o.get_time = get_corrected_time; + else if (time_sup.r.o.warp_mode == ERTS_MULTI_TIME_WARP_MODE) + time_sup.r.o.get_time = get_os_corrected_time; + else + time_sup.r.o.get_time = get_os_drift_corrected_time; + } + else #endif + { + ErtsMonotonicTime stime, offset; + time_sup.r.o.get_time = get_not_corrected_time; + stime = time_sup.inf.c.sinit = erts_os_system_time(); + offset = stime - ERTS_MONOTONIC_BEGIN; + time_sup.inf.c.not_corrected_moffset = offset; + init_time_offset(offset); + time_sup.f.c.last_not_corrected_time = 0; + } -/* -** The clock resolution should really be the resolution of the -** time function in use, which on most platforms -** is 1. On VxWorks the resolution should be -** the number of ticks per second (or 1, which would work nicely to). -** -** Setting lower resolutions is mostly interesting when timers are used -** instead of something like select. -*/ + prev_wall_clock_elapsed = 0; + + previous_now = ERTS_MONOTONIC_TO_USEC(get_time_offset()); + +#ifdef DEBUG + time_sup_initialized = 1; +#endif -static SysTimeval last_delivered; + return ERTS_CLKTCK_RESOLUTION/1000; +} -static void init_erts_deliver_time(const SysTimeval *inittv) +void +erts_late_init_time_sup(void) { - /* We set the initial values for deliver_time here */ - last_delivered = *inittv; - last_delivered.tv_usec = 1000 * (last_delivered.tv_usec / 1000); - /* ms resolution */ + erts_late_sys_init_time(); } -static void do_erts_deliver_time(const SysTimeval *current) +void +erts_sched_init_time_sup(ErtsSchedulerData *esdp) { - SysTimeval cur_time; - erts_time_t elapsed; - - /* calculate and deliver appropriate number of ticks */ - cur_time = *current; - cur_time.tv_usec = 1000 * (cur_time.tv_usec / 1000); /* ms resolution */ - elapsed = (1000 * (cur_time.tv_sec - last_delivered.tv_sec) + - (cur_time.tv_usec - last_delivered.tv_usec) / 1000) / - CLOCK_RESOLUTION; - - /* Sometimes the time jump backwards, - resulting in a negative elapsed time. We compensate for - this by simply pretend as if the time stood still. :) */ - - if (elapsed > 0) { + esdp->timer_wheel = erts_create_timer_wheel(esdp); + esdp->next_tmo_ref = erts_get_next_timeout_reference(esdp->timer_wheel); + esdp->timer_service = erts_create_timer_service(); +#ifdef ERTS_HAVE_OS_MONOTONIC_TIME_SUPPORT + if (esdp->no == 1) { + /* A timer wheel to use must have beeen initialized */ + if (time_sup.r.o.get_time != get_not_corrected_time) + late_init_time_correction(esdp); + } +#endif +} - ASSERT(elapsed < ((erts_time_t) ERTS_SHORT_TIME_T_MAX)); +ErtsTimeWarpMode erts_time_warp_mode(void) +{ + return time_sup.r.o.warp_mode; +} - erts_do_time_add((erts_short_time_t) elapsed); - last_delivered = cur_time; +ErtsTimeOffsetState erts_time_offset_state(void) +{ + switch (time_sup.r.o.warp_mode) { + case ERTS_NO_TIME_WARP_MODE: + return ERTS_TIME_OFFSET_FINAL; + case ERTS_SINGLE_TIME_WARP_MODE: + if (erts_smp_atomic32_read_nob(&time_sup.inf.c.preliminary_offset)) + return ERTS_TIME_OFFSET_PRELIMINARY; + return ERTS_TIME_OFFSET_FINAL; + case ERTS_MULTI_TIME_WARP_MODE: + return ERTS_TIME_OFFSET_VOLATILE; + default: + ERTS_INTERNAL_ERROR("Invalid time warp mode"); + return ERTS_TIME_OFFSET_VOLATILE; } } -int -erts_init_time_sup(void) +/* + * erts_finalize_time_offset() will only change time offset + * the first time it is called when the emulator has been + * started in "single time warp" mode. Returns previous + * state: + * * ERTS_TIME_OFFSET_PRELIMINARY - Finalization performed + * * ERTS_TIME_OFFSET_FINAL - Already finialized; nothing changed + * * ERTS_TIME_OFFSET_VOLATILE - Not supported, either in + * * no correction mode (or multi time warp mode; not yet implemented). + */ + +ErtsTimeOffsetState +erts_finalize_time_offset(void) { - erts_smp_mtx_init(&erts_timeofday_mtx, "timeofday"); + switch (time_sup.r.o.warp_mode) { + case ERTS_NO_TIME_WARP_MODE: + return ERTS_TIME_OFFSET_FINAL; + case ERTS_MULTI_TIME_WARP_MODE: + return ERTS_TIME_OFFSET_VOLATILE; + case ERTS_SINGLE_TIME_WARP_MODE: { + ErtsTimeOffsetState res = ERTS_TIME_OFFSET_FINAL; - init_approx_time(); + erts_smp_mtx_lock(&erts_get_time_mtx); - last_emu_time.tv_sec = 0; - last_emu_time.tv_usec = 0; + if (erts_smp_atomic32_read_nob(&time_sup.inf.c.preliminary_offset)) { + ErtsMonotonicTime mtime, new_offset; -#ifndef SYS_CLOCK_RESOLUTION - clock_resolution = sys_init_time(); -#else - (void) sys_init_time(); +#ifdef ERTS_HAVE_OS_MONOTONIC_TIME_SUPPORT + if (!time_sup.r.o.correction) #endif - sys_gettimeofday(&inittv); - -#ifdef HAVE_GETHRTIME - sys_init_hrtime(); + { + ErtsMonotonicTime stime = erts_os_system_time(); + + mtime = stime - time_sup.inf.c.not_corrected_moffset; + + if (mtime >= time_sup.f.c.last_not_corrected_time) { + time_sup.f.c.last_not_corrected_time = mtime; + new_offset = time_sup.inf.c.not_corrected_moffset; + } + else { + mtime = time_sup.f.c.last_not_corrected_time; + + ASSERT(time_sup.inf.c.not_corrected_moffset != stime - mtime); + new_offset = stime - mtime; + time_sup.inf.c.not_corrected_moffset = new_offset; + } + + } +#ifdef ERTS_HAVE_OS_MONOTONIC_TIME_SUPPORT + else { + ErtsSystemTime stime; + mtime = finalize_corrected_time_offset(&stime); + new_offset = stime - mtime; + } #endif - init_tolerant_timeofday(); + new_offset = ERTS_MONOTONIC_TO_USEC(new_offset); + new_offset = ERTS_USEC_TO_MONOTONIC(new_offset); - init_erts_deliver_time(&inittv); - gtv = inittv; - then.tv_sec = then.tv_usec = 0; + set_time_offset(new_offset); + schedule_send_time_offset_changed_notifications(new_offset); - erts_deliver_time(); + erts_smp_atomic32_set_nob(&time_sup.inf.c.preliminary_offset, 0); + res = ERTS_TIME_OFFSET_PRELIMINARY; + } + + erts_smp_mtx_unlock(&erts_get_time_mtx); + + return res; + } + default: + ERTS_INTERNAL_ERROR("Invalid time warp mode"); + return ERTS_TIME_OFFSET_VOLATILE; + } +} - return CLOCK_RESOLUTION; -} /* info functions */ void @@ -498,23 +1326,17 @@ elapsed_time_both(UWord *ms_user, UWord *ms_sys, void wall_clock_elapsed_time_both(UWord *ms_total, UWord *ms_diff) { - UWord prev_total; - SysTimeval tv; + ErtsMonotonicTime now, elapsed; erts_smp_mtx_lock(&erts_timeofday_mtx); - get_tolerant_timeofday(&tv); - - *ms_total = 1000 * (tv.tv_sec - inittv.tv_sec) + - (tv.tv_usec - inittv.tv_usec) / 1000; - - prev_total = 1000 * (gtv.tv_sec - inittv.tv_sec) + - (gtv.tv_usec - inittv.tv_usec) / 1000; - *ms_diff = *ms_total - prev_total; - gtv = tv; + now = time_sup.r.o.get_time(); + update_last_mtime(NULL, now); - /* must sync the machine's idea of time here */ - do_erts_deliver_time(&tv); + elapsed = ERTS_MONOTONIC_TO_MSEC(now); + *ms_total = (UWord) elapsed; + *ms_diff = (UWord) (elapsed - prev_wall_clock_elapsed); + prev_wall_clock_elapsed = elapsed; erts_smp_mtx_unlock(&erts_timeofday_mtx); } @@ -890,146 +1712,742 @@ univ_to_local(Sint *year, Sint *month, Sint *day, return 0; } - /* get a timestamp */ void get_now(Uint* megasec, Uint* sec, Uint* microsec) { - SysTimeval now; + ErtsMonotonicTime now_megasec, now_sec, now, mtime, time_offset; + mtime = time_sup.r.o.get_time(); + time_offset = get_time_offset(); + update_last_mtime(NULL, mtime); + now = ERTS_MONOTONIC_TO_USEC(mtime + time_offset); + erts_smp_mtx_lock(&erts_timeofday_mtx); - - get_tolerant_timeofday(&now); - do_erts_deliver_time(&now); - - /* Make sure time is later than last */ - if (then.tv_sec > now.tv_sec || - (then.tv_sec == now.tv_sec && then.tv_usec >= now.tv_usec)) { - now = then; - now.tv_usec++; - } - /* Check for carry from above + general reasonability */ - if (now.tv_usec >= 1000000) { - now.tv_usec = 0; - now.tv_sec++; - } - then = now; + + /* Make sure now time is later than last time */ + if (now <= previous_now) + now = previous_now + 1; + + previous_now = now; erts_smp_mtx_unlock(&erts_timeofday_mtx); - - *megasec = (Uint) (now.tv_sec / 1000000); - *sec = (Uint) (now.tv_sec % 1000000); - *microsec = (Uint) (now.tv_usec); - update_approx_time(&now); + now_megasec = now / ERTS_MONOTONIC_TIME_TERA; + now_sec = now / ERTS_MONOTONIC_TIME_MEGA; + *megasec = (Uint) now_megasec; + *sec = (Uint) (now_sec - now_megasec*ERTS_MONOTONIC_TIME_MEGA); + *microsec = (Uint) (now - now_sec*ERTS_MONOTONIC_TIME_MEGA); + + ASSERT(((ErtsMonotonicTime) *megasec)*ERTS_MONOTONIC_TIME_TERA + + ((ErtsMonotonicTime) *sec)*ERTS_MONOTONIC_TIME_MEGA + + ((ErtsMonotonicTime) *microsec) == now); +} + +ErtsMonotonicTime +erts_get_monotonic_time(ErtsSchedulerData *esdp) +{ + ErtsMonotonicTime mtime = time_sup.r.o.get_time(); + update_last_mtime(esdp, mtime); + return mtime; +} + +ErtsMonotonicTime +erts_get_time_offset(void) +{ + return get_time_offset(); +} + +static ERTS_INLINE void +make_timestamp_value(Uint* megasec, Uint* sec, Uint* microsec, + ErtsMonotonicTime mtime, ErtsMonotonicTime offset) +{ + ErtsMonotonicTime stime, as; + Uint ms; + + stime = ERTS_MONOTONIC_TO_USEC(mtime + offset); + + as = stime / ERTS_MONOTONIC_TIME_MEGA; + *megasec = ms = (Uint) (stime / ERTS_MONOTONIC_TIME_TERA); + *sec = (Uint) (as - (((ErtsMonotonicTime) ms) + * ERTS_MONOTONIC_TIME_MEGA)); + *microsec = (Uint) (stime - as*ERTS_MONOTONIC_TIME_MEGA); + + ASSERT(((ErtsMonotonicTime) ms)*ERTS_MONOTONIC_TIME_TERA + + ((ErtsMonotonicTime) *sec)*ERTS_MONOTONIC_TIME_MEGA + + *microsec == stime); +} + +void +erts_make_timestamp_value(Uint* megasec, Uint* sec, Uint* microsec, + ErtsMonotonicTime mtime, ErtsMonotonicTime offset) +{ + make_timestamp_value(megasec, sec, microsec, mtime, offset); } void get_sys_now(Uint* megasec, Uint* sec, Uint* microsec) { - SysTimeval now; - - sys_gettimeofday(&now); - - *megasec = (Uint) (now.tv_sec / 1000000); - *sec = (Uint) (now.tv_sec % 1000000); - *microsec = (Uint) (now.tv_usec); + ErtsSystemTime stime = erts_os_system_time(); + ErtsSystemTime ms, s, us; - update_approx_time(&now); + us = ERTS_MONOTONIC_TO_USEC(stime); + s = us / (1000*1000); + ms = s / (1000*1000); + + *megasec = (Uint) ms; + *sec = (Uint) (s - ms*(1000*1000)); + *microsec = (Uint) (us - s*(1000*1000)); } +#ifdef HAVE_ERTS_NOW_CPU +void erts_get_now_cpu(Uint* megasec, Uint* sec, Uint* microsec) { + SysCpuTime t; + SysTimespec tp; + + sys_get_proc_cputime(t, tp); + *microsec = (Uint)(tp.tv_nsec / 1000); + t = (tp.tv_sec / 1000000); + *megasec = (Uint)(t % 1000000); + *sec = (Uint)(tp.tv_sec % 1000000); +} +#endif -/* deliver elapsed *ticks* to the machine - takes a pointer - to a struct timeval representing current time (to save - a gettimeofday() where possible) or NULL */ +#include "big.h" -void erts_deliver_time(void) { - SysTimeval now; - - erts_smp_mtx_lock(&erts_timeofday_mtx); - - get_tolerant_timeofday(&now); - do_erts_deliver_time(&now); - - erts_smp_mtx_unlock(&erts_timeofday_mtx); +void +erts_monitor_time_offset(Eterm id, Eterm ref) +{ + erts_smp_mtx_lock(&erts_get_time_mtx); + erts_add_monitor(&time_offset_monitors, MON_TIME_OFFSET, ref, id, NIL); + no_time_offset_monitors++; + erts_smp_mtx_unlock(&erts_get_time_mtx); +} - update_approx_time(&now); +int +erts_demonitor_time_offset(Eterm ref) +{ + int res; + ErtsMonitor *mon; + ASSERT(is_internal_ref(ref)); + erts_smp_mtx_lock(&erts_get_time_mtx); + mon = erts_remove_monitor(&time_offset_monitors, ref); + if (!mon) + res = 0; + else { + ASSERT(no_time_offset_monitors > 0); + no_time_offset_monitors--; + res = 1; + } + erts_smp_mtx_unlock(&erts_get_time_mtx); + if (res) + erts_destroy_monitor(mon); + return res; } -/* get *real* time (not ticks) remaining until next timeout - if there - isn't one, give a "long" time, that is guaranteed - to not cause overflow when we report elapsed time later on */ +typedef struct { + Eterm pid; + Eterm ref; + Eterm heap[REF_THING_SIZE]; +} ErtsTimeOffsetMonitorInfo; -void erts_time_remaining(SysTimeval *rem_time) +typedef struct { + Uint ix; + ErtsTimeOffsetMonitorInfo *to_mon_info; +} ErtsTimeOffsetMonitorContext; + +static void +save_time_offset_monitor(ErtsMonitor *mon, void *vcntxt) { - erts_time_t ticks; - SysTimeval cur_time; - erts_time_t elapsed; + ErtsTimeOffsetMonitorContext *cntxt; + Eterm *from_hp, *to_hp; + Uint mix; + int hix; - /* erts_next_time() returns no of ticks to next timeout or -1 if none */ + cntxt = (ErtsTimeOffsetMonitorContext *) vcntxt; + mix = (cntxt->ix)++; + cntxt->to_mon_info[mix].pid = mon->pid; + to_hp = &cntxt->to_mon_info[mix].heap[0]; - ticks = (erts_time_t) erts_next_time(); - if (ticks == (erts_time_t) -1) { - /* timer queue empty */ - /* this will cause at most 100000000 ticks */ - rem_time->tv_sec = 100000; - rem_time->tv_usec = 0; - } else { - /* next timeout after ticks ticks */ - ticks *= CLOCK_RESOLUTION; - - erts_smp_mtx_lock(&erts_timeofday_mtx); - - get_tolerant_timeofday(&cur_time); - cur_time.tv_usec = 1000 * - (cur_time.tv_usec / 1000);/* ms resolution*/ - elapsed = 1000 * (cur_time.tv_sec - last_delivered.tv_sec) + - (cur_time.tv_usec - last_delivered.tv_usec) / 1000; - - erts_smp_mtx_unlock(&erts_timeofday_mtx); + ASSERT(is_internal_ref(mon->ref)); + from_hp = internal_ref_val(mon->ref); + ASSERT(thing_arityval(*from_hp) + 1 == REF_THING_SIZE); + + for (hix = 0; hix < REF_THING_SIZE; hix++) + to_hp[hix] = from_hp[hix]; + + cntxt->to_mon_info[mix].ref + = make_internal_ref(&cntxt->to_mon_info[mix].heap[0]); + +} + +static void +send_time_offset_changed_notifications(void *new_offsetp) +{ + ErtsMonotonicTime new_offset; + ErtsTimeOffsetMonitorInfo *to_mon_info = NULL; /* Shut up faulty warning */ + Uint no_monitors; + char *tmp = NULL; + +#ifdef ARCH_64 + new_offset = (ErtsMonotonicTime) new_offsetp; +#else + new_offset = *((ErtsMonotonicTime *) new_offsetp); + erts_free(ERTS_ALC_T_NEW_TIME_OFFSET, new_offsetp); +#endif + new_offset -= ERTS_MONOTONIC_OFFSET_NATIVE; + + erts_smp_mtx_lock(&erts_get_time_mtx); + + no_monitors = no_time_offset_monitors; + if (no_monitors) { + ErtsTimeOffsetMonitorContext cntxt; + Uint alloc_sz; - if (ticks <= elapsed) { /* Ooops, better hurry */ - rem_time->tv_sec = rem_time->tv_usec = 0; - return; + /* Monitor info array size */ + alloc_sz = no_monitors*sizeof(ErtsTimeOffsetMonitorInfo); + /* + template max size */ + alloc_sz += 6*sizeof(Eterm); /* 5-tuple */ + alloc_sz += ERTS_MAX_SINT64_HEAP_SIZE*sizeof(Eterm); /* max offset size */ + tmp = erts_alloc(ERTS_ALC_T_TMP, alloc_sz); + + to_mon_info = (ErtsTimeOffsetMonitorInfo *) tmp; + cntxt.ix = 0; + cntxt.to_mon_info = to_mon_info; + + erts_doforall_monitors(time_offset_monitors, + save_time_offset_monitor, + &cntxt); + + ASSERT(cntxt.ix == no_monitors); + } + + erts_smp_mtx_unlock(&erts_get_time_mtx); + + if (no_monitors) { + Eterm *hp, *patch_refp, new_offset_term, message_template; + Uint mix, hsz; + + /* Make message template */ + + hp = (Eterm *) (tmp + no_monitors*sizeof(ErtsTimeOffsetMonitorInfo)); + + hsz = 6; /* 5-tuple */ + hsz += REF_THING_SIZE; + hsz += ERTS_SINT64_HEAP_SIZE(new_offset); + + if (IS_SSMALL(new_offset)) + new_offset_term = make_small(new_offset); + else + new_offset_term = erts_sint64_to_big(new_offset, &hp); + message_template = TUPLE5(hp, + am_CHANGE, + THE_NON_VALUE, /* Patch point for ref */ + am_time_offset, + am_clock_service, + new_offset_term); + patch_refp = &hp[2]; + + ASSERT(*patch_refp == THE_NON_VALUE); + + for (mix = 0; mix < no_monitors; mix++) { + Process *rp = erts_proc_lookup(to_mon_info[mix].pid); + if (rp) { + Eterm ref = to_mon_info[mix].ref; + ErtsProcLocks rp_locks = ERTS_PROC_LOCK_LINK; + erts_smp_proc_lock(rp, ERTS_PROC_LOCK_LINK); + if (erts_lookup_monitor(ERTS_P_MONITORS(rp), ref)) { + ErtsMessage *mp; + ErlOffHeap *ohp; + Eterm message; + + mp = erts_alloc_message_heap(rp, &rp_locks, + hsz, &hp, &ohp); + *patch_refp = ref; + ASSERT(hsz == size_object(message_template)); + message = copy_struct(message_template, hsz, &hp, ohp); + erts_queue_message(rp, &rp_locks, mp, message, NIL); + } + erts_smp_proc_unlock(rp, rp_locks); + } } - rem_time->tv_sec = (ticks - elapsed) / 1000; - rem_time->tv_usec = 1000 * ((ticks - elapsed) % 1000); + + erts_free(ERTS_ALC_T_TMP, tmp); } } -void erts_get_timeval(SysTimeval *tv) +static void +schedule_send_time_offset_changed_notifications(ErtsMonotonicTime new_offset) { - erts_smp_mtx_lock(&erts_timeofday_mtx); - get_tolerant_timeofday(tv); - erts_smp_mtx_unlock(&erts_timeofday_mtx); - update_approx_time(tv); +#ifdef ARCH_64 + void *new_offsetp = (void *) new_offset; + ASSERT(sizeof(void *) == sizeof(ErtsMonotonicTime)); +#else + void *new_offsetp = erts_alloc(ERTS_ALC_T_NEW_TIME_OFFSET, + sizeof(ErtsMonotonicTime)); + *((ErtsMonotonicTime *) new_offsetp) = new_offset; +#endif + erts_schedule_misc_aux_work(1, + send_time_offset_changed_notifications, + new_offsetp); } -erts_time_t -erts_get_time(void) +static ERTS_INLINE Eterm +make_time_val(Process *c_p, ErtsMonotonicTime time_val) { - SysTimeval sys_tv; - - erts_smp_mtx_lock(&erts_timeofday_mtx); - - get_tolerant_timeofday(&sys_tv); - - erts_smp_mtx_unlock(&erts_timeofday_mtx); + Sint64 val = (Sint64) time_val; + Eterm *hp; + Uint sz; - update_approx_time(&sys_tv); + if (IS_SSMALL(val)) + return make_small(val); - return sys_tv.tv_sec; + sz = ERTS_SINT64_HEAP_SIZE(val); + hp = HAlloc(c_p, sz); + return erts_sint64_to_big(val, &hp); } -#ifdef HAVE_ERTS_NOW_CPU -void erts_get_now_cpu(Uint* megasec, Uint* sec, Uint* microsec) { - SysCpuTime t; - SysTimespec tp; +Eterm +erts_get_monotonic_start_time(struct process *c_p) +{ + return make_time_val(c_p, ERTS_MONOTONIC_TIME_START_EXTERNAL); +} - sys_get_proc_cputime(t, tp); - *microsec = (Uint)(tp.tv_nsec / 1000); - t = (tp.tv_sec / 1000000); - *megasec = (Uint)(t % 1000000); - *sec = (Uint)(tp.tv_sec % 1000000); +Eterm +erts_get_monotonic_end_time(struct process *c_p) +{ + return make_time_val(c_p, ERTS_MONOTONIC_TIME_END_EXTERNAL); } + +static Eterm +bld_monotonic_time_source(Uint **hpp, Uint *szp, Sint64 os_mtime) +{ +#ifndef ERTS_HAVE_OS_MONOTONIC_TIME_SUPPORT + return NIL; +#else + int i = 0; + Eterm k[6]; + Eterm v[6]; + + if (time_sup.r.o.os_monotonic_time_disable) + return NIL; + + k[i] = erts_bld_atom(hpp, szp, "function"); + v[i++] = erts_bld_atom(hpp, szp, + time_sup.r.o.os_monotonic_time_func); + + if (time_sup.r.o.os_monotonic_time_clock_id) { + k[i] = erts_bld_atom(hpp, szp, "clock_id"); + v[i++] = erts_bld_atom(hpp, szp, + time_sup.r.o.os_monotonic_time_clock_id); + } + + k[i] = erts_bld_atom(hpp, szp, "resolution"); + v[i++] = erts_bld_uint64(hpp, szp, + time_sup.r.o.os_monotonic_time_resolution); + + k[i] = erts_bld_atom(hpp, szp, "extended"); + v[i++] = time_sup.r.o.os_monotonic_time_extended ? am_yes : am_no; + + k[i] = erts_bld_atom(hpp, szp, "parallel"); + v[i++] = time_sup.r.o.os_monotonic_time_locked ? am_no : am_yes; + + k[i] = erts_bld_atom(hpp, szp, "time"); + v[i++] = erts_bld_sint64(hpp, szp, os_mtime); + + return erts_bld_2tup_list(hpp, szp, (Sint) i, k, v); #endif +} + +Eterm +erts_monotonic_time_source(struct process *c_p) +{ + Uint hsz = 0; + Eterm *hp = NULL; + Sint64 os_mtime = 0; +#ifdef ERTS_HAVE_OS_MONOTONIC_TIME_SUPPORT + if (!time_sup.r.o.os_monotonic_time_disable) + os_mtime = (Sint64) erts_os_monotonic_time(); +#endif + + bld_monotonic_time_source(NULL, &hsz, os_mtime); + if (hsz) + hp = HAlloc(c_p, hsz); + return bld_monotonic_time_source(&hp, NULL, os_mtime); +} + +static Eterm +bld_system_time_source(Uint **hpp, Uint *szp, Sint64 os_stime) +{ + int i = 0; + Eterm k[5]; + Eterm v[5]; + + k[i] = erts_bld_atom(hpp, szp, "function"); + v[i++] = erts_bld_atom(hpp, szp, + time_sup.r.o.os_system_time_func); + + if (time_sup.r.o.os_system_time_clock_id) { + k[i] = erts_bld_atom(hpp, szp, "clock_id"); + v[i++] = erts_bld_atom(hpp, szp, + time_sup.r.o.os_system_time_clock_id); + } + + k[i] = erts_bld_atom(hpp, szp, "resolution"); + v[i++] = erts_bld_uint64(hpp, szp, + time_sup.r.o.os_system_time_resolution); + + k[i] = erts_bld_atom(hpp, szp, "parallel"); + v[i++] = am_yes; + + k[i] = erts_bld_atom(hpp, szp, "time"); + v[i++] = erts_bld_sint64(hpp, szp, os_stime); + + return erts_bld_2tup_list(hpp, szp, (Sint) i, k, v); +} + +Eterm +erts_system_time_source(struct process *c_p) +{ + Uint hsz = 0; + Eterm *hp = NULL; + Sint64 os_stime = (Sint64) erts_os_system_time(); + + bld_system_time_source(NULL, &hsz, os_stime); + if (hsz) + hp = HAlloc(c_p, hsz); + return bld_system_time_source(&hp, NULL, os_stime); +} + + +#include "bif.h" + +static ERTS_INLINE Eterm +time_unit_conversion(Process *c_p, Eterm term, ErtsMonotonicTime val, ErtsMonotonicTime muloff) +{ + ErtsMonotonicTime result; + BIF_RETTYPE ret; + + if (val < 0) + goto trap_to_erlang_code; + + /* Convert to common user specified time units */ + switch (term) { + case am_seconds: + case make_small(1): + result = ERTS_MONOTONIC_TO_SEC(val) + muloff*ERTS_MONOTONIC_OFFSET_SEC; + ERTS_BIF_PREP_RET(ret, make_time_val(c_p, result)); + break; + case am_milli_seconds: + case make_small(1000): + result = ERTS_MONOTONIC_TO_MSEC(val) + muloff*ERTS_MONOTONIC_OFFSET_MSEC; + ERTS_BIF_PREP_RET(ret, make_time_val(c_p, result)); + break; + case am_micro_seconds: + case make_small(1000*1000): + result = ERTS_MONOTONIC_TO_USEC(val) + muloff*ERTS_MONOTONIC_OFFSET_USEC; + ERTS_BIF_PREP_RET(ret, make_time_val(c_p, result)); + break; +#ifdef ARCH_64 + case am_nano_seconds: + case make_small(1000*1000*1000): + result = ERTS_MONOTONIC_TO_NSEC(val) + muloff*ERTS_MONOTONIC_OFFSET_NSEC; + ERTS_BIF_PREP_RET(ret, make_time_val(c_p, result)); + break; +#endif + default: { + Eterm value, native_res; +#ifndef ARCH_64 + Sint user_res; + if (term == am_nano_seconds) + goto to_nano_seconds; + if (term_to_Sint(term, &user_res)) { + if (user_res == 1000*1000*1000) { + to_nano_seconds: + result = (ERTS_MONOTONIC_TO_NSEC(val) + + muloff*ERTS_MONOTONIC_OFFSET_NSEC); + ERTS_BIF_PREP_RET(ret, make_time_val(c_p, result)); + break; + } + if (user_res <= 0) + goto badarg; + } +#else + if (is_small(term)) { + if (signed_val(term) <= 0) + goto badarg; + } +#endif + else if (is_big(term)) { + if (big_sign(term)) + goto badarg; + } + else { + badarg: + ERTS_BIF_PREP_ERROR(ret, c_p, BADARG); + break; + } + + trap_to_erlang_code: + /* Do it in erlang code instead; pass along values to use... */ + value = make_time_val(c_p, val + muloff*ERTS_MONOTONIC_OFFSET_NATIVE); + native_res = make_time_val(c_p, ERTS_MONOTONIC_TIME_UNIT); + + ERTS_BIF_PREP_TRAP3(ret, erts_convert_time_unit_trap, c_p, + value, native_res, term); + + break; + } + } + + return ret; +} + + +/* + * Time Native API (drivers and NIFs) + */ + +#define ERTS_NAPI_TIME_ERROR ((ErtsMonotonicTime) ERTS_NAPI_TIME_ERROR__) + +static void +init_time_napi(void) +{ + /* Verify that time native api constants are as expected... */ + + ASSERT(sizeof(ErtsMonotonicTime) == sizeof(ErlDrvTime)); + ASSERT(ERL_DRV_TIME_ERROR == (ErlDrvTime) ERTS_NAPI_TIME_ERROR); + ASSERT(ERL_DRV_TIME_ERROR < (ErlDrvTime) 0); + ASSERT(ERTS_NAPI_SEC__ == (int) ERL_DRV_SEC); + ASSERT(ERTS_NAPI_MSEC__ == (int) ERL_DRV_MSEC); + ASSERT(ERTS_NAPI_USEC__ == (int) ERL_DRV_USEC); + ASSERT(ERTS_NAPI_NSEC__ == (int) ERL_DRV_NSEC); + + ASSERT(sizeof(ErtsMonotonicTime) == sizeof(ErlNifTime)); + ASSERT(ERL_NIF_TIME_ERROR == (ErlNifTime) ERTS_NAPI_TIME_ERROR); + ASSERT(ERL_NIF_TIME_ERROR < (ErlNifTime) 0); + ASSERT(ERTS_NAPI_SEC__ == (int) ERL_NIF_SEC); + ASSERT(ERTS_NAPI_MSEC__ == (int) ERL_NIF_MSEC); + ASSERT(ERTS_NAPI_USEC__ == (int) ERL_NIF_USEC); + ASSERT(ERTS_NAPI_NSEC__ == (int) ERL_NIF_NSEC); +} + +ErtsMonotonicTime +erts_napi_monotonic_time(int time_unit) +{ + ErtsSchedulerData *esdp; + ErtsMonotonicTime mtime; + + /* At least for now only allow schedulers to do this... */ + esdp = erts_get_scheduler_data(); + if (!esdp) + return ERTS_NAPI_TIME_ERROR; + + mtime = time_sup.r.o.get_time(); + update_last_mtime(esdp, mtime); + + switch (time_unit) { + case ERTS_NAPI_SEC__: + mtime = ERTS_MONOTONIC_TO_SEC(mtime); + mtime += ERTS_MONOTONIC_OFFSET_SEC; + break; + case ERTS_NAPI_MSEC__: + mtime = ERTS_MONOTONIC_TO_MSEC(mtime); + mtime += ERTS_MONOTONIC_OFFSET_MSEC; + break; + case ERTS_NAPI_USEC__: + mtime = ERTS_MONOTONIC_TO_USEC(mtime); + mtime += ERTS_MONOTONIC_OFFSET_USEC; + break; + case ERTS_NAPI_NSEC__: + mtime = ERTS_MONOTONIC_TO_NSEC(mtime); + mtime += ERTS_MONOTONIC_OFFSET_NSEC; + break; + default: + return ERTS_NAPI_TIME_ERROR; + } + + return mtime; +} + +ErtsMonotonicTime +erts_napi_time_offset(int time_unit) +{ + ErtsSchedulerData *esdp; + ErtsSystemTime offs; + + /* At least for now only allow schedulers to do this... */ + esdp = erts_get_scheduler_data(); + if (!esdp) + return ERTS_NAPI_TIME_ERROR; + + offs = get_time_offset(); + switch (time_unit) { + case ERTS_NAPI_SEC__: + offs = ERTS_MONOTONIC_TO_SEC(offs); + offs -= ERTS_MONOTONIC_OFFSET_SEC; + break; + case ERTS_NAPI_MSEC__: + offs = ERTS_MONOTONIC_TO_MSEC(offs); + offs -= ERTS_MONOTONIC_OFFSET_MSEC; + break; + case ERTS_NAPI_USEC__: + offs = ERTS_MONOTONIC_TO_USEC(offs); + offs -= ERTS_MONOTONIC_OFFSET_USEC; + break; + case ERTS_NAPI_NSEC__: + offs = ERTS_MONOTONIC_TO_NSEC(offs); + offs -= ERTS_MONOTONIC_OFFSET_NSEC; + break; + default: + return ERTS_NAPI_TIME_ERROR; + } + return offs; +} + +ErtsMonotonicTime +erts_napi_convert_time_unit(ErtsMonotonicTime val, int from, int to) +{ + ErtsMonotonicTime ffreq, tfreq, denom; + /* + * Convertion between time units using floor function. + * + * Note that this needs to work also for negative + * values. Ordinary integer division on a negative + * value will give ceiling... + */ + + switch ((int) from) { + case ERTS_NAPI_SEC__: ffreq = 1; break; + case ERTS_NAPI_MSEC__: ffreq = 1000; break; + case ERTS_NAPI_USEC__: ffreq = 1000*1000; break; + case ERTS_NAPI_NSEC__: ffreq = 1000*1000*1000; break; + default: return ERTS_NAPI_TIME_ERROR; + } + + switch ((int) to) { + case ERTS_NAPI_SEC__: tfreq = 1; break; + case ERTS_NAPI_MSEC__: tfreq = 1000; break; + case ERTS_NAPI_USEC__: tfreq = 1000*1000; break; + case ERTS_NAPI_NSEC__: tfreq = 1000*1000*1000; break; + default: return ERTS_NAPI_TIME_ERROR; + } + + if (tfreq >= ffreq) + return val * (tfreq / ffreq); + + denom = ffreq / tfreq; + if (val >= 0) + return val / denom; + + return (val - (denom - 1)) / denom; +} + +/* Built in functions */ + +BIF_RETTYPE monotonic_time_0(BIF_ALIST_0) +{ + ErtsMonotonicTime mtime = time_sup.r.o.get_time(); + update_last_mtime(ERTS_PROC_GET_SCHDATA(BIF_P), mtime); + mtime += ERTS_MONOTONIC_OFFSET_NATIVE; + BIF_RET(make_time_val(BIF_P, mtime)); +} + +BIF_RETTYPE monotonic_time_1(BIF_ALIST_1) +{ + ErtsMonotonicTime mtime = time_sup.r.o.get_time(); + update_last_mtime(ERTS_PROC_GET_SCHDATA(BIF_P), mtime); + BIF_RET(time_unit_conversion(BIF_P, BIF_ARG_1, mtime, 1)); +} + +BIF_RETTYPE system_time_0(BIF_ALIST_0) +{ + ErtsMonotonicTime mtime, offset; + mtime = time_sup.r.o.get_time(); + offset = get_time_offset(); + update_last_mtime(ERTS_PROC_GET_SCHDATA(BIF_P), mtime); + BIF_RET(make_time_val(BIF_P, mtime + offset)); +} + +BIF_RETTYPE system_time_1(BIF_ALIST_0) +{ + ErtsMonotonicTime mtime, offset; + mtime = time_sup.r.o.get_time(); + offset = get_time_offset(); + update_last_mtime(ERTS_PROC_GET_SCHDATA(BIF_P), mtime); + BIF_RET(time_unit_conversion(BIF_P, BIF_ARG_1, mtime + offset, 0)); +} + +BIF_RETTYPE erts_internal_time_unit_0(BIF_ALIST_0) +{ + BIF_RET(make_time_val(BIF_P, ERTS_MONOTONIC_TIME_UNIT)); +} + +BIF_RETTYPE time_offset_0(BIF_ALIST_0) +{ + ErtsMonotonicTime time_offset = get_time_offset(); + time_offset -= ERTS_MONOTONIC_OFFSET_NATIVE; + BIF_RET(make_time_val(BIF_P, time_offset)); +} + +BIF_RETTYPE time_offset_1(BIF_ALIST_1) +{ + BIF_RET(time_unit_conversion(BIF_P, BIF_ARG_1, get_time_offset(), -1)); +} + + +BIF_RETTYPE timestamp_0(BIF_ALIST_0) +{ + Eterm *hp, res; + ErtsMonotonicTime mtime, offset; + Uint mega_sec, sec, micro_sec; + + mtime = time_sup.r.o.get_time(); + offset = get_time_offset(); + update_last_mtime(ERTS_PROC_GET_SCHDATA(BIF_P), mtime); + + make_timestamp_value(&mega_sec, &sec, µ_sec, mtime, offset); + + /* + * Mega seconds is the only value that potentially + * ever could be a bignum. However, that wont happen + * during at least the next 4 million years... + * + * (System time will also have wrapped in the + * 64-bit integer before we get there...) + */ + + ASSERT(IS_USMALL(0, mega_sec)); + ASSERT(IS_USMALL(0, sec)); + ASSERT(IS_USMALL(0, micro_sec)); + + hp = HAlloc(BIF_P, 4); + res = TUPLE3(hp, + make_small(mega_sec), + make_small(sec), + make_small(micro_sec)); + BIF_RET(res); +} + +BIF_RETTYPE os_system_time_0(BIF_ALIST_0) +{ + ErtsSystemTime stime = erts_os_system_time(); + BIF_RET(make_time_val(BIF_P, stime)); +} + +BIF_RETTYPE os_system_time_1(BIF_ALIST_1) +{ + ErtsSystemTime stime = erts_os_system_time(); + BIF_RET(time_unit_conversion(BIF_P, BIF_ARG_1, stime, 0)); +} + +BIF_RETTYPE +os_perf_counter_0(BIF_ALIST_0) +{ + BIF_RET(make_time_val(BIF_P, erts_sys_perf_counter())); +} + +BIF_RETTYPE erts_internal_perf_counter_unit_0(BIF_ALIST_0) +{ + BIF_RET(make_time_val(BIF_P, erts_sys_perf_counter_unit())); +} diff --git a/erts/emulator/beam/erl_trace.c b/erts/emulator/beam/erl_trace.c index 305058ceff..9033c1b75c 100644 --- a/erts/emulator/beam/erl_trace.c +++ b/erts/emulator/beam/erl_trace.c @@ -1,18 +1,19 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 1999-2013. All Rights Reserved. + * Copyright Ericsson AB 1999-2014. All Rights Reserved. * - * The contents of this file are subject to the Erlang Public License, - * Version 1.1, (the "License"); you may not use this file except in - * compliance with the License. You should have received a copy of the - * Erlang Public License along with this software. If not, it can be - * retrieved online at http://www.erlang.org/. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Software distributed under the License is distributed on an "AS IS" - * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See - * the License for the specific language governing rights and limitations - * under the License. + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. * * %CopyrightEnd% */ @@ -37,6 +38,7 @@ #include "erl_binary.h" #include "erl_bits.h" #include "erl_thr_progress.h" +#include "erl_bif_unique.h" #if 0 #define DEBUG_PRINTOUTS @@ -76,6 +78,263 @@ enum ErtsSysMsgType { SYS_MSG_TYPE_SYSPROF }; +#define ERTS_TRACE_TS_NOW_MAX_SIZE \ + 4 +#define ERTS_TRACE_TS_MONOTONIC_MAX_SIZE \ + ERTS_MAX_SINT64_HEAP_SIZE +#define ERTS_TRACE_TS_STRICT_MONOTONIC_MAX_SIZE \ + (3 + ERTS_MAX_SINT64_HEAP_SIZE \ + + ERTS_MAX_UINT64_HEAP_SIZE) + +#define ERTS_TRACE_PATCH_TS_MAX_SIZE \ + (1 + ((ERTS_TRACE_TS_NOW_MAX_SIZE \ + > ERTS_TRACE_TS_MONOTONIC_MAX_SIZE) \ + ? ((ERTS_TRACE_TS_NOW_MAX_SIZE \ + > ERTS_TRACE_TS_STRICT_MONOTONIC_MAX_SIZE) \ + ? ERTS_TRACE_TS_NOW_MAX_SIZE \ + : ERTS_TRACE_TS_STRICT_MONOTONIC_MAX_SIZE) \ + : ((ERTS_TRACE_TS_MONOTONIC_MAX_SIZE \ + > ERTS_TRACE_TS_STRICT_MONOTONIC_MAX_SIZE) \ + ? ERTS_TRACE_TS_MONOTONIC_MAX_SIZE \ + : ERTS_TRACE_TS_STRICT_MONOTONIC_MAX_SIZE))) + +#define TFLGS_TS_TYPE(p) ERTS_TFLGS2TSTYPE(ERTS_TRACE_FLAGS((p))) + +/* + * FUTURE CHANGES: + * + * The timestamp functionality has intentionally been + * split in two parts for future use even though it + * is not used like this today. take_timestamp() takes + * the timestamp and calculate heap need for it (which + * is not constant). write_timestamp() writes the + * timestamp to the allocated heap. That is, one typically + * want to take the timestamp before allocating the heap + * and then write it to the heap. + * + * The trace output functionality now use patch_ts_size(), + * write_ts(), and patch_ts(). write_ts() both takes the + * timestamp and writes it. Since we don't know the + * heap need when allocating the heap area we need to + * over allocate (maximum size from patch_ts_size()) and + * then potentially (often) shrink the heap area after the + * timestamp has been written. The only reason it is + * currently done this way is because we do not want to + * make major changes of the trace behavior in a patch. + * This is planned to be changed in next major release. + */ + +typedef struct { + int ts_type_flag; + union { + struct { + Uint ms; + Uint s; + Uint us; + } now; + struct { + ErtsMonotonicTime time; + Sint64 raw_unique; + } monotonic; + } u; +} ErtsTraceTimeStamp; + +static ERTS_INLINE Uint +take_timestamp(ErtsTraceTimeStamp *tsp, int ts_type) +{ + int ts_type_flag = ts_type & -ts_type; /* least significant flag */ + + ASSERT(ts_type_flag == ERTS_TRACE_FLG_NOW_TIMESTAMP + || ts_type_flag == ERTS_TRACE_FLG_MONOTONIC_TIMESTAMP + || ts_type_flag == ERTS_TRACE_FLG_STRICT_MONOTONIC_TIMESTAMP + || ts_type_flag == 0); + + tsp->ts_type_flag = ts_type_flag; + switch (ts_type_flag) { + case 0: + return (Uint) 0; + case ERTS_TRACE_FLG_NOW_TIMESTAMP: +#ifdef HAVE_ERTS_NOW_CPU + if (erts_cpu_timestamp) + erts_get_now_cpu(&tsp->u.now.ms, &tsp->u.now.s, &tsp->u.now.us); + else +#endif + get_now(&tsp->u.now.ms, &tsp->u.now.s, &tsp->u.now.us); + return (Uint) 4; + case ERTS_TRACE_FLG_MONOTONIC_TIMESTAMP: + case ERTS_TRACE_FLG_STRICT_MONOTONIC_TIMESTAMP: { + Uint hsz = 0; + ErtsMonotonicTime mtime = erts_get_monotonic_time(NULL); + mtime = ERTS_MONOTONIC_TO_NSEC(mtime); + mtime += ERTS_MONOTONIC_OFFSET_NSEC; + hsz = (IS_SSMALL(mtime) ? + (Uint) 0 + : ERTS_SINT64_HEAP_SIZE((Sint64) mtime)); + tsp->u.monotonic.time = mtime; + if (ts_type_flag == ERTS_TRACE_FLG_STRICT_MONOTONIC_TIMESTAMP) { + Sint64 raw_unique; + hsz += 3; /* 2-tuple */ + raw_unique = erts_raw_get_unique_monotonic_integer(); + tsp->u.monotonic.raw_unique = raw_unique; + hsz += erts_raw_unique_monotonic_integer_heap_size(raw_unique, 0); + } + return hsz; + } + default: + ERTS_INTERNAL_ERROR("invalid timestamp type"); + return 0; + } +} + +static ERTS_INLINE Eterm +write_timestamp(ErtsTraceTimeStamp *tsp, Eterm **hpp) +{ + int ts_type_flag = tsp->ts_type_flag; + Eterm res; + + switch (ts_type_flag) { + case 0: + return NIL; + case ERTS_TRACE_FLG_NOW_TIMESTAMP: + res = TUPLE3(*hpp, + make_small(tsp->u.now.ms), + make_small(tsp->u.now.s), + make_small(tsp->u.now.us)); + *hpp += 4; + return res; + case ERTS_TRACE_FLG_MONOTONIC_TIMESTAMP: + case ERTS_TRACE_FLG_STRICT_MONOTONIC_TIMESTAMP: { + Sint64 mtime, raw; + Eterm unique, emtime; + + mtime = (Sint64) tsp->u.monotonic.time; + emtime = (IS_SSMALL(mtime) + ? make_small((Sint64) mtime) + : erts_sint64_to_big((Sint64) mtime, hpp)); + + if (ts_type_flag == ERTS_TRACE_FLG_MONOTONIC_TIMESTAMP) + return emtime; + + raw = tsp->u.monotonic.raw_unique; + unique = erts_raw_make_unique_monotonic_integer_value(hpp, raw, 0); + res = TUPLE2(*hpp, emtime, unique); + *hpp += 3; + return res; + } + default: + ERTS_INTERNAL_ERROR("invalid timestamp type"); + return THE_NON_VALUE; + } +} + +#define PATCH_TS_SIZE(p) patch_ts_size(TFLGS_TS_TYPE(p)) + +static ERTS_INLINE Uint +patch_ts_size(int ts_type) +{ + int ts_type_flag = ts_type & -ts_type; /* least significant flag */ + switch (ts_type_flag) { + case 0: + return 0; + case ERTS_TRACE_FLG_NOW_TIMESTAMP: + return 1 + ERTS_TRACE_TS_NOW_MAX_SIZE; + case ERTS_TRACE_FLG_MONOTONIC_TIMESTAMP: + return 1 + ERTS_TRACE_TS_MONOTONIC_MAX_SIZE; + case ERTS_TRACE_FLG_STRICT_MONOTONIC_TIMESTAMP: + return 1 + ERTS_TRACE_TS_STRICT_MONOTONIC_MAX_SIZE; + default: + ERTS_INTERNAL_ERROR("invalid timestamp type"); + return 0; + } +} + +/* + * Write a timestamp. The timestamp MUST be the last + * thing built on the heap. This since write_ts() might + * adjust the size of the used area. + */ +static Eterm +write_ts(int ts_type, Eterm *hp, ErlHeapFragment *bp, Process *tracer) +{ + ErtsTraceTimeStamp ts; + Sint shrink; + Eterm res, *ts_hp = hp; + Uint hsz; + + ASSERT(ts_type); + + hsz = take_timestamp(&ts, ts_type); + + res = write_timestamp(&ts, &ts_hp); + + ASSERT(ts_hp == hp + hsz); + + switch (ts.ts_type_flag) { + case ERTS_TRACE_FLG_MONOTONIC_TIMESTAMP: + shrink = ERTS_TRACE_TS_MONOTONIC_MAX_SIZE; + break; + case ERTS_TRACE_FLG_STRICT_MONOTONIC_TIMESTAMP: + shrink = ERTS_TRACE_TS_STRICT_MONOTONIC_MAX_SIZE; + break; + default: + return res; + } + + shrink -= hsz; + + ASSERT(shrink >= 0); + + if (shrink) { + if (bp) + bp->used_size -= shrink; +#ifndef ERTS_SMP + else if (tracer) { + Eterm *endp = ts_hp + shrink; + HRelease(tracer, endp, ts_hp); + } +#endif + } + + return res; +} + +/* + * Patch a timestamp into a tuple. The tuple MUST be the last thing + * built on the heap before the call, and the timestamp MUST be + * the last thing after the call. This since patch_ts() might adjust + * the size of the used area. + */ + +#define PATCH_TS__(Type, Tuple, Hp, Bp, Tracer) \ + do { \ + int ts_type__ = (Type); \ + if (ts_type__) \ + patch_ts(ts_type__, (Tuple), (Hp), (Bp), (Tracer)); \ + } while (0) + +#ifdef ERTS_SMP +#define PATCH_TS(Type, Tuple, Hp, Bp, Tracer) \ + PATCH_TS__((Type), (Tuple), (Hp), (Bp), NULL) +#else +#define PATCH_TS(Type, Tuple, Hp, Bp, Tracer) \ + PATCH_TS__((Type), (Tuple), (Hp), (Bp), (Tracer)) +#endif + +static ERTS_INLINE void +patch_ts(int ts_type, Eterm tuple, Eterm* hp, ErlHeapFragment *bp, Process *tracer) +{ + Eterm *tptr = tuple_val(tuple); + int arity = arityval(*tptr); + + ASSERT(ts_type); + ASSERT((tptr+arity+1) == hp); + + tptr[0] = make_arityval(arity+1); + tptr[1] = am_trace_ts; + + *hp = write_ts(ts_type, hp+1, bp, tracer); +} + #ifdef ERTS_SMP static void enqueue_sys_msg_unlocked(enum ErtsSysMsgType type, Eterm from, @@ -113,15 +372,10 @@ void erts_init_trace(void) { static Eterm system_seq_tracer; -#ifdef ERTS_SMP #define ERTS_ALLOC_SYSMSG_HEAP(SZ, BPP, OHPP, UNUSED) \ (*(BPP) = new_message_buffer((SZ)), \ *(OHPP) = &(*(BPP))->off_heap, \ (*(BPP))->mem) -#else -#define ERTS_ALLOC_SYSMSG_HEAP(SZ, BPP, OHPP, RPP) \ - erts_alloc_message_heap((SZ), (BPP), (OHPP), (RPP), 0) -#endif #ifdef ERTS_SMP #define ERTS_ENQ_TRACE_MSG(FPID, TPID, MSG, BP) \ @@ -130,13 +384,12 @@ do { \ enqueue_sys_msg_unlocked(SYS_MSG_TYPE_TRACE, (FPID), (TPID), (MSG), (BP)); \ } while(0) #else -#ifdef USE_VM_PROBES -#define ERTS_ENQ_TRACE_MSG(FPID, TPROC, MSG, BP) \ - erts_queue_message((TPROC), NULL, (BP), (MSG), NIL, NIL) -#else -#define ERTS_ENQ_TRACE_MSG(FPID, TPROC, MSG, BP) \ - erts_queue_message((TPROC), NULL, (BP), (MSG), NIL) -#endif +#define ERTS_ENQ_TRACE_MSG(FPID, TPROC, MSG, BP) \ + do { \ + ErtsMessage *mp__ = erts_alloc_message(0, NULL); \ + mp__->data.heap_frag = (BP); \ + erts_queue_message((TPROC), NULL, mp__, (MSG), NIL); \ + } while (0) #endif /* @@ -369,23 +622,6 @@ erts_get_system_profile(void) { return profile; } - -#ifdef HAVE_ERTS_NOW_CPU -# define GET_NOW(m, s, u) \ -do { \ - if (erts_cpu_timestamp) \ - erts_get_now_cpu(m, s, u); \ - else \ - get_now(m, s, u); \ -} while (0) -#else -# define GET_NOW(m, s, u) do {get_now(m, s, u);} while (0) -#endif - - - -static Eterm* patch_ts(Eterm tuple4, Eterm* hp); - #ifdef ERTS_SMP static void do_send_to_port(Eterm to, @@ -424,7 +660,7 @@ WRITE_SYS_MSG_TO_PORT(Eterm unused_to, erts_encode_ext(message, &ptr); if (!(ptr <= buffer+size)) { - erl_exit(1, "Internal error in do_send_to_port: %d\n", ptr-buffer); + erts_exit(ERTS_ERROR_EXIT, "Internal error in do_send_to_port: %d\n", ptr-buffer); } #ifndef ERTS_SMP @@ -440,11 +676,11 @@ WRITE_SYS_MSG_TO_PORT(Eterm unused_to, /* Send {trace_ts, Pid, out, 0, Timestamp} * followed by {trace_ts, Pid, in, 0, NewTimestamp} * - * 'NewTimestamp' is fetched from GET_NOW() through patch_ts(). + * 'NewTimestamp' through patch_ts(). */ static void -do_send_schedfix_to_port(Port *trace_port, Eterm pid, Eterm timestamp) { -#define LOCAL_HEAP_SIZE (4+5+5) +do_send_schedfix_to_port(Port *trace_port, Eterm pid, Eterm timestamp, int ts_type) { +#define LOCAL_HEAP_SIZE (5+5+ERTS_TRACE_PATCH_TS_MAX_SIZE) DeclareTmpHeapNoproc(local_heap,LOCAL_HEAP_SIZE); Eterm message; Eterm *hp; @@ -466,9 +702,11 @@ do_send_schedfix_to_port(Port *trace_port, Eterm pid, Eterm timestamp) { SYS_MSG_TYPE_UNDEFINED, message); - message = TUPLE4(hp, am_trace_ts, pid, am_in, mfarity); - hp += 5; - hp = patch_ts(message, hp); + + message = TUPLE5(hp, am_trace_ts, pid, am_in, mfarity, + NIL /* Will be overwritten by timestamp */); + hp += 6; + hp[-1] = write_ts(ts_type, hp, NULL, NULL); do_send_to_port(trace_port->common.id, trace_port, @@ -485,7 +723,7 @@ do_send_schedfix_to_port(Port *trace_port, Eterm pid, Eterm timestamp) { * It is assumed that 'message' is not an 'out' message. * * 'c_p' is the currently executing process, "tracee" is the traced process - * which 'message' concerns => if (*tracee_flags & F_TIMESTAMP), + * which 'message' concerns => if (*tracee_flags & F_TIMESTAMP_MASK), * 'message' must contain a timestamp. */ static void @@ -493,8 +731,9 @@ send_to_port(Process *c_p, Eterm message, Eterm *tracer_pid, Uint *tracee_flags) { Port* trace_port; #ifndef ERTS_SMP -#define LOCAL_HEAP_SIZE (4) - Eterm ts, *hp; + int ts_type; +#define LOCAL_HEAP_SIZE ERTS_TRACE_PATCH_TS_MAX_SIZE + Eterm ts; DeclareTmpHeapNoproc(local_heap,LOCAL_HEAP_SIZE); #endif @@ -523,7 +762,7 @@ send_to_port(Process *c_p, Eterm message, */ if ( c_p == NULL || - (! IS_TRACED_FL(c_p, F_TRACE_SCHED | F_TIMESTAMP))) { + (! IS_TRACED_FL(c_p, F_TRACE_SCHED | F_TIMESTAMP_MASK))) { #endif do_send_to_port(*tracer_pid, trace_port, @@ -542,22 +781,12 @@ send_to_port(Process *c_p, Eterm message, */ UseTmpHeapNoproc(LOCAL_HEAP_SIZE); - if (*tracee_flags & F_TIMESTAMP) { - ASSERT(is_tuple(message)); - hp = tuple_val(message); - ts = hp[arityval(hp[0])]; - } else { - /* A fake schedule might be needed, - * but this message does not contain a timestamp. - * Create a dummy trace message with timestamp to be - * passed to do_send_schedfix_to_port(). - */ - Uint ms,s,us; - GET_NOW(&ms, &s, &us); - hp = local_heap; - ts = TUPLE3(hp, make_small(ms), make_small(s), make_small(us)); - hp += 4; - } + /* A fake schedule might be needed. + * Create a dummy trace message with timestamp to be + * passed to do_send_schedfix_to_port(). + */ + ts_type = TFLGS_TS_TYPE(c_p); + ts = write_ts(ts_type, local_heap, NULL, NULL); trace_port->control_flags &= ~PORT_CONTROL_FLAG_HEAVY; do_send_to_port(*tracer_pid, @@ -576,7 +805,7 @@ send_to_port(Process *c_p, Eterm message, * just after writning the real trace message, and now gets scheduled * in again. */ - do_send_schedfix_to_port(trace_port, c_p->common.id, ts); + do_send_schedfix_to_port(trace_port, c_p->common.id, ts, ts_type); } erts_port_release(trace_port); @@ -595,11 +824,9 @@ send_to_port(Process *c_p, Eterm message, static void profile_send(Eterm from, Eterm message) { Uint sz = 0; - ErlHeapFragment *bp = NULL; Uint *hp = NULL; Eterm msg = NIL; Process *profile_p = NULL; - ErlOffHeap *off_heap = NULL; Eterm profiler = erts_get_system_profile(); @@ -625,6 +852,7 @@ profile_send(Eterm from, Eterm message) { } } else { + ErtsMessage *mp; ASSERT(is_internal_pid(profiler)); profile_p = erts_proc_lookup(profiler); @@ -633,14 +861,13 @@ profile_send(Eterm from, Eterm message) { return; sz = size_object(message); - hp = erts_alloc_message_heap(sz, &bp, &off_heap, profile_p, 0); - msg = copy_struct(message, sz, &hp, &bp->off_heap); - - erts_queue_message(profile_p, NULL, bp, msg, NIL -#ifdef USE_VM_PROBES - , NIL -#endif - ); + mp = erts_alloc_message(sz, &hp); + if (sz == 0) + msg = message; + else + msg = copy_struct(message, sz, &hp, &mp->hfrag.off_heap); + + erts_queue_message(profile_p, NULL, mp, msg, NIL); } } @@ -649,20 +876,19 @@ profile_send(Eterm from, Eterm message) { /* A fake schedule out/in message pair will be sent, * if the driver so requests. - * If (timestamp == NIL), one is fetched from GET_NOW(). * * 'c_p' is the currently executing process, may be NULL. */ static void seq_trace_send_to_port(Process *c_p, Eterm seq_tracer, - Eterm message, - Eterm timestamp) + Eterm message) { Port* trace_port; #ifndef ERTS_SMP - Eterm ts, *hp; -#define LOCAL_HEAP_SIZE (4) + int ts_type; + Eterm ts; +#define LOCAL_HEAP_SIZE ERTS_TRACE_PATCH_TS_MAX_SIZE DeclareTmpHeapNoproc(local_heap,LOCAL_HEAP_SIZE); UseTmpHeapNoproc(LOCAL_HEAP_SIZE); #endif @@ -687,7 +913,7 @@ seq_trace_send_to_port(Process *c_p, } if (c_p == NULL - || (! IS_TRACED_FL(c_p, F_TRACE_SCHED | F_TIMESTAMP))) { + || (! IS_TRACED_FL(c_p, F_TRACE_SCHED | F_TIMESTAMP_MASK))) { #endif do_send_to_port(seq_tracer, trace_port, @@ -704,20 +930,12 @@ seq_trace_send_to_port(Process *c_p, * with 'running' and 'timestamp'. */ - if (timestamp != NIL) { - ts = timestamp; - } else { - /* A fake schedule might be needed, - * but this message does not contain a timestamp. - * Create a dummy trace message with timestamp to be - * passed to do_send_schedfix_to_port(). - */ - Uint ms,s,us; - GET_NOW(&ms, &s, &us); - hp = local_heap; - ts = TUPLE3(hp, make_small(ms), make_small(s), make_small(us)); - hp += 4; - } + /* A fake schedule might be needed. + * Create a dummy trace message with timestamp to be + * passed to do_send_schedfix_to_port(). + */ + ts_type = TFLGS_TS_TYPE(c_p); + ts = write_ts(ts_type, local_heap, NULL, NULL); trace_port->control_flags &= ~PORT_CONTROL_FLAG_HEAVY; do_send_to_port(seq_tracer, @@ -736,7 +954,7 @@ seq_trace_send_to_port(Process *c_p, * just after writing the real trace message, and now gets scheduled * in again. */ - do_send_schedfix_to_port(trace_port, c_p->common.id, ts); + do_send_schedfix_to_port(trace_port, c_p->common.id, ts, ts_type); } erts_port_release(trace_port); @@ -746,32 +964,6 @@ seq_trace_send_to_port(Process *c_p, #endif } -#define TS_HEAP_WORDS 5 -#define TS_SIZE(p) ((ERTS_TRACE_FLAGS((p)) & F_TIMESTAMP) \ - ? TS_HEAP_WORDS \ - : 0) - -/* - * Patch a timestamp into a tuple. The tuple must be the last thing - * built on the heap. - * - * Returns the new hp pointer. -*/ -static Eterm* -patch_ts(Eterm tuple, Eterm* hp) -{ - Uint ms, s, us; - Eterm* ptr = tuple_val(tuple); - int arity = arityval(*ptr); - - ASSERT((ptr+arity+1) == hp); - ptr[0] = make_arityval(arity+1); - ptr[1] = am_trace_ts; - GET_NOW(&ms, &s, &us); - *hp = TUPLE3(hp+1, make_small(ms), make_small(s), make_small(us)); - return hp+5; -} - static ERTS_INLINE void send_to_tracer(Process *tracee, ERTS_TRACER_REF_TYPE tracer_ref, @@ -784,13 +976,13 @@ send_to_tracer(Process *tracee, erts_smp_mtx_lock(&smq_mtx); - if (ERTS_TRACE_FLAGS(tracee) & F_TIMESTAMP) - *hpp = patch_ts(msg, *hpp); - - if (is_internal_pid(ERTS_TRACER_PROC(tracee))) + if (is_internal_pid(ERTS_TRACER_PROC(tracee))) { + PATCH_TS(TFLGS_TS_TYPE(tracee), msg, *hpp, bp, tracer_ref); ERTS_ENQ_TRACE_MSG(tracee->common.id, tracer_ref, msg, bp); + } else { ASSERT(is_internal_port(ERTS_TRACER_PROC(tracee))); + PATCH_TS(TFLGS_TS_TYPE(tracee), msg, *hpp, NULL, NULL); send_to_port(no_fake_sched ? NULL : tracee, msg, &ERTS_TRACER_PROC(tracee), @@ -804,7 +996,7 @@ send_to_tracer(Process *tracee, static void trace_sched_aux(Process *p, Eterm what, int never_fake_sched) { -#define LOCAL_HEAP_SIZE (5+4+1+TS_HEAP_WORDS) +#define LOCAL_HEAP_SIZE (5+4+1+ERTS_TRACE_PATCH_TS_MAX_SIZE) DeclareTmpHeap(local_heap,LOCAL_HEAP_SIZE,p); Eterm tmp, mess, *hp; ErlHeapFragment *bp = NULL; @@ -860,7 +1052,7 @@ trace_sched_aux(Process *p, Eterm what, int never_fake_sched) size += 4; if (sched_no) size += 1; - size += TS_SIZE(p); + size += PATCH_TS_SIZE(p); hp = ERTS_ALLOC_SYSMSG_HEAP(size, &bp, &off_heap, tracer_ref); } @@ -934,7 +1126,7 @@ trace_send(Process *p, Eterm to, Eterm msg) } if (is_internal_port(ERTS_TRACER_PROC(p))) { -#define LOCAL_HEAP_SIZE (11) +#define LOCAL_HEAP_SIZE (6 + ERTS_TRACE_PATCH_TS_MAX_SIZE) DeclareTmpHeapNoproc(local_heap,LOCAL_HEAP_SIZE); UseTmpHeapNoproc(LOCAL_HEAP_SIZE); @@ -942,9 +1134,7 @@ trace_send(Process *p, Eterm to, Eterm msg) mess = TUPLE5(hp, am_trace, p->common.id, operation, msg, to); hp += 6; erts_smp_mtx_lock(&smq_mtx); - if (ERTS_TRACE_FLAGS(p) & F_TIMESTAMP) { - hp = patch_ts(mess, hp); - } + PATCH_TS(TFLGS_TS_TYPE(p), mess, hp, NULL, NULL); send_to_port(p, mess, &ERTS_TRACER_PROC(p), &ERTS_TRACE_FLAGS(p)); UnUseTmpHeapNoproc(LOCAL_HEAP_SIZE); #undef LOCAL_HEAP_SIZE @@ -963,7 +1153,7 @@ trace_send(Process *p, Eterm to, Eterm msg) sz_msg = size_object(msg); sz_to = size_object(to); - need = sz_msg + sz_to + 6 + TS_SIZE(p); + need = sz_msg + sz_to + 6 + PATCH_TS_SIZE(p); hp = ERTS_ALLOC_SYSMSG_HEAP(need, &bp, &off_heap, tracer_ref); @@ -980,10 +1170,7 @@ trace_send(Process *p, Eterm to, Eterm msg) erts_smp_mtx_lock(&smq_mtx); - if (ERTS_TRACE_FLAGS(p) & F_TIMESTAMP) { - patch_ts(mess, hp); - } - + PATCH_TS(TFLGS_TS_TYPE(p), mess, hp, bp, tracer_ref); ERTS_ENQ_TRACE_MSG(p->common.id, tracer_ref, mess, bp); erts_smp_mtx_unlock(&smq_mtx); } @@ -1000,7 +1187,7 @@ trace_receive(Process *rp, Eterm msg) Eterm* hp; if (is_internal_port(ERTS_TRACER_PROC(rp))) { -#define LOCAL_HEAP_SIZE (10) +#define LOCAL_HEAP_SIZE (5+ERTS_TRACE_PATCH_TS_MAX_SIZE) DeclareTmpHeapNoproc(local_heap,LOCAL_HEAP_SIZE); UseTmpHeapNoproc(LOCAL_HEAP_SIZE); @@ -1008,9 +1195,7 @@ trace_receive(Process *rp, Eterm msg) mess = TUPLE4(hp, am_trace, rp->common.id, am_receive, msg); hp += 5; erts_smp_mtx_lock(&smq_mtx); - if (ERTS_TRACE_FLAGS(rp) & F_TIMESTAMP) { - hp = patch_ts(mess, hp); - } + PATCH_TS(TFLGS_TS_TYPE(rp), mess, hp, NULL, NULL); send_to_port(rp, mess, &ERTS_TRACER_PROC(rp), &ERTS_TRACE_FLAGS(rp)); UnUseTmpHeapNoproc(LOCAL_HEAP_SIZE); #undef LOCAL_HEAP_SIZE @@ -1029,7 +1214,7 @@ trace_receive(Process *rp, Eterm msg) sz_msg = size_object(msg); - hsz = sz_msg + 5 + TS_SIZE(rp); + hsz = sz_msg + 5 + PATCH_TS_SIZE(rp); hp = ERTS_ALLOC_SYSMSG_HEAP(hsz, &bp, &off_heap, tracer_ref); @@ -1039,10 +1224,7 @@ trace_receive(Process *rp, Eterm msg) erts_smp_mtx_lock(&smq_mtx); - if (ERTS_TRACE_FLAGS(rp) & F_TIMESTAMP) { - patch_ts(mess, hp); - } - + PATCH_TS(TFLGS_TS_TYPE(rp), mess, hp, bp, tracer_ref); ERTS_ENQ_TRACE_MSG(rp->common.id, tracer_ref, mess, bp); erts_smp_mtx_unlock(&smq_mtx); } @@ -1053,11 +1235,7 @@ seq_trace_update_send(Process *p) { Eterm seq_tracer = erts_get_system_seq_tracer(); ASSERT((is_tuple(SEQ_TRACE_TOKEN(p)) || is_nil(SEQ_TRACE_TOKEN(p)))); - if ( (p->common.id == seq_tracer) || (SEQ_TRACE_TOKEN(p) == NIL) -#ifdef USE_VM_PROBES - || (SEQ_TRACE_TOKEN(p) == am_have_dt_utag) -#endif - ) { + if ((p->common.id == seq_tracer) || have_no_seqtrace(SEQ_TRACE_TOKEN(p))) { return 0; } SEQ_TRACE_TOKEN_SENDER(p) = p->common.id; @@ -1092,6 +1270,7 @@ seq_trace_output_generic(Eterm token, Eterm msg, Uint type, Eterm type_atom; int sz_exit; Eterm seq_tracer; + int ts_type; seq_tracer = erts_get_system_seq_tracer(); @@ -1106,7 +1285,7 @@ seq_trace_output_generic(Eterm token, Eterm msg, Uint type, case SEQ_TRACE_PRINT: type_atom = am_print; break; case SEQ_TRACE_RECEIVE: type_atom = am_receive; break; default: - erl_exit(1, "invalid type in seq_trace_output_generic: %d:\n", type); + erts_exit(ERTS_ERROR_EXIT, "invalid type in seq_trace_output_generic: %d:\n", type); return; /* To avoid warning */ } @@ -1119,8 +1298,10 @@ seq_trace_output_generic(Eterm token, Eterm msg, Uint type, return; /* no need to send anything */ } + ts_type = ERTS_SEQTFLGS2TSTYPE(unsigned_val(SEQ_TRACE_T_FLAGS(token))); + if (is_internal_port(seq_tracer)) { -#define LOCAL_HEAP_SIZE (64) +#define LOCAL_HEAP_SIZE (60 + ERTS_TRACE_PATCH_TS_MAX_SIZE) DeclareTmpHeapNoproc(local_heap,LOCAL_HEAP_SIZE); UseTmpHeapNoproc(LOCAL_HEAP_SIZE); @@ -1136,17 +1317,17 @@ seq_trace_output_generic(Eterm token, Eterm msg, Uint type, mess = TUPLE5(hp, type_atom, lastcnt_serial, SEQ_TRACE_T_SENDER(token), receiver, msg); hp += 6; + erts_smp_mtx_lock(&smq_mtx); - if ((unsigned_val(SEQ_TRACE_T_FLAGS(token)) & SEQ_TRACE_TIMESTAMP) == 0) { + if (!ts_type) { mess = TUPLE3(hp, am_seq_trace, label, mess); - seq_trace_send_to_port(NULL, seq_tracer, mess, NIL); + seq_trace_send_to_port(NULL, seq_tracer, mess); } else { - Uint ms,s,us,ts; - GET_NOW(&ms, &s, &us); - ts = TUPLE3(hp, make_small(ms),make_small(s), make_small(us)); - hp += 4; - mess = TUPLE4(hp, am_seq_trace, label, mess, ts); - seq_trace_send_to_port(process, seq_tracer, mess, ts); + mess = TUPLE4(hp, am_seq_trace, label, mess, + NIL /* Will be overwritten by timestamp */); + hp += 5; + hp[-1] = write_ts(ts_type, hp, NULL, NULL); + seq_trace_send_to_port(process, seq_tracer, mess); } UnUseTmpHeapNoproc(LOCAL_HEAP_SIZE); #undef LOCAL_HEAP_SIZE @@ -1181,8 +1362,7 @@ seq_trace_output_generic(Eterm token, Eterm msg, Uint type, sz_lastcnt_serial = 3; /* TUPLE2 */ sz_msg = size_object(msg); - sz_ts = ((unsigned_val(SEQ_TRACE_T_FLAGS(token)) & SEQ_TRACE_TIMESTAMP) ? - 5 : 0); + sz_ts = patch_ts_size(ts_type); if (exitfrom != NIL) { sz_exit = 4; /* create {'EXIT',exitfrom,msg} */ sz_exitfrom = size_object(exitfrom); @@ -1226,25 +1406,32 @@ seq_trace_output_generic(Eterm token, Eterm msg, Uint type, erts_smp_mtx_lock(&smq_mtx); - if (sz_ts) {/* timestamp should be included */ - Uint ms,s,us,ts; - GET_NOW(&ms, &s, &us); - ts = TUPLE3(hp, make_small(ms),make_small(s), make_small(us)); - hp += 4; - mess = TUPLE4(hp, am_seq_trace, label, mess, ts); - } else { + if (!ts_type) mess = TUPLE3(hp, am_seq_trace, label, mess); + else { + mess = TUPLE4(hp, am_seq_trace, label, mess, + NIL /* Will be overwritten by timestamp */); + hp += 5; + /* Write timestamp in element 6 of the 'msg' tuple */ + hp[-1] = write_ts(ts_type, hp, bp, +#ifndef ERTS_SMP + tracer +#else + NULL +#endif + ); } #ifdef ERTS_SMP enqueue_sys_msg_unlocked(SYS_MSG_TYPE_SEQTRACE, NIL, NIL, mess, bp); erts_smp_mtx_unlock(&smq_mtx); #else - erts_queue_message(tracer, NULL, bp, mess, NIL -#ifdef USE_VM_PROBES - , NIL -#endif - ); /* trace_token must be NIL here */ + /* trace_token must be NIL here */ + { + ErtsMessage *mp = erts_alloc_message(0, NULL); + mp->data.heap_frag = bp; + erts_queue_message(tracer, NULL, mp, mess, NIL); + } #endif } } @@ -1255,7 +1442,7 @@ seq_trace_output_generic(Eterm token, Eterm msg, Uint type, void erts_trace_return_to(Process *p, BeamInstr *pc) { -#define LOCAL_HEAP_SIZE (4+5+5) +#define LOCAL_HEAP_SIZE (4+5+ERTS_TRACE_PATCH_TS_MAX_SIZE) Eterm* hp; Eterm mfa; Eterm mess; @@ -1280,9 +1467,7 @@ erts_trace_return_to(Process *p, BeamInstr *pc) erts_smp_mtx_lock(&smq_mtx); - if (ERTS_TRACE_FLAGS(p) & F_TIMESTAMP) { - hp = patch_ts(mess, hp); - } + PATCH_TS(TFLGS_TS_TYPE(p), mess, hp, NULL, NULL); if (is_internal_port(ERTS_TRACER_PROC(p))) { send_to_port(p, mess, &ERTS_TRACER_PROC(p), &ERTS_TRACE_FLAGS(p)); @@ -1329,6 +1514,7 @@ erts_trace_return(Process* p, BeamInstr* fi, Eterm retval, Eterm *tracer_pid) Eterm mod, name; int arity; Uint meta_flags, *tracee_flags; + int ts_type; #ifdef ERTS_SMP Eterm tracee; #endif @@ -1364,7 +1550,7 @@ erts_trace_return(Process* p, BeamInstr* fi, Eterm retval, Eterm *tracer_pid) * meta trace => * use fixed flag set instead of process flags */ - meta_flags = F_TRACE_CALLS | F_TIMESTAMP; + meta_flags = F_TRACE_CALLS | F_NOW_TS; tracee_flags = &meta_flags; #ifdef ERTS_SMP tracee = NIL; @@ -1378,8 +1564,10 @@ erts_trace_return(Process* p, BeamInstr* fi, Eterm retval, Eterm *tracer_pid) name = fi[1]; arity = fi[2]; + ts_type = ERTS_TFLGS2TSTYPE(*tracee_flags); + if (is_internal_port(*tracer_pid)) { -#define LOCAL_HEAP_SIZE (4+6+5) +#define LOCAL_HEAP_SIZE (4+6+ERTS_TRACE_PATCH_TS_MAX_SIZE) DeclareTmpHeapNoproc(local_heap,LOCAL_HEAP_SIZE); UseTmpHeapNoproc(LOCAL_HEAP_SIZE); hp = local_heap; @@ -1388,9 +1576,7 @@ erts_trace_return(Process* p, BeamInstr* fi, Eterm retval, Eterm *tracer_pid) mess = TUPLE5(hp, am_trace, p->common.id, am_return_from, mfa, retval); hp += 6; erts_smp_mtx_lock(&smq_mtx); - if (*tracee_flags & F_TIMESTAMP) { - hp = patch_ts(mess, hp); - } + PATCH_TS(ts_type, mess, hp, NULL, NULL); send_to_port(p, mess, tracer_pid, tracee_flags); UnUseTmpHeapNoproc(LOCAL_HEAP_SIZE); #undef LOCAL_HEAP_SIZE @@ -1401,24 +1587,15 @@ erts_trace_return(Process* p, BeamInstr* fi, Eterm retval, Eterm *tracer_pid) ERTS_TRACER_REF_TYPE tracer_ref; unsigned size; unsigned retval_size; -#ifdef DEBUG - Eterm* limit; -#endif ASSERT(is_internal_pid(*tracer_pid)); ERTS_GET_TRACER_REF(tracer_ref, *tracer_pid, *tracee_flags); - + retval_size = size_object(retval); - size = 6 + 4 + retval_size; - if (*tracee_flags & F_TIMESTAMP) { - size += 1+4; - } + size = 6 + 4 + retval_size + patch_ts_size(ts_type); hp = ERTS_ALLOC_SYSMSG_HEAP(size, &bp, &off_heap, tracer_ref); -#ifdef DEBUG - limit = hp + size; -#endif /* * Build the trace tuple and put it into receive queue of the tracer process. @@ -1432,11 +1609,7 @@ erts_trace_return(Process* p, BeamInstr* fi, Eterm retval, Eterm *tracer_pid) erts_smp_mtx_lock(&smq_mtx); - if (*tracee_flags & F_TIMESTAMP) { - hp = patch_ts(mess, hp); - } - - ASSERT(hp == limit); + PATCH_TS(ts_type, mess, hp, bp, tracer_ref); ERTS_ENQ_TRACE_MSG(tracee, tracer_ref, mess, bp); erts_smp_mtx_unlock(&smq_mtx); @@ -1459,6 +1632,7 @@ erts_trace_exception(Process* p, BeamInstr mfa[3], Eterm class, Eterm value, Eterm cv; Eterm mess; Uint meta_flags, *tracee_flags; + int ts_type; #ifdef ERTS_SMP Eterm tracee; #endif @@ -1497,15 +1671,17 @@ erts_trace_exception(Process* p, BeamInstr mfa[3], Eterm class, Eterm value, * meta trace => * use fixed flag set instead of process flags */ - meta_flags = F_TRACE_CALLS | F_TIMESTAMP; + meta_flags = F_TRACE_CALLS | F_NOW_TS; tracee_flags = &meta_flags; #ifdef ERTS_SMP tracee = NIL; #endif } + ts_type = ERTS_TFLGS2TSTYPE(*tracee_flags); + if (is_internal_port(*tracer_pid)) { -#define LOCAL_HEAP_SIZE (4+3+6+5) +#define LOCAL_HEAP_SIZE (4+3+6+ERTS_TRACE_PATCH_TS_MAX_SIZE) DeclareTmpHeapNoproc(local_heap,LOCAL_HEAP_SIZE); UseTmpHeapNoproc(LOCAL_HEAP_SIZE); @@ -1518,10 +1694,7 @@ erts_trace_exception(Process* p, BeamInstr mfa[3], Eterm class, Eterm value, hp += 6; ASSERT((hp - local_heap) <= LOCAL_HEAP_SIZE); erts_smp_mtx_lock(&smq_mtx); - if (*tracee_flags & F_TIMESTAMP) { - hp = patch_ts(mess, hp); /* hp += 5 */ - ASSERT((hp - local_heap) == LOCAL_HEAP_SIZE); - } + PATCH_TS(ts_type, mess, hp, NULL, NULL); send_to_port(p, mess, tracer_pid, tracee_flags); UnUseTmpHeapNoproc(LOCAL_HEAP_SIZE); #undef LOCAL_HEAP_SIZE @@ -1532,24 +1705,15 @@ erts_trace_exception(Process* p, BeamInstr mfa[3], Eterm class, Eterm value, ERTS_TRACER_REF_TYPE tracer_ref; unsigned size; unsigned value_size; -#ifdef DEBUG - Eterm* limit; -#endif ASSERT(is_internal_pid(*tracer_pid)); ERTS_GET_TRACER_REF(tracer_ref, *tracer_pid, *tracee_flags); value_size = size_object(value); - size = 6 + 4 + 3 + value_size; - if (*tracee_flags & F_TIMESTAMP) { - size += 1+4; - } + size = 6 + 4 + 3 + value_size + patch_ts_size(ts_type); hp = ERTS_ALLOC_SYSMSG_HEAP(size, &bp, &off_heap, tracer_ref); -#ifdef DEBUG - limit = hp + size; -#endif /* * Build the trace tuple and put it into receive queue of the tracer process. @@ -1566,11 +1730,7 @@ erts_trace_exception(Process* p, BeamInstr mfa[3], Eterm class, Eterm value, erts_smp_mtx_lock(&smq_mtx); - if (*tracee_flags & F_TIMESTAMP) { - hp = patch_ts(mess, hp); - } - - ASSERT(hp == limit); + PATCH_TS(ts_type, mess, hp, bp, tracer_ref); ERTS_ENQ_TRACE_MSG(tracee, tracer_ref, mess, bp); erts_smp_mtx_unlock(&smq_mtx); @@ -1603,6 +1763,7 @@ erts_call_trace(Process* p, BeamInstr mfa[3], Binary *match_spec, Eterm pam_result = am_true; Eterm mess; Uint meta_flags, *tracee_flags; + int ts_type; #ifdef ERTS_SMP Eterm tracee; #endif @@ -1644,7 +1805,7 @@ erts_call_trace(Process* p, BeamInstr mfa[3], Binary *match_spec, /* No trace messages for sensitive processes. */ return 0; } - meta_flags = F_TRACE_CALLS | F_TIMESTAMP; + meta_flags = F_TRACE_CALLS | F_NOW_TS; tracee_flags = &meta_flags; #ifdef ERTS_SMP tracee = NIL; @@ -1687,13 +1848,10 @@ erts_call_trace(Process* p, BeamInstr mfa[3], Binary *match_spec, } args = transformed_args; + ts_type = ERTS_TFLGS2TSTYPE(*tracee_flags); + if (is_internal_port(*tracer_pid)) { -#if HEAP_ON_C_STACK - Eterm local_heap[64+MAX_ARG]; -#else - Eterm *local_heap = erts_alloc(ERTS_ALC_T_TEMP_TERM, - sizeof(Eterm)*(64+MAX_ARG)); -#endif + Eterm local_heap[64+ERTS_TRACE_PATCH_TS_MAX_SIZE+MAX_ARG]; hp = local_heap; if (!erts_is_valid_tracer_port(*tracer_pid)) { @@ -1708,9 +1866,6 @@ erts_call_trace(Process* p, BeamInstr mfa[3], Binary *match_spec, if (is_not_nil(tracee)) erts_smp_proc_unlock(p, ERTS_PROC_LOCKS_ALL_MINOR); #endif -#if !HEAP_ON_C_STACK - erts_free(ERTS_ALC_T_TEMP_TERM,local_heap); -#endif UnUseTmpHeap(ERL_SUB_BIN_SIZE,p); return 0; } @@ -1738,9 +1893,6 @@ erts_call_trace(Process* p, BeamInstr mfa[3], Binary *match_spec, ERTS_PAM_TMP_RESULT, &return_flags); if (is_non_value(pam_result)) { erts_match_set_release_result(p); -#if !HEAP_ON_C_STACK - erts_free(ERTS_ALC_T_TEMP_TERM,local_heap); -#endif UnUseTmpHeap(ERL_SUB_BIN_SIZE,p); return 0; } @@ -1749,9 +1901,6 @@ erts_call_trace(Process* p, BeamInstr mfa[3], Binary *match_spec, /* Meta trace */ if (pam_result == am_false) { erts_match_set_release_result(p); -#if !HEAP_ON_C_STACK - erts_free(ERTS_ALC_T_TEMP_TERM,local_heap); -#endif UnUseTmpHeap(ERL_SUB_BIN_SIZE,p); return return_flags; } @@ -1759,17 +1908,11 @@ erts_call_trace(Process* p, BeamInstr mfa[3], Binary *match_spec, /* Non-meta trace */ if (*tracee_flags & F_TRACE_SILENT) { erts_match_set_release_result(p); -#if !HEAP_ON_C_STACK - erts_free(ERTS_ALC_T_TEMP_TERM,local_heap); -#endif UnUseTmpHeap(ERL_SUB_BIN_SIZE,p); return 0; } if (pam_result == am_false) { erts_match_set_release_result(p); -#if !HEAP_ON_C_STACK - erts_free(ERTS_ALC_T_TEMP_TERM,local_heap); -#endif UnUseTmpHeap(ERL_SUB_BIN_SIZE,p); return return_flags; } @@ -1807,15 +1950,10 @@ erts_call_trace(Process* p, BeamInstr mfa[3], Binary *match_spec, *hp++ = pam_result; } erts_smp_mtx_lock(&smq_mtx); - if (*tracee_flags & F_TIMESTAMP) { - hp = patch_ts(mess, hp); - } + PATCH_TS(ts_type, mess, hp, NULL, NULL); send_to_port(p, mess, tracer_pid, tracee_flags); erts_smp_mtx_unlock(&smq_mtx); erts_match_set_release_result(p); -#if !HEAP_ON_C_STACK - erts_free(ERTS_ALC_T_TEMP_TERM,local_heap); -#endif UnUseTmpHeap(ERL_SUB_BIN_SIZE,p); return *tracer_pid == NIL ? 0 : return_flags; @@ -1831,9 +1969,6 @@ erts_call_trace(Process* p, BeamInstr mfa[3], Binary *match_spec, unsigned sizes[MAX_ARG]; unsigned pam_result_size = 0; int invalid_tracer; -#ifdef DEBUG - Eterm* limit; -#endif ASSERT(is_internal_pid(*tracer_pid)); @@ -1926,10 +2061,7 @@ erts_call_trace(Process* p, BeamInstr mfa[3], Binary *match_spec, size += sizes[i]; } } - if (*tracee_flags & F_TIMESTAMP) { - size += 1 + 4; - /* One element in trace tuple + timestamp tuple. */ - } + size += patch_ts_size(ts_type); if (pam_result != am_true) { pam_result_size = size_object(pam_result); size += 1 + pam_result_size; @@ -1937,9 +2069,6 @@ erts_call_trace(Process* p, BeamInstr mfa[3], Binary *match_spec, } hp = ERTS_ALLOC_SYSMSG_HEAP(size, &bp, &off_heap, tracer_ref); -#ifdef DEBUG - limit = hp + size; -#endif /* * Build the the {M,F,A} tuple in the message buffer. @@ -1982,11 +2111,7 @@ erts_call_trace(Process* p, BeamInstr mfa[3], Binary *match_spec, erts_smp_mtx_lock(&smq_mtx); - if (*tracee_flags & F_TIMESTAMP) { - hp = patch_ts(mess, hp); - } - - ASSERT(hp == limit); + PATCH_TS(ts_type, mess, hp, bp, tracer_ref); ERTS_ENQ_TRACE_MSG(tracee, tracer_ref, mess, bp); erts_smp_mtx_unlock(&smq_mtx); UnUseTmpHeap(ERL_SUB_BIN_SIZE,p); @@ -2021,9 +2146,7 @@ trace_proc(Process *c_p, Process *t_p, Eterm what, Eterm data) mess = TUPLE4(hp, am_trace, t_p->common.id, what, data); hp += 5; erts_smp_mtx_lock(&smq_mtx); - if (ERTS_TRACE_FLAGS(t_p) & F_TIMESTAMP) { - hp = patch_ts(mess, hp); - } + PATCH_TS(TFLGS_TS_TYPE(t_p), mess, hp, NULL, NULL); send_to_port( #ifndef ERTS_SMP /* No fake schedule out and in again after an exit */ @@ -2053,7 +2176,7 @@ trace_proc(Process *c_p, Process *t_p, Eterm what, Eterm data) sz_data = size_object(data); - need = sz_data + 5 + TS_SIZE(t_p); + need = sz_data + 5 + PATCH_TS_SIZE(t_p); hp = ERTS_ALLOC_SYSMSG_HEAP(need, &bp, &off_heap, tracer_ref); @@ -2063,9 +2186,7 @@ trace_proc(Process *c_p, Process *t_p, Eterm what, Eterm data) erts_smp_mtx_lock(&smq_mtx); - if (ERTS_TRACE_FLAGS(t_p) & F_TIMESTAMP) { - hp = patch_ts(mess, hp); - } + PATCH_TS(TFLGS_TS_TYPE(t_p), mess, hp, bp, tracer_ref); ERTS_ENQ_TRACE_MSG(t_p->common.id, tracer_ref, mess, bp); erts_smp_mtx_unlock(&smq_mtx); @@ -2099,9 +2220,7 @@ trace_proc_spawn(Process *p, Eterm pid, mess = TUPLE5(hp, am_trace, p->common.id, am_spawn, pid, mfa); hp += 6; erts_smp_mtx_lock(&smq_mtx); - if (ERTS_TRACE_FLAGS(p) & F_TIMESTAMP) { - hp = patch_ts(mess, hp); - } + PATCH_TS(TFLGS_TS_TYPE(p), mess, hp, NULL, NULL); send_to_port(p, mess, &ERTS_TRACER_PROC(p), &ERTS_TRACE_FLAGS(p)); UnUseTmpHeapNoproc(LOCAL_HEAP_SIZE); #undef LOCAL_HEAP_SIZE @@ -2122,7 +2241,7 @@ trace_proc_spawn(Process *p, Eterm pid, sz_args = size_object(args); sz_pid = size_object(pid); - need = sz_args + 4 + 6 + TS_SIZE(p); + need = sz_args + 4 + 6 + PATCH_TS_SIZE(p); hp = ERTS_ALLOC_SYSMSG_HEAP(need, &bp, &off_heap, tracer_ref); @@ -2135,9 +2254,7 @@ trace_proc_spawn(Process *p, Eterm pid, erts_smp_mtx_lock(&smq_mtx); - if (ERTS_TRACE_FLAGS(p) & F_TIMESTAMP) { - hp = patch_ts(mess, hp); - } + PATCH_TS(TFLGS_TS_TYPE(p), mess, hp, bp, tracer_ref); ERTS_ENQ_TRACE_MSG(p->common.id, tracer_ref, mess, bp); erts_smp_mtx_unlock(&smq_mtx); @@ -2174,11 +2291,6 @@ void save_calls(Process *p, Export *e) void trace_gc(Process *p, Eterm what) { - ERTS_DECL_AM(bin_vheap_size); - ERTS_DECL_AM(bin_vheap_block_size); - ERTS_DECL_AM(bin_old_vheap_size); - ERTS_DECL_AM(bin_old_vheap_block_size); - ErlHeapFragment *bp = NULL; ErlOffHeap *off_heap; ERTS_TRACER_REF_TYPE tracer_ref = ERTS_NULL_TRACER_REF; /* Initialized @@ -2189,43 +2301,10 @@ trace_gc(Process *p, Eterm what) Eterm msg = NIL; Uint size; - Eterm tags[] = { - am_old_heap_block_size, - am_heap_block_size, - am_mbuf_size, - am_recent_size, - am_stack_size, - am_old_heap_size, - am_heap_size, - AM_bin_vheap_size, - AM_bin_vheap_block_size, - AM_bin_old_vheap_size, - AM_bin_old_vheap_block_size - }; - - UWord values[] = { - OLD_HEAP(p) ? OLD_HEND(p) - OLD_HEAP(p) : 0, - HEAP_SIZE(p), - MBUF_SIZE(p), - HIGH_WATER(p) - HEAP_START(p), - STACK_START(p) - p->stop, - OLD_HEAP(p) ? OLD_HTOP(p) - OLD_HEAP(p) : 0, - HEAP_TOP(p) - HEAP_START(p), - MSO(p).overhead, - BIN_VHEAP_SZ(p), - BIN_OLD_VHEAP(p), - BIN_OLD_VHEAP_SZ(p) - }; #define LOCAL_HEAP_SIZE \ - (sizeof(values)/sizeof(*values)) * \ - (2/*cons*/ + 3/*2-tuple*/ + BIG_UINT_HEAP_SIZE) + \ - 5/*4-tuple */ + TS_HEAP_WORDS + (ERTS_PROCESS_GC_INFO_MAX_SIZE) + \ + 5/*4-tuple */ + ERTS_TRACE_PATCH_TS_MAX_SIZE DeclareTmpHeap(local_heap,LOCAL_HEAP_SIZE,p); -#ifdef DEBUG - Eterm* limit; -#endif - - ASSERT(sizeof(values)/sizeof(*values) == sizeof(tags)/sizeof(Eterm)); UseTmpHeap(LOCAL_HEAP_SIZE,p); @@ -2233,12 +2312,9 @@ trace_gc(Process *p, Eterm what) hp = local_heap; #ifdef DEBUG size = 0; - (void) erts_bld_atom_uword_2tup_list(NULL, - &size, - sizeof(values)/sizeof(*values), - tags, - values); - size += 5/*4-tuple*/ + TS_SIZE(p); + (void) erts_process_gc_info(p, &size, NULL); + + size += 5/*4-tuple*/ + PATCH_TS_SIZE(p); #endif } else { ASSERT(is_internal_pid(ERTS_TRACER_PROC(p))); @@ -2248,40 +2324,30 @@ trace_gc(Process *p, Eterm what) ERTS_TRACE_FLAGS(p)); size = 0; - (void) erts_bld_atom_uword_2tup_list(NULL, - &size, - sizeof(values)/sizeof(*values), - tags, - values); - size += 5/*4-tuple*/ + TS_SIZE(p); + (void) erts_process_gc_info(p, &size, NULL); + + size += 5/*4-tuple*/ + PATCH_TS_SIZE(p); hp = ERTS_ALLOC_SYSMSG_HEAP(size, &bp, &off_heap, tracer_ref); } -#ifdef DEBUG - limit = hp + size; ASSERT(size <= LOCAL_HEAP_SIZE); -#endif - msg = erts_bld_atom_uword_2tup_list(&hp, - NULL, - sizeof(values)/sizeof(*values), - tags, - values); + msg = erts_process_gc_info(p, NULL, &hp); msg = TUPLE4(hp, am_trace, p->common.id, what, msg); hp += 5; erts_smp_mtx_lock(&smq_mtx); - if (ERTS_TRACE_FLAGS(p) & F_TIMESTAMP) { - hp = patch_ts(msg, hp); - } - ASSERT(hp == limit); - if (is_internal_port(ERTS_TRACER_PROC(p))) + if (is_internal_port(ERTS_TRACER_PROC(p))) { + PATCH_TS(TFLGS_TS_TYPE(p), msg, hp, NULL, NULL); send_to_port(p, msg, &ERTS_TRACER_PROC(p), &ERTS_TRACE_FLAGS(p)); - else + } + else { + PATCH_TS(TFLGS_TS_TYPE(p), msg, hp, bp, tracer_ref); ERTS_ENQ_TRACE_MSG(p->common.id, tracer_ref, msg, bp); + } erts_smp_mtx_unlock(&smq_mtx); UnUseTmpHeap(LOCAL_HEAP_SIZE,p); #undef LOCAL_HEAP_SIZE @@ -2343,11 +2409,11 @@ monitor_long_schedule_proc(Process *p, BeamInstr *in_fp, BeamInstr *out_fp, Uint #ifdef ERTS_SMP enqueue_sys_msg(SYS_MSG_TYPE_SYSMON, p->common.id, NIL, msg, bp); #else - erts_queue_message(monitor_p, NULL, bp, msg, NIL -#ifdef USE_VM_PROBES - , NIL -#endif - ); + { + ErtsMessage *mp = erts_alloc_message(0, NULL); + mp->data.heap_frag = bp; + erts_queue_message(monitor_p, NULL, mp, msg, NIL); + } #endif } void @@ -2408,11 +2474,11 @@ monitor_long_schedule_port(Port *pp, ErtsPortTaskType type, Uint time) #ifdef ERTS_SMP enqueue_sys_msg(SYS_MSG_TYPE_SYSMON, pp->common.id, NIL, msg, bp); #else - erts_queue_message(monitor_p, NULL, bp, msg, NIL -#ifdef USE_VM_PROBES - , NIL -#endif - ); + { + ErtsMessage *mp = erts_alloc_message(0, NULL); + mp->data.heap_frag = bp; + erts_queue_message(monitor_p, NULL, mp, msg, NIL); + } #endif } @@ -2483,11 +2549,11 @@ monitor_long_gc(Process *p, Uint time) { #ifdef ERTS_SMP enqueue_sys_msg(SYS_MSG_TYPE_SYSMON, p->common.id, NIL, msg, bp); #else - erts_queue_message(monitor_p, NULL, bp, msg, NIL -#ifdef USE_VM_PROBES - , NIL -#endif - ); + { + ErtsMessage *mp = erts_alloc_message(0, NULL); + mp->data.heap_frag = bp; + erts_queue_message(monitor_p, NULL, mp, msg, NIL); + } #endif } @@ -2524,7 +2590,7 @@ monitor_large_heap(Process *p) { #ifndef ERTS_SMP ASSERT(is_internal_pid(system_monitor)); monitor_p = erts_proc_lookup(system_monitor); - if (monitor_p || p == monitor_p) { + if (!monitor_p || p == monitor_p) { return; } #endif @@ -2558,11 +2624,11 @@ monitor_large_heap(Process *p) { #ifdef ERTS_SMP enqueue_sys_msg(SYS_MSG_TYPE_SYSMON, p->common.id, NIL, msg, bp); #else - erts_queue_message(monitor_p, NULL, bp, msg, NIL -#ifdef USE_VM_PROBES - , NIL -#endif - ); + { + ErtsMessage *mp = erts_alloc_message(0, NULL); + mp->data.heap_frag = bp; + erts_queue_message(monitor_p, NULL, mp, msg, NIL); + } #endif } @@ -2590,11 +2656,11 @@ monitor_generic(Process *p, Eterm type, Eterm spec) { #ifdef ERTS_SMP enqueue_sys_msg(SYS_MSG_TYPE_SYSMON, p->common.id, NIL, msg, bp); #else - erts_queue_message(monitor_p, NULL, bp, msg, NIL -#ifdef USE_VM_PROBES - , NIL -#endif - ); + { + ErtsMessage *mp = erts_alloc_message(0, NULL); + mp->data.heap_frag = bp; + erts_queue_message(monitor_p, NULL, mp, msg, NIL); + } #endif } @@ -2605,19 +2671,18 @@ monitor_generic(Process *p, Eterm type, Eterm spec) { void profile_scheduler(Eterm scheduler_id, Eterm state) { - Eterm *hp, msg, timestamp; - Uint Ms, s, us; + Eterm *hp, msg; + ErlHeapFragment *bp = NULL; #ifndef ERTS_SMP -#define LOCAL_HEAP_SIZE (4 + 7) +#define LOCAL_HEAP_SIZE (7 + ERTS_TRACE_PATCH_TS_MAX_SIZE) DeclareTmpHeapNoproc(local_heap,LOCAL_HEAP_SIZE); UseTmpHeapNoproc(LOCAL_HEAP_SIZE); hp = local_heap; #else - ErlHeapFragment *bp; Uint hsz; - hsz = 4 + 7; + hsz = 7 + patch_ts_size(erts_system_profile_ts_type)-1; bp = new_message_buffer(hsz); hp = bp->mem; @@ -2637,10 +2702,13 @@ profile_scheduler(Eterm scheduler_id, Eterm state) { break; } - GET_NOW(&Ms, &s, &us); - timestamp = TUPLE3(hp, make_small(Ms), make_small(s), make_small(us)); hp += 4; - msg = TUPLE6(hp, am_profile, am_scheduler, scheduler_id, state, - make_small(active_sched), timestamp); hp += 7; + msg = TUPLE6(hp, am_profile, am_scheduler, scheduler_id, + state, make_small(active_sched), + NIL /* Will be overwritten by timestamp */); + hp += 7; + + /* Write timestamp in element 6 of the 'msg' tuple */ + hp[-1] = write_ts(erts_system_profile_ts_type, hp, bp, NULL); #ifndef ERTS_SMP profile_send(NIL, msg); @@ -2711,7 +2779,7 @@ trace_port_open(Port *p, Eterm calling_pid, Eterm drv_name) { Eterm* hp; if (is_internal_port(ERTS_TRACER_PROC(p))) { -#define LOCAL_HEAP_SIZE (5+6) +#define LOCAL_HEAP_SIZE (6+ERTS_TRACE_PATCH_TS_MAX_SIZE) DeclareTmpHeapNoproc(local_heap,LOCAL_HEAP_SIZE); UseTmpHeapNoproc(LOCAL_HEAP_SIZE); @@ -2720,9 +2788,7 @@ trace_port_open(Port *p, Eterm calling_pid, Eterm drv_name) { mess = TUPLE5(hp, am_trace, calling_pid, am_open, p->common.id, drv_name); hp += 6; erts_smp_mtx_lock(&smq_mtx); - if (ERTS_TRACE_FLAGS(p) & F_TIMESTAMP) { - hp = patch_ts(mess, hp); - } + PATCH_TS(TFLGS_TS_TYPE(p), mess, hp, NULL, NULL); /* No fake schedule */ send_to_port(NULL, mess, &ERTS_TRACER_PROC(p), &ERTS_TRACE_FLAGS(p)); UnUseTmpHeapNoproc(LOCAL_HEAP_SIZE); @@ -2736,7 +2802,7 @@ trace_port_open(Port *p, Eterm calling_pid, Eterm drv_name) { ASSERT(is_internal_pid(ERTS_TRACER_PROC(p))); - sz_data = 6 + TS_SIZE(p); + sz_data = 6 + PATCH_TS_SIZE(p); ERTS_GET_TRACER_REF(tracer_ref, ERTS_TRACER_PROC(p), @@ -2749,9 +2815,7 @@ trace_port_open(Port *p, Eterm calling_pid, Eterm drv_name) { erts_smp_mtx_lock(&smq_mtx); - if (ERTS_TRACE_FLAGS(p) & F_TIMESTAMP) { - hp = patch_ts(mess, hp); - } + PATCH_TS(TFLGS_TS_TYPE(p), mess, hp, bp, tracer_ref); ERTS_ENQ_TRACE_MSG(p->common.id, tracer_ref, mess, bp); erts_smp_mtx_unlock(&smq_mtx); @@ -2775,7 +2839,7 @@ trace_port(Port *t_p, Eterm what, Eterm data) { || erts_thr_progress_is_blocking()); if (is_internal_port(ERTS_TRACER_PROC(t_p))) { -#define LOCAL_HEAP_SIZE (5+5) +#define LOCAL_HEAP_SIZE (5+ERTS_TRACE_PATCH_TS_MAX_SIZE) DeclareTmpHeapNoproc(local_heap,LOCAL_HEAP_SIZE); UseTmpHeapNoproc(LOCAL_HEAP_SIZE); @@ -2783,9 +2847,7 @@ trace_port(Port *t_p, Eterm what, Eterm data) { mess = TUPLE4(hp, am_trace, t_p->common.id, what, data); hp += 5; erts_smp_mtx_lock(&smq_mtx); - if (ERTS_TRACE_FLAGS(t_p) & F_TIMESTAMP) { - hp = patch_ts(mess, hp); - } + PATCH_TS(TFLGS_TS_TYPE(t_p), mess, hp, NULL, NULL); /* No fake schedule */ send_to_port(NULL,mess,&ERTS_TRACER_PROC(t_p),&ERTS_TRACE_FLAGS(t_p)); UnUseTmpHeapNoproc(LOCAL_HEAP_SIZE); @@ -2799,7 +2861,7 @@ trace_port(Port *t_p, Eterm what, Eterm data) { ASSERT(is_internal_pid(ERTS_TRACER_PROC(t_p))); - sz_data = 5 + TS_SIZE(t_p); + sz_data = 5 + PATCH_TS_SIZE(t_p); ERTS_GET_TRACER_REF(tracer_ref, ERTS_TRACER_PROC(t_p), @@ -2812,9 +2874,7 @@ trace_port(Port *t_p, Eterm what, Eterm data) { erts_smp_mtx_lock(&smq_mtx); - if (ERTS_TRACE_FLAGS(t_p) & F_TIMESTAMP) { - hp = patch_ts(mess, hp); - } + PATCH_TS(TFLGS_TS_TYPE(t_p), mess, hp, bp, tracer_ref); ERTS_ENQ_TRACE_MSG(t_p->common.id, tracer_ref, mess, bp); erts_smp_mtx_unlock(&smq_mtx); @@ -2842,7 +2902,7 @@ trace_sched_ports_where(Port *p, Eterm what, Eterm where) { Eterm sched_id = am_undefined; if (is_internal_port(ERTS_TRACER_PROC(p))) { -#define LOCAL_HEAP_SIZE (5+6) +#define LOCAL_HEAP_SIZE (6+ERTS_TRACE_PATCH_TS_MAX_SIZE) DeclareTmpHeapNoproc(local_heap,LOCAL_HEAP_SIZE); UseTmpHeapNoproc(LOCAL_HEAP_SIZE); @@ -2865,9 +2925,8 @@ trace_sched_ports_where(Port *p, Eterm what, Eterm where) { hp += ws; erts_smp_mtx_lock(&smq_mtx); - if (ERTS_TRACE_FLAGS(p) & F_TIMESTAMP) { - hp = patch_ts(mess, hp); - } + + PATCH_TS(TFLGS_TS_TYPE(p), mess, hp, NULL, NULL); /* No fake scheduling */ send_to_port(NULL, mess, &ERTS_TRACER_PROC(p), &ERTS_TRACE_FLAGS(p)); @@ -2887,7 +2946,7 @@ trace_sched_ports_where(Port *p, Eterm what, Eterm where) { ERTS_TRACER_PROC(p), ERTS_TRACE_FLAGS(p)); - hp = ERTS_ALLOC_SYSMSG_HEAP(ws+TS_SIZE(p), &bp, &off_heap, tracer_ref); + hp = ERTS_ALLOC_SYSMSG_HEAP(ws+PATCH_TS_SIZE(p), &bp, &off_heap, tracer_ref); if (IS_TRACED_FL(p, F_TRACE_SCHED_NO)) { #ifdef ERTS_SMP @@ -2905,10 +2964,7 @@ trace_sched_ports_where(Port *p, Eterm what, Eterm where) { erts_smp_mtx_lock(&smq_mtx); - if (ERTS_TRACE_FLAGS(p) & F_TIMESTAMP) { - hp = patch_ts(mess, hp); - } - + PATCH_TS(TFLGS_TS_TYPE(p), mess, hp, bp, tracer_ref); ERTS_ENQ_TRACE_MSG(p->common.id, tracer_ref, mess, bp); erts_smp_mtx_unlock(&smq_mtx); } @@ -2918,13 +2974,12 @@ trace_sched_ports_where(Port *p, Eterm what, Eterm where) { void profile_runnable_port(Port *p, Eterm status) { - Uint Ms, s, us; - Eterm *hp, msg, timestamp; - + Eterm *hp, msg; + ErlHeapFragment *bp = NULL; Eterm count = make_small(0); #ifndef ERTS_SMP -#define LOCAL_HEAP_SIZE (4 + 6) +#define LOCAL_HEAP_SIZE (6 + ERTS_TRACE_PATCH_TS_MAX_SIZE) DeclareTmpHeapNoproc(local_heap,LOCAL_HEAP_SIZE); UseTmpHeapNoproc(LOCAL_HEAP_SIZE); @@ -2932,10 +2987,9 @@ profile_runnable_port(Port *p, Eterm status) { hp = local_heap; #else - ErlHeapFragment *bp; Uint hsz; - hsz = 4 + 6; + hsz = 6 + patch_ts_size(erts_system_profile_ts_type)-1; bp = new_message_buffer(hsz); hp = bp->mem; @@ -2943,9 +2997,12 @@ profile_runnable_port(Port *p, Eterm status) { erts_smp_mtx_lock(&smq_mtx); - GET_NOW(&Ms, &s, &us); - timestamp = TUPLE3(hp, make_small(Ms), make_small(s), make_small(us)); hp += 4; - msg = TUPLE5(hp, am_profile, p->common.id, status, count, timestamp); hp += 6; + msg = TUPLE5(hp, am_profile, p->common.id, status, count, + NIL /* Will be overwritten by timestamp */); + hp += 6; + + /* Write timestamp in element 5 of the 'msg' tuple */ + hp[-1] = write_ts(erts_system_profile_ts_type, hp, bp, NULL); #ifndef ERTS_SMP profile_send(p->common.id, msg); @@ -2960,20 +3017,19 @@ profile_runnable_port(Port *p, Eterm status) { /* Process profiling */ void profile_runnable_proc(Process *p, Eterm status){ - Uint Ms, s, us; - Eterm *hp, msg, timestamp; + Eterm *hp, msg; Eterm where = am_undefined; + ErlHeapFragment *bp = NULL; #ifndef ERTS_SMP -#define LOCAL_HEAP_SIZE (4 + 6 + 4) +#define LOCAL_HEAP_SIZE (4 + 6 + ERTS_TRACE_PATCH_TS_MAX_SIZE) DeclareTmpHeapNoproc(local_heap,LOCAL_HEAP_SIZE); UseTmpHeapNoproc(LOCAL_HEAP_SIZE); hp = local_heap; #else - ErlHeapFragment *bp; - Uint hsz = 4 + 6 + 4; + Uint hsz = 4 + 6 + patch_ts_size(erts_system_profile_ts_type)-1; #endif if (!p->current) { @@ -2982,7 +3038,7 @@ profile_runnable_proc(Process *p, Eterm status){ #ifdef ERTS_SMP if (!p->current) { - hsz = 4 + 6; + hsz -= 4; } bp = new_message_buffer(hsz); @@ -2997,9 +3053,13 @@ profile_runnable_proc(Process *p, Eterm status){ erts_smp_mtx_lock(&smq_mtx); - GET_NOW(&Ms, &s, &us); - timestamp = TUPLE3(hp, make_small(Ms), make_small(s), make_small(us)); hp += 4; - msg = TUPLE5(hp, am_profile, p->common.id, status, where, timestamp); hp += 6; + msg = TUPLE5(hp, am_profile, p->common.id, status, where, + NIL /* Will be overwritten by timestamp */); + hp += 6; + + /* Write timestamp in element 5 of the 'msg' tuple */ + hp[-1] = write_ts(erts_system_profile_ts_type, hp, bp, NULL); + #ifndef ERTS_SMP profile_send(p->common.id, msg); UnUseTmpHeapNoproc(LOCAL_HEAP_SIZE); @@ -3328,8 +3388,6 @@ sys_msg_dispatcher_func(void *unused) if (erts_thr_progress_update(NULL)) erts_thr_progress_leader_update(NULL); - ERTS_SCHED_FAIR_YIELD(); - #ifdef DEBUG_PRINTOUTS print_msg_type(smqp); #endif @@ -3388,12 +3446,11 @@ sys_msg_dispatcher_func(void *unused) goto failure; } else { + ErtsMessage *mp; queue_proc_msg: - erts_queue_message(proc,&proc_locks,smqp->bp,smqp->msg,NIL -#ifdef USE_VM_PROBES - , NIL -#endif - ); + mp = erts_alloc_message(0, NULL); + mp->data.heap_frag = smqp->bp; + erts_queue_message(proc,&proc_locks,mp,smqp->msg,NIL); #ifdef DEBUG_PRINTOUTS erts_fprintf(stderr, "delivered\n"); #endif @@ -3488,20 +3545,13 @@ static void init_sys_msg_dispatcher(void) { erts_smp_thr_opts_t thr_opts = ERTS_SMP_THR_OPTS_DEFAULT_INITER; -#ifdef __OSE__ - thr_opts.coreNo = 0; -#endif thr_opts.detached = 1; + thr_opts.name = "sys_msg_dispatcher"; init_smq_element_alloc(); sys_message_queue = NULL; sys_message_queue_end = NULL; erts_smp_cnd_init(&smq_cnd); erts_smp_mtx_init(&smq_mtx, "sys_msg_q"); - -#ifdef ETHR_HAVE_THREAD_NAMES - thr_opts.name = "sys_msg_dispatcher"; -#endif - erts_smp_thr_create(&sys_msg_dispatcher_tid, sys_msg_dispatcher_func, NULL, diff --git a/erts/emulator/beam/erl_trace.h b/erts/emulator/beam/erl_trace.h index 4f2c70d6e7..a0058264d7 100644 --- a/erts/emulator/beam/erl_trace.h +++ b/erts/emulator/beam/erl_trace.h @@ -3,22 +3,53 @@ * * Copyright Ericsson AB 2012-2013. All Rights Reserved. * - * The contents of this file are subject to the Erlang Public License, - * Version 1.1, (the "License"); you may not use this file except in - * compliance with the License. You should have received a copy of the - * Erlang Public License along with this software. If not, it can be - * retrieved online at http://www.erlang.org/. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Software distributed under the License is distributed on an "AS IS" - * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See - * the License for the specific language governing rights and limitations - * under the License. + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. * * %CopyrightEnd% */ +#ifndef ERL_TRACE_H__FLAGS__ +#define ERL_TRACE_H__FLAGS__ +/* + * NOTE! The bits used for these flags matter. The flag with + * the least significant bit will take precedence! + * + * The "now timestamp" has highest precedence due to + * compatibility reasons. + */ +#define ERTS_TRACE_FLG_NOW_TIMESTAMP (1 << 0) +#define ERTS_TRACE_FLG_STRICT_MONOTONIC_TIMESTAMP (1 << 1) +#define ERTS_TRACE_FLG_MONOTONIC_TIMESTAMP (1 << 2) + +/* + * The bits used effects trace flags (of processes and ports) + * as well as sequential trace flags. If changed make sure + * these arn't messed up... + */ +#define ERTS_TRACE_TS_TYPE_BITS 3 +#define ERTS_TRACE_TS_TYPE_MASK \ + ((1 << ERTS_TRACE_TS_TYPE_BITS) - 1) + +#define ERTS_TFLGS2TSTYPE(TFLGS) \ + ((int) (((TFLGS) >> ERTS_TRACE_FLAGS_TS_TYPE_SHIFT) \ + & ERTS_TRACE_TS_TYPE_MASK)) +#define ERTS_SEQTFLGS2TSTYPE(SEQTFLGS) \ + ((int) (((SEQTFLGS) >> ERTS_SEQ_TRACE_FLAGS_TS_TYPE_SHIFT) \ + & ERTS_TRACE_TS_TYPE_MASK)) + +#endif /* ERL_TRACE_H__FLAGS__ */ -#ifndef ERL_TRACE_H__ +#if !defined(ERL_TRACE_H__) && !defined(ERTS_ONLY_INCLUDE_TRACE_FLAGS) #define ERL_TRACE_H__ struct binary; diff --git a/erts/emulator/beam/erl_unicode.c b/erts/emulator/beam/erl_unicode.c index 3a968594f3..36d85b0a22 100644 --- a/erts/emulator/beam/erl_unicode.c +++ b/erts/emulator/beam/erl_unicode.c @@ -3,16 +3,17 @@ * * Copyright Ericsson AB 2008-2013. All Rights Reserved. * - * The contents of this file are subject to the Erlang Public License, - * Version 1.1, (the "License"); you may not use this file except in - * compliance with the License. You should have received a copy of the - * Erlang Public License along with this software. If not, it can be - * retrieved online at http://www.erlang.org/. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Software distributed under the License is distributed on an "AS IS" - * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See - * the License for the specific language governing rights and limitations - * under the License. + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. * * %CopyrightEnd% */ @@ -2126,6 +2127,8 @@ Eterm erts_convert_native_to_filename(Process *p, byte *bytes) mac = 1; case ERL_FILENAME_UTF8: size = strlen((char *) bytes); + if (size == 0) + return NIL; if (erts_analyze_utf8(bytes,size,&err_pos,&num_chars,NULL) != ERTS_UTF8_OK) { goto noconvert; } @@ -2504,7 +2507,7 @@ void erts_copy_utf8_to_utf16_little(byte *target, byte *bytes, int num_chars) ((Uint) (bytes[3] & ((byte) 0x3F))); bytes += 4; } else { - erl_exit(1,"Internal unicode error in prim_file:internal_name2native/1"); + erts_exit(ERTS_ERROR_EXIT,"Internal unicode error in prim_file:internal_name2native/1"); } *target++ = (byte) (unipoint & 0xFF); *target++ = (byte) ((unipoint >> 8) & 0xFF); diff --git a/erts/emulator/beam/erl_unicode.h b/erts/emulator/beam/erl_unicode.h index 1b63b797c2..4c25d89b7c 100644 --- a/erts/emulator/beam/erl_unicode.h +++ b/erts/emulator/beam/erl_unicode.h @@ -3,16 +3,17 @@ * * Copyright Ericsson AB 2008-2009. All Rights Reserved. * - * The contents of this file are subject to the Erlang Public License, - * Version 1.1, (the "License"); you may not use this file except in - * compliance with the License. You should have received a copy of the - * Erlang Public License along with this software. If not, it can be - * retrieved online at http://www.erlang.org/. - * - * Software distributed under the License is distributed on an "AS IS" - * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See - * the License for the specific language governing rights and limitations - * under the License. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. * * %CopyrightEnd% */ diff --git a/erts/emulator/beam/erl_unicode_normalize.h b/erts/emulator/beam/erl_unicode_normalize.h index fb0a111ca2..16c62db50e 100644 --- a/erts/emulator/beam/erl_unicode_normalize.h +++ b/erts/emulator/beam/erl_unicode_normalize.h @@ -1,21 +1,22 @@ /* -* %CopyrightBegin% -* -* Copyright Ericsson AB 1999-2010. All Rights Reserved. -* -* The contents of this file are subject to the Erlang Public License, -* Version 1.1, (the "License"); you may not use this file except in -* compliance with the License. You should have received a copy of the -* Erlang Public License along with this software. If not, it can be -* retrieved online at http://www.erlang.org/. -* -* Software distributed under the License is distributed on an "AS IS" -* basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See -* the License for the specific language governing rights and limitations -* under the License. -* -* %CopyrightEnd% -*/ + * %CopyrightBegin% + * + * Copyright Ericsson AB 1999-2010. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * %CopyrightEnd% + */ /* * This file is automatically generated by dec.erl, do not edit manually */ diff --git a/erts/emulator/beam/erl_utils.h b/erts/emulator/beam/erl_utils.h index 0807649ea1..640969daf0 100644 --- a/erts/emulator/beam/erl_utils.h +++ b/erts/emulator/beam/erl_utils.h @@ -3,16 +3,17 @@ * * Copyright Ericsson AB 2012-2014. All Rights Reserved. * - * The contents of this file are subject to the Erlang Public License, - * Version 1.1, (the "License"); you may not use this file except in - * compliance with the License. You should have received a copy of the - * Erlang Public License along with this software. If not, it can be - * retrieved online at http://www.erlang.org/. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Software distributed under the License is distributed on an "AS IS" - * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See - * the License for the specific language governing rights and limitations - * under the License. + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. * * %CopyrightEnd% */ @@ -32,11 +33,7 @@ typedef struct { #endif union { Uint64 not_atomic; -#ifdef ARCH_64 - erts_atomic_t atomic; -#else - erts_dw_atomic_t atomic; -#endif + erts_atomic64_t atomic; } counter; } erts_interval_t; @@ -50,9 +47,6 @@ Uint64 erts_ensure_later_interval_nob(erts_interval_t *, Uint64); Uint64 erts_ensure_later_interval_acqb(erts_interval_t *, Uint64); Uint64 erts_smp_ensure_later_interval_nob(erts_interval_t *, Uint64); Uint64 erts_smp_ensure_later_interval_acqb(erts_interval_t *, Uint64); -#ifdef ARCH_32 -ERTS_GLB_INLINE Uint64 erts_interval_dw_aint_to_val__(erts_dw_aint_t *); -#endif ERTS_GLB_INLINE Uint64 erts_current_interval_nob__(erts_interval_t *); ERTS_GLB_INLINE Uint64 erts_current_interval_acqb__(erts_interval_t *); ERTS_GLB_INLINE Uint64 erts_current_interval_nob(erts_interval_t *); @@ -62,46 +56,16 @@ ERTS_GLB_INLINE Uint64 erts_smp_current_interval_acqb(erts_interval_t *); #if ERTS_GLB_INLINE_INCL_FUNC_DEF -#ifdef ARCH_32 - -ERTS_GLB_INLINE Uint64 -erts_interval_dw_aint_to_val__(erts_dw_aint_t *dw) -{ -#ifdef ETHR_SU_DW_NAINT_T__ - return (Uint64) dw->dw_sint; -#else - Uint64 res; - res = (Uint64) ((Uint32) dw->sint[ERTS_DW_AINT_HIGH_WORD]); - res <<= 32; - res |= (Uint64) ((Uint32) dw->sint[ERTS_DW_AINT_LOW_WORD]); - return res; -#endif -} - -#endif - ERTS_GLB_INLINE Uint64 erts_current_interval_nob__(erts_interval_t *icp) { -#ifdef ARCH_64 - return (Uint64) erts_atomic_read_nob(&icp->counter.atomic); -#else - erts_dw_aint_t dw; - erts_dw_atomic_read_nob(&icp->counter.atomic, &dw); - return erts_interval_dw_aint_to_val__(&dw); -#endif + return (Uint64) erts_atomic64_read_nob(&icp->counter.atomic); } ERTS_GLB_INLINE Uint64 erts_current_interval_acqb__(erts_interval_t *icp) { -#ifdef ARCH_64 - return (Uint64) erts_atomic_read_acqb(&icp->counter.atomic); -#else - erts_dw_aint_t dw; - erts_dw_atomic_read_acqb(&icp->counter.atomic, &dw); - return erts_interval_dw_aint_to_val__(&dw); -#endif + return (Uint64) erts_atomic64_read_acqb(&icp->counter.atomic); } ERTS_GLB_INLINE Uint64 @@ -150,12 +114,14 @@ void erts_silence_warn_unused_result(long unused); int erts_fit_in_bits_int64(Sint64); int erts_fit_in_bits_int32(Sint32); -int erts_list_length(Eterm); +int erts_fit_in_bits_uint(Uint); +Sint erts_list_length(Eterm); int erts_is_builtin(Eterm, Eterm, int); Uint32 make_broken_hash(Eterm); Uint32 block_hash(byte *, unsigned, Uint32); Uint32 make_hash2(Eterm); Uint32 make_hash(Eterm); +Uint32 make_internal_hash(Eterm); void erts_save_emu_args(int argc, char **argv); Eterm erts_get_emu_args(struct process *c_p); @@ -191,49 +157,46 @@ void erts_init_utils_mem(void); erts_dsprintf_buf_t *erts_create_tmp_dsbuf(Uint); void erts_destroy_tmp_dsbuf(erts_dsprintf_buf_t *); -#if HALFWORD_HEAP -int eq_rel(Eterm a, Eterm* a_base, Eterm b, Eterm* b_base); -# define eq(A,B) eq_rel(A,NULL,B,NULL) -#else int eq(Eterm, Eterm); -# define eq_rel(A,A_BASE,B,B_BASE) eq(A,B) -#endif #define EQ(x,y) (((x) == (y)) || (is_not_both_immed((x),(y)) && eq((x),(y)))) -#if HALFWORD_HEAP -Sint erts_cmp_rel_opt(Eterm, Eterm*, Eterm, Eterm*, int); -#define cmp_rel(A,A_BASE,B,B_BASE) erts_cmp_rel_opt(A,A_BASE,B,B_BASE,0) -#define cmp_rel_term(A,A_BASE,B,B_BASE) erts_cmp_rel_opt(A,A_BASE,B,B_BASE,1) -#define CMP(A,B) erts_cmp_rel_opt(A,NULL,B,NULL,0) -#define CMP_TERM(A,B) erts_cmp_rel_opt(A,NULL,B,NULL,1) -#else -Sint cmp(Eterm, Eterm); -Sint erts_cmp(Eterm, Eterm, int); -#define cmp_rel(A,A_BASE,B,B_BASE) erts_cmp(A,B,0) -#define cmp_rel_term(A,A_BASE,B,B_BASE) erts_cmp(A,B,1) -#define CMP(A,B) erts_cmp(A,B,0) -#define CMP_TERM(A,B) erts_cmp(A,B,1) -#endif - -#define cmp_lt(a,b) (CMP((a),(b)) < 0) -#define cmp_le(a,b) (CMP((a),(b)) <= 0) -#define cmp_eq(a,b) (CMP((a),(b)) == 0) -#define cmp_ne(a,b) (CMP((a),(b)) != 0) -#define cmp_ge(a,b) (CMP((a),(b)) >= 0) -#define cmp_gt(a,b) (CMP((a),(b)) > 0) - -#define cmp_lt_term(a,b) (CMP_TERM((a),(b)) < 0) -#define cmp_le_term(a,b) (CMP_TERM((a),(b)) <= 0) -#define cmp_ge_term(a,b) (CMP_TERM((a),(b)) >= 0) -#define cmp_gt_term(a,b) (CMP_TERM((a),(b)) > 0) - -#define CMP_LT(a,b) ((a) != (b) && cmp_lt((a),(b))) -#define CMP_GE(a,b) ((a) == (b) || cmp_ge((a),(b))) -#define CMP_EQ(a,b) ((a) == (b) || cmp_eq((a),(b))) -#define CMP_NE(a,b) ((a) != (b) && cmp_ne((a),(b))) - -#define CMP_LT_TERM(a,b) ((a) != (b) && cmp_lt_term((a),(b))) -#define CMP_GE_TERM(a,b) ((a) == (b) || cmp_ge_term((a),(b))) +int erts_cmp_atoms(Eterm a, Eterm b); +Sint erts_cmp(Eterm, Eterm, int, int); +Sint erts_cmp_compound(Eterm, Eterm, int, int); +Sint cmp(Eterm a, Eterm b); +#define CMP(A,B) erts_cmp(A,B,0,0) +#define CMP_TERM(A,B) erts_cmp(A,B,1,0) +#define CMP_EQ_ONLY(A,B) erts_cmp(A,B,0,1) + +#define CMP_LT(a,b) ((a) != (b) && CMP((a),(b)) < 0) +#define CMP_LE(a,b) ((a) == (b) || CMP((a),(b)) <= 0) +#define CMP_EQ(a,b) ((a) == (b) || CMP_EQ_ONLY((a),(b)) == 0) +#define CMP_NE(a,b) ((a) != (b) && CMP_EQ_ONLY((a),(b)) != 0) +#define CMP_GE(a,b) ((a) == (b) || CMP((a),(b)) >= 0) +#define CMP_GT(a,b) ((a) != (b) && CMP((a),(b)) > 0) + +#define CMP_EQ_ACTION(X,Y,Action) \ + if ((X) != (Y)) { CMP_SPEC((X),(Y),!=,Action,1); } +#define CMP_NE_ACTION(X,Y,Action) \ + if ((X) == (Y)) { Action; } else { CMP_SPEC((X),(Y),==,Action,1); } +#define CMP_GE_ACTION(X,Y,Action) \ + if ((X) != (Y)) { CMP_SPEC((X),(Y),<,Action,0); } +#define CMP_LT_ACTION(X,Y,Action) \ + if ((X) == (Y)) { Action; } else { CMP_SPEC((X),(Y),>=,Action,0); } + +#define CMP_SPEC(X,Y,Op,Action,EqOnly) \ + if (is_atom(X) && is_atom(Y)) { \ + if (erts_cmp_atoms(X, Y) Op 0) { Action; }; \ + } else if (is_both_small(X, Y)) { \ + if (signed_val(X) Op signed_val(Y)) { Action; }; \ + } else if (is_float(X) && is_float(Y)) { \ + FloatDef af, bf; \ + GET_DOUBLE(X, af); \ + GET_DOUBLE(Y, bf); \ + if (af.fd Op bf.fd) { Action; }; \ + } else { \ + if (erts_cmp_compound(X,Y,0,EqOnly) Op 0) { Action; }; \ + } #endif diff --git a/erts/emulator/beam/erl_vm.h b/erts/emulator/beam/erl_vm.h index b7de8208ad..98f27a1725 100644 --- a/erts/emulator/beam/erl_vm.h +++ b/erts/emulator/beam/erl_vm.h @@ -3,16 +3,17 @@ * * Copyright Ericsson AB 1996-2014. All Rights Reserved. * - * The contents of this file are subject to the Erlang Public License, - * Version 1.1, (the "License"); you may not use this file except in - * compliance with the License. You should have received a copy of the - * Erlang Public License along with this software. If not, it can be - * retrieved online at http://www.erlang.org/. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Software distributed under the License is distributed on an "AS IS" - * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See - * the License for the specific language governing rights and limitations - * under the License. + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. * * %CopyrightEnd% */ @@ -20,8 +21,6 @@ #ifndef __ERL_VM_H__ #define __ERL_VM_H__ -/* #define ERTS_OPCODE_COUNTER_SUPPORT */ - /* FORCE_HEAP_FRAGS: * Debug provocation to make HAlloc always create heap fragments (if allowed) * even if there is room on heap. @@ -41,14 +40,6 @@ #define MAX_ARG 255 /* Max number of arguments allowed */ #define MAX_REG 1024 /* Max number of x(N) registers used */ -/* Scheduler stores data for temporary heaps if - !HEAP_ON_C_STACK. Macros (*TmpHeap*) in global.h selects if we put temporary - heap data on the C stack or if we use the buffers in the scheduler data. */ -#define TMP_HEAP_SIZE 128 /* Number of Eterm in the schedulers - small heap for transient heap data */ -#define ERL_ARITH_TMP_HEAP_SIZE 4 /* as does erl_arith... */ -#define BEAM_EMU_TMP_HEAP_SIZE 2 /* and beam_emu... */ - /* * The new arithmetic operations need some extra X registers in the register array. * so does the gc_bif's (i_gc_bif3 need 3 extra). @@ -118,12 +109,8 @@ #define HeapWordsLeft(p) (HEAP_LIMIT(p) - HEAP_TOP(p)) #if defined(DEBUG) || defined(CHECK_FOR_HOLES) -#if HALFWORD_HEAP -# define ERTS_HOLE_MARKER (0xaf5e78ccU) -#else -# define ERTS_HOLE_MARKER (((0xaf5e78ccUL << 24) << 8) | 0xaf5e78ccUL) -#endif -#endif +# define ERTS_HOLE_MARKER (((0xdeadbeef << 24) << 8) | 0xdeadbeef) +#endif /* egil: 32-bit ? */ /* * Allocate heap memory on the ordinary heap, NEVER in a heap @@ -154,6 +141,7 @@ typedef struct op_entry { char* name; /* Name of instruction. */ Uint32 mask[3]; /* Signature mask. */ + unsigned involves_r; /* Needs special attention when matching. */ int sz; /* Number of loaded words. */ char* pack; /* Instructions for packing engine. */ char* sign; /* Signature string. */ @@ -174,6 +162,7 @@ extern int H_MIN_SIZE; /* minimum (heap + stack) */ extern int BIN_VH_MIN_SIZE; /* minimum virtual (bin) heap */ extern int erts_atom_table_size;/* Atom table size */ +extern int erts_pd_initial_size;/* Initial Process dictionary table size */ #define ORIG_CREATION 0 #define INTERNAL_CREATION 255 diff --git a/erts/emulator/beam/erl_zlib.c b/erts/emulator/beam/erl_zlib.c index 8e33144f96..e44e86c39c 100644 --- a/erts/emulator/beam/erl_zlib.c +++ b/erts/emulator/beam/erl_zlib.c @@ -3,16 +3,17 @@ * * Copyright Ericsson AB 2009-2013. All Rights Reserved. * - * The contents of this file are subject to the Erlang Public License, - * Version 1.1, (the "License"); you may not use this file except in - * compliance with the License. You should have received a copy of the - * Erlang Public License along with this software. If not, it can be - * retrieved online at http://www.erlang.org/. - * - * Software distributed under the License is distributed on an "AS IS" - * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See - * the License for the specific language governing rights and limitations - * under the License. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. * * %CopyrightEnd% */ diff --git a/erts/emulator/beam/erl_zlib.h b/erts/emulator/beam/erl_zlib.h index 160166c66b..2ca1e3735e 100644 --- a/erts/emulator/beam/erl_zlib.h +++ b/erts/emulator/beam/erl_zlib.h @@ -3,16 +3,17 @@ * * Copyright Ericsson AB 2009-2013. All Rights Reserved. * - * The contents of this file are subject to the Erlang Public License, - * Version 1.1, (the "License"); you may not use this file except in - * compliance with the License. You should have received a copy of the - * Erlang Public License along with this software. If not, it can be - * retrieved online at http://www.erlang.org/. - * - * Software distributed under the License is distributed on an "AS IS" - * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See - * the License for the specific language governing rights and limitations - * under the License. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. * * %CopyrightEnd% */ diff --git a/erts/emulator/beam/erlang_dtrace.d b/erts/emulator/beam/erlang_dtrace.d index e3ebbb84f4..c682f76e3a 100644 --- a/erts/emulator/beam/erlang_dtrace.d +++ b/erts/emulator/beam/erlang_dtrace.d @@ -4,16 +4,17 @@ * Copyright Dustin Sallings, Michal Ptaszek, Scott Lystig Fritchie 2011-2012. * All Rights Reserved. * - * The contents of this file are subject to the Erlang Public License, - * Version 1.1, (the "License"); you may not use this file except in - * compliance with the License. You should have received a copy of the - * Erlang Public License along with this software. If not, it can be - * retrieved online at http://www.erlang.org/. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Software distributed under the License is distributed on an "AS IS" - * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See - * the License for the specific language governing rights and limitations - * under the License. + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. * * %CopyrightEnd% */ diff --git a/erts/emulator/beam/error.h b/erts/emulator/beam/error.h index ddc2c1396d..cba8672c68 100644 --- a/erts/emulator/beam/error.h +++ b/erts/emulator/beam/error.h @@ -3,16 +3,17 @@ * * Copyright Ericsson AB 1996-2010. All Rights Reserved. * - * The contents of this file are subject to the Erlang Public License, - * Version 1.1, (the "License"); you may not use this file except in - * compliance with the License. You should have received a copy of the - * Erlang Public License along with this software. If not, it can be - * retrieved online at http://www.erlang.org/. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Software distributed under the License is distributed on an "AS IS" - * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See - * the License for the specific language governing rights and limitations - * under the License. + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. * * %CopyrightEnd% */ @@ -140,7 +141,13 @@ #define EXC_NOTSUP ((17 << 8) | EXC_ERROR) /* Not supported */ -#define NUMBER_EXIT_CODES 18 /* The number of exit code indices */ +#define EXC_BADMAP ((18 << 8) | EXC_ERROR) + /* Bad map */ + +#define EXC_BADKEY ((19 << 8) | EXC_ERROR) + /* Bad key in map */ + +#define NUMBER_EXIT_CODES 20 /* The number of exit code indices */ /* * Internal pseudo-error codes. @@ -152,6 +159,8 @@ */ #define BADARG EXC_BADARG #define BADARITH EXC_BADARITH +#define BADKEY EXC_BADKEY +#define BADMAP EXC_BADMAP #define BADMATCH EXC_BADMATCH #define SYSTEM_LIMIT EXC_SYSTEM_LIMIT diff --git a/erts/emulator/beam/export.c b/erts/emulator/beam/export.c index b0f08d8245..581efe6eec 100644 --- a/erts/emulator/beam/export.c +++ b/erts/emulator/beam/export.c @@ -3,16 +3,17 @@ * * Copyright Ericsson AB 1996-2013. All Rights Reserved. * - * The contents of this file are subject to the Erlang Public License, - * Version 1.1, (the "License"); you may not use this file except in - * compliance with the License. You should have received a copy of the - * Erlang Public License along with this software. If not, it can be - * retrieved online at http://www.erlang.org/. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Software distributed under the License is distributed on an "AS IS" - * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See - * the License for the specific language governing rights and limitations - * under the License. + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. * * %CopyrightEnd% */ @@ -183,6 +184,9 @@ init_export_table(void) f.cmp = (HCMP_FUN) export_cmp; f.alloc = (HALLOC_FUN) export_alloc; f.free = (HFREE_FUN) export_free; + f.meta_alloc = (HMALLOC_FUN) erts_alloc; + f.meta_free = (HMFREE_FUN) erts_free; + f.meta_print = (HMPRINT_FUN) erts_print; for (i=0; i<ERTS_NUM_CODE_IX; i++) { erts_index_init(ERTS_ALC_T_EXPORT_TABLE, &export_tables[i], "export_list", diff --git a/erts/emulator/beam/export.h b/erts/emulator/beam/export.h index 61a54de59f..a8bc9d2f66 100644 --- a/erts/emulator/beam/export.h +++ b/erts/emulator/beam/export.h @@ -3,16 +3,17 @@ * * Copyright Ericsson AB 1996-2013. All Rights Reserved. * - * The contents of this file are subject to the Erlang Public License, - * Version 1.1, (the "License"); you may not use this file except in - * compliance with the License. You should have received a copy of the - * Erlang Public License along with this software. If not, it can be - * retrieved online at http://www.erlang.org/. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Software distributed under the License is distributed on an "AS IS" - * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See - * the License for the specific language governing rights and limitations - * under the License. + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. * * %CopyrightEnd% */ diff --git a/erts/emulator/beam/external.c b/erts/emulator/beam/external.c index 656de7c49a..10f03636ec 100644 --- a/erts/emulator/beam/external.c +++ b/erts/emulator/beam/external.c @@ -3,16 +3,17 @@ * * Copyright Ericsson AB 1996-2014. All Rights Reserved. * - * The contents of this file are subject to the Erlang Public License, - * Version 1.1, (the "License"); you may not use this file except in - * compliance with the License. You should have received a copy of the - * Erlang Public License along with this software. If not, it can be - * retrieved online at http://www.erlang.org/. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Software distributed under the License is distributed on an "AS IS" - * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See - * the License for the specific language governing rights and limitations - * under the License. + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. * * %CopyrightEnd% */ @@ -36,7 +37,9 @@ #include "erl_process.h" #include "error.h" #include "external.h" +#define ERL_WANT_HIPE_BIF_WRAPPER__ #include "bif.h" +#undef ERL_WANT_HIPE_BIF_WRAPPER__ #include "big.h" #include "dist.h" #include "erl_binary.h" @@ -44,9 +47,6 @@ #include "erl_zlib.h" #include "erl_map.h" -#ifdef HIPE -#include "hipe_mode_switch.h" -#endif #define in_area(ptr,start,nbytes) ((UWord)((char*)(ptr) - (char*)(start)) < (nbytes)) #define MAX_STRING_LEN 0xffff @@ -95,9 +95,9 @@ static Uint is_external_string(Eterm obj, int* p_is_string); static byte* enc_atom(ErtsAtomCacheMap *, Eterm, byte*, Uint32); static byte* enc_pid(ErtsAtomCacheMap *, Eterm, byte*, Uint32); struct B2TContext_t; -static byte* dec_term(ErtsDistExternal *, Eterm**, byte*, ErlOffHeap*, Eterm*, struct B2TContext_t*); +static byte* dec_term(ErtsDistExternal*, ErtsHeapFactory*, byte*, Eterm*, struct B2TContext_t*); static byte* dec_atom(ErtsDistExternal *, byte*, Eterm*); -static byte* dec_pid(ErtsDistExternal *, Eterm**, byte*, ErlOffHeap*, Eterm*); +static byte* dec_pid(ErtsDistExternal *, ErtsHeapFactory*, byte*, Eterm*); static Sint decoded_size(byte *ep, byte* endp, int internal_tags, struct B2TContext_t*); static BIF_RETTYPE term_to_binary_trap_1(BIF_ALIST_1); @@ -111,26 +111,17 @@ static int encode_size_struct_int(struct TTBSizeContext_*, ErtsAtomCacheMap *acm static Export binary_to_term_trap_export; static BIF_RETTYPE binary_to_term_trap_1(BIF_ALIST_1); -static Eterm binary_to_term_int(Process* p, Uint32 flags, Eterm bin, Binary* context_b); +static BIF_RETTYPE binary_to_term_int(Process* p, Uint32 flags, Eterm bin, Binary* context_b, + Export *bif, Eterm arg0, Eterm arg1); void erts_init_external(void) { -#if 1 /* In R16 */ erts_init_trap_export(&term_to_binary_trap_export, - am_erlang, am_term_to_binary_trap, 1, + am_erts_internal, am_term_to_binary_trap, 1, &term_to_binary_trap_1); erts_init_trap_export(&binary_to_term_trap_export, - am_erlang, am_binary_to_term_trap, 1, + am_erts_internal, am_binary_to_term_trap, 1, &binary_to_term_trap_1); -#else - sys_memset((void *) &term_to_binary_trap_export, 0, sizeof(Export)); - term_to_binary_trap_export.address = &term_to_binary_trap_export.code[3]; - term_to_binary_trap_export.code[0] = am_erlang; - term_to_binary_trap_export.code[1] = am_term_to_binary_trap; - term_to_binary_trap_export.code[2] = 1; - term_to_binary_trap_export.code[3] = (BeamInstr) em_apply_bif; - term_to_binary_trap_export.code[4] = (BeamInstr) &term_to_binary_trap_1; -#endif return; } @@ -510,15 +501,37 @@ byte *erts_encode_ext_dist_header_finalize(byte *ext, ErtsAtomCache *cache, Uint return ep; } -Uint erts_encode_dist_ext_size(Eterm term, Uint32 flags, ErtsAtomCacheMap *acmp) +int erts_encode_dist_ext_size(Eterm term, Uint32 flags, ErtsAtomCacheMap *acmp, + Uint* szp) { - Uint sz = 0; + Uint sz; + if (encode_size_struct_int(NULL, acmp, term, flags, NULL, &sz)) { + return -1; + } else { #ifndef ERTS_DEBUG_USE_DIST_SEP - if (!(flags & DFLAG_DIST_HDR_ATOM_CACHE)) + if (!(flags & DFLAG_DIST_HDR_ATOM_CACHE)) #endif - sz++ /* VERSION_MAGIC */; - sz += encode_size_struct2(acmp, term, flags); - return sz; + sz++ /* VERSION_MAGIC */; + + *szp += sz; + return 0; + } +} + +int erts_encode_dist_ext_size_int(Eterm term, struct erts_dsig_send_context* ctx, Uint* szp) +{ + Uint sz; + if (encode_size_struct_int(&ctx->u.sc, ctx->acmp, term, ctx->flags, &ctx->reds, &sz)) { + return -1; + } else { +#ifndef ERTS_DEBUG_USE_DIST_SEP + if (!(ctx->flags & DFLAG_DIST_HDR_ATOM_CACHE)) +#endif + sz++ /* VERSION_MAGIC */; + + *szp += sz; + return 0; + } } Uint erts_encode_ext_size(Eterm term) @@ -539,19 +552,16 @@ Uint erts_encode_ext_size_ets(Eterm term) } -void erts_encode_dist_ext(Eterm term, byte **ext, Uint32 flags, ErtsAtomCacheMap *acmp) +int erts_encode_dist_ext(Eterm term, byte **ext, Uint32 flags, ErtsAtomCacheMap *acmp, + TTBEncodeContext* ctx, Sint* reds) { - byte *ep = *ext; -#ifndef ERTS_DEBUG_USE_DIST_SEP - if (!(flags & DFLAG_DIST_HDR_ATOM_CACHE)) -#endif - *ep++ = VERSION_MAGIC; - ep = enc_term(acmp, term, ep, flags, NULL); - if (!ep) - erl_exit(ERTS_ABORT_EXIT, - "%s:%d:erts_encode_dist_ext(): Internal data structure error\n", - __FILE__, __LINE__); - *ext = ep; + if (!ctx || !ctx->wstack.wstart) { + #ifndef ERTS_DEBUG_USE_DIST_SEP + if (!(flags & DFLAG_DIST_HDR_ATOM_CACHE)) + #endif + *(*ext)++ = VERSION_MAGIC; + } + return enc_term_int(ctx, acmp, term, *ext, flags, NULL, reds, ext); } void erts_encode_ext(Eterm term, byte **ext) @@ -560,7 +570,7 @@ void erts_encode_ext(Eterm term, byte **ext) *ep++ = VERSION_MAGIC; ep = enc_term(NULL, term, ep, TERM_TO_BINARY_DFLAGS, NULL); if (!ep) - erl_exit(ERTS_ABORT_EXIT, + erts_exit(ERTS_ABORT_EXIT, "%s:%d:erts_encode_ext(): Internal data structure error\n", __FILE__, __LINE__); *ext = ep; @@ -921,8 +931,7 @@ Sint erts_decode_ext_size_ets(byte *ext, Uint size) ** on return hpp is updated to point after allocated data */ Eterm -erts_decode_dist_ext(Eterm** hpp, - ErlOffHeap* off_heap, +erts_decode_dist_ext(ErtsHeapFactory* factory, ErtsDistExternal *edep) { Eterm obj; @@ -942,7 +951,7 @@ erts_decode_dist_ext(Eterm** hpp, goto error; ep++; } - ep = dec_term(edep, hpp, ep, off_heap, &obj, NULL); + ep = dec_term(edep, factory, ep, &obj, NULL); if (!ep) goto error; @@ -951,33 +960,41 @@ erts_decode_dist_ext(Eterm** hpp, return obj; error: + erts_factory_undo(factory); bad_dist_ext(edep); return THE_NON_VALUE; } -Eterm erts_decode_ext(Eterm **hpp, ErlOffHeap *off_heap, byte **ext) +Eterm erts_decode_ext(ErtsHeapFactory* factory, byte **ext, Uint32 flags) { + ErtsDistExternal ede, *edep; Eterm obj; byte *ep = *ext; - if (*ep++ != VERSION_MAGIC) + if (*ep++ != VERSION_MAGIC) { + erts_factory_undo(factory); return THE_NON_VALUE; - ep = dec_term(NULL, hpp, ep, off_heap, &obj, NULL); + } + if (flags) { + ASSERT(flags == ERTS_DIST_EXT_BTT_SAFE); + ede.flags = flags; /* a dummy struct just for the flags */ + edep = &ede; + } else { + edep = NULL; + } + ep = dec_term(edep, factory, ep, &obj, NULL); if (!ep) { -#ifdef DEBUG - bin_write(ERTS_PRINT_STDERR,NULL,*ext,500); -#endif return THE_NON_VALUE; } *ext = ep; return obj; } -Eterm erts_decode_ext_ets(Eterm **hpp, ErlOffHeap *off_heap, byte *ext) +Eterm erts_decode_ext_ets(ErtsHeapFactory* factory, byte *ext) { Eterm obj; - ext = dec_term(NULL, hpp, ext, off_heap, &obj, NULL); + ext = dec_term(NULL, factory, ext, &obj, NULL); ASSERT(ext); return obj; } @@ -986,9 +1003,8 @@ Eterm erts_decode_ext_ets(Eterm **hpp, ErlOffHeap *off_heap, byte *ext) BIF_RETTYPE erts_debug_dist_ext_to_term_2(BIF_ALIST_2) { + ErtsHeapFactory factory; Eterm res; - Eterm *hp; - Eterm *hendp; Sint hsz; ErtsDistExternal ede; Eterm *tp; @@ -1035,12 +1051,9 @@ BIF_RETTYPE erts_debug_dist_ext_to_term_2(BIF_ALIST_2) if (hsz < 0) goto badarg; - hp = HAlloc(BIF_P, (Uint) hsz); - hendp = hp + hsz; - - res = erts_decode_dist_ext(&hp, &MSO(BIF_P), &ede); - - HRelease(BIF_P, hendp, hp); + erts_factory_proc_prealloc_init(&factory, BIF_P, hsz); + res = erts_decode_dist_ext(&factory, &ede); + erts_factory_close(&factory); if (is_value(res)) BIF_RET(res); @@ -1069,6 +1082,8 @@ static BIF_RETTYPE term_to_binary_trap_1(BIF_ALIST_1) } } +HIPE_WRAPPER_BIF_DISABLE_GC(term_to_binary, 1) + BIF_RETTYPE term_to_binary_1(BIF_ALIST_1) { Eterm res = erts_term_to_binary_int(BIF_P, BIF_ARG_1, 0, TERM_TO_BINARY_DFLAGS, NULL); @@ -1081,6 +1096,8 @@ BIF_RETTYPE term_to_binary_1(BIF_ALIST_1) } } +HIPE_WRAPPER_BIF_DISABLE_GC(term_to_binary, 2) + BIF_RETTYPE term_to_binary_2(BIF_ALIST_2) { Process* p = BIF_P; @@ -1164,12 +1181,11 @@ typedef struct { byte* ep; Eterm res; Eterm* next; - Eterm* hp_start; - Eterm* hp; - Eterm* hp_end; + ErtsHeapFactory factory; int remaining_n; char* remaining_bytes; - Eterm* maps_head; + ErtsWStack flat_maps; + ErtsPStack hamt_array; } B2TDecodeContext; typedef struct { @@ -1185,6 +1201,8 @@ typedef struct B2TContext_t { Uint32 flags; SWord reds; Eterm trap_bin; + Export *bif; + Eterm arg[2]; enum B2TState state; union { B2TSizeContext sc; @@ -1291,10 +1309,12 @@ binary2term_abort(ErtsBinary2TermState *state) } static ERTS_INLINE Eterm -binary2term_create(ErtsDistExternal *edep, ErtsBinary2TermState *state, Eterm **hpp, ErlOffHeap *ohp) +binary2term_create(ErtsDistExternal *edep, ErtsBinary2TermState *state, + ErtsHeapFactory* factory) { Eterm res; - if (!dec_term(edep, hpp, state->extp, ohp, &res, NULL)) + + if (!dec_term(edep, factory, state->extp, &res, NULL)) res = THE_NON_VALUE; if (state->exttmp) { state->exttmp = 0; @@ -1327,9 +1347,9 @@ erts_binary2term_abort(ErtsBinary2TermState *state) } Eterm -erts_binary2term_create(ErtsBinary2TermState *state, Eterm **hpp, ErlOffHeap *ohp) +erts_binary2term_create(ErtsBinary2TermState *state, ErtsHeapFactory* factory) { - return binary2term_create(NULL,state, hpp, ohp); + return binary2term_create(NULL,state, factory); } static void b2t_destroy_context(B2TContext* context) @@ -1338,8 +1358,21 @@ static void b2t_destroy_context(B2TContext* context) ERTS_ALC_T_EXT_TERM_DATA); context->aligned_alloc = NULL; binary2term_abort(&context->b2ts); - if (context->state == B2TUncompressChunk) { + switch (context->state) { + case B2TUncompressChunk: erl_zlib_inflate_finish(&context->u.uc.stream); + break; + case B2TDecode: + case B2TDecodeList: + case B2TDecodeTuple: + case B2TDecodeString: + case B2TDecodeBinary: + if (context->u.dc.hamt_array.pstart) { + erts_free(context->u.dc.hamt_array.alloc_type, + context->u.dc.hamt_array.pstart); + } + break; + default:; } } @@ -1356,7 +1389,8 @@ static BIF_RETTYPE binary_to_term_trap_1(BIF_ALIST_1) Binary *context_bin = ((ProcBin *) binary_val(BIF_ARG_1))->val; ASSERT(ERTS_MAGIC_BIN_DESTRUCTOR(context_bin) == b2t_context_destructor); - return binary_to_term_int(BIF_P, 0, THE_NON_VALUE, context_bin); + return binary_to_term_int(BIF_P, 0, THE_NON_VALUE, context_bin, NULL, + THE_NON_VALUE, THE_NON_VALUE); } @@ -1391,8 +1425,10 @@ static B2TContext* b2t_export_context(Process* p, B2TContext* src) return ctx; } -static Eterm binary_to_term_int(Process* p, Uint32 flags, Eterm bin, Binary* context_b) +static BIF_RETTYPE binary_to_term_int(Process* p, Uint32 flags, Eterm bin, Binary* context_b, + Export *bif_init, Eterm arg0, Eterm arg1) { + BIF_RETTYPE ret_val; #ifdef EXTREME_B2T_TRAPPING SWord initial_reds = 1 + b2t_rand() % 4; #else @@ -1409,6 +1445,9 @@ static Eterm binary_to_term_int(Process* p, Uint32 flags, Eterm bin, Binary* con ctx->state = B2TPrepare; ctx->aligned_alloc = NULL; ctx->flags = flags; + ctx->bif = bif_init; + ctx->arg[0] = arg0; + ctx->arg[1] = arg1; IF_DEBUG(ctx->trap_bin = THE_NON_VALUE;) } else { is_first_call = 0; @@ -1484,10 +1523,9 @@ static Eterm binary_to_term_int(Process* p, Uint32 flags, Eterm bin, Binary* con ctx->u.dc.ep = ctx->b2ts.extp; ctx->u.dc.res = (Eterm) (UWord) NULL; ctx->u.dc.next = &ctx->u.dc.res; - ctx->u.dc.hp_start = HAlloc(p, ctx->heap_size); - ctx->u.dc.hp = ctx->u.dc.hp_start; - ctx->u.dc.hp_end = ctx->u.dc.hp_start + ctx->heap_size; - ctx->u.dc.maps_head = NULL; + erts_factory_proc_prealloc_init(&ctx->u.dc.factory, p, ctx->heap_size); + ctx->u.dc.flat_maps.wstart = NULL; + ctx->u.dc.hamt_array.pstart = NULL; ctx->state = B2TDecode; /*fall through*/ case B2TDecode: @@ -1497,34 +1535,46 @@ static Eterm binary_to_term_int(Process* p, Uint32 flags, Eterm bin, Binary* con case B2TDecodeBinary: { ErtsDistExternal fakedep; fakedep.flags = ctx->flags; - dec_term(&fakedep, NULL, NULL, &MSO(p), NULL, ctx); + dec_term(&fakedep, NULL, NULL, NULL, ctx); break; } case B2TDecodeFail: - HRelease(p, ctx->u.dc.hp_end, ctx->u.dc.hp_start); /*fall through*/ case B2TBadArg: - b2t_destroy_context(ctx); - if (!is_first_call) { - erts_set_gc_state(p, 1); - } BUMP_REDS(p, (initial_reds - ctx->reds) / B2T_BYTES_PER_REDUCTION); - BIF_ERROR(p, BADARG & ~EXF_SAVETRACE); + + ASSERT(ctx->bif == bif_export[BIF_binary_to_term_1] + || ctx->bif == bif_export[BIF_binary_to_term_2]); + + if (is_first_call) + ERTS_BIF_PREP_ERROR(ret_val, p, BADARG); + else { + erts_set_gc_state(p, 1); + if (is_non_value(ctx->arg[1])) + ERTS_BIF_PREP_ERROR_TRAPPED1(ret_val, p, BADARG, ctx->bif, + ctx->arg[0]); + else + ERTS_BIF_PREP_ERROR_TRAPPED2(ret_val, p, BADARG, ctx->bif, + ctx->arg[0], ctx->arg[1]); + } + b2t_destroy_context(ctx); + return ret_val; case B2TDone: b2t_destroy_context(ctx); - if (ctx->u.dc.hp > ctx->u.dc.hp_end) { - erl_exit(1, ":%s, line %d: heap overrun by %d words(s)\n", - __FILE__, __LINE__, ctx->u.dc.hp - ctx->u.dc.hp_end); + if (ctx->u.dc.factory.hp > ctx->u.dc.factory.hp_end) { + erts_exit(ERTS_ERROR_EXIT, ":%s, line %d: heap overrun by %d words(s)\n", + __FILE__, __LINE__, ctx->u.dc.factory.hp - ctx->u.dc.factory.hp_end); } - HRelease(p, ctx->u.dc.hp_end, ctx->u.dc.hp); + erts_factory_close(&ctx->u.dc.factory); if (!is_first_call) { erts_set_gc_state(p, 1); } BUMP_REDS(p, (initial_reds - ctx->reds) / B2T_BYTES_PER_REDUCTION); - return ctx->u.dc.res; + ERTS_BIF_PREP_RET(ret_val, ctx->u.dc.res); + return ret_val; default: ASSERT(!"Unknown state in binary_to_term"); @@ -1541,15 +1591,24 @@ static Eterm binary_to_term_int(Process* p, Uint32 flags, Eterm bin, Binary* con erts_set_gc_state(p, 0); } BUMP_ALL_REDS(p); - BIF_TRAP1(&binary_to_term_trap_export, p, ctx->trap_bin); + + ERTS_BIF_PREP_TRAP1(ret_val, &binary_to_term_trap_export, + p, ctx->trap_bin); + + return ret_val; } -BIF_RETTYPE erts_internal_binary_to_term_1(BIF_ALIST_1) +HIPE_WRAPPER_BIF_DISABLE_GC(binary_to_term, 1) + +BIF_RETTYPE binary_to_term_1(BIF_ALIST_1) { - return binary_to_term_int(BIF_P, 0, BIF_ARG_1, NULL); + return binary_to_term_int(BIF_P, 0, BIF_ARG_1, NULL, bif_export[BIF_binary_to_term_1], + BIF_ARG_1, THE_NON_VALUE); } -BIF_RETTYPE erts_internal_binary_to_term_2(BIF_ALIST_2) +HIPE_WRAPPER_BIF_DISABLE_GC(binary_to_term, 2) + +BIF_RETTYPE binary_to_term_2(BIF_ALIST_2) { Eterm opts; Eterm opt; @@ -1570,7 +1629,8 @@ BIF_RETTYPE erts_internal_binary_to_term_2(BIF_ALIST_2) if (is_not_nil(opts)) goto error; - return binary_to_term_int(BIF_P, flags, BIF_ARG_1, NULL); + return binary_to_term_int(BIF_P, flags, BIF_ARG_1, NULL, bif_export[BIF_binary_to_term_2], + BIF_ARG_1, BIF_ARG_2); error: BIF_ERROR(BIF_P, BADARG); @@ -1653,12 +1713,12 @@ erts_term_to_binary_simple(Process* p, Eterm Term, Uint size, int level, Uint fl if ((endp = enc_term(NULL, Term, bytes, flags, NULL)) == NULL) { - erl_exit(1, "%s, line %d: bad term: %x\n", + erts_exit(ERTS_ERROR_EXIT, "%s, line %d: bad term: %x\n", __FILE__, __LINE__, Term); } real_size = endp - bytes; if (real_size > size) { - erl_exit(1, "%s, line %d: buffer overflow: %d word(s)\n", + erts_exit(ERTS_ERROR_EXIT, "%s, line %d: buffer overflow: %d word(s)\n", __FILE__, __LINE__, real_size - size); } @@ -1698,12 +1758,12 @@ erts_term_to_binary_simple(Process* p, Eterm Term, Uint size, int level, Uint fl bytes[0] = VERSION_MAGIC; if ((endp = enc_term(NULL, Term, bytes+1, flags, NULL)) == NULL) { - erl_exit(1, "%s, line %d: bad term: %x\n", + erts_exit(ERTS_ERROR_EXIT, "%s, line %d: bad term: %x\n", __FILE__, __LINE__, Term); } real_size = endp - bytes; if (real_size > size) { - erl_exit(1, "%s, line %d: buffer overflow: %d word(s)\n", + erts_exit(ERTS_ERROR_EXIT, "%s, line %d: buffer overflow: %d word(s)\n", __FILE__, __LINE__, endp - (bytes + size)); } return erts_realloc_binary(bin, real_size); @@ -1717,54 +1777,14 @@ erts_term_to_binary(Process* p, Eterm Term, int level, Uint flags) { return erts_term_to_binary_simple(p, Term, size, level, flags); } -/* Define for testing */ -/* #define EXTREME_TTB_TRAPPING 1 */ +/* Define EXTREME_TTB_TRAPPING for testing in dist.h */ #ifndef EXTREME_TTB_TRAPPING -#define TERM_TO_BINARY_LOOP_FACTOR 32 #define TERM_TO_BINARY_COMPRESS_CHUNK (1 << 18) #else -#define TERM_TO_BINARY_LOOP_FACTOR 1 #define TERM_TO_BINARY_COMPRESS_CHUNK 10 #endif - - -typedef enum { TTBSize, TTBEncode, TTBCompress } TTBState; -typedef struct TTBSizeContext_ { - Uint flags; - int level; - Uint result; - Eterm obj; - ErtsEStack estack; -} TTBSizeContext; - -typedef struct TTBEncodeContext_ { - Uint flags; - int level; - byte* ep; - Eterm obj; - ErtsWStack wstack; - Binary *result_bin; -} TTBEncodeContext; - -typedef struct { - Uint real_size; - Uint dest_len; - byte *dbytes; - Binary *result_bin; - Binary *destination_bin; - z_stream stream; -} TTBCompressContext; - -typedef struct { - int alive; - TTBState state; - union { - TTBSizeContext sc; - TTBEncodeContext ec; - TTBCompressContext cc; - } s; -} TTBContext; +#define TERM_TO_BINARY_MEMCPY_FACTOR 8 static void ttb_context_destructor(Binary *context_bin) { @@ -1773,7 +1793,7 @@ static void ttb_context_destructor(Binary *context_bin) context->alive = 0; switch (context->state) { case TTBSize: - DESTROY_SAVED_ESTACK(&context->s.sc.estack); + DESTROY_SAVED_WSTACK(&context->s.sc.wstack); break; case TTBEncode: DESTROY_SAVED_WSTACK(&context->s.ec.wstack); @@ -1841,7 +1861,7 @@ static Eterm erts_term_to_binary_int(Process* p, Eterm Term, int level, Uint fla /* Setup enough to get started */ context->state = TTBSize; context->alive = 1; - context->s.sc.estack.start = NULL; + context->s.sc.wstack.wstart = NULL; context->s.sc.flags = flags; context->s.sc.level = level; } else { @@ -1876,8 +1896,6 @@ static Eterm erts_term_to_binary_int(Process* p, Eterm Term, int level, Uint fla } result_bin = erts_bin_nrml_alloc(size); - result_bin->flags = 0; - result_bin->orig_size = size; erts_refc_init(&result_bin->refc, 0); result_bin->orig_bytes[0] = VERSION_MAGIC; /* Next state immediately, no need to export context */ @@ -1938,8 +1956,6 @@ static Eterm erts_term_to_binary_int(Process* p, Eterm Term, int level, Uint fla context->s.cc.result_bin = result_bin; result_bin = erts_bin_nrml_alloc(real_size); - result_bin->flags = 0; - result_bin->orig_size = real_size; erts_refc_init(&result_bin->refc, 0); result_bin->orig_bytes[0] = VERSION_MAGIC; @@ -2245,7 +2261,7 @@ static ERTS_INLINE ErlNode* dec_get_node(Eterm sysname, Uint creation) } static byte* -dec_pid(ErtsDistExternal *edep, Eterm** hpp, byte* ep, ErlOffHeap* off_heap, Eterm* objp) +dec_pid(ErtsDistExternal *edep, ErtsHeapFactory* factory, byte* ep, Eterm* objp) { Eterm sysname; Uint data; @@ -2284,15 +2300,15 @@ dec_pid(ErtsDistExternal *edep, Eterm** hpp, byte* ep, ErlOffHeap* off_heap, Ete if(node == erts_this_node) { *objp = make_internal_pid(data); } else { - ExternalThing *etp = (ExternalThing *) *hpp; - *hpp += EXTERNAL_THING_HEAD_SIZE + 1; + ExternalThing *etp = (ExternalThing *) factory->hp; + factory->hp += EXTERNAL_THING_HEAD_SIZE + 1; etp->header = make_external_pid_header(1); - etp->next = off_heap->first; + etp->next = factory->off_heap->first; etp->node = node; etp->data.ui[0] = data; - off_heap->first = (struct erl_off_heap_header*) etp; + factory->off_heap->first = (struct erl_off_heap_header*) etp; *objp = make_external_pid(etp); } return ep; @@ -2302,8 +2318,10 @@ dec_pid(ErtsDistExternal *edep, Eterm** hpp, byte* ep, ErlOffHeap* off_heap, Ete #define ENC_TERM ((Eterm) 0) #define ENC_ONE_CONS ((Eterm) 1) #define ENC_PATCH_FUN_SIZE ((Eterm) 2) -#define ENC_LAST_ARRAY_ELEMENT ((Eterm) 3) - +#define ENC_BIN_COPY ((Eterm) 3) +#define ENC_MAP_PAIR ((Eterm) 4) +#define ENC_HASHMAP_NODE ((Eterm) 5) +#define ENC_LAST_ARRAY_ELEMENT ((Eterm) 6) static byte* enc_term(ErtsAtomCacheMap *acmp, Eterm obj, byte* ep, Uint32 dflags, @@ -2326,10 +2344,6 @@ enc_term_int(TTBEncodeContext* ctx, ErtsAtomCacheMap *acmp, Eterm obj, byte* ep, Eterm val; FloatDef f; Sint r = 0; -#if HALFWORD_HEAP - UWord wobj; -#endif - if (ctx) { WSTACK_CHANGE_ALLOCATOR(s, ERTS_ALC_T_SAVED_ESTACK); @@ -2339,6 +2353,9 @@ enc_term_int(TTBEncodeContext* ctx, ErtsAtomCacheMap *acmp, Eterm obj, byte* ep, WSTACK_RESTORE(s, &ctx->wstack); ep = ctx->ep; obj = ctx->obj; + if (is_non_value(obj)) { + goto outer_loop; + } } } @@ -2346,11 +2363,8 @@ enc_term_int(TTBEncodeContext* ctx, ErtsAtomCacheMap *acmp, Eterm obj, byte* ep, outer_loop: while (!WSTACK_ISEMPTY(s)) { -#if HALFWORD_HEAP - obj = (Eterm) (wobj = WSTACK_POP(s)); -#else obj = WSTACK_POP(s); -#endif + switch (val = WSTACK_POP(s)) { case ENC_TERM: break; @@ -2362,49 +2376,76 @@ enc_term_int(TTBEncodeContext* ctx, ErtsAtomCacheMap *acmp, Eterm obj, byte* ep, obj = CAR(cons); tl = CDR(cons); - WSTACK_PUSH(s, is_list(tl) ? ENC_ONE_CONS : ENC_TERM); - WSTACK_PUSH(s, tl); + WSTACK_PUSH2(s, (is_list(tl) ? ENC_ONE_CONS : ENC_TERM), + tl); } break; case ENC_PATCH_FUN_SIZE: { -#if HALFWORD_HEAP - byte* size_p = (byte *) wobj; -#else byte* size_p = (byte *) obj; -#endif put_int32(ep - size_p, size_p); } goto outer_loop; + case ENC_BIN_COPY: { + Uint bits = (Uint)obj; + Uint bitoffs = WSTACK_POP(s); + byte* bytes = (byte*) WSTACK_POP(s); + byte* dst = (byte*) WSTACK_POP(s); + if (bits > r * (TERM_TO_BINARY_MEMCPY_FACTOR * 8)) { + Uint n = r * TERM_TO_BINARY_MEMCPY_FACTOR; + WSTACK_PUSH5(s, (UWord)(dst + n), (UWord)(bytes + n), bitoffs, + ENC_BIN_COPY, bits - 8*n); + bits = 8*n; + copy_binary_to_buffer(dst, 0, bytes, bitoffs, bits); + obj = THE_NON_VALUE; + r = 0; /* yield */ + break; + } else { + copy_binary_to_buffer(dst, 0, bytes, bitoffs, bits); + r -= bits / (TERM_TO_BINARY_MEMCPY_FACTOR * 8); + goto outer_loop; + } + } + case ENC_MAP_PAIR: { + Uint pairs_left = obj; + Eterm *vptr = (Eterm*) WSTACK_POP(s); + Eterm *kptr = (Eterm*) WSTACK_POP(s); + + obj = *kptr; + if (--pairs_left > 0) { + WSTACK_PUSH4(s, (UWord)(kptr+1), (UWord)(vptr+1), + ENC_MAP_PAIR, pairs_left); + } + WSTACK_PUSH2(s, ENC_TERM, *vptr); + break; + } + case ENC_HASHMAP_NODE: + if (is_list(obj)) { /* leaf node [K|V] */ + ptr = list_val(obj); + WSTACK_PUSH2(s, ENC_TERM, CDR(ptr)); + obj = CAR(ptr); + } + break; case ENC_LAST_ARRAY_ELEMENT: /* obj is the tuple */ { -#if HALFWORD_HEAP - Eterm* ptr = (Eterm *) wobj; -#else Eterm* ptr = (Eterm *) obj; -#endif obj = *ptr; } break; default: /* ENC_LAST_ARRAY_ELEMENT+1 and upwards */ { -#if HALFWORD_HEAP - Eterm* ptr = (Eterm *) wobj; -#else Eterm* ptr = (Eterm *) obj; -#endif - WSTACK_PUSH(s, val-1); obj = *ptr++; - WSTACK_PUSH(s, (UWord)ptr); + WSTACK_PUSH2(s, val-1, (UWord)ptr); } break; } L_jump_start: - if (ctx && --r == 0) { - *reds = r; + if (ctx && --r <= 0) { + *reds = 0; ctx->obj = obj; ctx->ep = ep; WSTACK_SAVE(s, &ctx->wstack); @@ -2553,39 +2594,59 @@ enc_term_int(TTBEncodeContext* ctx, ErtsAtomCacheMap *acmp, Eterm obj, byte* ep, ep += 4; } if (i > 0) { - WSTACK_PUSH(s, ENC_LAST_ARRAY_ELEMENT+i-1); - WSTACK_PUSH(s, (UWord)ptr); + WSTACK_PUSH2(s, ENC_LAST_ARRAY_ELEMENT+i-1, (UWord)ptr); } break; case MAP_DEF: - { - map_t *mp = (map_t*)map_val(obj); - Uint size = map_get_size(mp); + if (is_flatmap(obj)) { + flatmap_t *mp = (flatmap_t*)flatmap_val(obj); + Uint size = flatmap_get_size(mp); *ep++ = MAP_EXT; put_int32(size, ep); ep += 4; if (size > 0) { - Eterm *kptr = map_get_keys(mp); - Eterm *vptr = map_get_values(mp); - - for (i = size-1; i >= 1; i--) { - WSTACK_PUSH(s, ENC_TERM); - WSTACK_PUSH(s, (UWord) vptr[i]); - WSTACK_PUSH(s, ENC_TERM); - WSTACK_PUSH(s, (UWord) kptr[i]); - } + Eterm *kptr = flatmap_get_keys(mp); + Eterm *vptr = flatmap_get_values(mp); - WSTACK_PUSH(s, ENC_TERM); - WSTACK_PUSH(s, (UWord) vptr[0]); + WSTACK_PUSH4(s, (UWord)kptr, (UWord)vptr, + ENC_MAP_PAIR, size); + } + } else { + Eterm hdr; + Uint node_sz; + ptr = boxed_val(obj); + hdr = *ptr; + ASSERT(is_header(hdr)); + switch(hdr & _HEADER_MAP_SUBTAG_MASK) { + case HAMT_SUBTAG_HEAD_ARRAY: + *ep++ = MAP_EXT; + ptr++; + put_int32(*ptr, ep); ep += 4; + node_sz = 16; + break; + case HAMT_SUBTAG_HEAD_BITMAP: + *ep++ = MAP_EXT; + ptr++; + put_int32(*ptr, ep); ep += 4; + /*fall through*/ + case HAMT_SUBTAG_NODE_BITMAP: + node_sz = hashmap_bitcount(MAP_HEADER_VAL(hdr)); + ASSERT(node_sz < 17); + break; + default: + erts_exit(ERTS_ERROR_EXIT, "bad header\r\n"); + } - obj = kptr[0]; - goto L_jump_start; + ptr++; + WSTACK_RESERVE(s, node_sz*2); + while(node_sz--) { + WSTACK_FAST_PUSH(s, ENC_HASHMAP_NODE); + WSTACK_FAST_PUSH(s, *ptr++); } } break; - case FLOAT_DEF: GET_DOUBLE(obj, f); if (dflags & DFLAG_NEW_FLOATS) { @@ -2619,6 +2680,7 @@ enc_term_int(TTBEncodeContext* ctx, ErtsAtomCacheMap *acmp, Eterm obj, byte* ep, Uint bitoffs; Uint bitsize; byte* bytes; + byte* data_dst; ERTS_GET_BINARY_BYTES(obj, bytes, bitoffs, bitsize); if (dflags & DFLAG_INTERNAL_TAGS) { @@ -2664,7 +2726,7 @@ enc_term_int(TTBEncodeContext* ctx, ErtsAtomCacheMap *acmp, Eterm obj, byte* ep, j = binary_size(obj); put_int32(j, ep); ep += 4; - copy_binary_to_buffer(ep, 0, bytes, bitoffs, 8*j); + data_dst = ep; ep += j; } else if (dflags & DFLAG_BIT_BINARIES) { /* Bit-level binary. */ @@ -2674,7 +2736,7 @@ enc_term_int(TTBEncodeContext* ctx, ErtsAtomCacheMap *acmp, Eterm obj, byte* ep, ep += 4; *ep++ = bitsize; ep[j] = 0; /* Zero unused bits at end of binary */ - copy_binary_to_buffer(ep, 0, bytes, bitoffs, 8*j+bitsize); + data_dst = ep; ep += j + 1; } else { /* @@ -2688,11 +2750,18 @@ enc_term_int(TTBEncodeContext* ctx, ErtsAtomCacheMap *acmp, Eterm obj, byte* ep, put_int32((j+1), ep); ep += 4; ep[j] = 0; /* Zero unused bits at end of binary */ - copy_binary_to_buffer(ep, 0, bytes, bitoffs, 8*j+bitsize); + data_dst = ep; ep += j+1; *ep++ = SMALL_INTEGER_EXT; *ep++ = bitsize; } + if (ctx && j > r * TERM_TO_BINARY_MEMCPY_FACTOR) { + WSTACK_PUSH5(s, (UWord)data_dst, (UWord)bytes, bitoffs, + ENC_BIN_COPY, 8*j + bitsize); + } else { + copy_binary_to_buffer(data_dst, 0, bytes, bitoffs, + 8 * j + bitsize); + } } break; case EXPORT_DEF: @@ -2721,13 +2790,12 @@ enc_term_int(TTBEncodeContext* ctx, ErtsAtomCacheMap *acmp, Eterm obj, byte* ep, case FUN_DEF: { ErlFunThing* funp = (ErlFunThing *) fun_val(obj); + int ei; if ((dflags & DFLAG_NEW_FUN_TAGS) != 0) { - int ei; - *ep++ = NEW_FUN_EXT; - WSTACK_PUSH(s, ENC_PATCH_FUN_SIZE); - WSTACK_PUSH(s, (UWord) ep); /* Position for patching in size */ + WSTACK_PUSH2(s, ENC_PATCH_FUN_SIZE, + (UWord) ep); /* Position for patching in size */ ep += 4; *ep = funp->arity; ep += 1; @@ -2741,16 +2809,6 @@ enc_term_int(TTBEncodeContext* ctx, ErtsAtomCacheMap *acmp, Eterm obj, byte* ep, ep = enc_term(acmp, make_small(funp->fe->old_index), ep, dflags, off_heap); ep = enc_term(acmp, make_small(funp->fe->old_uniq), ep, dflags, off_heap); ep = enc_pid(acmp, funp->creator, ep, dflags); - - fun_env: - for (ei = funp->num_free-1; ei > 0; ei--) { - WSTACK_PUSH(s, ENC_TERM); - WSTACK_PUSH(s, (UWord) funp->env[ei]); - } - if (funp->num_free != 0) { - obj = funp->env[0]; - goto L_jump_start; - } } else { /* * Communicating with an obsolete erl_interface or @@ -2782,7 +2840,13 @@ enc_term_int(TTBEncodeContext* ctx, ErtsAtomCacheMap *acmp, Eterm obj, byte* ep, *ep++ = SMALL_TUPLE_EXT; put_int8(funp->num_free, ep); ep += 1; - goto fun_env; + } + for (ei = funp->num_free-1; ei > 0; ei--) { + WSTACK_PUSH2(s, ENC_TERM, (UWord) funp->env[ei]); + } + if (funp->num_free != 0) { + obj = funp->env[0]; + goto L_jump_start; } } break; @@ -2836,57 +2900,42 @@ is_external_string(Eterm list, int* p_is_string) return len; } -/* Assumes that the ones to undo are preluding the list. */ -static void -undo_offheap_in_area(ErlOffHeap* off_heap, Eterm* start, Eterm* end) -{ - const Uint area_sz = (end - start) * sizeof(Eterm); - struct erl_off_heap_header* hdr; - struct erl_off_heap_header** hdr_nextp = NULL; - - for (hdr = off_heap->first; ; hdr=hdr->next) { - if (!in_area(hdr, start, area_sz)) { - if (hdr_nextp != NULL) { - *hdr_nextp = NULL; - erts_cleanup_offheap(off_heap); - off_heap->first = hdr; - } - break; - } - hdr_nextp = &hdr->next; - } - /* Assert that the ones to undo were indeed preluding the list. */ -#ifdef DEBUG - for (hdr = off_heap->first; hdr != NULL; hdr = hdr->next) { - ASSERT(!in_area(hdr, start, area_sz)); - } -#endif /* DEBUG */ -} +struct dec_term_hamt +{ + Eterm* objp; /* write result here */ + Uint size; /* nr of leafs */ + Eterm* leaf_array; +}; /* Decode term from external format into *objp. -** On failure return NULL and (R13B04) *hpp will be unchanged. +** On failure calls erts_factory_undo() and returns NULL */ static byte* -dec_term(ErtsDistExternal *edep, Eterm** hpp, byte* ep, ErlOffHeap* off_heap, - Eterm* objp, B2TContext* ctx) +dec_term(ErtsDistExternal *edep, + ErtsHeapFactory* factory, + byte* ep, + Eterm* objp, + B2TContext* ctx) { - Eterm* hp_saved; +#define PSTACK_TYPE struct dec_term_hamt + PSTACK_DECLARE(hamt_array, 5); int n; ErtsAtomEncoding char_enc; register Eterm* hp; /* Please don't take the address of hp */ - Eterm *maps_head; /* for validation of maps */ + DECLARE_WSTACK(flat_maps); /* for preprocessing of small maps */ Eterm* next; SWord reds; +#ifdef DEBUG + Eterm* dbg_resultp = ctx ? &ctx->u.dc.res : objp; +#endif if (ctx) { - hp_saved = ctx->u.dc.hp_start; reds = ctx->reds; next = ctx->u.dc.next; ep = ctx->u.dc.ep; - hpp = &ctx->u.dc.hp; - maps_head = ctx->u.dc.maps_head; + factory = &ctx->u.dc.factory; if (ctx->state != B2TDecode) { int n_limit = reds; @@ -2913,7 +2962,7 @@ dec_term(ErtsDistExternal *edep, Eterm** hpp, byte* ep, ErlOffHeap* off_heap, case B2TDecodeList: objp = next - 2; while (n > 0) { - objp[0] = (Eterm) COMPRESS_POINTER(next); + objp[0] = (Eterm) next; objp[1] = make_list(next); next = objp; objp -= 2; @@ -2924,14 +2973,14 @@ dec_term(ErtsDistExternal *edep, Eterm** hpp, byte* ep, ErlOffHeap* off_heap, case B2TDecodeTuple: objp = next - 1; while (n-- > 0) { - objp[0] = (Eterm) COMPRESS_POINTER(next); + objp[0] = (Eterm) next; next = objp; objp--; } break; case B2TDecodeString: - hp = *hpp; + hp = factory->hp; hp[-1] = make_list(hp); /* overwrite the premature NIL */ while (n-- > 0) { hp[0] = make_small(*ep++); @@ -2939,7 +2988,7 @@ dec_term(ErtsDistExternal *edep, Eterm** hpp, byte* ep, ErlOffHeap* off_heap, hp += 2; } hp[-1] = NIL; - *hpp = hp; + factory->hp = hp; break; case B2TDecodeBinary: @@ -2961,20 +3010,26 @@ dec_term(ErtsDistExternal *edep, Eterm** hpp, byte* ep, ErlOffHeap* off_heap, return NULL; } } + PSTACK_CHANGE_ALLOCATOR(hamt_array, ERTS_ALC_T_SAVED_ESTACK); + WSTACK_CHANGE_ALLOCATOR(flat_maps, ERTS_ALC_T_SAVED_ESTACK); + if (ctx->u.dc.hamt_array.pstart) { + PSTACK_RESTORE(hamt_array, &ctx->u.dc.hamt_array); + } + if (ctx->u.dc.flat_maps.wstart) { + WSTACK_RESTORE(flat_maps, &ctx->u.dc.flat_maps); + } } else { - hp_saved = *hpp; reds = ERTS_SWORD_MAX; next = objp; *next = (Eterm) (UWord) NULL; - maps_head = NULL; } - hp = *hpp; + hp = factory->hp; while (next != NULL) { objp = next; - next = (Eterm *) EXPAND_POINTER(*objp); + next = (Eterm *) *objp; switch (*ep++) { case INTEGER_EXT: @@ -2982,7 +3037,7 @@ dec_term(ErtsDistExternal *edep, Eterm** hpp, byte* ep, ErlOffHeap* off_heap, Sint sn = get_int32(ep); ep += 4; -#if defined(ARCH_64) && !HALFWORD_HEAP +#if defined(ARCH_64) *objp = make_small(sn); #else if (MY_IS_SSMALL(sn)) { @@ -3031,6 +3086,8 @@ dec_term(ErtsDistExternal *edep, Eterm** hpp, byte* ep, ErlOffHeap* off_heap, big = make_small(0); } else { big = bytes_to_big(first, n, neg, hp); + if (is_nil(big)) + goto error; if (is_big(big)) { hp += big_arity(big) + 1; } @@ -3103,7 +3160,7 @@ dec_term_atom_common: reds -= n; } while (n-- > 0) { - objp[0] = (Eterm) COMPRESS_POINTER(next); + objp[0] = (Eterm) next; next = objp; objp--; } @@ -3121,8 +3178,8 @@ dec_term_atom_common: *objp = make_list(hp); hp += 2 * n; objp = hp - 2; - objp[0] = (Eterm) COMPRESS_POINTER((objp+1)); - objp[1] = (Eterm) COMPRESS_POINTER(next); + objp[0] = (Eterm) (objp+1); + objp[1] = (Eterm) next; next = objp; objp -= 2; n--; @@ -3135,7 +3192,7 @@ dec_term_atom_common: reds -= n; } while (n > 0) { - objp[0] = (Eterm) COMPRESS_POINTER(next); + objp[0] = (Eterm) next; objp[1] = make_list(next); next = objp; objp -= 2; @@ -3204,9 +3261,9 @@ dec_term_atom_common: break; } case PID_EXT: - *hpp = hp; - ep = dec_pid(edep, hpp, ep, off_heap, objp); - hp = *hpp; + factory->hp = hp; + ep = dec_pid(edep, factory, ep, objp); + hp = factory->hp; if (ep == NULL) { goto error; } @@ -3240,11 +3297,11 @@ dec_term_atom_common: hp += EXTERNAL_THING_HEAD_SIZE + 1; etp->header = make_external_port_header(1); - etp->next = off_heap->first; + etp->next = factory->off_heap->first; etp->node = node; etp->data.ui[0] = num; - off_heap->first = (struct erl_off_heap_header*)etp; + factory->off_heap->first = (struct erl_off_heap_header*)etp; *objp = make_external_port(etp); } @@ -3302,7 +3359,7 @@ dec_term_atom_common: RefThing *rtp = (RefThing *) hp; ref_num = (Uint32 *) (hp + REF_THING_HEAD_SIZE); -#if defined(ARCH_64) && !HALFWORD_HEAP +#if defined(ARCH_64) hp += REF_THING_HEAD_SIZE + ref_words/2 + 1; rtp->header = make_ref_thing_header(ref_words/2 + 1); #else @@ -3313,26 +3370,26 @@ dec_term_atom_common: } else { ExternalThing *etp = (ExternalThing *) hp; -#if defined(ARCH_64) && !HALFWORD_HEAP +#if defined(ARCH_64) hp += EXTERNAL_THING_HEAD_SIZE + ref_words/2 + 1; #else hp += EXTERNAL_THING_HEAD_SIZE + ref_words; #endif -#if defined(ARCH_64) && !HALFWORD_HEAP +#if defined(ARCH_64) etp->header = make_external_ref_header(ref_words/2 + 1); #else etp->header = make_external_ref_header(ref_words); #endif - etp->next = off_heap->first; + etp->next = factory->off_heap->first; etp->node = node; - off_heap->first = (struct erl_off_heap_header*)etp; + factory->off_heap->first = (struct erl_off_heap_header*)etp; *objp = make_external_ref(etp); ref_num = &(etp->data.ui32[0]); } -#if defined(ARCH_64) && !HALFWORD_HEAP +#if defined(ARCH_64) *(ref_num++) = ref_words /* 32-bit arity */; #endif ref_num[0] = r0; @@ -3340,7 +3397,7 @@ dec_term_atom_common: ref_num[i] = get_int32(ep); ep += 4; } -#if defined(ARCH_64) && !HALFWORD_HEAP +#if defined(ARCH_64) if ((1 + ref_words) % 2) ref_num[ref_words] = 0; #endif @@ -3362,15 +3419,14 @@ dec_term_atom_common: } else { Binary* dbin = erts_bin_nrml_alloc(n); ProcBin* pb; - dbin->flags = 0; - dbin->orig_size = n; erts_refc_init(&dbin->refc, 1); pb = (ProcBin *) hp; hp += PROC_BIN_SIZE; pb->thing_word = HEADER_PROC_BIN; pb->size = n; - pb->next = off_heap->first; - off_heap->first = (struct erl_off_heap_header*)pb; + pb->next = factory->off_heap->first; + factory->off_heap->first = (struct erl_off_heap_header*)pb; + OH_OVERHEAD(factory->off_heap, pb->size / sizeof(Eterm)); pb->val = dbin; pb->bytes = (byte*) dbin->orig_bytes; pb->flags = 0; @@ -3416,14 +3472,13 @@ dec_term_atom_common: Binary* dbin = erts_bin_nrml_alloc(n); ProcBin* pb; - dbin->flags = 0; - dbin->orig_size = n; erts_refc_init(&dbin->refc, 1); pb = (ProcBin *) hp; pb->thing_word = HEADER_PROC_BIN; pb->size = n; - pb->next = off_heap->first; - off_heap->first = (struct erl_off_heap_header*)pb; + pb->next = factory->off_heap->first; + factory->off_heap->first = (struct erl_off_heap_header*)pb; + OH_OVERHEAD(factory->off_heap, pb->size / sizeof(Eterm)); pb->val = dbin; pb->bytes = (byte*) dbin->orig_bytes; pb->flags = 0; @@ -3475,9 +3530,9 @@ dec_term_atom_common: if ((ep = dec_atom(edep, ep, &name)) == NULL) { goto error; } - *hpp = hp; - ep = dec_term(edep, hpp, ep, off_heap, &temp, NULL); - hp = *hpp; + factory->hp = hp; + ep = dec_term(edep, factory, ep, &temp, NULL); + hp = factory->hp; if (ep == NULL) { goto error; } @@ -3494,57 +3549,63 @@ dec_term_atom_common: } *objp = make_export(hp); *hp++ = HEADER_EXPORT; -#if HALFWORD_HEAP - *((UWord *) (UWord) hp) = (UWord) erts_export_get_or_make_stub(mod, name, arity); - hp += 2; -#else *hp++ = (Eterm) erts_export_get_or_make_stub(mod, name, arity); -#endif break; } break; case MAP_EXT: { - map_t *mp; Uint32 size,n; Eterm *kptr,*vptr; Eterm keys; size = get_int32(ep); ep += 4; - keys = make_tuple(hp); - *hp++ = make_arityval(size); - hp += size; - kptr = hp - 1; - - mp = (map_t*)hp; - hp += MAP_HEADER_SIZE; - hp += size; - vptr = hp - 1; - - /* kptr, last word for keys - * vptr, last word for values - */ - - /* - * Use thing_word to link through decoded maps. - * The list of maps is for later validation. - */ - - mp->thing_word = (Eterm) COMPRESS_POINTER(maps_head); - maps_head = (Eterm *) mp; - - mp->size = size; - mp->keys = keys; - *objp = make_map(mp); - - for (n = size; n; n--) { - *vptr = (Eterm) COMPRESS_POINTER(next); - *kptr = (Eterm) COMPRESS_POINTER(vptr); - next = kptr; - vptr--; - kptr--; - } + if (size <= MAP_SMALL_MAP_LIMIT) { + flatmap_t *mp; + + keys = make_tuple(hp); + *hp++ = make_arityval(size); + hp += size; + kptr = hp - 1; + + mp = (flatmap_t*)hp; + hp += MAP_HEADER_FLATMAP_SZ; + hp += size; + vptr = hp - 1; + + /* kptr, last word for keys + * vptr, last word for values + */ + + WSTACK_PUSH(flat_maps, (UWord)mp); + mp->thing_word = MAP_HEADER_FLATMAP; + mp->size = size; + mp->keys = keys; + *objp = make_flatmap(mp); + + for (n = size; n; n--) { + *vptr = (Eterm) next; + *kptr = (Eterm) vptr; + next = kptr; + vptr--; + kptr--; + } + } + else { /* Make hamt */ + struct dec_term_hamt* hamt = PSTACK_PUSH(hamt_array); + + hamt->objp = objp; + hamt->size = size; + hamt->leaf_array = hp; + + for (n = size; n; n--) { + CDR(hp) = (Eterm) next; + CAR(hp) = (Eterm) &CDR(hp); + next = &CAR(hp); + hp += 2; + } + } } break; case NEW_FUN_EXT: @@ -3578,9 +3639,9 @@ dec_term_atom_common: if ((ep = dec_atom(edep, ep, &module)) == NULL) { goto error; } - *hpp = hp; + factory->hp = hp; /* Index */ - if ((ep = dec_term(edep, hpp, ep, off_heap, &temp, NULL)) == NULL) { + if ((ep = dec_term(edep, factory, ep, &temp, NULL)) == NULL) { goto error; } if (!is_small(temp)) { @@ -3589,7 +3650,7 @@ dec_term_atom_common: old_index = unsigned_val(temp); /* Uniq */ - if ((ep = dec_term(edep, hpp, ep, off_heap, &temp, NULL)) == NULL) { + if ((ep = dec_term(edep, factory, ep, &temp, NULL)) == NULL) { goto error; } if (!is_small(temp)) { @@ -3601,8 +3662,8 @@ dec_term_atom_common: * It is safe to link the fun into the fun list only when * no more validity tests can fail. */ - funp->next = off_heap->first; - off_heap->first = (struct erl_off_heap_header*)funp; + funp->next = factory->off_heap->first; + factory->off_heap->first = (struct erl_off_heap_header*)funp; funp->fe = erts_put_fun_entry2(module, old_uniq, old_index, uniq, index, arity); @@ -3613,15 +3674,15 @@ dec_term_atom_common: } funp->native_address = funp->fe->native_address; #endif - hp = *hpp; + hp = factory->hp; /* Environment */ for (i = num_free-1; i >= 0; i--) { - funp->env[i] = (Eterm) COMPRESS_POINTER(next); + funp->env[i] = (Eterm) next; next = funp->env + i; } /* Creator */ - funp->creator = (Eterm) COMPRESS_POINTER(next); + funp->creator = (Eterm) next; next = &(funp->creator); break; } @@ -3639,14 +3700,14 @@ dec_term_atom_common: ep += 4; hp += ERL_FUN_SIZE; hp += num_free; - *hpp = hp; + factory->hp = hp; funp->thing_word = HEADER_FUN; funp->num_free = num_free; *objp = make_fun(funp); /* Creator pid */ if (*ep != PID_EXT - || (ep = dec_pid(edep, hpp, ++ep, off_heap, + || (ep = dec_pid(edep, factory, ++ep, &funp->creator))==NULL) { goto error; } @@ -3657,7 +3718,7 @@ dec_term_atom_common: } /* Index */ - if ((ep = dec_term(edep, hpp, ep, off_heap, &temp, NULL)) == NULL) { + if ((ep = dec_term(edep, factory, ep, &temp, NULL)) == NULL) { goto error; } if (!is_small(temp)) { @@ -3666,7 +3727,7 @@ dec_term_atom_common: old_index = unsigned_val(temp); /* Uniq */ - if ((ep = dec_term(edep, hpp, ep, off_heap, &temp, NULL)) == NULL) { + if ((ep = dec_term(edep, factory, ep, &temp, NULL)) == NULL) { goto error; } if (!is_small(temp)) { @@ -3677,8 +3738,8 @@ dec_term_atom_common: * It is safe to link the fun into the fun list only when * no more validity tests can fail. */ - funp->next = off_heap->first; - off_heap->first = (struct erl_off_heap_header*)funp; + funp->next = factory->off_heap->first; + factory->off_heap->first = (struct erl_off_heap_header*)funp; old_uniq = unsigned_val(temp); funp->fe = erts_put_fun_entry(module, old_uniq, old_index); @@ -3686,11 +3747,11 @@ dec_term_atom_common: #ifdef HIPE funp->native_address = funp->fe->native_address; #endif - hp = *hpp; + hp = factory->hp; /* Environment */ for (i = num_free-1; i >= 0; i--) { - funp->env[i] = (Eterm) COMPRESS_POINTER(next); + funp->env[i] = (Eterm) next; next = funp->env + i; } break; @@ -3720,8 +3781,9 @@ dec_term_atom_common: erts_refc_inc(&pb->val->refc, 1); hp += PROC_BIN_SIZE; - pb->next = off_heap->first; - off_heap->first = (struct erl_off_heap_header*)pb; + pb->next = factory->off_heap->first; + factory->off_heap->first = (struct erl_off_heap_header*)pb; + OH_OVERHEAD(factory->off_heap, pb->size / sizeof(Eterm)); pb->flags = 0; *objp = make_binary(pb); break; @@ -3737,8 +3799,9 @@ dec_term_atom_common: erts_refc_inc(&pb->val->refc, 1); hp += PROC_BIN_SIZE; - pb->next = off_heap->first; - off_heap->first = (struct erl_off_heap_header*)pb; + pb->next = factory->off_heap->first; + factory->off_heap->first = (struct erl_off_heap_header*)pb; + OH_OVERHEAD(factory->off_heap, pb->size / sizeof(Eterm)); pb->flags = 0; sub = (ErlSubBin*)hp; @@ -3764,8 +3827,13 @@ dec_term_atom_common: if (next || ctx->state != B2TDecode) { ctx->u.dc.ep = ep; ctx->u.dc.next = next; - ctx->u.dc.hp = hp; - ctx->u.dc.maps_head = maps_head; + ctx->u.dc.factory.hp = hp; + if (!WSTACK_ISEMPTY(flat_maps)) { + WSTACK_SAVE(flat_maps, &ctx->u.dc.flat_maps); + } + if (!PSTACK_IS_EMPTY(hamt_array)) { + PSTACK_SAVE(hamt_array, &ctx->u.dc.hamt_array); + } ctx->reds = 0; return NULL; } @@ -3776,24 +3844,47 @@ dec_term_atom_common: } } - /* Iterate through all the maps and check for validity and sort keys + ASSERT(hp <= factory->hp_end + || (factory->mode == FACTORY_CLOSED && is_immed(*dbg_resultp))); + factory->hp = hp; + /* + * From here on factory may produce (more) heap fragments + */ + + if (!PSTACK_IS_EMPTY(hamt_array)) { + do { + struct dec_term_hamt* hamt = PSTACK_TOP(hamt_array); + + *hamt->objp = erts_hashmap_from_array(factory, + hamt->leaf_array, + hamt->size, + 1); + if (is_non_value(*hamt->objp)) + goto error_hamt; + + (void) PSTACK_POP(hamt_array); + } while (!PSTACK_IS_EMPTY(hamt_array)); + PSTACK_DESTROY(hamt_array); + } + + /* Iterate through all the (flat)maps and check for validity and sort keys * - done here for when we know it is complete. */ - while (maps_head) { - next = (Eterm *)(EXPAND_POINTER(*maps_head)); - *maps_head = MAP_HEADER; - if (!erts_validate_and_sort_map((map_t*)maps_head)) - goto error; - maps_head = next; + while(!WSTACK_ISEMPTY(flat_maps)) { + next = (Eterm *)WSTACK_POP(flat_maps); + if (!erts_validate_and_sort_flatmap((flatmap_t*)next)) + goto error; } + WSTACK_DESTROY(flat_maps); + + ASSERT((Eterm*)*dbg_resultp != NULL); if (ctx) { ctx->state = B2TDone; ctx->reds = reds; } - *hpp = hp; return ep; error: @@ -3801,15 +3892,17 @@ error: * Must unlink all off-heap objects that may have been * linked into the process. */ - if (hp < *hpp) { /* Sometimes we used hp and sometimes *hpp */ - hp = *hpp; /* the largest must be the freshest */ + if (factory->hp < hp) { /* Sometimes we used hp and sometimes factory->hp */ + factory->hp = hp; /* the largest must be the freshest */ } - undo_offheap_in_area(off_heap, hp_saved, hp); - *hpp = hp_saved; +error_hamt: + erts_factory_undo(factory); + PSTACK_DESTROY(hamt_array); if (ctx) { ctx->state = B2TDecodeFail; ctx->reds = reds; } + WSTACK_DESTROY(flat_maps); return NULL; } @@ -3828,51 +3921,35 @@ static int encode_size_struct_int(TTBSizeContext* ctx, ErtsAtomCacheMap *acmp, Eterm obj, unsigned dflags, Sint *reds, Uint *res) { - DECLARE_ESTACK(s); + DECLARE_WSTACK(s); Uint m, i, arity; Uint result = 0; Sint r = 0; if (ctx) { - ESTACK_CHANGE_ALLOCATOR(s, ERTS_ALC_T_SAVED_ESTACK); + WSTACK_CHANGE_ALLOCATOR(s, ERTS_ALC_T_SAVED_ESTACK); r = *reds; - if (ctx->estack.start) { /* restore saved stack */ - ESTACK_RESTORE(s, &ctx->estack); + if (ctx->wstack.wstart) { /* restore saved stack */ + WSTACK_RESTORE(s, &ctx->wstack); result = ctx->result; obj = ctx->obj; } } - goto L_jump_start; +#define LIST_TAIL_OP ((0 << _TAG_PRIMARY_SIZE) | TAG_PRIMARY_HEADER) +#define TERM_ARRAY_OP(N) (((N) << _TAG_PRIMARY_SIZE) | TAG_PRIMARY_HEADER) +#define TERM_ARRAY_OP_DEC(OP) ((OP) - (1 << _TAG_PRIMARY_SIZE)) + + + for (;;) { + ASSERT(!is_header(obj)); - outer_loop: - while (!ESTACK_ISEMPTY(s)) { - obj = ESTACK_POP(s); - handle_popped_obj: - if (is_list(obj)) { - Eterm* cons = list_val(obj); - Eterm tl; - - tl = CDR(cons); - obj = CAR(cons); - ESTACK_PUSH(s, tl); - } else if (is_nil(obj)) { - result++; - goto outer_loop; - } else { - /* - * Other term (in the tail of a non-proper list or - * in a fun's environment). - */ - } - - L_jump_start: if (ctx && --r == 0) { *reds = r; ctx->obj = obj; ctx->result = result; - ESTACK_SAVE(s, &ctx->estack); + WSTACK_SAVE(s, &ctx->wstack); return -1; } switch (tag_val_def(obj)) { @@ -3955,70 +4032,79 @@ encode_size_struct_int(TTBSizeContext* ctx, ErtsAtomCacheMap *acmp, Eterm obj, result += m + 2 + 1; } else { result += 5; - goto handle_popped_obj; + WSTACK_PUSH2(s, (UWord)CDR(list_val(obj)), (UWord)LIST_TAIL_OP); + obj = CAR(list_val(obj)); + continue; /* big loop */ } break; case TUPLE_DEF: { Eterm* ptr = tuple_val(obj); - Uint i; arity = arityval(*ptr); if (arity <= 0xff) { result += 1 + 1; } else { result += 1 + 4; } - for (i = 1; i <= arity; ++i) { - if (is_list(ptr[i])) { - if ((m = is_string(obj)) && (m < MAX_STRING_LEN)) { - result += m + 2 + 1; - } else { - result += 5; - } - } - ESTACK_PUSH(s,ptr[i]); + if (arity > 1) { + WSTACK_PUSH2(s, (UWord) (ptr + 2), + (UWord) TERM_ARRAY_OP(arity-1)); } - goto outer_loop; + else if (arity == 0) { + break; + } + obj = ptr[1]; + continue; /* big loop */ } - break; case MAP_DEF: - { - map_t *mp = (map_t*)map_val(obj); - Uint size = map_get_size(mp); - Uint i; - Eterm *ptr; + if (is_flatmap(obj)) { + flatmap_t *mp = (flatmap_t*)flatmap_val(obj); + Uint size = flatmap_get_size(mp); result += 1 + 4; /* tag + 4 bytes size */ - /* push values first */ - ptr = map_get_values(mp); - i = size; - while(i--) { - if (is_list(*ptr)) { - if ((m = is_string(*ptr)) && (m < MAX_STRING_LEN)) { - result += m + 2 + 1; - } else { - result += 5; - } - } - ESTACK_PUSH(s,*ptr); - ++ptr; + if (size) { + WSTACK_PUSH4(s, (UWord) flatmap_get_values(mp), + (UWord) TERM_ARRAY_OP(size), + (UWord) flatmap_get_keys(mp), + (UWord) TERM_ARRAY_OP(size)); + } + } else { + Eterm *ptr; + Eterm hdr; + Uint node_sz; + ptr = boxed_val(obj); + hdr = *ptr; + ASSERT(is_header(hdr)); + switch(hdr & _HEADER_MAP_SUBTAG_MASK) { + case HAMT_SUBTAG_HEAD_ARRAY: + ptr++; + node_sz = 16; + result += 1 + 4; /* tag + 4 bytes size */ + break; + case HAMT_SUBTAG_HEAD_BITMAP: + ptr++; + result += 1 + 4; /* tag + 4 bytes size */ + /*fall through*/ + case HAMT_SUBTAG_NODE_BITMAP: + node_sz = hashmap_bitcount(MAP_HEADER_VAL(hdr)); + ASSERT(node_sz < 17); + break; + default: + erts_exit(ERTS_ERROR_EXIT, "bad header\r\n"); } - ptr = map_get_keys(mp); - i = size; - while(i--) { - if (is_list(*ptr)) { - if ((m = is_string(*ptr)) && (m < MAX_STRING_LEN)) { - result += m + 2 + 1; - } else { - result += 5; - } + ptr++; + WSTACK_RESERVE(s, node_sz*2); + while(node_sz--) { + if (is_list(*ptr)) { + WSTACK_FAST_PUSH(s, CAR(list_val(*ptr))); + WSTACK_FAST_PUSH(s, CDR(list_val(*ptr))); + } else { + WSTACK_FAST_PUSH(s, *ptr); } - ESTACK_PUSH(s,*ptr); - ++ptr; + ptr++; } - goto outer_loop; } break; case FLOAT_DEF: @@ -4071,25 +4157,13 @@ encode_size_struct_int(TTBSizeContext* ctx, ErtsAtomCacheMap *acmp, Eterm obj, result += 2 * (1 + 4); /* Index + Uniq */ result += 1 + (funp->num_free < 0x100 ? 1 : 4); } - for (i = 1; i < funp->num_free; i++) { - obj = funp->env[i]; - - if (is_not_list(obj)) { - /* Push any non-list terms on the stack */ - ESTACK_PUSH(s, obj); - } else { - /* Lists must be handled specially. */ - if ((m = is_string(obj)) && (m < MAX_STRING_LEN)) { - result += m + 2 + 1; - } else { - result += 5; - ESTACK_PUSH(s, obj); - } - } + if (funp->num_free > 1) { + WSTACK_PUSH2(s, (UWord) (funp->env + 1), + (UWord) TERM_ARRAY_OP(funp->num_free-1)); } if (funp->num_free != 0) { obj = funp->env[0]; - goto L_jump_start; + continue; /* big loop */ } break; } @@ -4097,11 +4171,7 @@ encode_size_struct_int(TTBSizeContext* ctx, ErtsAtomCacheMap *acmp, Eterm obj, case EXPORT_DEF: { Export* ep = *((Export **) (export_val(obj) + 1)); -#if HALFWORD_HEAP - result += 2; -#else result += 1; -#endif result += encode_size_struct2(acmp, ep->code[0], dflags); result += encode_size_struct2(acmp, ep->code[1], dflags); result += encode_size_struct2(acmp, make_small(ep->code[2]), dflags); @@ -4109,20 +4179,51 @@ encode_size_struct_int(TTBSizeContext* ctx, ErtsAtomCacheMap *acmp, Eterm obj, break; default: - erl_exit(1,"Internal data structure error (in encode_size_struct2)%x\n", + erts_exit(ERTS_ERROR_EXIT,"Internal data structure error (in encode_size_struct2)%x\n", obj); } + + if (WSTACK_ISEMPTY(s)) { + break; + } + obj = (Eterm) WSTACK_POP(s); + + if (is_header(obj)) { + switch (obj) { + case LIST_TAIL_OP: + obj = (Eterm) WSTACK_POP(s); + if (is_list(obj)) { + Eterm* cons = list_val(obj); + + WSTACK_PUSH2(s, (UWord)CDR(cons), (UWord)LIST_TAIL_OP); + obj = CAR(cons); + } + break; + + case TERM_ARRAY_OP(1): + obj = *(Eterm*)WSTACK_POP(s); + break; + default: { /* TERM_ARRAY_OP(N) when N > 1 */ + Eterm* ptr = (Eterm*) WSTACK_POP(s); + WSTACK_PUSH2(s, (UWord) (ptr+1), + (UWord) TERM_ARRAY_OP_DEC(obj)); + obj = *ptr; + } + } + } } - DESTROY_ESTACK(s); + WSTACK_DESTROY(s); if (ctx) { - ASSERT(ctx->estack.start == NULL); + ASSERT(ctx->wstack.wstart == NULL); *reds = r; } *res = result; return 0; } + + static Sint decoded_size(byte *ep, byte* endp, int internal_tags, B2TContext* ctx) { @@ -4185,7 +4286,7 @@ init_done: switch (tag) { case INTEGER_EXT: SKIP(4); -#if !defined(ARCH_64) || HALFWORD_HEAP +#if !defined(ARCH_64) heap_size += BIG_UINT_HEAP_SIZE; #endif break; @@ -4274,7 +4375,7 @@ init_done: ep += 2; atom_extra_skip = 1 + 4*id_words; /* In case it is an external ref */ -#if defined(ARCH_64) && !HALFWORD_HEAP +#if defined(ARCH_64) heap_size += EXTERNAL_THING_HEAD_SIZE + id_words/2 + 1; #else heap_size += EXTERNAL_THING_HEAD_SIZE + id_words; @@ -4316,7 +4417,11 @@ init_done: n = get_int32(ep); ep += 4; ADDTERMS(2*n); - heap_size += 3 + n + 1 + n; + if (n <= MAP_SMALL_MAP_LIMIT) { + heap_size += 3 + n + 1 + n; + } else { + heap_size += HASHMAP_ESTIMATED_HEAP_SIZE(n); + } break; case STRING_EXT: CHKSIZE(2); @@ -4356,11 +4461,7 @@ init_done: break; case EXPORT_EXT: terms += 3; -#if HALFWORD_HEAP - heap_size += 3; -#else heap_size += 2; -#endif break; case NEW_FUN_EXT: { @@ -4440,66 +4541,3 @@ error: #undef SKIP2 #undef CHKSIZE } - - -#ifdef HIPE -BIF_RETTYPE hipe_wrapper_term_to_binary_1(BIF_ALIST_1); -BIF_RETTYPE hipe_wrapper_term_to_binary_2(BIF_ALIST_2); -BIF_RETTYPE hipe_wrapper_erts_internal_binary_to_term_1(BIF_ALIST_1); -BIF_RETTYPE hipe_wrapper_erts_internal_binary_to_term_2(BIF_ALIST_2); - -/* Hipe wrappers used by native code for BIFs that disable GC while trapping. - * - * Problem: - * When native code calls a BIF that traps, hipe_mode_switch will push a - * "trap frame" on the Erlang stack in order to find its way back from beam_emu - * back to native caller when finally done. If GC is disabled and stack/heap - * is full there is no place to push the "trap frame". - * - * Solution: - * We reserve space on stack for the "trap frame" here before the BIF is called. - * If the BIF does not trap, the space is reclaimed here before returning. - * If the BIF traps, hipe_push_beam_trap_frame() will detect that a "trap frame" - * already is reserved and use it. - */ -BIF_RETTYPE hipe_wrapper_term_to_binary_1(BIF_ALIST_1) -{ - Eterm res; - hipe_reserve_beam_trap_frame(BIF_P, BIF__ARGS, 1); - res = term_to_binary_1(BIF_P, BIF__ARGS); - if (is_value(res) || BIF_P->freason != TRAP) { - hipe_unreserve_beam_trap_frame(BIF_P); - } - return res; -} -BIF_RETTYPE hipe_wrapper_term_to_binary_2(BIF_ALIST_2) -{ - Eterm res; - hipe_reserve_beam_trap_frame(BIF_P, BIF__ARGS, 2); - res = term_to_binary_2(BIF_P, BIF__ARGS); - if (is_value(res) || BIF_P->freason != TRAP) { - hipe_unreserve_beam_trap_frame(BIF_P); - } - return res; -} -BIF_RETTYPE hipe_wrapper_erts_internal_binary_to_term_1(BIF_ALIST_1) -{ - Eterm res; - hipe_reserve_beam_trap_frame(BIF_P, BIF__ARGS, 1); - res = erts_internal_binary_to_term_1(BIF_P, BIF__ARGS); - if (is_value(res) || BIF_P->freason != TRAP) { - hipe_unreserve_beam_trap_frame(BIF_P); - } - return res; -} -BIF_RETTYPE hipe_wrapper_erts_internal_binary_to_term_2(BIF_ALIST_2) -{ - Eterm res; - hipe_reserve_beam_trap_frame(BIF_P, BIF__ARGS, 2); - res = erts_internal_binary_to_term_2(BIF_P, BIF__ARGS); - if (is_value(res) || BIF_P->freason != TRAP) { - hipe_unreserve_beam_trap_frame(BIF_P); - } - return res; -} -#endif /*HIPE*/ diff --git a/erts/emulator/beam/external.h b/erts/emulator/beam/external.h index bf00958eb1..87eff2fe9f 100644 --- a/erts/emulator/beam/external.h +++ b/erts/emulator/beam/external.h @@ -1,18 +1,19 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 1996-2013. All Rights Reserved. + * Copyright Ericsson AB 1996-2014. All Rights Reserved. * - * The contents of this file are subject to the Erlang Public License, - * Version 1.1, (the "License"); you may not use this file except in - * compliance with the License. You should have received a copy of the - * Erlang Public License along with this software. If not, it can be - * retrieved online at http://www.erlang.org/. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Software distributed under the License is distributed on an "AS IS" - * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See - * the License for the specific language governing rights and limitations - * under the License. + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. * * %CopyrightEnd% */ @@ -148,21 +149,26 @@ typedef struct { byte *extp; int exttmp; Uint extsize; + Uint heap_size; } ErtsBinary2TermState; + /* -------------------------------------------------------------------------- */ void erts_init_atom_cache_map(ErtsAtomCacheMap *); void erts_reset_atom_cache_map(ErtsAtomCacheMap *); void erts_destroy_atom_cache_map(ErtsAtomCacheMap *); void erts_finalize_atom_cache_map(ErtsAtomCacheMap *, Uint32); -Uint erts_encode_ext_dist_header_size(ErtsAtomCacheMap *); Uint erts_encode_ext_dist_header_size(ErtsAtomCacheMap *); byte *erts_encode_ext_dist_header_setup(byte *, ErtsAtomCacheMap *); byte *erts_encode_ext_dist_header_finalize(byte *, ErtsAtomCache *, Uint32); -Uint erts_encode_dist_ext_size(Eterm, Uint32, ErtsAtomCacheMap *); -void erts_encode_dist_ext(Eterm, byte **, Uint32, ErtsAtomCacheMap *); +struct erts_dsig_send_context; +int erts_encode_dist_ext_size(Eterm, Uint32, ErtsAtomCacheMap*, Uint* szp); +int erts_encode_dist_ext_size_int(Eterm term, struct erts_dsig_send_context* ctx, Uint* szp); +struct TTBEncodeContext_; +int erts_encode_dist_ext(Eterm, byte **, Uint32, ErtsAtomCacheMap *, + struct TTBEncodeContext_ *, Sint* reds); Uint erts_encode_ext_size(Eterm); Uint erts_encode_ext_size_2(Eterm, unsigned); @@ -181,18 +187,18 @@ void erts_destroy_dist_ext_copy(ErtsDistExternal *); int erts_prepare_dist_ext(ErtsDistExternal *, byte *, Uint, DistEntry *, ErtsAtomCache *); Sint erts_decode_dist_ext_size(ErtsDistExternal *); -Eterm erts_decode_dist_ext(Eterm **, ErlOffHeap *, ErtsDistExternal *); +Eterm erts_decode_dist_ext(ErtsHeapFactory* factory, ErtsDistExternal *); Sint erts_decode_ext_size(byte*, Uint); Sint erts_decode_ext_size_ets(byte*, Uint); -Eterm erts_decode_ext(Eterm **, ErlOffHeap *, byte**); -Eterm erts_decode_ext_ets(Eterm **, ErlOffHeap *, byte*); +Eterm erts_decode_ext(ErtsHeapFactory*, byte**, Uint32 flags); +Eterm erts_decode_ext_ets(ErtsHeapFactory*, byte*); Eterm erts_term_to_binary(Process* p, Eterm Term, int level, Uint flags); Sint erts_binary2term_prepare(ErtsBinary2TermState *, byte *, Sint); void erts_binary2term_abort(ErtsBinary2TermState *); -Eterm erts_binary2term_create(ErtsBinary2TermState *, Eterm **hpp, ErlOffHeap *); +Eterm erts_binary2term_create(ErtsBinary2TermState *, ErtsHeapFactory*); int erts_debug_max_atom_out_cache_index(void); int erts_debug_atom_to_out_cache_index(Eterm); diff --git a/erts/emulator/beam/global.h b/erts/emulator/beam/global.h index 8fcb95d0e2..d7edf451be 100755..100644 --- a/erts/emulator/beam/global.h +++ b/erts/emulator/beam/global.h @@ -3,16 +3,17 @@ * * Copyright Ericsson AB 1996-2014. All Rights Reserved. * - * The contents of this file are subject to the Erlang Public License, - * Version 1.1, (the "License"); you may not use this file except in - * compliance with the License. You should have received a copy of the - * Erlang Public License along with this software. If not, it can be - * retrieved online at http://www.erlang.org/. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Software distributed under the License is distributed on an "AS IS" - * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See - * the License for the specific language governing rights and limitations - * under the License. + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. * * %CopyrightEnd% */ @@ -41,6 +42,7 @@ #include "error.h" #include "erl_utils.h" #include "erl_port.h" +#include "erl_gc.h" struct enif_environment_t /* ErlNifEnv */ { @@ -51,6 +53,7 @@ struct enif_environment_t /* ErlNifEnv */ ErlHeapFragment* heap_frag; int fpe_was_unmasked; struct enif_tmp_obj_t* tmp_obj_list; + int exception_thrown; /* boolean */ }; extern void erts_pre_nif(struct enif_environment_t*, Process*, struct erl_module_nif*); @@ -160,6 +163,7 @@ struct erts_driver_t_ { void (*ready_async)(ErlDrvData drv_data, ErlDrvThreadData thread_data); /* Might be NULL */ void (*process_exit)(ErlDrvData drv_data, ErlDrvMonitor *monitor); void (*stop_select)(ErlDrvEvent event, void*); /* Might be NULL */ + void (*emergency_close)(ErlDrvData drv_data); /* Might be NULL */ }; extern erts_driver_t *driver_list; @@ -227,9 +231,23 @@ typedef struct { ERTS_INTERNAL_BINARY_FIELDS SWord orig_size; void (*destructor)(Binary *); - char magic_bin_data[1]; + union { + struct { + ERTS_BINARY_STRUCT_ALIGNMENT + char data[1]; + } aligned; + struct { + char data[1]; + } unaligned; + } u; } ErtsMagicBinary; +#ifdef ARCH_32 +#define ERTS_MAGIC_BIN_BYTES_TO_ALIGN 4 +#else +#define ERTS_MAGIC_BIN_BYTES_TO_ALIGN 0 +#endif + typedef union { Binary binary; ErtsMagicBinary magic_binary; @@ -249,15 +267,30 @@ typedef union { #define ERTS_MAGIC_BIN_DESTRUCTOR(BP) \ ((ErtsBinary *) (BP))->magic_binary.destructor #define ERTS_MAGIC_BIN_DATA(BP) \ - ((void *) ((ErtsBinary *) (BP))->magic_binary.magic_bin_data) -#define ERTS_MAGIC_BIN_DATA_SIZE(BP) \ - ((BP)->orig_size - sizeof(void (*)(Binary *))) + ((void *) ((ErtsBinary *) (BP))->magic_binary.u.aligned.data) +#define ERTS_MAGIC_DATA_OFFSET \ + (offsetof(ErtsMagicBinary,u.aligned.data) - offsetof(Binary,orig_bytes)) #define ERTS_MAGIC_BIN_ORIG_SIZE(Sz) \ - (sizeof(void (*)(Binary *)) + (Sz)) + (ERTS_MAGIC_DATA_OFFSET + (Sz)) #define ERTS_MAGIC_BIN_SIZE(Sz) \ - (offsetof(ErtsMagicBinary,magic_bin_data) + (Sz)) -#define ERTS_MAGIC_BIN_FROM_DATA(DATA) \ - ((ErtsBinary*)((char*)(DATA) - offsetof(ErtsMagicBinary,magic_bin_data))) + (offsetof(ErtsMagicBinary,u.aligned.data) + (Sz)) + +/* On 32-bit arch these macro variants will save memory + by not forcing 8-byte alignment for the magic payload. +*/ +#define ERTS_MAGIC_BIN_UNALIGNED_DATA(BP) \ + ((void *) ((ErtsBinary *) (BP))->magic_binary.u.unaligned.data) +#define ERTS_MAGIC_UNALIGNED_DATA_OFFSET \ + (offsetof(ErtsMagicBinary,u.unaligned.data) - offsetof(Binary,orig_bytes)) +#define ERTS_MAGIC_BIN_UNALIGNED_DATA_SIZE(BP) \ + ((BP)->orig_size - ERTS_MAGIC_UNALIGNED_DATA_OFFSET) +#define ERTS_MAGIC_BIN_UNALIGNED_ORIG_SIZE(Sz) \ + (ERTS_MAGIC_UNALIGNED_DATA_OFFSET + (Sz)) +#define ERTS_MAGIC_BIN_UNALIGNED_SIZE(Sz) \ + (offsetof(ErtsMagicBinary,u.unaligned.data) + (Sz)) +#define ERTS_MAGIC_BIN_FROM_UNALIGNED_DATA(DATA) \ + ((ErtsBinary*)((char*)(DATA) - offsetof(ErtsMagicBinary,u.unaligned.data))) + #define Binary2ErlDrvBinary(B) (&((ErtsBinary *) (B))->driver.binary) #define ErlDrvBinary2Binary(D) ((Binary *) \ @@ -277,9 +310,6 @@ typedef union { typedef struct proc_bin { Eterm thing_word; /* Subtag REFC_BINARY_SUBTAG. */ Uint size; /* Binary size in bytes. */ -#if HALFWORD_HEAP - void* dummy_ptr_padding__; -#endif struct erl_off_heap_header *next; Binary *val; /* Pointer to Binary structure. */ byte *bytes; /* Pointer to the actual data bytes. */ @@ -346,13 +376,11 @@ extern Uint display_items; /* no of items to display in traces etc */ extern int erts_backtrace_depth; extern erts_smp_atomic32_t erts_max_gen_gcs; -extern int erts_disable_tolerant_timeofday; - extern int bif_reductions; /* reductions + fcalls (when doing call_bif) */ extern int stackdump_on_exit; /* - * Here is an implementation of a lightweiht stack. + * Here is an implementation of a lightweight stack. * * Use it like this: * @@ -370,16 +398,17 @@ extern int stackdump_on_exit; * DESTROY_ESTACK(Stack) */ -typedef struct { +typedef struct ErtsEStack_ { Eterm* start; Eterm* sp; Eterm* end; + Eterm* edefault; ErtsAlcType_t alloc_type; }ErtsEStack; #define DEF_ESTACK_SIZE (16) -void erl_grow_estack(ErtsEStack*, Eterm* def_stack); +void erl_grow_estack(ErtsEStack*, Uint need); #define ESTK_CONCAT(a,b) a##b #define ESTK_DEF_STACK(s) ESTK_CONCAT(s,_default_estack) @@ -389,22 +418,23 @@ void erl_grow_estack(ErtsEStack*, Eterm* def_stack); ESTK_DEF_STACK(s), /* start */ \ ESTK_DEF_STACK(s), /* sp */ \ ESTK_DEF_STACK(s) + DEF_ESTACK_SIZE, /* end */ \ + ESTK_DEF_STACK(s), /* default */ \ ERTS_ALC_T_ESTACK /* alloc_type */ \ } #define ESTACK_CHANGE_ALLOCATOR(s,t) \ do { \ - if (s.start != ESTK_DEF_STACK(s)) { \ - erl_exit(1, "Internal error - trying to change allocator " \ + if ((s).start != ESTK_DEF_STACK(s)) { \ + erts_exit(ERTS_ERROR_EXIT, "Internal error - trying to change allocator " \ "type of active estack\n"); \ } \ - s.alloc_type = (t); \ + (s).alloc_type = (t); \ } while (0) #define DESTROY_ESTACK(s) \ do { \ - if (s.start != ESTK_DEF_STACK(s)) { \ - erts_free(s.alloc_type, s.start); \ + if ((s).start != ESTK_DEF_STACK(s)) { \ + erts_free((s).alloc_type, (s).start); \ } \ } while(0) @@ -415,16 +445,17 @@ do { \ */ #define ESTACK_SAVE(s,dst)\ do {\ - if (s.start == ESTK_DEF_STACK(s)) {\ + if ((s).start == ESTK_DEF_STACK(s)) {\ UWord _wsz = ESTACK_COUNT(s);\ - (dst)->start = erts_alloc(s.alloc_type,\ + (dst)->start = erts_alloc((s).alloc_type,\ DEF_ESTACK_SIZE * sizeof(Eterm));\ - memcpy((dst)->start, s.start,_wsz*sizeof(Eterm));\ + memcpy((dst)->start, (s).start,_wsz*sizeof(Eterm));\ (dst)->sp = (dst)->start + _wsz;\ (dst)->end = (dst)->start + DEF_ESTACK_SIZE;\ - (dst)->alloc_type = s.alloc_type;\ + (dst)->edefault = NULL;\ + (dst)->alloc_type = (s).alloc_type;\ } else\ - *(dst) = s;\ + *(dst) = (s);\ } while (0) #define DESTROY_SAVED_ESTACK(estack)\ @@ -435,95 +466,146 @@ do {\ }\ } while(0) +#define CLEAR_SAVED_ESTACK(estack) ((void) ((estack)->start = NULL)) + /* * Use on empty stack, only the allocator can be changed before this. * The src stack is reset to NULL. */ #define ESTACK_RESTORE(s, src) \ do { \ - ASSERT(s.start == ESTK_DEF_STACK(s)); \ - s = *(src); /* struct copy */ \ + ASSERT((s).start == ESTK_DEF_STACK(s)); \ + (s) = *(src); /* struct copy */ \ (src)->start = NULL; \ - ASSERT(s.sp >= s.start); \ - ASSERT(s.sp <= s.end); \ + ASSERT((s).sp >= (s).start); \ + ASSERT((s).sp <= (s).end); \ } while (0) -#define ESTACK_IS_STATIC(s) (s.start == ESTK_DEF_STACK(s))) +#define ESTACK_IS_STATIC(s) ((s).start == ESTK_DEF_STACK(s)) -#define ESTACK_PUSH(s, x) \ -do { \ - if (s.sp == s.end) { \ - erl_grow_estack(&s, ESTK_DEF_STACK(s)); \ - } \ - *s.sp++ = (x); \ +#define ESTACK_PUSH(s, x) \ +do { \ + if ((s).sp == (s).end) { \ + erl_grow_estack(&(s), 1); \ + } \ + *(s).sp++ = (x); \ } while(0) #define ESTACK_PUSH2(s, x, y) \ do { \ - if (s.sp > s.end - 2) { \ - erl_grow_estack(&s, ESTK_DEF_STACK(s)); \ + if ((s).sp > (s).end - 2) { \ + erl_grow_estack(&(s), 2); \ } \ - *s.sp++ = (x); \ - *s.sp++ = (y); \ + *(s).sp++ = (x); \ + *(s).sp++ = (y); \ } while(0) #define ESTACK_PUSH3(s, x, y, z) \ do { \ - if (s.sp > s.end - 3) { \ - erl_grow_estack(&s, ESTK_DEF_STACK(s)); \ + if ((s).sp > (s).end - 3) { \ + erl_grow_estack(&s, 3); \ + } \ + *(s).sp++ = (x); \ + *(s).sp++ = (y); \ + *(s).sp++ = (z); \ +} while(0) + +#define ESTACK_PUSH4(s, E1, E2, E3, E4) \ +do { \ + if ((s).sp > (s).end - 4) { \ + erl_grow_estack(&s, 4); \ } \ - *s.sp++ = (x); \ - *s.sp++ = (y); \ - *s.sp++ = (z); \ + *(s).sp++ = (E1); \ + *(s).sp++ = (E2); \ + *(s).sp++ = (E3); \ + *(s).sp++ = (E4); \ } while(0) -#define ESTACK_COUNT(s) (s.sp - s.start) -#define ESTACK_ISEMPTY(s) (s.sp == s.start) -#define ESTACK_POP(s) (*(--s.sp)) +#define ESTACK_RESERVE(s, push_cnt) \ +do { \ + if ((s).sp > (s).end - (push_cnt)) { \ + erl_grow_estack(&(s), (push_cnt)); \ + } \ +} while(0) + +/* Must be preceded by ESTACK_RESERVE */ +#define ESTACK_FAST_PUSH(s, x) \ +do { \ + ASSERT((s).sp < (s).end); \ + *s.sp++ = (x); \ +} while(0) + +#define ESTACK_COUNT(s) ((s).sp - (s).start) +#define ESTACK_ISEMPTY(s) ((s).sp == (s).start) +#define ESTACK_POP(s) (*(--(s).sp)) /* * WSTACK: same as ESTACK but with UWord instead of Eterm */ -typedef struct { +typedef struct ErtsWStack_ { UWord* wstart; UWord* wsp; UWord* wend; + UWord* wdefault; ErtsAlcType_t alloc_type; }ErtsWStack; #define DEF_WSTACK_SIZE (16) -void erl_grow_wstack(ErtsWStack*, UWord* def_stack); +void erl_grow_wstack(ErtsWStack*, Uint need); #define WSTK_CONCAT(a,b) a##b #define WSTK_DEF_STACK(s) WSTK_CONCAT(s,_default_wstack) -#define DECLARE_WSTACK(s) \ +#define WSTACK_DECLARE(s) \ UWord WSTK_DEF_STACK(s)[DEF_WSTACK_SIZE]; \ ErtsWStack s = { \ WSTK_DEF_STACK(s), /* wstart */ \ WSTK_DEF_STACK(s), /* wsp */ \ WSTK_DEF_STACK(s) + DEF_WSTACK_SIZE, /* wend */ \ + WSTK_DEF_STACK(s), /* wdflt */ \ ERTS_ALC_T_ESTACK /* alloc_type */ \ } +#define DECLARE_WSTACK WSTACK_DECLARE + +typedef struct ErtsDynamicWStack_ { + UWord default_stack[DEF_WSTACK_SIZE]; + ErtsWStack ws; +}ErtsDynamicWStack; + +#define WSTACK_INIT(dwsp, ALC_TYPE) \ +do { \ + (dwsp)->ws.wstart = (dwsp)->default_stack; \ + (dwsp)->ws.wsp = (dwsp)->default_stack; \ + (dwsp)->ws.wend = (dwsp)->default_stack + DEF_WSTACK_SIZE;\ + (dwsp)->ws.wdefault = (dwsp)->default_stack; \ + (dwsp)->ws.alloc_type = ALC_TYPE; \ +} while (0) #define WSTACK_CHANGE_ALLOCATOR(s,t) \ do { \ if (s.wstart != WSTK_DEF_STACK(s)) { \ - erl_exit(1, "Internal error - trying to change allocator " \ + erts_exit(ERTS_ERROR_EXIT, "Internal error - trying to change allocator " \ "type of active wstack\n"); \ } \ s.alloc_type = (t); \ } while (0) -#define DESTROY_WSTACK(s) \ +#define WSTACK_DESTROY(s) \ do { \ - if (s.wstart != WSTK_DEF_STACK(s)) { \ + if (s.wstart != s.wdefault) { \ erts_free(s.alloc_type, s.wstart); \ } \ } while(0) +#define DESTROY_WSTACK WSTACK_DESTROY +#define WSTACK_DEBUG(s) \ + do { \ + fprintf(stderr, "wstack size = %ld\r\n", s.wsp - s.wstart); \ + fprintf(stderr, "wstack wstart = %p\r\n", s.wstart); \ + fprintf(stderr, "wstack wsp = %p\r\n", s.wsp); \ + } while(0) /* * Do not free the stack after this, it may have pointers into what @@ -538,6 +620,7 @@ do {\ memcpy((dst)->wstart, s.wstart,_wsz*sizeof(UWord));\ (dst)->wsp = (dst)->wstart + _wsz;\ (dst)->wend = (dst)->wstart + DEF_WSTACK_SIZE;\ + (dst)->wdefault = NULL;\ (dst)->alloc_type = s.alloc_type;\ } else\ *(dst) = s;\ @@ -551,6 +634,8 @@ do {\ }\ } while(0) +#define CLEAR_SAVED_WSTACK(wstack) ((void) ((wstack)->wstart = NULL)) + /* * Use on empty stack, only the allocator can be changed before this. * The src stack is reset to NULL. @@ -564,12 +649,12 @@ do { \ ASSERT(s.wsp <= s.wend); \ } while (0) -#define WSTACK_IS_STATIC(s) (s.wstart == WSTK_DEF_STACK(s))) +#define WSTACK_IS_STATIC(s) (s.wstart == WSTK_DEF_STACK(s)) #define WSTACK_PUSH(s, x) \ do { \ if (s.wsp == s.wend) { \ - erl_grow_wstack(&s, WSTK_DEF_STACK(s)); \ + erl_grow_wstack(&s, 1); \ } \ *s.wsp++ = (x); \ } while(0) @@ -577,7 +662,7 @@ do { \ #define WSTACK_PUSH2(s, x, y) \ do { \ if (s.wsp > s.wend - 2) { \ - erl_grow_wstack(&s, WSTK_DEF_STACK(s)); \ + erl_grow_wstack(&s, 2); \ } \ *s.wsp++ = (x); \ *s.wsp++ = (y); \ @@ -585,18 +670,268 @@ do { \ #define WSTACK_PUSH3(s, x, y, z) \ do { \ - if (s.wsp > s.wend - 3) { \ - erl_grow_wstack(&s, WSTK_DEF_STACK(s)); \ + if (s.wsp > s.wend - 3) { \ + erl_grow_wstack(&s, 3); \ } \ *s.wsp++ = (x); \ *s.wsp++ = (y); \ *s.wsp++ = (z); \ } while(0) +#define WSTACK_PUSH4(s, A1, A2, A3, A4) \ +do { \ + if (s.wsp > s.wend - 4) { \ + erl_grow_wstack(&s, 4); \ + } \ + *s.wsp++ = (A1); \ + *s.wsp++ = (A2); \ + *s.wsp++ = (A3); \ + *s.wsp++ = (A4); \ +} while(0) + +#define WSTACK_PUSH5(s, A1, A2, A3, A4, A5) \ +do { \ + if (s.wsp > s.wend - 5) { \ + erl_grow_wstack(&s, 5); \ + } \ + *s.wsp++ = (A1); \ + *s.wsp++ = (A2); \ + *s.wsp++ = (A3); \ + *s.wsp++ = (A4); \ + *s.wsp++ = (A5); \ +} while(0) + +#define WSTACK_PUSH6(s, A1, A2, A3, A4, A5, A6) \ +do { \ + if (s.wsp > s.wend - 6) { \ + erl_grow_wstack(&s, 6); \ + } \ + *s.wsp++ = (A1); \ + *s.wsp++ = (A2); \ + *s.wsp++ = (A3); \ + *s.wsp++ = (A4); \ + *s.wsp++ = (A5); \ + *s.wsp++ = (A6); \ +} while(0) + +#define WSTACK_RESERVE(s, push_cnt) \ +do { \ + if (s.wsp > s.wend - (push_cnt)) { \ + erl_grow_wstack(&s, (push_cnt)); \ + } \ +} while(0) + +/* Must be preceded by WSTACK_RESERVE */ +#define WSTACK_FAST_PUSH(s, x) \ +do { \ + ASSERT(s.wsp < s.wend); \ + *s.wsp++ = (x); \ +} while(0) + #define WSTACK_COUNT(s) (s.wsp - s.wstart) #define WSTACK_ISEMPTY(s) (s.wsp == s.wstart) -#define WSTACK_POP(s) (*(--s.wsp)) +#define WSTACK_POP(s) ((ASSERT(s.wsp > s.wstart)),*(--s.wsp)) + +#define WSTACK_ROLLBACK(s, count) (ASSERT(WSTACK_COUNT(s) >= (count)), \ + s.wsp = s.wstart + (count)) + +/* PSTACK - Stack of any type. + * Usage: + * { + * #define PSTACK_TYPE MyType + * PSTACK_DECLARE(s,16); + * MyType *sp = PSTACK_PUSH(s); + * + * sp->x = .... + * sp->y = .... + * sp = PSTACK_PUSH(s); + * ... + * sp = PSTACK_POP(s); + * if (PSTACK_IS_EMPTY(s)) { + * // sp is invalid when stack is empty after pop + * } + * + * PSTACK_DESTROY(s); + * } + */ + + +typedef struct ErtsPStack_ { + byte* pstart; + byte* psp; + byte* pend; + ErtsAlcType_t alloc_type; +}ErtsPStack; + +void erl_grow_pstack(ErtsPStack* s, void* default_pstack, unsigned need_bytes); +#define PSTK_CONCAT(a,b) a##b +#define PSTK_DEF_STACK(s) PSTK_CONCAT(s,_default_pstack) + +#define PSTACK_DECLARE(s, DEF_PSTACK_SIZE) \ +PSTACK_TYPE PSTK_DEF_STACK(s)[DEF_PSTACK_SIZE]; \ +ErtsPStack s = { (byte*)PSTK_DEF_STACK(s), /* pstart */ \ + (byte*)(PSTK_DEF_STACK(s) - 1), /* psp */ \ + (byte*)(PSTK_DEF_STACK(s) + (DEF_PSTACK_SIZE)), /* pend */\ + ERTS_ALC_T_ESTACK /* alloc_type */ \ +} + +#define PSTACK_CHANGE_ALLOCATOR(s,t) \ +do { \ + if (s.pstart != (byte*)PSTK_DEF_STACK(s)) { \ + erts_exit(ERTS_ERROR_EXIT, "Internal error - trying to change allocator " \ + "type of active pstack\n"); \ + } \ + s.alloc_type = (t); \ + } while (0) + +#define PSTACK_DESTROY(s) \ +do { \ + if (s.pstart != (byte*)PSTK_DEF_STACK(s)) { \ + erts_free(s.alloc_type, s.pstart); \ + } \ +} while(0) + +#define PSTACK_IS_EMPTY(s) (s.psp < s.pstart) + +#define PSTACK_COUNT(s) (((PSTACK_TYPE*)s.psp + 1) - (PSTACK_TYPE*)s.pstart) + +#define PSTACK_TOP(s) (ASSERT(!PSTACK_IS_EMPTY(s)), (PSTACK_TYPE*)(s.psp)) + +#define PSTACK_PUSH(s) \ + (s.psp += sizeof(PSTACK_TYPE), \ + ((s.psp == s.pend) ? erl_grow_pstack(&s, PSTK_DEF_STACK(s), \ + sizeof(PSTACK_TYPE)) : (void)0), \ + ((PSTACK_TYPE*) s.psp)) +#define PSTACK_POP(s) ((PSTACK_TYPE*) (s.psp -= sizeof(PSTACK_TYPE))) + +/* + * Do not free the stack after this, it may have pointers into what + * was saved in 'dst'. + */ +#define PSTACK_SAVE(s,dst)\ +do {\ + if (s.pstart == (byte*)PSTK_DEF_STACK(s)) {\ + UWord _pbytes = PSTACK_COUNT(s) * sizeof(PSTACK_TYPE);\ + (dst)->pstart = erts_alloc(s.alloc_type,\ + sizeof(PSTK_DEF_STACK(s)));\ + sys_memcpy((dst)->pstart, s.pstart, _pbytes);\ + (dst)->psp = (dst)->pstart + _pbytes - sizeof(PSTACK_TYPE);\ + (dst)->pend = (dst)->pstart + sizeof(PSTK_DEF_STACK(s));\ + (dst)->alloc_type = s.alloc_type;\ + } else\ + *(dst) = s;\ + } while (0) + +/* + * Use on empty stack, only the allocator can be changed before this. + * The src stack is reset to NULL. + */ +#define PSTACK_RESTORE(s, src) \ +do { \ + ASSERT(s.pstart == (byte*)PSTK_DEF_STACK(s)); \ + s = *(src); /* struct copy */ \ + (src)->pstart = NULL; \ + ASSERT(s.psp >= (s.pstart - sizeof(PSTACK_TYPE))); \ + ASSERT(s.psp < s.pend); \ +} while (0) + +#define PSTACK_DESTROY_SAVED(pstack)\ +do {\ + if ((pstack)->pstart) {\ + erts_free((pstack)->alloc_type, (pstack)->pstart);\ + (pstack)->pstart = NULL;\ + }\ +} while(0) + + +/* + * An implementation of lightweight unbounded queues, + * using a circular dynamic array. + * It does not include support for change_allocator. + * + * Use it like this: + * + * DECLARE_EQUEUE(Queue) (At the start of a block) + * ... + * EQUEUE_PUT(Queue, Term) + * ... + * if (EQUEUE_ISEMPTY(Queue)) { + * Queue is empty + * } else { + * Term = EQUEUE_GET(Stack); + * Process popped Term here + * } + * ... + * DESTROY_EQUEUE(Queue) + */ + +typedef struct { + Eterm* start; + Eterm* front; + Eterm* back; + int possibly_empty; + Eterm* end; + ErtsAlcType_t alloc_type; +} ErtsEQueue; + +#define DEF_EQUEUE_SIZE (16) + +void erl_grow_equeue(ErtsEQueue*, Eterm* def_queue); +#define EQUE_CONCAT(a,b) a##b +#define EQUE_DEF_QUEUE(q) EQUE_CONCAT(q,_default_equeue) + +#define DECLARE_EQUEUE(q) \ + UWord EQUE_DEF_QUEUE(q)[DEF_EQUEUE_SIZE]; \ + ErtsEQueue q = { \ + EQUE_DEF_QUEUE(q), /* start */ \ + EQUE_DEF_QUEUE(q), /* front */ \ + EQUE_DEF_QUEUE(q), /* back */ \ + 1, /* possibly_empty */ \ + EQUE_DEF_QUEUE(q) + DEF_EQUEUE_SIZE, /* end */ \ + ERTS_ALC_T_ESTACK /* alloc_type */ \ + } + +#define DESTROY_EQUEUE(q) \ +do { \ + if (q.start != EQUE_DEF_QUEUE(q)) { \ + erts_free(q.alloc_type, q.start); \ + } \ +} while(0) + +#define EQUEUE_PUT_UNCHECKED(q, x) \ +do { \ + q.possibly_empty = 0; \ + *(q.back) = (x); \ + if (++(q.back) == q.end) { \ + q.back = q.start; \ + } \ +} while(0) + +#define EQUEUE_PUT(q, x) \ +do { \ + if (q.back == q.front && !q.possibly_empty) { \ + erl_grow_equeue(&q, EQUE_DEF_QUEUE(q)); \ + } \ + EQUEUE_PUT_UNCHECKED(q, x); \ +} while(0) + +#define EQUEUE_ISEMPTY(q) (q.back == q.front && q.possibly_empty) + +ERTS_GLB_INLINE Eterm erts_equeue_get(ErtsEQueue *q); + +#if ERTS_GLB_INLINE_INCL_FUNC_DEF +ERTS_GLB_INLINE Eterm erts_equeue_get(ErtsEQueue *q) { + Eterm x; + q->possibly_empty = 1; + x = *(q->front); + if (++(q->front) == q->end) { + q->front = q->start; + } + return x; +} +#endif +#define EQUEUE_GET(q) erts_equeue_get(&(q)); /* binary.c */ @@ -618,9 +953,6 @@ erts_bld_port_info(Eterm **hpp, void erts_bif_info_init(void); /* bif.c */ -Eterm erts_make_ref(Process *); -Eterm erts_make_ref_in_buffer(Eterm buffer[REF_THING_SIZE]); -void erts_make_ref_in_array(Uint32 ref[ERTS_MAX_REF_NUMBERS]); ERTS_GLB_INLINE Eterm erts_proc_store_ref(Process *c_p, Uint32 ref[ERTS_MAX_REF_NUMBERS]); @@ -653,8 +985,18 @@ Eterm erl_send(Process *p, Eterm to, Eterm msg); Eterm erl_is_function(Process* p, Eterm arg1, Eterm arg2); /* beam_bif_load.c */ -Eterm erts_check_process_code(Process *c_p, Eterm module, int allow_gc, int *redsp); +#define ERTS_CPC_ALLOW_GC (1 << 0) +#define ERTS_CPC_COPY_LITERALS (1 << 1) +#define ERTS_CPC_ALL (ERTS_CPC_ALLOW_GC | ERTS_CPC_COPY_LITERALS) +Eterm erts_check_process_code(Process *c_p, Eterm module, Uint flags, int *redsp); + +typedef struct { + Eterm *ptr; + Uint sz; + Eterm pid; +} copy_literals_t; +extern copy_literals_t erts_clrange; /* beam_load.c */ typedef struct { @@ -666,6 +1008,7 @@ typedef struct { Binary* erts_alloc_loader_state(void); Eterm erts_module_for_prepared_code(Binary* magic); +Eterm erts_has_code_on_load(Binary* magic); Eterm erts_prepare_loading(Binary* loader_state, Process *c_p, Eterm group_leader, Eterm* modp, byte* code, Uint size); @@ -684,7 +1027,7 @@ Eterm erts_make_stub_module(Process* p, Eterm Mod, Eterm Beam, Eterm Info); /* beam_ranges.c */ void erts_init_ranges(void); -void erts_start_staging_ranges(void); +void erts_start_staging_ranges(int num_new); void erts_end_staging_ranges(int commit); void erts_update_ranges(BeamInstr* code, Uint size); void erts_remove_from_ranges(BeamInstr* code); @@ -700,41 +1043,80 @@ void print_process_info(int, void *, Process*); void info(int, void *); void loaded(int, void *); +/* erl_arith.c */ +double erts_get_positive_zero_float(void); + /* config.c */ -__decl_noreturn void __noreturn erl_exit(int n, char*, ...); -__decl_noreturn void __noreturn erl_exit_flush_async(int n, char*, ...); +__decl_noreturn void __noreturn erts_exit(int n, char*, ...); +__decl_noreturn void __noreturn erts_flush_async_exit(int n, char*, ...); void erl_error(char*, va_list); -/* copy.c */ -Eterm copy_object(Eterm, Process*); +/* This controls whether sharing-preserving copy is used by Erlang */ -#if HALFWORD_HEAP -Uint size_object_rel(Eterm, Eterm*); -# define size_object(A) size_object_rel(A,NULL) +#ifdef SHCOPY +#define SHCOPY_SEND +#define SHCOPY_SPAWN +#endif + +/* The persistent state while the sharing-preserving copier works */ -Eterm copy_struct_rel(Eterm, Uint, Eterm**, ErlOffHeap*, Eterm* src_base, Eterm* dst_base); -# define copy_struct(OBJ,SZ,HPP,OH) copy_struct_rel(OBJ,SZ,HPP,OH, NULL,NULL) +typedef struct { + Eterm queue_default[DEF_EQUEUE_SIZE]; + Eterm* queue_start; + Eterm* queue_end; + ErtsAlcType_t queue_alloc_type; + UWord bitstore_default[DEF_WSTACK_SIZE]; + UWord* bitstore_start; + ErtsAlcType_t bitstore_alloc_type; + Eterm shtable_default[DEF_ESTACK_SIZE]; + Eterm* shtable_start; + ErtsAlcType_t shtable_alloc_type; + Uint literal_size; + Eterm *range_ptr; + Uint range_sz; +} erts_shcopy_t; + +#define INITIALIZE_SHCOPY(info) \ +do { \ + info.queue_start = info.queue_default; \ + info.bitstore_start = info.bitstore_default; \ + info.shtable_start = info.shtable_default; \ + info.literal_size = 0; \ + info.range_ptr = erts_clrange.ptr; \ + info.range_sz = erts_clrange.sz; \ +} while(0) -Eterm copy_shallow_rel(Eterm*, Uint, Eterm**, ErlOffHeap*, Eterm* src_base); -# define copy_shallow(A,B,C,D) copy_shallow_rel(A,B,C,D,NULL) +#define DESTROY_SHCOPY(info) \ +do { \ + if (info.queue_start != info.queue_default) { \ + erts_free(info.queue_alloc_type, info.queue_start); \ + } \ + if (info.bitstore_start != info.bitstore_default) { \ + erts_free(info.bitstore_alloc_type, info.bitstore_start); \ + } \ + if (info.shtable_start != info.shtable_default) { \ + erts_free(info.shtable_alloc_type, info.shtable_start); \ + } \ +} while(0) -#else /* !HALFWORD_HEAP */ +/* copy.c */ +Eterm copy_object_x(Eterm, Process*, Uint); +#define copy_object(Term, Proc) copy_object_x(Term,Proc,0) Uint size_object(Eterm); -# define size_object_rel(A,B) size_object(A) +Uint copy_shared_calculate(Eterm, erts_shcopy_t*); +Eterm copy_shared_perform(Eterm, Uint, erts_shcopy_t*, Eterm**, ErlOffHeap*); -Eterm copy_struct(Eterm, Uint, Eterm**, ErlOffHeap*); -# define copy_struct_rel(OBJ,SZ,HPP,OH, SB,DB) copy_struct(OBJ,SZ,HPP,OH) +Uint size_shared(Eterm); +Eterm copy_struct_x(Eterm, Uint, Eterm**, ErlOffHeap*, Uint* bsz); +#define copy_struct(Obj,Sz,HPP,OH) \ + copy_struct_x(Obj,Sz,HPP,OH,NULL) Eterm copy_shallow(Eterm*, Uint, Eterm**, ErlOffHeap*); -# define copy_shallow_rel(A,B,C,D, BASE) copy_shallow(A,B,C,D) -#endif - - -void move_multi_frags(Eterm** hpp, ErlOffHeap*, ErlHeapFragment* first, - Eterm* refs, unsigned nrefs); +void erts_move_multi_frags(Eterm** hpp, ErlOffHeap*, ErlHeapFragment* first, + Eterm* refs, unsigned nrefs, int literals); /* Utilities */ extern void erts_delete_nodes_monitors(Process *, ErtsProcLocks); @@ -787,6 +1169,9 @@ extern ErtsModifiedTimings erts_modified_timings[]; extern int erts_no_line_info; extern Eterm erts_error_logger_warnings; extern int erts_initialized; +#if defined(USE_THREADS) && !defined(ERTS_SMP) +extern erts_tid_t erts_main_thread; +#endif extern int erts_compat_rel; extern int erts_use_sender_punish; void erts_short_init(void); @@ -805,23 +1190,6 @@ void MD5Init(MD5_CTX *); void MD5Update(MD5_CTX *, unsigned char *, unsigned int); void MD5Final(unsigned char [16], MD5_CTX *); -/* ggc.c */ - -void erts_gc_info(ErtsGCInfo *gcip); -void erts_init_gc(void); -int erts_garbage_collect(Process*, int, Eterm*, int); -void erts_garbage_collect_hibernate(Process* p); -Eterm erts_gc_after_bif_call(Process* p, Eterm result, Eterm* regs, Uint arity); -void erts_garbage_collect_literals(Process* p, Eterm* literals, - Uint lit_size, - struct erl_off_heap_header* oh); -Uint erts_next_heap_size(Uint, Uint); -Eterm erts_heap_sizes(Process* p); - -void erts_offset_off_heap(ErlOffHeap *, Sint, Eterm*, Eterm*); -void erts_offset_heap_ptr(Eterm*, Uint, Sint, Eterm*, Eterm*); -void erts_offset_heap(Eterm*, Uint, Sint, Eterm*, Eterm*); -void erts_free_heap_frags(Process* p); /* io.c */ @@ -848,6 +1216,7 @@ Uint erts_port_ioq_size(Port *pp); void erts_stale_drv_select(Eterm, ErlDrvPort, ErlDrvEvent, int, int); Port *erts_get_heart_port(void); +void erts_emergency_close_ports(void); #if defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_COUNT) void erts_lcnt_enable_io_lock_count(int enable); @@ -901,6 +1270,9 @@ Sint erts_binary_set_loop_limit(Sint limit); /* external.c */ void erts_init_external(void); +/* erl_map.c */ +void erts_init_map(void); + /* erl_unicode.c */ void erts_init_unicode(void); Sint erts_unicode_set_loop_limit(Sint limit); @@ -940,10 +1312,10 @@ int erts_utf8_to_latin1(byte* dest, const byte* source, int slen); #define ERTS_UTF8_OK_MAX_CHARS 4 void bin_write(int, void*, byte*, size_t); -int intlist_to_buf(Eterm, char*, int); /* most callers pass plain char*'s */ +Sint intlist_to_buf(Eterm, char*, Sint); /* most callers pass plain char*'s */ struct Sint_buf { -#if defined(ARCH_64) && !HALFWORD_HEAP +#if defined(ARCH_64) char s[22]; #else char s[12]; @@ -951,22 +1323,69 @@ struct Sint_buf { }; char* Sint_to_buf(Sint, struct Sint_buf*); +#define ERTS_IOLIST_STATE_INITER(C_P, OBJ) \ + {(C_P), 0, 0, (OBJ), {NULL, NULL, NULL, ERTS_ALC_T_INVALID}, 0, 0} + +#define ERTS_IOLIST_STATE_MOVE(TO, FROM) \ + sys_memcpy((void *) (TO), (void *) (FROM), sizeof(ErtsIOListState)) + +#define ERTS_IOLIST_SIZE_YIELDS_COUNT_PER_RED 8 + +typedef struct { + Process *c_p; + ErlDrvSizeT size; + Uint offs; + Eterm obj; + ErtsEStack estack; + int reds_left; + int have_size; +} ErtsIOListState; + +#define ERTS_IOLIST2BUF_STATE_INITER(C_P, OBJ) \ + {ERTS_IOLIST_STATE_INITER((C_P), (OBJ)), {NULL, 0, 0, 0}, NULL, 0, NULL, 0} + +#define ERTS_IOLIST2BUF_STATE_MOVE(TO, FROM) \ + sys_memcpy((void *) (TO), (void *) (FROM), sizeof(ErtsIOList2BufState)) + +#define ERTS_IOLIST_TO_BUF_BYTES_PER_YIELD_COUNT 32 +#define ERTS_IOLIST_TO_BUF_YIELD_COUNT_PER_RED 8 +#define ERTS_IOLIST_TO_BUF_BYTES_PER_RED \ + (ERTS_IOLIST_TO_BUF_YIELD_COUNT_PER_RED*ERTS_IOLIST_TO_BUF_BYTES_PER_YIELD_COUNT) + +typedef struct { + ErtsIOListState iolist; + struct { + byte *bptr; + size_t size; + Uint bitoffs; + Uint bitsize; + } bcopy; + char *buf; + ErlDrvSizeT len; + Eterm *objp; + int offset; +} ErtsIOList2BufState; + #define ERTS_IOLIST_OK 0 #define ERTS_IOLIST_OVERFLOW 1 #define ERTS_IOLIST_TYPE 2 +#define ERTS_IOLIST_YIELD 3 Eterm buf_to_intlist(Eterm**, const char*, size_t, Eterm); /* most callers pass plain char*'s */ #define ERTS_IOLIST_TO_BUF_OVERFLOW (~((ErlDrvSizeT) 0)) #define ERTS_IOLIST_TO_BUF_TYPE_ERROR (~((ErlDrvSizeT) 1)) +#define ERTS_IOLIST_TO_BUF_YIELD (~((ErlDrvSizeT) 2)) #define ERTS_IOLIST_TO_BUF_FAILED(R) \ - (((R) & (~((ErlDrvSizeT) 1))) == (~((ErlDrvSizeT) 1))) + (((R) & (~((ErlDrvSizeT) 3))) == (~((ErlDrvSizeT) 3))) #define ERTS_IOLIST_TO_BUF_SUCCEEDED(R) \ (!ERTS_IOLIST_TO_BUF_FAILED((R))) ErlDrvSizeT erts_iolist_to_buf(Eterm, char*, ErlDrvSizeT); +ErlDrvSizeT erts_iolist_to_buf_yielding(ErtsIOList2BufState *); +int erts_iolist_size_yielding(ErtsIOListState *state); int erts_iolist_size(Eterm, ErlDrvSizeT *); -int is_string(Eterm); +Sint is_string(Eterm); void erl_at_exit(void (*) (void*), void*); Eterm collect_memory(Process *); void dump_memory_to_fd(int); @@ -1012,6 +1431,29 @@ int erts_print_system_version(int to, void *arg, Process *c_p); int erts_hibernate(Process* c_p, Eterm module, Eterm function, Eterm args, Eterm* reg); +ERTS_GLB_FORCE_INLINE int erts_is_literal(Eterm tptr, Eterm *ptr); + +#if ERTS_GLB_INLINE_INCL_FUNC_DEF + +ERTS_GLB_FORCE_INLINE int erts_is_literal(Eterm tptr, Eterm *ptr) +{ + ASSERT(is_boxed(tptr) || is_list(tptr)); + ASSERT(ptr == ptr_val(tptr)); + +#if defined(ERTS_HAVE_IS_IN_LITERAL_RANGE) + return erts_is_in_literal_range(ptr); +#elif defined(TAG_LITERAL_PTR) + return is_literal_ptr(tptr); +#else +# error Not able to detect literals... +#endif + +} + +#endif + +Eterm erts_msacc_request(Process *c_p, int action, Eterm *threads); + /* ** Call_trace uses this API for the parameter matching functions */ @@ -1060,189 +1502,21 @@ extern void erts_match_prog_foreach_offheap(Binary *b, extern erts_driver_t vanilla_driver; extern erts_driver_t spawn_driver; +extern erts_driver_t forker_driver; extern erts_driver_t fd_driver; int erts_beam_jump_table(void); -/* Should maybe be placed in erl_message.h, but then we get an include mess. */ -ERTS_GLB_INLINE Eterm * -erts_alloc_message_heap_state(Uint size, - ErlHeapFragment **bpp, - ErlOffHeap **ohpp, - Process *receiver, - ErtsProcLocks *receiver_locks, - erts_aint32_t *statep); - -ERTS_GLB_INLINE Eterm * -erts_alloc_message_heap(Uint size, - ErlHeapFragment **bpp, - ErlOffHeap **ohpp, - Process *receiver, - ErtsProcLocks *receiver_locks); - -#if ERTS_GLB_INLINE_INCL_FUNC_DEF - -/* - * NOTE: erts_alloc_message_heap() releases msg q and status - * lock on receiver without ensuring that other locks are - * held. User is responsible to ensure that the receiver - * pointer cannot become invalid until after message has - * been passed. This is normal done either by increasing - * reference count on process (preferred) or by holding - * main or link lock over the whole message passing - * operation. - */ - -ERTS_GLB_INLINE Eterm * -erts_alloc_message_heap_state(Uint size, - ErlHeapFragment **bpp, - ErlOffHeap **ohpp, - Process *receiver, - ErtsProcLocks *receiver_locks, - erts_aint32_t *statep) -{ - Eterm *hp; - erts_aint32_t state; -#ifdef ERTS_SMP - int locked_main = 0; - state = erts_smp_atomic32_read_acqb(&receiver->state); - if (statep) - *statep = state; - if (state & (ERTS_PSFLG_EXITING|ERTS_PSFLG_PENDING_EXIT)) - goto allocate_in_mbuf; -#endif - - if (size > (Uint) INT_MAX) - erl_exit(ERTS_ABORT_EXIT, "HUGE size (%beu)\n", size); - - if ( -#if defined(ERTS_SMP) - *receiver_locks & ERTS_PROC_LOCK_MAIN -#else - 1 -#endif - ) { -#ifdef ERTS_SMP - try_allocate_on_heap: -#endif - state = erts_smp_atomic32_read_nob(&receiver->state); - if (statep) - *statep = state; - if ((state & (ERTS_PSFLG_EXITING|ERTS_PSFLG_PENDING_EXIT)) - || (receiver->flags & F_DISABLE_GC) - || HEAP_LIMIT(receiver) - HEAP_TOP(receiver) <= size) { - /* - * The heap is either potentially in an inconsistent - * state, or not large enough. - */ -#ifdef ERTS_SMP - if (locked_main) { - *receiver_locks &= ~ERTS_PROC_LOCK_MAIN; - erts_smp_proc_unlock(receiver, ERTS_PROC_LOCK_MAIN); - } -#endif - goto allocate_in_mbuf; - } - hp = HEAP_TOP(receiver); - HEAP_TOP(receiver) = hp + size; - *bpp = NULL; - *ohpp = &MSO(receiver); - } -#ifdef ERTS_SMP - else if (erts_smp_proc_trylock(receiver, ERTS_PROC_LOCK_MAIN) == 0) { - locked_main = 1; - *receiver_locks |= ERTS_PROC_LOCK_MAIN; - goto try_allocate_on_heap; - } -#endif - else { - ErlHeapFragment *bp; - allocate_in_mbuf: - bp = new_message_buffer(size); - hp = bp->mem; - *bpp = bp; - *ohpp = &bp->off_heap; - } - - return hp; -} - -ERTS_GLB_INLINE Eterm * -erts_alloc_message_heap(Uint size, - ErlHeapFragment **bpp, - ErlOffHeap **ohpp, - Process *receiver, - ErtsProcLocks *receiver_locks) -{ - return erts_alloc_message_heap_state(size, bpp, ohpp, receiver, - receiver_locks, NULL); -} - -#endif /* #if ERTS_GLB_INLINE_INCL_FUNC_DEF */ - -#if !HEAP_ON_C_STACK -# if defined(DEBUG) -# define DeclareTmpHeap(VariableName,Size,Process) \ - Eterm *VariableName = erts_debug_allocate_tmp_heap(Size,Process) -# define DeclareTypedTmpHeap(Type,VariableName,Process) \ - Type *VariableName = (Type *) erts_debug_allocate_tmp_heap(sizeof(Type)/sizeof(Eterm),Process) -# define DeclareTmpHeapNoproc(VariableName,Size) \ - Eterm *VariableName = erts_debug_allocate_tmp_heap(Size,NULL) -# define UseTmpHeap(Size,Proc) \ - do { \ - erts_debug_use_tmp_heap((Size),(Proc)); \ - } while (0) -# define UnUseTmpHeap(Size,Proc) \ - do { \ - erts_debug_unuse_tmp_heap((Size),(Proc)); \ - } while (0) -# define UseTmpHeapNoproc(Size) \ - do { \ - erts_debug_use_tmp_heap(Size,NULL); \ - } while (0) -# define UnUseTmpHeapNoproc(Size) \ - do { \ - erts_debug_unuse_tmp_heap(Size,NULL); \ - } while (0) -# else -# define DeclareTmpHeap(VariableName,Size,Process) \ - Eterm *VariableName = (ERTS_PROC_GET_SCHDATA(Process)->tmp_heap)+(ERTS_PROC_GET_SCHDATA(Process)->num_tmp_heap_used) -# define DeclareTypedTmpHeap(Type,VariableName,Process) \ - Type *VariableName = (Type *) (ERTS_PROC_GET_SCHDATA(Process)->tmp_heap)+(ERTS_PROC_GET_SCHDATA(Process)->num_tmp_heap_used) -# define DeclareTmpHeapNoproc(VariableName,Size) \ - Eterm *VariableName = (erts_get_scheduler_data()->tmp_heap)+(erts_get_scheduler_data()->num_tmp_heap_used) -# define UseTmpHeap(Size,Proc) \ - do { \ - ERTS_PROC_GET_SCHDATA(Proc)->num_tmp_heap_used += (Size); \ - } while (0) -# define UnUseTmpHeap(Size,Proc) \ - do { \ - ERTS_PROC_GET_SCHDATA(Proc)->num_tmp_heap_used -= (Size); \ - } while (0) -# define UseTmpHeapNoproc(Size) \ - do { \ - erts_get_scheduler_data()->num_tmp_heap_used += (Size); \ - } while (0) -# define UnUseTmpHeapNoproc(Size) \ - do { \ - erts_get_scheduler_data()->num_tmp_heap_used -= (Size); \ - } while (0) - - -# endif - -#else -# define DeclareTmpHeap(VariableName,Size,Process) \ +#define DeclareTmpHeap(VariableName,Size,Process) \ Eterm VariableName[Size] -# define DeclareTypedTmpHeap(Type,VariableName,Process) \ +#define DeclareTypedTmpHeap(Type,VariableName,Process) \ Type VariableName[1] -# define DeclareTmpHeapNoproc(VariableName,Size) \ +#define DeclareTmpHeapNoproc(VariableName,Size) \ Eterm VariableName[Size] -# define UseTmpHeap(Size,Proc) /* Nothing */ -# define UnUseTmpHeap(Size,Proc) /* Nothing */ -# define UseTmpHeapNoproc(Size) /* Nothing */ -# define UnUseTmpHeapNoproc(Size) /* Nothing */ -#endif /* HEAP_ON_C_STACK */ +#define UseTmpHeap(Size,Proc) /* Nothing */ +#define UnUseTmpHeap(Size,Proc) /* Nothing */ +#define UseTmpHeapNoproc(Size) /* Nothing */ +#define UnUseTmpHeapNoproc(Size) /* Nothing */ ERTS_GLB_INLINE void dtrace_pid_str(Eterm pid, char *process_buf); ERTS_GLB_INLINE void dtrace_proc_str(Process *process, char *process_buf); @@ -1290,6 +1564,7 @@ dtrace_fun_decode(Process *process, erts_snprintf(mfa_buf, DTRACE_TERM_BUF_SIZE, "%T:%T/%d", module, function, arity); } + #endif /* #if ERTS_GLB_INLINE_INCL_FUNC_DEF */ #endif /* !__GLOBAL_H__ */ diff --git a/erts/emulator/beam/hash.c b/erts/emulator/beam/hash.c index afaf32f8ce..5a0b93f693 100644 --- a/erts/emulator/beam/hash.c +++ b/erts/emulator/beam/hash.c @@ -3,16 +3,17 @@ * * Copyright Ericsson AB 1996-2009. All Rights Reserved. * - * The contents of this file are subject to the Erlang Public License, - * Version 1.1, (the "License"); you may not use this file except in - * compliance with the License. You should have received a copy of the - * Erlang Public License along with this software. If not, it can be - * retrieved online at http://www.erlang.org/. - * - * Software distributed under the License is distributed on an "AS IS" - * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See - * the License for the specific language governing rights and limitations - * under the License. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. * * %CopyrightEnd% */ @@ -26,8 +27,6 @@ #endif #include "sys.h" -#include "erl_vm.h" -#include "global.h" #include "hash.h" /* @@ -65,6 +64,7 @@ void hash_get_info(HashInfo *hi, Hash *h) int i; int max_depth = 0; int objects = 0; + int used = 0; for (i = 0; i < size; i++) { int depth = 0; @@ -75,14 +75,18 @@ void hash_get_info(HashInfo *hi, Hash *h) depth++; b = b->next; } - if (depth > max_depth) - max_depth = depth; + if (depth) { + used++; + if (depth > max_depth) + max_depth = depth; + } } + ASSERT(objects == h->nobjs); hi->name = h->name; hi->size = h->size; - hi->used = h->used; - hi->objs = objects; + hi->used = used; + hi->objs = h->nobjs; hi->depth = max_depth; } @@ -97,11 +101,11 @@ void hash_info(int to, void *arg, Hash* h) hash_get_info(&hi, h); - erts_print(to, arg, "=hash_table:%s\n", hi.name); - erts_print(to, arg, "size: %d\n", hi.size); - erts_print(to, arg, "used: %d\n", hi.used); - erts_print(to, arg, "objs: %d\n", hi.objs); - erts_print(to, arg, "depth: %d\n", hi.depth); + h->fun.meta_print(to, arg, "=hash_table:%s\n", hi.name); + h->fun.meta_print(to, arg, "size: %d\n", hi.size); + h->fun.meta_print(to, arg, "used: %d\n", hi.used); + h->fun.meta_print(to, arg, "objs: %d\n", hi.objs); + h->fun.meta_print(to, arg, "depth: %d\n", hi.depth); } @@ -118,47 +122,56 @@ hash_table_sz(Hash *h) } +static ERTS_INLINE void set_thresholds(Hash* h) +{ + h->grow_threshold = (8*h->size)/5; /* grow at 160% load */ + if (h->size_ix > h->min_size_ix) + h->shrink_threshold = h->size / 5; /* shrink at 20% load */ + else + h->shrink_threshold = -1; /* never shrink below inital size */ +} + /* ** init a pre allocated or static hash structure ** and allocate buckets. */ -Hash* hash_init(ErtsAlcType_t type, Hash* h, char* name, int size, HashFunctions fun) +Hash* hash_init(int type, Hash* h, char* name, int size, HashFunctions fun) { int sz; int ix = 0; - h->type = type; + h->meta_alloc_type = type; while (h_size_table[ix] != -1 && h_size_table[ix] < size) ix++; if (h_size_table[ix] == -1) - erl_exit(1, "panic: too large hash table size (%d)\n", size); + return NULL; size = h_size_table[ix]; sz = size*sizeof(HashBucket*); - h->bucket = (HashBucket**) erts_alloc(h->type, sz); + h->bucket = (HashBucket**) fun.meta_alloc(h->meta_alloc_type, sz); sys_memzero(h->bucket, sz); h->is_allocated = 0; h->name = name; h->fun = fun; h->size = size; - h->size20percent = h->size/5; - h->size80percent = (4*h->size)/5; - h->ix = ix; - h->used = 0; + h->size_ix = ix; + h->min_size_ix = ix; + h->nobjs = 0; + set_thresholds(h); return h; } /* ** Create a new hash table */ -Hash* hash_new(ErtsAlcType_t type, char* name, int size, HashFunctions fun) +Hash* hash_new(int type, char* name, int size, HashFunctions fun) { Hash* h; - h = erts_alloc(type, sizeof(Hash)); + h = fun.meta_alloc(type, sizeof(Hash)); h = hash_init(type, h, name, size, fun); h->is_allocated = 1; @@ -182,9 +195,9 @@ void hash_delete(Hash* h) b = b_next; } } - erts_free(h->type, h->bucket); + h->fun.meta_free(h->meta_alloc_type, h->bucket); if (h->is_allocated) - erts_free(h->type, (void*) h); + h->fun.meta_free(h->meta_alloc_type, (void*) h); } /* @@ -198,39 +211,34 @@ static void rehash(Hash* h, int grow) int i; if (grow) { - if ((h_size_table[h->ix+1]) == -1) + if ((h_size_table[h->size_ix+1]) == -1) return; - h->ix++; + h->size_ix++; } else { - if (h->ix == 0) + if (h->size_ix == 0) return; - h->ix--; + h->size_ix--; } - h->size = h_size_table[h->ix]; - h->size20percent = h->size/5; - h->size80percent = (4*h->size)/5; + h->size = h_size_table[h->size_ix]; sz = h->size*sizeof(HashBucket*); - new_bucket = (HashBucket **) erts_alloc(h->type, sz); + new_bucket = (HashBucket **) h->fun.meta_alloc(h->meta_alloc_type, sz); sys_memzero(new_bucket, sz); - h->used = 0; - for (i = 0; i < old_size; i++) { HashBucket* b = h->bucket[i]; while (b != (HashBucket*) 0) { HashBucket* b_next = b->next; int ix = b->hvalue % h->size; - if (new_bucket[ix] == NULL) - h->used++; b->next = new_bucket[ix]; new_bucket[ix] = b; b = b_next; } } - erts_free(h->type, (void *) h->bucket); + h->fun.meta_free(h->meta_alloc_type, (void *) h->bucket); h->bucket = new_bucket; + set_thresholds(h); } /* @@ -267,68 +275,15 @@ void* hash_put(Hash* h, void* tmpl) } b = (HashBucket*) h->fun.alloc(tmpl); - if (h->bucket[ix] == NULL) - h->used++; - b->hvalue = hval; b->next = h->bucket[ix]; h->bucket[ix] = b; - if (h->used > h->size80percent) /* rehash at 80% */ + if (++h->nobjs > h->grow_threshold) rehash(h, 1); return (void*) b; } -static void -hash_insert_entry(Hash* h, HashBucket* entry) -{ - HashValue hval = entry->hvalue; - int ix = hval % h->size; - HashBucket* b = h->bucket[ix]; - - while (b != (HashBucket*) 0) { - if ((b->hvalue == hval) && (h->fun.cmp((void*)entry, (void*)b) == 0)) { - abort(); /* Should not happen */ - } - b = b->next; - } - - if (h->bucket[ix] == NULL) - h->used++; - - entry->next = h->bucket[ix]; - h->bucket[ix] = entry; - - if (h->used > h->size80percent) /* rehash at 80% */ - rehash(h, 1); -} - - -/* - * Move all entries in src into dst; empty src. - * Entries in src must not exist in dst. - */ -void -erts_hash_merge(Hash* src, Hash* dst) -{ - int limit = src->size; - HashBucket** bucket = src->bucket; - int i; - - src->used = 0; - for (i = 0; i < limit; i++) { - HashBucket* b = bucket[i]; - HashBucket* next; - - bucket[i] = NULL; - while (b) { - next = b->next; - hash_insert_entry(dst, b); - b = next; - } - } -} - /* ** Erase hash entry return template if erased ** return 0 if not erased @@ -347,9 +302,7 @@ void* hash_erase(Hash* h, void* tmpl) else h->bucket[ix] = b->next; h->fun.free((void*)b); - if (h->bucket[ix] == NULL) - h->used--; - if (h->used < h->size20percent) /* rehash at 20% */ + if (--h->nobjs < h->shrink_threshold) rehash(h, 0); return tmpl; } @@ -380,9 +333,7 @@ hash_remove(Hash *h, void *tmpl) prev->next = b->next; else h->bucket[ix] = b->next; - if (h->bucket[ix] == NULL) - h->used--; - if (h->used < h->size20percent) /* rehash at 20% */ + if (--h->nobjs < h->shrink_threshold) rehash(h, 0); return (void *) b; } diff --git a/erts/emulator/beam/hash.h b/erts/emulator/beam/hash.h index 6dd66fc9b3..e94aaa0a84 100644 --- a/erts/emulator/beam/hash.h +++ b/erts/emulator/beam/hash.h @@ -3,16 +3,17 @@ * * Copyright Ericsson AB 1996-2009. All Rights Reserved. * - * The contents of this file are subject to the Erlang Public License, - * Version 1.1, (the "License"); you may not use this file except in - * compliance with the License. You should have received a copy of the - * Erlang Public License along with this software. If not, it can be - * retrieved online at http://www.erlang.org/. - * - * Software distributed under the License is distributed on an "AS IS" - * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See - * the License for the specific language governing rights and limitations - * under the License. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. * * %CopyrightEnd% */ @@ -28,14 +29,17 @@ #include "sys.h" #endif -#include "erl_alloc.h" - typedef unsigned long HashValue; +typedef struct hash Hash; typedef int (*HCMP_FUN)(void*, void*); typedef HashValue (*H_FUN)(void*); typedef void* (*HALLOC_FUN)(void*); typedef void (*HFREE_FUN)(void*); +/* Meta functions */ +typedef void* (*HMALLOC_FUN)(int,size_t); +typedef void (*HMFREE_FUN)(int,void*); +typedef int (*HMPRINT_FUN)(int,void*,char*, ...); /* ** This bucket must be placed in top of @@ -54,6 +58,9 @@ typedef struct hash_functions HCMP_FUN cmp; HALLOC_FUN alloc; HFREE_FUN free; + HMALLOC_FUN meta_alloc; + HMFREE_FUN meta_free; + HMPRINT_FUN meta_print; } HashFunctions; typedef struct { @@ -64,22 +71,23 @@ typedef struct { int depth; } HashInfo; -typedef struct hash +struct hash { HashFunctions fun; /* Function block */ int is_allocated; /* 0 iff hash structure is on stack or is static */ - ErtsAlcType_t type; + int meta_alloc_type; /* argument to pass to meta_alloc and meta_free */ char* name; /* Table name (static string, for debugging) */ int size; /* Number of slots */ - int size20percent; /* 20 percent of number of slots */ - int size80percent; /* 80 percent of number of slots */ - int ix; /* Size index in size table */ - int used; /* Number of slots used */ + int shrink_threshold; + int grow_threshold; + int size_ix; /* Size index in size table */ + int min_size_ix; /* Never shrink table smaller than this */ + int nobjs; /* Number of objects in table */ HashBucket** bucket; /* Vector of bucket pointers (objects) */ -} Hash; +}; -Hash* hash_new(ErtsAlcType_t, char*, int, HashFunctions); -Hash* hash_init(ErtsAlcType_t, Hash*, char*, int, HashFunctions); +Hash* hash_new(int, char*, int, HashFunctions); +Hash* hash_init(int, Hash*, char*, int, HashFunctions); void hash_delete(Hash*); void hash_get_info(HashInfo*, Hash*); @@ -92,6 +100,4 @@ void* hash_erase(Hash*, void*); void* hash_remove(Hash*, void*); void hash_foreach(Hash*, void (*func)(void *, void *), void *); -void erts_hash_merge(Hash* src, Hash* dst); - #endif diff --git a/erts/emulator/beam/index.c b/erts/emulator/beam/index.c index 79c3ecf1b3..5f6ea14732 100644 --- a/erts/emulator/beam/index.c +++ b/erts/emulator/beam/index.c @@ -3,16 +3,17 @@ * * Copyright Ericsson AB 1996-2012. All Rights Reserved. * - * The contents of this file are subject to the Erlang Public License, - * Version 1.1, (the "License"); you may not use this file except in - * compliance with the License. You should have received a copy of the - * Erlang Public License along with this software. If not, it can be - * retrieved online at http://www.erlang.org/. - * - * Software distributed under the License is distributed on an "AS IS" - * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See - * the License for the specific language governing rights and limitations - * under the License. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. * * %CopyrightEnd% */ @@ -83,7 +84,7 @@ index_put_entry(IndexTable* t, void* tmpl) Uint sz; if (ix >= t->limit) { /* A core dump is unnecessary */ - erl_exit(ERTS_DUMP_EXIT, "no more index entries in %s (max=%d)\n", + erts_exit(ERTS_DUMP_EXIT, "no more index entries in %s (max=%d)\n", t->htable.name, t->limit); } sz = INDEX_PAGE_SIZE*sizeof(IndexSlot*); @@ -122,7 +123,7 @@ void erts_index_merge(Hash* src, IndexTable* dst) ix = dst->entries++; if (ix >= dst->size) { if (ix >= dst->limit) { - erl_exit(1, "no more index entries in %s (max=%d)\n", + erts_exit(ERTS_ERROR_EXIT, "no more index entries in %s (max=%d)\n", dst->htable.name, dst->limit); } sz = INDEX_PAGE_SIZE*sizeof(IndexSlot*); diff --git a/erts/emulator/beam/index.h b/erts/emulator/beam/index.h index 537bc11056..99b2bdfab0 100644 --- a/erts/emulator/beam/index.h +++ b/erts/emulator/beam/index.h @@ -3,16 +3,17 @@ * * Copyright Ericsson AB 1996-2013. All Rights Reserved. * - * The contents of this file are subject to the Erlang Public License, - * Version 1.1, (the "License"); you may not use this file except in - * compliance with the License. You should have received a copy of the - * Erlang Public License along with this software. If not, it can be - * retrieved online at http://www.erlang.org/. - * - * Software distributed under the License is distributed on an "AS IS" - * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See - * the License for the specific language governing rights and limitations - * under the License. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. * * %CopyrightEnd% */ @@ -29,6 +30,10 @@ #include "hash.h" #endif +#ifndef ERL_ALLOC_H__ +#include "erl_alloc.h" +#endif + typedef struct index_slot { HashBucket bucket; diff --git a/erts/emulator/beam/io.c b/erts/emulator/beam/io.c index edf4a28784..29f28cc9dc 100644 --- a/erts/emulator/beam/io.c +++ b/erts/emulator/beam/io.c @@ -3,16 +3,17 @@ * * Copyright Ericsson AB 1996-2014. All Rights Reserved. * - * The contents of this file are subject to the Erlang Public License, - * Version 1.1, (the "License"); you may not use this file except in - * compliance with the License. You should have received a copy of the - * Erlang Public License along with this software. If not, it can be - * retrieved online at http://www.erlang.org/. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Software distributed under the License is distributed on an "AS IS" - * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See - * the License for the specific language governing rights and limitations - * under the License. + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. * * %CopyrightEnd% */ @@ -47,12 +48,16 @@ #include "external.h" #include "dtrace-wrapper.h" #include "erl_map.h" +#include "erl_bif_unique.h" +#include "erl_hl_timer.h" +#include "erl_time.h" extern ErlDrvEntry fd_driver_entry; -#ifndef __OSE__ extern ErlDrvEntry vanilla_driver_entry; -#endif extern ErlDrvEntry spawn_driver_entry; +#ifndef __WIN32__ +extern ErlDrvEntry forker_driver_entry; +#endif extern ErlDrvEntry *driver_tab[]; /* table of static drivers, only used during initialization */ erts_driver_t *driver_list; /* List of all drivers, static and dynamic. */ @@ -63,8 +68,6 @@ static erts_smp_tsd_key_t driver_list_last_error_key; /* Save last DDLL error o per thread basis (for BC interfaces) */ ErtsPTab erts_port erts_align_attribute(ERTS_CACHE_LINE_SIZE); /* The port table */ -erts_smp_atomic_t erts_bytes_out; /* No bytes sent out of the system */ -erts_smp_atomic_t erts_bytes_in; /* No bytes gotten into the system */ const ErlDrvTermData driver_term_nil = (ErlDrvTermData)NIL; @@ -72,16 +75,23 @@ const Port erts_invalid_port = {{ERTS_INVALID_PORT}}; erts_driver_t vanilla_driver; erts_driver_t spawn_driver; +#ifndef __WIN32__ +erts_driver_t forker_driver; +#endif erts_driver_t fd_driver; int erts_port_synchronous_ops = 0; int erts_port_schedule_all_ops = 0; int erts_port_parallelism = 0; +static erts_atomic64_t bytes_in; +static erts_atomic64_t bytes_out; + static void deliver_result(Eterm sender, Eterm pid, Eterm res); static int init_driver(erts_driver_t *, ErlDrvEntry *, DE_Handle *); static void terminate_port(Port *p); static void pdl_init(void); +static int driver_failure_term(ErlDrvPort ix, Eterm term, int eof); #ifdef ERTS_SMP static void driver_monitor_lock_pdl(Port *p); static void driver_monitor_unlock_pdl(Port *p); @@ -303,12 +313,9 @@ static Port *create_port(char *name, size_t port_size, busy_port_queue_size, size; erts_aint32_t state = ERTS_PORT_SFLG_CONNECTED; erts_aint32_t x_pts_flgs = 0; -#ifdef DEBUG - /* Make sure the debug flags survives until port is freed */ - state |= ERTS_PORT_SFLG_PORT_DEBUG; -#endif #ifdef ERTS_SMP + ErtsRunQueue *runq; if (!driver_lock) { /* Align size for mutex following port struct */ port_size = size = ERTS_ALC_DATA_ALIGN_SIZE(sizeof(Port)); @@ -318,6 +325,12 @@ static Port *create_port(char *name, #endif port_size = size = ERTS_ALC_DATA_ALIGN_SIZE(sizeof(Port)); +#ifdef DEBUG + /* Make sure the debug flags survives until port is freed */ + state |= ERTS_PORT_SFLG_PORT_DEBUG; +#endif + + busy_port_queue_size = ((driver->flags & ERL_DRV_FLAG_NO_BUSY_MSGQ) ? 0 @@ -353,8 +366,12 @@ static Port *create_port(char *name, p += sizeof(erts_mtx_t); state |= ERTS_PORT_SFLG_PORT_SPECIFIC_LOCK; } - erts_smp_atomic_set_nob(&prt->run_queue, - (erts_aint_t) erts_get_runq_current(NULL)); + if (erts_get_scheduler_data()) + runq = erts_get_runq_current(NULL); + else + runq = ERTS_RUNQ_IX(0); + erts_smp_atomic_set_nob(&prt->run_queue, (erts_aint_t) runq); + prt->xports = NULL; #else erts_atomic32_init_nob(&prt->refc, 1); @@ -378,20 +395,17 @@ static Port *create_port(char *name, prt->dist_entry = NULL; ERTS_PORT_INIT_CONNECTED(prt, pid); prt->common.u.alive.reg = NULL; -#ifdef ERTS_SMP - prt->common.u.alive.ptimer = NULL; -#else - sys_memset(&prt->common.u.alive.tm, 0, sizeof(ErlTimer)); -#endif + ERTS_PTMR_INIT(prt); erts_port_task_handle_init(&prt->timeout_task); - prt->psd = NULL; + erts_smp_atomic_init_nob(&prt->psd, (erts_aint_t) NULL); + prt->async_open_port = NULL; prt->drv_data = (SWord) 0; prt->os_pid = -1; /* Set default tracing */ erts_get_default_tracing(&ERTS_TRACE_FLAGS(prt), &ERTS_TRACER_PROC(prt)); - ASSERT(((char *) prt) == ((char *) &prt->common)); + ERTS_CT_ASSERT(offsetof(Port,common) == 0); #if !ERTS_PORT_INIT_INSTR_NEED_ID /* @@ -462,14 +476,15 @@ erts_port_free(Port *prt) | ERTS_PORT_SFLG_FREE)); ASSERT(state & ERTS_PORT_SFLG_PORT_DEBUG); -#ifdef ERTS_SMP - ERTS_LC_ASSERT(erts_atomic32_read_nob(&prt->common.refc) == 0); -#else - ERTS_LC_ASSERT(erts_atomic32_read_nob(&prt->refc) == 0); -#endif + ERTS_LC_ASSERT(erts_atomic_read_nob(&prt->common.refc.atmc) == 0); erts_port_task_fini_sched(&prt->sched); + if (prt->async_open_port) { + erts_free(ERTS_ALC_T_PRTSD, prt->async_open_port); + prt->async_open_port = NULL; + } + #ifdef ERTS_SMP ASSERT(prt->lock); if (state & ERTS_PORT_SFLG_PORT_SPECIFIC_LOCK) @@ -691,6 +706,7 @@ erts_open_driver(erts_driver_t* driver, /* Pointer to driver. */ error_number = error_type = 0; if (driver->start) { + ERTS_MSACC_PUSH_STATE_M(); if (IS_TRACED_FL(port, F_TRACE_SCHED_PORTS)) { trace_sched_ports_where(port, am_in, am_start); } @@ -701,6 +717,7 @@ erts_open_driver(erts_driver_t* driver, /* Pointer to driver. */ DTRACE3(driver_start, process_str, driver->name, port_str); } #endif + ERTS_MSACC_SET_STATE_CACHED_M(ERTS_MSACC_STATE_PORT); fpe_was_unmasked = erts_block_fpe(); drv_data = (*driver->start)(ERTS_Port2ErlDrvPort(port), name, opts); if (((SWord) drv_data) == -1) @@ -720,6 +737,7 @@ erts_open_driver(erts_driver_t* driver, /* Pointer to driver. */ } erts_unblock_fpe(fpe_was_unmasked); + ERTS_MSACC_POP_STATE_M(); port->caller = NIL; if (IS_TRACED_FL(port, F_TRACE_SCHED_PORTS)) { trace_sched_ports_where(port, am_out, am_start); @@ -735,11 +753,7 @@ erts_open_driver(erts_driver_t* driver, /* Pointer to driver. */ /* * Must clean up the port. */ -#ifdef ERTS_SMP - erts_cancel_smp_ptimer(port->common.u.alive.ptimer); -#else - erts_cancel_timer(&(port->common.u.alive.tm)); -#endif + erts_cancel_port_timer(port); stopq(port); if (port->linebuf != NULL) { erts_free(ERTS_ALC_T_LINEBUF, @@ -1113,7 +1127,7 @@ io_list_vec_len(Eterm obj, int* vsize, Uint* csize, Uint p_v_size = 0; Uint p_c_size = 0; Uint p_in_clist = 0; - Uint total; /* Uint due to halfword emulator */ + Uint total; goto L_jump_start; /* avoid a push */ @@ -1218,9 +1232,10 @@ typedef struct { static ERTS_INLINE ErtsTryImmDrvCallResult try_imm_drv_call(ErtsTryImmDrvCallState *sp) { + unsigned int prof_runnable_ports; ErtsTryImmDrvCallResult res; int reds_left_in; - erts_aint32_t invalid_state, invalid_sched_flags; + erts_aint32_t act, exp, invalid_state, invalid_sched_flags; Port *prt = sp->port; Process *c_p = sp->c_p; @@ -1247,18 +1262,39 @@ try_imm_drv_call(ErtsTryImmDrvCallState *sp) goto locked_fail; } - sp->sched_flags = erts_smp_atomic32_read_nob(&prt->sched.flags); - if (sp->sched_flags & invalid_sched_flags) { - res = ERTS_TRY_IMM_DRV_CALL_INVALID_SCHED_FLAGS; - goto locked_fail; - } + prof_runnable_ports = erts_system_profile_flags.runnable_ports; + if (prof_runnable_ports) + erts_port_task_sched_lock(&prt->sched); + act = erts_smp_atomic32_read_nob(&prt->sched.flags); + + do { + erts_aint32_t new; + + if (act & invalid_sched_flags) { + res = ERTS_TRY_IMM_DRV_CALL_INVALID_SCHED_FLAGS; + sp->sched_flags = act; + goto locked_fail; + } + exp = act; + new = act | ERTS_PTS_FLG_EXEC_IMM; + act = erts_smp_atomic32_cmpxchg_mb(&prt->sched.flags, new, exp); + } while (act != exp); + + sp->sched_flags = act; if (!c_p) reds_left_in = CONTEXT_REDS/10; else { if (IS_TRACED_FL(c_p, F_TRACE_SCHED_PROCS)) trace_virtual_sched(c_p, am_out); + /* + * No status lock held while sending runnable + * proc trace messages. It is however not needed + * in this case, since only this thread can send + * such messages for this process until the process + * has been scheduled out. + */ if (erts_system_profile_flags.runnable_procs && erts_system_profile_flags.exclusive) profile_runnable_proc(c_p, am_inactive); @@ -1273,11 +1309,14 @@ try_imm_drv_call(ErtsTryImmDrvCallState *sp) ERTS_SMP_CHK_NO_PROC_LOCKS; - if (IS_TRACED_FL(prt, F_TRACE_SCHED_PORTS)) - trace_sched_ports_where(prt, am_in, sp->port_op); - if (erts_system_profile_flags.runnable_ports - && !erts_port_is_scheduled(prt)) - profile_runnable_port(prt, am_active); + if (prof_runnable_ports | IS_TRACED_FL(prt, F_TRACE_SCHED_PORTS)) { + if (prof_runnable_ports && !(act & (ERTS_PTS_FLG_IN_RUNQ|ERTS_PTS_FLG_EXEC))) + profile_runnable_port(prt, am_active); + if (IS_TRACED_FL(prt, F_TRACE_SCHED_PORTS)) + trace_sched_ports_where(prt, am_in, sp->port_op); + if (prof_runnable_ports) + erts_port_task_sched_unlock(&prt->sched); + } sp->fpe_was_unmasked = erts_block_fpe(); @@ -1294,17 +1333,31 @@ finalize_imm_drv_call(ErtsTryImmDrvCallState *sp) int reds; Port *prt = sp->port; Process *c_p = sp->c_p; + erts_aint32_t act; + unsigned int prof_runnable_ports; reds = prt->reds; reds += erts_port_driver_callback_epilogue(prt, NULL); erts_unblock_fpe(sp->fpe_was_unmasked); - if (IS_TRACED_FL(prt, F_TRACE_SCHED_PORTS)) - trace_sched_ports_where(prt, am_out, sp->port_op); - if (erts_system_profile_flags.runnable_ports - && !erts_port_is_scheduled(prt)) - profile_runnable_port(prt, am_inactive); + prof_runnable_ports = erts_system_profile_flags.runnable_ports; + if (prof_runnable_ports) + erts_port_task_sched_lock(&prt->sched); + + act = erts_smp_atomic32_read_band_mb(&prt->sched.flags, + ~ERTS_PTS_FLG_EXEC_IMM); + ERTS_SMP_LC_ASSERT(act & ERTS_PTS_FLG_EXEC_IMM); + + if (prof_runnable_ports | IS_TRACED_FL(prt, F_TRACE_SCHED_PORTS)) { + if (IS_TRACED_FL(prt, F_TRACE_SCHED_PORTS)) + trace_sched_ports_where(prt, am_out, sp->port_op); + if (prof_runnable_ports) { + if (!(act & (ERTS_PTS_FLG_IN_RUNQ|ERTS_PTS_FLG_EXEC))) + profile_runnable_port(prt, am_inactive); + erts_port_task_sched_unlock(&prt->sched); + } + } erts_port_release(prt); @@ -1319,6 +1372,13 @@ finalize_imm_drv_call(ErtsTryImmDrvCallState *sp) if (IS_TRACED_FL(c_p, F_TRACE_SCHED_PROCS)) trace_virtual_sched(c_p, am_in); + /* + * No status lock held while sending runnable + * proc trace messages. It is however not needed + * in this case, since only this thread can send + * such messages for this process until the process + * has been scheduled out. + */ if (erts_system_profile_flags.runnable_procs && erts_system_profile_flags.exclusive) profile_runnable_proc(c_p, am_active); @@ -1359,39 +1419,22 @@ finalize_force_imm_drv_call(ErtsTryImmDrvCallState *sp) static ERTS_INLINE void queue_port_sched_op_reply(Process *rp, ErtsProcLocks *rp_locksp, - Eterm *hp_start, - Eterm *hp, - Uint h_size, - ErlHeapFragment* bp, + ErtsHeapFactory* factory, Uint32 *ref_num, Eterm msg) { - Eterm ref = make_internal_ref(hp); + Eterm* hp = erts_produce_heap(factory, ERTS_QUEUE_PORT_SCHED_OP_REPLY_SIZE, 0); + Eterm ref; + + ref= make_internal_ref(hp); write_ref_thing(hp, ref_num[0], ref_num[1], ref_num[2]); hp += REF_THING_SIZE; msg = TUPLE2(hp, ref, msg); - hp += 3; - if (!bp) { - HRelease(rp, hp_start + h_size, hp); - } - else { - Uint used_h_size = hp - hp_start; - ASSERT(h_size >= used_h_size); - if (h_size > used_h_size) - bp = erts_resize_message_buffer(bp, used_h_size, &msg, 1); - } + erts_factory_trim_and_close(factory, &msg, 1); - erts_queue_message(rp, - rp_locksp, - bp, - msg, - NIL -#ifdef USE_VM_PROBES - , NIL -#endif - ); + erts_queue_message(rp, rp_locksp, factory->message, msg, NIL); } static void @@ -1399,11 +1442,9 @@ port_sched_op_reply(Eterm to, Uint32 *ref_num, Eterm msg) { Process *rp = erts_proc_lookup_raw(to); if (rp) { - ErlOffHeap *ohp; - ErlHeapFragment* bp; + ErtsHeapFactory factory; Eterm msg_copy; Uint hsz, msg_sz; - Eterm *hp, *hp_start; ErtsProcLocks rp_locks = 0; hsz = ERTS_QUEUE_PORT_SCHED_OP_REPLY_SIZE; @@ -1414,22 +1455,17 @@ port_sched_op_reply(Eterm to, Uint32 *ref_num, Eterm msg) hsz += msg_sz; } - hp_start = hp = erts_alloc_message_heap(hsz, - &bp, - &ohp, - rp, - &rp_locks); - if (is_immed(msg)) - msg_copy = msg; - else - msg_copy = copy_struct(msg, msg_sz, &hp, ohp); + (void) erts_factory_message_create(&factory, rp, + &rp_locks, hsz); + msg_copy = (is_immed(msg) + ? msg + : copy_struct(msg, msg_sz, + &factory.hp, + factory.off_heap)); queue_port_sched_op_reply(rp, &rp_locks, - hp_start, - hp, - hsz, - bp, + &factory, ref_num, msg_copy); @@ -1506,20 +1542,49 @@ erts_schedule_proc2port_signal(Process *c_p, erts_smp_proc_lock(c_p, ERTS_PROC_LOCK_MAIN); if (sched_res != 0) { - if (refp) + if (refp) { + /* + * We need to restore the message queue save + * pointer to the beginning of the message queue + * since the caller now wont wait for a message + * containing the reference created above... + */ + ASSERT(c_p); + erts_smp_proc_lock(c_p, ERTS_PROC_LOCKS_MSG_RECEIVE); + JOIN_MESSAGE(c_p); + erts_smp_proc_unlock(c_p, ERTS_PROC_LOCKS_MSG_RECEIVE); *refp = NIL; + } return ERTS_PORT_OP_DROPPED; } return ERTS_PORT_OP_SCHEDULED; } -static ERTS_INLINE void -send_badsig(Port *prt) +static int +erts_schedule_port2port_signal(Eterm port_num, ErtsProc2PortSigData *sigdp, + int task_flags, + ErtsProc2PortSigCallback callback) { + Port *prt = erts_port_lookup_raw(port_num); + + if (!prt) + return -1; + + sigdp->caller = ERTS_INVALID_PID; + + return erts_port_task_schedule(prt->common.id, + NULL, + ERTS_PORT_TASK_PROC_SIG, + sigdp, + callback, + task_flags); +} + +static ERTS_INLINE void +send_badsig(Port *prt) { ErtsProcLocks rp_locks = ERTS_PROC_LOCKS_XSIG_SEND; Process* rp; Eterm connected = ERTS_PORT_GET_CONNECTED(prt); - ERTS_SMP_CHK_NO_PROC_LOCKS; ERTS_LC_ASSERT(erts_get_scheduler_id()); @@ -1539,15 +1604,13 @@ send_badsig(Port *prt) 0); if (rp_locks) erts_smp_proc_unlock(rp, rp_locks); - } -} + } /* exit sent */ +} /* send_badsig */ static void -badsig_received(int bang_op, - Port *prt, +badsig_received(int bang_op, Port *prt, erts_aint32_t state, - int bad_output_value) -{ + int bad_output_value) { /* * if (bang_op) * we are part of a "Prt ! Something" operation @@ -1563,12 +1626,12 @@ badsig_received(int bang_op, } if (bang_op) send_badsig(prt); - } -} + } /* not invalid */ +} /* behaved accordingly */ static int -port_badsig(Port *prt, erts_aint32_t state, int op, ErtsProc2PortSigData *sigdp) -{ +port_badsig(Port *prt, erts_aint32_t state, int op, + ErtsProc2PortSigData *sigdp) { if (op == ERTS_PROC2PORT_SIG_EXEC) badsig_received(sigdp->flags & ERTS_P2P_SIG_DATA_FLG_BANG_OP, prt, @@ -1577,16 +1640,14 @@ port_badsig(Port *prt, erts_aint32_t state, int op, ErtsProc2PortSigData *sigdp) if (sigdp->flags & ERTS_P2P_SIG_DATA_FLG_REPLY) port_sched_op_reply(sigdp->caller, sigdp->ref, am_badarg); return ERTS_PORT_REDS_BADSIG; -} - - -/* - * bad_port_signal() will +} /* port_badsig */ +/* bad_port_signal() will * - preserve signal order of signals. * - send a 'badsig' exit signal to connected process if 'from' is an * internal pid and the port is alive when the bad signal reaches * it. */ + static ErtsPortOpResult bad_port_signal(Process *c_p, int flags, @@ -1661,7 +1722,9 @@ call_driver_outputv(int bang_op, if (bang_op && from != ERTS_PORT_GET_CONNECTED(prt)) send_badsig(prt); else { + ErtsSchedulerData *esdp = erts_get_scheduler_data(); ErlDrvSizeT size = evp->size; + ERTS_MSACC_PUSH_AND_SET_STATE_M(ERTS_MSACC_STATE_PORT); ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(prt) || ERTS_IS_CRASH_DUMPING); @@ -1678,7 +1741,12 @@ call_driver_outputv(int bang_op, prt->caller = NIL; prt->bytes_out += size; - erts_smp_atomic_add_nob(&erts_bytes_out, size); + if (esdp) + esdp->io.out += (Uint64) size; + else + erts_atomic64_add_nob(&bytes_out, (erts_aint64_t) size); + + ERTS_MSACC_POP_STATE_M(); } } @@ -1692,7 +1760,6 @@ cleanup_scheduled_outputv(ErlIOVec *ev, ErlDrvBinary *cbinp) driver_free_binary(ev->binv[i]); if (cbinp) driver_free_binary(cbinp); - erts_free(ERTS_ALC_T_DRV_CMD_DATA, ev); } static int @@ -1758,7 +1825,8 @@ call_driver_output(int bang_op, if (bang_op && from != ERTS_PORT_GET_CONNECTED(prt)) send_badsig(prt); else { - + ErtsSchedulerData *esdp = erts_get_scheduler_data(); + ERTS_MSACC_PUSH_AND_SET_STATE_M(ERTS_MSACC_STATE_PORT); ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(prt) || ERTS_IS_CRASH_DUMPING); @@ -1774,7 +1842,12 @@ call_driver_output(int bang_op, prt->caller = NIL; prt->bytes_out += size; - erts_smp_atomic_add_nob(&erts_bytes_out, size); + if (esdp) + esdp->io.out += (Uint64) size; + else + erts_atomic64_add_nob(&bytes_out, (erts_aint64_t) size); + + ERTS_MSACC_POP_STATE_M(); } } @@ -1824,6 +1897,188 @@ port_sig_output(Port *prt, erts_aint32_t state, int op, ErtsProc2PortSigData *si return ERTS_PORT_REDS_CMD_OUTPUT; } + +/* + * This erts_port_output will always create a port task. + * The call is treated as a port_command call, i.e. no + * badsig i generated if the input in invalid. However + * an error_logger message is generated. + */ +int +erts_port_output_async(Port *prt, Eterm from, Eterm list) +{ + + ErtsPortOpResult res; + ErtsProc2PortSigData *sigdp; + erts_driver_t *drv = prt->drv_ptr; + size_t size; + int task_flags; + ErtsProc2PortSigCallback port_sig_callback; + ErlDrvBinary *cbin = NULL; + ErlIOVec *evp = NULL; + char *buf = NULL; + ErtsPortTaskHandle *ns_pthp; + + if (drv->outputv) { + ErlIOVec ev; + SysIOVec* ivp; + ErlDrvBinary** bvp; + int vsize; + Uint csize; + Uint pvsize; + Uint pcsize; + size_t iov_offset, binv_offset, alloc_size; + Uint blimit = 0; + char *ptr; + int i; + + Eterm* bptr = NULL; + Uint offset; + + if (is_binary(list)) { + /* We optimize for when we get a procbin without offset */ + Eterm real_bin; + int bitoffs; + int bitsize; + ERTS_GET_REAL_BIN(list, real_bin, offset, bitoffs, bitsize); + bptr = binary_val(real_bin); + if (*bptr == HEADER_PROC_BIN && bitoffs == 0) { + size = binary_size(list); + vsize = 1; + } else + bptr = NULL; + } + + if (!bptr) { + if (io_list_vec_len(list, &vsize, &csize, &pvsize, &pcsize, &size)) + goto bad_value; + + /* To pack or not to pack (small binaries) ...? */ + if (vsize >= SMALL_WRITE_VEC) { + /* Do pack */ + vsize = pvsize + 1; + csize = pcsize; + blimit = ERL_SMALL_IO_BIN_LIMIT; + } + cbin = driver_alloc_binary(csize); + if (!cbin) + erts_alloc_enomem(ERTS_ALC_T_DRV_BINARY, ERTS_SIZEOF_Binary(csize)); + } + + + iov_offset = ERTS_ALC_DATA_ALIGN_SIZE(sizeof(ErlIOVec)); + binv_offset = iov_offset; + binv_offset += ERTS_ALC_DATA_ALIGN_SIZE((vsize+1)*sizeof(SysIOVec)); + alloc_size = binv_offset; + alloc_size += (vsize+1)*sizeof(ErlDrvBinary *); + + sigdp = erts_port_task_alloc_p2p_sig_data_extra(alloc_size, (void**)&ptr); + + evp = (ErlIOVec *) ptr; + ivp = evp->iov = (SysIOVec *) (ptr + iov_offset); + bvp = evp->binv = (ErlDrvBinary **) (ptr + binv_offset); + + ivp[0].iov_base = NULL; + ivp[0].iov_len = 0; + bvp[0] = NULL; + + if (bptr) { + ProcBin* pb = (ProcBin *) bptr; + + ivp[1].iov_base = pb->bytes+offset; + ivp[1].iov_len = size; + bvp[1] = Binary2ErlDrvBinary(pb->val); + + evp->vsize = 1; + } else { + + evp->vsize = io_list_to_vec(list, ivp+1, bvp+1, cbin, blimit); + if (evp->vsize < 0) { + if (evp != &ev) + erts_free(ERTS_ALC_T_DRV_CMD_DATA, evp); + driver_free_binary(cbin); + goto bad_value; + } + } +#if 0 + /* This assertion may say something useful, but it can + be falsified during the emulator test suites. */ + ASSERT(evp->vsize == vsize); +#endif + evp->vsize++; + evp->size = size; /* total size */ + + /* Need to increase refc on all binaries */ + for (i = 1; i < evp->vsize; i++) + if (bvp[i]) + driver_binary_inc_refc(bvp[i]); + + sigdp->flags = ERTS_P2P_SIG_TYPE_OUTPUTV; + sigdp->u.outputv.from = from; + sigdp->u.outputv.evp = evp; + sigdp->u.outputv.cbinp = cbin; + port_sig_callback = port_sig_outputv; + } else { + ErlDrvSizeT ERTS_DECLARE_DUMMY(r); + + /* + * Apperently there exist code that write 1 byte to + * much in buffer. Where it resides I don't know, but + * we can live with one byte extra allocated... + */ + + if (erts_iolist_size(list, &size)) + goto bad_value; + + buf = erts_alloc(ERTS_ALC_T_DRV_CMD_DATA, size + 1); + + r = erts_iolist_to_buf(list, buf, size); + ASSERT(ERTS_IOLIST_TO_BUF_SUCCEEDED(r)); + + sigdp = erts_port_task_alloc_p2p_sig_data(); + sigdp->flags = ERTS_P2P_SIG_TYPE_OUTPUT; + sigdp->u.output.from = from; + sigdp->u.output.bufp = buf; + sigdp->u.output.size = size; + port_sig_callback = port_sig_output; + } + sigdp->flags = 0; + ns_pthp = NULL; + task_flags = 0; + + res = erts_schedule_proc2port_signal(NULL, + prt, + ERTS_INVALID_PID, + NULL, + sigdp, + task_flags, + ns_pthp, + port_sig_callback); + + if (res != ERTS_PORT_OP_SCHEDULED) { + if (drv->outputv) + cleanup_scheduled_outputv(evp, cbin); + else + cleanup_scheduled_output(buf); + return 1; + } + return 1; + +bad_value: + + /* + * We call badsig directly here as this function is called with + * the main lock of the calling process still held. + * At the moment this operation is always not a bang_op, so + * only an error_logger message should be generated, no badsig. + */ + + badsig_received(0, prt, erts_atomic32_read_nob(&prt->state), 1); + + return 0; + +} + ErtsPortOpResult erts_port_output(Process *c_p, int flags, @@ -1833,7 +2088,7 @@ erts_port_output(Process *c_p, Eterm *refp) { ErtsPortOpResult res; - ErtsProc2PortSigData *sigdp; + ErtsProc2PortSigData *sigdp = NULL; erts_driver_t *drv = prt->drv_ptr; size_t size; int try_call; @@ -1915,10 +2170,13 @@ erts_port_output(Process *c_p, evp = &ev; } else { - char *ptr = erts_alloc((try_call - ? ERTS_ALC_T_TMP - : ERTS_ALC_T_DRV_CMD_DATA), alloc_size); - + char *ptr; + if (try_call) { + ptr = erts_alloc(ERTS_ALC_T_TMP, alloc_size); + } else { + sigdp = erts_port_task_alloc_p2p_sig_data_extra( + alloc_size, (void**)&ptr); + } evp = (ErlIOVec *) ptr; ivp = evp->iov = (SysIOVec *) (ptr + iov_offset); bvp = evp->binv = (ErlDrvBinary **) (ptr + binv_offset); @@ -1947,9 +2205,12 @@ erts_port_output(Process *c_p, bvp[0] = NULL; evp->vsize = io_list_to_vec(list, ivp+1, bvp+1, cbin, blimit); if (evp->vsize < 0) { - if (evp != &ev) - erts_free(try_call ? ERTS_ALC_T_TMP : ERTS_ALC_T_DRV_CMD_DATA, - evp); + if (evp != &ev) { + if (try_call) + erts_free(ERTS_ALC_T_TMP, evp); + else + erts_port_task_free_p2p_sig_data(sigdp); + } driver_free_binary(cbin); goto bad_value; } @@ -2001,8 +2262,10 @@ erts_port_output(Process *c_p, /* Fall through... */ case ERTS_TRY_IMM_DRV_CALL_INVALID_PORT: driver_free_binary(cbin); - if (evp != &ev) + if (evp != &ev) { + ASSERT(!sigdp); erts_free(ERTS_ALC_T_TMP, evp); + } if (try_call_res != ERTS_TRY_IMM_DRV_CALL_OK) return ERTS_PORT_OP_DROPPED; if (c_p) @@ -2013,8 +2276,10 @@ erts_port_output(Process *c_p, if (async_nosuspend && (sched_flags & (busy_flgs|ERTS_PTS_FLG_EXIT))) { driver_free_binary(cbin); - if (evp != &ev) + if (evp != &ev) { + ASSERT(!sigdp); erts_free(ERTS_ALC_T_TMP, evp); + } return ((sched_flags & ERTS_PTS_FLG_EXIT) ? ERTS_PORT_OP_DROPPED : ERTS_PORT_OP_BUSY); @@ -2029,9 +2294,16 @@ erts_port_output(Process *c_p, if (bvp[i]) driver_binary_inc_refc(bvp[i]); - new_evp = erts_alloc(ERTS_ALC_T_DRV_CMD_DATA, alloc_size); + /* The port task and iovec is allocated in the + same structure as an optimization. This + is especially important in erts_port_output_async + of when !try_call */ + ASSERT(sigdp == NULL); + sigdp = erts_port_task_alloc_p2p_sig_data_extra( + alloc_size, (void**)&new_evp); if (evp != &ev) { + /* Copy from TMP alloc to port task */ sys_memcpy((void *) new_evp, (void *) evp, alloc_size); new_evp->iov = (SysIOVec *) (((char *) new_evp) + iov_offset); @@ -2079,7 +2351,6 @@ erts_port_output(Process *c_p, evp = new_evp; } - sigdp = erts_port_task_alloc_p2p_sig_data(); sigdp->flags = ERTS_P2P_SIG_TYPE_OUTPUTV; sigdp->u.outputv.from = from; sigdp->u.outputv.evp = evp; @@ -2347,6 +2618,11 @@ erts_port_exit(Process *c_p, | ERTS_PORT_SIG_FLG_BROKEN_LINK | ERTS_PORT_SIG_FLG_FORCE_SCHED)) == 0); +#ifndef __WIN32__ + if (prt->drv_ptr == &forker_driver) + return ERTS_PORT_OP_DROPPED; +#endif + if (!(flags & ERTS_PORT_SIG_FLG_FORCE_SCHED)) { ErtsTryImmDrvCallState try_call_state = ERTS_INIT_TRY_IMM_DRV_CALL_STATE(c_p, @@ -2711,6 +2987,72 @@ erts_port_link(Process *c_p, Port *prt, Eterm to, Eterm *refp) port_sig_link); } +static void +init_ack_send_reply(Port *port, Eterm resp) +{ + + if (!is_internal_port(resp)) { + Process *rp = erts_proc_lookup_raw(port->async_open_port->to); + erts_smp_proc_lock(rp, ERTS_PROC_LOCK_LINK); + erts_remove_link(&ERTS_P_LINKS(port), port->async_open_port->to); + erts_remove_link(&ERTS_P_LINKS(rp), port->common.id); + erts_smp_proc_unlock(rp, ERTS_PROC_LOCK_LINK); + } + port_sched_op_reply(port->async_open_port->to, + port->async_open_port->ref, + resp); + + erts_free(ERTS_ALC_T_PRTSD, port->async_open_port); + port->async_open_port = NULL; +} + +void +erl_drv_init_ack(ErlDrvPort ix, ErlDrvData res) { + Port *port = erts_drvport2port(ix); + SWord err_type = (SWord)res; + Eterm resp; + + if (port == ERTS_INVALID_ERL_DRV_PORT && port->async_open_port) + return; + + if (port->async_open_port) { + switch(err_type) { + case -3: + resp = am_badarg; + break; + case -2: { + char *str = erl_errno_id(errno); + resp = erts_atom_put((byte *) str, strlen(str), + ERTS_ATOM_ENC_LATIN1, 1); + break; + } + case -1: + resp = am_einval; + break; + default: + resp = port->common.id; + break; + } + + init_ack_send_reply(port, resp); + + if (err_type == -1 || err_type == -2 || err_type == -3) + driver_failure_term(ix, am_normal, 0); + port->drv_data = err_type; + } +} + +void +erl_drv_set_os_pid(ErlDrvPort ix, ErlDrvSInt pid) { + Port *port = erts_drvport2port(ix); + + if (port == ERTS_INVALID_ERL_DRV_PORT) + return; + + port->os_pid = (SWord)pid; + +} + void erts_init_io(int port_tab_size, int port_tab_size_ignore_files, int legacy_port_tab) @@ -2721,6 +3063,9 @@ void erts_init_io(int port_tab_size, drv_list_rwmtx_opts.type = ERTS_SMP_RWMTX_TYPE_EXTREMELY_FREQUENT_READ; drv_list_rwmtx_opts.lived = ERTS_SMP_RWMTX_LONG_LIVED; + erts_atomic64_init_nob(&bytes_in, 0); + erts_atomic64_init_nob(&bytes_out, 0); + common_element_size = ERTS_ALC_DATA_ALIGN_SIZE(sizeof(Port)); common_element_size += ERTS_ALC_DATA_ALIGN_SIZE(sizeof(ErtsPortTaskBusyPortQ)); common_element_size += 10; /* name */ @@ -2759,10 +3104,8 @@ void erts_init_io(int port_tab_size, port_tab_size, common_element_size, /* Doesn't need to be excact */ "port_table", - legacy_port_tab); - - erts_smp_atomic_init_nob(&erts_bytes_out, 0); - erts_smp_atomic_init_nob(&erts_bytes_in, 0); + legacy_port_tab, + 1); sys_init_io(); @@ -2770,10 +3113,11 @@ void erts_init_io(int port_tab_size, erts_smp_rwmtx_rwlock(&erts_driver_list_lock); init_driver(&fd_driver, &fd_driver_entry, NULL); -#ifndef __OSE__ init_driver(&vanilla_driver, &vanilla_driver_entry, NULL); -#endif init_driver(&spawn_driver, &spawn_driver_entry, NULL); +#ifndef __WIN32__ + init_driver(&forker_driver, &forker_driver_entry, NULL); +#endif erts_init_static_drivers(); for (dp = driver_tab; *dp != NULL; dp++) erts_add_driver_entry(*dp, NULL, 1); @@ -2783,7 +3127,6 @@ void erts_init_io(int port_tab_size, } #if defined(ERTS_ENABLE_LOCK_COUNT) && defined(ERTS_SMP) - static ERTS_INLINE void lcnt_enable_drv_lock_count(erts_driver_t *dp, int enable) { if (dp->lock) { @@ -2823,25 +3166,29 @@ static ERTS_INLINE void lcnt_enable_port_lock_count(Port *prt, int enable) } } -void erts_lcnt_enable_io_lock_count(int enable) -{ +void erts_lcnt_enable_io_lock_count(int enable) { erts_driver_t *dp; - int i, max = erts_ptab_max(&erts_port); + int ix, max = erts_ptab_max(&erts_port); + Port *prt; - for (i = 0; i < max; i++) { - Port *prt = erts_pix2port(i); - if (prt) + for (ix = 0; ix < max; ix++) { + if ((prt = erts_pix2port(ix)) != NULL) { lcnt_enable_port_lock_count(prt, enable); - } + } + } /* for all ports */ lcnt_enable_drv_lock_count(&vanilla_driver, enable); lcnt_enable_drv_lock_count(&spawn_driver, enable); +#ifndef __WIN32__ + lcnt_enable_drv_lock_count(&forker_driver, enable); +#endif lcnt_enable_drv_lock_count(&fd_driver, enable); - for (dp = driver_list; dp; dp = dp->next) + /* enable lock counting in all drivers */ + for (dp = driver_list; dp; dp = dp->next) { lcnt_enable_drv_lock_count(dp, enable); -} -#endif - + } +} /* enable/disable lock counting of ports */ +#endif /* defined(ERTS_ENABLE_LOCK_COUNT) && defined(ERTS_SMP) */ /* * Buffering of data when using line oriented I/O on ports */ @@ -3026,29 +3373,26 @@ deliver_result(Eterm sender, Eterm pid, Eterm res) rp = (scheduler ? erts_proc_lookup(pid) - : erts_pid2proc_opt(NULL, 0, pid, 0, ERTS_P2P_FLG_SMP_INC_REFC)); + : erts_pid2proc_opt(NULL, 0, pid, 0, ERTS_P2P_FLG_INC_REFC)); if (rp) { Eterm tuple; - ErlHeapFragment *bp; + ErtsMessage *mp; ErlOffHeap *ohp; Eterm* hp; Uint sz_res; sz_res = size_object(res); - hp = erts_alloc_message_heap(sz_res + 3, &bp, &ohp, rp, &rp_locks); + mp = erts_alloc_message_heap(rp, &rp_locks, + sz_res + 3, &hp, &ohp); res = copy_struct(res, sz_res, &hp, ohp); tuple = TUPLE2(hp, sender, res); - erts_queue_message(rp, &rp_locks, bp, tuple, NIL -#ifdef USE_VM_PROBES - , NIL -#endif - ); + erts_queue_message(rp, &rp_locks, mp, tuple, NIL); if (rp_locks) erts_smp_proc_unlock(rp, rp_locks); if (!scheduler) - erts_smp_proc_dec_refc(rp); + erts_proc_dec_refc(rp); } } @@ -3071,7 +3415,7 @@ static void deliver_read_message(Port* prt, erts_aint32_t state, Eterm to, Eterm tuple; Process* rp; Eterm* hp; - ErlHeapFragment *bp; + ErtsMessage *mp; ErlOffHeap *ohp; ErtsProcLocks rp_locks = 0; int scheduler = erts_get_scheduler_id() != 0; @@ -3092,12 +3436,12 @@ static void deliver_read_message(Port* prt, erts_aint32_t state, Eterm to, rp = (scheduler ? erts_proc_lookup(to) - : erts_pid2proc_opt(NULL, 0, to, 0, ERTS_P2P_FLG_SMP_INC_REFC)); + : erts_pid2proc_opt(NULL, 0, to, 0, ERTS_P2P_FLG_INC_REFC)); if (!rp) return; - hp = erts_alloc_message_heap(need, &bp, &ohp, rp, &rp_locks); + mp = erts_alloc_message_heap(rp, &rp_locks, need, &hp, &ohp); listp = NIL; if ((state & ERTS_PORT_SFLG_BINARY_IO) == 0) { @@ -3107,8 +3451,6 @@ static void deliver_read_message(Port* prt, erts_aint32_t state, Eterm to, Binary* bptr; bptr = erts_bin_nrml_alloc(len); - bptr->flags = 0; - bptr->orig_size = len; erts_refc_init(&bptr->refc, 1); sys_memcpy(bptr->orig_bytes, buf, len); @@ -3141,15 +3483,11 @@ static void deliver_read_message(Port* prt, erts_aint32_t state, Eterm to, tuple = TUPLE2(hp, prt->common.id, tuple); hp += 3; - erts_queue_message(rp, &rp_locks, bp, tuple, am_undefined -#ifdef USE_VM_PROBES - , NIL -#endif - ); + erts_queue_message(rp, &rp_locks, mp, tuple, am_undefined); if (rp_locks) erts_smp_proc_unlock(rp, rp_locks); if (!scheduler) - erts_smp_proc_dec_refc(rp); + erts_proc_dec_refc(rp); } /* @@ -3219,7 +3557,7 @@ deliver_vec_message(Port* prt, /* Port */ Eterm tuple; Process* rp; Eterm* hp; - ErlHeapFragment *bp; + ErtsMessage *mp; ErlOffHeap *ohp; ErtsProcLocks rp_locks = 0; int scheduler = erts_get_scheduler_id() != 0; @@ -3235,7 +3573,7 @@ deliver_vec_message(Port* prt, /* Port */ rp = (scheduler ? erts_proc_lookup(to) - : erts_pid2proc_opt(NULL, 0, to, 0, ERTS_P2P_FLG_SMP_INC_REFC)); + : erts_pid2proc_opt(NULL, 0, to, 0, ERTS_P2P_FLG_INC_REFC)); if (!rp) return; @@ -3251,7 +3589,7 @@ deliver_vec_message(Port* prt, /* Port */ need += (hlen+csize)*2; } - hp = erts_alloc_message_heap(need, &bp, &ohp, rp, &rp_locks); + mp = erts_alloc_message_heap(rp, &rp_locks, need, &hp, &ohp); listp = NIL; iov += vsize; @@ -3312,14 +3650,10 @@ deliver_vec_message(Port* prt, /* Port */ tuple = TUPLE2(hp, prt->common.id, tuple); hp += 3; - erts_queue_message(rp, &rp_locks, bp, tuple, am_undefined -#ifdef USE_VM_PROBES - , NIL -#endif - ); + erts_queue_message(rp, &rp_locks, mp, tuple, am_undefined); erts_smp_proc_unlock(rp, rp_locks); if (!scheduler) - erts_smp_proc_dec_refc(rp); + erts_proc_dec_refc(rp); } @@ -3356,6 +3690,7 @@ static void flush_port(Port *p) ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(p)); if (p->drv_ptr->flush != NULL) { + ERTS_MSACC_PUSH_STATE_M(); #ifdef USE_VM_PROBES if (DTRACE_ENABLED(driver_flush)) { DTRACE_FORMAT_COMMON_PID_AND_PORT(ERTS_PORT_GET_CONNECTED(p), p) @@ -3365,9 +3700,11 @@ static void flush_port(Port *p) if (IS_TRACED_FL(p, F_TRACE_SCHED_PORTS)) { trace_sched_ports_where(p, am_in, am_flush); } + ERTS_MSACC_SET_STATE_CACHED_M(ERTS_MSACC_STATE_PORT); fpe_was_unmasked = erts_block_fpe(); (*p->drv_ptr->flush)((ErlDrvData)p->drv_data); erts_unblock_fpe(fpe_was_unmasked); + ERTS_MSACC_POP_STATE_M(); if (IS_TRACED_FL(p, F_TRACE_SCHED_PORTS)) { trace_sched_ports_where(p, am_out, am_flush); } @@ -3391,6 +3728,7 @@ terminate_port(Port *prt) Eterm connected_id = NIL /* Initialize to silence compiler */; erts_driver_t *drv; erts_aint32_t state; + ErtsPrtSD *psd; ERTS_SMP_CHK_NO_PROC_LOCKS; ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(prt)); @@ -3409,15 +3747,13 @@ terminate_port(Port *prt) send_closed_port_id = NIL; } -#ifdef ERTS_SMP - erts_cancel_smp_ptimer(prt->common.u.alive.ptimer); -#else - erts_cancel_timer(&prt->common.u.alive.tm); -#endif + if (ERTS_PTMR_IS_SET(prt)) + erts_cancel_port_timer(prt); drv = prt->drv_ptr; if ((drv != NULL) && (drv->stop != NULL)) { int fpe_was_unmasked = erts_block_fpe(); + ERTS_MSACC_PUSH_AND_SET_STATE_M(ERTS_MSACC_STATE_PORT); #ifdef USE_VM_PROBES if (DTRACE_ENABLED(driver_stop)) { DTRACE_FORMAT_COMMON_PID_AND_PORT(connected_id, prt) @@ -3426,6 +3762,7 @@ terminate_port(Port *prt) #endif (*drv->stop)((ErlDrvData)prt->drv_data); erts_unblock_fpe(fpe_was_unmasked); + ERTS_MSACC_POP_STATE_M(); #ifdef ERTS_SMP if (prt->xports) erts_port_handle_xports(prt); @@ -3445,8 +3782,9 @@ terminate_port(Port *prt) erts_cleanup_port_data(prt); - if (prt->psd) - erts_free(ERTS_ALC_T_PRTSD, prt->psd); + psd = (ErtsPrtSD *) erts_smp_atomic_read_nob(&prt->psd); + if (psd) + erts_free(ERTS_ALC_T_PRTSD, psd); ASSERT(prt->dist_entry == NULL); @@ -3459,7 +3797,7 @@ terminate_port(Port *prt) if ((state & ERTS_PORT_SFLG_HALT) && (erts_smp_atomic32_dec_read_nob(&erts_halt_progress) == 0)) { erts_port_release(prt); /* We will exit and never return */ - erl_exit_flush_async(erts_halt_code, ""); + erts_flush_async_exit(erts_halt_code, ""); } if (is_internal_port(send_closed_port_id)) deliver_result(send_closed_port_id, connected_id, am_closed); @@ -3736,6 +4074,7 @@ call_driver_control(Eterm caller, ErlDrvSizeT *from_size) { ErlDrvSSizeT cres; + ERTS_MSACC_PUSH_STATE_M(); if (!prt->drv_ptr->control) return ERTS_PORT_OP_BADARG; @@ -3749,6 +4088,8 @@ call_driver_control(Eterm caller, command, size); } #endif + + ERTS_MSACC_SET_STATE_CACHED_M(ERTS_MSACC_STATE_PORT); prt->caller = caller; cres = prt->drv_ptr->control((ErlDrvData) prt->drv_data, @@ -3759,6 +4100,8 @@ call_driver_control(Eterm caller, *from_size); prt->caller = NIL; + ERTS_MSACC_POP_STATE_M(); + if (cres < 0) return ERTS_PORT_OP_BADARG; @@ -3810,7 +4153,6 @@ write_port_control_result(int control_flags, ErlDrvSizeT resp_size, char *pre_alloc_buf, Eterm **hpp, - ErlHeapFragment *bp, ErlOffHeap *ohp) { Eterm res; @@ -3884,46 +4226,37 @@ port_sig_control(Port *prt, if (res == ERTS_PORT_OP_DONE) { Eterm msg; - Eterm *hp, *hp_start; - ErlHeapFragment *bp; - ErlOffHeap *ohp; + ErtsHeapFactory factory; Process *rp; ErtsProcLocks rp_locks = 0; - Uint hsz; + Uint hsz, rsz; int control_flags; - rp = erts_proc_lookup_raw(sigdp->caller); + rp = sigdp->caller == ERTS_INVALID_PID ? NULL : erts_proc_lookup_raw(sigdp->caller); if (!rp) goto done; control_flags = prt->control_flags; - hsz = ERTS_QUEUE_PORT_SCHED_OP_REPLY_SIZE; - hsz += port_control_result_size(control_flags, + rsz = port_control_result_size(control_flags, resp_bufp, &resp_size, &resp_buf[0]); + hsz = rsz + ERTS_QUEUE_PORT_SCHED_OP_REPLY_SIZE; + - hp_start = hp = erts_alloc_message_heap(hsz, - &bp, - &ohp, - rp, - &rp_locks); + (void) erts_factory_message_create(&factory, rp, + &rp_locks, hsz); msg = write_port_control_result(control_flags, resp_bufp, resp_size, &resp_buf[0], - &hp, - bp, - ohp); - + &factory.hp, + factory.off_heap); queue_port_sched_op_reply(rp, &rp_locks, - hp_start, - hp, - hsz, - bp, + &factory, sigdp->ref, msg); @@ -3935,7 +4268,8 @@ port_sig_control(Port *prt, /* failure */ - port_sched_op_reply(sigdp->caller, sigdp->ref, am_badarg); + if (sigdp->caller != ERTS_INVALID_PID) + port_sched_op_reply(sigdp->caller, sigdp->ref, am_badarg); done: @@ -3945,6 +4279,23 @@ done: return ERTS_PORT_REDS_CONTROL; } +/* + * This is an asynchronous control call. I.e. it will not return anything + * to the caller. + */ +int +erl_drv_port_control(Eterm port_num, char cmd, char* buff, ErlDrvSizeT size) +{ + ErtsProc2PortSigData *sigdp = erts_port_task_alloc_p2p_sig_data(); + + sigdp->flags = ERTS_P2P_SIG_TYPE_CONTROL | ERTS_P2P_SIG_DATA_FLG_REPLY; + sigdp->u.control.binp = NULL; + sigdp->u.control.command = cmd; + sigdp->u.control.bufp = buff; + sigdp->u.control.size = size; + + return erts_schedule_port2port_signal(port_num, sigdp, 0, port_sig_control); +} ErtsPortOpResult erts_port_control(Process* c_p, @@ -4041,6 +4392,9 @@ erts_port_control(Process* c_p, size, &resp_bufp, &resp_size); + + control_flags = prt->control_flags; + finalize_imm_drv_call(&try_call_state); if (tmp_alloced) erts_free(ERTS_ALC_T_TMP, bufp); @@ -4048,8 +4402,6 @@ erts_port_control(Process* c_p, return ERTS_PORT_OP_BADARG; } - control_flags = prt->control_flags; - hsz = port_control_result_size(control_flags, resp_bufp, &resp_size, @@ -4060,7 +4412,6 @@ erts_port_control(Process* c_p, resp_size, &resp_buf[0], &hp, - NULL, &c_p->off_heap); BUMP_REDS(c_p, ERTS_PORT_REDS_CONTROL); return ERTS_PORT_OP_DONE; @@ -4082,10 +4433,10 @@ erts_port_control(Process* c_p, binp = NULL; if (is_binary(data) && binary_bitoffset(data) == 0) { - Eterm *ebinp = binary_val_rel(data, NULL); + Eterm *ebinp = binary_val(data); ASSERT(!tmp_alloced); if (*ebinp == HEADER_SUB_BIN) - ebinp = binary_val_rel(((ErlSubBin *) ebinp)->orig, NULL); + ebinp = binary_val(((ErlSubBin *) ebinp)->orig); if (*ebinp != HEADER_PROC_BIN) copy = 1; else { @@ -4138,6 +4489,7 @@ call_driver_call(Eterm caller, unsigned *ret_flagsp) { ErlDrvSSizeT cres; + ERTS_MSACC_PUSH_STATE_M(); if (!prt->drv_ptr->call) return ERTS_PORT_OP_BADARG; @@ -4153,6 +4505,8 @@ call_driver_call(Eterm caller, } #endif + ERTS_MSACC_SET_STATE_CACHED_M(ERTS_MSACC_STATE_PORT); + prt->caller = caller; cres = prt->drv_ptr->call((ErlDrvData) prt->drv_data, command, @@ -4163,6 +4517,8 @@ call_driver_call(Eterm caller, ret_flagsp); prt->caller = NIL; + ERTS_MSACC_POP_STATE_M(); + if (cres <= 0 || ((byte) (*resp_bufp)[0]) != VERSION_MAGIC) return ERTS_PORT_OP_BADARG; @@ -4209,8 +4565,6 @@ port_sig_call(Port *prt, if (res == ERTS_PORT_OP_DONE) { Eterm msg; Eterm *hp; - ErlHeapFragment *bp; - ErlOffHeap *ohp; Process *rp; ErtsProcLocks rp_locks = 0; Sint hsz; @@ -4221,29 +4575,24 @@ port_sig_call(Port *prt, hsz = erts_decode_ext_size((byte *) resp_bufp, resp_size); if (hsz >= 0) { - Eterm *hp_start; + ErtsHeapFactory factory; byte *endp; hsz += 3; /* ok tuple */ hsz += ERTS_QUEUE_PORT_SCHED_OP_REPLY_SIZE; - hp_start = hp = erts_alloc_message_heap(hsz, - &bp, - &ohp, - rp, - &rp_locks); + (void) erts_factory_message_create(&factory, rp, &rp_locks, hsz); endp = (byte *) resp_bufp; - msg = erts_decode_ext(&hp, ohp, &endp); + msg = erts_decode_ext(&factory, &endp, 0); if (is_value(msg)) { + hp = erts_produce_heap(&factory, + 3, + ERTS_QUEUE_PORT_SCHED_OP_REPLY_SIZE); msg = TUPLE2(hp, am_ok, msg); - hp += 3; queue_port_sched_op_reply(rp, &rp_locks, - hp_start, - hp, - hsz, - bp, + &factory, sigdp->ref, msg); @@ -4251,8 +4600,6 @@ port_sig_call(Port *prt, erts_smp_proc_unlock(rp, rp_locks); goto done; } - if (bp) - free_message_buffer(bp); if (rp_locks) erts_smp_proc_unlock(rp, rp_locks); } @@ -4329,10 +4676,11 @@ erts_port_call(Process* c_p, try_call_res = try_imm_drv_call(&try_call_state); switch (try_call_res) { case ERTS_TRY_IMM_DRV_CALL_OK: { - Eterm *hp, *hp_end; + ErtsHeapFactory factory; Sint hsz; unsigned ret_flags = 0U; Eterm term; + Eterm* hp; res = call_driver_call(c_p->common.id, prt, @@ -4352,15 +4700,14 @@ erts_port_call(Process* c_p, if (hsz < 0) return ERTS_PORT_OP_BADARG; hsz += 3; - hp = HAlloc(c_p, hsz); - hp_end = hp + hsz; + erts_factory_proc_prealloc_init(&factory, c_p, hsz); endp = (byte *) resp_bufp; - term = erts_decode_ext(&hp, &MSO(c_p), &endp); + term = erts_decode_ext(&factory, &endp, 0); if (term == THE_NON_VALUE) return ERTS_PORT_OP_BADARG; + hp = erts_produce_heap(&factory,3,0); *retvalp = TUPLE2(hp, am_ok, term); - hp += 3; - HRelease(c_p, hp_end, hp); + erts_factory_close(&factory); if (resp_bufp != &resp_buf[0] && !(ret_flags & DRIVER_CALL_KEEP_BUFFER)) driver_free(resp_bufp); @@ -4438,7 +4785,7 @@ make_port_info_term(Eterm **hpp_start, int len; int start; static Eterm item[] = ERTS_PORT_INFO_1_ITEMS; - static Eterm value[sizeof(item)/sizeof(item[0])]; + Eterm value[sizeof(item)/sizeof(item[0])]; start = 0; len = sizeof(item)/sizeof(item[0]); @@ -4495,12 +4842,13 @@ port_sig_info(Port *prt, prt, sigdp->u.info.item); if (is_value(value)) { + ErtsHeapFactory factory; + ErtsMessage *mp = erts_alloc_message(0, NULL); + mp->data.heap_frag = bp; + erts_factory_selfcontained_message_init(&factory, mp, hp); queue_port_sched_op_reply(rp, &rp_locks, - hp_start, - hp, - hsz, - bp, + &factory, sigdp->ref, value); } @@ -4571,6 +4919,102 @@ erts_port_info(Process* c_p, } typedef struct { + Uint sched_id; + Eterm pid; + Uint32 refn[ERTS_REF_NUMBERS]; + erts_smp_atomic32_t refc; +} ErtsIOBytesReq; + +static void +reply_io_bytes(void *vreq) +{ + ErtsIOBytesReq *req = (ErtsIOBytesReq *) vreq; + Process *rp; + + rp = erts_proc_lookup(req->pid); + if (rp) { + ErlOffHeap *ohp; + ErtsMessage *mp; + ErtsProcLocks rp_locks; + Eterm ref, msg, ein, eout, *hp; + Uint64 in, out; + Uint hsz; + ErtsSchedulerData *esdp = erts_get_scheduler_data(); + Uint sched_id = esdp->no; + in = esdp->io.in; + out = esdp->io.out; + if (req->sched_id != sched_id) + rp_locks = 0; + else { + in += (Uint64) erts_atomic64_read_nob(&bytes_in); + out += (Uint64) erts_atomic64_read_nob(&bytes_out); + rp_locks = ERTS_PROC_LOCK_MAIN; + } + + hsz = 5 /* 4-tuple */ + REF_THING_SIZE; + + erts_bld_uint64(NULL, &hsz, in); + erts_bld_uint64(NULL, &hsz, out); + + mp = erts_alloc_message_heap(rp, &rp_locks, hsz, &hp, &ohp); + + ref = make_internal_ref(hp); + write_ref_thing(hp, req->refn[0], req->refn[1], req->refn[2]); + hp += REF_THING_SIZE; + + ein = erts_bld_uint64(&hp, NULL, in); + eout = erts_bld_uint64(&hp, NULL, out); + + msg = TUPLE4(hp, ref, make_small(sched_id), ein, eout); + erts_queue_message(rp, &rp_locks, mp, msg, NIL); + + if (req->sched_id == sched_id) + rp_locks &= ~ERTS_PROC_LOCK_MAIN; + if (rp_locks) + erts_smp_proc_unlock(rp, rp_locks); + } + + if (erts_smp_atomic32_dec_read_nob(&req->refc) == 0) + erts_free(ERTS_ALC_T_IOB_REQ, req); +} + +Eterm +erts_request_io_bytes(Process *c_p) +{ + Uint *hp; + Eterm ref; + Uint32 *refn; + ErtsSchedulerData *esdp = ERTS_PROC_GET_SCHDATA(c_p); + ErtsIOBytesReq *req = erts_alloc(ERTS_ALC_T_IOB_REQ, + sizeof(ErtsIOBytesReq)); + + hp = HAlloc(c_p, REF_THING_SIZE); + ref = erts_sched_make_ref_in_buffer(esdp, hp); + refn = internal_ref_numbers(ref); + + req->sched_id = esdp->no; + req->pid = c_p->common.id; + req->refn[0] = refn[0]; + req->refn[1] = refn[1]; + req->refn[2] = refn[2]; + erts_smp_atomic32_init_nob(&req->refc, + (erts_aint32_t) erts_no_schedulers); + +#ifdef ERTS_SMP + if (erts_no_schedulers > 1) + erts_schedule_multi_misc_aux_work(1, + erts_no_schedulers, + reply_io_bytes, + (void *) req); +#endif + + reply_io_bytes((void *) req); + + return ref; +} + + +typedef struct { int to; void *arg; } prt_one_lnk_data; @@ -4628,6 +5072,10 @@ print_port_info(Port *p, int to, void *arg) erts_print(to, arg, "Port is a file: %s\n",p->name); } else if (p->drv_ptr == &spawn_driver) { erts_print(to, arg, "Port controls external process: %s\n",p->name); +#ifndef __WIN32__ + } else if (p->drv_ptr == &forker_driver) { + erts_print(to, arg, "Port controls forker process: %s\n",p->name); +#endif } else { erts_print(to, arg, "Port controls linked-in driver: %s\n",p->name); } @@ -4774,12 +5222,13 @@ int get_port_flags(ErlDrvPort ix) void erts_raw_port_command(Port* p, byte* buf, Uint len) { int fpe_was_unmasked; + ERTS_MSACC_PUSH_STATE_M(); ERTS_SMP_CHK_NO_PROC_LOCKS; ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(p)); if (len > (Uint) INT_MAX) - erl_exit(ERTS_ABORT_EXIT, + erts_exit(ERTS_ABORT_EXIT, "Absurdly large data buffer (%beu bytes) passed to" "output callback of %s driver.\n", len, @@ -4794,9 +5243,11 @@ void erts_raw_port_command(Port* p, byte* buf, Uint len) DTRACE4(driver_output, "-raw-", port_str, p->name, len); } #endif + ERTS_MSACC_SET_STATE_CACHED_M(ERTS_MSACC_STATE_PORT); fpe_was_unmasked = erts_block_fpe(); (*p->drv_ptr->output)((ErlDrvData)p->drv_data, (char*) buf, (int) len); erts_unblock_fpe(fpe_was_unmasked); + ERTS_MSACC_POP_STATE_M(); } int async_ready(Port *p, void* data) @@ -4808,6 +5259,7 @@ int async_ready(Port *p, void* data) if (p) { ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(p)); if (p->drv_ptr->ready_async != NULL) { + ERTS_MSACC_PUSH_AND_SET_STATE_M(ERTS_MSACC_STATE_PORT); #ifdef USE_VM_PROBES if (DTRACE_ENABLED(driver_ready_async)) { DTRACE_FORMAT_COMMON_PID_AND_PORT(ERTS_PORT_GET_CONNECTED(p), p) @@ -4816,6 +5268,7 @@ int async_ready(Port *p, void* data) #endif (*p->drv_ptr->ready_async)((ErlDrvData)p->drv_data, data); need_free = 0; + ERTS_MSACC_POP_STATE_M(); } erts_port_driver_callback_epilogue(p, NULL); @@ -4959,24 +5412,6 @@ erts_free_port_names(ErtsPortNames *pnp) erts_free(ERTS_ALC_T_PORT_NAMES, pnp); } -static void schedule_port_timeout(Port *p) -{ - /* - * Scheduling of port timeouts can be done without port locking, but - * since the task handle is stored in the port structure and the ptimer - * structure is protected by the port lock we require the port to be - * locked for now... - * - * TODO: Implement scheduling of port timeouts without locking - * the port. - * /Rickard - */ - ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(p)); - erts_port_task_schedule(p->common.id, - &p->timeout_task, - ERTS_PORT_TASK_TIMEOUT); -} - ErlDrvTermData driver_mk_term_nil(void) { return driver_term_nil; @@ -4985,11 +5420,11 @@ ErlDrvTermData driver_mk_term_nil(void) void driver_report_exit(ErlDrvPort ix, int status) { Eterm* hp; + ErlOffHeap *ohp; Eterm tuple; Process *rp; Eterm pid; - ErlHeapFragment *bp = NULL; - ErlOffHeap *ohp; + ErtsMessage *mp; ErtsProcLocks rp_locks = 0; int scheduler = erts_get_scheduler_id() != 0; Port* prt = erts_drvport2port(ix); @@ -5005,25 +5440,21 @@ void driver_report_exit(ErlDrvPort ix, int status) rp = (scheduler ? erts_proc_lookup(pid) - : erts_pid2proc_opt(NULL, 0, pid, 0, ERTS_P2P_FLG_SMP_INC_REFC)); + : erts_pid2proc_opt(NULL, 0, pid, 0, ERTS_P2P_FLG_INC_REFC)); if (!rp) return; - hp = erts_alloc_message_heap(3+3, &bp, &ohp, rp, &rp_locks); + mp = erts_alloc_message_heap(rp, &rp_locks, 3+3, &hp, &ohp); tuple = TUPLE2(hp, am_exit_status, make_small(status)); hp += 3; tuple = TUPLE2(hp, prt->common.id, tuple); - erts_queue_message(rp, &rp_locks, bp, tuple, am_undefined -#ifdef USE_VM_PROBES - , NIL -#endif - ); + erts_queue_message(rp, &rp_locks, mp, tuple, am_undefined); erts_smp_proc_unlock(rp, rp_locks); if (!scheduler) - erts_smp_proc_dec_refc(rp); + erts_proc_dec_refc(rp); } #define ERTS_B2T_STATES_DEF_STATES_SZ 5 @@ -5115,22 +5546,26 @@ cleanup_b2t_states(struct b2t_states__ *b2tsp) static int driver_deliver_term(Eterm to, ErlDrvTermData* data, int len) { +#define HEAP_EXTRA 200 #define ERTS_DDT_FAIL do { res = -1; goto done; } while (0) Uint need = 0; int depth = 0; int res; - Eterm *hp = NULL, *hp_start = NULL, *hp_end = NULL; ErlDrvTermData* ptr; ErlDrvTermData* ptr_end; DECLARE_ESTACK(stack); - Eterm mess = NIL; /* keeps compiler happy */ + Eterm mess; Process* rp = NULL; - ErlHeapFragment *bp = NULL; - ErlOffHeap *ohp; + ErtsHeapFactory factory; ErtsProcLocks rp_locks = 0; struct b2t_states__ b2t; - int scheduler = 1; /* Silence erroneous warning... */ + int scheduler; + ErtsSchedulerData *esdp = erts_get_scheduler_data(); + ERTS_UNDEF(mess,NIL); + ERTS_UNDEF(scheduler,1); + + factory.mode = FACTORY_CLOSED; init_b2t_states(&b2t); /* @@ -5163,25 +5598,17 @@ driver_deliver_term(Eterm to, ErlDrvTermData* data, int len) break; case ERL_DRV_INT: /* signed int argument */ ERTS_DDT_CHK_ENOUGH_ARGS(1); -#if HALFWORD_HEAP - erts_bld_sint64(NULL, &need, (Sint64)ptr[0]); -#else /* check for bignum */ if (!IS_SSMALL((Sint)ptr[0])) need += BIG_UINT_HEAP_SIZE; /* use small_to_big */ -#endif ptr++; depth++; break; case ERL_DRV_UINT: /* unsigned int argument */ ERTS_DDT_CHK_ENOUGH_ARGS(1); -#if HALFWORD_HEAP - erts_bld_uint64(NULL, &need, (Uint64)ptr[0]); -#else /* check for bignum */ if (!IS_USMALL(0, (Uint)ptr[0])) need += BIG_UINT_HEAP_SIZE; /* use small_to_big */ -#endif ptr++; depth++; break; @@ -5294,9 +5721,10 @@ driver_deliver_term(Eterm to, ErlDrvTermData* data, int len) #ifdef DEBUG b2t.org_ext[b2t.ix] = ext; #endif - hsz = erts_binary2term_prepare(&b2t.state[b2t.ix++], ext, size); + hsz = erts_binary2term_prepare(&b2t.state[b2t.ix], ext, size); if (hsz < 0) ERTS_DDT_FAIL; /* Invalid data */ + b2t.state[b2t.ix++].heap_size = hsz; need += hsz; ptr += 2; depth++; @@ -5305,7 +5733,11 @@ driver_deliver_term(Eterm to, ErlDrvTermData* data, int len) case ERL_DRV_MAP: { /* int */ ERTS_DDT_CHK_ENOUGH_ARGS(1); if ((int) ptr[0] < 0) ERTS_DDT_FAIL; - need += MAP_HEADER_SIZE + 1 + 2*ptr[0]; + if (ptr[0] > MAP_SMALL_MAP_LIMIT) { + need += HASHMAP_ESTIMATED_HEAP_SIZE(ptr[0]); + } else { + need += MAP_HEADER_FLATMAP_SZ + 1 + 2*ptr[0]; + } depth -= 2*ptr[0]; if (depth < 0) ERTS_DDT_FAIL; ptr++; @@ -5335,14 +5767,13 @@ driver_deliver_term(Eterm to, ErlDrvTermData* data, int len) scheduler = erts_get_scheduler_id() != 0; rp = (scheduler ? erts_proc_lookup(to) - : erts_pid2proc_opt(NULL, 0, to, 0, ERTS_P2P_FLG_SMP_INC_REFC)); + : erts_pid2proc_opt(NULL, 0, to, 0, ERTS_P2P_FLG_INC_REFC)); if (!rp) { res = 0; goto done; } - hp_start = hp = erts_alloc_message_heap(need, &bp, &ohp, rp, &rp_locks); - hp_end = hp + need; + (void) erts_factory_message_create(&factory, rp, &rp_locks, need); /* * Interpret the instructions and build the term. @@ -5362,40 +5793,36 @@ driver_deliver_term(Eterm to, ErlDrvTermData* data, int len) break; case ERL_DRV_INT: /* signed int argument */ -#if HALFWORD_HEAP - mess = erts_bld_sint64(&hp, NULL, (Sint64)ptr[0]); -#else + erts_reserve_heap(&factory, BIG_UINT_HEAP_SIZE); if (IS_SSMALL((Sint)ptr[0])) mess = make_small((Sint)ptr[0]); else { - mess = small_to_big((Sint)ptr[0], hp); - hp += BIG_UINT_HEAP_SIZE; + mess = small_to_big((Sint)ptr[0], factory.hp); + factory.hp += BIG_UINT_HEAP_SIZE; } -#endif ptr++; break; case ERL_DRV_UINT: /* unsigned int argument */ -#if HALFWORD_HEAP - mess = erts_bld_uint64(&hp, NULL, (Uint64)ptr[0]); -#else + erts_reserve_heap(&factory, BIG_UINT_HEAP_SIZE); if (IS_USMALL(0, (Uint)ptr[0])) mess = make_small((Uint)ptr[0]); else { - mess = uint_to_big((Uint)ptr[0], hp); - hp += BIG_UINT_HEAP_SIZE; + mess = uint_to_big((Uint)ptr[0], factory.hp); + factory.hp += BIG_UINT_HEAP_SIZE; } -#endif ptr++; break; case ERL_DRV_INT64: /* pointer to unsigned 64-bit int argument */ - mess = erts_bld_sint64(&hp, NULL, *((Sint64 *) ptr[0])); + erts_reserve_heap(&factory, BIG_NEED_FOR_BITS(64)); + mess = erts_bld_sint64(&factory.hp, NULL, *((Sint64 *) ptr[0])); ptr++; break; case ERL_DRV_UINT64: /* pointer to unsigned 64-bit int argument */ - mess = erts_bld_uint64(&hp, NULL, *((Uint64 *) ptr[0])); + erts_reserve_heap(&factory, BIG_NEED_FOR_BITS(64)); + mess = erts_bld_uint64(&factory.hp, NULL, *((Uint64 *) ptr[0])); ptr++; break; @@ -5409,11 +5836,14 @@ driver_deliver_term(Eterm to, ErlDrvTermData* data, int len) Uint size = ptr[1]; Uint offset = ptr[2]; - erts_smp_atomic_add_nob(&erts_bytes_in, (erts_aint_t) size); + if (esdp) + esdp->io.in += (Uint64) size; + else + erts_atomic64_add_nob(&bytes_in, (erts_aint64_t) size); if (size <= ERL_ONHEAP_BIN_LIMIT) { - ErlHeapBin* hbp = (ErlHeapBin *) hp; - hp += heap_bin_size(size); + ErlHeapBin* hbp = (ErlHeapBin *) erts_produce_heap(&factory, + heap_bin_size(size), HEAP_EXTRA); hbp->thing_word = header_heap_bin(size); hbp->size = size; if (size > 0) { @@ -5422,18 +5852,18 @@ driver_deliver_term(Eterm to, ErlDrvTermData* data, int len) mess = make_binary(hbp); } else { - ProcBin* pb = (ProcBin *) hp; + ProcBin* pb = (ProcBin *) erts_produce_heap(&factory, + PROC_BIN_SIZE, HEAP_EXTRA); driver_binary_inc_refc(b); /* caller will free binary */ pb->thing_word = HEADER_PROC_BIN; pb->size = size; - pb->next = ohp->first; - ohp->first = (struct erl_off_heap_header*)pb; + pb->next = factory.off_heap->first; + factory.off_heap->first = (struct erl_off_heap_header*)pb; pb->val = ErlDrvBinary2Binary(b); pb->bytes = ((byte*) b->orig_bytes) + offset; pb->flags = 0; mess = make_binary(pb); - hp += PROC_BIN_SIZE; - OH_OVERHEAD(ohp, pb->size / sizeof(Eterm)); + OH_OVERHEAD(factory.off_heap, pb->size / sizeof(Eterm)); } ptr += 3; break; @@ -5443,11 +5873,15 @@ driver_deliver_term(Eterm to, ErlDrvTermData* data, int len) byte *bufp = (byte *) ptr[0]; Uint size = (Uint) ptr[1]; - erts_smp_atomic_add_nob(&erts_bytes_in, (erts_aint_t) size); + if (esdp) + esdp->io.in += (Uint64) size; + else + erts_atomic64_add_nob(&bytes_in, (erts_aint64_t) size); if (size <= ERL_ONHEAP_BIN_LIMIT) { - ErlHeapBin* hbp = (ErlHeapBin *) hp; - hp += heap_bin_size(size); + ErlHeapBin* hbp = (ErlHeapBin *) erts_produce_heap(&factory, + heap_bin_size(size), + HEAP_EXTRA); hbp->thing_word = header_heap_bin(size); hbp->size = size; if (size > 0) { @@ -5460,20 +5894,18 @@ driver_deliver_term(Eterm to, ErlDrvTermData* data, int len) ProcBin* pbp; Binary* bp = erts_bin_nrml_alloc(size); ASSERT(bufp); - bp->flags = 0; - bp->orig_size = (SWord) size; erts_refc_init(&bp->refc, 1); sys_memcpy((void *) bp->orig_bytes, (void *) bufp, size); - pbp = (ProcBin *) hp; - hp += PROC_BIN_SIZE; + pbp = (ProcBin *) erts_produce_heap(&factory, + PROC_BIN_SIZE, HEAP_EXTRA); pbp->thing_word = HEADER_PROC_BIN; pbp->size = size; - pbp->next = ohp->first; - ohp->first = (struct erl_off_heap_header*)pbp; + pbp->next = factory.off_heap->first; + factory.off_heap->first = (struct erl_off_heap_header*)pbp; pbp->val = bp; pbp->bytes = (byte*) bp->orig_bytes; pbp->flags = 0; - OH_OVERHEAD(ohp, pbp->size / sizeof(Eterm)); + OH_OVERHEAD(factory.off_heap, pbp->size / sizeof(Eterm)); mess = make_binary(pbp); } ptr += 2; @@ -5481,14 +5913,19 @@ driver_deliver_term(Eterm to, ErlDrvTermData* data, int len) } case ERL_DRV_STRING: /* char*, length */ - erts_smp_atomic_add_nob(&erts_bytes_in, (erts_aint_t) ptr[1]); - mess = buf_to_intlist(&hp, (char*)ptr[0], ptr[1], NIL); + if (esdp) + esdp->io.in += (Uint64) ptr[1]; + else + erts_atomic64_add_nob(&bytes_in, (erts_aint64_t) ptr[1]); + erts_reserve_heap(&factory, 2*ptr[1]); + mess = buf_to_intlist(&factory.hp, (char*)ptr[0], ptr[1], NIL); ptr += 2; break; case ERL_DRV_STRING_CONS: /* char*, length */ mess = ESTACK_POP(stack); - mess = buf_to_intlist(&hp, (char*)ptr[0], ptr[1], mess); + erts_reserve_heap(&factory, 2*ptr[1]); + mess = buf_to_intlist(&factory.hp, (char*)ptr[0], ptr[1], mess); ptr += 2; break; @@ -5497,11 +5934,12 @@ driver_deliver_term(Eterm to, ErlDrvTermData* data, int len) mess = ESTACK_POP(stack); i--; + erts_reserve_heap(&factory, 2*i); while(i > 0) { Eterm hd = ESTACK_POP(stack); - mess = CONS(hp, hd, mess); - hp += 2; + mess = CONS(factory.hp, hd, mess); + factory.hp += 2; i--; } ptr++; @@ -5510,13 +5948,12 @@ driver_deliver_term(Eterm to, ErlDrvTermData* data, int len) case ERL_DRV_TUPLE: { /* int */ int size = (int)ptr[0]; - Eterm* tp = hp; + Eterm* tp = erts_produce_heap(&factory, size+1, HEAP_EXTRA); *tp = make_arityval(size); mess = make_tuple(tp); tp += size; /* point at last element */ - hp = tp+1; /* advance "heap" pointer */ while(size--) { *tp-- = ESTACK_POP(stack); @@ -5532,18 +5969,22 @@ driver_deliver_term(Eterm to, ErlDrvTermData* data, int len) case ERL_DRV_FLOAT: { /* double * */ FloatDef f; + Eterm* fp = erts_produce_heap(&factory, FLOAT_SIZE_OBJECT, HEAP_EXTRA); - mess = make_float(hp); + mess = make_float(fp); f.fd = *((double *) ptr[0]); - PUT_DOUBLE(f, hp); - hp += FLOAT_SIZE_OBJECT; + if (!erts_isfinite(f.fd)) + ERTS_DDT_FAIL; + PUT_DOUBLE(f, fp); ptr++; break; } case ERL_DRV_EXT2TERM: /* char *ext, int size */ ASSERT(b2t.org_ext[b2t.ix] == (byte *) ptr[0]); - mess = erts_binary2term_create(&b2t.state[b2t.ix++], &hp, ohp); + + erts_reserve_heap(&factory, b2t.state[b2t.ix].heap_size); + mess = erts_binary2term_create(&b2t.state[b2t.ix++], &factory); if (mess == THE_NON_VALUE) ERTS_DDT_FAIL; ptr += 2; @@ -5551,31 +5992,43 @@ driver_deliver_term(Eterm to, ErlDrvTermData* data, int len) case ERL_DRV_MAP: { /* int */ int size = (int)ptr[0]; - Eterm* tp = hp; - Eterm* vp; - map_t *mp; - - *tp = make_arityval(size); - - hp += 1 + size; - mp = (map_t*)hp; - mp->thing_word = MAP_HEADER; - mp->size = size; - mp->keys = make_tuple(tp); - mess = make_map(mp); - - hp += MAP_HEADER_SIZE + size; /* advance "heap" pointer */ - - tp += size; /* point at last key */ - vp = hp - 1; /* point at last value */ - - while(size--) { - *vp-- = ESTACK_POP(stack); - *tp-- = ESTACK_POP(stack); - } - if (!erts_validate_and_sort_map(mp)) - ERTS_DDT_FAIL; - ptr++; + if (size > MAP_SMALL_MAP_LIMIT) { + int ix = 2*size; + Eterm* leafs; + + erts_produce_heap(&factory, ix, HEAP_EXTRA); + leafs = factory.hp; + while(ix--) { *--leafs = ESTACK_POP(stack); } + + mess = erts_hashmap_from_array(&factory, leafs, size, 1); + if (is_non_value(mess)) + ERTS_DDT_FAIL; + } else { + Eterm* vp; + flatmap_t *mp; + Eterm* tp = erts_produce_heap(&factory, + 2*size + 1 + MAP_HEADER_FLATMAP_SZ, + HEAP_EXTRA); + + *tp = make_arityval(size); + + mp = (flatmap_t*) (tp + 1 + size); + mp->thing_word = MAP_HEADER_FLATMAP; + mp->size = size; + mp->keys = make_tuple(tp); + mess = make_flatmap(mp); + + tp += size; /* point at last key */ + vp = factory.hp - 1; /* point at last value */ + + while(size--) { + *vp-- = ESTACK_POP(stack); + *tp-- = ESTACK_POP(stack); + } + if (!erts_validate_and_sort_flatmap(mp)) + ERTS_DDT_FAIL; + } + ptr++; break; } @@ -5589,42 +6042,28 @@ driver_deliver_term(Eterm to, ErlDrvTermData* data, int len) if (res > 0) { mess = ESTACK_POP(stack); /* get resulting value */ - if (bp) - bp = erts_resize_message_buffer(bp, hp - hp_start, &mess, 1); - else { - ASSERT(hp); - HRelease(rp, hp_end, hp); - } + erts_factory_trim_and_close(&factory, &mess, 1); /* send message */ - erts_queue_message(rp, &rp_locks, bp, mess, am_undefined -#ifdef USE_VM_PROBES - , NIL -#endif - ); + erts_queue_message(rp, &rp_locks, factory.message, mess, am_undefined); } else { if (b2t.ix > b2t.used) b2t.used = b2t.ix; for (b2t.ix = 0; b2t.ix < b2t.used; b2t.ix++) erts_binary2term_abort(&b2t.state[b2t.ix]); - if (bp) - free_message_buffer(bp); - else if (hp) { - HRelease(rp, hp_end, hp); - } + erts_factory_undo(&factory); } -#ifdef ERTS_SMP if (rp) { if (rp_locks) erts_smp_proc_unlock(rp, rp_locks); if (!scheduler) - erts_smp_proc_dec_refc(rp); + erts_proc_dec_refc(rp); } -#endif cleanup_b2t_states(&b2t); DESTROY_ESTACK(stack); return res; #undef ERTS_DDT_FAIL +#undef HEAP_EXTRA } static ERTS_INLINE int @@ -5753,6 +6192,7 @@ int driver_output_binary(ErlDrvPort ix, char* hbuf, ErlDrvSizeT hlen, { erts_aint32_t state; Port* prt = erts_drvport2port_state(ix, &state); + ErtsSchedulerData *esdp = erts_get_scheduler_data(); ERTS_SMP_CHK_NO_PROC_LOCKS; @@ -5763,7 +6203,10 @@ int driver_output_binary(ErlDrvPort ix, char* hbuf, ErlDrvSizeT hlen, return 0; prt->bytes_in += (hlen + len); - erts_smp_atomic_add_nob(&erts_bytes_in, (erts_aint_t) (hlen + len)); + if (esdp) + esdp->io.in += (Uint64) (hlen + len); + else + erts_atomic64_add_nob(&bytes_in, (erts_aint64_t) (hlen + len)); if (state & ERTS_PORT_SFLG_DISTRIBUTION) { return erts_net_message(prt, prt->dist_entry, @@ -5788,6 +6231,7 @@ int driver_output2(ErlDrvPort ix, char* hbuf, ErlDrvSizeT hlen, { erts_aint32_t state; Port* prt = erts_drvport2port_state(ix, &state); + ErtsSchedulerData *esdp = erts_get_scheduler_data(); ERTS_SMP_CHK_NO_PROC_LOCKS; @@ -5799,7 +6243,10 @@ int driver_output2(ErlDrvPort ix, char* hbuf, ErlDrvSizeT hlen, return 0; prt->bytes_in += (hlen + len); - erts_smp_atomic_add_nob(&erts_bytes_in, (erts_aint_t) (hlen + len)); + if (esdp) + esdp->io.in += (Uint64) (hlen + len); + else + erts_atomic64_add_nob(&bytes_in, (erts_aint64_t) (hlen + len)); if (state & ERTS_PORT_SFLG_DISTRIBUTION) { if (len == 0) return erts_net_message(prt, @@ -5839,6 +6286,7 @@ int driver_outputv(ErlDrvPort ix, char* hbuf, ErlDrvSizeT hlen, ErlDrvBinary** binv; Port* prt; erts_aint32_t state; + ErtsSchedulerData *esdp = erts_get_scheduler_data(); ERTS_SMP_CHK_NO_PROC_LOCKS; @@ -5877,7 +6325,10 @@ int driver_outputv(ErlDrvPort ix, char* hbuf, ErlDrvSizeT hlen, /* XXX handle distribution !!! */ prt->bytes_in += (hlen + size); - erts_smp_atomic_add_nob(&erts_bytes_in, (erts_aint_t) (hlen + size)); + if (esdp) + esdp->io.in += (Uint64) (hlen + size); + else + erts_atomic64_add_nob(&bytes_in, (erts_aint64_t) (hlen + size)); deliver_vec_message(prt, ERTS_PORT_GET_CONNECTED(prt), hbuf, hlen, binv, iov, n, size); return 0; @@ -5953,9 +6404,7 @@ driver_alloc_binary(ErlDrvSizeT size) bin = erts_bin_drv_alloc_fnf((Uint) size); if (!bin) return NULL; /* The driver write must take action */ - bin->flags = BIN_FLAG_DRV; erts_refc_init(&bin->refc, 1); - bin->orig_size = (SWord) size; return Binary2ErlDrvBinary(bin); } @@ -5985,7 +6434,6 @@ ErlDrvBinary* driver_realloc_binary(ErlDrvBinary* bin, ErlDrvSizeT size) if (!newbin) return NULL; - newbin->orig_size = size; return Binary2ErlDrvBinary(newbin); } @@ -6129,7 +6577,7 @@ driver_pdl_create(ErlDrvPort dp) return NULL; pdl = erts_alloc(ERTS_ALC_T_PORT_DATA_LOCK, sizeof(struct erl_drv_port_data_lock)); - erts_mtx_init(&pdl->mtx, "port_data_lock"); + erts_mtx_init_x(&pdl->mtx, "port_data_lock", pp->common.id, 1); pdl_init_refc(pdl); erts_port_inc_refc(pp); pdl->prt = pp; @@ -6569,18 +7017,6 @@ int driver_pushq(ErlDrvPort ix, char* buffer, ErlDrvSizeT len) return code; } -static ERTS_INLINE void -drv_cancel_timer(Port *prt) -{ -#ifdef ERTS_SMP - erts_cancel_smp_ptimer(prt->common.u.alive.ptimer); -#else - erts_cancel_timer(&prt->common.u.alive.tm); -#endif - if (erts_port_task_is_scheduled(&prt->timeout_task)) - erts_port_task_abort(&prt->timeout_task); -} - int driver_set_timer(ErlDrvPort ix, unsigned long t) { Port* prt = erts_drvport2port(ix); @@ -6592,19 +7028,8 @@ int driver_set_timer(ErlDrvPort ix, unsigned long t) if (prt->drv_ptr->timeout == NULL) return -1; - drv_cancel_timer(prt); -#ifdef ERTS_SMP - erts_create_smp_ptimer(&prt->common.u.alive.ptimer, - prt->common.id, - (ErlTimeoutProc) schedule_port_timeout, - t); -#else - erts_set_timer(&prt->common.u.alive.tm, - (ErlTimeoutProc) schedule_port_timeout, - NULL, - prt, - t); -#endif + + erts_set_port_timer(prt, (Sint64) t); return 0; } @@ -6614,28 +7039,28 @@ int driver_cancel_timer(ErlDrvPort ix) if (prt == ERTS_INVALID_ERL_DRV_PORT) return -1; ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(prt)); - drv_cancel_timer(prt); + erts_cancel_port_timer(prt); return 0; } - int driver_read_timer(ErlDrvPort ix, unsigned long* t) { Port* prt = erts_drvport2port(ix); + Sint64 left; ERTS_SMP_CHK_NO_PROC_LOCKS; if (prt == ERTS_INVALID_ERL_DRV_PORT) return -1; ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(prt)); -#ifdef ERTS_SMP - *t = (prt->common.u.alive.ptimer - ? erts_time_left(&prt->common.u.alive.ptimer->timer.tm) - : 0); -#else - *t = erts_time_left(&prt->common.u.alive.tm); -#endif + + left = erts_read_port_timer(prt); + if (left < 0) + left = 0; + + *t = (unsigned long) left; + return 0; } @@ -6655,11 +7080,33 @@ driver_get_now(ErlDrvNowData *now_data) return 0; } +ErlDrvTime +erl_drv_monotonic_time(ErlDrvTimeUnit time_unit) +{ + return (ErlDrvTime) erts_napi_monotonic_time((int) time_unit); +} + +ErlDrvTime +erl_drv_time_offset(ErlDrvTimeUnit time_unit) +{ + return (ErlDrvTime) erts_napi_time_offset((int) time_unit); +} + +ErlDrvTime +erl_drv_convert_time_unit(ErlDrvTime val, + ErlDrvTimeUnit from, + ErlDrvTimeUnit to) +{ + return (ErlDrvTime) erts_napi_convert_time_unit((ErtsMonotonicTime) val, + (int) from, + (int) to); +} + static void ref_to_driver_monitor(Eterm ref, ErlDrvMonitor *mon) { RefThing *refp; ASSERT(is_internal_ref(ref)); - ASSERT(sizeof(RefThing) <= sizeof(ErlDrvMonitor)); + ERTS_CT_ASSERT(sizeof(RefThing) <= sizeof(ErlDrvMonitor)); refp = ref_thing_ptr(ref); memset(mon,0,sizeof(ErlDrvMonitor)); memcpy(mon,refp,sizeof(RefThing)); @@ -6702,7 +7149,7 @@ int driver_monitor_process(ErlDrvPort drvport, { Port *prt; int ret; -#if !HEAP_ON_C_STACK || (defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_CHECK)) +#if defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_CHECK) ErtsSchedulerData *sched = erts_get_scheduler_data(); #endif @@ -6713,16 +7160,6 @@ int driver_monitor_process(ErlDrvPort drvport, /* Now (in SMP) we should have either the port lock (if we have a scheduler) or the port data lock (if we're a driver thread) */ ERTS_SMP_LC_ASSERT((sched != NULL || prt->port_data_lock)); - -#if !HEAP_ON_C_STACK - if (!sched) { - /* Need a separate allocation for the ref :( */ - Eterm *buf = erts_alloc(ERTS_ALC_T_TEMP_TERM, - sizeof(Eterm)*REF_THING_SIZE); - ret = do_driver_monitor_process(prt,buf,process,monitor); - erts_free(ERTS_ALC_T_TEMP_TERM,buf); - } else -#endif { DeclareTmpHeapNoproc(buf,REF_THING_SIZE); UseTmpHeapNoproc(REF_THING_SIZE); @@ -6775,7 +7212,7 @@ int driver_demonitor_process(ErlDrvPort drvport, { Port *prt; int ret; -#if !HEAP_ON_C_STACK || (defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_CHECK)) +#if defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_CHECK) ErtsSchedulerData *sched = erts_get_scheduler_data(); #endif @@ -6786,15 +7223,6 @@ int driver_demonitor_process(ErlDrvPort drvport, /* Now we should have either the port lock (if we have a scheduler) or the port data lock (if we're a driver thread) */ ERTS_SMP_LC_ASSERT((sched != NULL || prt->port_data_lock)); -#if !HEAP_ON_C_STACK - if (!sched) { - /* Need a separate allocation for the ref :( */ - Eterm *buf = erts_alloc(ERTS_ALC_T_TEMP_TERM, - sizeof(Eterm)*REF_THING_SIZE); - ret = do_driver_demonitor_process(prt,buf,monitor); - erts_free(ERTS_ALC_T_TEMP_TERM,buf); - } else -#endif { DeclareTmpHeapNoproc(buf,REF_THING_SIZE); UseTmpHeapNoproc(REF_THING_SIZE); @@ -6830,7 +7258,7 @@ ErlDrvTermData driver_get_monitored_process(ErlDrvPort drvport, { Port *prt; ErlDrvTermData ret; -#if !HEAP_ON_C_STACK || (defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_CHECK)) +#if defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_CHECK) ErtsSchedulerData *sched = erts_get_scheduler_data(); #endif @@ -6841,16 +7269,6 @@ ErlDrvTermData driver_get_monitored_process(ErlDrvPort drvport, /* Now we should have either the port lock (if we have a scheduler) or the port data lock (if we're a driver thread) */ ERTS_SMP_LC_ASSERT((sched != NULL || prt->port_data_lock)); - -#if !HEAP_ON_C_STACK - if (!sched) { - /* Need a separate allocation for the ref :( */ - Eterm *buf = erts_alloc(ERTS_ALC_T_TEMP_TERM, - sizeof(Eterm)*REF_THING_SIZE); - ret = do_driver_get_monitored_process(prt,buf,monitor); - erts_free(ERTS_ALC_T_TEMP_TERM,buf); - } else -#endif { DeclareTmpHeapNoproc(buf,REF_THING_SIZE); UseTmpHeapNoproc(REF_THING_SIZE); @@ -6874,6 +7292,7 @@ void erts_fire_port_monitor(Port *prt, Eterm ref) void (*callback)(ErlDrvData drv_data, ErlDrvMonitor *monitor); ErlDrvMonitor drv_monitor; int fpe_was_unmasked; + ERTS_MSACC_PUSH_STATE_M(); ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(prt)); ASSERT(prt->drv_ptr != NULL); @@ -6885,6 +7304,7 @@ void erts_fire_port_monitor(Port *prt, Eterm ref) callback = prt->drv_ptr->process_exit; ASSERT(callback != NULL); ref_to_driver_monitor(ref,&drv_monitor); + ERTS_MSACC_SET_STATE_CACHED_M(ERTS_MSACC_STATE_PORT); DRV_MONITOR_UNLOCK_PDL(prt); #ifdef USE_VM_PROBES if (DTRACE_ENABLED(driver_process_exit)) { @@ -6896,6 +7316,7 @@ void erts_fire_port_monitor(Port *prt, Eterm ref) (*callback)((ErlDrvData) (prt->drv_data), &drv_monitor); erts_unblock_fpe(fpe_was_unmasked); DRV_MONITOR_LOCK_PDL(prt); + ERTS_MSACC_POP_STATE_M(); /* remove monitor *after* callback */ rmon = erts_remove_monitor(&ERTS_P_MONITORS(prt), ref); DRV_MONITOR_UNLOCK_PDL(prt); @@ -6916,6 +7337,9 @@ driver_failure_term(ErlDrvPort ix, Eterm term, int eof) if (prt == ERTS_INVALID_ERL_DRV_PORT) return -1; ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(prt)); + + if (prt->async_open_port) + init_ack_send_reply(prt, prt->common.id); if (eof) flush_linebuf_messages(prt, state); if (state & ERTS_PORT_SFLG_CLOSING) { @@ -7166,7 +7590,7 @@ char *driver_dl_error(void) #define ERL_DRV_SYS_INFO_SIZE(LAST_FIELD) \ - (((size_t) &((ErlDrvSysInfo *) 0)->LAST_FIELD) \ + (offsetof(ErlDrvSysInfo, LAST_FIELD) \ + sizeof(((ErlDrvSysInfo *) 0)->LAST_FIELD)) void @@ -7182,7 +7606,7 @@ driver_system_info(ErlDrvSysInfo *sip, size_t si_size) * of ErlDrvSysInfo (introduced in driver version 1.0). */ if (!sip || si_size < ERL_DRV_SYS_INFO_SIZE(smp_support)) - erl_exit(1, + erts_exit(ERTS_ERROR_EXIT, "driver_system_info(%p, %ld) called with invalid arguments\n", sip, si_size); @@ -7228,6 +7652,18 @@ driver_system_info(ErlDrvSysInfo *sip, size_t si_size) sip->nif_major_version = ERL_NIF_MAJOR_VERSION; sip->nif_minor_version = ERL_NIF_MINOR_VERSION; } + /* + * 'dirty_scheduler_support' is the last field in the 4th version + * (driver version 3.1, NIF version 2.7) + */ + if (si_size >= ERL_DRV_SYS_INFO_SIZE(dirty_scheduler_support)) { +#if defined(ERL_NIF_DIRTY_SCHEDULER_SUPPORT) && defined(USE_THREADS) + sip->dirty_scheduler_support = 1; +#else + sip->dirty_scheduler_support = 0; +#endif + } + } @@ -7291,6 +7727,8 @@ no_stop_select_callback(ErlDrvEvent event, void* private) erts_send_error_to_logger_nogl(dsbufp); } +#define IS_DRIVER_VERSION_GE(DE,MAJOR,MINOR) \ + ((DE)->major_version >= (MAJOR) && (DE)->minor_version >= (MINOR)) static int init_driver(erts_driver_t *drv, ErlDrvEntry *de, DE_Handle *handle) @@ -7338,6 +7776,7 @@ init_driver(erts_driver_t *drv, ErlDrvEntry *de, DE_Handle *handle) drv->timeout = de->timeout ? de->timeout : no_timeout_callback; drv->ready_async = de->ready_async; drv->process_exit = de->process_exit; + drv->emergency_close = IS_DRIVER_VERSION_GE(de,3,2) ? de->emergency_close : NULL; if (de->stop_select) drv->stop_select = de->stop_select; else @@ -7356,6 +7795,8 @@ init_driver(erts_driver_t *drv, ErlDrvEntry *de, DE_Handle *handle) } } +#undef IS_DRIVER_VERSION_GE + void erts_destroy_driver(erts_driver_t *drv) { @@ -7472,15 +7913,15 @@ int null_func(void) } int -erl_drv_putenv(char *key, char *value) +erl_drv_putenv(const char *key, char *value) { - return erts_sys_putenv_raw(key, value); + return erts_sys_putenv_raw((char*)key, value); } int -erl_drv_getenv(char *key, char *value, size_t *value_size) +erl_drv_getenv(const char *key, char *value, size_t *value_size) { - return erts_sys_getenv_raw(key, value, value_size); + return erts_sys_getenv_raw((char*)key, value, value_size); } /* get heart_port @@ -7499,7 +7940,7 @@ Port *erts_get_heart_port(void) if (!port) continue; /* only examine undead or alive ports */ - if (erts_atomic32_read_nob(&port->state) & ERTS_PORT_SFLGS_DEAD) + if (erts_atomic32_read_nob(&port->state) & ERTS_PORT_SFLGS_INVALID_DRIVER_LOOKUP) continue; /* immediate atom compare */ reg = port->common.u.alive.reg; @@ -7510,3 +7951,23 @@ Port *erts_get_heart_port(void) return NULL; } + +void erts_emergency_close_ports(void) +{ + int ix, max = erts_ptab_max(&erts_port); + + for (ix = 0; ix < max; ix++) { + Port *port = erts_pix2port(ix); + + if (!port) + continue; + /* only examine undead or alive ports */ + if (erts_atomic32_read_nob(&port->state) & ERTS_PORT_SFLGS_INVALID_DRIVER_LOOKUP) + continue; + + /* emergency close socket */ + if (port->drv_ptr->emergency_close) { + port->drv_ptr->emergency_close((ErlDrvData) port->drv_data); + } + } +} diff --git a/erts/emulator/beam/module.c b/erts/emulator/beam/module.c index daa6e136c5..f5c7b177d3 100644 --- a/erts/emulator/beam/module.c +++ b/erts/emulator/beam/module.c @@ -3,16 +3,17 @@ * * Copyright Ericsson AB 1996-2013. All Rights Reserved. * - * The contents of this file are subject to the Erlang Public License, - * Version 1.1, (the "License"); you may not use this file except in - * compliance with the License. You should have received a copy of the - * Erlang Public License along with this software. If not, it can be - * retrieved online at http://www.erlang.org/. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Software distributed under the License is distributed on an "AS IS" - * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See - * the License for the specific language governing rights and limitations - * under the License. + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. * * %CopyrightEnd% */ @@ -73,8 +74,8 @@ static Module* module_alloc(Module* tmpl) erts_smp_atomic_add_nob(&tot_module_bytes, sizeof(Module)); obj->module = tmpl->module; - obj->curr.code = 0; - obj->old.code = 0; + obj->curr.code_hdr = 0; + obj->old.code_hdr = 0; obj->curr.code_length = 0; obj->old.code_length = 0; obj->slot.index = -1; @@ -102,6 +103,9 @@ void init_module_table(void) f.cmp = (HCMP_FUN) module_cmp; f.alloc = (HALLOC_FUN) module_alloc; f.free = (HFREE_FUN) module_free; + f.meta_alloc = (HMALLOC_FUN) erts_alloc; + f.meta_free = (HMFREE_FUN) erts_free; + f.meta_print = (HMPRINT_FUN) erts_print; for (i = 0; i < ERTS_NUM_CODE_IX; i++) { erts_index_init(ERTS_ALC_T_MODULE_TABLE, &module_tables[i], "module_code", diff --git a/erts/emulator/beam/module.h b/erts/emulator/beam/module.h index 5235528e98..b7468b0926 100644 --- a/erts/emulator/beam/module.h +++ b/erts/emulator/beam/module.h @@ -3,16 +3,17 @@ * * Copyright Ericsson AB 1996-2013. All Rights Reserved. * - * The contents of this file are subject to the Erlang Public License, - * Version 1.1, (the "License"); you may not use this file except in - * compliance with the License. You should have received a copy of the - * Erlang Public License along with this software. If not, it can be - * retrieved online at http://www.erlang.org/. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Software distributed under the License is distributed on an "AS IS" - * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See - * the License for the specific language governing rights and limitations - * under the License. + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. * * %CopyrightEnd% */ @@ -25,7 +26,7 @@ #endif struct erl_module_instance { - BeamInstr* code; + BeamCodeHeader* code_hdr; int code_length; /* Length of loaded code in bytes. */ unsigned catches; struct erl_module_nif* nif; @@ -36,6 +37,7 @@ struct erl_module_instance { typedef struct erl_module { IndexSlot slot; /* Must be located at top of struct! */ int module; /* Atom index for module (not tagged). */ + int seen; /* Used by finish_loading() */ struct erl_module_instance curr; struct erl_module_instance old; /* protected by "old_code" rwlock */ diff --git a/erts/emulator/beam/ops.tab b/erts/emulator/beam/ops.tab index 68fcc177ae..9e53b4bfcc 100644 --- a/erts/emulator/beam/ops.tab +++ b/erts/emulator/beam/ops.tab @@ -3,16 +3,17 @@ # # Copyright Ericsson AB 1997-2013. All Rights Reserved. # -# The contents of this file are subject to the Erlang Public License, -# Version 1.1, (the "License"); you may not use this file except in -# compliance with the License. You should have received a copy of the -# Erlang Public License along with this software. If not, it can be -# retrieved online at http://www.erlang.org/. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# Software distributed under the License is distributed on an "AS IS" -# basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See -# the License for the specific language governing rights and limitations -# under the License. +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. # # %CopyrightEnd% # @@ -38,8 +39,8 @@ too_old_compiler | never() => # necessary.) Since the instructions don't work correctly in R12B, simply # refuse to load the module. -func_info M=a a==am_module_info A=u==0 | label L | move n r => too_old_compiler -func_info M=a a==am_module_info A=u==1 | label L | move n r => too_old_compiler +func_info M=a a==am_module_info A=u==0 | label L | move n x==0 => too_old_compiler +func_info M=a a==am_module_info A=u==1 | label L | move n x==0 => too_old_compiler # The undocumented and unsupported guard BIF is_constant/1 was removed # in R13. The is_constant/2 operation is marked as obsolete in genop.tab, @@ -75,17 +76,6 @@ return # with the following call instruction, we need to make sure that # there is no line/1 instruction between the move and the call. # - -move S r | line Loc | call_ext Ar Func => \ - line Loc | move S r | call_ext Ar Func -move S r | line Loc | call_ext_last Ar Func=u$is_bif D => \ - line Loc | move S r | call_ext_last Ar Func D -move S r | line Loc | call_ext_only Ar Func=u$is_bif => \ - line Loc | move S r | call_ext_only Ar Func -move S r | line Loc | call Ar Func => \ - line Loc | move S r | call Ar Func - -# # A tail-recursive call to an external function (non-BIF) will # never be saved on the stack, so there is no reason to keep # the line instruction. (The compiler did not remove the line @@ -93,10 +83,14 @@ move S r | line Loc | call Ar Func => \ # BIFs and ordinary Erlang functions.) # -line Loc | call_ext_last Ar Func=u$is_not_bif D => \ - call_ext_last Ar Func D -line Loc | call_ext_only Ar Func=u$is_not_bif => \ - call_ext_only Ar Func +move S X0=x==0 | line Loc | call_ext Ar Func => \ + line Loc | move S X0 | call_ext Ar Func +move S X0=x==0 | line Loc | call_ext_last Ar Func=u$is_not_bif D => \ + move S X0 | call_ext_last Ar Func D +move S X0=x==0 | line Loc | call_ext_only Ar Func=u$is_not_bif => \ + move S X0 | call_ext_only Ar Func +move S X0=x==0 | line Loc | call Ar Func => \ + line Loc | move S X0 | call Ar Func line Loc | func_info M F A => func_info M F A | line Loc @@ -166,27 +160,24 @@ is_tuple Fail=f S | select_tuple_arity S=d Fail=f Size=u Rest=* => \ select_tuple_arity S=d Fail=f Size=u Rest=* => \ gen_select_tuple_arity(S, Fail, Size, Rest) -i_select_val r f I -i_select_val x f I -i_select_val y f I +i_select_val_bins x f I +i_select_val_bins y f I -i_select_val2 r f c f c f -i_select_val2 x f c f c f -i_select_val2 y f c f c f +i_select_val_lins x f I +i_select_val_lins y f I -i_select_tuple_arity2 r f A f A f -i_select_tuple_arity2 x f A f A f -i_select_tuple_arity2 y f A f A f +i_select_val2 x f c c f f +i_select_val2 y f c c f f -i_select_tuple_arity r f I i_select_tuple_arity x f I i_select_tuple_arity y f I -i_jump_on_val_zero r f I +i_select_tuple_arity2 x f A A f f +i_select_tuple_arity2 y f A A f f + i_jump_on_val_zero x f I i_jump_on_val_zero y f I -i_jump_on_val r f I I i_jump_on_val x f I I i_jump_on_val y f I I @@ -198,30 +189,21 @@ is_ne_exact L1 S1 S2 | jump Fail | label L2 | same_label(L1, L2) => \ %macro: get_list GetList -pack get_list x x x get_list x x y -get_list x x r get_list x y x get_list x y y -get_list x y r -get_list x r x -get_list x r y get_list y x x get_list y x y -get_list y x r get_list y y x get_list y y y -get_list y y r -get_list y r x -get_list y r y +# The following get_list instructions using x(0) are frequently used. get_list r x x +get_list r r y +get_list x r x get_list r x y -get_list r x r -get_list r y x -get_list r y y get_list r y r -get_list r r x -get_list r r y +get_list r x r # Old-style catch. catch y f @@ -232,33 +214,31 @@ try Y F => catch Y F try_case Y => try_end Y try_end y -try_case_end Literal=q => move Literal x | try_case_end x try_case_end s # Destructive set tuple element -set_tuple_element Lit=q Tuple Pos => move Lit x | set_tuple_element x Tuple Pos set_tuple_element s d P # Get tuple element %macro: i_get_tuple_element GetTupleElement -pack i_get_tuple_element x P x -i_get_tuple_element r P x i_get_tuple_element y P x -i_get_tuple_element x P r -i_get_tuple_element y P r %cold -i_get_tuple_element r P r i_get_tuple_element x P y -i_get_tuple_element r P y i_get_tuple_element y P y %hot +%macro: i_get_tuple_element2 GetTupleElement2 -pack +i_get_tuple_element2 x P x + +%macro: i_get_tuple_element3 GetTupleElement3 -pack +i_get_tuple_element3 x P x + %macro: is_number IsNumber -fail_action %cold -is_number f r is_number f x is_number f y %hot @@ -268,16 +248,12 @@ is_number Fail Literal=q => move Literal x | is_number Fail x jump f -case_end Literal=cq => move Literal x | case_end x -badmatch Literal=cq => move Literal x | badmatch x +case_end NotInX=cy => move NotInX x | case_end x +badmatch NotInX=cy => move NotInX x | badmatch x -case_end r case_end x -case_end y -badmatch r badmatch x -badmatch y if_end raise s s @@ -286,7 +262,7 @@ raise s s badarg j system_limit j -move C=cxy r | jump Lbl => move_jump Lbl C +move C=cxy x==0 | jump Lbl => move_jump Lbl C %macro: move_jump MoveJump -nonext move_jump f n @@ -294,9 +270,98 @@ move_jump f c move_jump f x move_jump f y -move X1=x Y1=y | move X2=x Y2=y => move2 X1 Y1 X2 Y2 -move Y1=y X1=x | move Y2=y X2=x => move2 Y1 X1 Y2 X2 -move X1=x X2=x | move X3=x X4=x => move2 X1 X2 X3 X4 + +# Movement to and from the stack is common +# Try to pack as much as we can into one instruction + +# Window move +move_window/5 +move_window/6 + +# x -> y + +move X1=x Y1=y | move X2=x Y2=y | move X3=x Y3=y | succ(Y1,Y2) | succ(Y2,Y3) => \ + move_window X1 X2 X3 Y1 Y3 + +move_window X1=x X2=x X3=x Y1=y Y3=y | move X4=x Y4=y | succ(Y3,Y4) => \ + move_window X1 X2 X3 X4 Y1 Y4 + +move_window X1=x X2=x X3=x X4=x Y1=y Y4=y | move X5=x Y5=y | succ(Y4,Y5) => \ + move_window5 X1 X2 X3 X4 X5 Y1 + +move_window X1=x X2=x X3=x Y1=y Y3=y => move_window3 X1 X2 X3 Y1 +move_window X1=x X2=x X3=x X4=x Y1=y Y4=y => move_window4 X1 X2 X3 X4 Y1 + +%macro: move_window3 MoveWindow3 -pack +%macro: move_window4 MoveWindow4 -pack +%macro: move_window5 MoveWindow5 -pack + +move_window3 x x x y +move_window4 x x x x y +move_window5 x x x x x y + +# Swap registers. +move R1=x Tmp=x | move R2=xy R1 | move Tmp R2 => swap_temp R1 R2 Tmp + +swap_temp R1 R2 Tmp | line Loc | apply Live | is_killed(Tmp, Live) => \ + swap R1 R2 | line Loc | apply Live + +swap_temp R1 R2 Tmp | line Loc | call Live Addr | is_killed(Tmp, Live) => \ + swap R1 R2 | line Loc | call Live Addr +swap_temp R1 R2 Tmp | call_only Live Addr | \ + is_killed(Tmp, Live) => swap R1 R2 | call_only Live Addr +swap_temp R1 R2 Tmp | call_last Live Addr D | \ + is_killed(Tmp, Live) => swap R1 R2 | call_last Live Addr D + +swap_temp R1 R2 Tmp | line Loc | call_ext Live Addr | is_killed(Tmp, Live) => \ + swap R1 R2 | line Loc | call_ext Live Addr +swap_temp R1 R2 Tmp | line Loc | call_ext_only Live Addr | \ + is_killed(Tmp, Live) => swap R1 R2 | line Loc | call_ext_only Live Addr +swap_temp R1 R2 Tmp | line Loc | call_ext_last Live Addr D | \ + is_killed(Tmp, Live) => swap R1 R2 | line Loc | call_ext_last Live Addr D + +%macro: swap_temp SwapTemp -pack +swap_temp x x x +swap_temp x y x + +%macro: swap Swap -pack +swap x x +swap x y + +move Src=x D1=x | move Src=x D2=x => move_dup Src D1 D2 +move Src=x SD=x | move SD=x D=x => move_dup Src SD D +move Src=x D1=x | move Src=x D2=y => move_dup Src D1 D2 +move Src=y SD=x | move SD=x D=y => move_dup Src SD D +move Src=x SD=x | move SD=x D=y => move_dup Src SD D +move Src=y SD=x | move SD=x D=x => move_dup Src SD D + +move SD=x D=x | move Src=xy SD=x => move_shift Src SD D +move SD=y D=x | move Src=x SD=y => move_shift Src SD D +move SD=x D=y | move Src=x SD=x => move_shift Src SD D + +# The transformations above guarantee that the source for +# the second move is not the same as the destination for +# the first move. That means that we can do the moves in +# parallel (fetch both values, then store them) which could +# be faster. + +move X1=x Y1=y | move X2=x Y2=y => move2_par X1 Y1 X2 Y2 +move Y1=y X1=x | move Y2=y X2=x => move2_par Y1 X1 Y2 X2 + +move X1=x X2=x | move X3=x X4=x => move2_par X1 X2 X3 X4 + +move X1=x X2=x | move X3=x Y1=y => move2_par X1 X2 X3 Y1 + +move S1=x S2=x | move X1=x Y1=y => move2_par S1 S2 X1 Y1 + +move S1=y S2=x | move X1=x Y1=y => move2_par S1 S2 X1 Y1 + +move Y1=y X1=x | move S1=x D1=x => move2_par Y1 X1 S1 D1 +move S1=x D1=x | move Y1=y X1=x => move2_par S1 D1 Y1 X1 + +move2_par X1=x Y1=y X2=x Y2=y | move X3=x Y3=y => move3 X1 Y1 X2 Y2 X3 Y3 +move2_par Y1=y X1=x Y2=y X2=x | move Y3=y X3=x => move3 Y1 X1 Y2 X2 Y3 X3 +move2_par X1=x X2=x X3=x X4=x | move X5=x X6=x => move3 X1 X2 X3 X4 X5 X6 move C=aiq X=x==1 => move_x1 C move C=aiq X=x==2 => move_x2 C @@ -304,10 +369,35 @@ move C=aiq X=x==2 => move_x2 C move_x1 c move_x2 c -%macro: move2 Move2 -pack -move2 x y x y -move2 y x y x -move2 x x x x +%macro: move_shift MoveShift -pack +move_shift x x x +move_shift y x x +move_shift x y x +move_shift x x y + +%macro: move_dup MoveDup -pack +move_dup x x x +move_dup x x y +move_dup y x x +move_dup y x y + +%macro: move2_par Move2Par -pack + +move2_par x y x y +move2_par y x y x +move2_par x x x x + +move2_par x x x y + +move2_par y x x y + +move2_par x x y x +move2_par y x x x + +%macro: move3 Move3 -pack +move3 x y x y x y +move3 y x y x y x +move3 x x x x x x # The compiler almost never generates a "move Literal y(Y)" instruction, # so let's cheat if we encounter one. @@ -317,20 +407,22 @@ move S=c D=y => move S x | move x D %macro:move Move -pack -gen_dest move x x move x y -move x r move y x -move y r -move r x -move r y -move c r move c x move n x -move n r move y y +# The following move instructions using x(0) are frequently used. + +move x r +move r x +move y r +move c r +move r y + # Receive operations. -loop_rec Fail Src | smp_mark_target_label(Fail) => i_loop_rec Fail Src +loop_rec Fail x==0 | smp_mark_target_label(Fail) => i_loop_rec Fail label L | wait_timeout Fail Src | smp_already_locked(L) => label L | i_wait_timeout_locked Fail Src wait_timeout Fail Src => i_wait_timeout Fail Src @@ -345,7 +437,7 @@ label L | timeout | smp_already_locked(L) => label L | timeout_locked remove_message timeout timeout_locked -i_loop_rec f r +i_loop_rec f loop_rec_end f wait f wait_locked f @@ -363,48 +455,57 @@ send # Optimized comparisons with one immediate/literal operand. # -is_eq_exact Lbl R=rxy C=ian => i_is_eq_exact_immed Lbl R C -is_eq_exact Lbl R=rxy C=q => i_is_eq_exact_literal R Lbl C +is_eq_exact Lbl R=xy C=ian => i_is_eq_exact_immed Lbl R C +is_eq_exact Lbl R=xy C=q => i_is_eq_exact_literal Lbl R C -is_ne_exact Lbl R=rxy C=ian => i_is_ne_exact_immed Lbl R C -is_ne_exact Lbl R=rxy C=q => i_is_ne_exact_literal R Lbl C +is_ne_exact Lbl R=xy C=ian => i_is_ne_exact_immed Lbl R C +is_ne_exact Lbl R=xy C=q => i_is_ne_exact_literal Lbl R C %macro: i_is_eq_exact_immed EqualImmed -fail_action i_is_eq_exact_immed f r c i_is_eq_exact_immed f x c i_is_eq_exact_immed f y c -i_is_eq_exact_literal r f c -i_is_eq_exact_literal x f c -i_is_eq_exact_literal y f c +i_is_eq_exact_literal f x c +i_is_eq_exact_literal f y c %macro: i_is_ne_exact_immed NotEqualImmed -fail_action -i_is_ne_exact_immed f r c i_is_ne_exact_immed f x c i_is_ne_exact_immed f y c -i_is_ne_exact_literal r f c -i_is_ne_exact_literal x f c -i_is_ne_exact_literal y f c +i_is_ne_exact_literal f x c +i_is_ne_exact_literal f y c -# -# All other comparisons. -# +is_eq_exact Lbl Y=y X=x => is_eq_exact Lbl X Y +%macro: is_eq_exact EqualExact -fail_action -pack +is_eq_exact f x x +is_eq_exact f x y +is_eq_exact f s s -is_eq_exact Lbl S1 S2 => i_fetch S1 S2 | i_is_eq_exact Lbl -is_ne_exact Lbl S1 S2 => i_fetch S1 S2 | i_is_ne_exact Lbl +%macro: is_lt IsLessThan -fail_action +is_lt f x x +is_lt f x c +is_lt f c x +%cold +is_lt f s s +%hot + +%macro: is_ge IsGreaterEqual -fail_action +is_ge f x x +is_ge f x c +is_ge f c x +%cold +is_ge f s s +%hot -is_ge Lbl S1 S2 => i_fetch S1 S2 | i_is_ge Lbl -is_lt Lbl S1 S2 => i_fetch S1 S2 | i_is_lt Lbl -is_eq Lbl S1 S2 => i_fetch S1 S2 | i_is_eq Lbl -is_ne Lbl S1 S2 => i_fetch S1 S2 | i_is_ne Lbl +%macro: is_ne_exact NotEqualExact -fail_action +is_ne_exact f s s -i_is_eq_exact f -i_is_ne_exact f -i_is_lt f -i_is_ge f -i_is_eq f -i_is_ne f +%macro: is_eq Equal -fail_action +is_eq f s s + +%macro: is_ne NotEqual -fail_action +is_ne f s s # # Putting things. @@ -422,7 +523,6 @@ i_put_tuple Dst Arity Puts=* | put S => \ i_put_tuple/2 %macro:i_put_tuple PutTuple -pack -goto:do_put_tuple -i_put_tuple r I i_put_tuple x I i_put_tuple y I @@ -439,74 +539,36 @@ put_list x n x put_list y n x put_list x x x put_list y x x -put_list x x r -put_list y r r put_list y y x put_list x y x -put_list r x x -put_list r y x -put_list r x r -put_list y y r -put_list y r x -put_list r n x -put_list x r x -put_list x y r -put_list y x r put_list y x x -put_list x r r - # put_list SrcReg Constant Dst -put_list r c r -put_list r c x -put_list r c y -put_list x c r put_list x c x put_list x c y -put_list y c r put_list y c x -put_list y c y # put_list Constant SrcReg Dst -put_list c r r -put_list c r x -put_list c r y -put_list c x r put_list c x x -put_list c x y - -put_list c y r put_list c y x -put_list c y y -%cold -put_list s s d -%hot +# The following put_list instructions using x(0) are frequently used. -%macro: i_fetch FetchArgs -pack -i_fetch c c -i_fetch c r -i_fetch c x -i_fetch c y -i_fetch r c -i_fetch r x -i_fetch r y -i_fetch x c -i_fetch x r -i_fetch x x -i_fetch x y -i_fetch y c -i_fetch y r -i_fetch y x -i_fetch y y +put_list y r r +put_list x r r +put_list r n r +put_list r n x +put_list r x x +put_list r x r +put_list x x r %cold -i_fetch s s +put_list s s d %hot # @@ -528,27 +590,27 @@ return_trace # Note: There is no 'move_return y r', since there never are any y registers # when we do move_return (if we have y registers, we must do move_deallocate_return). -move S r | return => move_return S r +move S x==0 | return => move_return S %macro: move_return MoveReturn -nonext -move_return x r -move_return c r -move_return n r +move_return x +move_return c +move_return n -move S r | deallocate D | return => move_deallocate_return S r D +move S x==0 | deallocate D | return => move_deallocate_return S D %macro: move_deallocate_return MoveDeallocateReturn -pack -nonext -move_deallocate_return x r Q -move_deallocate_return y r Q -move_deallocate_return c r Q -move_deallocate_return n r Q +move_deallocate_return x Q +move_deallocate_return y Q +move_deallocate_return c Q +move_deallocate_return n Q deallocate D | return => deallocate_return D %macro: deallocate_return DeallocateReturn -nonext deallocate_return Q -test_heap Need u==1 | put_list Y=y r r => test_heap_1_put_list Need Y +test_heap Need u==1 | put_list Y=y x==0 x==0 => test_heap_1_put_list Need Y %macro: test_heap_1_put_list TestHeapPutList -pack test_heap_1_put_list I y @@ -557,18 +619,18 @@ test_heap_1_put_list I y is_tuple Fail Literal=q => move Literal x | is_tuple Fail x is_tuple Fail=f c => jump Fail -is_tuple Fail=f S=rxy | test_arity Fail=f S=rxy Arity => is_tuple_of_arity Fail S Arity +is_tuple Fail=f S=xy | test_arity Fail=f S=xy Arity => is_tuple_of_arity Fail S Arity %macro:is_tuple_of_arity IsTupleOfArity -fail_action +is_tuple_of_arity f r A is_tuple_of_arity f x A is_tuple_of_arity f y A -is_tuple_of_arity f r A %macro: is_tuple IsTuple -fail_action +is_tuple f r is_tuple f x is_tuple f y -is_tuple f r test_arity Fail Literal=q Arity => move Literal x | test_arity Fail x Arity test_arity Fail=f c Arity => jump Fail @@ -576,95 +638,63 @@ test_arity Fail=f c Arity => jump Fail %macro: test_arity IsArity -fail_action test_arity f x A test_arity f y A -test_arity f r A - -is_tuple_of_arity Fail=f Reg Arity | get_tuple_element Reg P=u==0 Dst=xy => \ - is_tuple_of_arity Fail Reg Arity | extract_next_element Dst | original_reg Reg P - -test_arity Fail Reg Arity | get_tuple_element Reg P=u==0 Dst=xy => \ - test_arity Fail Reg Arity | extract_next_element Dst | original_reg Reg P - -original_reg Reg P1 | get_tuple_element Reg P2 Dst=xy | succ(P1, P2) => \ - extract_next_element Dst | original_reg Reg P2 - -get_tuple_element Reg P Dst => i_get_tuple_element Reg P Dst | original_reg Reg P - -original_reg Reg Pos => - -original_reg/2 - -extract_next_element D1=xy | original_reg Reg P1 | get_tuple_element Reg P2 D2=xy | \ -succ(P1, P2) | succ(D1, D2) => \ - extract_next_element2 D1 | original_reg Reg P2 -extract_next_element2 D1=xy | original_reg Reg P1 | get_tuple_element Reg P2 D2=xy | \ -succ(P1, P2) | succ2(D1, D2) => \ - extract_next_element3 D1 | original_reg Reg P2 +get_tuple_element Reg=x P1 D1=x | get_tuple_element Reg=x P2 D2=x | \ + get_tuple_element Reg=x P3 D3=x | \ + succ(P1, P2) | succ(P2, P3) | \ + succ(D1, D2) | succ(D2, D3) => i_get_tuple_element3 Reg P1 D1 -#extract_next_element3 D1=xy | original_reg Reg P1 | get_tuple_element Reg P2 D2=xy | \ -#succ(P1, P2) | succ3(D1, D2) => \ -# extract_next_element4 D1 | original_reg Reg P2 +get_tuple_element Reg=x P1 D1=x | get_tuple_element Reg=x P2 D2=x | \ + succ(P1, P2) | succ(D1, D2) => i_get_tuple_element2 Reg P1 D1 -%macro: extract_next_element ExtractNextElement -pack -extract_next_element x -extract_next_element y - -%macro: extract_next_element2 ExtractNextElement2 -pack -extract_next_element2 x -extract_next_element2 y - -%macro: extract_next_element3 ExtractNextElement3 -pack -extract_next_element3 x -extract_next_element3 y - -#%macro: extract_next_element4 ExtractNextElement4 -pack -#extract_next_element4 x -#extract_next_element4 y +get_tuple_element Reg P Dst => i_get_tuple_element Reg P Dst is_integer Fail=f i => is_integer Fail=f an => jump Fail is_integer Fail Literal=q => move Literal x | is_integer Fail x -is_integer Fail=f S=rx | allocate Need Regs => is_integer_allocate Fail S Need Regs +is_integer Fail=f S=x | allocate Need Regs => is_integer_allocate Fail S Need Regs %macro: is_integer_allocate IsIntegerAllocate -fail_action is_integer_allocate f x I I -is_integer_allocate f r I I %macro: is_integer IsInteger -fail_action is_integer f x is_integer f y -is_integer f r is_list Fail=f n => is_list Fail Literal=q => move Literal x | is_list Fail x is_list Fail=f c => jump Fail %macro: is_list IsList -fail_action -is_list f r is_list f x %cold is_list f y %hot -is_nonempty_list Fail=f S=rx | allocate Need Rs => is_nonempty_list_allocate Fail S Need Rs +is_nonempty_list Fail=f S=x | allocate Need Rs => is_nonempty_list_allocate Fail S Need Rs %macro:is_nonempty_list_allocate IsNonemptyListAllocate -fail_action -pack -is_nonempty_list_allocate f x I t is_nonempty_list_allocate f r I t +is_nonempty_list_allocate f x I t -is_nonempty_list F=f r | test_heap I1 I2 => is_non_empty_list_test_heap F r I1 I2 +is_nonempty_list F=f x==0 | test_heap I1 I2 => is_non_empty_list_test_heap F I1 I2 %macro: is_non_empty_list_test_heap IsNonemptyListTestHeap -fail_action -pack -is_non_empty_list_test_heap f r I t +is_non_empty_list_test_heap f I t + +is_nonempty_list Fail=f S=x | get_list S D1=x D2=x => \ + is_nonempty_list_get_list Fail S D1 D2 + +%macro: is_nonempty_list_get_list IsNonemptyListGetList -fail_action -pack +is_nonempty_list_get_list f r x x +is_nonempty_list_get_list f x x x %macro: is_nonempty_list IsNonemptyList -fail_action is_nonempty_list f x is_nonempty_list f y -is_nonempty_list f r %macro: is_atom IsAtom -fail_action is_atom f x -is_atom f r %cold is_atom f y %hot @@ -672,7 +702,6 @@ is_atom Fail=f a => is_atom Fail=f niq => jump Fail %macro: is_float IsFloat -fail_action -is_float f r is_float f x %cold is_float f y @@ -686,12 +715,10 @@ is_nil Fail=f qia => jump Fail %macro: is_nil IsNil -fail_action is_nil f x is_nil f y -is_nil f r is_binary Fail Literal=q => move Literal x | is_binary Fail x is_binary Fail=f c => jump Fail %macro: is_binary IsBinary -fail_action -is_binary f r is_binary f x %cold is_binary f y @@ -703,7 +730,6 @@ is_bitstr Fail Term => is_bitstring Fail Term is_bitstring Fail Literal=q => move Literal x | is_bitstring Fail x is_bitstring Fail=f c => jump Fail %macro: is_bitstring IsBitstring -fail_action -is_bitstring f r is_bitstring f x %cold is_bitstring f y @@ -711,7 +737,6 @@ is_bitstring f y is_reference Fail=f cq => jump Fail %macro: is_reference IsRef -fail_action -is_reference f r is_reference f x %cold is_reference f y @@ -719,7 +744,6 @@ is_reference f y is_pid Fail=f cq => jump Fail %macro: is_pid IsPid -fail_action -is_pid f r is_pid f x %cold is_pid f y @@ -727,7 +751,6 @@ is_pid f y is_port Fail=f cq => jump Fail %macro: is_port IsPort -fail_action -is_port f r is_port f x %cold is_port f y @@ -739,14 +762,12 @@ is_boolean Fail=f ac => jump Fail %cold %macro: is_boolean IsBoolean -fail_action -is_boolean f r is_boolean f x is_boolean f y %hot is_function2 Fail=f acq Arity => jump Fail is_function2 Fail=f Fun a => jump Fail -is_function2 Fail Fun Literal=q => move Literal x | is_function2 Fail Fun x is_function2 f s s %macro: is_function2 IsFunction2 -fail_action @@ -886,76 +907,76 @@ call_ext_only u==3 u$func:erlang:hibernate/3 => i_hibernate %unless USE_VM_PROBES call_ext Arity u$func:erlang:dt_get_tag/0 => \ - move a=am_undefined r + move a=am_undefined x=0 call_ext_last Arity u$func:erlang:dt_get_tag/0 D => \ - move a=am_undefined r | deallocate D | return + move a=am_undefined x=0 | deallocate D | return call_ext_only Arity u$func:erlang:dt_get_tag/0 => \ - move a=am_undefined r | return - -move Any r | call_ext Arity u$func:erlang:dt_put_tag/1 => \ - move a=am_undefined r -move Any r | call_ext_last Arity u$func:erlang:dt_put_tag/1 D => \ - move a=am_undefined r | deallocate D | return -move Any r | call_ext_only Arity u$func:erlang:dt_put_tag/1 => \ - move a=am_undefined r | return + move a=am_undefined x=0 | return + +move Any x==0 | call_ext Arity u$func:erlang:dt_put_tag/1 => \ + move a=am_undefined x=0 +move Any x==0 | call_ext_last Arity u$func:erlang:dt_put_tag/1 D => \ + move a=am_undefined x=0 | deallocate D | return +move Any x==0 | call_ext_only Arity u$func:erlang:dt_put_tag/1 => \ + move a=am_undefined x=0 | return call_ext Arity u$func:erlang:dt_put_tag/1 => \ - move a=am_undefined r + move a=am_undefined x=0 call_ext_last Arity u$func:erlang:dt_put_tag/1 D => \ - move a=am_undefined r | deallocate D | return + move a=am_undefined x=0 | deallocate D | return call_ext_only Arity u$func:erlang:dt_put_tag/1 => \ - move a=am_undefined r | return + move a=am_undefined x=0 | return call_ext Arity u$func:erlang:dt_get_tag_data/0 => \ - move a=am_undefined r + move a=am_undefined x=0 call_ext_last Arity u$func:erlang:dt_get_tag_data/0 D => \ - move a=am_undefined r | deallocate D | return + move a=am_undefined x=0 | deallocate D | return call_ext_only Arity u$func:erlang:dt_get_tag_data/0 => \ - move a=am_undefined r | return - -move Any r | call_ext Arity u$func:erlang:dt_spread_tag/1 => \ - move a=am_true r -move Any r | call_ext_last Arity u$func:erlang:dt_spread_tag/1 D => \ - move a=am_true r | deallocate D | return -move Any r | call_ext_only Arity u$func:erlang:dt_spread_tag/1 => \ - move a=am_true r | return + move a=am_undefined x=0 | return + +move Any x==0 | call_ext Arity u$func:erlang:dt_spread_tag/1 => \ + move a=am_true x=0 +move Any x==0 | call_ext_last Arity u$func:erlang:dt_spread_tag/1 D => \ + move a=am_true x=0 | deallocate D | return +move Any x==0 | call_ext_only Arity u$func:erlang:dt_spread_tag/1 => \ + move a=am_true x=0 | return call_ext Arity u$func:erlang:dt_spread_tag/1 => \ - move a=am_true r + move a=am_true x=0 call_ext_last Arity u$func:erlang:dt_spread_tag/1 D => \ - move a=am_true r | deallocate D | return + move a=am_true x=0 | deallocate D | return call_ext_only Arity u$func:erlang:dt_spread_tag/1 => \ - move a=am_true r | return - -move Any r | call_ext Arity u$func:erlang:dt_restore_tag/1 => \ - move a=am_true r -move Any r | call_ext_last Arity u$func:erlang:dt_restore_tag/1 D => \ - move a=am_true r | deallocate D | return -move Any r | call_ext_only Arity u$func:erlang:dt_restore_tag/1 => \ - move a=am_true r | return + move a=am_true x=0 | return + +move Any x==0 | call_ext Arity u$func:erlang:dt_restore_tag/1 => \ + move a=am_true x=0 +move Any x==0 | call_ext_last Arity u$func:erlang:dt_restore_tag/1 D => \ + move a=am_true x=0 | deallocate D | return +move Any x==0 | call_ext_only Arity u$func:erlang:dt_restore_tag/1 => \ + move a=am_true x=0 | return call_ext Arity u$func:erlang:dt_restore_tag/1 => \ - move a=am_true r + move a=am_true x=0 call_ext_last Arity u$func:erlang:dt_restore_tag/1 D => \ - move a=am_true r | deallocate D | return + move a=am_true x=0 | deallocate D | return call_ext_only Arity u$func:erlang:dt_restore_tag/1 => \ - move a=am_true r | return - -move Any r | call_ext Arity u$func:erlang:dt_prepend_vm_tag_data/1 => \ - move Any r -move Any r | call_ext_last Arity u$func:erlang:dt_prepend_vm_tag_data/1 D => \ - move Any r | deallocate D | return -move Any r | call_ext_only Arity u$func:erlang:dt_prepend_vm_tag_data/1 => \ - move Any r | return + move a=am_true x=0 | return + +move Any x==0 | call_ext Arity u$func:erlang:dt_prepend_vm_tag_data/1 => \ + move Any x=0 +move Any x==0 | call_ext_last Arity u$func:erlang:dt_prepend_vm_tag_data/1 D => \ + move Any x=0 | deallocate D | return +move Any x==0 | call_ext_only Arity u$func:erlang:dt_prepend_vm_tag_data/1 => \ + move Any x=0 | return call_ext Arity u$func:erlang:dt_prepend_vm_tag_data/1 => call_ext_last Arity u$func:erlang:dt_prepend_vm_tag_data/1 D => \ deallocate D | return call_ext_only Arity u$func:erlang:dt_prepend_vm_tag_data/1 => \ return -move Any r | call_ext Arity u$func:erlang:dt_append_vm_tag_data/1 => \ - move Any r -move Any r | call_ext_last Arity u$func:erlang:dt_append_vm_tag_data/1 D => \ - move Any r | deallocate D | return -move Any r | call_ext_only Arity u$func:erlang:dt_append_vm_tag_data/1 => \ - move Any r | return +move Any x==0 | call_ext Arity u$func:erlang:dt_append_vm_tag_data/1 => \ + move Any x=0 +move Any x==0 | call_ext_last Arity u$func:erlang:dt_append_vm_tag_data/1 D => \ + move Any x=0 | deallocate D | return +move Any x==0 | call_ext_only Arity u$func:erlang:dt_append_vm_tag_data/1 => \ + move Any x=0 | return call_ext Arity u$func:erlang:dt_append_vm_tag_data/1 => call_ext_last Arity u$func:erlang:dt_append_vm_tag_data/1 D => \ deallocate D | return @@ -963,10 +984,17 @@ call_ext_only Arity u$func:erlang:dt_append_vm_tag_data/1 => \ return # Can happen after one of the transformations above. -move Discarded r | move Something r => move Something r +move Discarded x==0 | move Something x==0 => move Something x=0 %endif +call_ext u==0 u$func:os:perf_counter/0 => \ + i_perf_counter +call_ext_last u==0 u$func:os:perf_counter/0 D => \ + i_perf_counter | deallocate_return D +call_ext_only u==0 u$func:os:perf_counter/0 => \ + i_perf_counter | return + # # The general case for BIFs that have no special instructions. # A BIF used in the tail must be followed by a return instruction. @@ -988,9 +1016,9 @@ call_ext_only Ar=u Bif=u$is_bif => \ # with call instructions. # -move S=c r | call_ext Ar=u Func=u$is_not_bif => i_move_call_ext S r Func -move S=c r | call_ext_last Ar=u Func=u$is_not_bif D => i_move_call_ext_last Func D S r -move S=c r | call_ext_only Ar=u Func=u$is_not_bif => i_move_call_ext_only Func S r +move S=c x==0 | call_ext Ar=u Func=u$is_not_bif => i_move_call_ext S Func +move S=c x==0 | call_ext_last Ar=u Func=u$is_not_bif D => i_move_call_ext_last Func D S +move S=c x==0 | call_ext_only Ar=u Func=u$is_not_bif => i_move_call_ext_only Func S call_ext Ar Func => i_call_ext Func call_ext_last Ar Func D => i_call_ext_last Func D @@ -1006,6 +1034,8 @@ i_apply_fun_only i_hibernate +i_perf_counter + call_bif e # @@ -1015,80 +1045,74 @@ call_bif e bif0 u$bif:erlang:self/0 Dst=d => self Dst bif0 u$bif:erlang:node/0 Dst=d => node Dst -bif1 Fail Bif=u$bif:erlang:get/1 Src=s Dst=d => i_get Src Dst +bif1 Fail Bif=u$bif:erlang:get/1 Src=s Dst=d => gen_get(Src, Dst) -bif2 Jump=j u$bif:erlang:element/2 S1=s S2=rxy Dst=d => gen_element(Jump, S1, S2, Dst) +bif2 Jump=j u$bif:erlang:element/2 S1=s S2=xy Dst=d => gen_element(Jump, S1, S2, Dst) -bif1 Fail Bif Literal=q Dst => move Literal x | bif1 Fail Bif x Dst bif1 p Bif S1 Dst => bif1_body Bif S1 Dst -bif1_body Bif Literal=q Dst => move Literal x | bif1_body Bif x Dst - -bif2 p Bif S1 S2 Dst => i_fetch S1 S2 | i_bif2_body Bif Dst -bif2 Fail Bif S1 S2 Dst => i_fetch S1 S2 | i_bif2 Fail Bif Dst +bif2 p Bif S1 S2 Dst => i_bif2_body Bif S1 S2 Dst +bif2 Fail Bif S1 S2 Dst => i_bif2 Fail Bif S1 S2 Dst +i_get_hash c I d i_get s d %macro: self Self -self r self x self y %macro: node Node -node r node x %cold node y %hot -i_fast_element r j I d -i_fast_element x j I d -i_fast_element y j I d +i_fast_element j x I d +i_fast_element j y I d -i_element r j s d -i_element x j s d -i_element y j s d +i_element j x s d +i_element j y s d bif1 f b s d bif1_body b s d -i_bif2 f b d -i_bif2_body b d +i_bif2 f b s s d +i_bif2_body b s s d # # Internal calls. # -move S=c r | call Ar P=f => i_move_call S r P -move S=s r | call Ar P=f => move_call S r P +move S=c x==0 | call Ar P=f => i_move_call S P +move S=s x==0 | call Ar P=f => move_call S P -i_move_call c r f +i_move_call c f %macro:move_call MoveCall -arg_f -size -nonext -move_call/3 +move_call/2 -move_call x r f -move_call y r f +move_call x f +move_call y f -move S=c r | call_last Ar P=f D => i_move_call_last P D S r -move S r | call_last Ar P=f D => move_call_last S r P D +move S=c x==0 | call_last Ar P=f D => i_move_call_last P D S +move S x==0 | call_last Ar P=f D => move_call_last S P D -i_move_call_last f P c r +i_move_call_last f P c %macro:move_call_last MoveCallLast -arg_f -nonext -pack -move_call_last/4 -move_call_last x r f Q -move_call_last y r f Q +move_call_last/3 +move_call_last x f Q +move_call_last y f Q -move S=c r | call_only Ar P=f => i_move_call_only P S r -move S=x r | call_only Ar P=f => move_call_only S r P +move S=c x==0 | call_only Ar P=f => i_move_call_only P S +move S=x x==0 | call_only Ar P=f => move_call_only S P -i_move_call_only f c r +i_move_call_only f c %macro:move_call_only MoveCallOnly -arg_f -nonext -move_call_only/3 +move_call_only/2 -move_call_only x r f +move_call_only x f call Ar Func => i_call Func call_last Ar Func D => i_call_last Func D @@ -1102,9 +1126,9 @@ i_call_ext e i_call_ext_last e P i_call_ext_only e -i_move_call_ext c r e -i_move_call_ext_last e P c r -i_move_call_ext_only e c r +i_move_call_ext c e +i_move_call_ext_last e P c +i_move_call_ext_only e c # Fun calls. @@ -1124,7 +1148,6 @@ i_make_fun I t %macro: is_function IsFunction -fail_action is_function f x is_function f y -is_function f r is_function Fail=f c => jump Fail func_info M F A => i_func_info u M F A @@ -1136,131 +1159,105 @@ func_info M F A => i_func_info u M F A %cold bs_start_match2 Fail=f ica X Y D => jump Fail bs_start_match2 Fail Bin X Y D => i_bs_start_match2 Bin Fail X Y D -i_bs_start_match2 r f I I d i_bs_start_match2 x f I I d i_bs_start_match2 y f I I d bs_save2 Reg Index => gen_bs_save(Reg, Index) -i_bs_save2 r I i_bs_save2 x I bs_restore2 Reg Index => gen_bs_restore(Reg, Index) -i_bs_restore2 r I i_bs_restore2 x I # Matching integers bs_match_string Fail Ms Bits Val => i_bs_match_string Ms Fail Bits Val -i_bs_match_string r f I I i_bs_match_string x f I I # Fetching integers from binaries. -bs_get_integer2 Fail=f Ms=rx Live=u Sz=sq Unit=u Flags=u Dst=d => \ +bs_get_integer2 Fail=f Ms=x Live=u Sz=sq Unit=u Flags=u Dst=d => \ gen_get_integer2(Fail, Ms, Live, Sz, Unit, Flags, Dst) -i_bs_get_integer_small_imm r I f I d i_bs_get_integer_small_imm x I f I d -i_bs_get_integer_imm r I I f I d i_bs_get_integer_imm x I I f I d -i_bs_get_integer f I I d -i_bs_get_integer_8 r f d +i_bs_get_integer f I I s s d i_bs_get_integer_8 x f d -i_bs_get_integer_16 r f d i_bs_get_integer_16 x f d -i_bs_get_integer_32 r f I d i_bs_get_integer_32 x f I d # Fetching binaries from binaries. -bs_get_binary2 Fail=f Ms=rx Live=u Sz=sq Unit=u Flags=u Dst=d => \ +bs_get_binary2 Fail=f Ms=x Live=u Sz=sq Unit=u Flags=u Dst=d => \ gen_get_binary2(Fail, Ms, Live, Sz, Unit, Flags, Dst) %macro: i_bs_get_binary_imm2 BsGetBinaryImm_2 -fail_action -gen_dest %macro: i_bs_get_binary2 BsGetBinary_2 -fail_action -gen_dest %macro: i_bs_get_binary_all2 BsGetBinaryAll_2 -fail_action -gen_dest -i_bs_get_binary_imm2 f r I I I d i_bs_get_binary_imm2 f x I I I d -i_bs_get_binary2 f r I s I d i_bs_get_binary2 f x I s I d -i_bs_get_binary_all2 f r I I d i_bs_get_binary_all2 f x I I d -i_bs_get_binary_all_reuse r f I i_bs_get_binary_all_reuse x f I # Fetching float from binaries. -bs_get_float2 Fail=f Ms=rx Live=u Sz=s Unit=u Flags=u Dst=d => \ +bs_get_float2 Fail=f Ms=x Live=u Sz=s Unit=u Flags=u Dst=d => \ gen_get_float2(Fail, Ms, Live, Sz, Unit, Flags, Dst) -bs_get_float2 Fail=f Ms=rx Live=u Sz=q Unit=u Flags=u Dst=d => jump Fail +bs_get_float2 Fail=f Ms=x Live=u Sz=q Unit=u Flags=u Dst=d => jump Fail %macro: i_bs_get_float2 BsGetFloat2 -fail_action -gen_dest -i_bs_get_float2 f r I s I d i_bs_get_float2 f x I s I d # Miscellanous -bs_skip_bits2 Fail=f Ms=rx Sz=s Unit=u Flags=u => \ - gen_skip_bits2(Fail, Ms, Sz, Unit, Flags) -bs_skip_bits2 Fail=f Ms=rx Sz=q Unit=u Flags=u => \ +bs_skip_bits2 Fail=f Ms=x Sz=sq Unit=u Flags=u => \ gen_skip_bits2(Fail, Ms, Sz, Unit, Flags) %macro: i_bs_skip_bits_imm2 BsSkipBitsImm2 -fail_action -i_bs_skip_bits_imm2 f r I i_bs_skip_bits_imm2 f x I %macro: i_bs_skip_bits2 BsSkipBits2 -fail_action -i_bs_skip_bits2 f r x I -i_bs_skip_bits2 f r y I i_bs_skip_bits2 f x x I -i_bs_skip_bits2 f x r I i_bs_skip_bits2 f x y I %macro: i_bs_skip_bits_all2 BsSkipBitsAll2 -fail_action -i_bs_skip_bits_all2 f r I i_bs_skip_bits_all2 f x I -bs_test_tail2 Fail=f Ms=rx Bits=u==0 => bs_test_zero_tail2 Fail Ms -bs_test_tail2 Fail=f Ms=rx Bits=u => bs_test_tail_imm2 Fail Ms Bits -bs_test_zero_tail2 f r +bs_test_tail2 Fail=f Ms=x Bits=u==0 => bs_test_zero_tail2 Fail Ms +bs_test_tail2 Fail=f Ms=x Bits=u => bs_test_tail_imm2 Fail Ms Bits bs_test_zero_tail2 f x -bs_test_tail_imm2 f r I bs_test_tail_imm2 f x I bs_test_unit F Ms Unit=u==8 => bs_test_unit8 F Ms -bs_test_unit f r I bs_test_unit f x I -bs_test_unit8 f r bs_test_unit8 f x -bs_context_to_binary r +# An y register operand for bs_context_to_binary is rare, +# but can happen because of inlining. + +bs_context_to_binary Y=y => move Y x | bs_context_to_binary x + bs_context_to_binary x -bs_context_to_binary y # # Utf8/utf16/utf32 support. (R12B-5) # -bs_get_utf8 Fail=f Ms=rx u u Dst=d => i_bs_get_utf8 Ms Fail Dst -i_bs_get_utf8 r f d +bs_get_utf8 Fail=f Ms=x u u Dst=d => i_bs_get_utf8 Ms Fail Dst i_bs_get_utf8 x f d -bs_skip_utf8 Fail=f Ms=rx u u => i_bs_get_utf8 Ms Fail x +bs_skip_utf8 Fail=f Ms=x u u => i_bs_get_utf8 Ms Fail x -bs_get_utf16 Fail=f Ms=rx u Flags=u Dst=d => i_bs_get_utf16 Ms Fail Flags Dst -bs_skip_utf16 Fail=f Ms=rx u Flags=u => i_bs_get_utf16 Ms Fail Flags x +bs_get_utf16 Fail=f Ms=x u Flags=u Dst=d => i_bs_get_utf16 Ms Fail Flags Dst +bs_skip_utf16 Fail=f Ms=x u Flags=u => i_bs_get_utf16 Ms Fail Flags x -i_bs_get_utf16 r f I d i_bs_get_utf16 x f I d -bs_get_utf32 Fail=f Ms=rx Live=u Flags=u Dst=d => \ +bs_get_utf32 Fail=f Ms=x Live=u Flags=u Dst=d => \ bs_get_integer2 Fail Ms Live i=32 u=1 Flags Dst | \ - i_fetch Dst Ms | \ - i_bs_validate_unicode_retract Fail -bs_skip_utf32 Fail=f Ms=rx Live=u Flags=u => \ + i_bs_validate_unicode_retract Fail Dst Ms +bs_skip_utf32 Fail=f Ms=x Live=u Flags=u => \ bs_get_integer2 Fail Ms Live i=32 u=1 Flags x | \ - i_fetch x Ms | \ - i_bs_validate_unicode_retract Fail + i_bs_validate_unicode_retract Fail x Ms -i_bs_validate_unicode_retract j +i_bs_validate_unicode_retract j s s %hot # @@ -1282,13 +1279,12 @@ bs_init2 Fail Sz=u Words Regs Flags Dst => \ bs_init2 Fail Sz Words=u==0 Regs Flags Dst => \ i_bs_init_fail Sz Fail Regs Dst bs_init2 Fail Sz Words Regs Flags Dst => \ - i_fetch Sz r | i_bs_init_fail_heap Words Fail Regs Dst + i_bs_init_fail_heap Sz Words Fail Regs Dst -i_bs_init_fail r j I d i_bs_init_fail x j I d i_bs_init_fail y j I d -i_bs_init_fail_heap I j I d +i_bs_init_fail_heap s I j I d i_bs_init I I d i_bs_init_heap_bin I I d @@ -1305,39 +1301,35 @@ bs_init_bits Fail Sz=u Words Regs Flags Dst => i_bs_init_bits_heap Sz Words Reg bs_init_bits Fail Sz Words=u==0 Regs Flags Dst => \ i_bs_init_bits_fail Sz Fail Regs Dst bs_init_bits Fail Sz Words Regs Flags Dst => \ - i_fetch Sz r | i_bs_init_bits_fail_heap Words Fail Regs Dst + i_bs_init_bits_fail_heap Sz Words Fail Regs Dst -i_bs_init_bits_fail r j I d i_bs_init_bits_fail x j I d i_bs_init_bits_fail y j I d -i_bs_init_bits_fail_heap I j I d +i_bs_init_bits_fail_heap s I j I d i_bs_init_bits I I d i_bs_init_bits_heap I I I d bs_add Fail S1=i==0 S2 Unit=u==1 D => move S2 D -bs_add Fail S1 S2 Unit D => i_fetch S1 S2 | i_bs_add Fail Unit D -i_bs_add j I d +bs_add j s s I d bs_append Fail Size Extra Live Unit Bin Flags Dst => \ - i_fetch Size Bin | i_bs_append Fail Extra Live Unit Dst + move Bin x | i_bs_append Fail Extra Live Unit Size Dst bs_private_append Fail Size Unit Bin Flags Dst => \ - i_fetch Size Bin | i_bs_private_append Fail Unit Dst + i_bs_private_append Fail Unit Size Bin Dst bs_init_writable -i_bs_append j I I I d -i_bs_private_append j I d +i_bs_append j I I I s d +i_bs_private_append j I s s d # # Storing integers into binaries. # -bs_put_integer Fail=j Sz=s Unit=u Flags=u Literal=q => \ - move Literal x | bs_put_integer Fail Sz Unit Flags x bs_put_integer Fail=j Sz=sq Unit=u Flags=u Src=s => \ gen_put_integer(Fail, Sz, Unit, Flags, Src) @@ -1351,32 +1343,22 @@ i_new_bs_put_integer_imm j I I s # Utf8/utf16/utf32 support. (R12B-5) # -bs_utf8_size Fail Literal=q Dst=d => \ - move Literal x | bs_utf8_size Fail x Dst bs_utf8_size j Src=s Dst=d => i_bs_utf8_size Src Dst i_bs_utf8_size s d -bs_utf16_size Fail Literal=q Dst=d => \ - move Literal x | bs_utf16_size Fail x Dst bs_utf16_size j Src=s Dst=d => i_bs_utf16_size Src Dst i_bs_utf16_size s d -bs_put_utf8 Fail=j Flags=u Literal=q => \ - move Literal x | bs_put_utf8 Fail Flags x bs_put_utf8 Fail u Src=s => i_bs_put_utf8 Fail Src i_bs_put_utf8 j s -bs_put_utf16 Fail=j Flags=u Literal=q => \ - move Literal x | bs_put_utf16 Fail Flags x bs_put_utf16 Fail Flags=u Src=s => i_bs_put_utf16 Fail Flags Src i_bs_put_utf16 j I s -bs_put_utf32 Fail=j Flags=u Literal=q => \ - move Literal x | bs_put_utf32 Fail Flags x bs_put_utf32 Fail=j Flags=u Src=s => \ i_bs_validate_unicode Fail Src | bs_put_integer Fail i=32 u=1 Flags Src @@ -1387,9 +1369,6 @@ i_bs_validate_unicode j s # bs_put_float Fail Sz=q Unit Flags Val => badarg Fail -bs_put_float Fail=j Sz Unit=u Flags=u Literal=q => \ - move Literal x | bs_put_float Fail Sz Unit Flags x - bs_put_float Fail=j Sz=s Unit=u Flags=u Src=s => \ gen_put_float(Fail, Sz, Unit, Flags, Src) @@ -1403,8 +1382,6 @@ i_new_bs_put_float_imm j I I s # Storing binaries into binaries. # -bs_put_binary Fail Sz Unit Flags Literal=q => \ - move Literal x | bs_put_binary Fail Sz Unit Flags x bs_put_binary Fail=j Sz=s Unit=u Flags=u Src=s => \ gen_put_binary(Fail, Sz, Unit, Flags, Src) @@ -1469,144 +1446,158 @@ apply_last I P # Map instructions in R17. # -put_map_assoc F n Dst Live Size Rest=* => new_map F Dst Live Size Rest -put_map_assoc F Src=s Dst Live Size Rest=* => \ +sorted_put_map_assoc/5 +put_map_assoc F Map Dst Live Size Rest=* | map_key_sort(Size, Rest) => \ + sorted_put_map_assoc F Map Dst Live Size Rest + +sorted_put_map_exact/5 +put_map_exact F Map Dst Live Size Rest=* | map_key_sort(Size, Rest) => \ + sorted_put_map_exact F Map Dst Live Size Rest + +sorted_put_map_assoc j Map Dst Live Size Rest=* | is_empty_map(Map) => \ + new_map Dst Live Size Rest +sorted_put_map_assoc F Src=s Dst Live Size Rest=* => \ update_map_assoc F Src Dst Live Size Rest -put_map_assoc F Src Dst Live Size Rest=* => \ +sorted_put_map_assoc F Src Dst Live Size Rest=* => \ move Src x | update_map_assoc F x Dst Live Size Rest -put_map_exact F n Dst Live Size Rest=* => new_map F Dst Live Size Rest -put_map_exact F Src=s Dst Live Size Rest=* => \ + +sorted_put_map_exact F Src=s Dst Live Size Rest=* => \ update_map_exact F Src Dst Live Size Rest -put_map_exact F Src Dst Live Size Rest=* => \ +sorted_put_map_exact F Src Dst Live Size Rest=* => \ move Src x | update_map_exact F x Dst Live Size Rest -new_map j d I I +new_map d I I update_map_assoc j s d I I update_map_exact j s d I I -is_map Fail Literal=q => move Literal x | is_map Fail x -is_map Fail c => jump Fail +is_map Fail Lit=q | literal_is_map(Lit) => +is_map Fail cq => jump Fail %macro: is_map IsMap -fail_action -is_map f r is_map f x is_map f y -## Transform has_map_field(s) #{ K1 := _, K2 := _ } +## Transform has_map_fields #{ K1 := _, K2 := _ } to has_map_elements -has_map_field/3 - -has_map_fields Fail Src Size=u==1 Rest=* => gen_has_map_field(Fail,Src,Size,Rest) -has_map_fields Fail Src Size Rest=* => i_has_map_fields Fail Src Size Rest - -i_has_map_fields f s I - -has_map_field Fail Src=rxy Key=arxy => i_has_map_field Fail Src Key -has_map_field Fail Src Key => move Key x | i_has_map_field Fail Src x - -%macro: i_has_map_field HasMapField -fail_action -i_has_map_field f r a -i_has_map_field f x a -i_has_map_field f y a -i_has_map_field f r r -i_has_map_field f x r -i_has_map_field f y r -i_has_map_field f r x -i_has_map_field f x x -i_has_map_field f y x -i_has_map_field f r y -i_has_map_field f x y -i_has_map_field f y y +has_map_fields Fail Src Size Rest=* => \ + gen_has_map_fields(Fail, Src, Size, Rest) ## Transform get_map_elements(s) #{ K1 := V1, K2 := V2 } -get_map_element/4 - -get_map_elements Fail Src=rxy Size=u==2 Rest=* => gen_get_map_element(Fail,Src,Size,Rest) -get_map_elements Fail Src Size Rest=* => i_get_map_elements Fail Src Size Rest +get_map_elements Fail Src=xy Size=u==2 Rest=* => \ + gen_get_map_element(Fail, Src, Size, Rest) +get_map_elements Fail Src Size Rest=* | map_key_sort(Size, Rest) => \ + gen_get_map_elements(Fail, Src, Size, Rest) i_get_map_elements f s I -get_map_element Fail Src=rxy Key=ax Dst => i_get_map_element Fail Src Key Dst -get_map_element Fail Src=rxy Key=rycq Dst => \ +i_get_map_element Fail Src=xy Key=y Dst => \ move Key x | i_get_map_element Fail Src x Dst -get_map_element Fail Src Key Dst => jump Fail + +%macro: i_get_map_element_hash GetMapElementHash -fail_action +i_get_map_element_hash f x c I x +i_get_map_element_hash f y c I x +i_get_map_element_hash f x c I y +i_get_map_element_hash f y c I y %macro: i_get_map_element GetMapElement -fail_action -i_get_map_element f r a r -i_get_map_element f x a r -i_get_map_element f y a r -i_get_map_element f r a x -i_get_map_element f x a x -i_get_map_element f y a x -i_get_map_element f r a y -i_get_map_element f x a y -i_get_map_element f y a y -i_get_map_element f r x r -i_get_map_element f x x r -i_get_map_element f y x r -i_get_map_element f r x x i_get_map_element f x x x i_get_map_element f y x x -i_get_map_element f r x y i_get_map_element f x x y i_get_map_element f y x y # +# Convert the plus operations to a generic plus instruction. +# +gen_plus/5 +gen_minus/5 + +gc_bif1 Fail Live u$bif:erlang:splus/1 Src Dst => \ + gen_plus Fail Live Src i Dst +gc_bif2 Fail Live u$bif:erlang:splus/2 S1 S2 Dst => \ + gen_plus Fail Live S1 S2 Dst + +gc_bif1 Fail Live u$bif:erlang:sminus/1 Src Dst => \ + gen_minus Fail Live i Src Dst +gc_bif2 Fail Live u$bif:erlang:sminus/2 S1 S2 Dst => \ + gen_minus Fail Live S1 S2 Dst + +# # Optimize addition and subtraction of small literals using # the i_increment/4 instruction (in bodies, not in guards). # -gc_bif2 p Live u$bif:erlang:splus/2 Int=i Reg=d Dst => \ +gen_plus p Live Int=i Reg=d Dst => \ gen_increment(Reg, Int, Live, Dst) -gc_bif2 p Live u$bif:erlang:splus/2 Reg=d Int=i Dst => \ +gen_plus p Live Reg=d Int=i Dst => \ gen_increment(Reg, Int, Live, Dst) -gc_bif2 p Live u$bif:erlang:sminus/2 Reg=d Int=i Dst | \ - negation_is_small(Int) => \ +gen_minus p Live Reg=d Int=i Dst | negation_is_small(Int) => \ gen_increment_from_minus(Reg, Int, Live, Dst) # # GCing arithmetic instructions. # -gc_bif2 Fail I u$bif:erlang:splus/2 S1 S2 Dst=d => i_fetch S1 S2 | i_plus Fail I Dst -gc_bif2 Fail I u$bif:erlang:sminus/2 S1 S2 Dst=d => i_fetch S1 S2 | i_minus Fail I Dst -gc_bif2 Fail I u$bif:erlang:stimes/2 S1 S2 Dst=d => i_fetch S1 S2 | i_times Fail I Dst -gc_bif2 Fail I u$bif:erlang:div/2 S1 S2 Dst=d => i_fetch S1 S2 | i_m_div Fail I Dst +gen_plus Fail Live Y=y X=x Dst => i_plus Fail Live X Y Dst +gen_plus Fail Live S1 S2 Dst => i_plus Fail Live S1 S2 Dst -gc_bif2 Fail I u$bif:erlang:intdiv/2 S1 S2 Dst=d => i_fetch S1 S2 | i_int_div Fail I Dst -gc_bif2 Fail I u$bif:erlang:rem/2 S1 S2 Dst=d => i_fetch S1 S2 | i_rem Fail I Dst +gen_minus Fail Live S1 S2 Dst => i_minus Fail Live S1 S2 Dst -gc_bif2 Fail I u$bif:erlang:bsl/2 S1 S2 Dst=d => i_fetch S1 S2 | i_bsl Fail I Dst -gc_bif2 Fail I u$bif:erlang:bsr/2 S1 S2 Dst=d => i_fetch S1 S2 | i_bsr Fail I Dst +gc_bif2 Fail Live u$bif:erlang:stimes/2 S1 S2 Dst => \ + i_times Fail Live S1 S2 Dst -gc_bif2 Fail I u$bif:erlang:band/2 S1 S2 Dst=d => i_fetch S1 S2 | i_band Fail I Dst -gc_bif2 Fail I u$bif:erlang:bor/2 S1 S2 Dst=d => i_fetch S1 S2 | i_bor Fail I Dst -gc_bif2 Fail I u$bif:erlang:bxor/2 S1 S2 Dst=d => i_fetch S1 S2 | i_bxor Fail I Dst +gc_bif2 Fail Live u$bif:erlang:div/2 S1 S2 Dst => \ + i_m_div Fail Live S1 S2 Dst +gc_bif2 Fail Live u$bif:erlang:intdiv/2 S1 S2 Dst => \ + i_int_div Fail Live S1 S2 Dst -gc_bif1 Fail I u$bif:erlang:bnot/1 Src Dst=d => i_int_bnot Fail Src I Dst +gc_bif2 Fail Live u$bif:erlang:rem/2 S1 S2 Dst => \ + i_rem Fail Live S1 S2 Dst + +gc_bif2 Fail Live u$bif:erlang:bsl/2 S1 S2 Dst => \ + i_bsl Fail Live S1 S2 Dst +gc_bif2 Fail Live u$bif:erlang:bsr/2 S1 S2 Dst => \ + i_bsr Fail Live S1 S2 Dst -gc_bif1 Fail I u$bif:erlang:sminus/1 Src Dst=d => i_fetch i Src | i_minus Fail I Dst -gc_bif1 Fail I u$bif:erlang:splus/1 Src Dst=d => i_fetch i Src | i_plus Fail I Dst +gc_bif2 Fail Live u$bif:erlang:band/2 S1 S2 Dst => \ + i_band Fail Live S1 S2 Dst + +gc_bif2 Fail Live u$bif:erlang:bor/2 S1 S2 Dst => \ + i_bor Fail Live S1 S2 Dst + +gc_bif2 Fail Live u$bif:erlang:bxor/2 S1 S2 Dst => \ + i_bxor Fail Live S1 S2 Dst + +gc_bif1 Fail I u$bif:erlang:bnot/1 Src Dst=d => i_int_bnot Fail Src I Dst i_increment r I I d i_increment x I I d i_increment y I I d -i_plus j I d -i_minus j I d -i_times j I d -i_m_div j I d -i_int_div j I d -i_rem j I d +i_plus j I x x d +i_plus j I x y d +i_plus j I s s d -i_bsl j I d -i_bsr j I d +i_minus j I x x d +i_minus j I s s d -i_band j I d -i_bor j I d -i_bxor j I d +i_times j I s s d + +i_m_div j I s s d +i_int_div j I s s d + +i_rem j I x x d +i_rem j I s s d + +i_bsl j I s s d +i_bsr j I s s d + +i_band j I x c d +i_band j I s s d + +i_bor j I s s d +i_bxor j I s s d i_int_bnot j s I d @@ -1632,21 +1623,18 @@ gc_bif2 Fail I Bif S1 S2 Dst => \ gc_bif3 Fail I Bif S1 S2 S3 Dst => \ gen_guard_bif3(Fail, I, Bif, S1, S2, S3, Dst) -i_gc_bif1 Fail Bif V=q Live D => move V x | i_gc_bif1 Fail Bif x Live D - i_gc_bif1 j I s I d -ii_gc_bif2/6 - -ii_gc_bif2 Fail Bif S1 S2 Live D => i_fetch S1 S2 | i_gc_bif2 Fail Bif Live D - -i_gc_bif2 j I I d +i_gc_bif2 j I I s s d ii_gc_bif3/7 -ii_gc_bif3 Fail Bif S1 S2 S3 Live D => move S1 x | i_fetch S2 S3 | i_gc_bif3 Fail Bif x Live D +# A specific instruction can only have 6 operands, so we must +# pass one of the arguments in an x register. +ii_gc_bif3 Fail Bif Live S1 S2 S3 Dst => \ + move S1 x | i_gc_bif3 Fail Bif Live S2 S3 Dst -i_gc_bif3 j I s I d +i_gc_bif3 j I I s s d # # The following instruction is specially handled in beam_load.c diff --git a/erts/emulator/beam/packet_parser.c b/erts/emulator/beam/packet_parser.c index db0e78b1a7..a737a86f14 100644 --- a/erts/emulator/beam/packet_parser.c +++ b/erts/emulator/beam/packet_parser.c @@ -3,16 +3,17 @@ * * Copyright Ericsson AB 2008-2013. All Rights Reserved. * - * The contents of this file are subject to the Erlang Public License, - * Version 1.1, (the "License"); you may not use this file except in - * compliance with the License. You should have received a copy of the - * Erlang Public License along with this software. If not, it can be - * retrieved online at http://www.erlang.org/. - * - * Software distributed under the License is distributed on an "AS IS" - * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See - * the License for the specific language governing rights and limitations - * under the License. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. * * %CopyrightEnd% */ @@ -255,6 +256,7 @@ int packet_get_length(enum PacketParseType htype, const char* ptr, unsigned n, /* Bytes read so far */ unsigned max_plen, /* Max packet length, 0=no limit */ unsigned trunc_len, /* Truncate (lines) if longer, 0=no limit */ + char delimiter, /* Line delimiting character */ int* statep) /* Protocol specific state */ { unsigned hlen, plen; @@ -298,9 +300,9 @@ int packet_get_length(enum PacketParseType htype, goto remain; case TCP_PB_LINE_LF: { - /* TCP_PB_LINE_LF: [Data ... \n] */ + /* TCP_PB_LINE_LF: [Data ... Delimiter] */ const char* ptr2; - if ((ptr2 = memchr(ptr, '\n', n)) == NULL) { + if ((ptr2 = memchr(ptr, delimiter, n)) == NULL) { if (n > max_plen && max_plen != 0) { /* packet full */ DEBUGF((" => packet full (no NL)=%d\r\n", n)); goto error; diff --git a/erts/emulator/beam/packet_parser.h b/erts/emulator/beam/packet_parser.h index 1c3a9aa3da..717d905fad 100644 --- a/erts/emulator/beam/packet_parser.h +++ b/erts/emulator/beam/packet_parser.h @@ -3,16 +3,17 @@ * * Copyright Ericsson AB 2008-2009. All Rights Reserved. * - * The contents of this file are subject to the Erlang Public License, - * Version 1.1, (the "License"); you may not use this file except in - * compliance with the License. You should have received a copy of the - * Erlang Public License along with this software. If not, it can be - * retrieved online at http://www.erlang.org/. - * - * Software distributed under the License is distributed on an "AS IS" - * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See - * the License for the specific language governing rights and limitations - * under the License. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. * * %CopyrightEnd% */ @@ -104,7 +105,8 @@ int packet_get_length(enum PacketParseType htype, const char* ptr, unsigned n, /* Bytes read so far */ unsigned max_plen, /* Packet max length, 0=no limit */ unsigned trunc_len, /* Truncate (lines) if longer, 0=no limit */ - int* statep); /* Internal protocol state */ + char delimiter, /* Line delimiting character */ + int* statep); /* Internal protocol state */ ERTS_GLB_INLINE void packet_get_body(enum PacketParseType htype, diff --git a/erts/emulator/beam/register.c b/erts/emulator/beam/register.c index c626cb2780..ebc35b0c4d 100644 --- a/erts/emulator/beam/register.c +++ b/erts/emulator/beam/register.c @@ -3,16 +3,17 @@ * * Copyright Ericsson AB 1996-2013. All Rights Reserved. * - * The contents of this file are subject to the Erlang Public License, - * Version 1.1, (the "License"); you may not use this file except in - * compliance with the License. You should have received a copy of the - * Erlang Public License along with this software. If not, it can be - * retrieved online at http://www.erlang.org/. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Software distributed under the License is distributed on an "AS IS" - * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See - * the License for the specific language governing rights and limitations - * under the License. + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. * * %CopyrightEnd% */ @@ -124,7 +125,7 @@ static RegProc* reg_alloc(RegProc *tmpl) { RegProc* obj = (RegProc*) erts_alloc(ERTS_ALC_T_REG_PROC, sizeof(RegProc)); if (!obj) { - erl_exit(1, "Can't allocate %d bytes of memory\n", sizeof(RegProc)); + erts_exit(ERTS_ERROR_EXIT, "Can't allocate %d bytes of memory\n", sizeof(RegProc)); } obj->name = tmpl->name; obj->p = tmpl->p; @@ -150,6 +151,9 @@ void init_register_table(void) f.cmp = (HCMP_FUN) reg_cmp; f.alloc = (HALLOC_FUN) reg_alloc; f.free = (HFREE_FUN) reg_free; + f.meta_alloc = (HMALLOC_FUN) erts_alloc; + f.meta_free = (HMFREE_FUN) erts_free; + f.meta_print = (HMPRINT_FUN) erts_print; hash_init(ERTS_ALC_T_REG_TABLE, &process_reg, "process_reg", PREG_HASH_SIZE, f); @@ -269,7 +273,10 @@ erts_whereis_name_to_id(Process *c_p, Eterm name) #ifdef ERTS_SMP ErtsProcLocks c_p_locks = c_p ? ERTS_PROC_LOCK_MAIN : 0; - ERTS_SMP_CHK_HAVE_ONLY_MAIN_PROC_LOCK(c_p); +#ifdef ERTS_ENABLE_LOCK_CHECK + if (c_p) ERTS_SMP_CHK_HAVE_ONLY_MAIN_PROC_LOCK(c_p); +#endif + reg_safe_read_lock(c_p, &c_p_locks); if (c_p && !c_p_locks) erts_smp_proc_lock(c_p, ERTS_PROC_LOCK_MAIN); @@ -380,8 +387,6 @@ erts_whereis_name(Process *c_p, erts_smp_proc_unlock(rp->p, need_locks); *proc = NULL; } - if (*proc && (flags & ERTS_P2P_FLG_SMP_INC_REFC)) - erts_smp_proc_inc_refc(rp->p); } #else if (rp->p @@ -390,6 +395,8 @@ erts_whereis_name(Process *c_p, else *proc = NULL; #endif + if (*proc && (flags & ERTS_P2P_FLG_INC_REFC)) + erts_proc_inc_refc(*proc); } } diff --git a/erts/emulator/beam/register.h b/erts/emulator/beam/register.h index 7170463375..144536f34b 100644 --- a/erts/emulator/beam/register.h +++ b/erts/emulator/beam/register.h @@ -3,16 +3,17 @@ * * Copyright Ericsson AB 1996-2012. All Rights Reserved. * - * The contents of this file are subject to the Erlang Public License, - * Version 1.1, (the "License"); you may not use this file except in - * compliance with the License. You should have received a copy of the - * Erlang Public License along with this software. If not, it can be - * retrieved online at http://www.erlang.org/. - * - * Software distributed under the License is distributed on an "AS IS" - * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See - * the License for the specific language governing rights and limitations - * under the License. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. * * %CopyrightEnd% */ diff --git a/erts/emulator/beam/safe_hash.c b/erts/emulator/beam/safe_hash.c index 3326e5cc2a..3f039c8dfd 100644 --- a/erts/emulator/beam/safe_hash.c +++ b/erts/emulator/beam/safe_hash.c @@ -3,16 +3,17 @@ * * Copyright Ericsson AB 2008-2011. All Rights Reserved. * - * The contents of this file are subject to the Erlang Public License, - * Version 1.1, (the "License"); you may not use this file except in - * compliance with the License. You should have received a copy of the - * Erlang Public License along with this software. If not, it can be - * retrieved online at http://www.erlang.org/. - * - * Software distributed under the License is distributed on an "AS IS" - * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See - * the License for the specific language governing rights and limitations - * under the License. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. * * %CopyrightEnd% */ diff --git a/erts/emulator/beam/safe_hash.h b/erts/emulator/beam/safe_hash.h index c691126ef9..a11370813c 100644 --- a/erts/emulator/beam/safe_hash.h +++ b/erts/emulator/beam/safe_hash.h @@ -3,16 +3,17 @@ * * Copyright Ericsson AB 2008-2009. All Rights Reserved. * - * The contents of this file are subject to the Erlang Public License, - * Version 1.1, (the "License"); you may not use this file except in - * compliance with the License. You should have received a copy of the - * Erlang Public License along with this software. If not, it can be - * retrieved online at http://www.erlang.org/. - * - * Software distributed under the License is distributed on an "AS IS" - * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See - * the License for the specific language governing rights and limitations - * under the License. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. * * %CopyrightEnd% */ diff --git a/erts/emulator/beam/sys.h b/erts/emulator/beam/sys.h index 05f07e57b2..44735c0ec0 100644 --- a/erts/emulator/beam/sys.h +++ b/erts/emulator/beam/sys.h @@ -3,16 +3,17 @@ * * Copyright Ericsson AB 1996-2013. All Rights Reserved. * - * The contents of this file are subject to the Erlang Public License, - * Version 1.1, (the "License"); you may not use this file except in - * compliance with the License. You should have received a copy of the - * Erlang Public License along with this software. If not, it can be - * retrieved online at http://www.erlang.org/. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Software distributed under the License is distributed on an "AS IS" - * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See - * the License for the specific language governing rights and limitations - * under the License. + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. * * %CopyrightEnd% */ @@ -20,27 +21,80 @@ #ifndef __SYS_H__ #define __SYS_H__ +#if !defined(__GNUC__) +# define ERTS_AT_LEAST_GCC_VSN__(MAJ, MIN, PL) 0 +#elif !defined(__GNUC_MINOR__) +# define ERTS_AT_LEAST_GCC_VSN__(MAJ, MIN, PL) \ + ((__GNUC__ << 24) >= (((MAJ) << 24) | ((MIN) << 12) | (PL))) +#elif !defined(__GNUC_PATCHLEVEL__) +# define ERTS_AT_LEAST_GCC_VSN__(MAJ, MIN, PL) \ + (((__GNUC__ << 24) | (__GNUC_MINOR__ << 12)) >= (((MAJ) << 24) | ((MIN) << 12) | (PL))) +#else +# define ERTS_AT_LEAST_GCC_VSN__(MAJ, MIN, PL) \ + (((__GNUC__ << 24) | (__GNUC_MINOR__ << 12) | __GNUC_PATCHLEVEL__) >= (((MAJ) << 24) | ((MIN) << 12) | (PL))) +#endif -#if defined(VALGRIND) && !defined(NO_FPE_SIGNALS) -# define NO_FPE_SIGNALS +#ifdef ERTS_INLINE +# ifndef ERTS_CAN_INLINE +# define ERTS_CAN_INLINE 1 +# endif +#else +# if defined(__GNUC__) +# define ERTS_CAN_INLINE 1 +# define ERTS_INLINE __inline__ +# elif defined(__WIN32__) +# define ERTS_CAN_INLINE 1 +# define ERTS_INLINE __inline +# else +# define ERTS_CAN_INLINE 0 +# define ERTS_INLINE +# endif +#endif + +#ifndef ERTS_FORCE_INLINE +# if ERTS_AT_LEAST_GCC_VSN__(3,1,1) +# define ERTS_FORCE_INLINE __inline__ __attribute__((__always_inline__)) +# elif defined(__WIN32__) +# define ERTS_FORCE_INLINE __forceinline +# endif +# ifndef ERTS_FORCE_INLINE +# define ERTS_FORCE_INLINE ERTS_INLINE +# endif +#endif + +#if defined(DEBUG) || defined(ERTS_ENABLE_LOCK_CHECK) +# undef ERTS_CAN_INLINE +# define ERTS_CAN_INLINE 0 +# undef ERTS_INLINE +# define ERTS_INLINE +#endif + +#if ERTS_CAN_INLINE +#define ERTS_GLB_FORCE_INLINE static ERTS_FORCE_INLINE +#define ERTS_GLB_INLINE static ERTS_INLINE +#else +#define ERTS_GLB_FORCE_INLINE +#define ERTS_GLB_INLINE #endif -#ifdef DISABLE_CHILD_WAITER_THREAD -#undef ENABLE_CHILD_WAITER_THREAD +#if ERTS_CAN_INLINE || defined(ERTS_DO_INCL_GLB_INLINE_FUNC_DEF) +# define ERTS_GLB_INLINE_INCL_FUNC_DEF 1 +#else +# define ERTS_GLB_INLINE_INCL_FUNC_DEF 0 #endif -#if defined(ERTS_SMP) && !defined(DISABLE_CHILD_WAITER_THREAD) -#undef ENABLE_CHILD_WAITER_THREAD -#define ENABLE_CHILD_WAITER_THREAD 1 +#if defined(VALGRIND) && !defined(NO_FPE_SIGNALS) +# define NO_FPE_SIGNALS #endif #define ERTS_I64_LITERAL(X) X##LL +#define ErtsInArea(ptr,start,nbytes) \ + ((UWord)((char*)(ptr) - (char*)(start)) < (nbytes)) + #if defined (__WIN32__) # include "erl_win_sys.h" -#elif defined (__OSE__) -# include "erl_ose_sys.h" -#else +#else # include "erl_unix_sys.h" #ifndef UNIX # define UNIX 1 @@ -66,39 +120,13 @@ */ #ifndef ERTS_SYS_FD_TYPE +#define ERTS_SYS_FD_INVALID ((ErtsSysFdType) -1) typedef int ErtsSysFdType; #else -typedef ERTS_SYS_FD_TYPE ErtsSysFdType; +#ifndef ERTS_SYS_FD_INVALID +# error missing ERTS_SYS_FD_INVALID #endif - -#ifdef ERTS_INLINE -# ifndef ERTS_CAN_INLINE -# define ERTS_CAN_INLINE 1 -# endif -#else -# if defined(__GNUC__) -# define ERTS_CAN_INLINE 1 -# define ERTS_INLINE __inline__ -# elif defined(__WIN32__) -# define ERTS_CAN_INLINE 1 -# define ERTS_INLINE __inline -# else -# define ERTS_CAN_INLINE 0 -# define ERTS_INLINE -# endif -#endif - -#if !defined(__GNUC__) -# define ERTS_AT_LEAST_GCC_VSN__(MAJ, MIN, PL) 0 -#elif !defined(__GNUC_MINOR__) -# define ERTS_AT_LEAST_GCC_VSN__(MAJ, MIN, PL) \ - ((__GNUC__ << 24) >= (((MAJ) << 24) | ((MIN) << 12) | (PL))) -#elif !defined(__GNUC_PATCHLEVEL__) -# define ERTS_AT_LEAST_GCC_VSN__(MAJ, MIN, PL) \ - (((__GNUC__ << 24) | (__GNUC_MINOR__ << 12)) >= (((MAJ) << 24) | ((MIN) << 12) | (PL))) -#else -# define ERTS_AT_LEAST_GCC_VSN__(MAJ, MIN, PL) \ - (((__GNUC__ << 24) | (__GNUC_MINOR__ << 12) | __GNUC_PATCHLEVEL__) >= (((MAJ) << 24) | ((MIN) << 12) | (PL))) +typedef ERTS_SYS_FD_TYPE ErtsSysFdType; #endif #if ERTS_AT_LEAST_GCC_VSN__(2, 96, 0) @@ -108,6 +136,17 @@ typedef ERTS_SYS_FD_TYPE ErtsSysFdType; # define ERTS_LIKELY(BOOL) (BOOL) # define ERTS_UNLIKELY(BOOL) (BOOL) #endif + +#if ERTS_AT_LEAST_GCC_VSN__(2, 96, 0) +#if (defined(__APPLE__) && defined(__MACH__)) || defined(__DARWIN__) +# define ERTS_WRITE_UNLIKELY(X) X __attribute__ ((section ("__DATA,ERTS_LOW_WRITE") )) +#else +# define ERTS_WRITE_UNLIKELY(X) X __attribute__ ((section ("ERTS_LOW_WRITE") )) +#endif +#else +# define ERTS_WRITE_UNLIKELY(X) X +#endif + #ifdef __GNUC__ # if __GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ > 5) # define ERTS_DECLARE_DUMMY(X) X __attribute__ ((unused)) @@ -128,24 +167,8 @@ typedef ERTS_SYS_FD_TYPE ErtsSysFdType; # endif #endif -#if defined(DEBUG) || defined(ERTS_ENABLE_LOCK_CHECK) -# undef ERTS_CAN_INLINE -# define ERTS_CAN_INLINE 0 -# undef ERTS_INLINE -# define ERTS_INLINE -#endif - -#if ERTS_CAN_INLINE -#define ERTS_GLB_INLINE static ERTS_INLINE -#else -#define ERTS_GLB_INLINE -#endif - -#if ERTS_CAN_INLINE || defined(ERTS_DO_INCL_GLB_INLINE_FUNC_DEF) -# define ERTS_GLB_INLINE_INCL_FUNC_DEF 1 -#else -# define ERTS_GLB_INLINE_INCL_FUNC_DEF 0 -#endif +#define ERTS_MK_VSN_INT(Major, Minor, Build) \ + ((((Major) & 0x3ff) << 20) | (((Minor) & 0x3ff) << 10) | ((Build) & 0x3ff)) #ifndef ERTS_EXIT_AFTER_DUMP # define ERTS_EXIT_AFTER_DUMP exit @@ -184,6 +207,38 @@ __decl_noreturn void __noreturn erl_assert_error(const char* expr, const char *f # define ASSERT(e) ((void) 1) #endif +#ifdef ERTS_SMP +# define ERTS_SMP_ASSERT(e) ASSERT(e) +#else +# define ERTS_SMP_ASSERT(e) ((void)1) +#endif + +/* ERTS_UNDEF can be used to silence false warnings about + * "variable may be used uninitialized" while keeping the variable + * marked as undefined by valgrind. + */ +#ifdef VALGRIND +# define ERTS_UNDEF(V,I) +#else +# define ERTS_UNDEF(V,I) V = I +#endif + +/* + * Compile time assert + * (the actual compiler error msg can be a bit confusing) + */ +#if ERTS_AT_LEAST_GCC_VSN__(3,1,1) +# define ERTS_CT_ASSERT(e) \ + do { \ + enum { compile_time_assert__ = __builtin_choose_expr((e),0,(void)0) }; \ + } while(0) +#else +# define ERTS_CT_ASSERT(e) \ + do { \ + enum { compile_time_assert__ = 1/(e) }; \ + } while (0) +#endif + /* * Microsoft C/C++: We certainly want to use stdarg.h and prototypes. * But MSC doesn't define __STDC__, unless we compile with the -Za @@ -252,61 +307,11 @@ __decl_noreturn void __noreturn erl_assert_error(const char* expr, const char *f #else #error Neither 32 nor 64 bit architecture #endif -#if defined(ARCH_64) && defined(HALFWORD_HEAP_EMULATOR) -# define HALFWORD_HEAP 1 -# define HALFWORD_ASSERT 0 -# define ASSERT_HALFWORD(COND) ASSERT(COND) -# undef ERTS_SIZEOF_TERM -# define ERTS_SIZEOF_TERM 4 -#else -# define HALFWORD_HEAP 0 -# define HALFWORD_ASSERT 0 -# define ASSERT_HALFWORD(COND) -#endif #if SIZEOF_VOID_P != SIZEOF_SIZE_T #error sizeof(void*) != sizeof(size_t) #endif -#if HALFWORD_HEAP - -#if SIZEOF_INT == 4 -typedef unsigned int Eterm; -typedef unsigned int Uint; -typedef int Sint; -#define ERTS_SIZEOF_ETERM SIZEOF_INT -#define ErtsStrToSint strtol -#else -#error Found no appropriate type to use for 'Eterm', 'Uint' and 'Sint' -#endif - -#if SIZEOF_VOID_P == SIZEOF_LONG -typedef unsigned long UWord; -typedef long SWord; -#define SWORD_CONSTANT(Const) Const##L -#define UWORD_CONSTANT(Const) Const##UL -#define ERTS_UWORD_MAX ULONG_MAX -#define ERTS_SWORD_MAX LONG_MAX -#elif SIZEOF_VOID_P == SIZEOF_INT -typedef unsigned int UWord; -typedef int SWord; -#define SWORD_CONSTANT(Const) Const -#define UWORD_CONSTANT(Const) Const##U -#define ERTS_UWORD_MAX UINT_MAX -#define ERTS_SWORD_MAX INT_MAX -#elif SIZEOF_VOID_P == SIZEOF_LONG_LONG -typedef unsigned long long UWord; -typedef long long SWord; -#define SWORD_CONSTANT(Const) Const##LL -#define UWORD_CONSTANT(Const) Const##ULL -#define ERTS_UWORD_MAX ULLONG_MAX -#define ERTS_SWORD_MAX LLONG_MAX -#else -#error Found no appropriate type to use for 'Eterm', 'Uint' and 'Sint' -#endif - -#else /* !HALFWORD_HEAP */ - #if SIZEOF_VOID_P == SIZEOF_LONG typedef unsigned long Eterm; typedef unsigned long Uint; @@ -347,23 +352,50 @@ typedef long long Sint; typedef Uint UWord; typedef Sint SWord; - -#endif /* HALFWORD_HEAP */ +#define ERTS_UINT_MAX ERTS_UWORD_MAX typedef UWord BeamInstr; #ifndef HAVE_INT64 -#if SIZEOF_LONG == 8 -#define HAVE_INT64 1 +# if SIZEOF_LONG == 8 +# define HAVE_INT64 1 typedef unsigned long Uint64; typedef long Sint64; -#elif SIZEOF_LONG_LONG == 8 -#define HAVE_INT64 1 +# ifdef ULONG_MAX +# define ERTS_UINT64_MAX ULONG_MAX +# endif +# ifdef LONG_MAX +# define ERTS_SINT64_MAX LONG_MAX +# endif +# ifdef LONG_MIN +# define ERTS_SINT64_MIN LONG_MIN +# endif +# elif SIZEOF_LONG_LONG == 8 +# define HAVE_INT64 1 typedef unsigned long long Uint64; typedef long long Sint64; -#else -#define HAVE_INT64 0 +# ifdef ULLONG_MAX +# define ERTS_UINT64_MAX ULLONG_MAX +# endif +# ifdef LLONG_MAX +# define ERTS_SINT64_MAX LLONG_MAX +# endif +# ifdef LLONG_MIN +# define ERTS_SINT64_MIN LLONG_MIN +# endif +# else +# error "No 64-bit integer type found" +# endif +#endif + +#ifndef ERTS_UINT64_MAX +# define ERTS_UINT64_MAX (~((Uint64) 0)) +#endif +#ifndef ERTS_SINT64_MAX +# define ERTS_SINT64_MAX ((Sint64) ((((Uint64) 1) << 63)-1)) #endif +#ifndef ERTS_SINT64_MIN +# define ERTS_SINT64_MIN (-1*(((Sint64) 1) << 63)) #endif #if SIZEOF_LONG == 4 @@ -499,7 +531,7 @@ extern volatile int erts_writing_erl_crash_dump; # define NO_ERF # define NO_ERFC /* This definition doesn't take NaN into account, but matherr() gets those */ -# define finite(x) (fabs(x) != HUGE_VAL) +# define isfinite(x) (fabs(x) != HUGE_VAL) # define USE_MATHERR # define HAVE_FINITE #endif @@ -548,15 +580,16 @@ static unsigned long zero_value = 0, one_value = 1; # endif /* !__WIN32__ */ #endif /* WANT_NONBLOCKING */ -__decl_noreturn void __noreturn erl_exit(int n, char*, ...); +__decl_noreturn void __noreturn erts_exit(int n, char*, ...); -/* Some special erl_exit() codes: */ -#define ERTS_INTR_EXIT INT_MIN /* called from signal handler */ -#define ERTS_ABORT_EXIT (INT_MIN + 1) /* no crash dump; only abort() */ -#define ERTS_DUMP_EXIT (INT_MIN + 2) /* crash dump; then exit() */ +/* Some special erts_exit() codes: */ +#define ERTS_INTR_EXIT -1 /* called from signal handler */ +#define ERTS_ABORT_EXIT -2 /* no crash dump; only abort() */ +#define ERTS_DUMP_EXIT -3 /* crash dump; then exit() */ +#define ERTS_ERROR_EXIT -4 /* crash dump; then abort() */ #define ERTS_INTERNAL_ERROR(What) \ - erl_exit(ERTS_ABORT_EXIT, "%s:%d:%s(): Internal error: %s\n", \ + erts_exit(ERTS_ABORT_EXIT, "%s:%d:%s(): Internal error: %s\n", \ __FILE__, __LINE__, __func__, What) Eterm erts_check_io_info(void *p); @@ -571,7 +604,6 @@ Uint erts_sys_misc_mem_sz(void); /* Io constants to erts_print and erts_putc */ #define ERTS_PRINT_STDERR (2) #define ERTS_PRINT_STDOUT (1) -#define ERTS_PRINT_INVALID (0) /* Don't want to use 0 since CBUF was 0 */ #define ERTS_PRINT_FILE (-1) #define ERTS_PRINT_SBUF (-2) #define ERTS_PRINT_SNBUF (-3) @@ -594,6 +626,7 @@ erts_dsprintf_buf_t *erts_create_logger_dsbuf(void); int erts_send_info_to_logger(Eterm, erts_dsprintf_buf_t *); int erts_send_warning_to_logger(Eterm, erts_dsprintf_buf_t *); int erts_send_error_to_logger(Eterm, erts_dsprintf_buf_t *); +int erts_send_error_term_to_logger(Eterm, erts_dsprintf_buf_t *, Eterm); int erts_send_info_to_logger_str(Eterm, char *); int erts_send_warning_to_logger_str(Eterm, char *); int erts_send_error_to_logger_str(Eterm, char *); @@ -640,14 +673,37 @@ extern char *erts_default_arg0; extern char os_type[]; -extern int sys_init_time(void); +typedef struct { + int have_os_monotonic_time; + int have_corrected_os_monotonic_time; + ErtsMonotonicTime os_monotonic_time_unit; + ErtsMonotonicTime sys_clock_resolution; + struct { + Uint64 resolution; + char *func; + char *clock_id; + int locked_use; + int extended; + } os_monotonic_time_info; + struct { + Uint64 resolution; + char *func; + char *clock_id; + int locked_use; + } os_system_time_info; +} ErtsSysInitTimeResult; + +#define ERTS_SYS_INIT_TIME_RESULT_INITER \ + {0, 0, (ErtsMonotonicTime) -1, (ErtsMonotonicTime) 1} + +extern void erts_init_sys_time_sup(void); +extern void sys_init_time(ErtsSysInitTimeResult *); +extern void erts_late_sys_init_time(void); extern void erts_deliver_time(void); extern void erts_time_remaining(SysTimeval *); -extern int erts_init_time_sup(void); extern void erts_sys_init_float(void); extern void erts_thread_init_float(void); extern void erts_thread_disable_fpe(void); - ERTS_GLB_INLINE int erts_block_fpe(void); ERTS_GLB_INLINE void erts_unblock_fpe(int); @@ -690,17 +746,16 @@ extern char *erts_sys_ddll_error(int code); /* * System interfaces for startup. */ -#include "erl_time.h" - void erts_sys_schedule_interrupt(int set); #ifdef ERTS_SMP -void erts_sys_schedule_interrupt_timed(int set, erts_short_time_t msec); +void erts_sys_schedule_interrupt_timed(int, ErtsMonotonicTime); void erts_sys_main_thread(void); #endif extern int erts_sys_prepare_crash_dump(int secs); extern void erts_sys_pre_init(void); extern void erl_sys_init(void); +extern void erl_sys_late_init(void); extern void erl_sys_args(int *argc, char **argv); extern void erl_sys_schedule(int); void sys_tty_reset(int); @@ -733,6 +788,12 @@ int univ_to_local( int local_to_univ(Sint *year, Sint *month, Sint *day, Sint *hour, Sint *minute, Sint *second, int isdst); void get_now(Uint*, Uint*, Uint*); +struct ErtsSchedulerData_; +ErtsMonotonicTime erts_get_monotonic_time(struct ErtsSchedulerData_ *); +ErtsMonotonicTime erts_get_time_offset(void); +void +erts_make_timestamp_value(Uint* megasec, Uint* sec, Uint* microsec, + ErtsMonotonicTime mtime, ErtsMonotonicTime offset); void get_sys_now(Uint*, Uint*, Uint*); void set_break_quit(void (*)(void), void (*)(void)); @@ -742,6 +803,16 @@ void init_getenv_state(GETENV_STATE *); char * getenv_string(GETENV_STATE *); void fini_getenv_state(GETENV_STATE *); +#define HAVE_ERTS_CHECK_IO_DEBUG +typedef struct { + int no_used_fds; + int no_driver_select_structs; + int no_driver_event_structs; +} ErtsCheckIoDebugInfo; +int erts_check_io_debug(ErtsCheckIoDebugInfo *ip); + +int erts_sys_is_area_readable(char *start, char *stop); + /* xxxP */ #define SYS_DEFAULT_FLOAT_DECIMALS 20 void init_sys_float(void); @@ -770,6 +841,11 @@ int erts_sys_unsetenv(char *key); char *erts_read_env(char *key); void erts_free_read_env(void *value); +#if defined(ERTS_THR_HAVE_SIG_FUNCS) && !defined(ETHR_UNUSABLE_SIGUSRX) +extern void sys_thr_resume(erts_tid_t tid); +extern void sys_thr_suspend(erts_tid_t tid); +#endif + /* utils.c */ /* Options to sys_alloc_opt */ @@ -828,7 +904,7 @@ erts_refc_inc(erts_refc_t *refcp, erts_aint_t min_val) #ifdef ERTS_REFC_DEBUG erts_aint_t val = erts_smp_atomic_inc_read_nob((erts_smp_atomic_t *) refcp); if (val < min_val) - erl_exit(ERTS_ABORT_EXIT, + erts_exit(ERTS_ABORT_EXIT, "erts_refc_inc(): Bad refc found (refc=%ld < %ld)!\n", val, min_val); #else @@ -842,7 +918,7 @@ erts_refc_inctest(erts_refc_t *refcp, erts_aint_t min_val) erts_aint_t val = erts_smp_atomic_inc_read_nob((erts_smp_atomic_t *) refcp); #ifdef ERTS_REFC_DEBUG if (val < min_val) - erl_exit(ERTS_ABORT_EXIT, + erts_exit(ERTS_ABORT_EXIT, "erts_refc_inctest(): Bad refc found (refc=%ld < %ld)!\n", val, min_val); #endif @@ -855,7 +931,7 @@ erts_refc_dec(erts_refc_t *refcp, erts_aint_t min_val) #ifdef ERTS_REFC_DEBUG erts_aint_t val = erts_smp_atomic_dec_read_nob((erts_smp_atomic_t *) refcp); if (val < min_val) - erl_exit(ERTS_ABORT_EXIT, + erts_exit(ERTS_ABORT_EXIT, "erts_refc_dec(): Bad refc found (refc=%ld < %ld)!\n", val, min_val); #else @@ -869,7 +945,7 @@ erts_refc_dectest(erts_refc_t *refcp, erts_aint_t min_val) erts_aint_t val = erts_smp_atomic_dec_read_nob((erts_smp_atomic_t *) refcp); #ifdef ERTS_REFC_DEBUG if (val < min_val) - erl_exit(ERTS_ABORT_EXIT, + erts_exit(ERTS_ABORT_EXIT, "erts_refc_dectest(): Bad refc found (refc=%ld < %ld)!\n", val, min_val); #endif @@ -882,7 +958,7 @@ erts_refc_add(erts_refc_t *refcp, erts_aint_t diff, erts_aint_t min_val) #ifdef ERTS_REFC_DEBUG erts_aint_t val = erts_smp_atomic_add_read_nob((erts_smp_atomic_t *) refcp, diff); if (val < min_val) - erl_exit(ERTS_ABORT_EXIT, + erts_exit(ERTS_ABORT_EXIT, "erts_refc_add(%ld): Bad refc found (refc=%ld < %ld)!\n", diff, val, min_val); #else @@ -896,7 +972,7 @@ erts_refc_read(erts_refc_t *refcp, erts_aint_t min_val) erts_aint_t val = erts_smp_atomic_read_nob((erts_smp_atomic_t *) refcp); #ifdef ERTS_REFC_DEBUG if (val < min_val) - erl_exit(ERTS_ABORT_EXIT, + erts_exit(ERTS_ABORT_EXIT, "erts_refc_read(): Bad refc found (refc=%ld < %ld)!\n", val, min_val); #endif @@ -997,7 +1073,6 @@ extern int erts_use_kernel_poll; #define put_int8(i, s) do {((unsigned char*)(s))[0] = (i) & 0xff;} while (0) - /* * Use DEBUGF as you would use printf, but use double parentheses: * diff --git a/erts/emulator/beam/time.c b/erts/emulator/beam/time.c index 2fd8e0cf00..0ce4f443f3 100644 --- a/erts/emulator/beam/time.c +++ b/erts/emulator/beam/time.c @@ -3,16 +3,17 @@ * * Copyright Ericsson AB 1996-2013. All Rights Reserved. * - * The contents of this file are subject to the Erlang Public License, - * Version 1.1, (the "License"); you may not use this file except in - * compliance with the License. You should have received a copy of the - * Erlang Public License along with this software. If not, it can be - * retrieved online at http://www.erlang.org/. - * - * Software distributed under the License is distributed on an "AS IS" - * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See - * the License for the specific language governing rights and limitations - * under the License. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. * * %CopyrightEnd% */ @@ -76,6 +77,11 @@ #include "sys.h" #include "erl_vm.h" #include "global.h" +#define ERTS_WANT_TIMER_WHEEL_API +#include "erl_time.h" + +#define ERTS_MONOTONIC_DAY ERTS_SEC_TO_MONOTONIC(60*60*24) +#define ERTS_CLKTCKS_DAY ERTS_MONOTONIC_TO_CLKTCKS(ERTS_MONOTONIC_DAY) #ifdef ERTS_ENABLE_LOCK_CHECK #define ASSERT_NO_LOCKED_LOCKS erts_lc_check_exact(NULL, 0) @@ -83,26 +89,25 @@ #define ASSERT_NO_LOCKED_LOCKS #endif -static erts_smp_mtx_t tiw_lock; - - -/* BEGIN tiw_lock protected variables -** -** The individual timer cells in tiw are also protected by the same mutex. -*/ +#if 0 +# define ERTS_TW_DEBUG +#endif +#if defined(DEBUG) && !defined(ERTS_TW_DEBUG) +# define ERTS_TW_DEBUG +#endif -#ifdef SMALL_MEMORY -#define TIW_SIZE 8192 +#undef ERTS_TW_ASSERT +#if defined(ERTS_TW_DEBUG) +# define ERTS_TW_ASSERT(E) ERTS_ASSERT(E) #else -#define TIW_SIZE 65536 /* timing wheel size (should be a power of 2) */ +# define ERTS_TW_ASSERT(E) ((void) 1) #endif -static ErlTimer** tiw; /* the timing wheel, allocated in init_time() */ -static Uint tiw_pos; /* current position in wheel */ -static Uint tiw_nto; /* number of timeouts in wheel */ -static Uint tiw_min; -static ErlTimer *tiw_min_ptr; -/* END tiw_lock protected variables */ +#ifdef ERTS_TW_DEBUG +# define ERTS_TWHEEL_BUMP_YIELD_LIMIT 5 +#else +# define ERTS_TWHEEL_BUMP_YIELD_LIMIT 100 +#endif /* Actual interval time chosen by sys_init_time() */ @@ -114,392 +119,596 @@ static int tiw_itime; /* Constant after init */ # define TIW_ITIME tiw_itime #endif -erts_smp_atomic32_t do_time; /* set at clock interrupt */ -static ERTS_INLINE erts_short_time_t do_time_read(void) -{ - return erts_smp_atomic32_read_acqb(&do_time); -} - -static ERTS_INLINE erts_short_time_t do_time_update(void) +struct ErtsTimerWheel_ { + ErtsTWheelTimer *w[ERTS_TIW_SIZE]; + ErtsMonotonicTime pos; + Uint nto; + struct { + ErtsTWheelTimer *head; + ErtsTWheelTimer *tail; + Uint nto; + } at_once; + int yield_slot; + int yield_slots_left; + int yield_start_pos; + ErtsTWheelTimer sentinel; + int true_next_timeout_time; + ErtsMonotonicTime next_timeout_time; +}; + +static ERTS_INLINE ErtsMonotonicTime +find_next_timeout(ErtsSchedulerData *esdp, + ErtsTimerWheel *tiw, + int search_all, + ErtsMonotonicTime curr_time, /* When !search_all */ + ErtsMonotonicTime max_search_time) /* When !search_all */ { - return do_time_read(); -} - -static ERTS_INLINE void do_time_init(void) -{ - erts_smp_atomic32_init_nob(&do_time, 0); -} + int start_ix, tiw_pos_ix; + ErtsTWheelTimer *p; + int true_min_timeout = 0; + ErtsMonotonicTime min_timeout, min_timeout_pos, slot_timeout_pos; + + if (tiw->nto == 0) { /* no timeouts in wheel */ + if (!search_all) + min_timeout_pos = tiw->pos; + else { + curr_time = erts_get_monotonic_time(esdp); + tiw->pos = min_timeout_pos = ERTS_MONOTONIC_TO_CLKTCKS(curr_time); + } + min_timeout_pos += ERTS_MONOTONIC_TO_CLKTCKS(ERTS_MONOTONIC_DAY); + goto found_next; + } -/* get the time (in units of TIW_ITIME) to the next timeout, - or -1 if there are no timeouts */ + slot_timeout_pos = min_timeout_pos = tiw->pos; + if (search_all) + min_timeout_pos += ERTS_MONOTONIC_TO_CLKTCKS(ERTS_MONOTONIC_DAY); + else + min_timeout_pos = ERTS_MONOTONIC_TO_CLKTCKS(curr_time + max_search_time); -static erts_short_time_t next_time_internal(void) /* PRE: tiw_lock taken by caller */ -{ - int i, tm, nto; - Uint32 min; - ErlTimer* p; - erts_short_time_t dt; - - if (tiw_nto == 0) - return -1; /* no timeouts in wheel */ + start_ix = tiw_pos_ix = (int) (tiw->pos & (ERTS_TIW_SIZE-1)); - if (tiw_min_ptr) { - min = tiw_min; - dt = do_time_read(); - return ((min >= dt) ? (min - dt) : 0); - } - - /* start going through wheel to find next timeout */ - tm = nto = 0; - min = (Uint32) -1; /* max Uint32 */ - i = tiw_pos; do { - p = tiw[i]; - while (p != NULL) { - nto++; - if (p->count == 0) { - /* found next timeout */ - dt = do_time_read(); - /* p->count is zero */ - tiw_min_ptr = p; - tiw_min = tm; - return ((tm >= dt) ? (tm - dt) : 0); - } else { - /* keep shortest time in 'min' */ - if (tm + p->count*TIW_SIZE < min) { - min = tm + p->count*TIW_SIZE; - tiw_min_ptr = p; - tiw_min = min; + if (++slot_timeout_pos >= min_timeout_pos) + break; + + p = tiw->w[tiw_pos_ix]; + + if (p) { + ErtsTWheelTimer *end = p; + + do { + ErtsMonotonicTime timeout_pos; + timeout_pos = p->timeout_pos; + if (min_timeout_pos > timeout_pos) { + true_min_timeout = 1; + min_timeout_pos = timeout_pos; + if (min_timeout_pos <= slot_timeout_pos) + goto found_next; } - } - p = p->next; + p = p->next; + } while (p != end); } - /* when we have found all timeouts the shortest time will be in min */ - if (nto == tiw_nto) break; - tm++; - i = (i + 1) % TIW_SIZE; - } while (i != tiw_pos); - dt = do_time_read(); - if (min <= (Uint32) dt) - return 0; - if ((min - (Uint32) dt) > (Uint32) ERTS_SHORT_TIME_T_MAX) - return ERTS_SHORT_TIME_T_MAX; - return (erts_short_time_t) (min - (Uint32) dt); + + tiw_pos_ix++; + if (tiw_pos_ix == ERTS_TIW_SIZE) + tiw_pos_ix = 0; + } while (start_ix != tiw_pos_ix); + +found_next: + + min_timeout = ERTS_CLKTCKS_TO_MONOTONIC(min_timeout_pos); + tiw->next_timeout_time = min_timeout; + tiw->true_next_timeout_time = true_min_timeout; + + return min_timeout; } -static void remove_timer(ErlTimer *p) { - /* first */ - if (!p->prev) { - tiw[p->slot] = p->next; - if(p->next) - p->next->prev = NULL; - } else { - p->prev->next = p->next; +static ERTS_INLINE void +insert_timer_into_slot(ErtsTimerWheel *tiw, int slot, ErtsTWheelTimer *p) +{ + ERTS_TW_ASSERT(slot >= 0); + ERTS_TW_ASSERT(slot < ERTS_TIW_SIZE); + p->slot = slot; + if (!tiw->w[slot]) { + tiw->w[slot] = p; + p->next = p; + p->prev = p; + } + else { + ErtsTWheelTimer *next, *prev; + next = tiw->w[slot]; + prev = next->prev; + p->next = next; + p->prev = prev; + prev->next = p; + next->prev = p; } +} - /* last */ - if (!p->next) { +static ERTS_INLINE void +remove_timer(ErtsTimerWheel *tiw, ErtsTWheelTimer *p) +{ + int slot = p->slot; + ERTS_TW_ASSERT(slot != ERTS_TWHEEL_SLOT_INACTIVE); + + if (slot >= 0) { + /* + * Timer in wheel or in circular + * list of timers currently beeing + * triggered (referred by sentinel). + */ + ERTS_TW_ASSERT(slot < ERTS_TIW_SIZE); + + if (p->next == p) { + ERTS_TW_ASSERT(tiw->w[slot] == p); + tiw->w[slot] = NULL; + } + else { + if (tiw->w[slot] == p) + tiw->w[slot] = p->next; + p->prev->next = p->next; + p->next->prev = p->prev; + } + } + else { + /* Timer in "at once" queue... */ + ERTS_TW_ASSERT(slot == ERTS_TWHEEL_SLOT_AT_ONCE); if (p->prev) - p->prev->next = NULL; - } else { - p->next->prev = p->prev; + p->prev->next = p->next; + else { + ERTS_TW_ASSERT(tiw->at_once.head == p); + tiw->at_once.head = p->next; + } + if (p->next) + p->next->prev = p->prev; + else { + ERTS_TW_ASSERT(tiw->at_once.tail == p); + tiw->at_once.tail = p->prev; + } + ERTS_TW_ASSERT(tiw->at_once.nto > 0); + tiw->at_once.nto--; } - p->next = NULL; - p->prev = NULL; - /* Make sure cancel callback isn't called */ - p->active = 0; - tiw_nto--; + p->slot = ERTS_TWHEEL_SLOT_INACTIVE; + + tiw->nto--; } -/* Private export to erl_time_sup.c */ -erts_short_time_t erts_next_time(void) +ErtsMonotonicTime +erts_check_next_timeout_time(ErtsSchedulerData *esdp) { - erts_short_time_t ret; + ErtsTimerWheel *tiw = esdp->timer_wheel; + ErtsMonotonicTime time; + ERTS_MSACC_DECLARE_CACHE_X(); + if (tiw->true_next_timeout_time) + return tiw->next_timeout_time; + ERTS_MSACC_PUSH_AND_SET_STATE_CACHED_X(ERTS_MSACC_STATE_TIMERS); + time = find_next_timeout(esdp, tiw, 1, 0, 0); + ERTS_MSACC_POP_STATE_M_X(); + return time; +} + +#ifndef ERTS_TW_DEBUG +#define ERTS_DBG_CHK_SAFE_TO_SKIP_TO(TIW, TO) ((void) 0) +#else +#define ERTS_DBG_CHK_SAFE_TO_SKIP_TO(TIW, TO) debug_check_safe_to_skip_to((TIW), (TO)) +static void +debug_check_safe_to_skip_to(ErtsTimerWheel *tiw, ErtsMonotonicTime skip_to_pos) +{ + int slots, ix; + ErtsTWheelTimer *tmr; + ErtsMonotonicTime tmp; + + ix = (int) (tiw->pos & (ERTS_TIW_SIZE-1)); + tmp = skip_to_pos - tiw->pos; + ERTS_TW_ASSERT(tmp >= 0); + if (tmp < (ErtsMonotonicTime) ERTS_TIW_SIZE) + slots = (int) tmp; + else + slots = ERTS_TIW_SIZE; + + while (slots > 0) { + tmr = tiw->w[ix]; + if (tmr) { + ErtsTWheelTimer *end = tmr; + do { + ERTS_TW_ASSERT(tmr->timeout_pos > skip_to_pos); + tmr = tmr->next; + } while (tmr != end); + } + ix++; + if (ix == ERTS_TIW_SIZE) + ix = 0; + slots--; + } +} +#endif - erts_smp_mtx_lock(&tiw_lock); - (void)do_time_update(); - ret = next_time_internal(); - erts_smp_mtx_unlock(&tiw_lock); - return ret; +static ERTS_INLINE void +timeout_timer(ErtsTWheelTimer *p) +{ + ErlTimeoutProc timeout; + void *arg; + p->slot = ERTS_TWHEEL_SLOT_INACTIVE; + timeout = p->u.func.timeout; + arg = p->u.func.arg; + (*timeout)(arg); + ASSERT_NO_LOCKED_LOCKS; } -static ERTS_INLINE void bump_timer_internal(erts_short_time_t dt) /* PRE: tiw_lock is write-locked */ +void +erts_bump_timers(ErtsTimerWheel *tiw, ErtsMonotonicTime curr_time) { - Uint keep_pos; - Uint count; - ErlTimer *p, **prev, *timeout_head, **timeout_tail; - Uint dtime = (Uint) dt; - - /* no need to bump the position if there aren't any timeouts */ - if (tiw_nto == 0) { - erts_smp_mtx_unlock(&tiw_lock); - return; + int tiw_pos_ix, slots, yielded_slot_restarted, yield_count; + ErtsMonotonicTime bump_to, tmp_slots, old_pos; + ERTS_MSACC_PUSH_AND_SET_STATE_M_X(ERTS_MSACC_STATE_TIMERS); + + yield_count = ERTS_TWHEEL_BUMP_YIELD_LIMIT; + + /* + * In order to be fair we always continue with work + * where we left off when restarting after a yield. + */ + + if (tiw->yield_slot >= 0) { + yielded_slot_restarted = 1; + tiw_pos_ix = tiw->yield_slot; + slots = tiw->yield_slots_left; + bump_to = tiw->pos; + old_pos = tiw->yield_start_pos; + goto restart_yielded_slot; } - /* if do_time > TIW_SIZE we want to go around just once */ - count = (Uint)(dtime / TIW_SIZE) + 1; - keep_pos = (tiw_pos + dtime) % TIW_SIZE; - if (dtime > TIW_SIZE) dtime = TIW_SIZE; - - timeout_head = NULL; - timeout_tail = &timeout_head; - while (dtime > 0) { - /* this is to decrease the counters with the right amount */ - /* when dtime >= TIW_SIZE */ - if (tiw_pos == keep_pos) count--; - prev = &tiw[tiw_pos]; - while ((p = *prev) != NULL) { - ASSERT( p != p->next); - if (p->count < count) { /* we have a timeout */ - /* remove min time */ - if (tiw_min_ptr == p) { - tiw_min_ptr = NULL; - tiw_min = 0; + do { + + yielded_slot_restarted = 0; + + bump_to = ERTS_MONOTONIC_TO_CLKTCKS(curr_time); + + while (1) { + ErtsTWheelTimer *p; + + old_pos = tiw->pos; + + if (tiw->nto == 0) { + empty_wheel: + ERTS_DBG_CHK_SAFE_TO_SKIP_TO(tiw, bump_to); + tiw->true_next_timeout_time = 0; + tiw->next_timeout_time = curr_time + ERTS_MONOTONIC_DAY; + tiw->pos = bump_to; + tiw->yield_slot = ERTS_TWHEEL_SLOT_INACTIVE; + ERTS_MSACC_POP_STATE_M_X(); + return; + } + + p = tiw->at_once.head; + while (p) { + if (--yield_count <= 0) { + ERTS_TW_ASSERT(tiw->nto > 0); + ERTS_TW_ASSERT(tiw->at_once.nto > 0); + tiw->yield_slot = ERTS_TWHEEL_SLOT_AT_ONCE; + tiw->true_next_timeout_time = 1; + tiw->next_timeout_time = ERTS_CLKTCKS_TO_MONOTONIC(old_pos); + ERTS_MSACC_POP_STATE_M_X(); + return; } - /* Remove from list */ - remove_timer(p); - *timeout_tail = p; /* Insert in timeout queue */ - timeout_tail = &p->next; + ERTS_TW_ASSERT(tiw->nto > 0); + ERTS_TW_ASSERT(tiw->at_once.nto > 0); + tiw->nto--; + tiw->at_once.nto--; + tiw->at_once.head = p->next; + if (p->next) + p->next->prev = NULL; + else + tiw->at_once.tail = NULL; + + timeout_timer(p); + + p = tiw->at_once.head; } - else { - /* no timeout, just decrease counter */ - p->count -= count; - prev = &p->next; + + if (tiw->pos >= bump_to) { + ERTS_MSACC_POP_STATE_M_X(); + break; + } + + if (tiw->nto == 0) + goto empty_wheel; + + if (tiw->true_next_timeout_time) { + ErtsMonotonicTime skip_until_pos; + /* + * No need inspecting slots where we know no timeouts + * to trigger should reside. + */ + + skip_until_pos = ERTS_MONOTONIC_TO_CLKTCKS(tiw->next_timeout_time); + if (skip_until_pos > bump_to) + skip_until_pos = bump_to; + + skip_until_pos--; + + if (skip_until_pos > tiw->pos) { + ERTS_DBG_CHK_SAFE_TO_SKIP_TO(tiw, skip_until_pos); + + tiw->pos = skip_until_pos; + } + } + + tiw_pos_ix = (int) ((tiw->pos+1) & (ERTS_TIW_SIZE-1)); + tmp_slots = (bump_to - tiw->pos); + if (tmp_slots < (ErtsMonotonicTime) ERTS_TIW_SIZE) + slots = (int) tmp_slots; + else + slots = ERTS_TIW_SIZE; + + tiw->pos = bump_to; + + while (slots > 0) { + + p = tiw->w[tiw_pos_ix]; + if (p) { + if (p->next == p) { + ERTS_TW_ASSERT(tiw->sentinel.next == &tiw->sentinel); + ERTS_TW_ASSERT(tiw->sentinel.prev == &tiw->sentinel); + } + else { + tiw->sentinel.next = p->next; + tiw->sentinel.prev = p->prev; + tiw->sentinel.next->prev = &tiw->sentinel; + tiw->sentinel.prev->next = &tiw->sentinel; + } + tiw->w[tiw_pos_ix] = NULL; + + while (1) { + + if (p->timeout_pos > bump_to) { + /* Very unusual case... */ + ++yield_count; + insert_timer_into_slot(tiw, tiw_pos_ix, p); + } + else { + /* Normal case... */ + timeout_timer(p); + tiw->nto--; + } + + restart_yielded_slot: + + p = tiw->sentinel.next; + if (p == &tiw->sentinel) { + ERTS_TW_ASSERT(tiw->sentinel.prev == &tiw->sentinel); + break; + } + + if (--yield_count <= 0) { + tiw->true_next_timeout_time = 1; + tiw->next_timeout_time = ERTS_CLKTCKS_TO_MONOTONIC(old_pos); + tiw->yield_slot = tiw_pos_ix; + tiw->yield_slots_left = slots; + tiw->yield_start_pos = old_pos; + ERTS_MSACC_POP_STATE_M_X(); + return; /* Yield! */ + } + + tiw->sentinel.next = p->next; + p->next->prev = &tiw->sentinel; + } + } + tiw_pos_ix++; + if (tiw_pos_ix == ERTS_TIW_SIZE) + tiw_pos_ix = 0; + slots--; } } - tiw_pos = (tiw_pos + 1) % TIW_SIZE; - dtime--; - } - tiw_pos = keep_pos; - if (tiw_min_ptr) - tiw_min -= dt; - - erts_smp_mtx_unlock(&tiw_lock); - - /* Call timedout timers callbacks */ - while (timeout_head) { - p = timeout_head; - timeout_head = p->next; - /* Here comes hairy use of the timer fields! - * They are reset without having the lock. - * It is assumed that no code but this will - * accesses any field until the ->timeout - * callback is called. - */ - p->next = NULL; - p->prev = NULL; - p->slot = 0; - (*p->timeout)(p->arg); - } -} -void erts_bump_timer(erts_short_time_t dt) /* dt is value from do_time */ -{ - erts_smp_mtx_lock(&tiw_lock); - bump_timer_internal(dt); + } while (yielded_slot_restarted); + + tiw->yield_slot = ERTS_TWHEEL_SLOT_INACTIVE; + tiw->true_next_timeout_time = 0; + tiw->next_timeout_time = curr_time + ERTS_MONOTONIC_DAY; + + /* Search at most two seconds ahead... */ + (void) find_next_timeout(NULL, tiw, 0, curr_time, ERTS_SEC_TO_MONOTONIC(2)); + ERTS_MSACC_POP_STATE_M_X(); } Uint erts_timer_wheel_memory_size(void) { - return (Uint) TIW_SIZE * sizeof(ErlTimer*); + return sizeof(ErtsTimerWheel)*erts_no_schedulers; +} + +ErtsTimerWheel * +erts_create_timer_wheel(ErtsSchedulerData *esdp) +{ + ErtsMonotonicTime mtime; + int i; + ErtsTimerWheel *tiw; + tiw = erts_alloc_permanent_cache_aligned(ERTS_ALC_T_TIMER_WHEEL, + sizeof(ErtsTimerWheel)); + for(i = 0; i < ERTS_TIW_SIZE; i++) + tiw->w[i] = NULL; + + mtime = erts_get_monotonic_time(esdp); + tiw->pos = ERTS_MONOTONIC_TO_CLKTCKS(mtime); + tiw->nto = 0; + tiw->at_once.head = NULL; + tiw->at_once.tail = NULL; + tiw->at_once.nto = 0; + tiw->yield_slot = ERTS_TWHEEL_SLOT_INACTIVE; + tiw->true_next_timeout_time = 0; + tiw->next_timeout_time = mtime + ERTS_MONOTONIC_DAY; + tiw->sentinel.next = &tiw->sentinel; + tiw->sentinel.prev = &tiw->sentinel; + tiw->sentinel.u.func.timeout = NULL; + tiw->sentinel.u.func.cancel = NULL; + tiw->sentinel.u.func.arg = NULL; + return tiw; +} + +ErtsNextTimeoutRef +erts_get_next_timeout_reference(ErtsTimerWheel *tiw) +{ + return (ErtsNextTimeoutRef) &tiw->next_timeout_time; } + /* this routine links the time cells into a free list at the start and sets the time queue as empty */ void -erts_init_time(void) +erts_init_time(int time_correction, ErtsTimeWarpMode time_warp_mode) { - int i, itime; + int itime; /* system dependent init; must be done before do_time_init() if timer thread is enabled */ - itime = erts_init_time_sup(); + itime = erts_init_time_sup(time_correction, time_warp_mode); #ifdef TIW_ITIME_IS_CONSTANT if (itime != TIW_ITIME) { - erl_exit(ERTS_ABORT_EXIT, "timer resolution mismatch %d != %d", itime, TIW_ITIME); + erts_exit(ERTS_ABORT_EXIT, "timer resolution mismatch %d != %d", itime, TIW_ITIME); } #else tiw_itime = itime; #endif +} - erts_smp_mtx_init(&tiw_lock, "timer_wheel"); +void +erts_twheel_set_timer(ErtsTimerWheel *tiw, + ErtsTWheelTimer *p, ErlTimeoutProc timeout, + ErlCancelProc cancel, void *arg, + ErtsMonotonicTime timeout_pos) +{ + ErtsMonotonicTime timeout_time; + ERTS_MSACC_PUSH_AND_SET_STATE_M_X(ERTS_MSACC_STATE_TIMERS); - tiw = (ErlTimer**) erts_alloc(ERTS_ALC_T_TIMER_WHEEL, - TIW_SIZE * sizeof(ErlTimer*)); - for(i = 0; i < TIW_SIZE; i++) - tiw[i] = NULL; - do_time_init(); - tiw_pos = tiw_nto = 0; - tiw_min_ptr = NULL; - tiw_min = 0; -} + p->u.func.timeout = timeout; + p->u.func.cancel = cancel; + p->u.func.arg = arg; + ERTS_TW_ASSERT(p->slot == ERTS_TWHEEL_SLOT_INACTIVE); + if (timeout_pos <= tiw->pos) { + tiw->nto++; + tiw->at_once.nto++; + p->next = NULL; + p->prev = tiw->at_once.tail; + if (tiw->at_once.tail) { + ERTS_TW_ASSERT(tiw->at_once.head); + tiw->at_once.tail->next = p; + } + else { + ERTS_TW_ASSERT(!tiw->at_once.head); + tiw->at_once.head = p; + } + tiw->at_once.tail = p; + p->timeout_pos = tiw->pos; + p->slot = ERTS_TWHEEL_SLOT_AT_ONCE; + timeout_time = ERTS_CLKTCKS_TO_MONOTONIC(tiw->pos); + } + else { + int slot; + /* calculate slot */ + slot = (int) (timeout_pos & (ERTS_TIW_SIZE-1)); -/* -** Insert a process into the time queue, with a timeout 't' -*/ -static void -insert_timer(ErlTimer* p, Uint t) -{ - Uint tm; - Uint64 ticks; + insert_timer_into_slot(tiw, slot, p); - /* The current slot (tiw_pos) in timing wheel is the next slot to be - * be processed. Hence no extra time tick is needed. - * - * (x + y - 1)/y is precisely the "number of bins" formula. - */ - ticks = (t + (TIW_ITIME - 1)) / TIW_ITIME; + tiw->nto++; - /* - * Ticks must be a Uint64, or the addition may overflow here, - * resulting in an incorrect value for p->count below. - */ - ticks += do_time_update(); /* Add backlog of unprocessed time */ - - /* calculate slot */ - tm = (ticks + tiw_pos) % TIW_SIZE; - p->slot = (Uint) tm; - p->count = (Uint) (ticks / TIW_SIZE); - - /* insert at head of list at slot */ - p->next = tiw[tm]; - p->prev = NULL; - if (p->next != NULL) - p->next->prev = p; - tiw[tm] = p; - - - /* insert min time */ - if ((tiw_nto == 0) || ((tiw_min_ptr != NULL) && (ticks < tiw_min))) { - tiw_min = ticks; - tiw_min_ptr = p; - } - if ((tiw_min_ptr == p) && (ticks > tiw_min)) { - /* some other timer might be 'min' now */ - tiw_min = 0; - tiw_min_ptr = NULL; + timeout_time = ERTS_CLKTCKS_TO_MONOTONIC(timeout_pos); + p->timeout_pos = timeout_pos; } - tiw_nto++; + if (timeout_time < tiw->next_timeout_time) { + tiw->true_next_timeout_time = 1; + tiw->next_timeout_time = timeout_time; + } + ERTS_MSACC_POP_STATE_M_X(); } void -erts_set_timer(ErlTimer* p, ErlTimeoutProc timeout, ErlCancelProc cancel, - void* arg, Uint t) +erts_twheel_cancel_timer(ErtsTimerWheel *tiw, ErtsTWheelTimer *p) { - - erts_deliver_time(); - erts_smp_mtx_lock(&tiw_lock); - if (p->active) { /* XXX assert ? */ - erts_smp_mtx_unlock(&tiw_lock); - return; + if (p->slot != ERTS_TWHEEL_SLOT_INACTIVE) { + ErlCancelProc cancel; + void *arg; + ERTS_MSACC_PUSH_AND_SET_STATE_M_X(ERTS_MSACC_STATE_TIMERS); + remove_timer(tiw, p); + cancel = p->u.func.cancel; + arg = p->u.func.arg; + if (cancel) + (*cancel)(arg); + ERTS_MSACC_POP_STATE_M_X(); } - p->timeout = timeout; - p->cancel = cancel; - p->arg = arg; - p->active = 1; - insert_timer(p, t); - erts_smp_mtx_unlock(&tiw_lock); -#if defined(ERTS_SMP) - if (t <= (Uint) ERTS_SHORT_TIME_T_MAX) - erts_sys_schedule_interrupt_timed(1, (erts_short_time_t) t); -#endif } void -erts_cancel_timer(ErlTimer* p) +erts_twheel_debug_foreach(ErtsTimerWheel *tiw, + void (*tclbk)(void *), + void (*func)(void *, + ErtsMonotonicTime, + void *), + void *arg) { - erts_smp_mtx_lock(&tiw_lock); - if (!p->active) { /* allow repeated cancel (drivers) */ - erts_smp_mtx_unlock(&tiw_lock); - return; - } - - /* is it the 'min' timer, remove min */ - if (p == tiw_min_ptr) { - tiw_min_ptr = NULL; - tiw_min = 0; + ErtsTWheelTimer *tmr; + int ix; + + tmr = tiw->sentinel.next; + while (tmr != &tiw->sentinel) { + if (tmr->u.func.timeout == tclbk) + (*func)(arg, tmr->timeout_pos, tmr->u.func.arg); + tmr = tmr->next; } - remove_timer(p); - p->slot = p->count = 0; - - if (p->cancel != NULL) { - erts_smp_mtx_unlock(&tiw_lock); - (*p->cancel)(p->arg); - return; + for (tmr = tiw->at_once.head; tmr; tmr = tmr->next) { + if (tmr->u.func.timeout == tclbk) + (*func)(arg, tmr->timeout_pos, tmr->u.func.arg); } - erts_smp_mtx_unlock(&tiw_lock); -} - -/* - Returns the amount of time left in ms until the timer 'p' is triggered. - 0 is returned if 'p' isn't active. - 0 is returned also if the timer is overdue (i.e., would have triggered - immediately if it hadn't been cancelled). -*/ -Uint -erts_time_left(ErlTimer *p) -{ - Uint left; - erts_short_time_t dt; - erts_smp_mtx_lock(&tiw_lock); - - if (!p->active) { - erts_smp_mtx_unlock(&tiw_lock); - return 0; + for (ix = 0; ix < ERTS_TIW_SIZE; ix++) { + tmr = tiw->w[ix]; + if (tmr) { + do { + if (tmr->u.func.timeout == tclbk) + (*func)(arg, tmr->timeout_pos, tmr->u.func.arg); + tmr = tmr->next; + } while (tmr != tiw->w[ix]); + } } - - if (p->slot < tiw_pos) - left = (p->count + 1) * TIW_SIZE + p->slot - tiw_pos; - else - left = p->count * TIW_SIZE + p->slot - tiw_pos; - dt = do_time_read(); - if (left < dt) - left = 0; - else - left -= dt; - - erts_smp_mtx_unlock(&tiw_lock); - - return (Uint) left * TIW_ITIME; } -#ifdef DEBUG +#ifdef ERTS_TW_DEBUG void erts_p_slpq(void) { + erts_printf("Not yet implemented...\n"); +#if 0 + ErtsMonotonicTime current_time = erts_get_monotonic_time(NULL); int i; - ErlTimer* p; + ErtsTWheelTimer* p; - erts_smp_mtx_lock(&tiw_lock); - /* print the whole wheel, starting at the current position */ - erts_printf("\ntiw_pos = %d tiw_nto %d\n", tiw_pos, tiw_nto); - i = tiw_pos; - if (tiw[i] != NULL) { + erts_printf("\ncurrent time = %bps tiw_pos = %d tiw_nto %d\n", + current_time, tiw->pos, tiw->nto); + i = tiw->pos; + if (tiw->w[i] != NULL) { erts_printf("%d:\n", i); - for(p = tiw[i]; p != NULL; p = p->next) { - erts_printf(" (count %d, slot %d)\n", - p->count, p->slot); + for(p = tiw->w[i]; p != NULL; p = p->next) { + erts_printf(" (timeout time %bps, slot %d)\n", + ERTS_CLKTCKS_TO_MONOTONIC(p->timeout_pos), + p->slot); } } - for(i = (i+1)%TIW_SIZE; i != tiw_pos; i = (i+1)%TIW_SIZE) { - if (tiw[i] != NULL) { + for(i = ((i+1) & (ERTS_TIW_SIZE-1)); i != (tiw->pos & (ERTS_TIW_SIZE-1)); i = ((i+1) & (ERTS_TIW_SIZE-1))) { + if (tiw->w[i] != NULL) { erts_printf("%d:\n", i); - for(p = tiw[i]; p != NULL; p = p->next) { - erts_printf(" (count %d, slot %d)\n", - p->count, p->slot); + for(p = tiw->w[i]; p != NULL; p = p->next) { + erts_printf(" (timeout time %bps, slot %d)\n", + ERTS_CLKTCKS_TO_MONOTONIC(p->timeout_pos), p->slot); } } } - - erts_smp_mtx_unlock(&tiw_lock); +#endif } -#endif /* DEBUG */ +#endif /* ERTS_TW_DEBUG */ diff --git a/erts/emulator/beam/utils.c b/erts/emulator/beam/utils.c index 738f793020..66fcfcf6ce 100644 --- a/erts/emulator/beam/utils.c +++ b/erts/emulator/beam/utils.c @@ -3,16 +3,17 @@ * * Copyright Ericsson AB 1996-2014. All Rights Reserved. * - * The contents of this file are subject to the Erlang Public License, - * Version 1.1, (the "License"); you may not use this file except in - * compliance with the License. You should have received a copy of the - * Erlang Public License along with this software. If not, it can be - * retrieved online at http://www.erlang.org/. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Software distributed under the License is distributed on an "AS IS" - * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See - * the License for the specific language governing rights and limitations - * under the License. + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. * * %CopyrightEnd% */ @@ -48,6 +49,13 @@ #include "erl_sched_spec_pre_alloc.h" #include "beam_bp.h" #include "erl_ptab.h" +#include "erl_check_io.h" +#include "erl_bif_unique.h" +#define ERTS_WANT_TIMER_WHEEL_API +#include "erl_time.h" +#ifdef HIPE +# include "hipe_mode_switch.h" +#endif #undef M_TRIM_THRESHOLD #undef M_TOP_PAD @@ -102,7 +110,6 @@ Eterm* erts_heap_alloc(Process* p, Uint need, Uint xtra) { ErlHeapFragment* bp; - Eterm* htop; Uint n; #if defined(DEBUG) || defined(CHECK_FOR_HOLES) Uint i; @@ -148,16 +155,6 @@ erts_heap_alloc(Process* p, Uint need, Uint xtra) n--; #endif - /* - * When we have created a heap fragment, we are no longer allowed - * to store anything more on the heap. - */ - htop = HEAP_TOP(p); - if (htop < HEAP_LIMIT(p)) { - *htop = make_pos_bignum_header(HEAP_LIMIT(p)-htop-1); - HEAP_TOP(p) = HEAP_LIMIT(p); - } - bp->next = MBUF(p); MBUF(p) = bp; bp->alloc_size = n; @@ -186,12 +183,18 @@ erts_set_hole_marker(Eterm* ptr, Uint sz) * Helper function for the ESTACK macros defined in global.h. */ void -erl_grow_estack(ErtsEStack* s, Eterm* default_estack) +erl_grow_estack(ErtsEStack* s, Uint need) { Uint old_size = (s->end - s->start); - Uint new_size = old_size * 2; + Uint new_size; Uint sp_offs = s->sp - s->start; - if (s->start != default_estack) { + + if (need < old_size) + new_size = 2*old_size; + else + new_size = ((need / old_size) + 2) * old_size; + + if (s->start != s->edefault) { s->start = erts_realloc(s->alloc_type, s->start, new_size*sizeof(Eterm)); } else { @@ -206,12 +209,18 @@ erl_grow_estack(ErtsEStack* s, Eterm* default_estack) * Helper function for the WSTACK macros defined in global.h. */ void -erl_grow_wstack(ErtsWStack* s, UWord* default_wstack) +erl_grow_wstack(ErtsWStack* s, Uint need) { Uint old_size = (s->wend - s->wstart); - Uint new_size = old_size * 2; + Uint new_size; Uint sp_offs = s->wsp - s->wstart; - if (s->wstart != default_wstack) { + + if (need < old_size) + new_size = 2 * old_size; + else + new_size = ((need / old_size) + 2) * old_size; + + if (s->wstart != s->wdefault) { s->wstart = erts_realloc(s->alloc_type, s->wstart, new_size*sizeof(UWord)); } else { @@ -223,6 +232,57 @@ erl_grow_wstack(ErtsWStack* s, UWord* default_wstack) s->wsp = s->wstart + sp_offs; } +/* + * Helper function for the PSTACK macros defined in global.h. + */ +void +erl_grow_pstack(ErtsPStack* s, void* default_pstack, unsigned need_bytes) +{ + Uint old_size = s->pend - s->pstart; + Uint new_size; + Uint sp_offs = s->psp - s->pstart; + + if (need_bytes < old_size) + new_size = 2 * old_size; + else + new_size = ((need_bytes / old_size) + 2) * old_size; + + if (s->pstart != default_pstack) { + s->pstart = erts_realloc(s->alloc_type, s->pstart, new_size); + } else { + byte* new_ptr = erts_alloc(s->alloc_type, new_size); + sys_memcpy(new_ptr, s->pstart, old_size); + s->pstart = new_ptr; + } + s->pend = s->pstart + new_size; + s->psp = s->pstart + sp_offs; +} + +/* + * Helper function for the EQUEUE macros defined in global.h. + */ + +void +erl_grow_equeue(ErtsEQueue* q, Eterm* default_equeue) +{ + Uint old_size = (q->end - q->start); + Uint new_size = old_size * 2; + Uint first_part = (q->end - q->front); + Uint second_part = (q->back - q->start); + Eterm* new_ptr = erts_alloc(q->alloc_type, new_size*sizeof(Eterm)); + ASSERT(q->back == q->front); // of course the queue is full now! + if (first_part > 0) + sys_memcpy(new_ptr, q->front, first_part*sizeof(Eterm)); + if (second_part > 0) + sys_memcpy(new_ptr+first_part, q->start, second_part*sizeof(Eterm)); + if (q->start != default_equeue) + erts_free(q->alloc_type, q->start); + q->start = new_ptr; + q->end = q->start + new_size; + q->front = q->start; + q->back = q->start + old_size; +} + /* CTYPE macros */ #define LATIN1 @@ -257,10 +317,10 @@ erl_grow_wstack(ErtsWStack* s, UWord* default_wstack) * Calculate length of a list. * Returns -1 if not a proper list (i.e. not terminated with NIL) */ -int +Sint erts_list_length(Eterm list) { - int i = 0; + Sint i = 0; while(is_list(list)) { i++; @@ -310,6 +370,17 @@ int erts_fit_in_bits_int32(Sint32 value) return fit_in_bits((Sint64) (Uint32) value, 4); } +int erts_fit_in_bits_uint(Uint value) +{ +#if ERTS_SIZEOF_ETERM == 4 + return fit_in_bits((Sint64) (Uint32) value, 4); +#elif ERTS_SIZEOF_ETERM == 8 + return fit_in_bits(value, 5); +#else +# error "No way, Jose" +#endif +} + int erts_print(int to, void *arg, char *format, ...) { @@ -342,9 +413,6 @@ erts_print(int to, void *arg, char *format, ...) case ERTS_PRINT_DSBUF: res = erts_vdsprintf((erts_dsprintf_buf_t *) arg, format, arg_list); break; - case ERTS_PRINT_INVALID: - res = -EINVAL; - break; default: res = erts_vfdprintf((int) to, format, arg_list); break; @@ -706,7 +774,7 @@ erts_bld_atom_2uint_3tup_list(Uint **hpp, Uint *szp, Sint length, ** If N < 0, Y = FUNNY_NUMBER4 else Y = FUNNY_NUMBER3. ** The hash value is Y*h(J) mod 2^32 where h(J) is calculated like ** h(0) = <initial hash> -** h(i) = h(i-i)*X + B(i-1) +** h(i) = h(i-1)*X + B(i-1) ** The above should hold regardless of internal representation. ** Pids are hashed like small numbers but with differrent constants, as are ** ports. @@ -787,11 +855,10 @@ Uint32 make_hash(Eterm term_arg) Eterm hash = 0; unsigned op; - /* Must not collide with the real tag_val_def's: */ -#define MAKE_HASH_TUPLE_OP 0x11 -#define MAKE_HASH_TERM_ARRAY_OP 0x12 -#define MAKE_HASH_CDR_PRE_OP 0x13 -#define MAKE_HASH_CDR_POST_OP 0x14 +#define MAKE_HASH_TUPLE_OP (FIRST_VACANT_TAG_DEF) +#define MAKE_HASH_TERM_ARRAY_OP (FIRST_VACANT_TAG_DEF+1) +#define MAKE_HASH_CDR_PRE_OP (FIRST_VACANT_TAG_DEF+2) +#define MAKE_HASH_CDR_POST_OP (FIRST_VACANT_TAG_DEF+3) /* ** Convenience macro for calculating a bytewise hash on an unsigned 32 bit @@ -841,7 +908,7 @@ tail_recur: Uint y2 = y1 < 0 ? -(Uint)y1 : y1; UINT32_HASH_STEP(y2, FUNNY_NUMBER2); -#if defined(ARCH_64) && !HALFWORD_HEAP +#if defined(ARCH_64) if (y2 >> 32) UINT32_HASH_STEP(y2 >> 32, FUNNY_NUMBER2); #endif @@ -901,12 +968,15 @@ tail_recur: UINT32_HASH_RET(external_ref_numbers(term)[0],FUNNY_NUMBER9,FUNNY_NUMBER10); case FLOAT_DEF: { - FloatDef ff; - GET_DOUBLE(term, ff); - hash = hash*FUNNY_NUMBER6 + (ff.fw[0] ^ ff.fw[1]); - break; + FloatDef ff; + GET_DOUBLE(term, ff); + if (ff.fd == 0.0f) { + /* ensure positive 0.0 */ + ff.fd = erts_get_positive_zero_float(); + } + hash = hash*FUNNY_NUMBER6 + (ff.fw[0] ^ ff.fw[1]); + break; } - case MAKE_HASH_CDR_PRE_OP: term = (Eterm) WSTACK_POP(stack); if (is_not_list(term)) { @@ -959,7 +1029,7 @@ tail_recur: } d = BIG_DIGIT(ptr, k); k = sizeof(ErtsDigit); -#if defined(ARCH_64) && !HALFWORD_HEAP +#if defined(ARCH_64) if (!(d >> 32)) k /= 2; #endif @@ -971,23 +1041,8 @@ tail_recur: break; } case MAP_DEF: - { - map_t *mp = (map_t *)map_val(term); - int size = map_get_size(mp); - Eterm *ks = map_get_keys(mp); - Eterm *vs = map_get_values(mp); - - /* Use a prime with size to remedy some of - * the {} and <<>> hash problems */ - hash = hash*FUNNY_NUMBER13 + FUNNY_NUMBER14 + size; - if (size == 0) - break; - - /* push values first */ - WSTACK_PUSH3(stack, (UWord)vs, (UWord) size, MAKE_HASH_TERM_ARRAY_OP); - WSTACK_PUSH3(stack, (UWord)ks, (UWord) size, MAKE_HASH_TERM_ARRAY_OP); - break; - } + hash = hash*FUNNY_NUMBER13 + FUNNY_NUMBER14 + make_hash2(term); + break; case TUPLE_DEF: { Eterm* ptr = tuple_val(term); @@ -1014,7 +1069,7 @@ tail_recur: } default: - erl_exit(1, "Invalid tag in make_hash(0x%X,0x%X)\n", term, op); + erts_exit(ERTS_ERROR_EXIT, "Invalid tag in make_hash(0x%X,0x%X)\n", term, op); return 0; } if (WSTACK_ISEMPTY(stack)) break; @@ -1091,11 +1146,12 @@ Uint32 make_hash2(Eterm term) { Uint32 hash; - Uint32 hash_xor_keys = 0; - Uint32 hash_xor_values = 0; + Uint32 hash_xor_pairs; DeclareTmpHeapNoproc(tmp_big,2); -/* (HCONST * {2, ..., 16}) mod 2^32 */ + ERTS_UNDEF(hash_xor_pairs, 0); + +/* (HCONST * {2, ..., 22}) mod 2^32 */ #define HCONST_2 0x3c6ef372UL #define HCONST_3 0xdaa66d2bUL #define HCONST_4 0x78dde6e4UL @@ -1111,10 +1167,16 @@ make_hash2(Eterm term) #define HCONST_14 0xa708a81eUL #define HCONST_15 0x454021d7UL #define HCONST_16 0xe3779b90UL +#define HCONST_17 0x81af1549UL +#define HCONST_18 0x1fe68f02UL +#define HCONST_19 0xbe1e08bbUL +#define HCONST_20 0x5c558274UL +#define HCONST_21 0xfa8cfc2dUL +#define HCONST_22 0x98c475e6UL #define HASH_MAP_TAIL (_make_header(1,_TAG_HEADER_REF)) -#define HASH_MAP_KEY (_make_header(2,_TAG_HEADER_REF)) -#define HASH_MAP_VAL (_make_header(3,_TAG_HEADER_REF)) +#define HASH_MAP_PAIR (_make_header(2,_TAG_HEADER_REF)) +#define HASH_CDR (_make_header(3,_TAG_HEADER_REF)) #define UINT32_HASH_2(Expr1, Expr2, AConst) \ do { \ @@ -1137,6 +1199,13 @@ make_hash2(Eterm term) } while(0) #define IS_SSMALL28(x) (((Uint) (((x) >> (28-1)) + 1)) < 2) + +#ifdef ARCH_64 +# define POINTER_HASH(Ptr, AConst) UINT32_HASH_2((Uint32)(UWord)(Ptr), (((UWord)(Ptr)) >> 32), AConst) +#else +# define POINTER_HASH(Ptr, AConst) UINT32_HASH(Ptr, AConst) +#endif + /* Optimization. Simple cases before declaration of estack. */ if (primary_tag(term) == TAG_PRIMARY_IMMED1) { switch (term & _TAG_IMMED1_MASK) { @@ -1191,9 +1260,9 @@ make_hash2(Eterm term) if (c > 0) UINT32_HASH(sh, HCONST_4); if (is_list(term)) { - term = *ptr; - tmp = *++ptr; - ESTACK_PUSH(s, tmp); + tmp = CDR(ptr); + ESTACK_PUSH(s, tmp); + term = CAR(ptr); } } break; @@ -1210,51 +1279,90 @@ make_hash2(Eterm term) UINT32_HASH(arity, HCONST_9); if (arity == 0) /* Empty tuple */ goto hash2_common; - for (i = arity; i >= 1; i--) { - tmp = elem[i]; - ESTACK_PUSH(s, tmp); + for (i = arity; ; i--) { + term = elem[i]; + if (i == 1) + break; + ESTACK_PUSH(s, term); } - goto hash2_common; - } - break; - case MAP_SUBTAG: - { - map_t *mp = (map_t *)map_val(term); - int i; - int size = map_get_size(mp); - Eterm *ks = map_get_keys(mp); - Eterm *vs = map_get_values(mp); - UINT32_HASH(size, HCONST_16); - if (size == 0) { - goto hash2_common; - } - ESTACK_PUSH(s, hash_xor_values); - ESTACK_PUSH(s, hash_xor_keys); - ESTACK_PUSH(s, hash); - ESTACK_PUSH(s, HASH_MAP_TAIL); - hash = 0; - hash_xor_keys = 0; - hash_xor_values = 0; - for (i = size - 1; i >= 0; i--) { - tmp = vs[i]; - ESTACK_PUSH(s, HASH_MAP_VAL); - ESTACK_PUSH(s, tmp); - } - /* We do not want to expose the tuple representation. - * Do not push the keys as a tuple. - */ - for (i = size - 1; i >= 0; i--) { - tmp = ks[i]; - ESTACK_PUSH(s, HASH_MAP_KEY); - ESTACK_PUSH(s, tmp); - } - goto hash2_common; } break; + case MAP_SUBTAG: + { + Eterm* ptr = boxed_val(term) + 1; + Uint size; + int i; + switch (hdr & _HEADER_MAP_SUBTAG_MASK) { + case HAMT_SUBTAG_HEAD_FLATMAP: + { + flatmap_t *mp = (flatmap_t *)flatmap_val(term); + Eterm *ks = flatmap_get_keys(mp); + Eterm *vs = flatmap_get_values(mp); + size = flatmap_get_size(mp); + UINT32_HASH(size, HCONST_16); + if (size == 0) + goto hash2_common; + + /* We want a portable hash function that is *independent* of + * the order in which keys and values are encountered. + * We therefore calculate context independent hashes for all . + * key-value pairs and then xor them together. + */ + ESTACK_PUSH(s, hash_xor_pairs); + ESTACK_PUSH(s, hash); + ESTACK_PUSH(s, HASH_MAP_TAIL); + hash = 0; + hash_xor_pairs = 0; + for (i = size - 1; i >= 0; i--) { + ESTACK_PUSH(s, HASH_MAP_PAIR); + ESTACK_PUSH(s, vs[i]); + ESTACK_PUSH(s, ks[i]); + } + goto hash2_common; + } + + case HAMT_SUBTAG_HEAD_ARRAY: + case HAMT_SUBTAG_HEAD_BITMAP: + size = *ptr++; + UINT32_HASH(size, HCONST_16); + if (size == 0) + goto hash2_common; + ESTACK_PUSH(s, hash_xor_pairs); + ESTACK_PUSH(s, hash); + ESTACK_PUSH(s, HASH_MAP_TAIL); + hash = 0; + hash_xor_pairs = 0; + } + switch (hdr & _HEADER_MAP_SUBTAG_MASK) { + case HAMT_SUBTAG_HEAD_ARRAY: + i = 16; + break; + case HAMT_SUBTAG_HEAD_BITMAP: + case HAMT_SUBTAG_NODE_BITMAP: + i = hashmap_bitcount(MAP_HEADER_VAL(hdr)); + break; + default: + erts_exit(ERTS_ERROR_EXIT, "bad header"); + } + while (i) { + if (is_list(*ptr)) { + Eterm* cons = list_val(*ptr); + ESTACK_PUSH(s, HASH_MAP_PAIR); + ESTACK_PUSH(s, CDR(cons)); + ESTACK_PUSH(s, CAR(cons)); + } + else { + ASSERT(is_boxed(*ptr)); + ESTACK_PUSH(s, *ptr); + } + i--; ptr++; + } + goto hash2_common; + } + break; case EXPORT_SUBTAG: { Export* ep = *((Export **) (export_val(term) + 1)); - UINT32_HASH_2 (ep->code[2], atom_tab(atom_val(ep->code[0]))->slot.bucket.hvalue, @@ -1269,7 +1377,6 @@ make_hash2(Eterm term) { ErlFunThing* funp = (ErlFunThing *) fun_val(term); Uint num_free = funp->num_free; - UINT32_HASH_2 (num_free, atom_tab(atom_val(funp->fe->module))->slot.bucket.hvalue, @@ -1350,7 +1457,8 @@ make_hash2(Eterm term) do { Uint t; Uint32 x, y; - t = i < n ? BIG_DIGIT(ptr, i++) : 0; + ASSERT(i < n); + t = BIG_DIGIT(ptr, i++); x = t & 0xffffffff; y = t >> 32; UINT32_HASH_2(x, y, con); @@ -1383,6 +1491,10 @@ make_hash2(Eterm term) { FloatDef ff; GET_DOUBLE(term, ff); + if (ff.fd == 0.0f) { + /* ensure positive 0.0 */ + ff.fd = erts_get_positive_zero_float(); + } #if defined(WORDS_BIGENDIAN) || defined(DOUBLE_MIDDLE_ENDIAN) UINT32_HASH_2(ff.fw[0], ff.fw[1], HCONST_12); #else @@ -1393,7 +1505,7 @@ make_hash2(Eterm term) break; default: - erl_exit(1, "Invalid tag in make_hash2(0x%X)\n", term); + erts_exit(ERTS_ERROR_EXIT, "Invalid tag in make_hash2(0x%X)\n", term); } } break; @@ -1424,7 +1536,7 @@ make_hash2(Eterm term) UINT32_HASH(NIL_DEF, HCONST_2); goto hash2_common; default: - erl_exit(1, "Invalid tag in make_hash2(0x%X)\n", term); + erts_exit(ERTS_ERROR_EXIT, "Invalid tag in make_hash2(0x%X)\n", term); } case _TAG_IMMED1_SMALL: { @@ -1440,7 +1552,7 @@ make_hash2(Eterm term) } break; default: - erl_exit(1, "Invalid tag in make_hash2(0x%X)\n", term); + erts_exit(ERTS_ERROR_EXIT, "Invalid tag in make_hash2(0x%X)\n", term); hash2_common: /* Uint32 hash always has the hash value of the previous term, @@ -1458,20 +1570,397 @@ make_hash2(Eterm term) switch (term) { case HASH_MAP_TAIL: { hash = (Uint32) ESTACK_POP(s); - UINT32_HASH(hash_xor_keys, HCONST_16); - UINT32_HASH(hash_xor_values, HCONST_16); - hash_xor_keys = (Uint32) ESTACK_POP(s); - hash_xor_values = (Uint32) ESTACK_POP(s); + UINT32_HASH(hash_xor_pairs, HCONST_19); + hash_xor_pairs = (Uint32) ESTACK_POP(s); goto hash2_common; } - case HASH_MAP_KEY: - hash_xor_keys ^= hash; + case HASH_MAP_PAIR: + hash_xor_pairs ^= hash; hash = 0; goto hash2_common; - case HASH_MAP_VAL: - hash_xor_values ^= hash; + default: + break; + } + } + } + } +} + +/* Term hash function for internal use. + * + * Limitation #1: Is not "portable" in any way between different VM instances. + * + * Limitation #2: The hash value is only valid as long as the term exists + * somewhere in the VM. Why? Because external pids, ports and refs are hashed + * by mixing the node *pointer* value. If a node disappears and later reappears + * with a new ErlNode struct, externals from that node will hash different than + * before. + * + * One IMPORTANT property must hold (for hamt). + * EVERY BIT of the term that is significant for equality (see EQ) + * MUST BE USED AS INPUT FOR THE HASH. Two different terms must always have a + * chance of hashing different when salted: hash([Salt|A]) vs hash([Salt|B]). + * + * This is why we can not use cached hash values for atoms for example. + * + */ + +#define CONST_HASH(AConst) \ +do { /* Lightweight mixing of constant (type info) */ \ + hash ^= AConst; \ + hash = (hash << 17) ^ (hash >> (32-17)); \ +} while (0) + +Uint32 +make_internal_hash(Eterm term) +{ + Uint32 hash; + Uint32 hash_xor_pairs; + + ERTS_UNDEF(hash_xor_pairs, 0); + + /* Optimization. Simple cases before declaration of estack. */ + if (primary_tag(term) == TAG_PRIMARY_IMMED1) { + hash = 0; + #if ERTS_SIZEOF_ETERM == 8 + UINT32_HASH_2((Uint32)term, (Uint32)(term >> 32), HCONST); + #elif ERTS_SIZEOF_ETERM == 4 + UINT32_HASH(term, HCONST); + #else + # error "No you don't" + #endif + return hash; + } + { + Eterm tmp; + DECLARE_ESTACK(s); + + hash = 0; + for (;;) { + switch (primary_tag(term)) { + case TAG_PRIMARY_LIST: + { + int c = 0; + Uint32 sh = 0; + Eterm* ptr = list_val(term); + while (is_byte(*ptr)) { + /* Optimization for strings. */ + sh = (sh << 8) + unsigned_val(*ptr); + if (c == 3) { + UINT32_HASH(sh, HCONST_4); + c = sh = 0; + } else { + c++; + } + term = CDR(ptr); + if (is_not_list(term)) + break; + ptr = list_val(term); + } + if (c > 0) + UINT32_HASH_2(sh, (Uint32)c, HCONST_22); + + if (is_list(term)) { + tmp = CDR(ptr); + CONST_HASH(HCONST_17); /* Hash CAR in cons cell */ + ESTACK_PUSH(s, tmp); + if (is_not_list(tmp)) { + ESTACK_PUSH(s, HASH_CDR); + } + term = CAR(ptr); + } + } + break; + case TAG_PRIMARY_BOXED: + { + Eterm hdr = *boxed_val(term); + ASSERT(is_header(hdr)); + switch (hdr & _TAG_HEADER_MASK) { + case ARITYVAL_SUBTAG: + { + int i; + int arity = header_arity(hdr); + Eterm* elem = tuple_val(term); + UINT32_HASH(arity, HCONST_9); + if (arity == 0) /* Empty tuple */ + goto pop_next; + for (i = arity; ; i--) { + term = elem[i]; + if (i == 1) + break; + ESTACK_PUSH(s, term); + } + } + break; + + case MAP_SUBTAG: + { + Eterm* ptr = boxed_val(term) + 1; + Uint size; + int i; + switch (hdr & _HEADER_MAP_SUBTAG_MASK) { + case HAMT_SUBTAG_HEAD_FLATMAP: + { + flatmap_t *mp = (flatmap_t *)flatmap_val(term); + Eterm *ks = flatmap_get_keys(mp); + Eterm *vs = flatmap_get_values(mp); + size = flatmap_get_size(mp); + UINT32_HASH(size, HCONST_16); + if (size == 0) + goto pop_next; + + /* We want a hash function that is *independent* of + * the order in which keys and values are encountered. + * We therefore calculate context independent hashes for all . + * key-value pairs and then xor them together. + */ + ESTACK_PUSH(s, hash_xor_pairs); + ESTACK_PUSH(s, hash); + ESTACK_PUSH(s, HASH_MAP_TAIL); + hash = 0; + hash_xor_pairs = 0; + for (i = size - 1; i >= 0; i--) { + ESTACK_PUSH(s, HASH_MAP_PAIR); + ESTACK_PUSH(s, vs[i]); + ESTACK_PUSH(s, ks[i]); + } + goto pop_next; + } + case HAMT_SUBTAG_HEAD_ARRAY: + case HAMT_SUBTAG_HEAD_BITMAP: + size = *ptr++; + UINT32_HASH(size, HCONST_16); + if (size == 0) + goto pop_next; + ESTACK_PUSH(s, hash_xor_pairs); + ESTACK_PUSH(s, hash); + ESTACK_PUSH(s, HASH_MAP_TAIL); + hash = 0; + hash_xor_pairs = 0; + } + switch (hdr & _HEADER_MAP_SUBTAG_MASK) { + case HAMT_SUBTAG_HEAD_ARRAY: + i = 16; + break; + case HAMT_SUBTAG_HEAD_BITMAP: + case HAMT_SUBTAG_NODE_BITMAP: + i = hashmap_bitcount(MAP_HEADER_VAL(hdr)); + break; + default: + erts_exit(ERTS_ERROR_EXIT, "bad header"); + } + while (i) { + if (is_list(*ptr)) { + Eterm* cons = list_val(*ptr); + ESTACK_PUSH(s, HASH_MAP_PAIR); + ESTACK_PUSH(s, CDR(cons)); + ESTACK_PUSH(s, CAR(cons)); + } + else { + ASSERT(is_boxed(*ptr)); + ESTACK_PUSH(s, *ptr); + } + i--; ptr++; + } + goto pop_next; + } + break; + case EXPORT_SUBTAG: + { + Export* ep = *((Export **) (export_val(term) + 1)); + /* Assumes Export entries never moves */ + POINTER_HASH(ep, HCONST_14); + goto pop_next; + } + + case FUN_SUBTAG: + { + ErlFunThing* funp = (ErlFunThing *) fun_val(term); + Uint num_free = funp->num_free; + UINT32_HASH_2(num_free, funp->fe->module, HCONST_20); + UINT32_HASH_2(funp->fe->old_index, funp->fe->old_uniq, HCONST_21); + if (num_free == 0) { + goto pop_next; + } else { + Eterm* bptr = funp->env + num_free - 1; + while (num_free-- > 1) { + term = *bptr--; + ESTACK_PUSH(s, term); + } + term = *bptr; + } + } + break; + case REFC_BINARY_SUBTAG: + case HEAP_BINARY_SUBTAG: + case SUB_BINARY_SUBTAG: + { + byte* bptr; + unsigned sz = binary_size(term); + Uint32 con = HCONST_13 + hash; + Uint bitoffs; + Uint bitsize; + + ERTS_GET_BINARY_BYTES(term, bptr, bitoffs, bitsize); + if (sz == 0 && bitsize == 0) { + hash = con; + } else { + if (bitoffs == 0) { + hash = block_hash(bptr, sz, con); + if (bitsize > 0) { + UINT32_HASH_2(bitsize, (bptr[sz] >> (8 - bitsize)), + HCONST_15); + } + } else { + byte* buf = (byte *) erts_alloc(ERTS_ALC_T_TMP, + sz + (bitsize != 0)); + erts_copy_bits(bptr, bitoffs, 1, buf, 0, 1, sz*8+bitsize); + hash = block_hash(buf, sz, con); + if (bitsize > 0) { + UINT32_HASH_2(bitsize, (buf[sz] >> (8 - bitsize)), + HCONST_15); + } + erts_free(ERTS_ALC_T_TMP, (void *) buf); + } + } + goto pop_next; + } + break; + case POS_BIG_SUBTAG: + case NEG_BIG_SUBTAG: + { + Eterm* ptr = big_val(term); + Uint i = 0; + Uint n = BIG_SIZE(ptr); + Uint32 con = BIG_SIGN(ptr) ? HCONST_10 : HCONST_11; +#if D_EXP == 16 + do { + Uint32 x, y; + x = i < n ? BIG_DIGIT(ptr, i++) : 0; + x += (Uint32)(i < n ? BIG_DIGIT(ptr, i++) : 0) << 16; + y = i < n ? BIG_DIGIT(ptr, i++) : 0; + y += (Uint32)(i < n ? BIG_DIGIT(ptr, i++) : 0) << 16; + UINT32_HASH_2(x, y, con); + } while (i < n); +#elif D_EXP == 32 + do { + Uint32 x, y; + x = i < n ? BIG_DIGIT(ptr, i++) : 0; + y = i < n ? BIG_DIGIT(ptr, i++) : 0; + UINT32_HASH_2(x, y, con); + } while (i < n); +#elif D_EXP == 64 + do { + Uint t; + Uint32 x, y; + ASSERT(i < n); + t = BIG_DIGIT(ptr, i++); + x = t & 0xffffffff; + y = t >> 32; + UINT32_HASH_2(x, y, con); + } while (i < n); +#else +#error "unsupported D_EXP size" +#endif + goto pop_next; + } + break; + case REF_SUBTAG: + UINT32_HASH(internal_ref_numbers(term)[0], HCONST_7); + ASSERT(internal_ref_no_of_numbers(term) == 3); + UINT32_HASH_2(internal_ref_numbers(term)[1], + internal_ref_numbers(term)[2], HCONST_8); + goto pop_next; + + case EXTERNAL_REF_SUBTAG: + { + ExternalThing* thing = external_thing_ptr(term); + + ASSERT(external_thing_ref_no_of_numbers(thing) == 3); + /* See limitation #2 */ + #ifdef ARCH_64 + POINTER_HASH(thing->node, HCONST_7); + UINT32_HASH(external_thing_ref_numbers(thing)[0], HCONST_7); + #else + UINT32_HASH_2(thing->node, + external_thing_ref_numbers(thing)[0], HCONST_7); + #endif + UINT32_HASH_2(external_thing_ref_numbers(thing)[1], + external_thing_ref_numbers(thing)[2], HCONST_8); + goto pop_next; + } + case EXTERNAL_PID_SUBTAG: { + ExternalThing* thing = external_thing_ptr(term); + /* See limitation #2 */ + #ifdef ARCH_64 + POINTER_HASH(thing->node, HCONST_5); + UINT32_HASH(thing->data.ui[0], HCONST_5); + #else + UINT32_HASH_2(thing->node, thing->data.ui[0], HCONST_5); + #endif + goto pop_next; + } + case EXTERNAL_PORT_SUBTAG: { + ExternalThing* thing = external_thing_ptr(term); + /* See limitation #2 */ + #ifdef ARCH_64 + POINTER_HASH(thing->node, HCONST_6); + UINT32_HASH(thing->data.ui[0], HCONST_6); + #else + UINT32_HASH_2(thing->node, thing->data.ui[0], HCONST_6); + #endif + goto pop_next; + } + case FLOAT_SUBTAG: + { + FloatDef ff; + GET_DOUBLE(term, ff); + if (ff.fd == 0.0f) { + /* ensure positive 0.0 */ + ff.fd = erts_get_positive_zero_float(); + } + UINT32_HASH_2(ff.fw[0], ff.fw[1], HCONST_12); + goto pop_next; + } + default: + erts_exit(ERTS_ERROR_EXIT, "Invalid tag in make_hash2(0x%X)\n", term); + } + } + break; + case TAG_PRIMARY_IMMED1: + #if ERTS_SIZEOF_ETERM == 8 + UINT32_HASH_2((Uint32)term, (Uint32)(term >> 32), HCONST); + #else + UINT32_HASH(term, HCONST); + #endif + goto pop_next; + + default: + erts_exit(ERTS_ERROR_EXIT, "Invalid tag in make_hash2(0x%X)\n", term); + + pop_next: + if (ESTACK_ISEMPTY(s)) { + DESTROY_ESTACK(s); + return hash; + } + + term = ESTACK_POP(s); + + switch (term) { + case HASH_MAP_TAIL: { + hash = (Uint32) ESTACK_POP(s); + UINT32_HASH(hash_xor_pairs, HCONST_19); + hash_xor_pairs = (Uint32) ESTACK_POP(s); + goto pop_next; + } + case HASH_MAP_PAIR: + hash_xor_pairs ^= hash; hash = 0; - goto hash2_common; + goto pop_next; + + case HASH_CDR: + CONST_HASH(HCONST_18); /* Hash CDR i cons cell */ + goto pop_next; default: break; } @@ -1479,9 +1968,10 @@ make_hash2(Eterm term) } } +#undef CONST_HASH #undef HASH_MAP_TAIL -#undef HASH_MAP_KEY -#undef HASH_MAP_VAL +#undef HASH_MAP_PAIR +#undef HASH_CDR #undef UINT32_HASH_2 #undef UINT32_HASH @@ -1509,7 +1999,7 @@ tail_recur: (atom_tab(atom_val(term))->slot.bucket.hvalue); break; case SMALL_DEF: -#if defined(ARCH_64) && !HALFWORD_HEAP +#if defined(ARCH_64) { Sint y1 = signed_val(term); Uint y2 = y1 < 0 ? -(Uint)y1 : y1; @@ -1612,12 +2102,15 @@ tail_recur: break; case FLOAT_DEF: { - FloatDef ff; - GET_DOUBLE(term, ff); - hash = hash*FUNNY_NUMBER6 + (ff.fw[0] ^ ff.fw[1]); + FloatDef ff; + GET_DOUBLE(term, ff); + if (ff.fd == 0.0f) { + /* ensure positive 0.0 */ + ff.fd = erts_get_positive_zero_float(); + } + hash = hash*FUNNY_NUMBER6 + (ff.fw[0] ^ ff.fw[1]); } break; - case MAKE_HASH_CDR_PRE_OP: term = (Eterm) WSTACK_POP(stack); if (is_not_list(term)) { @@ -1698,23 +2191,8 @@ tail_recur: break; case MAP_DEF: - { - map_t *mp = (map_t *)map_val(term); - int size = map_get_size(mp); - Eterm *ks = map_get_keys(mp); - Eterm *vs = map_get_values(mp); - - /* Use a prime with size to remedy some of - * the {} and <<>> hash problems */ - hash = hash*FUNNY_NUMBER13 + FUNNY_NUMBER14 + size; - if (size == 0) - break; - - /* push values first */ - WSTACK_PUSH3(stack, (UWord)vs, (UWord) size, MAKE_HASH_TERM_ARRAY_OP); - WSTACK_PUSH3(stack, (UWord)ks, (UWord) size, MAKE_HASH_TERM_ARRAY_OP); - break; - } + hash = hash*FUNNY_NUMBER13 + FUNNY_NUMBER14 + make_hash2(term); + break; case TUPLE_DEF: { Eterm* ptr = tuple_val(term); @@ -1741,7 +2219,7 @@ tail_recur: } default: - erl_exit(1, "Invalid tag in make_broken_hash\n"); + erts_exit(ERTS_ERROR_EXIT, "Invalid tag in make_broken_hash\n"); return 0; } if (WSTACK_ISEMPTY(stack)) break; @@ -1757,101 +2235,161 @@ tail_recur: #undef MAKE_HASH_CDR_POST_OP } -static int do_send_to_logger(Eterm tag, Eterm gleader, char *buf, int len) +static Eterm +do_allocate_logger_message(Eterm gleader, Eterm **hp, ErlOffHeap **ohp, + ErlHeapFragment **bp, Process **p, Uint sz) { - /* error_logger ! - {notify,{info_msg,gleader,{emulator,"~s~n",[<message as list>]}}} | - {notify,{error,gleader,{emulator,"~s~n",[<message as list>]}}} | - {notify,{warning_msg,gleader,{emulator,"~s~n",[<message as list>}]}} */ - Eterm* hp; - Uint sz; Uint gl_sz; - Eterm gl; - Eterm list,plist,format,tuple1,tuple2,tuple3; - ErlOffHeap *ohp; - ErlHeapFragment *bp = NULL; -#if !defined(ERTS_SMP) - Process *p; -#endif - - ASSERT(is_atom(tag)); - - if (len <= 0) { - return -1; - } + gl_sz = IS_CONST(gleader) ? 0 : size_object(gleader); + sz = sz + gl_sz; #ifndef ERTS_SMP #ifdef USE_THREADS - p = NULL; if (erts_get_scheduler_data()) /* Must be scheduler thread */ #endif { - p = erts_whereis_process(NULL, 0, am_error_logger, 0, 0); - if (p) { - erts_aint32_t state = erts_smp_atomic32_read_acqb(&p->state); + *p = erts_whereis_process(NULL, 0, am_error_logger, 0, 0); + if (*p) { + erts_aint32_t state = erts_smp_atomic32_read_acqb(&(*p)->state); if (state & (ERTS_PSFLG_RUNNING|ERTS_PSFLG_RUNNING_SYS)) - p = NULL; + *p = NULL; } } - if (!p) { - /* buf *always* points to a null terminated string */ - erts_fprintf(stderr, "(no error logger present) %T: \"%s\"\n", - tag, buf); - return 0; + if (!*p) { + return NIL; } - /* So we have an error logger, lets build the message */ -#endif - gl_sz = IS_CONST(gleader) ? 0 : size_object(gleader); - sz = len * 2 /* message list */+ 2 /* cons surrounding message list */ - + gl_sz + - 3 /*outer 2-tuple*/ + 4 /* middle 3-tuple */ + 4 /*inner 3-tuple */ + - 8 /* "~s~n" */; -#ifndef ERTS_SMP - if (sz <= HeapWordsLeft(p)) { - ohp = &MSO(p); - hp = HEAP_TOP(p); - HEAP_TOP(p) += sz; + /* So we have an error logger, lets build the message */ + if (sz <= HeapWordsLeft(*p)) { + *ohp = &MSO(*p); + *hp = HEAP_TOP(*p); + HEAP_TOP(*p) += sz; } else { #endif - bp = new_message_buffer(sz); - ohp = &bp->off_heap; - hp = bp->mem; + *bp = new_message_buffer(sz); + *ohp = &(*bp)->off_heap; + *hp = (*bp)->mem; #ifndef ERTS_SMP } #endif - gl = (is_nil(gleader) + + return (is_nil(gleader) ? am_noproc : (IS_CONST(gleader) ? gleader - : copy_struct(gleader,gl_sz,&hp,ohp))); - list = buf_to_intlist(&hp, buf, len, NIL); - plist = CONS(hp,list,NIL); - hp += 2; - format = buf_to_intlist(&hp, "~s~n", 4, NIL); - tuple1 = TUPLE3(hp, am_emulator, format, plist); - hp += 4; - tuple2 = TUPLE3(hp, tag, gl, tuple1); - hp += 4; - tuple3 = TUPLE2(hp, am_notify, tuple2); + : copy_struct(gleader,gl_sz,hp,*ohp))); +} + +static void do_send_logger_message(Eterm *hp, ErlOffHeap *ohp, ErlHeapFragment *bp, + Process *p, Eterm message) +{ #ifdef HARDDEBUG - erts_fprintf(stderr, "%T\n", tuple3); + erts_fprintf(stderr, "%T\n", message); #endif #ifdef ERTS_SMP { Eterm from = erts_get_current_pid(); if (is_not_internal_pid(from)) from = NIL; - erts_queue_error_logger_message(from, tuple3, bp); + erts_queue_error_logger_message(from, message, bp); } #else - erts_queue_message(p, NULL /* only used for smp build */, bp, tuple3, NIL -#ifdef USE_VM_PROBES - , NIL -#endif - ); + { + ErtsMessage *mp = erts_alloc_message(0, NULL); + mp->data.heap_frag = bp; + erts_queue_message(p, NULL /* only used for smp build */, mp, message, NIL); + } #endif +} + +/* error_logger ! + {notify,{info_msg,gleader,{emulator,format,[args]}}} | + {notify,{error,gleader,{emulator,format,[args]}}} | + {notify,{warning_msg,gleader,{emulator,format,[args}]}} */ +static int do_send_to_logger(Eterm tag, Eterm gleader, char *buf, int len) +{ + Uint sz; + Eterm gl; + Eterm list,args,format,tuple1,tuple2,tuple3; + + Eterm *hp = NULL; + ErlOffHeap *ohp = NULL; + ErlHeapFragment *bp = NULL; + Process *p = NULL; + + ASSERT(is_atom(tag)); + + if (len <= 0) { + return -1; + } + + sz = len * 2 /* message list */ + 2 /* cons surrounding message list */ + + 3 /*outer 2-tuple*/ + 4 /* middle 3-tuple */ + 4 /*inner 3-tuple */ + + 8 /* "~s~n" */; + + /* gleader size is accounted and allocated next */ + gl = do_allocate_logger_message(gleader, &hp, &ohp, &bp, &p, sz); + + if(is_nil(gl)) { + /* buf *always* points to a null terminated string */ + erts_fprintf(stderr, "(no error logger present) %T: \"%s\"\n", + tag, buf); + return 0; + } + + list = buf_to_intlist(&hp, buf, len, NIL); + args = CONS(hp,list,NIL); + hp += 2; + format = buf_to_intlist(&hp, "~s~n", 4, NIL); + tuple1 = TUPLE3(hp, am_emulator, format, args); + hp += 4; + tuple2 = TUPLE3(hp, tag, gl, tuple1); + hp += 4; + tuple3 = TUPLE2(hp, am_notify, tuple2); + + do_send_logger_message(hp, ohp, bp, p, tuple3); + return 0; +} + +static int do_send_term_to_logger(Eterm tag, Eterm gleader, + char *buf, int len, Eterm args) +{ + Uint sz; + Eterm gl; + Uint args_sz; + Eterm format,tuple1,tuple2,tuple3; + + Eterm *hp = NULL; + ErlOffHeap *ohp = NULL; + ErlHeapFragment *bp = NULL; + Process *p = NULL; + + ASSERT(is_atom(tag)); + + args_sz = size_object(args); + sz = len * 2 /* format */ + args_sz + + 3 /*outer 2-tuple*/ + 4 /* middle 3-tuple */ + 4 /*inner 3-tuple */; + + /* gleader size is accounted and allocated next */ + gl = do_allocate_logger_message(gleader, &hp, &ohp, &bp, &p, sz); + + if(is_nil(gl)) { + /* buf *always* points to a null terminated string */ + erts_fprintf(stderr, "(no error logger present) %T: \"%s\" %T\n", + tag, buf, args); + return 0; + } + + format = buf_to_intlist(&hp, buf, len, NIL); + args = copy_struct(args, args_sz, &hp, ohp); + tuple1 = TUPLE3(hp, am_emulator, format, args); + hp += 4; + tuple2 = TUPLE3(hp, tag, gl, tuple1); + hp += 4; + tuple3 = TUPLE2(hp, am_notify, tuple2); + + do_send_logger_message(hp, ohp, bp, p, tuple3); return 0; } @@ -1879,6 +2417,12 @@ send_error_to_logger(Eterm gleader, char *buf, int len) return do_send_to_logger(am_error, gleader, buf, len); } +static ERTS_INLINE int +send_error_term_to_logger(Eterm gleader, char *buf, int len, Eterm args) +{ + return do_send_term_to_logger(am_error, gleader, buf, len, args); +} + #define LOGGER_DSBUF_INC_SZ 256 static erts_dsprintf_buf_t * @@ -1954,6 +2498,15 @@ erts_send_error_to_logger(Eterm gleader, erts_dsprintf_buf_t *dsbufp) } int +erts_send_error_term_to_logger(Eterm gleader, erts_dsprintf_buf_t *dsbufp, Eterm args) +{ + int res; + res = send_error_term_to_logger(gleader, dsbufp->str, dsbufp->str_len, args); + destroy_logger_dsbuf(dsbufp); + return res; +} + +int erts_send_info_to_logger_str(Eterm gleader, char *str) { return send_info_to_logger(gleader, str, sys_strlen(str)); @@ -2059,11 +2612,7 @@ erts_destroy_tmp_dsbuf(erts_dsprintf_buf_t *dsbufp) * Test for equality of two terms. * Returns 0 if not equal, or a non-zero value otherwise. */ -#if HALFWORD_HEAP -int eq_rel(Eterm a, Eterm* a_base, Eterm b, Eterm* b_base) -#else int eq(Eterm a, Eterm b) -#endif { DECLARE_WSTACK(stack); Sint sz; @@ -2071,18 +2620,18 @@ int eq(Eterm a, Eterm b) Eterm* bb; tailrecur: - if (is_same(a, a_base, b, b_base)) goto pop_next; + if (is_same(a, b)) goto pop_next; tailrecur_ne: switch (primary_tag(a)) { case TAG_PRIMARY_LIST: if (is_list(b)) { - Eterm* aval = list_val_rel(a, a_base); - Eterm* bval = list_val_rel(b, b_base); + Eterm* aval = list_val(a); + Eterm* bval = list_val(b); while (1) { Eterm atmp = CAR(aval); Eterm btmp = CAR(bval); - if (!is_same(atmp,a_base,btmp,b_base)) { + if (!is_same(atmp,btmp)) { WSTACK_PUSH2(stack,(UWord) CDR(bval),(UWord) CDR(aval)); a = atmp; b = btmp; @@ -2090,7 +2639,7 @@ tailrecur_ne: } atmp = CDR(aval); btmp = CDR(bval); - if (is_same(atmp,a_base,btmp,b_base)) { + if (is_same(atmp,btmp)) { goto pop_next; } if (is_not_list(atmp) || is_not_list(btmp)) { @@ -2098,43 +2647,27 @@ tailrecur_ne: b = btmp; goto tailrecur_ne; } - aval = list_val_rel(atmp, a_base); - bval = list_val_rel(btmp, b_base); + aval = list_val(atmp); + bval = list_val(btmp); } } break; /* not equal */ case TAG_PRIMARY_BOXED: { - Eterm hdr = *boxed_val_rel(a,a_base); + Eterm hdr = *boxed_val(a); switch (hdr & _TAG_HEADER_MASK) { case ARITYVAL_SUBTAG: { - aa = tuple_val_rel(a, a_base); - if (!is_boxed(b) || *boxed_val_rel(b,b_base) != *aa) + aa = tuple_val(a); + if (!is_boxed(b) || *boxed_val(b) != *aa) goto not_equal; - bb = tuple_val_rel(b,b_base); + bb = tuple_val(b); if ((sz = arityval(*aa)) == 0) goto pop_next; ++aa; ++bb; goto term_array; } - case MAP_SUBTAG: - { - aa = map_val_rel(a, a_base); - if (!is_boxed(b) || *boxed_val_rel(b,b_base) != *aa) - goto not_equal; - bb = map_val_rel(b,b_base); - sz = map_get_size((map_t*)aa); - - if (sz != map_get_size((map_t*)bb)) goto not_equal; - if (sz == 0) goto pop_next; - - aa += 2; - bb += 2; - sz += 1; /* increment for tuple-keys */ - goto term_array; - } case REFC_BINARY_SUBTAG: case HEAP_BINARY_SUBTAG: case SUB_BINARY_SUBTAG: @@ -2148,16 +2681,16 @@ tailrecur_ne: Uint a_bitoffs; Uint b_bitoffs; - if (!is_binary_rel(b,b_base)) { + if (!is_binary(b)) { goto not_equal; } - a_size = binary_size_rel(a,a_base); - b_size = binary_size_rel(b,b_base); + a_size = binary_size(a); + b_size = binary_size(b); if (a_size != b_size) { goto not_equal; } - ERTS_GET_BINARY_BYTES_REL(a, a_ptr, a_bitoffs, a_bitsize, a_base); - ERTS_GET_BINARY_BYTES_REL(b, b_ptr, b_bitoffs, b_bitsize, b_base); + ERTS_GET_BINARY_BYTES(a, a_ptr, a_bitoffs, a_bitsize); + ERTS_GET_BINARY_BYTES(b, b_ptr, b_bitoffs, b_bitsize); if ((a_bitsize | b_bitsize | a_bitoffs | b_bitoffs) == 0) { if (sys_memcmp(a_ptr, b_ptr, a_size) == 0) goto pop_next; } else if (a_bitsize == b_bitsize) { @@ -2168,9 +2701,9 @@ tailrecur_ne: } case EXPORT_SUBTAG: { - if (is_export_rel(b,b_base)) { - Export* a_exp = *((Export **) (export_val_rel(a,a_base) + 1)); - Export* b_exp = *((Export **) (export_val_rel(b,b_base) + 1)); + if (is_export(b)) { + Export* a_exp = *((Export **) (export_val(a) + 1)); + Export* b_exp = *((Export **) (export_val(b) + 1)); if (a_exp == b_exp) goto pop_next; } break; /* not equal */ @@ -2180,10 +2713,10 @@ tailrecur_ne: ErlFunThing* f1; ErlFunThing* f2; - if (!is_fun_rel(b,b_base)) + if (!is_fun(b)) goto not_equal; - f1 = (ErlFunThing *) fun_val_rel(a,a_base); - f2 = (ErlFunThing *) fun_val_rel(b,b_base); + f1 = (ErlFunThing *) fun_val(a); + f2 = (ErlFunThing *) fun_val(b); if (f1->fe->module != f2->fe->module || f1->fe->old_index != f2->fe->old_index || f1->fe->old_uniq != f2->fe->old_uniq || @@ -2201,15 +2734,15 @@ tailrecur_ne: ExternalThing *ap; ExternalThing *bp; - if(!is_external_rel(b,b_base)) + if(!is_external(b)) goto not_equal; - ap = external_thing_ptr_rel(a,a_base); - bp = external_thing_ptr_rel(b,b_base); + ap = external_thing_ptr(a); + bp = external_thing_ptr(b); if(ap->header == bp->header && ap->node == bp->node) { - ASSERT(1 == external_data_words_rel(a,a_base)); - ASSERT(1 == external_data_words_rel(b,b_base)); + ASSERT(1 == external_data_words(a)); + ASSERT(1 == external_data_words(b)); if (ap->data.ui[0] == bp->data.ui[0]) goto pop_next; } @@ -2230,11 +2763,11 @@ tailrecur_ne: ExternalThing* athing; ExternalThing* bthing; - if(!is_external_ref_rel(b,b_base)) + if(!is_external_ref(b)) goto not_equal; - athing = external_thing_ptr_rel(a,a_base); - bthing = external_thing_ptr_rel(b,b_base); + athing = external_thing_ptr(a); + bthing = external_thing_ptr(b); if(athing->node != bthing->node) goto not_equal; @@ -2246,12 +2779,12 @@ tailrecur_ne: goto ref_common; case REF_SUBTAG: - if (!is_internal_ref_rel(b,b_base)) + if (!is_internal_ref(b)) goto not_equal; { - RefThing* athing = ref_thing_ptr_rel(a,a_base); - RefThing* bthing = ref_thing_ptr_rel(b,b_base); + RefThing* athing = ref_thing_ptr(a); + RefThing* bthing = ref_thing_ptr(b); alen = internal_thing_ref_no_of_numbers(athing); blen = internal_thing_ref_no_of_numbers(bthing); anum = internal_thing_ref_numbers(athing); @@ -2301,10 +2834,10 @@ tailrecur_ne: { int i; - if (!is_big_rel(b,b_base)) + if (!is_big(b)) goto not_equal; - aa = big_val_rel(a,a_base); - bb = big_val_rel(b,b_base); + aa = big_val(a); + bb = big_val(b); if (*aa != *bb) goto not_equal; i = BIG_ARITY(aa); @@ -2319,13 +2852,53 @@ tailrecur_ne: FloatDef af; FloatDef bf; - if (is_float_rel(b,b_base)) { - GET_DOUBLE_REL(a, af, a_base); - GET_DOUBLE_REL(b, bf, b_base); + if (is_float(b)) { + GET_DOUBLE(a, af); + GET_DOUBLE(b, bf); if (af.fd == bf.fd) goto pop_next; } break; /* not equal */ } + case MAP_SUBTAG: + if (is_flatmap(a)) { + aa = flatmap_val(a); + if (!is_boxed(b) || *boxed_val(b) != *aa) + goto not_equal; + bb = flatmap_val(b); + sz = flatmap_get_size((flatmap_t*)aa); + + if (sz != flatmap_get_size((flatmap_t*)bb)) goto not_equal; + if (sz == 0) goto pop_next; + + aa += 2; + bb += 2; + sz += 1; /* increment for tuple-keys */ + goto term_array; + + } else { + if (!is_boxed(b) || *boxed_val(b) != hdr) + goto not_equal; + + aa = hashmap_val(a) + 1; + bb = hashmap_val(b) + 1; + switch (hdr & _HEADER_MAP_SUBTAG_MASK) { + case HAMT_SUBTAG_HEAD_ARRAY: + aa++; bb++; + sz = 16; + break; + case HAMT_SUBTAG_HEAD_BITMAP: + aa++; bb++; + case HAMT_SUBTAG_NODE_BITMAP: + sz = hashmap_bitcount(MAP_HEADER_VAL(hdr)); + ASSERT(sz > 0 && sz < 17); + break; + default: + erts_exit(ERTS_ERROR_EXIT, "Unknown hashmap subsubtag\n"); + } + goto term_array; + } + default: + ASSERT(!"Unknown boxed subtab in EQ"); } break; } @@ -2340,7 +2913,7 @@ term_array: /* arrays in 'aa' and 'bb', length in 'sz' */ Eterm* bp = bb; Sint i = sz; for (;;) { - if (!is_same(*ap,a_base,*bp,b_base)) break; + if (!is_same(*ap,*bp)) break; if (--i == 0) goto pop_next; ++ap; ++bp; @@ -2418,7 +2991,7 @@ static int cmpbytes(byte *s1, int l1, byte *s2, int l2) #define float_comp(x,y) (((x)<(y)) ? -1 : (((x)==(y)) ? 0 : 1)) -static int cmp_atoms(Eterm a, Eterm b) +int erts_cmp_atoms(Eterm a, Eterm b) { Atom *aa = atom_tab(atom_val(a)); Atom *bb = atom_tab(atom_val(b)); @@ -2429,27 +3002,50 @@ static int cmp_atoms(Eterm a, Eterm b) bb->name+3, bb->len-3); } -#if !HALFWORD_HEAP /* cmp(Eterm a, Eterm b) * For compatibility with HiPE - arith-based compare. */ Sint cmp(Eterm a, Eterm b) { - return erts_cmp(a, b, 0); + return erts_cmp(a, b, 0, 0); } -#endif + +Sint erts_cmp_compound(Eterm a, Eterm b, int exact, int eq_only); + +Sint erts_cmp(Eterm a, Eterm b, int exact, int eq_only) +{ + if (is_atom(a) && is_atom(b)) { + return erts_cmp_atoms(a, b); + } else if (is_both_small(a, b)) { + return (signed_val(a) - signed_val(b)); + } else if (is_float(a) && is_float(b)) { + FloatDef af, bf; + GET_DOUBLE(a, af); + GET_DOUBLE(b, bf); + return float_comp(af.fd, bf.fd); + } + return erts_cmp_compound(a,b,exact,eq_only); +} + /* erts_cmp(Eterm a, Eterm b, int exact) * exact = 1 -> term-based compare * exact = 0 -> arith-based compare */ -#if HALFWORD_HEAP -Sint erts_cmp_rel_opt(Eterm a, Eterm* a_base, Eterm b, Eterm* b_base, int exact) -#else -Sint erts_cmp(Eterm a, Eterm b, int exact) -#endif +Sint erts_cmp_compound(Eterm a, Eterm b, int exact, int eq_only) { - DECLARE_WSTACK(stack); +#define PSTACK_TYPE struct erts_cmp_hashmap_state + struct erts_cmp_hashmap_state { + Sint wstack_rollback; + int was_exact; + Eterm *ap; + Eterm *bp; + Eterm min_key; + Sint cmp_res; /* result so far -1,0,+1 */ + }; + PSTACK_DECLARE(hmap_stack, 1); + WSTACK_DECLARE(stack); + WSTACK_DECLARE(b_stack); /* only used by hashmaps */ Eterm* aa; Eterm* bb; int i; @@ -2465,6 +3061,26 @@ Sint erts_cmp(Eterm a, Eterm b, int exact) Uint32 *anum; Uint32 *bnum; +/* The WSTACK contains naked Eterms and Operations marked with header-tags */ +#define OP_BITS 4 +#define OP_MASK 0xF +#define TERM_ARRAY_OP 0 +#define SWITCH_EXACT_OFF_OP 1 +#define HASHMAP_PHASE1_ARE_KEYS_EQUAL 2 +#define HASHMAP_PHASE1_IS_MIN_KEY 3 +#define HASHMAP_PHASE1_CMP_VALUES 4 +#define HASHMAP_PHASE2_ARE_KEYS_EQUAL 5 +#define HASHMAP_PHASE2_IS_MIN_KEY_A 6 +#define HASHMAP_PHASE2_IS_MIN_KEY_B 7 + + +#define OP_WORD(OP) (((OP) << _TAG_PRIMARY_SIZE) | TAG_PRIMARY_HEADER) +#define TERM_ARRAY_OP_WORD(SZ) OP_WORD(((SZ) << OP_BITS) | TERM_ARRAY_OP) + +#define GET_OP(WORD) (ASSERT(is_header(WORD)), ((WORD) >> _TAG_PRIMARY_SIZE) & OP_MASK) +#define GET_OP_ARG(WORD) (ASSERT(is_header(WORD)), ((WORD) >> (OP_BITS + _TAG_PRIMARY_SIZE))) + + #define RETURN_NEQ(cmp) { j=(cmp); ASSERT(j != 0); goto not_equal; } #define ON_CMP_GOTO(cmp) if ((j=(cmp)) == 0) goto pop_next; else goto not_equal @@ -2473,15 +3089,17 @@ Sint erts_cmp(Eterm a, Eterm b, int exact) do { \ if((AN) != (BN)) { \ if((AN)->sysname != (BN)->sysname) \ - RETURN_NEQ(cmp_atoms((AN)->sysname, (BN)->sysname)); \ + RETURN_NEQ(erts_cmp_atoms((AN)->sysname, (BN)->sysname)); \ ASSERT((AN)->creation != (BN)->creation); \ RETURN_NEQ(((AN)->creation < (BN)->creation) ? -1 : 1); \ } \ } while (0) +bodyrecur: + j = 0; tailrecur: - if (is_same(a,a_base,b,b_base)) { /* Equal values or pointers. */ + if (is_same(a,b)) { /* Equal values or pointers. */ goto pop_next; } tailrecur_ne: @@ -2489,7 +3107,7 @@ tailrecur_ne: /* deal with majority (?) cases by brute-force */ if (is_atom(a)) { if (is_atom(b)) { - ON_CMP_GOTO(cmp_atoms(a, b)); + ON_CMP_GOTO(erts_cmp_atoms(a, b)); } } else if (is_both_small(a, b)) { ON_CMP_GOTO(signed_val(a) - signed_val(b)); @@ -2507,9 +3125,9 @@ tailrecur_ne: if (is_internal_port(b)) { bnode = erts_this_node; bdata = internal_port_data(b); - } else if (is_external_port_rel(b,b_base)) { - bnode = external_port_node_rel(b,b_base); - bdata = external_port_data_rel(b,b_base); + } else if (is_external_port(b)) { + bnode = external_port_node(b); + bdata = external_port_data(b); } else { a_tag = PORT_DEF; goto mixed_types; @@ -2525,9 +3143,9 @@ tailrecur_ne: if (is_internal_pid(b)) { bnode = erts_this_node; bdata = internal_pid_data(b); - } else if (is_external_pid_rel(b,b_base)) { - bnode = external_pid_node_rel(b,b_base); - bdata = external_pid_data_rel(b,b_base); + } else if (is_external_pid(b)) { + bnode = external_pid_node(b); + bdata = external_pid_data(b); } else { a_tag = PID_DEF; goto mixed_types; @@ -2560,12 +3178,12 @@ tailrecur_ne: a_tag = LIST_DEF; goto mixed_types; } - aa = list_val_rel(a,a_base); - bb = list_val_rel(b,b_base); + aa = list_val(a); + bb = list_val(b); while (1) { Eterm atmp = CAR(aa); Eterm btmp = CAR(bb); - if (!is_same(atmp,a_base,btmp,b_base)) { + if (!is_same(atmp,btmp)) { WSTACK_PUSH2(stack,(UWord) CDR(bb),(UWord) CDR(aa)); a = atmp; b = btmp; @@ -2573,7 +3191,7 @@ tailrecur_ne: } atmp = CDR(aa); btmp = CDR(bb); - if (is_same(atmp,a_base,btmp,b_base)) { + if (is_same(atmp,btmp)) { goto pop_next; } if (is_not_list(atmp) || is_not_list(btmp)) { @@ -2581,20 +3199,20 @@ tailrecur_ne: b = btmp; goto tailrecur_ne; } - aa = list_val_rel(atmp,a_base); - bb = list_val_rel(btmp,b_base); + aa = list_val(atmp); + bb = list_val(btmp); } case TAG_PRIMARY_BOXED: { - Eterm ahdr = *boxed_val_rel(a,a_base); + Eterm ahdr = *boxed_val(a); switch ((ahdr & _TAG_HEADER_MASK) >> _TAG_PRIMARY_SIZE) { case (_TAG_HEADER_ARITYVAL >> _TAG_PRIMARY_SIZE): - if (!is_tuple_rel(b,b_base)) { + if (!is_tuple(b)) { a_tag = TUPLE_DEF; goto mixed_types; } - aa = tuple_val_rel(a,a_base); - bb = tuple_val_rel(b,b_base); + aa = tuple_val(a); + bb = tuple_val(b); /* compare the arities */ i = arityval(ahdr); /* get the arity*/ if (i != arityval(*bb)) { @@ -2606,68 +3224,139 @@ tailrecur_ne: ++aa; ++bb; goto term_array; - case (_TAG_HEADER_MAP >> _TAG_PRIMARY_SIZE) : - if (!is_map_rel(b,b_base)) { - a_tag = MAP_DEF; - goto mixed_types; - } - aa = (Eterm *)map_val_rel(a,a_base); - bb = (Eterm *)map_val_rel(b,b_base); - - i = map_get_size((map_t*)aa); - if (i != map_get_size((map_t*)bb)) { - RETURN_NEQ((int)(i - map_get_size((map_t*)bb))); - } - if (i == 0) { - goto pop_next; + case (_TAG_HEADER_MAP >> _TAG_PRIMARY_SIZE) : + { + struct erts_cmp_hashmap_state* sp; + if (is_flatmap_header(ahdr)) { + if (!is_flatmap(b)) { + if (is_hashmap(b)) { + aa = (Eterm *)flatmap_val(a); + i = flatmap_get_size((flatmap_t*)aa) - hashmap_size(b); + ASSERT(i != 0); + RETURN_NEQ(i); + } + a_tag = MAP_DEF; + goto mixed_types; + } + aa = (Eterm *)flatmap_val(a); + bb = (Eterm *)flatmap_val(b); + + i = flatmap_get_size((flatmap_t*)aa); + if (i != flatmap_get_size((flatmap_t*)bb)) { + RETURN_NEQ((int)(i - flatmap_get_size((flatmap_t*)bb))); + } + if (i == 0) { + goto pop_next; + } + aa += 2; + bb += 2; + if (exact) { + i += 1; /* increment for tuple-keys */ + goto term_array; + } + else { + /* Value array */ + WSTACK_PUSH3(stack,(UWord)(bb+1),(UWord)(aa+1),TERM_ARRAY_OP_WORD(i)); + /* Switch back from 'exact' key compare */ + WSTACK_PUSH(stack,OP_WORD(SWITCH_EXACT_OFF_OP)); + /* Now do 'exact' compare of key tuples */ + a = *aa; + b = *bb; + exact = 1; + goto bodyrecur; + } + } + if (!is_hashmap(b)) { + if (is_flatmap(b)) { + bb = (Eterm *)flatmap_val(b); + i = hashmap_size(a) - flatmap_get_size((flatmap_t*)bb); + ASSERT(i != 0); + RETURN_NEQ(i); + } + a_tag = MAP_DEF; + goto mixed_types; + } + i = hashmap_size(a) - hashmap_size(b); + if (i) { + RETURN_NEQ(i); + } + if (hashmap_size(a) == 0) { + goto pop_next; + } + + /* Hashmap compare strategy: + Phase 1. While keys are identical + Do synchronous stepping through leafs of both trees in hash + order. Maintain value compare result of minimal key. + + Phase 2. If key diff was found in phase 1 + Ignore values from now on. + Continue iterate trees by always advancing the one + lagging behind hash-wise. Identical keys are skipped. + A minimal key can only be candidate as tie-breaker if we + have passed that hash value in the other tree (which means + the key did not exist in the other tree). + */ + + sp = PSTACK_PUSH(hmap_stack); + hashmap_iterator_init(&stack, a, 0); + hashmap_iterator_init(&b_stack, b, 0); + sp->ap = hashmap_iterator_next(&stack); + sp->bp = hashmap_iterator_next(&b_stack); + sp->cmp_res = 0; + ASSERT(sp->ap && sp->bp); + + a = CAR(sp->ap); + b = CAR(sp->bp); + sp->was_exact = exact; + exact = 1; + WSTACK_PUSH(stack, OP_WORD(HASHMAP_PHASE1_ARE_KEYS_EQUAL)); + sp->wstack_rollback = WSTACK_COUNT(stack); + goto bodyrecur; } - aa += 2; - bb += 2; - i += 1; /* increment for tuple-keys */ - goto term_array; case (_TAG_HEADER_FLOAT >> _TAG_PRIMARY_SIZE): - if (!is_float_rel(b,b_base)) { + if (!is_float(b)) { a_tag = FLOAT_DEF; goto mixed_types; } else { FloatDef af; FloatDef bf; - GET_DOUBLE_REL(a, af, a_base); - GET_DOUBLE_REL(b, bf, b_base); + GET_DOUBLE(a, af); + GET_DOUBLE(b, bf); ON_CMP_GOTO(float_comp(af.fd, bf.fd)); } case (_TAG_HEADER_POS_BIG >> _TAG_PRIMARY_SIZE): case (_TAG_HEADER_NEG_BIG >> _TAG_PRIMARY_SIZE): - if (!is_big_rel(b,b_base)) { + if (!is_big(b)) { a_tag = BIG_DEF; goto mixed_types; } - ON_CMP_GOTO(big_comp(rterm2wterm(a,a_base), rterm2wterm(b,b_base))); + ON_CMP_GOTO(big_comp(a, b)); case (_TAG_HEADER_EXPORT >> _TAG_PRIMARY_SIZE): - if (!is_export_rel(b,b_base)) { + if (!is_export(b)) { a_tag = EXPORT_DEF; goto mixed_types; } else { - Export* a_exp = *((Export **) (export_val_rel(a,a_base) + 1)); - Export* b_exp = *((Export **) (export_val_rel(b,b_base) + 1)); + Export* a_exp = *((Export **) (export_val(a) + 1)); + Export* b_exp = *((Export **) (export_val(b) + 1)); - if ((j = cmp_atoms(a_exp->code[0], b_exp->code[0])) != 0) { + if ((j = erts_cmp_atoms(a_exp->code[0], b_exp->code[0])) != 0) { RETURN_NEQ(j); } - if ((j = cmp_atoms(a_exp->code[1], b_exp->code[1])) != 0) { + if ((j = erts_cmp_atoms(a_exp->code[1], b_exp->code[1])) != 0) { RETURN_NEQ(j); } ON_CMP_GOTO((Sint) a_exp->code[2] - (Sint) b_exp->code[2]); } break; case (_TAG_HEADER_FUN >> _TAG_PRIMARY_SIZE): - if (!is_fun_rel(b,b_base)) { + if (!is_fun(b)) { a_tag = FUN_DEF; goto mixed_types; } else { - ErlFunThing* f1 = (ErlFunThing *) fun_val_rel(a,a_base); - ErlFunThing* f2 = (ErlFunThing *) fun_val_rel(b,b_base); + ErlFunThing* f1 = (ErlFunThing *) fun_val(a); + ErlFunThing* f2 = (ErlFunThing *) fun_val(b); Sint diff; diff = cmpbytes(atom_tab(atom_val(f1->fe->module))->name, @@ -2699,29 +3388,29 @@ tailrecur_ne: if (is_internal_pid(b)) { bnode = erts_this_node; bdata = internal_pid_data(b); - } else if (is_external_pid_rel(b,b_base)) { - bnode = external_pid_node_rel(b,b_base); - bdata = external_pid_data_rel(b,b_base); + } else if (is_external_pid(b)) { + bnode = external_pid_node(b); + bdata = external_pid_data(b); } else { a_tag = EXTERNAL_PID_DEF; goto mixed_types; } - anode = external_pid_node_rel(a,a_base); - adata = external_pid_data_rel(a,a_base); + anode = external_pid_node(a); + adata = external_pid_data(a); goto pid_common; case (_TAG_HEADER_EXTERNAL_PORT >> _TAG_PRIMARY_SIZE): if (is_internal_port(b)) { bnode = erts_this_node; bdata = internal_port_data(b); - } else if (is_external_port_rel(b,b_base)) { - bnode = external_port_node_rel(b,b_base); - bdata = external_port_data_rel(b,b_base); + } else if (is_external_port(b)) { + bnode = external_port_node(b); + bdata = external_port_data(b); } else { a_tag = EXTERNAL_PORT_DEF; goto mixed_types; } - anode = external_port_node_rel(a,a_base); - adata = external_port_data_rel(a,a_base); + anode = external_port_node(a); + adata = external_port_data(a); goto port_common; case (_TAG_HEADER_REF >> _TAG_PRIMARY_SIZE): /* @@ -2729,14 +3418,13 @@ tailrecur_ne: * (32-bit words), *not* ref data words. */ - - if (is_internal_ref_rel(b,b_base)) { - RefThing* bthing = ref_thing_ptr_rel(b,b_base); + if (is_internal_ref(b)) { + RefThing* bthing = ref_thing_ptr(b); bnode = erts_this_node; bnum = internal_thing_ref_numbers(bthing); blen = internal_thing_ref_no_of_numbers(bthing); - } else if(is_external_ref_rel(b,b_base)) { - ExternalThing* bthing = external_thing_ptr_rel(b,b_base); + } else if(is_external_ref(b)) { + ExternalThing* bthing = external_thing_ptr(b); bnode = bthing->node; bnum = external_thing_ref_numbers(bthing); blen = external_thing_ref_no_of_numbers(bthing); @@ -2745,7 +3433,7 @@ tailrecur_ne: goto mixed_types; } { - RefThing* athing = ref_thing_ptr_rel(a,a_base); + RefThing* athing = ref_thing_ptr(a); anode = erts_this_node; anum = internal_thing_ref_numbers(athing); alen = internal_thing_ref_no_of_numbers(athing); @@ -2778,13 +3466,13 @@ tailrecur_ne: RETURN_NEQ((Sint32) (anum[i] - bnum[i])); goto pop_next; case (_TAG_HEADER_EXTERNAL_REF >> _TAG_PRIMARY_SIZE): - if (is_internal_ref_rel(b,b_base)) { - RefThing* bthing = ref_thing_ptr_rel(b,b_base); + if (is_internal_ref(b)) { + RefThing* bthing = ref_thing_ptr(b); bnode = erts_this_node; bnum = internal_thing_ref_numbers(bthing); blen = internal_thing_ref_no_of_numbers(bthing); - } else if (is_external_ref_rel(b,b_base)) { - ExternalThing* bthing = external_thing_ptr_rel(b,b_base); + } else if (is_external_ref(b)) { + ExternalThing* bthing = external_thing_ptr(b); bnode = bthing->node; bnum = external_thing_ref_numbers(bthing); blen = external_thing_ref_no_of_numbers(bthing); @@ -2793,7 +3481,7 @@ tailrecur_ne: goto mixed_types; } { - ExternalThing* athing = external_thing_ptr_rel(a,a_base); + ExternalThing* athing = external_thing_ptr(a); anode = athing->node; anum = external_thing_ref_numbers(athing); alen = external_thing_ref_no_of_numbers(athing); @@ -2801,13 +3489,13 @@ tailrecur_ne: goto ref_common; default: /* Must be a binary */ - ASSERT(is_binary_rel(a,a_base)); - if (!is_binary_rel(b,b_base)) { + ASSERT(is_binary(a)); + if (!is_binary(b)) { a_tag = BINARY_DEF; goto mixed_types; } else { - Uint a_size = binary_size_rel(a,a_base); - Uint b_size = binary_size_rel(b,b_base); + Uint a_size = binary_size(a); + Uint b_size = binary_size(b); Uint a_bitsize; Uint b_bitsize; Uint a_bitoffs; @@ -2816,8 +3504,8 @@ tailrecur_ne: int cmp; byte* a_ptr; byte* b_ptr; - ERTS_GET_BINARY_BYTES_REL(a, a_ptr, a_bitoffs, a_bitsize, a_base); - ERTS_GET_BINARY_BYTES_REL(b, b_ptr, b_bitoffs, b_bitsize, b_base); + ERTS_GET_BINARY_BYTES(a, a_ptr, a_bitoffs, a_bitsize); + ERTS_GET_BINARY_BYTES(b, b_ptr, b_bitoffs, b_bitsize); if ((a_bitsize | b_bitsize | a_bitoffs | b_bitoffs) == 0) { min_size = (a_size < b_size) ? a_size : b_size; if ((cmp = sys_memcmp(a_ptr, b_ptr, min_size)) != 0) { @@ -2848,13 +3536,8 @@ tailrecur_ne: { FloatDef f1, f2; Eterm big; -#if HALFWORD_HEAP - Wterm aw = is_immed(a) ? a : rterm2wterm(a,a_base); - Wterm bw = is_immed(b) ? b : rterm2wterm(b,b_base); -#else Eterm aw = a; Eterm bw = b; -#endif #define MAX_LOSSLESS_FLOAT ((double)((1LL << 53) - 2)) #define MIN_LOSSLESS_FLOAT ((double)(((1LL << 53) - 2)*-1)) #define BIG_ARITY_FLOAT_MAX (1024 / D_EXP) /* arity of max float as a bignum */ @@ -2926,7 +3609,7 @@ tailrecur_ne: } } else { big = double_to_big(f2.fd, big_buf, sizeof(big_buf)/sizeof(Eterm)); - j = big_comp(aw, rterm2wterm(big,big_buf)); + j = big_comp(aw, big); } if (_NUMBER_CODE(a_tag, b_tag) == FLOAT_BIG) { j = -j; @@ -2974,9 +3657,9 @@ term_array: /* arrays in 'aa' and 'bb', length in 'i' */ while (--i) { a = *aa++; b = *bb++; - if (!is_same(a,a_base, b,b_base)) { + if (!is_same(a, b)) { if (is_atom(a) && is_atom(b)) { - if ((j = cmp_atoms(a, b)) != 0) { + if ((j = erts_cmp_atoms(a, b)) != 0) { goto not_equal; } } else if (is_both_small(a, b)) { @@ -2984,8 +3667,7 @@ term_array: /* arrays in 'aa' and 'bb', length in 'i' */ goto not_equal; } } else { - /* (ab)Use TAG_PRIMARY_HEADER to recognize a term_array */ - WSTACK_PUSH3(stack, i, (UWord)bb, (UWord)aa | TAG_PRIMARY_HEADER); + WSTACK_PUSH3(stack, (UWord)bb, (UWord)aa, TERM_ARRAY_OP_WORD(i)); goto tailrecur_ne; } } @@ -2997,22 +3679,179 @@ term_array: /* arrays in 'aa' and 'bb', length in 'i' */ pop_next: if (!WSTACK_ISEMPTY(stack)) { UWord something = WSTACK_POP(stack); - if (primary_tag((Eterm) something) == TAG_PRIMARY_HEADER) { /* a term_array */ - aa = (Eterm*) something; - bb = (Eterm*) WSTACK_POP(stack); - i = WSTACK_POP(stack); - goto term_array; + struct erts_cmp_hashmap_state* sp; + if (primary_tag((Eterm) something) == TAG_PRIMARY_HEADER) { /* an operation */ + switch (GET_OP(something)) { + case TERM_ARRAY_OP: + i = GET_OP_ARG(something); + aa = (Eterm*)WSTACK_POP(stack); + bb = (Eterm*) WSTACK_POP(stack); + goto term_array; + + case SWITCH_EXACT_OFF_OP: + /* Done with exact compare of map keys, switch back */ + ASSERT(exact); + exact = 0; + goto pop_next; + + case HASHMAP_PHASE1_ARE_KEYS_EQUAL: { + sp = PSTACK_TOP(hmap_stack); + if (j) { + /* Key diff found, enter phase 2 */ + if (hashmap_key_hash_cmp(sp->ap, sp->bp) < 0) { + sp->min_key = CAR(sp->ap); + sp->cmp_res = -1; + sp->ap = hashmap_iterator_next(&stack); + } + else { + sp->min_key = CAR(sp->bp); + sp->cmp_res = 1; + sp->bp = hashmap_iterator_next(&b_stack); + } + exact = 1; /* only exact key compares in phase 2 */ + goto case_HASHMAP_PHASE2_LOOP; + } + + /* No key diff found so far, compare values if min key */ + + if (sp->cmp_res) { + a = CAR(sp->ap); + b = sp->min_key; + exact = 1; + WSTACK_PUSH(stack, OP_WORD(HASHMAP_PHASE1_IS_MIN_KEY)); + sp->wstack_rollback = WSTACK_COUNT(stack); + goto bodyrecur; + } + /* no min key-value found yet */ + a = CDR(sp->ap); + b = CDR(sp->bp); + exact = sp->was_exact; + WSTACK_PUSH(stack, OP_WORD(HASHMAP_PHASE1_CMP_VALUES)); + sp->wstack_rollback = WSTACK_COUNT(stack); + goto bodyrecur; + } + case HASHMAP_PHASE1_IS_MIN_KEY: + sp = PSTACK_TOP(hmap_stack); + if (j < 0) { + a = CDR(sp->ap); + b = CDR(sp->bp); + exact = sp->was_exact; + WSTACK_PUSH(stack, OP_WORD(HASHMAP_PHASE1_CMP_VALUES)); + sp->wstack_rollback = WSTACK_COUNT(stack); + goto bodyrecur; + } + goto case_HASHMAP_PHASE1_LOOP; + + case HASHMAP_PHASE1_CMP_VALUES: + sp = PSTACK_TOP(hmap_stack); + if (j) { + sp->cmp_res = j; + sp->min_key = CAR(sp->ap); + } + case_HASHMAP_PHASE1_LOOP: + sp->ap = hashmap_iterator_next(&stack); + sp->bp = hashmap_iterator_next(&b_stack); + if (!sp->ap) { + /* end of maps with identical keys */ + ASSERT(!sp->bp); + j = sp->cmp_res; + exact = sp->was_exact; + (void) PSTACK_POP(hmap_stack); + ON_CMP_GOTO(j); + } + a = CAR(sp->ap); + b = CAR(sp->bp); + exact = 1; + WSTACK_PUSH(stack, OP_WORD(HASHMAP_PHASE1_ARE_KEYS_EQUAL)); + sp->wstack_rollback = WSTACK_COUNT(stack); + goto bodyrecur; + + case_HASHMAP_PHASE2_LOOP: + if (sp->ap && sp->bp) { + a = CAR(sp->ap); + b = CAR(sp->bp); + ASSERT(exact); + WSTACK_PUSH(stack, OP_WORD(HASHMAP_PHASE2_ARE_KEYS_EQUAL)); + sp->wstack_rollback = WSTACK_COUNT(stack); + goto bodyrecur; + } + goto case_HASHMAP_PHASE2_NEXT_STEP; + + case HASHMAP_PHASE2_ARE_KEYS_EQUAL: + sp = PSTACK_TOP(hmap_stack); + if (j == 0) { + /* keys are equal, skip them */ + sp->ap = hashmap_iterator_next(&stack); + sp->bp = hashmap_iterator_next(&b_stack); + goto case_HASHMAP_PHASE2_LOOP; + } + /* fall through */ + case_HASHMAP_PHASE2_NEXT_STEP: + if (sp->ap || sp->bp) { + if (hashmap_key_hash_cmp(sp->ap, sp->bp) < 0) { + ASSERT(sp->ap); + a = CAR(sp->ap); + b = sp->min_key; + ASSERT(exact); + WSTACK_PUSH(stack, OP_WORD(HASHMAP_PHASE2_IS_MIN_KEY_A)); + } + else { /* hash_cmp > 0 */ + ASSERT(sp->bp); + a = CAR(sp->bp); + b = sp->min_key; + ASSERT(exact); + WSTACK_PUSH(stack, OP_WORD(HASHMAP_PHASE2_IS_MIN_KEY_B)); + } + sp->wstack_rollback = WSTACK_COUNT(stack); + goto bodyrecur; + } + /* End of both maps */ + j = sp->cmp_res; + exact = sp->was_exact; + (void) PSTACK_POP(hmap_stack); + ON_CMP_GOTO(j); + + case HASHMAP_PHASE2_IS_MIN_KEY_A: + sp = PSTACK_TOP(hmap_stack); + if (j < 0) { + sp->min_key = CAR(sp->ap); + sp->cmp_res = -1; + } + sp->ap = hashmap_iterator_next(&stack); + goto case_HASHMAP_PHASE2_LOOP; + + case HASHMAP_PHASE2_IS_MIN_KEY_B: + sp = PSTACK_TOP(hmap_stack); + if (j < 0) { + sp->min_key = CAR(sp->bp); + sp->cmp_res = 1; + } + sp->bp = hashmap_iterator_next(&b_stack); + goto case_HASHMAP_PHASE2_LOOP; + + default: + ASSERT(!"Invalid cmp op"); + } /* switch */ } a = (Eterm) something; b = (Eterm) WSTACK_POP(stack); goto tailrecur; } - DESTROY_WSTACK(stack); + ASSERT(PSTACK_IS_EMPTY(hmap_stack)); + PSTACK_DESTROY(hmap_stack); + WSTACK_DESTROY(stack); + WSTACK_DESTROY(b_stack); return 0; not_equal: - DESTROY_WSTACK(stack); + if (!PSTACK_IS_EMPTY(hmap_stack) && !eq_only) { + WSTACK_ROLLBACK(stack, PSTACK_TOP(hmap_stack)->wstack_rollback); + goto pop_next; + } + PSTACK_DESTROY(hmap_stack); + WSTACK_DESTROY(stack); + WSTACK_DESTROY(b_stack); return j; #undef CMP_NODES @@ -3091,11 +3930,11 @@ void bin_write(int to, void *to_arg, byte* buf, size_t sz) /* Fill buf with the contents of bytelist list return number of chars in list or -1 for error */ -int -intlist_to_buf(Eterm list, char *buf, int len) +Sint +intlist_to_buf(Eterm list, char *buf, Sint len) { Eterm* listptr; - int sz = 0; + Sint sz = 0; if (is_nil(list)) return 0; @@ -3197,106 +4036,303 @@ buf_to_intlist(Eterm** hpp, const char *buf, size_t len, Eterm tail) ** */ -ErlDrvSizeT erts_iolist_to_buf(Eterm obj, char* buf, ErlDrvSizeT alloced_len) +typedef enum { + ERTS_IL2B_BCOPY_OK, + ERTS_IL2B_BCOPY_YIELD, + ERTS_IL2B_BCOPY_OVERFLOW, + ERTS_IL2B_BCOPY_TYPE_ERROR +} ErtsIL2BBCopyRes; + +static ErtsIL2BBCopyRes +iolist_to_buf_bcopy(ErtsIOList2BufState *state, Eterm obj, int *yield_countp); + +static ERTS_INLINE ErlDrvSizeT +iolist_to_buf(const int yield_support, + ErtsIOList2BufState *state, + Eterm obj, + char* buf, + ErlDrvSizeT alloced_len) { - ErlDrvSizeT len = (ErlDrvSizeT) alloced_len; - Eterm* objp; +#undef IOLIST_TO_BUF_BCOPY +#define IOLIST_TO_BUF_BCOPY(CONSP) \ +do { \ + size_t size = binary_size(obj); \ + if (size > 0) { \ + Uint bitsize; \ + byte* bptr; \ + Uint bitoffs; \ + Uint num_bits; \ + if (yield_support) { \ + size_t max_size = ERTS_IOLIST_TO_BUF_BYTES_PER_YIELD_COUNT; \ + if (yield_count > 0) \ + max_size *= yield_count+1; \ + if (size > max_size) { \ + state->objp = CONSP; \ + goto L_bcopy_yield; \ + } \ + if (size >= ERTS_IOLIST_TO_BUF_BYTES_PER_YIELD_COUNT) { \ + int cost = (int) size; \ + cost /= ERTS_IOLIST_TO_BUF_BYTES_PER_YIELD_COUNT; \ + yield_count -= cost; \ + } \ + } \ + if (len < size) \ + goto L_overflow; \ + ERTS_GET_BINARY_BYTES(obj, bptr, bitoffs, bitsize); \ + if (bitsize != 0) \ + goto L_type_error; \ + num_bits = 8*size; \ + copy_binary_to_buffer(buf, 0, bptr, bitoffs, num_bits); \ + buf += size; \ + len -= size; \ + } \ +} while (0) + + ErlDrvSizeT res, len; + Eterm* objp = NULL; + int init_yield_count; + int yield_count; DECLARE_ESTACK(s); - goto L_again; - - while (!ESTACK_ISEMPTY(s)) { - obj = ESTACK_POP(s); - L_again: - if (is_list(obj)) { - L_iter_list: - objp = list_val(obj); - obj = CAR(objp); - if (is_byte(obj)) { - if (len == 0) { - goto L_overflow; - } - *buf++ = unsigned_val(obj); - len--; - } else if (is_binary(obj)) { - byte* bptr; - size_t size = binary_size(obj); - Uint bitsize; - Uint bitoffs; - Uint num_bits; - - if (len < size) { + + len = (ErlDrvSizeT) alloced_len; + + if (!yield_support) { + yield_count = init_yield_count = 0; /* Shut up faulty warning... >:-( */ + goto L_again; + } + else { + + if (state->iolist.reds_left <= 0) + return ERTS_IOLIST_TO_BUF_YIELD; + + ESTACK_CHANGE_ALLOCATOR(s, ERTS_ALC_T_SAVED_ESTACK); + init_yield_count = (ERTS_IOLIST_TO_BUF_YIELD_COUNT_PER_RED + * state->iolist.reds_left); + yield_count = init_yield_count; + + if (!state->iolist.estack.start) + goto L_again; + else { + int chk_stack; + /* Restart; restore state... */ + ESTACK_RESTORE(s, &state->iolist.estack); + + if (!state->bcopy.bptr) + chk_stack = 0; + else { + chk_stack = 1; + switch (iolist_to_buf_bcopy(state, THE_NON_VALUE, &yield_count)) { + case ERTS_IL2B_BCOPY_OK: + break; + case ERTS_IL2B_BCOPY_YIELD: + BUMP_ALL_REDS(state->iolist.c_p); + state->iolist.reds_left = 0; + ESTACK_SAVE(s, &state->iolist.estack); + return ERTS_IOLIST_TO_BUF_YIELD; + case ERTS_IL2B_BCOPY_OVERFLOW: goto L_overflow; - } - ERTS_GET_BINARY_BYTES(obj, bptr, bitoffs, bitsize); - if (bitsize != 0) { + case ERTS_IL2B_BCOPY_TYPE_ERROR: goto L_type_error; } - num_bits = 8*size; - copy_binary_to_buffer(buf, 0, bptr, bitoffs, num_bits); - buf += size; - len -= size; - } else if (is_list(obj)) { - ESTACK_PUSH(s, CDR(objp)); - goto L_iter_list; /* on head */ - } else if (is_not_nil(obj)) { - goto L_type_error; } - obj = CDR(objp); - if (is_list(obj)) { - goto L_iter_list; /* on tail */ - } else if (is_binary(obj)) { - byte* bptr; - size_t size = binary_size(obj); - Uint bitsize; - Uint bitoffs; - Uint num_bits; - if (len < size) { - goto L_overflow; + obj = state->iolist.obj; + buf = state->buf; + len = state->len; + objp = state->objp; + state->objp = NULL; + if (objp) + goto L_tail; + if (!chk_stack) + goto L_again; + /* check stack */ + } + } + + while (!ESTACK_ISEMPTY(s)) { + obj = ESTACK_POP(s); + L_again: + if (is_list(obj)) { + while (1) { /* Tail loop */ + while (1) { /* Head loop */ + if (yield_support && --yield_count <= 0) + goto L_yield; + objp = list_val(obj); + obj = CAR(objp); + if (is_byte(obj)) { + if (len == 0) { + goto L_overflow; + } + *buf++ = unsigned_val(obj); + len--; + } else if (is_binary(obj)) { + IOLIST_TO_BUF_BCOPY(objp); + } else if (is_list(obj)) { + ESTACK_PUSH(s, CDR(objp)); + continue; /* Head loop */ + } else if (is_not_nil(obj)) { + goto L_type_error; + } + break; } - ERTS_GET_BINARY_BYTES(obj, bptr, bitoffs, bitsize); - if (bitsize != 0) { + + L_tail: + + obj = CDR(objp); + + if (is_list(obj)) { + continue; /* Tail loop */ + } else if (is_binary(obj)) { + IOLIST_TO_BUF_BCOPY(NULL); + } else if (is_not_nil(obj)) { goto L_type_error; } - num_bits = 8*size; - copy_binary_to_buffer(buf, 0, bptr, bitoffs, num_bits); - buf += size; - len -= size; - } else if (is_not_nil(obj)) { - goto L_type_error; + break; } } else if (is_binary(obj)) { - byte* bptr; - size_t size = binary_size(obj); - Uint bitsize; - Uint bitoffs; - Uint num_bits; - if (len < size) { - goto L_overflow; - } - ERTS_GET_BINARY_BYTES(obj, bptr, bitoffs, bitsize); - if (bitsize != 0) { - goto L_type_error; - } - num_bits = 8*size; - copy_binary_to_buffer(buf, 0, bptr, bitoffs, num_bits); - buf += size; - len -= size; + IOLIST_TO_BUF_BCOPY(NULL); } else if (is_not_nil(obj)) { goto L_type_error; - } + } else if (yield_support && --yield_count <= 0) + goto L_yield; } + res = len; + + L_return: + DESTROY_ESTACK(s); - return len; + + if (yield_support) { + int reds; + CLEAR_SAVED_ESTACK(&state->iolist.estack); + reds = ((init_yield_count - yield_count - 1) + / ERTS_IOLIST_TO_BUF_YIELD_COUNT_PER_RED) + 1; + BUMP_REDS(state->iolist.c_p, reds); + state->iolist.reds_left -= reds; + if (state->iolist.reds_left < 0) + state->iolist.reds_left = 0; + } + + + return res; L_type_error: - DESTROY_ESTACK(s); - return ERTS_IOLIST_TO_BUF_TYPE_ERROR; + res = ERTS_IOLIST_TO_BUF_TYPE_ERROR; + goto L_return; L_overflow: - DESTROY_ESTACK(s); - return ERTS_IOLIST_TO_BUF_OVERFLOW; + res = ERTS_IOLIST_TO_BUF_OVERFLOW; + goto L_return; + + L_bcopy_yield: + + state->buf = buf; + state->len = len; + + switch (iolist_to_buf_bcopy(state, obj, &yield_count)) { + case ERTS_IL2B_BCOPY_OK: + ERTS_INTERNAL_ERROR("Missing yield"); + case ERTS_IL2B_BCOPY_YIELD: + BUMP_ALL_REDS(state->iolist.c_p); + state->iolist.reds_left = 0; + ESTACK_SAVE(s, &state->iolist.estack); + return ERTS_IOLIST_TO_BUF_YIELD; + case ERTS_IL2B_BCOPY_OVERFLOW: + goto L_overflow; + case ERTS_IL2B_BCOPY_TYPE_ERROR: + goto L_type_error; + } + + L_yield: + + BUMP_ALL_REDS(state->iolist.c_p); + state->iolist.reds_left = 0; + state->iolist.obj = obj; + state->buf = buf; + state->len = len; + ESTACK_SAVE(s, &state->iolist.estack); + return ERTS_IOLIST_TO_BUF_YIELD; + +#undef IOLIST_TO_BUF_BCOPY +} + +static ErtsIL2BBCopyRes +iolist_to_buf_bcopy(ErtsIOList2BufState *state, Eterm obj, int *yield_countp) +{ + ErtsIL2BBCopyRes res; + char *buf = state->buf; + ErlDrvSizeT len = state->len; + byte* bptr; + size_t size; + size_t max_size; + Uint bitoffs; + Uint num_bits; + int yield_count = *yield_countp; + + if (state->bcopy.bptr) { + bptr = state->bcopy.bptr; + size = state->bcopy.size; + bitoffs = state->bcopy.bitoffs; + state->bcopy.bptr = NULL; + } + else { + Uint bitsize; + + ASSERT(is_binary(obj)); + + size = binary_size(obj); + if (size <= 0) + return ERTS_IL2B_BCOPY_OK; + + if (len < size) + return ERTS_IL2B_BCOPY_OVERFLOW; + + ERTS_GET_BINARY_BYTES(obj, bptr, bitoffs, bitsize); + if (bitsize != 0) + return ERTS_IL2B_BCOPY_TYPE_ERROR; + } + + ASSERT(size > 0); + max_size = (size_t) ERTS_IOLIST_TO_BUF_BYTES_PER_YIELD_COUNT; + if (yield_count > 0) + max_size *= (size_t) (yield_count+1); + + if (size <= max_size) { + if (size >= ERTS_IOLIST_TO_BUF_BYTES_PER_YIELD_COUNT) { + int cost = (int) size; + cost /= ERTS_IOLIST_TO_BUF_BYTES_PER_YIELD_COUNT; + yield_count -= cost; + } + res = ERTS_IL2B_BCOPY_OK; + } + else { + ASSERT(0 < max_size && max_size < size); + yield_count = 0; + state->bcopy.bptr = bptr + max_size; + state->bcopy.bitoffs = bitoffs; + state->bcopy.size = size - max_size; + size = max_size; + res = ERTS_IL2B_BCOPY_YIELD; + } + + num_bits = 8*size; + copy_binary_to_buffer(buf, 0, bptr, bitoffs, num_bits); + state->buf += size; + state->len -= size; + *yield_countp = yield_count; + + return res; +} + +ErlDrvSizeT erts_iolist_to_buf_yielding(ErtsIOList2BufState *state) +{ + return iolist_to_buf(1, state, state->iolist.obj, state->buf, state->len); +} + +ErlDrvSizeT erts_iolist_to_buf(Eterm obj, char* buf, ErlDrvSizeT alloced_len) +{ + return iolist_to_buf(0, NULL, obj, buf, alloced_len); } /* @@ -3307,11 +4343,32 @@ ErlDrvSizeT erts_iolist_to_buf(Eterm obj, char* buf, ErlDrvSizeT alloced_len) * Any input term error detected in erts_iolist_to_buf should also * be detected in this function! */ -int erts_iolist_size(Eterm obj, ErlDrvSizeT* sizep) + +static ERTS_INLINE int +iolist_size(const int yield_support, ErtsIOListState *state, Eterm obj, ErlDrvSizeT* sizep) { + int res, init_yield_count, yield_count; Eterm* objp; - Uint size = 0; /* Intentionally Uint due to halfword heap */ + Uint size = (Uint) *sizep; DECLARE_ESTACK(s); + + if (!yield_support) + yield_count = init_yield_count = 0; /* Shut up faulty warning... >:-( */ + else { + if (state->reds_left <= 0) + return ERTS_IOLIST_YIELD; + ESTACK_CHANGE_ALLOCATOR(s, ERTS_ALC_T_SAVED_ESTACK); + init_yield_count = ERTS_IOLIST_SIZE_YIELDS_COUNT_PER_RED; + init_yield_count *= state->reds_left; + yield_count = init_yield_count; + if (state->estack.start) { + /* Restart; restore state... */ + ESTACK_RESTORE(s, &state->estack); + size = (Uint) state->size; + obj = state->obj; + } + } + goto L_again; #define SAFE_ADD(Var, Val) \ @@ -3327,58 +4384,109 @@ int erts_iolist_size(Eterm obj, ErlDrvSizeT* sizep) obj = ESTACK_POP(s); L_again: if (is_list(obj)) { - L_iter_list: - objp = list_val(obj); - /* Head */ - obj = CAR(objp); - if (is_byte(obj)) { - size++; - if (size == 0) { - goto L_overflow_error; + while (1) { /* Tail loop */ + while (1) { /* Head loop */ + if (yield_support && --yield_count <= 0) + goto L_yield; + objp = list_val(obj); + /* Head */ + obj = CAR(objp); + if (is_byte(obj)) { + size++; + if (size == 0) { + goto L_overflow_error; + } + } else if (is_binary(obj) && binary_bitsize(obj) == 0) { + SAFE_ADD(size, binary_size(obj)); + } else if (is_list(obj)) { + ESTACK_PUSH(s, CDR(objp)); + continue; /* Head loop */ + } else if (is_not_nil(obj)) { + goto L_type_error; + } + break; } - } else if (is_binary(obj) && binary_bitsize(obj) == 0) { - SAFE_ADD(size, binary_size(obj)); - } else if (is_list(obj)) { - ESTACK_PUSH(s, CDR(objp)); - goto L_iter_list; /* on head */ - } else if (is_not_nil(obj)) { - goto L_type_error; + /* Tail */ + obj = CDR(objp); + if (is_list(obj)) + continue; /* Tail loop */ + else if (is_binary(obj) && binary_bitsize(obj) == 0) { + SAFE_ADD(size, binary_size(obj)); + } else if (is_not_nil(obj)) { + goto L_type_error; + } + break; } - /* Tail */ - obj = CDR(objp); - if (is_list(obj)) - goto L_iter_list; /* on tail */ - else if (is_binary(obj) && binary_bitsize(obj) == 0) { + } else { + if (yield_support && --yield_count <= 0) + goto L_yield; + if (is_binary(obj) && binary_bitsize(obj) == 0) { /* Tail was binary */ SAFE_ADD(size, binary_size(obj)); } else if (is_not_nil(obj)) { goto L_type_error; } - } else if (is_binary(obj) && binary_bitsize(obj) == 0) { /* Tail was binary */ - SAFE_ADD(size, binary_size(obj)); - } else if (is_not_nil(obj)) { - goto L_type_error; } } #undef SAFE_ADD - DESTROY_ESTACK(s); *sizep = (ErlDrvSizeT) size; - return ERTS_IOLIST_OK; - L_overflow_error: + res = ERTS_IOLIST_OK; + + L_return: + DESTROY_ESTACK(s); - return ERTS_IOLIST_OVERFLOW; + + if (yield_support) { + int yc, reds; + CLEAR_SAVED_ESTACK(&state->estack); + yc = init_yield_count - yield_count; + reds = ((yc - 1) / ERTS_IOLIST_SIZE_YIELDS_COUNT_PER_RED) + 1; + BUMP_REDS(state->c_p, reds); + state->reds_left -= reds; + state->size = (ErlDrvSizeT) size; + state->have_size = 1; + } + + return res; + + L_overflow_error: + res = ERTS_IOLIST_OVERFLOW; + size = 0; + goto L_return; L_type_error: - DESTROY_ESTACK(s); - return ERTS_IOLIST_TYPE; + res = ERTS_IOLIST_TYPE; + size = 0; + goto L_return; + + L_yield: + BUMP_ALL_REDS(state->c_p); + state->reds_left = 0; + state->size = size; + state->obj = obj; + ESTACK_SAVE(s, &state->estack); + return ERTS_IOLIST_YIELD; } -/* return 0 if item is not a non-empty flat list of bytes */ -int +int erts_iolist_size_yielding(ErtsIOListState *state) +{ + ErlDrvSizeT size = state->size; + return iolist_size(1, state, state->obj, &size); +} + +int erts_iolist_size(Eterm obj, ErlDrvSizeT* sizep) +{ + *sizep = 0; + return iolist_size(0, NULL, obj, sizep); +} + +/* return 0 if item is not a non-empty flat list of bytes + otherwise return the nonzero length of the list */ +Sint is_string(Eterm list) { - int len = 0; + Sint len = 0; while(is_list(list)) { Eterm* consp = list_val(list); @@ -3394,145 +4502,6 @@ is_string(Eterm list) return 0; } -#ifdef ERTS_SMP - -/* - * Process and Port timers in smp case - */ - -ERTS_SCHED_PREF_PRE_ALLOC_IMPL(ptimer_pre, ErtsSmpPTimer, 1000) - -#define ERTS_PTMR_FLGS_ALLCD_SIZE \ - 2 -#define ERTS_PTMR_FLGS_ALLCD_MASK \ - ((((Uint32) 1) << ERTS_PTMR_FLGS_ALLCD_SIZE) - 1) - -#define ERTS_PTMR_FLGS_PREALLCD ((Uint32) 1) -#define ERTS_PTMR_FLGS_SLALLCD ((Uint32) 2) -#define ERTS_PTMR_FLGS_LLALLCD ((Uint32) 3) -#define ERTS_PTMR_FLG_CANCELLED (((Uint32) 1) << (ERTS_PTMR_FLGS_ALLCD_SIZE+0)) - -static void -init_ptimers(void) -{ - init_ptimer_pre_alloc(); -} - -static ERTS_INLINE void -free_ptimer(ErtsSmpPTimer *ptimer) -{ - switch (ptimer->timer.flags & ERTS_PTMR_FLGS_ALLCD_MASK) { - case ERTS_PTMR_FLGS_PREALLCD: - (void) ptimer_pre_free(ptimer); - break; - case ERTS_PTMR_FLGS_SLALLCD: - erts_free(ERTS_ALC_T_SL_PTIMER, (void *) ptimer); - break; - case ERTS_PTMR_FLGS_LLALLCD: - erts_free(ERTS_ALC_T_LL_PTIMER, (void *) ptimer); - break; - default: - erl_exit(ERTS_ABORT_EXIT, - "Internal error: Bad ptimer alloc type\n"); - break; - } -} - -/* Callback for process timeout cancelled */ -static void -ptimer_cancelled(ErtsSmpPTimer *ptimer) -{ - free_ptimer(ptimer); -} - -/* Callback for process timeout */ -static void -ptimer_timeout(ErtsSmpPTimer *ptimer) -{ - if (is_internal_pid(ptimer->timer.id)) { - Process *p; - p = erts_pid2proc_opt(NULL, - 0, - ptimer->timer.id, - ERTS_PROC_LOCK_MAIN|ERTS_PROC_LOCK_STATUS, - ERTS_P2P_FLG_ALLOW_OTHER_X); - if (p) { - if (!ERTS_PROC_IS_EXITING(p) - && !(ptimer->timer.flags & ERTS_PTMR_FLG_CANCELLED)) { - ASSERT(*ptimer->timer.timer_ref == ptimer); - *ptimer->timer.timer_ref = NULL; - (*ptimer->timer.timeout_func)(p); - } - erts_smp_proc_unlock(p, ERTS_PROC_LOCK_MAIN|ERTS_PROC_LOCK_STATUS); - } - } - else { - Port *p; - ASSERT(is_internal_port(ptimer->timer.id)); - p = erts_id2port_sflgs(ptimer->timer.id, - NULL, - 0, - ERTS_PORT_SFLGS_DEAD); - if (p) { - if (!(ptimer->timer.flags & ERTS_PTMR_FLG_CANCELLED)) { - ASSERT(*ptimer->timer.timer_ref == ptimer); - *ptimer->timer.timer_ref = NULL; - (*ptimer->timer.timeout_func)(p); - } - erts_port_release(p); - } - } - free_ptimer(ptimer); -} - -void -erts_create_smp_ptimer(ErtsSmpPTimer **timer_ref, - Eterm id, - ErlTimeoutProc timeout_func, - Uint timeout) -{ - ErtsSmpPTimer *res = ptimer_pre_alloc(); - if (res) - res->timer.flags = ERTS_PTMR_FLGS_PREALLCD; - else { - if (timeout < ERTS_ALC_MIN_LONG_LIVED_TIME) { - res = erts_alloc(ERTS_ALC_T_SL_PTIMER, sizeof(ErtsSmpPTimer)); - res->timer.flags = ERTS_PTMR_FLGS_SLALLCD; - } - else { - res = erts_alloc(ERTS_ALC_T_LL_PTIMER, sizeof(ErtsSmpPTimer)); - res->timer.flags = ERTS_PTMR_FLGS_LLALLCD; - } - } - res->timer.timeout_func = timeout_func; - res->timer.timer_ref = timer_ref; - res->timer.id = id; - res->timer.tm.active = 0; /* MUST be initalized */ - - ASSERT(!*timer_ref); - - *timer_ref = res; - - erts_set_timer(&res->timer.tm, - (ErlTimeoutProc) ptimer_timeout, - (ErlCancelProc) ptimer_cancelled, - (void*) res, - timeout); -} - -void -erts_cancel_smp_ptimer(ErtsSmpPTimer *ptimer) -{ - if (ptimer) { - ASSERT(*ptimer->timer.timer_ref == ptimer); - *ptimer->timer.timer_ref = NULL; - ptimer->timer.flags |= ERTS_PTMR_FLG_CANCELLED; - erts_cancel_timer(&ptimer->timer.tm); - } -} - -#endif - static int trim_threshold; static int top_pad; static int mmap_threshold; @@ -3542,9 +4511,7 @@ Uint tot_bin_allocated; void erts_init_utils(void) { -#ifdef ERTS_SMP - init_ptimers(); -#endif + } void erts_init_utils_mem(void) @@ -3680,6 +4647,9 @@ erts_save_emu_args(int argc, char **argv) size += sz+1; } ptr = (char *) malloc(size); + if (!ptr) { + ERTS_INTERNAL_ERROR("malloc failed to allocate memory!"); + } #ifdef DEBUG end_ptr = ptr + size; #endif @@ -3944,19 +4914,7 @@ void erts_silence_warn_unused_result(long unused) void erts_interval_init(erts_interval_t *icp) { -#ifdef ARCH_64 - erts_atomic_init_nob(&icp->counter.atomic, 0); -#else - erts_dw_aint_t dw; -#ifdef ETHR_SU_DW_NAINT_T__ - dw.dw_sint = 0; -#else - dw.sint[ERTS_DW_AINT_HIGH_WORD] = 0; - dw.sint[ERTS_DW_AINT_LOW_WORD] = 0; -#endif - erts_dw_atomic_init_nob(&icp->counter.atomic, &dw); - -#endif + erts_atomic64_init_nob(&icp->counter.atomic, 0); #ifdef DEBUG icp->smp_api = 0; #endif @@ -3978,55 +4936,13 @@ erts_smp_interval_init(erts_interval_t *icp) static ERTS_INLINE Uint64 step_interval_nob(erts_interval_t *icp) { -#ifdef ARCH_64 - return (Uint64) erts_atomic_inc_read_nob(&icp->counter.atomic); -#else - erts_dw_aint_t exp; - - erts_dw_atomic_read_nob(&icp->counter.atomic, &exp); - while (1) { - erts_dw_aint_t new = exp; - -#ifdef ETHR_SU_DW_NAINT_T__ - new.dw_sint++; -#else - new.sint[ERTS_DW_AINT_LOW_WORD]++; - if (new.sint[ERTS_DW_AINT_LOW_WORD] == 0) - new.sint[ERTS_DW_AINT_HIGH_WORD]++; -#endif - - if (erts_dw_atomic_cmpxchg_nob(&icp->counter.atomic, &new, &exp)) - return erts_interval_dw_aint_to_val__(&new); - - } -#endif + return (Uint64) erts_atomic64_inc_read_nob(&icp->counter.atomic); } static ERTS_INLINE Uint64 step_interval_relb(erts_interval_t *icp) { -#ifdef ARCH_64 - return (Uint64) erts_atomic_inc_read_relb(&icp->counter.atomic); -#else - erts_dw_aint_t exp; - - erts_dw_atomic_read_nob(&icp->counter.atomic, &exp); - while (1) { - erts_dw_aint_t new = exp; - -#ifdef ETHR_SU_DW_NAINT_T__ - new.dw_sint++; -#else - new.sint[ERTS_DW_AINT_LOW_WORD]++; - if (new.sint[ERTS_DW_AINT_LOW_WORD] == 0) - new.sint[ERTS_DW_AINT_HIGH_WORD]++; -#endif - - if (erts_dw_atomic_cmpxchg_relb(&icp->counter.atomic, &new, &exp)) - return erts_interval_dw_aint_to_val__(&new); - - } -#endif + return (Uint64) erts_atomic64_inc_read_relb(&icp->counter.atomic); } @@ -4034,38 +4950,10 @@ static ERTS_INLINE Uint64 ensure_later_interval_nob(erts_interval_t *icp, Uint64 ic) { Uint64 curr_ic; -#ifdef ARCH_64 - curr_ic = (Uint64) erts_atomic_read_nob(&icp->counter.atomic); + curr_ic = (Uint64) erts_atomic64_read_nob(&icp->counter.atomic); if (curr_ic > ic) return curr_ic; - return (Uint64) erts_atomic_inc_read_nob(&icp->counter.atomic); -#else - erts_dw_aint_t exp; - - erts_dw_atomic_read_nob(&icp->counter.atomic, &exp); - curr_ic = erts_interval_dw_aint_to_val__(&exp); - if (curr_ic > ic) - return curr_ic; - - while (1) { - erts_dw_aint_t new = exp; - -#ifdef ETHR_SU_DW_NAINT_T__ - new.dw_sint++; -#else - new.sint[ERTS_DW_AINT_LOW_WORD]++; - if (new.sint[ERTS_DW_AINT_LOW_WORD] == 0) - new.sint[ERTS_DW_AINT_HIGH_WORD]++; -#endif - - if (erts_dw_atomic_cmpxchg_nob(&icp->counter.atomic, &new, &exp)) - return erts_interval_dw_aint_to_val__(&new); - - curr_ic = erts_interval_dw_aint_to_val__(&exp); - if (curr_ic > ic) - return curr_ic; - } -#endif + return (Uint64) erts_atomic64_inc_read_nob(&icp->counter.atomic); } @@ -4073,38 +4961,10 @@ static ERTS_INLINE Uint64 ensure_later_interval_acqb(erts_interval_t *icp, Uint64 ic) { Uint64 curr_ic; -#ifdef ARCH_64 - curr_ic = (Uint64) erts_atomic_read_acqb(&icp->counter.atomic); + curr_ic = (Uint64) erts_atomic64_read_acqb(&icp->counter.atomic); if (curr_ic > ic) return curr_ic; - return (Uint64) erts_atomic_inc_read_acqb(&icp->counter.atomic); -#else - erts_dw_aint_t exp; - - erts_dw_atomic_read_acqb(&icp->counter.atomic, &exp); - curr_ic = erts_interval_dw_aint_to_val__(&exp); - if (curr_ic > ic) - return curr_ic; - - while (1) { - erts_dw_aint_t new = exp; - -#ifdef ETHR_SU_DW_NAINT_T__ - new.dw_sint++; -#else - new.sint[ERTS_DW_AINT_LOW_WORD]++; - if (new.sint[ERTS_DW_AINT_LOW_WORD] == 0) - new.sint[ERTS_DW_AINT_HIGH_WORD]++; -#endif - - if (erts_dw_atomic_cmpxchg_acqb(&icp->counter.atomic, &new, &exp)) - return erts_interval_dw_aint_to_val__(&new); - - curr_ic = erts_interval_dw_aint_to_val__(&exp); - if (curr_ic > ic) - return curr_ic; - } -#endif + return (Uint64) erts_atomic64_inc_read_acqb(&icp->counter.atomic); } Uint64 @@ -4191,8 +5051,8 @@ erts_smp_ensure_later_interval_acqb(erts_interval_t *icp, Uint64 ic) */ Uint64 erts_timestamp_millis(void) { -#ifdef HAVE_GETHRTIME - return (Uint64) (sys_gethrtime() / 1000000); +#ifdef ERTS_HAVE_OS_MONOTONIC_TIME_SUPPORT + return ERTS_MONOTONIC_TO_MSEC(erts_os_monotonic_time()); #else Uint64 res; SysTimeval tv; diff --git a/erts/emulator/beam/version.h b/erts/emulator/beam/version.h index 3952c751b7..004725eaf5 100644 --- a/erts/emulator/beam/version.h +++ b/erts/emulator/beam/version.h @@ -3,16 +3,17 @@ * * Copyright Ericsson AB 1996-2009. All Rights Reserved. * - * The contents of this file are subject to the Erlang Public License, - * Version 1.1, (the "License"); you may not use this file except in - * compliance with the License. You should have received a copy of the - * Erlang Public License along with this software. If not, it can be - * retrieved online at http://www.erlang.org/. - * - * Software distributed under the License is distributed on an "AS IS" - * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See - * the License for the specific language governing rights and limitations - * under the License. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. * * %CopyrightEnd% */ |